diff --git a/requirements.txt b/requirements.txt index 923abd14..597e5178 100644 --- a/requirements.txt +++ b/requirements.txt @@ -84,4 +84,3 @@ werkzeug==3.1.8 # flask # flask-cors -opengeodeweb-microservice==1.*,>=1.4.0rc1 diff --git a/src/opengeodeweb_back/crs/crs_assignment.py b/src/opengeodeweb_back/crs/crs_assignment.py index 04b2365f..0066bf26 100644 --- a/src/opengeodeweb_back/crs/crs_assignment.py +++ b/src/opengeodeweb_back/crs/crs_assignment.py @@ -88,9 +88,14 @@ def assign_all( for target_id, _, target_object in targets: if target_id in assigned: _assign(target_object, authority, code) + # Local import: utils_functions imports the crs modules + from opengeodeweb_back.utils_functions import wait_native_saved # noqa: PLC0415 + # Assigning only relabels the coordinates: the viewables are unchanged for target_id, target_data, target_object in targets: if target_id in assigned: + # A background save of the same data, started meanwhile, must not write with it + wait_native_saved(target_id) target_object.save(geode_functions.data_file_path(target_id, target_data.native_file)) entries: dict[str, dict[str, Any]] = {} for target_id, _, target_object in targets: diff --git a/src/opengeodeweb_back/crs/crs_functions.py b/src/opengeodeweb_back/crs/crs_functions.py index be0350aa..4dde27fa 100644 --- a/src/opengeodeweb_back/crs/crs_functions.py +++ b/src/opengeodeweb_back/crs/crs_functions.py @@ -105,7 +105,7 @@ def known_geographic_info( def geographic_info( - crs_manager: og.CoordinateReferenceSystemManager2D | og.CoordinateReferenceSystemManager3D, + crs_manager: (og.CoordinateReferenceSystemManager2D | og.CoordinateReferenceSystemManager3D), ) -> og_geosciences.GeographicCoordinateSystemInfo | None: crs = crs_manager.active_coordinate_reference_system() if isinstance( @@ -282,7 +282,8 @@ def frame( crs: og.CoordinateReferenceSystem2D | og.CoordinateReferenceSystem3D, ) -> og.CoordinateSystem2D: return og.CoordinateSystem2D( - og.Point2D(_xy(crs, 0)), [og.Point2D(_xy(crs, first)), og.Point2D(_xy(crs, second))] + og.Point2D(_xy(crs, 0)), + [og.Point2D(_xy(crs, first)), og.Point2D(_xy(crs, second))], ) return frame(active), frame(original) @@ -310,7 +311,11 @@ def coordinate_system(points: list[og.Point2D]) -> og.CoordinateSystem2D: def load_crs_object(data_id: str) -> tuple[Data, CrsGeodeObject]: + # Local import: utils_functions imports this module + from opengeodeweb_back.utils_functions import wait_native_saved # noqa: PLC0415 + data = geode_functions.get_data_info(data_id) + wait_native_saved(data_id) geode_object = geode_functions.geode_object_from_string(data.geode_object_type).load_one( geode_functions.data_file_path(data_id, data.native_file) ) diff --git a/src/opengeodeweb_back/geode_functions.py b/src/opengeodeweb_back/geode_functions.py index d92144e6..022bb695 100644 --- a/src/opengeodeweb_back/geode_functions.py +++ b/src/opengeodeweb_back/geode_functions.py @@ -38,10 +38,14 @@ def geode_object_from_string(value: str) -> type[GeodeObject]: def load_geode_object(data_id: str) -> GeodeObject: + # Local import: utils_functions imports this module + from .utils_functions import wait_native_saved # noqa: PLC0415 + data = Data.get(data_id) if not data: flask.abort(404, f"Data with id {data_id} not found") + wait_native_saved(data_id) file_absolute_path = data_file_path(data_id, data.native_file) logger.debug( "Loading file: %s (exists: %s)", diff --git a/src/opengeodeweb_back/routes/basemap/blueprint_basemap.py b/src/opengeodeweb_back/routes/basemap/blueprint_basemap.py index 064904ea..73d01a5d 100644 --- a/src/opengeodeweb_back/routes/basemap/blueprint_basemap.py +++ b/src/opengeodeweb_back/routes/basemap/blueprint_basemap.py @@ -2,7 +2,6 @@ # Standard library imports import logging -import shutil from pathlib import Path from typing import TYPE_CHECKING, Any @@ -84,7 +83,7 @@ def _save_tiles(data_id: str, tiles: basemap_tiles.TileRange, images: Images) -> ) except OSError: # The request rolls the data row back: its folder must not stay behind - shutil.rmtree(data_folder, ignore_errors=True) + utils_functions.remove_data_folder(data_id, data_folder, ignore_errors=True) raise @@ -133,5 +132,5 @@ def delete(params: schemas.Delete) -> schemas.DeleteResponse: if not basemap_tiles.is_basemap(params.id): flask.abort(400, f"Data {params.id} is not a basemap") get_session().delete(data) - shutil.rmtree(geode_functions.data_file_path(params.id)) + utils_functions.remove_data_folder(params.id) return schemas.DeleteResponse(id=params.id) diff --git a/src/opengeodeweb_back/routes/blueprint_routes.py b/src/opengeodeweb_back/routes/blueprint_routes.py index 0c23976a..14fc73cd 100644 --- a/src/opengeodeweb_back/routes/blueprint_routes.py +++ b/src/opengeodeweb_back/routes/blueprint_routes.py @@ -5,7 +5,6 @@ import logging import math import os -import shutil import sqlite3 import tempfile import time @@ -297,6 +296,7 @@ def apply_time_series( _check_time_series_extension(params.filename) if data.geode_object_type != GeodeBRep.geode_object_type(): flask.abort(400, f"{data.geode_object_type} is not a BRep: time series cannot be applied") + utils_functions.wait_native_saved(params.id) geode_object = GeodeBRep.load_one(geode_functions.data_file_path(params.id, data.native_file)) # Applied in memory first: if OpenGeode fails, nothing is written. geode_object.load_time_series(geode_functions.upload_file_path(params.filename)) @@ -841,6 +841,7 @@ def export_project() -> flask.Response: flask.abort(400, "Requested filename must end with .vease") export_vease_path = Path(project_folder) / filename + utils_functions.wait_all_native_saved() with get_session() as session: rows = session.query(Data.id, Data.native_file).all() # Basemaps are rebuilt from the project style: neither their rows nor their tiles are exported @@ -889,7 +890,7 @@ def _reset_data_folder(data_folder_path: Path) -> None: if data_folder_path.exists(): for item in data_folder_path.iterdir(): if item.is_dir() and not item.is_symlink(): - shutil.rmtree(item) + utils_functions.remove_data_folder(item.name, item) else: item.unlink() else: diff --git a/src/opengeodeweb_back/utils_functions.py b/src/opengeodeweb_back/utils_functions.py index 233b6cc8..e45eea89 100644 --- a/src/opengeodeweb_back/utils_functions.py +++ b/src/opengeodeweb_back/utils_functions.py @@ -9,7 +9,8 @@ import time import uuid import zipfile -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import Future, ThreadPoolExecutor, wait +from dataclasses import dataclass, field from importlib import metadata from pathlib import Path from typing import TYPE_CHECKING, Any @@ -33,6 +34,11 @@ logger = logging.getLogger(__name__) +# Native files are saved in background: the viewer only needs the viewables. +# Several files can be imported at once, hence the default pool size. +_native_executor = ThreadPoolExecutor(thread_name_prefix="native_save") +_native_saves_lock = threading.Lock() + if TYPE_CHECKING: from collections.abc import Callable, Sequence @@ -303,18 +309,150 @@ def model_components(data_id: str, model: GeodeModel, viewable_file: str | None) } +@dataclass +class _NativeSave: + """A background native save. The object is kept until it is saved, to retry a failure.""" + + future: Future[list[str]] + geode_object: GeodeObject + native_path: str + # Held by a synchronous retry: a later save of the same data waits for it + retry_lock: threading.Lock = field(default_factory=threading.Lock) + retried: bool = False + + +_native_saves: dict[str, _NativeSave] = {} + + +def _save_native_in_background(geode_object: GeodeObject, data_id: str, native_path: str) -> None: + """Save the native file without blocking the request. + + The task only uses the captured geode object: no app context nor DB session. + The Werkzeug debug reloader may kill a save mid-write (dev only). + """ + + def save_native(previous: _NativeSave | None) -> list[str]: + if previous is not None: + # A re-save of the same data supersedes the previous one, whatever its outcome, + # but never writes the same file at the same time. No deadlock: FIFO order puts + # the previous save ahead in the queue or running. + wait([previous.future]) + with previous.retry_lock: + pass + return geode_object.save(native_path) + + # One critical section: two re-saves of the same data cannot chain on the same previous + with _native_saves_lock: + previous = _native_saves.get(data_id) + future = _native_executor.submit(save_native, previous) + _native_saves[data_id] = _NativeSave(future, geode_object, native_path) + # Outside the lock: the callback takes it, and runs here if the save is already done + future.add_done_callback(lambda done: _forget_native_save(data_id, done)) + + +def _forget_native_save(data_id: str, future: Future[list[str]]) -> None: + """Drop a successful save from the registry; failed ones stay to be retried.""" + if future.cancelled() or future.exception() is not None: + return + with _native_saves_lock: + entry = _native_saves.get(data_id) + if entry is not None and entry.future is future: + del _native_saves[data_id] + + +def _drop_native_save(data_id: str, entry: _NativeSave) -> None: + with _native_saves_lock: + if _native_saves.get(data_id) is entry: + del _native_saves[data_id] + + +def discard_native_save(data_id: str) -> None: + """Wait for the native save of data_id, if any, and forget it whatever its outcome.""" + with _native_saves_lock: + entry = _native_saves.get(data_id) + if entry is None: + return + try: + entry.future.result() + except Exception: + logger.warning("Discarded failed native save of %s", data_id, exc_info=True) + # A retry running on another thread must not write in a folder being removed, + # and none may start once the entry is dropped + with entry.retry_lock: + _drop_native_save(data_id, entry) + + +def remove_data_folder( + data_id: str, folder: str | Path | None = None, *, ignore_errors: bool = False +) -> None: + """Remove the folder of data_id once its native save is done (by default its data folder).""" + discard_native_save(data_id) + if folder is None: + folder = geode_functions.data_file_path(data_id) + shutil.rmtree(folder, ignore_errors=ignore_errors) + + +def _settle_native_save(data_id: str, entry: _NativeSave) -> Exception | None: + """Wait for entry; on failure, retry its save once on this thread. None once saved.""" + try: + entry.future.result() + except Exception as error: + with entry.retry_lock: + with _native_saves_lock: + superseded = _native_saves.get(data_id) is not entry + if not entry.retried and not superseded: + logger.warning("Native save of %s failed, retrying", data_id, exc_info=error) + try: + entry.geode_object.save(entry.native_path) + except Exception as retry_error: + logger.exception("Native file of %s could not be saved", data_id) + return retry_error + entry.retried = True + _drop_native_save(data_id, entry) + return None + + +def _ensure_native_saved(data_id: str) -> Exception | None: + """Wait until data_id has no pending save. The error of a save that failed twice, if any.""" + while True: + with _native_saves_lock: + entry = _native_saves.get(data_id) + if entry is None: + return None + # A superseded entry is not dropped: the loop then waits for the newer save + error = _settle_native_save(data_id, entry) + if error is not None: + return error + + +def wait_native_saved(data_id: str) -> None: + """Wait for the background native save of data_id, if any, retrying a failure once.""" + error = _ensure_native_saved(data_id) + if error is not None: + # The failed save stays registered: a later call retries it again + flask.abort(500, f"Native file of {data_id} could not be saved: {error}") + + +def wait_all_native_saved() -> None: + """Wait for every pending background native save, retrying each failure once. + + Aborts once, naming every data whose native file could not be saved. + """ + with _native_saves_lock: + pending = list(_native_saves) + errors = {data_id: _ensure_native_saved(data_id) for data_id in pending} + failed = [f"{data_id} ({error})" for data_id, error in errors.items() if error] + if failed: + flask.abort(500, f"Native files could not be saved: {', '.join(failed)}") + + def save_all_viewables_and_return_info( geode_object: GeodeObject, data: Data, data_path: str, ) -> dict[str, Any]: with ThreadPoolExecutor() as executor: - tasks: list[tuple[Callable[[str], Any], str]] = [ - ( - geode_object.save, - str(Path(data_path) / ("native." + geode_object.native_extension())), - ) - ] + tasks: list[tuple[Callable[[str], Any], str]] = [] if geode_object.is_viewable(): tasks.extend( [ @@ -327,10 +465,9 @@ def save_all_viewables_and_return_info( ) results = list(executor.map(lambda args: args[0](args[1]), tasks)) - native_files = results[0] if geode_object.is_viewable(): - viewable_path = results[1] - light_path = results[2] + viewable_path = results[0] + light_path = results[1] binary_light_viewable = Path(light_path).read_bytes() binary_light_viewable_str = base64.b64encode(binary_light_viewable).decode("ascii") data.viewable_file = Path(viewable_path).name @@ -340,7 +477,7 @@ def save_all_viewables_and_return_info( data.viewable_file = None data.light_viewable_file = None - data.native_file = Path(native_files[0]).name + data.native_file = f"native.{geode_object.native_extension()}" name = geode_object.identifier.name() if not name: @@ -365,6 +502,7 @@ def save_all_viewables_and_return_info( crs = active_crs(geode_object) if crs is not None: response["crs"] = crs + _save_native_in_background(geode_object, data.id, str(Path(data_path) / data.native_file)) return response @@ -426,7 +564,7 @@ def _remove_created_data(created: list[Data]) -> None: session = get_session() for data in created: session.delete(data) - shutil.rmtree(geode_functions.data_file_path(data.id), ignore_errors=True) + remove_data_folder(data.id, ignore_errors=True) session.flush() diff --git a/tests/test_background_native_save.py b/tests/test_background_native_save.py new file mode 100644 index 00000000..d50b9159 --- /dev/null +++ b/tests/test_background_native_save.py @@ -0,0 +1,413 @@ +"""The native file of an import is saved in background, after the route answered.""" + +from __future__ import annotations + +# Standard library imports +import contextlib +import logging +import threading +import time +import zipfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Any + +# Third party imports +import pytest +from opengeodeweb_microservice.database import connection +from werkzeug.exceptions import HTTPException + +# Local application imports +from opengeodeweb_back import geode_functions, test_utils, utils_functions +from opengeodeweb_back.geode_objects.geode_triangulated_surface3d import ( + GeodeTriangulatedSurface3D, +) + +if TYPE_CHECKING: + from collections.abc import Callable, Generator + + from flask.testing import FlaskClient + from werkzeug.test import TestResponse + + from opengeodeweb_back.geode_objects.geode_object import GeodeObject + +GEODE_OBJECT_TYPE = "TriangulatedSurface3D" +FILENAME = "test.og_tsf3d" +NATIVE_FILE = "native.og_tsf3d" +ASSIGN_ROUTE = "/opengeodeweb_back/crs/assign_geographic" +LAMBERT = {"authority": "EPSG", "code": "2154", "name": "RGF93 v1 / Lambert-93"} +SAVE_ROUTE = "/opengeodeweb_back/save_viewable_file" +EXPORT_ROUTE = "/opengeodeweb_back/export_project" +BASEMAP_BUILD_ROUTE = "/opengeodeweb_back/basemap/build" +BASEMAP_DELETE_ROUTE = "/opengeodeweb_back/basemap/delete" +ROUTE_TIMEOUT = 10.0 +# Larger than ROUTE_TIMEOUT: a blocked route fails the test before the save gives up +TIMEOUT = 25.0 +STILL_RUNNING_DELAY = 0.5 + + +@dataclass +class NativeSave: + """Blocks every native save of a TriangulatedSurface3D until released.""" + + release: threading.Event = field(default_factory=threading.Event) + condition: threading.Condition = field(default_factory=threading.Condition) + entered: int = 0 + error: Exception | None = None + # How many saves fail with error, every one if None + failures: int | None = None + # Seconds a released save stays in progress, to widen the window of a write race + hold: float = 0.0 + active: int = 0 + max_active: int = 0 + threads: list[threading.Thread] = field(default_factory=list) + + def wait_entered(self, count: int) -> bool: + with self.condition: + return self.condition.wait_for(lambda: self.entered >= count, TIMEOUT) + + +@dataclass +class Background[T]: + """A call running in a thread, whose result is awaited with a timeout.""" + + thread: threading.Thread + outcome: dict[str, Any] = field(default_factory=dict) + + def is_running_after(self, delay: float) -> bool: + self.thread.join(delay) + return self.thread.is_alive() + + def result(self, timeout: float = ROUTE_TIMEOUT) -> T: + self.thread.join(timeout) + if self.thread.is_alive(): + pytest.fail(f"still running after {timeout}s") + if "error" in self.outcome: + raise self.outcome["error"] + value: T = self.outcome["value"] + return value + + +def _close_thread_session() -> None: + registry = connection._state.scoped_session_registry # noqa: SLF001 + if registry is not None: + registry.remove() + + +@pytest.fixture +def native_save(client: FlaskClient, monkeypatch: pytest.MonkeyPatch) -> Generator[NativeSave]: + state = NativeSave() + original_save = GeodeTriangulatedSurface3D.save + + def blocked_save(self: GeodeTriangulatedSurface3D, filename: str) -> list[str]: + with state.condition: + state.entered += 1 + state.condition.notify_all() + if not state.release.wait(TIMEOUT): + msg = "native save never released" + raise TimeoutError(msg) + with state.condition: + failing = state.error is not None and state.failures != 0 + if failing and state.failures is not None: + state.failures -= 1 + state.active += 1 + state.max_active = max(state.max_active, state.active) + try: + time.sleep(state.hold) + if failing and state.error is not None: + raise state.error + return original_save(self, filename) + finally: + with state.condition: + state.active -= 1 + + monkeypatch.setattr(GeodeTriangulatedSurface3D, "save", blocked_save) + test_utils.check_upload_file(client, FILENAME) + try: + yield state + finally: + state.release.set() + for thread in state.threads: + thread.join(TIMEOUT) + # No pending nor failed native save may leak into the next test + with client.application.app_context(), contextlib.suppress(HTTPException): + utils_functions.wait_all_native_saved() + utils_functions._native_saves.clear() # noqa: SLF001 + + +def _in_thread[T](state: NativeSave, function: Callable[[], T]) -> Background[T]: + outcome: dict[str, Any] = {} + + def run() -> None: + try: + outcome["value"] = function() + except BaseException as error: # noqa: BLE001 + outcome["error"] = error + finally: + _close_thread_session() + + thread = threading.Thread(target=run, daemon=True) + state.threads.append(thread) + thread.start() + return Background(thread, outcome) + + +def _import(client: FlaskClient, state: NativeSave) -> Background[TestResponse]: + return _in_thread( + state, + lambda: client.post( + SAVE_ROUTE, + json={"geode_object_type": GEODE_OBJECT_TYPE, "filename": FILENAME}, + ), + ) + + +def _imported_item(client: FlaskClient, state: NativeSave) -> dict[str, Any]: + """Import while the native save is blocked: the route must answer anyway.""" + response = _import(client, state).result() + assert response.status_code == 200, response.get_json() + item: dict[str, Any] = response.get_json()["data"][0] + assert len(response.get_json()["data"]) == 1 + assert item["native_file"] == NATIVE_FILE + return item + + +def _load_in_app(client: FlaskClient, data_id: str) -> GeodeObject: + with client.application.app_context(): + return geode_functions.load_geode_object(data_id) + + +def test_route_answers_before_native_save(client: FlaskClient, native_save: NativeSave) -> None: + """(a) save_viewable_file answers while the native save is still blocked.""" + item = _imported_item(client, native_save) + assert not native_save.release.is_set() + assert native_save.wait_entered(1) + assert isinstance(item["binary_light_viewable"], str) + + +def test_native_file_saved_after_wait(client: FlaskClient, native_save: NativeSave) -> None: + """(b) After wait_native_saved, the native file exists and loads.""" + item = _imported_item(client, native_save) + native_save.release.set() + with client.application.app_context(): + utils_functions.wait_native_saved(item["id"]) + native_path = Path(geode_functions.data_file_path(item["id"], NATIVE_FILE)) + assert native_path.is_file() + GeodeTriangulatedSurface3D.load_one(str(native_path)) + assert isinstance(geode_functions.load_geode_object(item["id"]), GeodeTriangulatedSurface3D) + + +def test_load_waits_for_native_save(client: FlaskClient, native_save: NativeSave) -> None: + """(c) load_geode_object does not return before the native save is done.""" + item = _imported_item(client, native_save) + load = _in_thread(native_save, lambda: _load_in_app(client, item["id"])) + assert load.is_running_after(STILL_RUNNING_DELAY), "load returned before the native save" + native_save.release.set() + assert isinstance(load.result(), GeodeTriangulatedSurface3D) + + +def test_failed_native_save_aborts_load( + client: FlaskClient, native_save: NativeSave, caplog: pytest.LogCaptureFixture +) -> None: + """(d) A failed native save makes load_geode_object abort with 500, and is logged.""" + native_save.error = RuntimeError("disk exploded") + item = _imported_item(client, native_save) + with caplog.at_level(logging.ERROR): + native_save.release.set() + # The error must be logged by wait_native_saved on the calling thread, + # not from a done-callback: caplog only sees it reliably there. + with pytest.raises(HTTPException) as error: + _load_in_app(client, item["id"]) + assert error.value.code == 500 + assert "could not be saved" in str(error.value.description) + assert "disk exploded" in caplog.text + # A failed save keeps failing the same way + with pytest.raises(HTTPException) as error: + _load_in_app(client, item["id"]) + assert error.value.code == 500 + + +def test_concurrent_imports_save_in_parallel(client: FlaskClient, native_save: NativeSave) -> None: + """(e) Two imports issued concurrently run their native saves in parallel.""" + first = _import(client, native_save) + second = _import(client, native_save) + for import_call in (first, second): + assert import_call.result().status_code == 200 + assert native_save.wait_entered(2), "both native saves should run at once" + native_save.release.set() + with client.application.app_context(): + utils_functions.wait_all_native_saved() + + +def test_export_contains_pending_native_file( + client: FlaskClient, native_save: NativeSave, tmp_path: Path +) -> None: + """(f) export_project waits for pending native saves and exports their files.""" + item = _imported_item(client, native_save) + filename = "background_native_save.vease" + export = _in_thread( + native_save, + lambda: client.post(EXPORT_ROUTE, json={"snapshot": {}, "filename": filename}), + ) + assert export.is_running_after(STILL_RUNNING_DELAY), "export did not wait for the save" + native_save.release.set() + response = export.result() + assert response.status_code == 200 + response.direct_passthrough = False + vease = tmp_path / filename + vease.write_bytes(response.get_data()) + response.close() + (Path(client.application.config["DATA_FOLDER_PATH"]) / filename).unlink(missing_ok=True) + with zipfile.ZipFile(vease) as zip_file: + native_bytes = zip_file.read(f"{item['id']}/{NATIVE_FILE}") + native_copy = tmp_path / NATIVE_FILE + native_copy.write_bytes(native_bytes) + GeodeTriangulatedSurface3D.load_one(str(native_copy)) + + +def test_crs_assignment_waits_for_native_save(client: FlaskClient, native_save: NativeSave) -> None: + """(g) A CRS assignment waits for the native save, then its change is in the final file.""" + item = _imported_item(client, native_save) + assign = _in_thread( + native_save, + lambda: client.post(ASSIGN_ROUTE, json={"id": item["id"], **LAMBERT}), + ) + assert assign.is_running_after(STILL_RUNNING_DELAY), ( + "assignment did not wait for the native save" + ) + native_save.release.set() + response = assign.result() + assert response.status_code == 200, response.get_json() + with client.application.app_context(): + utils_functions.wait_all_native_saved() + geode_object = _load_in_app(client, item["id"]) + assert isinstance(geode_object, GeodeTriangulatedSurface3D) + manager = geode_object.crs_manager() + assert manager.active_coordinate_reference_system_name() == "EPSG:2154" + + +def _test_surface() -> GeodeTriangulatedSurface3D: + return GeodeTriangulatedSurface3D.load_one(str(Path(__file__).parent / "data" / FILENAME)) + + +def _wait_registered_save(data_id: str) -> None: + """Wait for the background save of data_id to end, without settling it.""" + with utils_functions._native_saves_lock: # noqa: SLF001 + entry = utils_functions._native_saves.get(data_id) # noqa: SLF001 + if entry is not None: + entry.future.exception(TIMEOUT) + + +def test_failed_native_save_is_retried(client: FlaskClient, native_save: NativeSave) -> None: + """A failed background save is retried once by the next load, which then succeeds.""" + native_save.error = RuntimeError("transient lock") + native_save.failures = 1 + item = _imported_item(client, native_save) + native_save.release.set() + _wait_registered_save(item["id"]) + assert native_save.failures == 0, "the background save should have failed" + response = client.post(ASSIGN_ROUTE, json={"id": item["id"], **LAMBERT}) + assert response.status_code == 200, response.get_json() + with client.application.app_context(): + utils_functions.wait_all_native_saved() + assert isinstance(_load_in_app(client, item["id"]), GeodeTriangulatedSurface3D) + + +def test_failed_native_save_retry_fails(client: FlaskClient, native_save: NativeSave) -> None: + """When the retry fails too, the load answers 500, and the next call retries again.""" + native_save.error = RuntimeError("disk exploded") + native_save.failures = 2 + item = _imported_item(client, native_save) + native_save.release.set() + response = client.post(ASSIGN_ROUTE, json={"id": item["id"], **LAMBERT}) + assert response.status_code == 500 + assert "disk exploded" in response.get_json()["description"] + assert native_save.failures == 0 + # The data is not lost for the session: the next call retries and heals it + response = client.post(ASSIGN_ROUTE, json={"id": item["id"], **LAMBERT}) + assert response.status_code == 200, response.get_json() + + +def test_export_names_every_failed_native_save( + client: FlaskClient, native_save: NativeSave, tmp_path: Path +) -> None: + """export_project waits for every save, then answers 500 naming each failed data.""" + native_save.error = RuntimeError("disk exploded") + item = _imported_item(client, native_save) + other_id = "other-failing-data" + utils_functions._save_native_in_background( # noqa: SLF001 + _test_surface(), other_id, str(tmp_path / NATIVE_FILE) + ) + native_save.release.set() + response = client.post( + EXPORT_ROUTE, json={"snapshot": {}, "filename": "failed_native_save.vease"} + ) + assert response.status_code == 500 + description = response.get_json()["description"] + assert item["id"] in description + assert other_id in description + + +def test_resaves_of_one_data_never_overlap(client: FlaskClient, native_save: NativeSave) -> None: + """Two re-saves of one data, requested at once, write its native file one at a time.""" + item = _imported_item(client, native_save) + with client.application.app_context(): + native_path = geode_functions.data_file_path(item["id"], NATIVE_FILE) + geode_object = _test_surface() + start = threading.Barrier(2) + + def resave() -> None: + start.wait(TIMEOUT) + utils_functions._save_native_in_background( # noqa: SLF001 + geode_object, item["id"], native_path + ) + + resaves = [_in_thread(native_save, resave) for _ in range(2)] + for call in resaves: + call.result() + native_save.hold = 0.2 + native_save.release.set() + with client.application.app_context(): + utils_functions.wait_native_saved(item["id"]) + assert native_save.entered == 3 + assert native_save.max_active == 1 + + +def test_basemap_delete_waits_for_native_save( + client: FlaskClient, + native_save: NativeSave, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Deleting a basemap whose native save is pending waits for it, then removes it all.""" + any_tile = Path(__file__).parent / "data" / "basemap_tiles" / "1" / "0" / "0.png" + monkeypatch.setenv("BASEMAP_TILE_URL", any_tile.as_uri()) + monkeypatch.setenv("BASEMAP_CACHE_FOLDER", str(tmp_path)) + build = client.post( + BASEMAP_BUILD_ROUTE, + json={ + "maps": ["satellite"], + "bounds": { + "x_min": 650_000.0, + "y_min": 6_860_000.0, + "x_max": 655_000.0, + "y_max": 6_865_000.0, + }, + "crs": LAMBERT, + "z": 0.0, + }, + ) + assert build.status_code == 200, build.get_json() + data_id = build.get_json()["id"] + with client.application.app_context(): + folder = Path(geode_functions.data_file_path(data_id)) + delete = _in_thread( + native_save, lambda: client.post(BASEMAP_DELETE_ROUTE, json={"id": data_id}) + ) + assert delete.is_running_after(STILL_RUNNING_DELAY), "delete did not wait for the native save" + native_save.release.set() + response = delete.result() + assert response.status_code == 200, response.get_json() + assert not folder.exists() + with utils_functions._native_saves_lock: # noqa: SLF001 + assert data_id not in utils_functions._native_saves # noqa: SLF001 diff --git a/tests/test_utils_functions.py b/tests/test_utils_functions.py index a4d781d2..eb7ff3ab 100644 --- a/tests/test_utils_functions.py +++ b/tests/test_utils_functions.py @@ -218,6 +218,8 @@ def test_generate_files_from_object( assert data.light_viewable_file.endswith(".vtp") data_path = Path(app.config["DATA_FOLDER_PATH"]) / result["id"] + # The native file is saved in background + utils_functions.wait_native_saved(result["id"]) assert (data_path / result["native_file"]).exists() assert (data_path / result["viewable_file"]).exists() assert (data_path / data.light_viewable_file).exists() @@ -331,13 +333,15 @@ def test_generate_files_from_mixed_file_gives_distinct_geode_ids( app = client.application with app.app_context(): [surface] = utils_functions.generate_files_from_file( - GeodeTriangulatedSurface3D.geode_object_type(), "mixed_polygons_polyhedra.vtu" + GeodeTriangulatedSurface3D.geode_object_type(), + "mixed_polygons_polyhedra.vtu", ) [solid] = utils_functions.generate_files_from_file( GeodeTetrahedralSolid3D.geode_object_type(), "mixed_polygons_polyhedra.vtu" ) [surface_again] = utils_functions.generate_files_from_file( - GeodeTriangulatedSurface3D.geode_object_type(), "mixed_polygons_polyhedra.vtu" + GeodeTriangulatedSurface3D.geode_object_type(), + "mixed_polygons_polyhedra.vtu", ) # Each content family of one file is a distinct object assert surface["geode_id"] != solid["geode_id"]