From 79e511693481775ad99a496b63537fd6af09e401 Mon Sep 17 00:00:00 2001 From: Arnaud Botella Date: Fri, 20 Nov 2026 14:32:07 +0100 Subject: [PATCH 1/4] perf(model): prefetch vtm pieces in parallel before reading a model The XML multiblock reader opens the pieces of a .vtm one by one. Right after an import they are new files, and an on-access antivirus scan (Windows Defender) makes each first open slow: about 6 s for a model of 764 pieces, against 0.6 s of VTK parsing. register_model now reads every piece once from a thread pool before reader.Update(): the scans run in parallel and VTK reads files already scanned and cached. On a 284 MB structural model, the read drops from 6.1 s to 1.8 s and the first image arrives about 6 s earlier. Co-Authored-By: Claude Opus 5.5 --- .../rpc/model/model_protocols.py | 5 ++- src/opengeodeweb_viewer/vtm_prefetch.py | 35 ++++++++++++++++++ tests/test_vtm_prefetch.py | 37 +++++++++++++++++++ 3 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 src/opengeodeweb_viewer/vtm_prefetch.py create mode 100644 tests/test_vtm_prefetch.py diff --git a/src/opengeodeweb_viewer/rpc/model/model_protocols.py b/src/opengeodeweb_viewer/rpc/model/model_protocols.py index b6591e26..10d094f7 100644 --- a/src/opengeodeweb_viewer/rpc/model/model_protocols.py +++ b/src/opengeodeweb_viewer/rpc/model/model_protocols.py @@ -37,6 +37,7 @@ PointsSizeActor, VtkPipeline, ) +from opengeodeweb_viewer.vtm_prefetch import prefetch_vtm_pieces from . import schemas @@ -487,8 +488,10 @@ def register_model(self, params: schemas.Register) -> schemas.RegisterResponse: viewer_data = self.get_viewer_data(data_id) file_name = str(viewer_data.viewable_file) + vtm_path = Path(self.DATA_FOLDER_PATH) / data_id / file_name reader = vtkXMLMultiBlockDataReader() - reader.SetFileName(str(Path(self.DATA_FOLDER_PATH) / data_id / file_name)) + reader.SetFileName(str(vtm_path)) + prefetch_vtm_pieces(vtm_path) reader.Update() mapper = vtkCompositePolyDataMapper() attributes = vtkCompositeDataDisplayAttributes() diff --git a/src/opengeodeweb_viewer/vtm_prefetch.py b/src/opengeodeweb_viewer/vtm_prefetch.py new file mode 100644 index 00000000..b4d0ed28 --- /dev/null +++ b/src/opengeodeweb_viewer/vtm_prefetch.py @@ -0,0 +1,35 @@ +"""Read the pieces of a VTK multiblock file ahead of the VTK reader.""" + +# Standard library imports +import contextlib +import re +from concurrent.futures import ThreadPoolExecutor +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path + +_PIECE_FILE = re.compile(r'\bfile="([^"]+)"') +_PREFETCH_WORKERS = 32 + + +def _read_piece(path: Path) -> None: + # The VTK reader reports missing or unreadable pieces itself + with contextlib.suppress(OSError): + path.read_bytes() + + +def prefetch_vtm_pieces(vtm_path: Path) -> int: + """Read every piece of a .vtm file once, in parallel, and return how many were read. + + The XML multiblock reader opens its pieces one by one. Right after an import they + are new files, and an on-access antivirus scan (Windows Defender) makes each first + open slow: about 5 s for a model of 764 pieces, against 0.6 s for the VTK parsing + itself. Opening them first from threads, which release the GIL during I/O, runs + those scans in parallel; the VTK reader then opens files already scanned and cached. + """ + folder = vtm_path.parent + pieces = [folder / name for name in _PIECE_FILE.findall(vtm_path.read_text())] + with ThreadPoolExecutor(min(_PREFETCH_WORKERS, len(pieces) or 1)) as executor: + list(executor.map(_read_piece, pieces)) + return len(pieces) diff --git a/tests/test_vtm_prefetch.py b/tests/test_vtm_prefetch.py new file mode 100644 index 00000000..f4bff457 --- /dev/null +++ b/tests/test_vtm_prefetch.py @@ -0,0 +1,37 @@ +from pathlib import Path +from typing import TYPE_CHECKING + +from opengeodeweb_viewer import vtm_prefetch +from opengeodeweb_viewer.vtm_prefetch import prefetch_vtm_pieces + +if TYPE_CHECKING: + import pytest + +DATA = Path(__file__).parent / "data" + + +def test_prefetch_reads_every_piece(monkeypatch: pytest.MonkeyPatch) -> None: + read: list[Path] = [] + monkeypatch.setattr(vtm_prefetch, "_read_piece", read.append) + assert prefetch_vtm_pieces(DATA / "cube.vtm") == 45 + assert len(read) == 45 + assert all(path.is_file() for path in read) + assert all(path.parent.parent == DATA for path in read) + + +def test_prefetch_ignores_missing_pieces(tmp_path: Path) -> None: + (tmp_path / "piece.vtp").write_bytes(b"data") + vtm = tmp_path / "model.vtm" + vtm.write_text( + '' + '' + '' + "" + ) + assert prefetch_vtm_pieces(vtm) == 2 + + +def test_prefetch_without_pieces(tmp_path: Path) -> None: + vtm = tmp_path / "empty.vtm" + vtm.write_text("") + assert prefetch_vtm_pieces(vtm) == 0 From 0ed825323f394a0b53c9a895ec967ed8da235cae Mon Sep 17 00:00:00 2001 From: BotellaA <3213882+BotellaA@users.noreply.github.com> Date: Fri, 20 Nov 2026 18:33:39 +0000 Subject: [PATCH 2/4] Apply prepare changes --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e87b5c40..0e8e27eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -61,4 +61,3 @@ wslink==2.6.0 yarl==1.* # via aiohttp -opengeodeweb-microservice==1.*,>=1.3.3 From 1e476f5a8cf7fecb899bec69cfe43c6fbefef2ed Mon Sep 17 00:00:00 2001 From: JulienChampagnol <91873154+JulienChampagnol@users.noreply.github.com> Date: Fri, 20 Nov 2026 19:01:02 +0000 Subject: [PATCH 3/4] Apply prepare changes --- opengeodeweb_viewer_typed_schemas.d.ts | 110 +++++-------------------- 1 file changed, 22 insertions(+), 88 deletions(-) diff --git a/opengeodeweb_viewer_typed_schemas.d.ts b/opengeodeweb_viewer_typed_schemas.d.ts index 9181c7f1..ea639c93 100644 --- a/opengeodeweb_viewer_typed_schemas.d.ts +++ b/opengeodeweb_viewer_typed_schemas.d.ts @@ -23,10 +23,7 @@ export interface MeshCellsAttributeCellAttributeParams { minimum: number; name: string; no_data_color?: MeshCellsAttributeCellAttributeParamsNoDataColor; - /** - * Flat array of [value, r, g, b, ...] - */ - points: number[]; + points: number[]; // Flat array of [value, r, g, b, ...] } export interface MeshCellsAttributeCellAttributeParamsNoDataColor { @@ -46,10 +43,7 @@ export interface MeshCellsAttributeVertexAttributeParams { minimum: number; name: string; no_data_color?: MeshCellsAttributeVertexAttributeParamsNoDataColor; - /** - * Flat array of [value, r, g, b, ...] - */ - points: number[]; + points: number[]; // Flat array of [value, r, g, b, ...] } export interface MeshCellsAttributeVertexAttributeParamsNoDataColor { @@ -100,10 +94,7 @@ export interface MeshEdgesAttributeEdgeAttributeParams { minimum: number; name: string; no_data_color?: MeshEdgesAttributeEdgeAttributeParamsNoDataColor; - /** - * Flat array of [value, r, g, b, ...] - */ - points: number[]; + points: number[]; // Flat array of [value, r, g, b, ...] } export interface MeshEdgesAttributeEdgeAttributeParamsNoDataColor { @@ -123,10 +114,7 @@ export interface MeshEdgesAttributeVertexAttributeParams { minimum: number; name: string; no_data_color?: MeshEdgesAttributeVertexAttributeParamsNoDataColor; - /** - * Flat array of [value, r, g, b, ...] - */ - points: number[]; + points: number[]; // Flat array of [value, r, g, b, ...] } export interface MeshEdgesAttributeVertexAttributeParamsNoDataColor { @@ -185,10 +173,7 @@ export interface MeshPointsAttributeVertexAttributeParams { minimum: number; name: string; no_data_color?: MeshPointsAttributeVertexAttributeParamsNoDataColor; - /** - * Flat array of [value, r, g, b, ...] - */ - points: number[]; + points: number[]; // Flat array of [value, r, g, b, ...] } export interface MeshPointsAttributeVertexAttributeParamsNoDataColor { @@ -249,10 +234,7 @@ export interface MeshPolygonsAttributePolygonAttributeParams { minimum: number; name: string; no_data_color?: MeshPolygonsAttributePolygonAttributeParamsNoDataColor; - /** - * Flat array of [value, r, g, b, ...] - */ - points: number[]; + points: number[]; // Flat array of [value, r, g, b, ...] } export interface MeshPolygonsAttributePolygonAttributeParamsNoDataColor { @@ -272,10 +254,7 @@ export interface MeshPolygonsAttributeVertexAttributeParams { minimum: number; name: string; no_data_color?: MeshPolygonsAttributeVertexAttributeParamsNoDataColor; - /** - * Flat array of [value, r, g, b, ...] - */ - points: number[]; + points: number[]; // Flat array of [value, r, g, b, ...] } export interface MeshPolygonsAttributeVertexAttributeParamsNoDataColor { @@ -326,10 +305,7 @@ export interface MeshPolyhedraAttributePolyhedronAttributeParams { minimum: number; name: string; no_data_color?: MeshPolyhedraAttributePolyhedronAttributeParamsNoDataColor; - /** - * Flat array of [value, r, g, b, ...] - */ - points: number[]; + points: number[]; // Flat array of [value, r, g, b, ...] } export interface MeshPolyhedraAttributePolyhedronAttributeParamsNoDataColor { @@ -349,10 +325,7 @@ export interface MeshPolyhedraAttributeVertexAttributeParams { minimum: number; name: string; no_data_color?: MeshPolyhedraAttributeVertexAttributeParamsNoDataColor; - /** - * Flat array of [value, r, g, b, ...] - */ - points: number[]; + points: number[]; // Flat array of [value, r, g, b, ...] } export interface MeshPolyhedraAttributeVertexAttributeParamsNoDataColor { @@ -510,10 +483,7 @@ export interface ModelBlocksAttributePolyhedronAttributeParams { minimum: number; name: string; no_data_color?: ModelBlocksAttributePolyhedronAttributeParamsNoDataColor; - /** - * Flat array of [value, r, g, b, ...] - */ - points: number[]; + points: number[]; // Flat array of [value, r, g, b, ...] } export interface ModelBlocksAttributePolyhedronAttributeParamsNoDataColor { @@ -534,10 +504,7 @@ export interface ModelBlocksAttributeVertexAttributeParams { minimum: number; name: string; no_data_color?: ModelBlocksAttributeVertexAttributeParamsNoDataColor; - /** - * Flat array of [value, r, g, b, ...] - */ - points: number[]; + points: number[]; // Flat array of [value, r, g, b, ...] } export interface ModelBlocksAttributeVertexAttributeParamsNoDataColor { @@ -558,10 +525,7 @@ export interface ModelBlocksEdgesAttributeVertexAttributeParams { minimum: number; name: string; no_data_color?: ModelBlocksEdgesAttributeVertexAttributeParamsNoDataColor; - /** - * Flat array of [value, r, g, b, ...] - */ - points: number[]; + points: number[]; // Flat array of [value, r, g, b, ...] } export interface ModelBlocksEdgesAttributeVertexAttributeParamsNoDataColor { @@ -616,10 +580,7 @@ export interface ModelBlocksPointsAttributeVertexAttributeParams { minimum: number; name: string; no_data_color?: ModelBlocksPointsAttributeVertexAttributeParamsNoDataColor; - /** - * Flat array of [value, r, g, b, ...] - */ - points: number[]; + points: number[]; // Flat array of [value, r, g, b, ...] } export interface ModelBlocksPointsAttributeVertexAttributeParamsNoDataColor { @@ -717,10 +678,7 @@ export interface ModelCornersAttributeVertexAttributeParams { minimum: number; name: string; no_data_color?: ModelCornersAttributeVertexAttributeParamsNoDataColor; - /** - * Flat array of [value, r, g, b, ...] - */ - points: number[]; + points: number[]; // Flat array of [value, r, g, b, ...] } export interface ModelCornersAttributeVertexAttributeParamsNoDataColor { @@ -798,10 +756,7 @@ export interface ModelLinesAttributeEdgeAttributeParams { minimum: number; name: string; no_data_color?: ModelLinesAttributeEdgeAttributeParamsNoDataColor; - /** - * Flat array of [value, r, g, b, ...] - */ - points: number[]; + points: number[]; // Flat array of [value, r, g, b, ...] } export interface ModelLinesAttributeEdgeAttributeParamsNoDataColor { @@ -822,10 +777,7 @@ export interface ModelLinesAttributeVertexAttributeParams { minimum: number; name: string; no_data_color?: ModelLinesAttributeVertexAttributeParamsNoDataColor; - /** - * Flat array of [value, r, g, b, ...] - */ - points: number[]; + points: number[]; // Flat array of [value, r, g, b, ...] } export interface ModelLinesAttributeVertexAttributeParamsNoDataColor { @@ -846,10 +798,7 @@ export interface ModelLinesPointsAttributeVertexAttributeParams { minimum: number; name: string; no_data_color?: ModelLinesPointsAttributeVertexAttributeParamsNoDataColor; - /** - * Flat array of [value, r, g, b, ...] - */ - points: number[]; + points: number[]; // Flat array of [value, r, g, b, ...] } export interface ModelLinesPointsAttributeVertexAttributeParamsNoDataColor { @@ -1002,10 +951,7 @@ export interface ModelSurfacesAttributePolygonAttributeParams { minimum: number; name: string; no_data_color?: ModelSurfacesAttributePolygonAttributeParamsNoDataColor; - /** - * Flat array of [value, r, g, b, ...] - */ - points: number[]; + points: number[]; // Flat array of [value, r, g, b, ...] } export interface ModelSurfacesAttributePolygonAttributeParamsNoDataColor { @@ -1026,10 +972,7 @@ export interface ModelSurfacesAttributeVertexAttributeParams { minimum: number; name: string; no_data_color?: ModelSurfacesAttributeVertexAttributeParamsNoDataColor; - /** - * Flat array of [value, r, g, b, ...] - */ - points: number[]; + points: number[]; // Flat array of [value, r, g, b, ...] } export interface ModelSurfacesAttributeVertexAttributeParamsNoDataColor { @@ -1050,10 +993,7 @@ export interface ModelSurfacesEdgesAttributeVertexAttributeParams { minimum: number; name: string; no_data_color?: ModelSurfacesEdgesAttributeVertexAttributeParamsNoDataColor; - /** - * Flat array of [value, r, g, b, ...] - */ - points: number[]; + points: number[]; // Flat array of [value, r, g, b, ...] } export interface ModelSurfacesEdgesAttributeVertexAttributeParamsNoDataColor { @@ -1108,10 +1048,7 @@ export interface ModelSurfacesPointsAttributeVertexAttributeParams { minimum: number; name: string; no_data_color?: ModelSurfacesPointsAttributeVertexAttributeParamsNoDataColor; - /** - * Flat array of [value, r, g, b, ...] - */ - points: number[]; + points: number[]; // Flat array of [value, r, g, b, ...] } export interface ModelSurfacesPointsAttributeVertexAttributeParamsNoDataColor { @@ -1290,10 +1227,7 @@ export interface ViewerTakeScreenshotParams { export type OutputExtension = "png" | "jpg"; export interface ViewerTakeScreenshotResponse { - /** - * Image file content, sent as binary - */ - blob: any; + blob: any; // Image file content, sent as binary } export interface ViewerSliceParams { From b753a4c33bc7bfa2aa3fb749298d18c3619010b9 Mon Sep 17 00:00:00 2001 From: BotellaA <3213882+BotellaA@users.noreply.github.com> Date: Fri, 20 Nov 2026 19:19:49 +0000 Subject: [PATCH 4/4] Apply prepare changes --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5283b0a5..0e8e27eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -61,4 +61,3 @@ wslink==2.6.0 yarl==1.* # via aiohttp -opengeodeweb-microservice==1.*,>=1.4.0rc1