diff --git a/requirements.txt b/requirements.txt index 5283b0a5..0e8e27eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -61,4 +61,3 @@ wslink==2.6.0 yarl==1.* # via aiohttp -opengeodeweb-microservice==1.*,>=1.4.0rc1 diff --git a/src/opengeodeweb_viewer/rpc/model/model_protocols.py b/src/opengeodeweb_viewer/rpc/model/model_protocols.py index b6591e26..10d094f7 100644 --- a/src/opengeodeweb_viewer/rpc/model/model_protocols.py +++ b/src/opengeodeweb_viewer/rpc/model/model_protocols.py @@ -37,6 +37,7 @@ PointsSizeActor, VtkPipeline, ) +from opengeodeweb_viewer.vtm_prefetch import prefetch_vtm_pieces from . import schemas @@ -487,8 +488,10 @@ def register_model(self, params: schemas.Register) -> schemas.RegisterResponse: viewer_data = self.get_viewer_data(data_id) file_name = str(viewer_data.viewable_file) + vtm_path = Path(self.DATA_FOLDER_PATH) / data_id / file_name reader = vtkXMLMultiBlockDataReader() - reader.SetFileName(str(Path(self.DATA_FOLDER_PATH) / data_id / file_name)) + reader.SetFileName(str(vtm_path)) + prefetch_vtm_pieces(vtm_path) reader.Update() mapper = vtkCompositePolyDataMapper() attributes = vtkCompositeDataDisplayAttributes() diff --git a/src/opengeodeweb_viewer/vtm_prefetch.py b/src/opengeodeweb_viewer/vtm_prefetch.py new file mode 100644 index 00000000..b4d0ed28 --- /dev/null +++ b/src/opengeodeweb_viewer/vtm_prefetch.py @@ -0,0 +1,35 @@ +"""Read the pieces of a VTK multiblock file ahead of the VTK reader.""" + +# Standard library imports +import contextlib +import re +from concurrent.futures import ThreadPoolExecutor +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path + +_PIECE_FILE = re.compile(r'\bfile="([^"]+)"') +_PREFETCH_WORKERS = 32 + + +def _read_piece(path: Path) -> None: + # The VTK reader reports missing or unreadable pieces itself + with contextlib.suppress(OSError): + path.read_bytes() + + +def prefetch_vtm_pieces(vtm_path: Path) -> int: + """Read every piece of a .vtm file once, in parallel, and return how many were read. + + The XML multiblock reader opens its pieces one by one. Right after an import they + are new files, and an on-access antivirus scan (Windows Defender) makes each first + open slow: about 5 s for a model of 764 pieces, against 0.6 s for the VTK parsing + itself. Opening them first from threads, which release the GIL during I/O, runs + those scans in parallel; the VTK reader then opens files already scanned and cached. + """ + folder = vtm_path.parent + pieces = [folder / name for name in _PIECE_FILE.findall(vtm_path.read_text())] + with ThreadPoolExecutor(min(_PREFETCH_WORKERS, len(pieces) or 1)) as executor: + list(executor.map(_read_piece, pieces)) + return len(pieces) diff --git a/tests/test_vtm_prefetch.py b/tests/test_vtm_prefetch.py new file mode 100644 index 00000000..f4bff457 --- /dev/null +++ b/tests/test_vtm_prefetch.py @@ -0,0 +1,37 @@ +from pathlib import Path +from typing import TYPE_CHECKING + +from opengeodeweb_viewer import vtm_prefetch +from opengeodeweb_viewer.vtm_prefetch import prefetch_vtm_pieces + +if TYPE_CHECKING: + import pytest + +DATA = Path(__file__).parent / "data" + + +def test_prefetch_reads_every_piece(monkeypatch: pytest.MonkeyPatch) -> None: + read: list[Path] = [] + monkeypatch.setattr(vtm_prefetch, "_read_piece", read.append) + assert prefetch_vtm_pieces(DATA / "cube.vtm") == 45 + assert len(read) == 45 + assert all(path.is_file() for path in read) + assert all(path.parent.parent == DATA for path in read) + + +def test_prefetch_ignores_missing_pieces(tmp_path: Path) -> None: + (tmp_path / "piece.vtp").write_bytes(b"data") + vtm = tmp_path / "model.vtm" + vtm.write_text( + '' + '' + '' + "" + ) + assert prefetch_vtm_pieces(vtm) == 2 + + +def test_prefetch_without_pieces(tmp_path: Path) -> None: + vtm = tmp_path / "empty.vtm" + vtm.write_text("") + assert prefetch_vtm_pieces(vtm) == 0