Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -61,4 +61,3 @@ wslink==2.6.0
yarl==1.*
# via aiohttp

opengeodeweb-microservice==1.*,>=1.4.0rc1
5 changes: 4 additions & 1 deletion src/opengeodeweb_viewer/rpc/model/model_protocols.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
PointsSizeActor,
VtkPipeline,
)
from opengeodeweb_viewer.vtm_prefetch import prefetch_vtm_pieces

from . import schemas

Expand Down Expand Up @@ -487,8 +488,10 @@ def register_model(self, params: schemas.Register) -> schemas.RegisterResponse:
viewer_data = self.get_viewer_data(data_id)
file_name = str(viewer_data.viewable_file)

vtm_path = Path(self.DATA_FOLDER_PATH) / data_id / file_name
reader = vtkXMLMultiBlockDataReader()
reader.SetFileName(str(Path(self.DATA_FOLDER_PATH) / data_id / file_name))
reader.SetFileName(str(vtm_path))
prefetch_vtm_pieces(vtm_path)
reader.Update()
mapper = vtkCompositePolyDataMapper()
attributes = vtkCompositeDataDisplayAttributes()
Expand Down
35 changes: 35 additions & 0 deletions src/opengeodeweb_viewer/vtm_prefetch.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
"""Read the pieces of a VTK multiblock file ahead of the VTK reader."""

# Standard library imports
import contextlib
import re
from concurrent.futures import ThreadPoolExecutor
from typing import TYPE_CHECKING

if TYPE_CHECKING:
from pathlib import Path

_PIECE_FILE = re.compile(r'\bfile="([^"]+)"')
_PREFETCH_WORKERS = 32


def _read_piece(path: Path) -> None:
# The VTK reader reports missing or unreadable pieces itself
with contextlib.suppress(OSError):
path.read_bytes()


def prefetch_vtm_pieces(vtm_path: Path) -> int:
"""Read every piece of a .vtm file once, in parallel, and return how many were read.

The XML multiblock reader opens its pieces one by one. Right after an import they
are new files, and an on-access antivirus scan (Windows Defender) makes each first

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should this be done only on windows ?

open slow: about 5 s for a model of 764 pieces, against 0.6 s for the VTK parsing
itself. Opening them first from threads, which release the GIL during I/O, runs
those scans in parallel; the VTK reader then opens files already scanned and cached.
"""
folder = vtm_path.parent
pieces = [folder / name for name in _PIECE_FILE.findall(vtm_path.read_text())]
with ThreadPoolExecutor(min(_PREFETCH_WORKERS, len(pieces) or 1)) as executor:
list(executor.map(_read_piece, pieces))
return len(pieces)
37 changes: 37 additions & 0 deletions tests/test_vtm_prefetch.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
from pathlib import Path
from typing import TYPE_CHECKING

from opengeodeweb_viewer import vtm_prefetch
from opengeodeweb_viewer.vtm_prefetch import prefetch_vtm_pieces

if TYPE_CHECKING:
import pytest

DATA = Path(__file__).parent / "data"


def test_prefetch_reads_every_piece(monkeypatch: pytest.MonkeyPatch) -> None:
read: list[Path] = []
monkeypatch.setattr(vtm_prefetch, "_read_piece", read.append)
assert prefetch_vtm_pieces(DATA / "cube.vtm") == 45
assert len(read) == 45
assert all(path.is_file() for path in read)
assert all(path.parent.parent == DATA for path in read)


def test_prefetch_ignores_missing_pieces(tmp_path: Path) -> None:
(tmp_path / "piece.vtp").write_bytes(b"data")
vtm = tmp_path / "model.vtm"
vtm.write_text(
'<VTKFile><vtkMultiBlockDataSet><Block index="0">'
'<DataSet index="0" file="piece.vtp" />'
'<DataSet index="1" file="missing.vtp" />'
"</Block></vtkMultiBlockDataSet></VTKFile>"
)
assert prefetch_vtm_pieces(vtm) == 2


def test_prefetch_without_pieces(tmp_path: Path) -> None:
vtm = tmp_path / "empty.vtm"
vtm.write_text("<VTKFile><vtkMultiBlockDataSet /></VTKFile>")
assert prefetch_vtm_pieces(vtm) == 0
Loading