diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index f84c481..b22f7e5 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -237,7 +237,7 @@ jobs: uv pip install "black==22.12.0" "coveralls>=4" \ "cytoolz==0.12.2" "dask>=2026" "isort>=5.12.0" \ "multimethod<2.0" "nbmake>=1.4.6" "numba>=0.57" \ - "numpy>=2,<3" "openmatrix==0.3.5.0" \ + "numpy>=2,<3" "openmatrix>=0.4.0" \ "pandera>=0.30,<0.35" "pandas>=2,<3" "platformdirs>=3.2" \ "psutil>=5.9" "pyarrow>=11.0" "pydantic>=2.6,<3" "pypyr>=5.8" \ "tables>=3.9" "pytest>=7.2" "pytest-cov" "pytest-regressions" \ diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..dcc6ec7 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,66 @@ +# Sharrow OMX benchmarks + +`benchmark_mtc.py` is a standalone, cross-platform benchmark for the full-scale +MTC `skims.omx`. It needs only Python and [uv](https://docs.astral.sh/uv/). +The script installs its own Python dependencies and creates isolated cached +environments for the release and development versions of Sharrow. It does not +use `git`, `curl`, or a system archive utility. + +Run the default comparison with either command: + +```shell +uv run benchmarks/benchmark_mtc.py +python benchmarks/benchmark_mtc.py +``` + +The script downloads `data_full.tar.zst` from the MTC v1.3.4 release, validates +its published SHA256 checksum, and extracts it with `wring`. Downloads, +extracted data, exact GitHub source snapshots, and uv environments are cached in +the platform's user cache directory. Set `SHARROW_BENCHMARK_CACHE` or pass +`--cache-dir` to put the cache elsewhere. + +By default, the benchmark compares the latest ActivitySim/Sharrow GitHub release +with: + +```text +https://github.com/driftlesslabs/sharrow/tree/codex/openmatrix-0.4-performance +``` + +Select another development source using a branch name, an `OWNER/REPO@REF` +identifier, or a GitHub tree URL: + +```shell +uv run benchmarks/benchmark_mtc.py --development feature/my-branch +uv run benchmarks/benchmark_mtc.py --development ActivitySim/sharrow@my-branch +uv run benchmarks/benchmark_mtc.py --development https://github.com/OWNER/REPO/tree/BRANCH +``` + +The default run executes one warmup and three measured repetitions of lazy, +eager (when available), and ActivitySim-style shared-memory reloads. Every trial +runs in a fresh process. Peak memory is sampled across the complete process +tree, so it includes the final dataset, temporary allocations, and child +workers. Both Sharrow builds use the same pinned Python and dependency versions +so the comparison isolates their code changes and remains repeatable across +machines. Results and exact source commits are written to +`mtc-sharrow-benchmark.json` by default. + +Useful options include: + +```shell +# Quick comparison of the directly comparable lazy loaders +uv run benchmarks/benchmark_mtc.py --strategies lazy --warmups 0 --repetitions 1 + +# Use a previously available MTC skim file +uv run benchmarks/benchmark_mtc.py --omx /path/to/skims.omx + +# Download data and build environments now, run trials later +uv run benchmarks/benchmark_mtc.py --prepare-only + +# Force cache refreshes +uv run benchmarks/benchmark_mtc.py --refresh-data --rebuild-environments +``` + +The materialized dataset is about 6.8 GiB, while the release lazy loader can +peak near 17 GiB. A machine with at least 24 GiB of available memory and several +GiB of free cache space is recommended. Set `GITHUB_TOKEN` if unauthenticated +GitHub API rate limits are an issue. diff --git a/benchmarks/benchmark_mtc.py b/benchmarks/benchmark_mtc.py new file mode 100644 index 0000000..550e325 --- /dev/null +++ b/benchmarks/benchmark_mtc.py @@ -0,0 +1,1181 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "psutil>=5.9", +# "wring==0.0.6", +# ] +# /// +"""Reproduce Sharrow OMX load benchmarks with the full-scale MTC skims. + +This one-file benchmark downloads and caches the MTC ``data_full`` archive, +extracts it with wring, resolves the latest ActivitySim/Sharrow GitHub release +and a development branch to exact commits, and installs each source snapshot in +an isolated uv environment. No system tools such as git, curl, or tar are used. + +Run with either of these commands:: + + uv run benchmark_mtc.py + python benchmark_mtc.py + +The default development source is:: + + https://github.com/driftlesslabs/sharrow/tree/codex/openmatrix-0.4-performance + +Use ``--development BRANCH``, ``--development OWNER/REPO@BRANCH``, or another +GitHub tree URL to benchmark a different development branch. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib +import importlib.util +import inspect +import json +import os +import platform +import re +import shutil +import statistics +import subprocess +import sys +import time +import traceback +import types +import urllib.error +import urllib.parse +import urllib.request +import uuid +from dataclasses import asdict, dataclass +from pathlib import Path + +DATA_URL = ( + "https://github.com/ActivitySim/activitysim-prototype-mtc/" + "releases/download/v1.3.4/data_full.tar.zst" +) +DATA_SHA256 = "b402506a61055e2d38621416dd9a5c7e3cf7517c0a9ae5869f6d760c03284ef3" +DEFAULT_DEVELOPMENT = ( + "https://github.com/driftlesslabs/sharrow/tree/codex/openmatrix-0.4-performance" +) +GITHUB_API = "https://api.github.com" +RESULT_PREFIX = "SHARROW_MTC_BENCHMARK_RESULT=" +GIB = 1024**3 + +# Install identical runtime dependencies in both benchmark environments. The +# source package itself is installed without dependencies because release +# 2.16.2 made optional HDF5 plugins mandatory in its metadata even though the +# gzip-compressed MTC OMX file does not use them. Lightweight import stubs in +# the worker preserve that release's code path without requiring binary plugins +# that are unavailable on some Windows architectures. +BENCHMARK_DEPENDENCIES = ( + "numpy==2.2.6", + "pandas==2.3.3", + "pyarrow==23.0.1", + "h5py==3.16.0", + "openmatrix==0.4.0", + "xarray==2025.6.1", + "numba==0.65.0", + "numexpr==2.14.1", + "filelock==3.25.2", + "dask==2023.11.0", + "networkx==3.4.2", + "psutil==7.2.2", +) + + +def _bootstrap_with_uv() -> int | None: + """Re-run the controller through uv when script dependencies are absent.""" + if "--worker" in sys.argv: + return None + required = ("psutil", "wring") + if all(importlib.util.find_spec(name) is not None for name in required): + return None + if os.environ.get("SHARROW_BENCHMARK_BOOTSTRAPPED") == "1": + missing = [name for name in required if importlib.util.find_spec(name) is None] + raise RuntimeError(f"uv did not install required packages: {missing}") + uv = shutil.which("uv") + if uv is None: + raise RuntimeError("uv is required and was not found on PATH") + environment = os.environ.copy() + environment["SHARROW_BENCHMARK_BOOTSTRAPPED"] = "1" + command = [ + uv, + "run", + "--no-project", + "--script", + str(Path(__file__).resolve()), + *sys.argv[1:], + ] + return subprocess.call(command, env=environment) + + +_bootstrap_result = _bootstrap_with_uv() +if _bootstrap_result is not None: + raise SystemExit(_bootstrap_result) + +import psutil # noqa: E402 (available after the optional uv bootstrap) + +try: + import resource +except ImportError: # Windows has no resource module; the controller samples RSS. + resource = None + + +@dataclass(frozen=True) +class SourceSpec: + """An immutable GitHub source snapshot used to build one environment.""" + + kind: str + label: str + owner: str + repository: str + ref: str + revision: str + archive_url: str + requested: str + version: str + + @property + def cache_key(self) -> str: + """Return a short filesystem-safe identity for this source snapshot.""" + prefix = re.sub(r"[^A-Za-z0-9_.-]+", "-", f"{self.owner}-{self.repository}") + return f"{self.kind}-{prefix}-{self.revision[:16]}" + + +@dataclass(frozen=True) +class BenchmarkCase: + """One isolated source/environment and loading-strategy combination.""" + + label: str + source_kind: str + strategy: str + python: str + source: SourceSpec + + +def _default_cache_dir() -> Path: + """Return a conventional per-user cache directory on each operating system.""" + override = os.environ.get("SHARROW_BENCHMARK_CACHE") + if override: + return Path(override).expanduser() + if os.name == "nt": + root = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local")) + elif sys.platform == "darwin": + root = Path.home() / "Library" / "Caches" + else: + root = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) + return root / "sharrow-mtc-benchmark" + + +def _github_headers(authenticated: bool = True) -> dict[str, str]: + """Build public GitHub API headers, optionally using GITHUB_TOKEN.""" + headers = { + "Accept": "application/vnd.github+json", + "User-Agent": "sharrow-mtc-benchmark", + "X-GitHub-Api-Version": "2022-11-28", + } + token = os.environ.get("GITHUB_TOKEN") + if authenticated and token: + headers["Authorization"] = f"Bearer {token}" + return headers + + +def _github_json(path: str) -> dict | list: + """Read one GitHub API object with useful rate-limit diagnostics.""" + request = urllib.request.Request( + f"{GITHUB_API}{path}", headers=_github_headers(authenticated=True) + ) + try: + with urllib.request.urlopen(request, timeout=60) as response: + return json.load(response) + except urllib.error.HTTPError as error: + rate_limit = error.headers.get("X-RateLimit-Remaining") == "0" + hint = " Set GITHUB_TOKEN to raise the API rate limit." if rate_limit else "" + raise RuntimeError( + f"GitHub API request failed ({error.code}) for {path}.{hint}" + ) from error + + +def _resolve_commit(owner: str, repository: str, ref: str) -> str: + """Resolve a GitHub branch, tag, or SHA to an exact commit hash.""" + encoded_ref = urllib.parse.quote(ref, safe="") + payload = _github_json(f"/repos/{owner}/{repository}/commits/{encoded_ref}") + if not isinstance(payload, dict) or not re.fullmatch( + r"[0-9a-fA-F]{40}", str(payload.get("sha", "")) + ): + raise RuntimeError(f"GitHub returned no commit for {owner}/{repository}@{ref}") + return str(payload["sha"]).lower() + + +def _source_archive_url(owner: str, repository: str, revision: str) -> str: + """Return a public archive URL that does not require a local git client.""" + return f"https://github.com/{owner}/{repository}/archive/{revision}.tar.gz" + + +def _release_source(selector: str) -> SourceSpec: + """Resolve the latest release, or an explicitly requested Sharrow tag.""" + owner, repository = "ActivitySim", "sharrow" + if selector == "latest": + payload = _github_json(f"/repos/{owner}/{repository}/releases/latest") + if not isinstance(payload, dict) or not payload.get("tag_name"): + raise RuntimeError("the latest Sharrow GitHub release has no tag") + ref = str(payload["tag_name"]) + else: + ref = selector + revision = _resolve_commit(owner, repository, ref) + version = ref[1:] if ref.startswith("v") else ref + if not re.fullmatch(r"[0-9]+(?:\.[0-9]+)*(?:[A-Za-z0-9_.+-]*)", version): + version = "0.0" + return SourceSpec( + kind="release", + label=f"Sharrow {ref}", + owner=owner, + repository=repository, + ref=ref, + revision=revision, + archive_url=_source_archive_url(owner, repository, revision), + requested=selector, + version=version, + ) + + +def _parse_development_selector(selector: str) -> tuple[str, str, str]: + """Parse a branch name, OWNER/REPO@REF, or GitHub tree/commit URL.""" + default_owner, default_repository = "driftlesslabs", "sharrow" + parsed = urllib.parse.urlparse(selector) + if parsed.scheme in {"http", "https"}: + if parsed.netloc.lower() not in {"github.com", "www.github.com"}: + raise ValueError("development URLs must refer to github.com") + parts = [urllib.parse.unquote(part) for part in parsed.path.split("/") if part] + if len(parts) < 4 or parts[2] not in {"tree", "commit"}: + raise ValueError( + "development URL must look like https://github.com/OWNER/REPO/tree/REF" + ) + return parts[0], parts[1].removesuffix(".git"), "/".join(parts[3:]) + + if "@" in selector: + repository_name, ref = selector.rsplit("@", 1) + repository_parts = repository_name.split("/", 1) + if len(repository_parts) != 2 or not all(repository_parts) or not ref: + raise ValueError("development selector must be OWNER/REPO@REF") + return repository_parts[0], repository_parts[1].removesuffix(".git"), ref + + if not selector: + raise ValueError("development branch must not be empty") + return default_owner, default_repository, selector + + +def _development_source(selector: str) -> SourceSpec: + """Resolve a named or linked development branch to an exact commit.""" + owner, repository, ref = _parse_development_selector(selector) + revision = _resolve_commit(owner, repository, ref) + return SourceSpec( + kind="development", + label=f"{owner}/{repository}@{ref}", + owner=owner, + repository=repository, + ref=ref, + revision=revision, + archive_url=_source_archive_url(owner, repository, revision), + requested=selector, + version=f"9999.0.dev0+g{revision[:12]}", + ) + + +def _sha256(path: Path) -> str: + """Hash a potentially large file without loading it into memory.""" + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _download_once(url: str, destination: Path) -> None: + """Download one URL, resuming a partial response when the server permits.""" + destination.parent.mkdir(parents=True, exist_ok=True) + partial = destination.with_name(destination.name + ".part") + offset = partial.stat().st_size if partial.exists() else 0 + headers = _github_headers(authenticated=False) + if offset: + headers["Range"] = f"bytes={offset}-" + request = urllib.request.Request(url, headers=headers) + with urllib.request.urlopen(request, timeout=60) as response: + status = getattr(response, "status", None) + append = offset > 0 and status == 206 + mode = "ab" if append else "wb" + downloaded = offset if append else 0 + content_length = response.headers.get("Content-Length") + total = downloaded + int(content_length) if content_length else None + last_update = 0.0 + with partial.open(mode) as stream: + while True: + block = response.read(1024 * 1024) + if not block: + break + stream.write(block) + downloaded += len(block) + now = time.monotonic() + if now - last_update >= 0.5: + if total: + progress = f"{downloaded / GIB:.2f}/{total / GIB:.2f} GiB" + else: + progress = f"{downloaded / GIB:.2f} GiB" + print( + f"\rDownloading {destination.name}: {progress}", + end="", + flush=True, + ) + last_update = now + print() + partial.replace(destination) + + +def _download(url: str, destination: Path, expected_sha256: str | None = None) -> Path: + """Return a cached download, verifying and retrying known content hashes.""" + if destination.is_file(): + if expected_sha256 is None or _sha256(destination) == expected_sha256: + print(f"Using cached download: {destination}") + return destination + print(f"Discarding cached file with an invalid checksum: {destination}") + destination.unlink() + + for attempt in range(2): + _download_once(url, destination) + if expected_sha256 is None or _sha256(destination) == expected_sha256: + return destination + destination.unlink(missing_ok=True) + destination.with_name(destination.name + ".part").unlink(missing_ok=True) + if attempt == 0: + print("Checksum mismatch; retrying the download from the beginning.") + raise RuntimeError(f"SHA256 mismatch for {url}; expected {expected_sha256}") + + +def _extract_with_wring(archive: Path, destination: Path) -> None: + """Load wring's tar.zst extractor without its unrelated OMX extras. + + Wring 0.0.6 imports its optional OMX module from ``wring.__init__``, which + otherwise makes the archive tool require NumPy and PyTables. Loading the + packaged ``tar_zst`` submodule directly keeps extraction portable while + still using wring's cross-platform implementation. + """ + package_spec = importlib.util.find_spec("wring") + if package_spec is None or not package_spec.submodule_search_locations: + raise RuntimeError("wring is not installed in the controller environment") + if "wring" not in sys.modules: + package = types.ModuleType("wring") + package.__package__ = "wring" + package.__path__ = list(package_spec.submodule_search_locations) + package.__spec__ = package_spec + sys.modules["wring"] = package + tar_zst = importlib.import_module("wring.tar_zst") + tar_zst.extract_zst(archive, destination) + + +def _prepare_mtc_data(cache_dir: Path, refresh: bool = False) -> Path: + """Download, verify, and transactionally extract the full MTC data archive.""" + data_root = cache_dir / "data" / "v1.3.4" + archive = data_root / "data_full.tar.zst" + extracted = data_root / "extracted" + marker = extracted / ".sharrow-benchmark-data.json" + if refresh: + archive.unlink(missing_ok=True) + archive.with_name(archive.name + ".part").unlink(missing_ok=True) + if extracted.exists(): + shutil.rmtree(extracted) + + if marker.is_file(): + try: + metadata = json.loads(marker.read_text(encoding="utf-8")) + except (OSError, ValueError): + metadata = {} + else: + candidate = extracted / metadata.get("skims_relative_path", "") + if metadata.get("sha256") == DATA_SHA256 and candidate.is_file(): + print(f"Using cached MTC skims: {candidate}") + return candidate + + archive = _download(DATA_URL, archive, DATA_SHA256) + temporary = data_root / f"extracting-{os.getpid()}-{uuid.uuid4().hex[:8]}" + temporary.mkdir(parents=True, exist_ok=False) + try: + # wring supplies the cross-platform zstandard/tar extraction workflow. + _extract_with_wring(archive, temporary) + candidates = sorted(temporary.rglob("skims.omx")) + if len(candidates) != 1: + raise RuntimeError( + f"expected one skims.omx in the MTC archive, found {len(candidates)}" + ) + relative = candidates[0].relative_to(temporary) + if extracted.exists(): + shutil.rmtree(extracted) + temporary.replace(extracted) + marker.write_text( + json.dumps( + {"sha256": DATA_SHA256, "skims_relative_path": str(relative)}, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + except BaseException: + if temporary.exists(): + shutil.rmtree(temporary) + raise + skims = extracted / relative + print(f"Cached MTC skims: {skims}") + return skims + + +def _environment_python(environment: Path) -> Path: + """Return a virtual environment's interpreter path on Windows or POSIX.""" + if os.name == "nt": + return environment / "Scripts" / "python.exe" + return environment / "bin" / "python" + + +def _run_checked(command: list[str], **kwargs) -> None: + """Print and run an environment-setup command, failing immediately.""" + print(f"+ {subprocess.list2cmdline([str(item) for item in command])}") + subprocess.run([str(item) for item in command], check=True, **kwargs) + + +def _isolated_environment() -> dict[str, str]: + """Prevent a local checkout or user site from contaminating a trial.""" + environment = os.environ.copy() + environment.pop("PYTHONPATH", None) + environment.pop("PYTHONHOME", None) + environment["PYTHONNOUSERSITE"] = "1" + return environment + + +def _source_archive(source: SourceSpec, cache_dir: Path) -> Path: + """Download and cache an exact GitHub source archive.""" + archive = cache_dir / "sources" / f"{source.cache_key}.tar.gz" + return _download(source.archive_url, archive) + + +def _probe_environment(python: Path, cache_dir: Path) -> dict: + """Import the installed Sharrow build and report supported load modes.""" + command = [str(python), str(Path(__file__).resolve()), "--worker", "--probe"] + completed = subprocess.run( + command, + check=True, + cwd=cache_dir, + env=_isolated_environment(), + capture_output=True, + text=True, + ) + for line in reversed(completed.stdout.splitlines()): + if line.startswith(RESULT_PREFIX): + return json.loads(line[len(RESULT_PREFIX) :]) + raise RuntimeError( + "installed Sharrow environment could not be probed:\n" + f"stdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + + +def _ensure_environment( + source: SourceSpec, + cache_dir: Path, + python_request: str, + rebuild: bool, +) -> tuple[Path, dict]: + """Build or reuse a uv environment for one exact Sharrow source snapshot.""" + environment = cache_dir / "environments" / source.cache_key + interpreter = _environment_python(environment) + marker = environment / ".sharrow-benchmark-environment.json" + expected = { + "source": asdict(source), + "python_request": python_request, + "dependencies": list(BENCHMARK_DEPENDENCIES), + } + if not rebuild and interpreter.is_file() and marker.is_file(): + try: + cached_metadata = json.loads(marker.read_text(encoding="utf-8")) + except (OSError, ValueError): + cached_metadata = None + if cached_metadata == expected: + print(f"Using cached environment: {environment}") + return interpreter, _probe_environment(interpreter, cache_dir) + + if environment.exists(): + shutil.rmtree(environment) + environment.parent.mkdir(parents=True, exist_ok=True) + archive = _source_archive(source, cache_dir) + uv = shutil.which("uv") + if uv is None: + raise RuntimeError("uv is required and was not found on PATH") + _run_checked( + [uv, "venv", "--python", python_request, str(environment)], cwd=cache_dir + ) + interpreter = _environment_python(environment) + _run_checked( + [ + uv, + "pip", + "install", + "--python", + str(interpreter), + *BENCHMARK_DEPENDENCIES, + ], + cwd=cache_dir, + ) + install_environment = os.environ.copy() + install_environment["SETUPTOOLS_SCM_PRETEND_VERSION_FOR_SHARROW"] = source.version + _run_checked( + [ + uv, + "pip", + "install", + "--python", + str(interpreter), + "--no-deps", + "--reinstall", + str(archive), + ], + cwd=cache_dir, + env=install_environment, + ) + probe = _probe_environment(interpreter, cache_dir) + marker.write_text(json.dumps(expected, indent=2, sort_keys=True) + "\n") + return interpreter, probe + + +def _install_optional_filter_stubs() -> None: + """Allow gzip-only release benchmarks without unavailable binary plugins.""" + for name in ("blosc2", "hdf5plugin"): + try: + importlib.import_module(name) + except (ImportError, OSError): + module = types.ModuleType(name) + module.__doc__ = "Unavailable optional filter stub for the MTC benchmark." + sys.modules[name] = module + + +def _rss_high_water_bytes() -> int: + """Return this process's maximum resident set size in bytes.""" + if resource is None: + return psutil.Process().memory_info().rss + value = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + if platform.system() == "Darwin": + return int(value) + return int(value * 1024) + + +def _dataset_fingerprint(dataset) -> str: + """Hash array metadata and edge values without copying full skim arrays.""" + digest = hashlib.sha256() + for name in sorted(dataset.data_vars): + array = dataset[name].data + digest.update(name.encode()) + digest.update(repr((array.shape, str(array.dtype))).encode()) + if array.size: + first = array[(0,) * array.ndim] + last = array[tuple(size - 1 for size in array.shape)] + digest.update(repr((first, last)).encode()) + return digest.hexdigest() + + +class _H5pyOmxCompatibilityHandle: + """Present the legacy OMX shape/root protocol over an h5py file. + + The MTC file predates the optional root ``SHAPE`` attribute expected by + release 2.16.2's direct h5py-handle path. This adapter exposes the same + small protocol as ``openmatrix.File`` while leaving all matrix reads on the + h5py fast path. Current Sharrow uses the OpenMatrix handle directly. + """ + + def __init__(self, handle, filename: str): + self.filename = filename + self.root = handle["/"] + first_matrix = next(iter(handle["/data"].values())) + self._shape = tuple(int(size) for size in first_matrix.shape) + + def shape(self) -> tuple[int, ...]: + """Return the common OMX matrix shape.""" + return self._shape + + +def _worker_probe() -> int: + """Emit installed-version and API-capability metadata for the controller.""" + _install_optional_filter_stubs() + import sharrow as sh + + parameters = inspect.signature(sh.dataset.from_omx_3d).parameters + payload = { + "status": "ok", + "sharrow_file": str(Path(sh.__file__).resolve()), + "sharrow_version": getattr(sh, "__version__", "unknown"), + "supports_eager": "load" in parameters, + "supports_shared_reload": hasattr(sh.dataset, "reload_from_omx_3d"), + "python_version": platform.python_version(), + } + print(f"{RESULT_PREFIX}{json.dumps(payload, sort_keys=True)}", flush=True) + return 0 + + +def _worker(args: argparse.Namespace) -> int: + """Load the MTC OMX file once and emit a machine-readable trial record.""" + if args.probe: + return _worker_probe() + + import openmatrix + + _install_optional_filter_stubs() + import sharrow as sh + + process = psutil.Process() + omx_path = str(Path(args.omx).resolve()) + result = { + "label": args.label, + "source_kind": args.source_kind, + "source_revision": args.source_revision, + "strategy": args.strategy, + "omx": omx_path, + "sharrow_file": str(Path(sh.__file__).resolve()), + "sharrow_version": getattr(sh, "__version__", "unknown"), + "python_version": platform.python_version(), + "baseline_rss_bytes": process.memory_info().rss, + } + dataset = None + started = time.perf_counter() + try: + kwargs = { + "time_periods": args.periods.split(","), + "max_float_precision": args.max_float_precision, + } + if args.strategy == "shared-reload": + with openmatrix.open_file(omx_path, "r") as handle: + source = ( + handle + if "load" in inspect.signature(sh.dataset.from_omx_3d).parameters + else _H5pyOmxCompatibilityHandle(handle, omx_path) + ) + template = sh.dataset.from_omx_3d(source, **kwargs) + shared_key = f"sharrow-mtc-benchmark-{os.getpid()}-{time.time_ns()}" + dataset = template.shm.to_shared_memory(shared_key, mode="r+", load=False) + sh.dataset.reload_from_omx_3d(dataset, [omx_path]) + elif args.strategy == "eager": + kwargs["load"] = "eager" + with openmatrix.open_file(omx_path, "r") as handle: + source = ( + handle + if "load" in inspect.signature(sh.dataset.from_omx_3d).parameters + else _H5pyOmxCompatibilityHandle(handle, omx_path) + ) + dataset = sh.dataset.from_omx_3d(source, **kwargs) + else: + with openmatrix.open_file(omx_path, "r") as handle: + source = ( + handle + if "load" in inspect.signature(sh.dataset.from_omx_3d).parameters + else _H5pyOmxCompatibilityHandle(handle, omx_path) + ) + dataset = sh.dataset.from_omx_3d(source, **kwargs) + dataset.load() + result.update( + { + "status": "ok", + "elapsed_seconds": time.perf_counter() - started, + "dataset_nbytes": int(dataset.nbytes), + "final_rss_bytes": process.memory_info().rss, + "variable_count": len(dataset.data_vars), + "dimensions": {name: int(size) for name, size in dataset.sizes.items()}, + "fingerprint": _dataset_fingerprint(dataset), + } + ) + except Exception as error: # noqa: BLE001 - failures belong in benchmark JSON. + result.update( + { + "status": "error", + "elapsed_seconds": time.perf_counter() - started, + "final_rss_bytes": process.memory_info().rss, + "error_type": type(error).__name__, + "error": str(error), + "traceback": traceback.format_exc(), + } + ) + finally: + result["worker_peak_rss_bytes"] = _rss_high_water_bytes() + if args.strategy == "shared-reload" and dataset is not None: + try: + dataset.shm.release_shared_memory() + except Exception: # noqa: BLE001 - retain the primary trial result. + result["cleanup_traceback"] = traceback.format_exc() + + print(f"{RESULT_PREFIX}{json.dumps(result, sort_keys=True)}", flush=True) + time.sleep(args.hold_seconds) + return 0 + + +def _process_tree_rss(process: psutil.Process) -> int: + """Sum current RSS across a worker and all live descendants.""" + try: + descendants = process.children(recursive=True) + except (psutil.Error, OSError): + descendants = [] + total = 0 + for item in [process, *descendants]: + try: + total += item.memory_info().rss + except (psutil.Error, OSError): + continue + return total + + +def _extract_worker_result(stdout: str) -> dict: + """Extract the final JSON record from worker output.""" + for line in reversed(stdout.splitlines()): + if line.startswith(RESULT_PREFIX): + return json.loads(line[len(RESULT_PREFIX) :]) + raise RuntimeError("worker produced no benchmark result") + + +def _run_trial( + args: argparse.Namespace, + case: BenchmarkCase, + cache_dir: Path, + omx: Path, + run_number: int, + warmup: bool, +) -> dict: + """Launch and externally monitor one isolated benchmark trial.""" + command = [ + case.python, + str(Path(__file__).resolve()), + "--worker", + "--omx", + str(omx), + "--label", + case.label, + "--source-kind", + case.source_kind, + "--source-revision", + case.source.revision, + "--strategy", + case.strategy, + "--periods", + args.periods, + "--max-float-precision", + str(args.max_float_precision), + "--hold-seconds", + str(args.hold_seconds), + ] + child = subprocess.Popen( + command, + cwd=cache_dir, + env=_isolated_environment(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + monitored = psutil.Process(child.pid) + peak_tree_rss = 0 + while child.poll() is None: + peak_tree_rss = max(peak_tree_rss, _process_tree_rss(monitored)) + time.sleep(args.sample_interval) + peak_tree_rss = max(peak_tree_rss, _process_tree_rss(monitored)) + stdout, stderr = child.communicate() + try: + result = _extract_worker_result(stdout) + except Exception as error: # noqa: BLE001 - preserve process diagnostics. + result = { + "label": case.label, + "source_kind": case.source_kind, + "source_revision": case.source.revision, + "strategy": case.strategy, + "status": "error", + "error_type": type(error).__name__, + "error": str(error), + "elapsed_seconds": 0.0, + "final_rss_bytes": 0, + "worker_peak_rss_bytes": 0, + "stdout": stdout, + } + result.update( + { + "run": run_number, + "warmup": warmup, + "returncode": child.returncode, + "peak_tree_rss_bytes": peak_tree_rss, + "peak_rss_bytes": max( + peak_tree_rss, result.get("worker_peak_rss_bytes", 0) + ), + "stderr": stderr, + } + ) + return result + + +def _gib(value: int | float) -> float: + """Convert bytes to GiB for terminal reporting.""" + return value / GIB + + +def _print_trial(result: dict) -> None: + """Print one compact benchmark trial.""" + message = ( + f"{result['label']:24} {result['strategy']:13} " + f"run={result['run']} warmup={result['warmup']} " + f"status={result['status']:5} time={result['elapsed_seconds']:8.3f}s " + f"peak={_gib(result['peak_rss_bytes']):7.3f} GiB" + ) + if result["status"] == "error": + message += f" error={result['error_type']}: {result['error']}" + print(message, flush=True) + + +def _summaries(results: list[dict]) -> list[dict]: + """Compute medians and ranges for successful measured trials.""" + summaries = [] + labels = dict.fromkeys(result["label"] for result in results) + for label in labels: + trials = [ + result + for result in results + if result["label"] == label + and not result["warmup"] + and result["status"] == "ok" + ] + if not trials: + continue + elapsed = [trial["elapsed_seconds"] for trial in trials] + peak = [trial["peak_rss_bytes"] for trial in trials] + summaries.append( + { + "label": label, + "source_kind": trials[0]["source_kind"], + "source_revision": trials[0]["source_revision"], + "strategy": trials[0]["strategy"], + "trials": len(trials), + "median_elapsed_seconds": statistics.median(elapsed), + "min_elapsed_seconds": min(elapsed), + "max_elapsed_seconds": max(elapsed), + "median_peak_rss_bytes": statistics.median(peak), + "median_final_rss_bytes": statistics.median( + trial["final_rss_bytes"] for trial in trials + ), + "median_baseline_rss_bytes": statistics.median( + trial["baseline_rss_bytes"] for trial in trials + ), + "median_peak_increase_bytes": statistics.median( + trial["peak_rss_bytes"] - trial["baseline_rss_bytes"] + for trial in trials + ), + "dataset_nbytes": trials[0]["dataset_nbytes"], + "fingerprint": trials[0]["fingerprint"], + } + ) + return summaries + + +def _comparisons(summaries: list[dict]) -> list[dict]: + """Compare release and development medians for matching strategies.""" + comparisons = [] + for strategy in dict.fromkeys(summary["strategy"] for summary in summaries): + release = next( + ( + summary + for summary in summaries + if summary["strategy"] == strategy + and summary["source_kind"] == "release" + ), + None, + ) + development = next( + ( + summary + for summary in summaries + if summary["strategy"] == strategy + and summary["source_kind"] == "development" + ), + None, + ) + if release and development: + comparisons.append( + { + "strategy": strategy, + "development_speedup": release["median_elapsed_seconds"] + / development["median_elapsed_seconds"], + "development_peak_rss_ratio": development["median_peak_rss_bytes"] + / release["median_peak_rss_bytes"], + } + ) + return comparisons + + +def _parse_strategies(value: str) -> list[str]: + """Normalize the public lazy/eager/shared strategy list.""" + aliases = {"lazy": "lazy", "eager": "eager", "shared": "shared-reload"} + requested = [item.strip().lower() for item in value.split(",") if item.strip()] + invalid = [item for item in requested if item not in aliases] + if invalid or not requested: + raise argparse.ArgumentTypeError( + "strategies must be a comma-separated subset of lazy,eager,shared" + ) + return list(dict.fromkeys(aliases[item] for item in requested)) + + +def _build_cases( + strategies: list[str], + release: SourceSpec, + release_python: Path, + release_probe: dict, + development: SourceSpec, + development_python: Path, + development_probe: dict, +) -> list[BenchmarkCase]: + """Create only the source/strategy combinations supported by each API.""" + cases = [] + source_rows = ( + ("release", release, release_python, release_probe), + ("development", development, development_python, development_probe), + ) + for strategy in strategies: + for source_kind, source, python, probe in source_rows: + if strategy == "eager" and not probe["supports_eager"]: + print(f"Skipping eager mode for {source.label}: API not available") + continue + if strategy == "shared-reload" and not probe["supports_shared_reload"]: + print(f"Skipping shared mode for {source.label}: API not available") + continue + cases.append( + BenchmarkCase( + label=f"{source_kind}-{strategy.replace('-reload', '')}", + source_kind=source_kind, + strategy=strategy, + python=str(python), + source=source, + ) + ) + return cases + + +def _controller(args: argparse.Namespace) -> int: + """Prepare inputs/environments, execute trials, and write the final report.""" + cache_dir = Path(args.cache_dir).expanduser().resolve() + cache_dir.mkdir(parents=True, exist_ok=True) + if args.omx: + omx = Path(args.omx).expanduser().resolve() + if not omx.is_file(): + raise FileNotFoundError(omx) + data_provenance = {"kind": "local", "omx": str(omx)} + else: + omx = _prepare_mtc_data(cache_dir, refresh=args.refresh_data) + data_provenance = { + "kind": "download", + "url": DATA_URL, + "sha256": DATA_SHA256, + "omx": str(omx), + } + + available = psutil.virtual_memory().available + if available < 20 * GIB: + print( + f"Warning: only {_gib(available):.1f} GiB of memory is currently " + "available; the release lazy load may peak near 17 GiB." + ) + + print("Resolving exact Sharrow source revisions...") + release = _release_source(args.release) + development = _development_source(args.development) + print(f"Release: {release.label} ({release.revision})") + print(f"Development: {development.label} ({development.revision})") + + release_python, release_probe = _ensure_environment( + release, cache_dir, args.python, args.rebuild_environments + ) + development_python, development_probe = _ensure_environment( + development, cache_dir, args.python, args.rebuild_environments + ) + print(f"Release environment: {release_probe['sharrow_file']}") + print(f"Development environment: {development_probe['sharrow_file']}") + if args.prepare_only: + print("Preparation complete; benchmark trials were not requested.") + return 0 + + cases = _build_cases( + args.strategies, + release, + release_python, + release_probe, + development, + development_python, + development_probe, + ) + if not cases: + raise RuntimeError("none of the requested strategies is supported") + + results = [] + for warmup_number in range(1, args.warmups + 1): + for case in cases: + result = _run_trial(args, case, cache_dir, omx, warmup_number, warmup=True) + results.append(result) + _print_trial(result) + + # Rotate order between repetitions to distribute filesystem-cache effects. + for run_number in range(1, args.repetitions + 1): + offset = (run_number - 1) % len(cases) + for case in cases[offset:] + cases[:offset]: + result = _run_trial(args, case, cache_dir, omx, run_number, warmup=False) + results.append(result) + _print_trial(result) + + summaries = _summaries(results) + comparisons = _comparisons(summaries) + successful_fingerprints = { + result["fingerprint"] for result in results if result["status"] == "ok" + } + fingerprints_match = len(successful_fingerprints) == 1 + payload = { + "benchmark": "Sharrow full-scale MTC OMX loading", + "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "system": { + "platform": platform.platform(), + "machine": platform.machine(), + "processor": platform.processor(), + "logical_cpu_count": os.cpu_count(), + "total_memory_bytes": psutil.virtual_memory().total, + "controller_python": platform.python_version(), + "benchmark_python_request": args.python, + "uv_version": subprocess.check_output( + [shutil.which("uv") or "uv", "--version"], text=True + ).strip(), + }, + "data": data_provenance, + "omx_size_bytes": omx.stat().st_size, + "periods": args.periods.split(","), + "max_float_precision": args.max_float_precision, + "memory_metric": "peak RSS across the complete benchmark process tree", + "sample_interval_seconds": args.sample_interval, + "sources": { + "release": asdict(release), + "development": asdict(development), + }, + "environment_probes": { + "release": release_probe, + "development": development_probe, + }, + "fingerprints_match": fingerprints_match, + "results": results, + "summaries": summaries, + "comparisons": comparisons, + } + + print("\nSummary (medians of measured trials)") + for summary in summaries: + print( + f"{summary['label']:24} {summary['strategy']:13} " + f"time={summary['median_elapsed_seconds']:8.3f}s " + f"peak={_gib(summary['median_peak_rss_bytes']):7.3f} GiB " + f"dataset={_gib(summary['dataset_nbytes']):7.3f} GiB" + ) + for comparison in comparisons: + print( + f"{comparison['strategy']:24} development speedup=" + f"{comparison['development_speedup']:.2f}x, peak RSS=" + f"{comparison['development_peak_rss_ratio']:.1%} of release" + ) + + output = Path(args.output).expanduser().resolve() + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + print(f"\nWrote {output}") + + failures = [result for result in results if result["status"] != "ok"] + if failures: + print(f"Benchmark completed with {len(failures)} failed trial(s).") + return 1 + if not fingerprints_match: + print("Benchmark outputs did not have matching fingerprints.") + return 2 + return 0 + + +def _parser() -> argparse.ArgumentParser: + """Build the public controller and private worker command-line interface.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--development", + default=DEFAULT_DEVELOPMENT, + help=( + "development branch name, OWNER/REPO@REF, or GitHub tree URL " + f"(default: {DEFAULT_DEVELOPMENT})" + ), + ) + parser.add_argument( + "--release", + default="latest", + help="ActivitySim/Sharrow release tag, or 'latest' (default: latest)", + ) + parser.add_argument( + "--cache-dir", + default=str(_default_cache_dir()), + help="download, source, and environment cache directory", + ) + parser.add_argument( + "--omx", + help="use an existing skims.omx instead of downloading the MTC archive", + ) + parser.add_argument( + "--python", + default="3.12", + help="Python version used in both benchmark environments (default: 3.12)", + ) + parser.add_argument( + "--strategies", + type=_parse_strategies, + default=_parse_strategies("lazy,eager,shared"), + help="comma-separated subset of lazy,eager,shared", + ) + parser.add_argument("--warmups", type=int, default=1) + parser.add_argument("--repetitions", type=int, default=3) + parser.add_argument("--periods", default="EA,AM,MD,PM,EV") + parser.add_argument("--max-float-precision", type=int, default=32) + parser.add_argument("--sample-interval", type=float, default=0.005) + parser.add_argument("--hold-seconds", type=float, default=0.25) + parser.add_argument("--output", default="mtc-sharrow-benchmark.json") + parser.add_argument( + "--refresh-data", action="store_true", help="redownload and re-extract MTC data" + ) + parser.add_argument( + "--rebuild-environments", + action="store_true", + help="recreate cached release and development environments", + ) + parser.add_argument( + "--prepare-only", + action="store_true", + help="prepare data and environments without running load trials", + ) + + parser.add_argument("--worker", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--probe", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--label", help=argparse.SUPPRESS) + parser.add_argument("--source-kind", help=argparse.SUPPRESS) + parser.add_argument("--source-revision", help=argparse.SUPPRESS) + parser.add_argument( + "--strategy", + choices=("lazy", "eager", "shared-reload"), + help=argparse.SUPPRESS, + ) + return parser + + +def main() -> int: + """Dispatch to an isolated load worker or the benchmark controller.""" + args = _parser().parse_args() + if args.worker: + return _worker(args) + if args.warmups < 0 or args.repetitions < 1: + raise SystemExit("warmups must be nonnegative and repetitions positive") + if args.sample_interval <= 0 or args.hold_seconds < 0: + raise SystemExit("sample interval must be positive and hold time nonnegative") + return _controller(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/benchmark_omx_load.py b/benchmarks/benchmark_omx_load.py new file mode 100644 index 0000000..053d661 --- /dev/null +++ b/benchmarks/benchmark_omx_load.py @@ -0,0 +1,412 @@ +#!/usr/bin/env python3 +"""Benchmark fully materialized Sharrow OMX loads in isolated processes. + +The controller launches one fresh Python process per trial, samples the RSS of +the complete process tree, and combines that with the operating system's +high-water RSS for the worker. This captures the final Dataset as well as +temporary arrays created while HDF5 data is decoded, stacked, or cast. + +Example +------- +python benchmarks/benchmark_omx_load.py \ + --omx /path/to/traffic_skims_EA.omx \ + --omx /path/to/traffic_skims_AM.omx \ + --case 'release-2.16.2|/tmp/sharrow-2.16.2|lazy' \ + --case 'current-lazy|/path/to/current/sharrow|lazy' \ + --case 'current-eager|/path/to/current/sharrow|eager' \ + --warmups 1 --repetitions 3 --output /tmp/omx-benchmark.json + +Case strategies are ``lazy`` (construct the default Dask-backed Dataset, then +call ``load``), ``lazy-matrix`` (use one Dask task per physical matrix), +``shared-reload`` (the ActivitySim shared-memory workflow), and ``eager`` (use +the newer ``load="eager"`` path). Peak RSS excludes the operating system's file +cache and memory owned by unrelated processes. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import statistics +import subprocess +import sys +import time +import traceback +from pathlib import Path + +import psutil + +try: + import resource +except ImportError: # Windows has no resource module; the controller samples RSS. + resource = None + +RESULT_PREFIX = "SHARROW_BENCHMARK_RESULT=" + + +def _rss_high_water_bytes() -> int: + """Return this process's maximum resident set size in bytes.""" + if resource is None: + return psutil.Process().memory_info().rss + value = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + # macOS reports bytes, while Linux and most BSDs report KiB. + if platform.system() == "Darwin": + return int(value) + return int(value * 1024) + + +def _dataset_fingerprint(dataset) -> str: + """Hash array metadata and edge values without scanning or copying arrays.""" + digest = hashlib.sha256() + for name in sorted(dataset.data_vars): + array = dataset[name].data + digest.update(name.encode()) + digest.update(repr((array.shape, str(array.dtype))).encode()) + if array.size: + first = array[(0,) * array.ndim] + last = array[tuple(size - 1 for size in array.shape)] + digest.update(repr((first, last)).encode()) + return digest.hexdigest() + + +def _worker(args: argparse.Namespace) -> int: + """Run one load trial and emit a machine-readable result.""" + import sharrow as sh + + process = psutil.Process() + omx_paths = [str(Path(path).resolve()) for path in args.omx] + omx_sources = omx_paths[0] if len(omx_paths) == 1 else omx_paths + source_commit = subprocess.check_output( + ["git", "-C", args.source_root, "rev-parse", "HEAD"], text=True + ).strip() + result = { + "label": args.label, + "strategy": args.strategy, + "omx": omx_paths, + "source_root": str(Path(args.source_root).resolve()), + "source_commit": source_commit, + "sharrow_file": str(Path(sh.__file__).resolve()), + "sharrow_version": getattr(sh, "__version__", "unknown"), + "baseline_rss_bytes": process.memory_info().rss, + } + started = time.perf_counter() + try: + kwargs = { + "time_periods": args.periods.split(","), + "max_float_precision": args.max_float_precision, + } + if args.strategy == "shared-reload": + template = sh.dataset.from_omx_3d(omx_sources, **kwargs) + shared_key = f"sharrow-benchmark-{os.getpid()}-{time.time_ns()}" + dataset = template.shm.to_shared_memory(shared_key, mode="r", load=False) + sh.dataset.reload_from_omx_3d(dataset, omx_paths) + elif args.strategy == "eager": + kwargs["load"] = "eager" + dataset = sh.dataset.from_omx_3d(omx_sources, **kwargs) + elif args.strategy == "lazy-matrix": + kwargs["task_granularity"] = "matrix" + dataset = sh.dataset.from_omx_3d(omx_sources, **kwargs) + else: + dataset = sh.dataset.from_omx_3d(omx_sources, **kwargs) + if args.strategy in {"lazy", "lazy-matrix"}: + dataset.load() + result.update( + { + "status": "ok", + "elapsed_seconds": time.perf_counter() - started, + "dataset_nbytes": int(dataset.nbytes), + "final_rss_bytes": process.memory_info().rss, + "variable_count": len(dataset.data_vars), + "dimensions": {name: int(size) for name, size in dataset.sizes.items()}, + "fingerprint": _dataset_fingerprint(dataset), + } + ) + except Exception as error: # noqa: BLE001 - failures are benchmark results + result.update( + { + "status": "error", + "elapsed_seconds": time.perf_counter() - started, + "final_rss_bytes": process.memory_info().rss, + "error_type": type(error).__name__, + "error": str(error), + "traceback": traceback.format_exc(), + } + ) + finally: + result["worker_peak_rss_bytes"] = _rss_high_water_bytes() + + print(f"{RESULT_PREFIX}{json.dumps(result, sort_keys=True)}", flush=True) + # Give the external sampler one final opportunity to observe the loaded + # Dataset before process teardown releases its resident pages. + time.sleep(args.hold_seconds) + return 0 + + +def _process_tree_rss(process: psutil.Process) -> int: + """Sum current RSS across a worker and all of its live descendants.""" + try: + descendants = process.children(recursive=True) + except (psutil.Error, OSError): + descendants = [] + total = 0 + for item in [process, *descendants]: + try: + total += item.memory_info().rss + except (psutil.Error, OSError): + continue + return total + + +def _parse_case(value: str) -> dict[str, str]: + """Parse a LABEL|SOURCE_ROOT|STRATEGY command-line case.""" + try: + label, source_root, strategy = value.split("|", 2) + except ValueError as error: + raise argparse.ArgumentTypeError( + "case must be LABEL|SOURCE_ROOT|STRATEGY" + ) from error + if strategy not in {"lazy", "lazy-matrix", "shared-reload", "eager"}: + raise argparse.ArgumentTypeError( + "case strategy must be lazy, lazy-matrix, shared-reload, or eager" + ) + root = Path(source_root).expanduser().resolve() + if not (root / "sharrow" / "__init__.py").is_file(): + raise argparse.ArgumentTypeError(f"not a Sharrow source tree: {root}") + return {"label": label, "source_root": str(root), "strategy": strategy} + + +def _extract_result(stdout: str) -> dict: + """Extract the worker's JSON record from captured standard output.""" + for line in reversed(stdout.splitlines()): + if line.startswith(RESULT_PREFIX): + return json.loads(line[len(RESULT_PREFIX) :]) + raise RuntimeError(f"worker produced no benchmark result:\n{stdout}") + + +def _run_trial( + args: argparse.Namespace, + case: dict[str, str], + run_number: int, + warmup: bool, +) -> dict: + """Launch and externally monitor one isolated benchmark trial.""" + command = [ + sys.executable, + str(Path(__file__).resolve()), + "--worker", + ] + for omx in args.omx: + command.extend(["--omx", str(Path(omx).resolve())]) + command.extend( + [ + "--label", + case["label"], + "--source-root", + case["source_root"], + "--strategy", + case["strategy"], + "--periods", + args.periods, + "--max-float-precision", + str(args.max_float_precision), + "--hold-seconds", + str(args.hold_seconds), + ] + ) + environment = os.environ.copy() + environment["PYTHONPATH"] = os.pathsep.join( + filter(None, [case["source_root"], environment.get("PYTHONPATH")]) + ) + child = subprocess.Popen( + command, + cwd=case["source_root"], + env=environment, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + monitored = psutil.Process(child.pid) + peak_tree_rss = 0 + while child.poll() is None: + peak_tree_rss = max(peak_tree_rss, _process_tree_rss(monitored)) + time.sleep(args.sample_interval) + peak_tree_rss = max(peak_tree_rss, _process_tree_rss(monitored)) + stdout, stderr = child.communicate() + result = _extract_result(stdout) + result.update( + { + "run": run_number, + "warmup": warmup, + "returncode": child.returncode, + "peak_tree_rss_bytes": peak_tree_rss, + "peak_rss_bytes": max( + peak_tree_rss, result.get("worker_peak_rss_bytes", 0) + ), + "stderr": stderr, + } + ) + return result + + +def _gib(value: int | float) -> float: + """Convert bytes to GiB for compact terminal reporting.""" + return value / 1024**3 + + +def _print_trial(result: dict) -> None: + """Print one compact human-readable result row.""" + status = result["status"] + peak = _gib(result["peak_rss_bytes"]) + message = ( + f"{result['label']:20} {result['strategy']:11} " + f"run={result['run']} warmup={result['warmup']} status={status:5} " + f"time={result['elapsed_seconds']:8.3f}s peak={peak:7.3f} GiB" + ) + if status == "error": + message += f" error={result['error_type']}: {result['error']}" + print(message, flush=True) + + +def _summaries(results: list[dict]) -> list[dict]: + """Compute medians and ranges for successful measured trials by case.""" + summaries = [] + labels = dict.fromkeys(result["label"] for result in results) + for label in labels: + trials = [ + result + for result in results + if result["label"] == label + and not result["warmup"] + and result["status"] == "ok" + ] + if not trials: + continue + elapsed = [result["elapsed_seconds"] for result in trials] + peak = [result["peak_rss_bytes"] for result in trials] + final_rss = [result["final_rss_bytes"] for result in trials] + baseline = [result["baseline_rss_bytes"] for result in trials] + summaries.append( + { + "label": label, + "strategy": trials[0]["strategy"], + "trials": len(trials), + "median_elapsed_seconds": statistics.median(elapsed), + "min_elapsed_seconds": min(elapsed), + "max_elapsed_seconds": max(elapsed), + "median_peak_rss_bytes": statistics.median(peak), + "median_final_rss_bytes": statistics.median(final_rss), + "median_baseline_rss_bytes": statistics.median(baseline), + "median_peak_increase_bytes": statistics.median( + trial_peak - trial_baseline + for trial_peak, trial_baseline in zip(peak, baseline) + ), + "dataset_nbytes": trials[0]["dataset_nbytes"], + "fingerprint": trials[0]["fingerprint"], + } + ) + return summaries + + +def _controller(args: argparse.Namespace) -> int: + """Run warmups and measured trials, rotating order between repetitions.""" + omx_paths = [Path(path).expanduser().resolve() for path in args.omx] + for omx in omx_paths: + if not omx.is_file(): + raise FileNotFoundError(omx) + + results = [] + for warmup_number in range(1, args.warmups + 1): + for case in args.case: + result = _run_trial(args, case, warmup_number, warmup=True) + results.append(result) + _print_trial(result) + + # Rotate trial order so no implementation systematically benefits from + # immediately following the cache-warming run. + for run_number in range(1, args.repetitions + 1): + offset = (run_number - 1) % len(args.case) + ordered_cases = args.case[offset:] + args.case[:offset] + for case in ordered_cases: + result = _run_trial(args, case, run_number, warmup=False) + results.append(result) + _print_trial(result) + + payload = { + "omx": [str(path) for path in omx_paths], + "omx_size_bytes": sum(path.stat().st_size for path in omx_paths), + "periods": args.periods.split(","), + "max_float_precision": args.max_float_precision, + "memory_metric": "peak RSS across the complete benchmark process tree", + "sample_interval_seconds": args.sample_interval, + "results": results, + "summaries": _summaries(results), + } + print("\nSummary (medians of successful measured trials)") + for summary in payload["summaries"]: + print( + f"{summary['label']:20} {summary['strategy']:11} " + f"time={summary['median_elapsed_seconds']:8.3f}s " + f"peak={_gib(summary['median_peak_rss_bytes']):7.3f} GiB " + f"final={_gib(summary['median_final_rss_bytes']):7.3f} GiB " + f"dataset={_gib(summary['dataset_nbytes']):7.3f} GiB" + ) + + if args.output: + output = Path(args.output).expanduser().resolve() + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + print(f"\nWrote {output}") + return 0 + + +def _parser() -> argparse.ArgumentParser: + """Build the command-line parser used by the controller and worker.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--omx", + action="append", + required=True, + help="OMX file to load; repeat to load several files as one Dataset", + ) + parser.add_argument( + "--case", + action="append", + type=_parse_case, + default=[], + help="benchmark case as LABEL|SOURCE_ROOT|STRATEGY", + ) + parser.add_argument("--warmups", type=int, default=1) + parser.add_argument("--repetitions", type=int, default=3) + parser.add_argument("--periods", default="EA,AM,MD,PM,EV") + parser.add_argument("--max-float-precision", type=int, default=32) + parser.add_argument("--sample-interval", type=float, default=0.005) + parser.add_argument("--hold-seconds", type=float, default=0.25) + parser.add_argument("--output") + parser.add_argument("--worker", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--label", help=argparse.SUPPRESS) + parser.add_argument("--source-root", help=argparse.SUPPRESS) + parser.add_argument( + "--strategy", + choices=("lazy", "lazy-matrix", "shared-reload", "eager"), + help=argparse.SUPPRESS, + ) + return parser + + +def main() -> int: + """Dispatch to the isolated worker or multi-case controller.""" + args = _parser().parse_args() + if args.worker: + return _worker(args) + if not args.case: + raise SystemExit("at least one --case is required") + if args.warmups < 0 or args.repetitions < 1: + raise SystemExit("warmups must be nonnegative and repetitions positive") + return _controller(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/api.rst b/docs/api.rst index 14a8d20..001691f 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -93,6 +93,85 @@ These functions can be found in the :py:mod:`sharrow.dataset` module. .. autofunction:: sharrow.dataset.from_zarr .. autofunction:: sharrow.dataset.from_named_objects +OMX Loading Portability +~~~~~~~~~~~~~~~~~~~~~~~ + +Sharrow uses OpenMatrix 0.4 or newer, whose file objects are backed by h5py. +Paths, OpenMatrix handles, and plain h5py handles are supported. Sharrow's +bounded parallel chunk decoder operates directly on their HDF5 datasets, +including direct reads into shared memory and dtype conversion during decoding. +Caller-owned handles remain open. + +Standard gzip/shuffle OMX files work with Sharrow's base installation. Files +that use Blosc, Zstandard, or other optional HDF5 filters may require the +additional filter packages:: + + pip install "sharrow[hdf5-plugins]" + +These third-party filter packages do not currently publish Windows ARM64 +wheels, so Windows on ARM uses the standard HDF5 filters supported by h5py. + +All selected OMX matrices must match the shape declared or inferred from the +first source file. Inconsistent matrices now raise an error during construction, +before loading or reserving shared storage. The error identifies the matrix, +source file, actual shape, and expected shape. Correct stale ``SHAPE`` metadata +and verify coordinates rather than relying on earlier permissive behavior. +``ignore`` can exclude an unrelated matrix, but does not repair incorrect +metadata for matrices that must be loaded. + +Lazy loading defaults to one Dask task per matrix (time-period page), so +selecting a period does not load the other periods. Use +``task_granularity="variable"`` explicitly when loading whole variables and +reducing task overhead matters more than selective reads. Eager and direct +shared/memmap loads continue to fill their final destination arrays directly. + +``reload_from_omx_3d`` loads files serially by default, with parallel chunk +decoding. Existing callers do not acquire multiprocessing requirements. +Pass ``workers>1`` to opt into file-level parallel reloads. + +The ``shared`` and ``memmap`` loading modes can read several source files +concurrently. On Windows, Sharrow uses file-level threads so these modes work +from ordinary scripts, notebooks, and frozen applications without a +``__main__`` multiprocessing guard or Windows' process-pool size limit. On +other platforms, Sharrow uses spawned worker processes to avoid inheriting +open HDF5 state. Calls made at module scope in a standalone script should +therefore use the standard guarded entry point:: + + if __name__ == "__main__": + skims = sharrow.dataset.from_omx_3d( + skim_files, + time_periods=["EA", "AM", "MD", "PM", "EV"], + load="shared", + ) + +Process-based file workers divide the CPU budget between their decoder pools +instead of each creating a machine-sized pool. Chunk queue limits are still +per process, and at least one decoder is needed per active worker; this is not +a global byte limit. Windows file threads share one decoder pool. + +Set ``workers=1`` for deterministic serial I/O on every platform. Successful +``memmap`` loads retain their backing data and metadata files intentionally. +Releasing a memmap flushes it and relinquishes the accessor's ownership; +existing Dataset arrays and retained NumPy views remain valid. The mapping +closes when the last array reference is discarded. Before deleting the backing +files on Windows, discard the Dataset and all retained views:: + + key = result.shm.shared_memory_key + result.shm.release_shared_memory() + del result # also discard any retained DataArrays or NumPy views + gc.collect() # import gc; Xarray accessor cycles may delay collection + sharrow.Dataset.shm.delete_shared_memory_files(key) + +File deletion is a separate, explicit operation; release never deletes memmap +files. Do not force-close the underlying mapping while views remain alive. + +OMX to Zarr conversion preserves float64 values by default. Pass +``max_float_precision=32`` to ``sharrow.translate.omx_to_zarr`` only when the +precision reduction is intended. Output uses Zarr format 2 with spatial chunks +of at most 256 by 256 cells and one period per chunk. This bounds stored chunks +independently of the source loader's tasks; decoding still materializes a source +matrix, rather than streaming individual spatial tiles from OMX. + Editing ~~~~~~~ .. automethod:: sharrow.Dataset.ensure_integer diff --git a/docs/walkthrough/loading-skims.ipynb b/docs/walkthrough/loading-skims.ipynb index ad8d601..321fc51 100644 --- a/docs/walkthrough/loading-skims.ipynb +++ b/docs/walkthrough/loading-skims.ipynb @@ -31,7 +31,7 @@ "from tempfile import TemporaryDirectory\n", "\n", "import numpy as np\n", - "import openmatrix\n", + "import openmatrix as omx\n", "import pandas as pd\n", "import xarray as xr\n", "\n", @@ -132,7 +132,7 @@ "source": [ "## OMX: time period in matrix names\n", "\n", - "The openmatrix (OMX) standard defines a data format that stores two-dimensional matrix data. To represent a third dimension, one approach is to store a matrix for each time period and include the time period after a double underscore in the matrix name, such as `autotime__AM`." + "The Open Matrix (OMX) standard defines an HDF5 layout that stores two-dimensional matrix data. To represent a third dimension, one approach is to store a matrix for each time period and include the time period after a double underscore in the matrix name, such as `autotime__AM`." ] }, { @@ -143,15 +143,13 @@ "outputs": [], "source": [ "omx_path = data_directory / \"skims.omx\"\n", - "with openmatrix.open_file(omx_path, mode=\"w\") as omx_file:\n", + "with omx.open_file(omx_path, mode=\"w\") as omx_file:\n", " for timeperiod in timeperiods:\n", - " omx_file.create_carray(\n", - " \"/data\",\n", + " omx_file.create_matrix(\n", " f\"autotime__{timeperiod}\",\n", " obj=source_skims.autotime.sel(timeperiod=timeperiod).values,\n", " )\n", - " omx_file.create_carray(\"/lookup\", \"taz\", obj=zones)\n", - " omx_file.root._v_attrs.SHAPE = np.array([len(zones), len(zones)])" + " omx_file.create_mapping(\"taz\", zones)" ] }, { diff --git a/envs/development.yml b/envs/development.yml index c0e0c0e..49a5cef 100644 --- a/envs/development.yml +++ b/envs/development.yml @@ -21,7 +21,6 @@ dependencies: - numba>=0.57 - numexpr - numpy>=1.19,<2 - - openmatrix - pandas>=1.2 - pyarrow - pytest @@ -33,5 +32,6 @@ dependencies: - zarr - pip: + - openmatrix>=0.4.0 - larch>=6 - -e .. diff --git a/envs/testing.yml b/envs/testing.yml index 92b1eab..7987390 100644 --- a/envs/testing.yml +++ b/envs/testing.yml @@ -23,10 +23,10 @@ dependencies: - pytest-regressions - pytest-xdist - nbmake - - openmatrix - h5py - python-blosc2 - hdf5plugin - zarr - pip: + - openmatrix>=0.4.0 - larch>=6 diff --git a/pyproject.toml b/pyproject.toml index 76c8423..46ae8b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,8 +14,7 @@ dependencies = [ "pandas >= 1.2", "pyarrow", "h5py", - "blosc2", - "hdf5plugin", + "openmatrix>=0.4.0", "xarray >= 2025.3.1", "numba >= 0.57", "numexpr", @@ -35,6 +34,12 @@ description = "numba for ActivitySim-style spec files" readme = "README.md" keywords = ["activitysim", "discrete choice"] +[project.optional-dependencies] +hdf5-plugins = [ + "blosc2; platform_system != 'Windows' or platform_machine != 'ARM64'", + "hdf5plugin; platform_system != 'Windows' or platform_machine != 'ARM64'", +] + [project.urls] Documentation = "https://activitysim.github.io/sharrow/" Repository = "https://github.com/activitysim/sharrow" @@ -91,9 +96,10 @@ testpaths = [ [dependency-groups] dev = [ + "blosc2; platform_system != 'Windows' or platform_machine != 'ARM64'", + "hdf5plugin; platform_system != 'Windows' or platform_machine != 'ARM64'", "ipykernel>=6.31.0", "nbmake>=1.5.5", - "openmatrix>=0.3.5.0", "pytest>=8.4.2", "pytest-regressions>=2.8.3", "scipy>=1.13.1", diff --git a/sharrow/dataset.py b/sharrow/dataset.py index 024a23c..4896296 100755 --- a/sharrow/dataset.py +++ b/sharrow/dataset.py @@ -1,17 +1,26 @@ from __future__ import annotations import ast +import atexit import base64 +import concurrent.futures +import contextlib import hashlib import logging +import multiprocessing +import os import re +import secrets +import threading import time import zipfile from collections.abc import Hashable, Iterable, Mapping, Sequence from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import Any +import h5py import numpy as np +import openmatrix import pandas as pd import pyarrow as pa import xarray as xr @@ -24,11 +33,77 @@ from .shared_memory import si_units from .table import Table -if TYPE_CHECKING: - import openmatrix +logger = logging.getLogger("sharrow") -logger = logging.getLogger("sharrow") +class _OmxReadCoordinator: + """Share a bounded decoder pool across all matrices in one load.""" + + def __init__(self, max_workers=None): + if max_workers is None: + # Match ThreadPoolExecutor's Python 3.9+ default while keeping the + # queue budget explicit and stable across concurrent Dask tasks. + max_workers = min(32, (os.cpu_count() or 1) + 4) + self.executor = concurrent.futures.ThreadPoolExecutor( + max_workers=max_workers, thread_name_prefix="sharrow-omx" + ) + self.pending_semaphore = threading.BoundedSemaphore(max(2 * max_workers, 4)) + + def read(self, dset, out): + """Decode one HDF5 dataset into an existing destination array.""" + return omx_reader.read_dataset( + dset, + out=out, + executor=self.executor, + pending_semaphore=self.pending_semaphore, + ) + + def close(self): + """Wait for decoder work and release the pool's threads.""" + self.executor.shutdown(wait=True) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.close() + + +_lazy_omx_reader = None +_lazy_omx_reader_lock = threading.Lock() + + +def _reset_lazy_omx_reader_after_fork(): + """Discard inherited thread state without joining vanished parent threads.""" + global _lazy_omx_reader, _lazy_omx_reader_lock + _lazy_omx_reader = None + # The parent may have forked while another thread held this lock. + _lazy_omx_reader_lock = threading.Lock() + + +if hasattr(os, "register_at_fork"): + os.register_at_fork(after_in_child=_reset_lazy_omx_reader_after_fork) + + +def _get_lazy_omx_reader(): + """Return the process-wide decoder pool shared by concurrent Dask tasks.""" + global _lazy_omx_reader + if _lazy_omx_reader is None: + with _lazy_omx_reader_lock: + if _lazy_omx_reader is None: + _lazy_omx_reader = _OmxReadCoordinator() + return _lazy_omx_reader + + +def _shutdown_lazy_omx_reader(): + """Release the process-wide lazy decoder pool during interpreter shutdown.""" + global _lazy_omx_reader + if _lazy_omx_reader is not None: + _lazy_omx_reader.close() + _lazy_omx_reader = None + + +atexit.register(_shutdown_lazy_omx_reader) well_known_names = { "nb", @@ -292,20 +367,57 @@ def from_table( def _group_names(grp) -> list[str]: - """List the child node names of a pytables or h5py group.""" - try: - return list(grp._v_children) # pytables - except AttributeError: - return list(grp.keys()) # h5py + """List the child names of an HDF5 group.""" + return list(grp.keys()) + + +def _omx_shape(omx: h5py.File) -> tuple[int, int]: + """Read OMX shape metadata or infer it from the first matrix. + + Older OpenMatrix writers did not always persist the optional ``SHAPE`` + attribute. Their readers inferred the dimensions from matrix data, so the + h5py implementation retains that compatibility behavior. + """ + # OpenMatrix handles supply the format-level shape inference. Plain h5py + # handles remain supported, including in-memory files. + shape = omx.shape() if isinstance(omx, openmatrix.File) else omx.attrs.get("SHAPE") + if shape is None: + data_group = omx["/data"] + try: + first_name = next(iter(data_group)) + except StopIteration: + raise ValueError( + "cannot infer OMX shape from an empty data group" + ) from None + shape = data_group[first_name].shape + shape = tuple(int(i) for i in shape) + if len(shape) != 2: + raise ValueError(f"OMX matrices must be two-dimensional, found shape {shape}") + return shape def omx_file_name(omx) -> str | None: - """Resolve the on-disk filename of an OMX-like object, if possible.""" - return omx_reader.h5_filename(omx) + """Resolve the on-disk filename of an OMX HDF5 file, if possible. + + Filename discovery is deliberately structural so callers can continue to + pass handles created by optional OMX libraries. In particular, + legacy PyTables handles expose a ``filename`` attribute. OpenMatrix 0.4 + handles are h5py files and can be used directly without reopening. + """ + filename = omx_reader.h5_filename(omx) + if filename is None: + filename = getattr(omx, "filename", None) + try: + filename = os.fspath(filename) + except TypeError: + return None + if not os.path.isfile(filename): + return None + return filename def from_omx( - omx: openmatrix.File, + omx: h5py.File | str | os.PathLike, index_names=("otaz", "dtaz"), indexes="one-based", renames=None, @@ -315,13 +427,12 @@ def from_omx( Parameters ---------- - omx : openmatrix.File or larch.OMX - An OMX-format file, opened for reading. + omx : openmatrix.File, h5py.File, path-like, or filename-bearing OMX handle + An OMX-format HDF5 file, its path, or a compatible open handle such as + an ``openmatrix.File``. OpenMatrix and h5py handles remain owned by + the caller; legacy filename-bearing handles are reopened with OpenMatrix. index_names : tuple, default ("otaz", "dtaz") - Should be a tuple of length 3, giving the names of the three - dimensions. The first two names are the native dimensions from - the open matrix file, the last is the name of the implicit - dimension that is created by parsing array names. + The names of the two matrix dimensions. indexes : str or tuple[str], optional The name of a 'lookup' in the OMX file, which will be used to populate the coordinates for the two native dimensions. Or, @@ -341,21 +452,32 @@ def from_omx( ------- Dataset """ - import h5py - - # handle larch.OMX, openmatrix.open_file, and h5py.File versions - if "lar" in type(omx).__module__: - omx_data = omx.data - omx_lookup = omx.lookup - omx_shape = omx.shape - elif isinstance(omx, h5py.File): - omx_data = omx["data"] - omx_lookup = omx["lookup"] - omx_shape = tuple(int(i) for i in omx.attrs["SHAPE"]) - else: - omx_data = omx.root["data"] - omx_lookup = omx.root["lookup"] - omx_shape = omx.shape() + if isinstance(omx, (str, os.PathLike)): + with openmatrix.open_file(omx, "r") as handle: + return from_omx( + handle, + index_names=index_names, + indexes=indexes, + renames=renames, + ) + if not isinstance(omx, h5py.File): + filename = omx_file_name(omx) + if filename is None: + raise TypeError( + "omx must be an h5py.File, path-like, or filename-bearing OMX handle" + ) + with openmatrix.open_file(filename, "r") as handle: + return from_omx( + handle, + index_names=index_names, + indexes=indexes, + renames=renames, + ) + + # Absolute paths bypass OpenMatrix's matrix-name lookup semantics. + omx_data = omx["/data"] + omx_lookup = omx["/lookup"] + omx_shape = _omx_shape(omx) if renames is None: data_names = _group_names(omx_data) @@ -369,10 +491,8 @@ def from_omx( filename = omx_file_name(omx) if _is_reopenable(filename): # fast path: parallel chunk decoding via h5py - import concurrent.futures - - with h5py.File(filename, "r") as f5: - f5_data = f5["data"] + with openmatrix.open_file(filename, "r") as f5: + f5_data = f5["/data"] with concurrent.futures.ThreadPoolExecutor() as pool: for new_k, old_k in rename_pairs: arrays[new_k] = omx_reader.read_dataset( @@ -435,12 +555,269 @@ def _should_ignore(ignore, x): return False -def _fast_load_omx_array(filename, name): - """Load one matrix table from an OMX file, decoding chunks in parallel.""" - import h5py +def _omx_target_dtype(dtype, max_float_precision): + """Return the in-memory dtype after applying the precision limit.""" + dtype = np.dtype(dtype) + if np.issubdtype(dtype, np.floating): + max_dtype = np.dtype(f"float{max_float_precision}") + if dtype.itemsize > max_dtype.itemsize: + return max_dtype + return dtype + + +def _empty_omx_3d(shape, dtype): + """Allocate a 3-D array whose individual last-axis pages are contiguous. + + OMX stores each time-period page as an independent C-contiguous 2-D HDF5 + dataset. Making the logical last axis physically outermost lets h5py read + each source page directly into the result without a matrix-sized temporary. + """ + dtype = np.dtype(dtype) + nbytes = int(np.prod(shape)) * dtype.itemsize + buffer = np.empty(nbytes, dtype=np.uint8) + strides = ( + shape[1] * dtype.itemsize, + dtype.itemsize, + shape[0] * shape[1] * dtype.itemsize, + ) + return np.ndarray(shape, dtype=dtype, buffer=buffer, strides=strides) + + +def _read_omx_dataset(dset, out=None, dtype=None, reader=None): + """Read one HDF5 dataset through the bounded parallel chunk decoder.""" + if out is None: + target_dtype = dset.dtype if dtype is None else np.dtype(dtype) + out = np.empty(dset.shape, dtype=target_dtype) + elif dtype is not None and np.dtype(dtype) != out.dtype: + raise ValueError(f"out has dtype {out.dtype}, requested {np.dtype(dtype)}") + if reader is None: + reader = _get_lazy_omx_reader() + return reader.read(dset, out) + + +def _fast_load_omx_array(filename, name, dtype=None): + """Load one matrix table through the shared parallel decoder pool.""" + with openmatrix.open_file(filename, "r") as f: + return _read_omx_dataset(f["/data"][name], dtype=dtype) + + +def _load_omx_variable(page_sources, shape, dtype): + """Load all pages of one logical OMX variable in a single Dask task.""" + result = _empty_omx_3d(shape, dtype) + open_files = {} + reader = _get_lazy_omx_reader() + try: + for period, source in enumerate(page_sources): + if source is None: + result[..., period].fill(0) + continue + filename, data_name, eager_array = source + if eager_array is not None: + result[..., period] = eager_array + continue + if filename not in open_files: + open_files[filename] = openmatrix.open_file(filename, "r") + _read_omx_dataset( + open_files[filename]["/data"][data_name], + result[..., period], + reader=reader, + ) + finally: + for handle in open_files.values(): + handle.close() + return result + + +def _load_omx_assignments(dataset, source, assignments, reader=None): + """Load all selected matrices from one source into a prepared Dataset.""" + if reader is None: + with _OmxReadCoordinator() as reader: + return _load_omx_assignments(dataset, source, assignments, reader=reader) + if isinstance(source, h5py.File): + file_context = contextlib.nullcontext(source) + else: + filename = omx_file_name(source) + if filename is None: + raise TypeError( + "OMX sources must be h5py.File, path-like, or filename-bearing handles" + ) + file_context = openmatrix.open_file(filename, "r") + bytes_loaded = 0 + with file_context as handle: + data_group = handle["/data"] + for data_name, variable_name, period in assignments: + target = dataset[variable_name].data + if period is not None: + target = target[..., period] + _read_omx_dataset(data_group[data_name], target, reader=reader) + bytes_loaded += target.nbytes + return bytes_loaded + + +def _load_omx_shared_worker(shared_memory_key, source, assignments, decoder_workers): + """Process worker that fills disjoint pages of a shared OMX Dataset.""" + target = xr.Dataset.shm.from_shared_memory(shared_memory_key, mode="r+") + with _OmxReadCoordinator(max_workers=decoder_workers) as reader: + bytes_loaded = _load_omx_assignments(target, source, assignments, reader=reader) + if shared_memory_key.startswith("memmap:"): + for memory_object in target.shm._shared_memory_objs_: + flush = getattr(memory_object, "flush", None) + if flush is not None: + flush() + return bytes_loaded + + +def _omx_worker_count(workers, batch_count): + """Choose a file-level worker count without creating idle workers.""" + if batch_count < 1: + return 0 + requested = (os.cpu_count() or 1) if workers is None else workers + return max(1, min(requested, batch_count)) + + +def _omx_uses_processes(): + """Return whether file-level parallelism should use spawned processes.""" + return os.name != "nt" + + +def _load_omx_batches(dataset, active_batches, worker_count): + """Load independent source batches with a platform-safe executor. + + Windows uses threads because its spawn-only multiprocessing model requires + callers to guard their program entry point and limits process pools to 61 + workers. Raw HDF5 reads remain serialized by h5py, while chunk decompression + still runs concurrently in the shared decoder pool. Other platforms retain + process-level file parallelism without inheriting HDF5 state. + """ + if not active_batches: + return 0 + if worker_count == 1: + with _OmxReadCoordinator() as reader: + return sum( + _load_omx_assignments(dataset, source, batch, reader=reader) + for source, batch in active_batches + ) + + if not _omx_uses_processes(): + with _OmxReadCoordinator() as reader: + with concurrent.futures.ThreadPoolExecutor( + max_workers=worker_count, thread_name_prefix="sharrow-omx-file" + ) as pool: + futures = [ + pool.submit( + _load_omx_assignments, + dataset, + source, + batch, + reader, + ) + for source, batch in active_batches + ] + return sum(future.result() for future in futures) + + shared_memory_key = dataset.shm.shared_memory_key + # Share the machine's CPU budget across file workers instead of giving + # every process a machine-sized decoder pool. Each process needs at least + # one decoder, even when an explicit file-worker count exceeds CPU count. + decoder_workers = max(1, min(32, (os.cpu_count() or 1) // worker_count)) + mp_context = multiprocessing.get_context("spawn") + with concurrent.futures.ProcessPoolExecutor( + max_workers=worker_count, mp_context=mp_context + ) as pool: + futures = [ + pool.submit( + _load_omx_shared_worker, + shared_memory_key, + source, + batch, + decoder_workers, + ) + for source, batch in active_batches + ] + return sum(future.result() for future in futures) + + +def _prepare_omx_reload_assignments(dataset, sources, time_period_sep, ignore): + """Map OMX matrices to disjoint target arrays, preserving last-file-wins.""" + assignments_by_target = {} + filenames = [] + + for source_number, source in enumerate(sources): + filename = omx_file_name(source) + filenames.append(filename) + if isinstance(source, h5py.File): + file_context = contextlib.nullcontext(source) + elif filename is not None: + file_context = openmatrix.open_file(filename, "r") + else: + raise TypeError( + "OMX sources must be h5py.File, path-like, or filename-bearing handles" + ) + + with file_context as handle: + for data_name in handle["/data"]: + if _should_ignore(ignore, data_name): + logger.info("ignoring %s", data_name) + continue + + if time_period_sep in data_name: + variable_name, period_name = data_name.split(time_period_sep, 1) + if variable_name not in dataset: + logger.info( + "skipping %s because %s not in dataset", + data_name, + variable_name, + ) + continue + variable = dataset[variable_name] + if variable.ndim != 3: + raise ValueError( + f"dataset variable {variable_name} has " + f"{variable.ndim} dimensions, expected 3" + ) + period_dimension = variable.dims[-1] + try: + period = variable.get_index(period_dimension).get_loc( + period_name + ) + except KeyError: + raise KeyError( + f"time period {period_name!r} from {data_name!r} is not " + f"in dataset coordinate {period_dimension!r}" + ) from None + if not isinstance(period, (int, np.integer)): + raise ValueError( + f"time period {period_name!r} does not identify one page" + ) + period = int(period) + else: + variable_name = data_name + period = None + if variable_name not in dataset: + logger.info( + "skipping %s because it is not in dataset", data_name + ) + continue + if dataset[variable_name].ndim != 2: + raise ValueError( + f"dataset variable {variable_name} has " + f"{dataset[variable_name].ndim} dimensions, expected 2" + ) + + # Multiple source files can contain the same matrix. Matching + # from_omx_3d, the last source wins without concurrent writes. + assignments_by_target[(variable_name, period)] = ( + source_number, + data_name, + ) - with h5py.File(filename, "r") as f: - return omx_reader.read_dataset(f["data"][name]) + assignments = [[] for _ in sources] + for (variable_name, period), ( + source_number, + data_name, + ) in assignments_by_target.items(): + assignments[source_number].append((data_name, variable_name, period)) + return filenames, assignments def _is_reopenable(filename) -> bool: @@ -451,10 +828,8 @@ def _is_reopenable(filename) -> bool: """ if filename is None: return False - import h5py - try: - with h5py.File(filename, "r"): + with openmatrix.open_file(filename, "r"): pass except Exception: # noqa: BLE001 return False @@ -462,7 +837,7 @@ def _is_reopenable(filename) -> bool: def from_omx_3d( - omx: openmatrix.File | str | Iterable[openmatrix.File | str], + omx: h5py.File | str | os.PathLike | Iterable[h5py.File | str | os.PathLike], index_names=("otaz", "dtaz", "time_period"), indexes=None, *, @@ -470,14 +845,22 @@ def from_omx_3d( time_period_sep="__", max_float_precision=32, ignore=None, + load="lazy", + task_granularity="matrix", + workers=None, + memory_path=None, + shared_memory_key=None, ): """ Create a Dataset from an OMX file with an implicit third dimension. Parameters ---------- - omx : openmatrix.File or larch.OMX - An OMX-format file, opened for reading. + omx : openmatrix.File, h5py.File, path-like, filename-bearing OMX handle, or iterable + One or more OMX-format HDF5 files, paths, or compatible open handles + such as ``openmatrix.File``. OpenMatrix and h5py handles are used + directly; legacy filename-bearing handles are reopened with OpenMatrix. + Caller-owned handles remain open. index_names : tuple, default ("otaz", "dtaz", "time_period") Should be a tuple of length 3, giving the names of the three dimensions. The first two names are the native dimensions from @@ -510,73 +893,111 @@ def from_omx_3d( match the name of a variable, that variable will not be included in the loaded dataset. This is useful for excluding variables that are not needed in the current application. + load : {"lazy", "eager", "shared", "memmap"}, default "lazy" + Loading mode. ``"lazy"`` returns Dask arrays. ``"eager"`` loads into + ordinary NumPy arrays in the calling process. ``"shared"`` uses + parallel reads into shared memory and is the fastest mode for multiple + large OMX files. ``"memmap"`` uses the same parallel loader with + disk-backed arrays, substantially reducing resident memory at the cost + of additional storage I/O. + task_granularity : {"matrix", "variable"}, default "matrix" + Dask task granularity for lazy loading. The default keeps one task per + period, avoiding reads of unselected periods and limiting intermediate + arrays during copies or realignment. Opt into "variable" to group all + periods and reduce graph and file-open overhead for whole-variable reads. + workers : int, optional + Number of source files to load concurrently for ``"shared"`` or + ``"memmap"``. The default uses up to one worker per source file. + Windows uses threads so this API is safe in ordinary scripts and + notebooks; other platforms use spawned processes. Eager loading uses + one worker in the calling process. + memory_path : path-like, optional + New backing file to create when ``load="memmap"``. The associated + metadata is stored alongside it with a ``.meta.pkl`` suffix. Release + the accessor and discard the Dataset and all retained array views before + deleting both files with ``Dataset.shm.delete_shared_memory_files(key)``. + Windows requires all mappings to be closed before file deletion. + shared_memory_key : str, optional + Key used to identify an explicitly shared dataset. A unique key is + generated by default. Call ``result.shm.release_shared_memory()`` when + a shared result is no longer needed. Returns ------- Dataset + Lazy Dask-backed, ordinary in-memory, shared-memory-backed, or + memory-mapped according to ``load``. """ - if not isinstance(omx, (list, tuple)): - omx = [omx] + if load is True: + load = "eager" + elif load is False: + load = "lazy" + if load not in {"lazy", "eager", "shared", "memmap"}: + raise ValueError("load must be 'lazy', 'eager', 'shared', or 'memmap'") + if task_granularity not in {"variable", "matrix"}: + raise ValueError("task_granularity must be 'variable' or 'matrix'") + if workers is not None and (not isinstance(workers, int) or workers < 1): + raise ValueError("workers must be a positive integer") + if load == "eager" and workers not in {None, 1}: + raise ValueError( + "load='eager' uses one worker; use load='shared' for parallel reads" + ) + if load == "memmap" and memory_path is None: + raise ValueError("memory_path is required when load='memmap'") + if load != "memmap" and memory_path is not None: + raise ValueError("memory_path is only used when load='memmap'") + + if isinstance(omx, (h5py.File, str, os.PathLike)) or omx_file_name(omx): + omx_sources = [omx] + else: + omx_sources = list(omx) + if not omx_sources: + raise ValueError("at least one OMX file is required") use_file_handles = [] opened_file_handles = [] - for filename in omx: - if isinstance(filename, str): - import openmatrix - - h = openmatrix.open_file(filename) - opened_file_handles.append(h) - use_file_handles.append(h) - else: - use_file_handles.append(filename) - omx = use_file_handles - try: - import h5py - - # handle larch.OMX, openmatrix.open_file, and h5py.File versions - if "larch" in type(omx[0]).__module__: - omx_shape = omx[0].shape - omx_lookup = omx[0].lookup - elif isinstance(omx[0], h5py.File): - omx_shape = tuple(int(i) for i in omx[0].attrs["SHAPE"]) - omx_lookup = omx[0]["lookup"] - else: - omx_shape = omx[0].shape() - omx_lookup = omx[0].root["lookup"] - omx_data = [] - omx_data_map = {} - for n, i in enumerate(omx): - if "larch" in type(i).__module__: - omx_data.append(i.data) - elif isinstance(i, h5py.File): - omx_data.append(i["data"]) + for source in omx_sources: + if isinstance(source, (str, os.PathLike)): + h = openmatrix.open_file(source, "r") + opened_file_handles.append(h) + use_file_handles.append(h) + elif isinstance(source, h5py.File): + use_file_handles.append(source) else: - omx_data.append(i.root["data"]) - for k in _group_names(omx_data[-1]): - omx_data_map[k] = n + # Preserve compatibility with legacy PyTables and similar + # optional wrappers without importing those dependencies. They + # expose the backing HDF5 path through ``filename``. + filename = omx_file_name(source) + if filename is None: + raise TypeError( + "omx entries must be h5py.File, path-like, or " + "filename-bearing OMX handles" + ) + h = openmatrix.open_file(filename, "r") + opened_file_handles.append(h) + use_file_handles.append(h) + except Exception: + for handle in opened_file_handles: + handle.close() + raise + omx_handles = use_file_handles - omx_filenames = [omx_file_name(i) for i in omx] + try: + omx_shape = _omx_shape(omx_handles[0]) + omx_lookup = omx_handles[0]["/lookup"] + omx_data = [handle["/data"] for handle in omx_handles] + omx_data_map = {} + matrix_metadata = {} + for source_number, data_group in enumerate(omx_data): + for data_name in _group_names(data_group): + node = data_group[data_name] + omx_data_map[data_name] = source_number + matrix_metadata[data_name] = (tuple(node.shape), np.dtype(node.dtype)) + + omx_filenames = [omx_file_name(i) for i in omx_handles] omx_reopenable = [_is_reopenable(i) for i in omx_filenames] - import dask.array - - def _lazy_omx_array(k): - # Build a lazy dask array for one matrix table. When the source - # file can be independently reopened, defer to the parallel - # chunk-decoding reader; otherwise read the data eagerly, as the - # open file handle may be closed before the dask graph is computed. - n = omx_data_map[k] - filename = omx_filenames[n] - node = omx_data[n][k] - if not omx_reopenable[n]: - return dask.array.from_array(np.asarray(node[:])) - return dask.array.from_delayed( - dask.delayed(_fast_load_omx_array)(filename, k), - shape=tuple(node.shape), - dtype=node.dtype, - ) - data_names = list(omx_data_map.keys()) if ignore is not None: if isinstance(ignore, str): @@ -605,7 +1026,7 @@ def _lazy_omx_array(k): r1 = ranger(n1) r2 = ranger(n2) else: - r1 = r2 = pd.Index(omx_lookup[indexes]) + r1 = r2 = pd.Index(np.asarray(omx_lookup[indexes])) if time_periods is None: raise ValueError("must give time periods explicitly") @@ -613,50 +1034,243 @@ def _lazy_omx_array(k): time_periods_map = {t: n for n, t in enumerate(time_periods)} pending_3d = {} - content = {} - - for k in data_names: - if time_period_sep in k: - base_k, time_k = k.split(time_period_sep, 1) - if base_k not in pending_3d: - pending_3d[base_k] = [None] * len(time_periods) - pending_3d[base_k][time_periods_map[time_k]] = _lazy_omx_array(k) + variable_specs = {} + for data_name in data_names: + source_number = omx_data_map[data_name] + matrix_shape, matrix_dtype = matrix_metadata[data_name] + if matrix_shape != omx_shape: + source_label = omx_filenames[source_number] or "" + reference_label = omx_filenames[0] or "" + raise ValueError( + f"matrix {data_name!r} in OMX source {source_label!r} has shape " + f"{matrix_shape}, expected {omx_shape} from the first source " + f"{reference_label!r}. Check SHAPE metadata and coordinate " + "consistency. Use ignore to exclude this matrix only if it " + "is not needed; ignore cannot repair incorrect shape metadata." + ) + entry = (source_number, data_name, matrix_dtype) + if time_period_sep in data_name: + base_name, period_name = data_name.split(time_period_sep, 1) + if period_name not in time_periods_map: + raise KeyError( + f"time period {period_name!r} from {data_name!r} is not in " + "time_periods" + ) + pending_3d.setdefault(base_name, [None] * len(time_periods))[ + time_periods_map[period_name] + ] = entry else: - content[k] = xr.DataArray( - _lazy_omx_array(k), - dims=index_names[:2], - coords={ - index_names[0]: r1, - index_names[1]: r2, - }, + variable_specs[data_name] = { + "pages": [entry], + "shape": omx_shape, + "dtype": _omx_target_dtype(matrix_dtype, max_float_precision), + } + for base_name, pages in pending_3d.items(): + source_dtypes = [page[2] for page in pages if page is not None] + variable_specs[base_name] = { + "pages": pages, + "shape": omx_shape + (len(time_periods),), + "dtype": _omx_target_dtype( + np.result_type(*source_dtypes), max_float_precision + ), + } + + coords = {index_names[0]: r1, index_names[1]: r2} + if pending_3d: + coords[index_names[2]] = time_periods + + # Each assignment belongs to exactly one source file. Duplicate OMX + # names retain the established last-file-wins behavior. + assignments = [[] for _ in omx_handles] + for variable_name, spec in variable_specs.items(): + is_3d = len(spec["shape"]) == 3 + for period, page in enumerate(spec["pages"]): + if page is None: + continue + source_number, data_name, _ = page + assignments[source_number].append( + (data_name, variable_name, period if is_3d else None) ) - for base_k, darrs in pending_3d.items(): - # find a prototype array - prototype = None - for i in darrs: - prototype = i - if prototype is not None: - break - if prototype is None: - raise ValueError("no prototype") - darrs_ = [ - (i if i is not None else dask.array.zeros_like(prototype)) - for i in darrs - ] - content[base_k] = xr.DataArray( - dask.array.stack(darrs_, axis=-1), - dims=index_names, - coords={ - index_names[0]: r1, - index_names[1]: r2, - index_names[2]: time_periods, - }, + + if load == "lazy": + import dask + import dask.array + + eager_arrays = {} + + def source_descriptor(page): + if page is None: + return None + source_number, data_name, _ = page + if omx_reopenable[source_number]: + return (str(omx_filenames[source_number]), data_name, None) + if data_name not in eager_arrays: + eager_arrays[data_name] = np.asarray( + omx_data[source_number][data_name][()] + ) + return (None, data_name, eager_arrays[data_name]) + + content = {} + for variable_name, spec in variable_specs.items(): + dtype = spec["dtype"] + if len(spec["shape"]) == 2: + page = spec["pages"][0] + descriptor = source_descriptor(page) + filename, data_name, eager_array = descriptor + if eager_array is not None: + array = dask.array.from_array(eager_array).astype(dtype) + else: + array = dask.array.from_delayed( + dask.delayed(_fast_load_omx_array)( + filename, data_name, dtype + ), + shape=spec["shape"], + dtype=dtype, + ) + elif task_granularity == "variable": + page_sources = [source_descriptor(i) for i in spec["pages"]] + array = dask.array.from_delayed( + dask.delayed(_load_omx_variable)( + page_sources, spec["shape"], dtype + ), + shape=spec["shape"], + dtype=dtype, + ) + else: + page_arrays = [] + for page in spec["pages"]: + descriptor = source_descriptor(page) + if descriptor is None: + page_arrays.append( + dask.array.zeros( + omx_shape, chunks=omx_shape, dtype=dtype + ) + ) + continue + filename, data_name, eager_array = descriptor + if eager_array is not None: + page_array = dask.array.from_array(eager_array).astype( + dtype + ) + else: + page_array = dask.array.from_delayed( + dask.delayed(_fast_load_omx_array)( + filename, data_name, dtype + ), + shape=omx_shape, + dtype=dtype, + ) + page_arrays.append(page_array) + array = dask.array.stack(page_arrays, axis=-1) + dims = index_names if array.ndim == 3 else index_names[:2] + content[variable_name] = (dims, array) + return xr.Dataset(content, coords=coords) + + if load == "eager": + content = {} + for variable_name, spec in variable_specs.items(): + if len(spec["shape"]) == 3: + array = _empty_omx_3d(spec["shape"], spec["dtype"]) + for period, page in enumerate(spec["pages"]): + if page is None: + array[..., period].fill(0) + else: + array = np.empty(spec["shape"], dtype=spec["dtype"]) + dims = index_names if array.ndim == 3 else index_names[:2] + content[variable_name] = (dims, array) + result = xr.Dataset(content, coords=coords) + with _OmxReadCoordinator() as reader: + for source, source_assignments in zip(omx_handles, assignments): + if source_assignments: + _load_omx_assignments( + result, source, source_assignments, reader=reader + ) + return result + + # Shared and memory-mapped modes use a lightweight template to reserve + # one contiguous backing buffer, then independent workers fill each + # source file directly into disjoint final-array pages. + if not all(omx_reopenable[n] for n, batch in enumerate(assignments) if batch): + raise ValueError( + f"load={load!r} requires path-backed OMX files that can be reopened" ) - for i in content: - if np.issubdtype(content[i].dtype, np.floating): - if content[i].dtype.itemsize > max_float_precision / 8: - content[i] = content[i].astype(f"float{max_float_precision}") - return xr.Dataset(content) + import dask.array + + template_content = {} + array_order = {} + for variable_name, spec in variable_specs.items(): + dims = index_names if len(spec["shape"]) == 3 else index_names[:2] + template_content[variable_name] = ( + dims, + dask.array.empty( + spec["shape"], chunks=spec["shape"], dtype=spec["dtype"] + ), + ) + if len(spec["shape"]) == 3: + array_order[variable_name] = "last-axis-first" + template = xr.Dataset(template_content, coords=coords) + + if load == "memmap": + memory_path = Path(memory_path).expanduser().resolve() + metadata_path = Path(f"{memory_path}.meta.pkl") + if memory_path.exists() or metadata_path.exists(): + raise FileExistsError( + f"memory_path and metadata path must not already exist: {memory_path}" + ) + memory_path.parent.mkdir(parents=True, exist_ok=True) + key = f"memmap:{memory_path}" + else: + if shared_memory_key is not None and shared_memory_key.startswith( + "memmap:" + ): + raise ValueError("shared_memory_key must not start with 'memmap:'") + key = shared_memory_key or f"omx-{secrets.token_hex(8)}" + + result = template.shm.to_shared_memory( + key, mode="r+", load=False, array_order=array_order + ) + if hasattr(result.shm, "tasks"): + del result.shm.tasks + if hasattr(result.shm, "task_names"): + del result.shm.task_names + + active_batches = [ + (str(omx_filenames[n]), batch) + for n, batch in enumerate(assignments) + if batch + ] + worker_count = _omx_worker_count(workers, len(active_batches)) + started = time.time() + try: + bytes_loaded = _load_omx_batches(result, active_batches, worker_count) + if load == "memmap": + for memory_object in result.shm._shared_memory_objs_: + flush = getattr(memory_object, "flush", None) + if flush is not None: + flush() + logger.info( + "loaded %s from %d OMX files with %d worker(s) in %.2fs", + si_units(bytes_loaded), + len(active_batches), + worker_count, + time.time() - started, + ) + return result + except Exception: + if load == "shared": + result.shm.release_shared_memory() + else: + # Close the mapping before unlinking its backing files. Windows + # does not permit deletion while the memmap is still open. + # No result has escaped this constructor, and worker pools have + # stopped. These unpublished mappings can be closed immediately. + memory_objects = list(result.shm._shared_memory_objs_) + result.shm.release_shared_memory() + from .shared_memory import _close_memmap_objects + + _close_memmap_objects(memory_objects) + result.shm.delete_shared_memory_files(key) + raise finally: for h in opened_file_handles: h.close() @@ -664,27 +1278,28 @@ def _lazy_omx_array(k): def reload_from_omx_3d( dataset: xr.Dataset, - omx: Iterable[str], + omx: h5py.File | str | os.PathLike | Iterable[h5py.File | str | os.PathLike], *, time_period_sep="__", ignore=None, + workers=None, ) -> None: """ Reload the content of a dataset from OMX files. This loads the data from the OMX files into the dataset, replacing the existing data in the dataset. The dataset must have been created - by `from_omx_3d` or a similar function. Note that `from_omx_3d` will - create a dataset backed by `dask.array` objects; this function allows for - loading the data without going through dask, which may have poor performance - on some platforms. + by `from_omx_3d` or a similar function. By default, `from_omx_3d` creates + a dataset backed by `dask.array` objects; this function allows for loading + the data without going through Dask. Parameters ---------- dataset : xr.Dataset The dataset to reload into. - omx : Iterable[str] - The list of OMX file names to load from. + omx : openmatrix.File, h5py.File, path-like, filename-bearing OMX handle, or iterable + One or more OMX-format HDF5 files, paths, or compatible open handles + such as ``openmatrix.File``. time_period_sep : str, default "__" The separator used to identify time periods in the dataset. ignore : list-like, optional @@ -693,90 +1308,66 @@ def reload_from_omx_3d( match the name of a variable, that variable will not be included in the load process. This is useful for excluding variables that are not found in the target dataset. + workers : int, optional + Number of source files to load concurrently. The default loads files + serially, retaining parallel chunk decoding within each file. Opt into + file-level parallelism with workers > 1 and a shared-memory destination + plus reopenable sources. Windows uses threads; other platforms spawn + processes and require a guarded script entry point. """ if isinstance(ignore, str): ignore = [ignore] + if isinstance(omx, (h5py.File, str, os.PathLike)) or omx_file_name(omx): + sources = [omx] + else: + sources = list(omx) + if workers is not None and (not isinstance(workers, int) or workers < 1): + raise ValueError("workers must be a positive integer") + + filenames, assignments = _prepare_omx_reload_assignments( + dataset, sources, time_period_sep, ignore + ) + active_batches = [ + (source_number, batch) + for source_number, batch in enumerate(assignments) + if batch + ] + if not active_batches: + logger.info("no OMX matrices selected for reload") + return + + parallel_capable = dataset.shm.is_shared_memory and all( + _is_reopenable(filenames[source_number]) for source_number, _ in active_batches + ) + if workers is None: + worker_count = 1 + else: + worker_count = _omx_worker_count(workers, len(active_batches)) + if worker_count > 1 and not parallel_capable: + raise ValueError( + "parallel OMX reload requires a shared-memory-backed dataset " + "and path-backed source files" + ) - import concurrent.futures - - import h5py - - bytes_loaded = 0 - t0 = time.time() - - def _load_into(raw, dset, executor): - """Fill the array `raw` from the h5py or pytables dataset `dset`.""" - if isinstance(dset, h5py.Dataset): - if raw.dtype == dset.dtype: - omx_reader.read_dataset(dset, out=raw, executor=executor) - else: - raw[:, :] = omx_reader.read_dataset(dset, executor=executor) - else: - raw[:, :] = dset[:, :] - - def _load_one(dset, data_name, filter_note, executor): - nonlocal bytes_loaded - t1 = time.time() - if time_period_sep in data_name: - data_name_x, data_name_t = data_name.split(time_period_sep, 1) - if data_name_x not in dataset: - logger.info( - f"skipping {data_name} because {data_name_x} not in dataset" - ) - return - if len(dataset[data_name_x].dims) != 3: - raise ValueError( - f"dataset variable {data_name_x} has " - f"{len(dataset[data_name_x].dims)} dimensions, expected 3" - ) - raw = dataset[data_name_x].sel(time_period=data_name_t).data - else: - if len(dataset[data_name].dims) != 2: - raise ValueError( - f"dataset variable {data_name} has " - f"{len(dataset[data_name].dims)} dimensions, expected 2" - ) - raw = dataset[data_name].data - _load_into(raw, dset, executor) - bytes_loaded += raw.nbytes - logger.debug( - f"loaded {data_name} ({filter_note}) to dataset " - f"in {time.time() - t1:.2f}s, {si_units(bytes_loaded)}" + started = time.time() + load_batches = [ + ( + sources[source_number] + if worker_count == 1 + else str(filenames[source_number]), + batch, ) - - opened_file_handles = [] - try: - with concurrent.futures.ThreadPoolExecutor() as pool: - for source in omx: - filename = omx_file_name(source) - if filename is not None: - # fast path: parallel chunk decoding via h5py - logger.info(f"loading into dataset from {filename}") - f5 = h5py.File(filename, "r") - opened_file_handles.append(f5) - data_group = f5["data"] - for data_name in data_group.keys(): - if _should_ignore(ignore, data_name): - logger.info(f"ignoring {data_name}") - continue - dset = data_group[data_name] - filter_note = f"{dset.compression}/{dset.compression_opts}" - _load_one(dset, data_name, filter_note, pool) - else: - # source is an open file handle with no resolvable - # on-disk filename; read through the handle directly - f = source - for data_name in f.root.data._v_children: - if _should_ignore(ignore, data_name): - logger.info(f"ignoring {data_name}") - continue - filters = f.root.data[data_name].filters - filter_note = f"{filters.complib}/{filters.complevel}" - _load_one(f.root.data[data_name], data_name, filter_note, pool) - logger.info(f"loading to dataset complete in {time.time() - t0:.2f}s") - finally: - for h in opened_file_handles: - h.close() + for source_number, batch in active_batches + ] + bytes_loaded = _load_omx_batches(dataset, load_batches, worker_count) + + logger.info( + "reloaded %s from %d OMX files with %d worker(s) in %.2fs", + si_units(bytes_loaded), + len(active_batches), + worker_count, + time.time() - started, + ) def _parquet_layout(labels_0, labels_1): diff --git a/sharrow/example_data.py b/sharrow/example_data.py index 10ec665..62bc20d 100644 --- a/sharrow/example_data.py +++ b/sharrow/example_data.py @@ -4,6 +4,7 @@ from pathlib import Path import numpy as np +import openmatrix import pandas as pd @@ -28,13 +29,10 @@ def get_skims_filename() -> str: def get_skims_omx(): - import openmatrix - from . import dataset with get_example_data_path("skims.omx") as filename: - skims = None - with openmatrix.open_file(str(filename)) as f: + with openmatrix.open_file(filename, "r") as f: skims = dataset.from_omx_3d( f, index_names=("otaz", "dtaz", "time_period"), diff --git a/sharrow/omx.py b/sharrow/omx.py index 1340779..191be58 100644 --- a/sharrow/omx.py +++ b/sharrow/omx.py @@ -1,59 +1,87 @@ import logging -import os +from pathlib import Path + +import h5py +import openmatrix logger = logging.getLogger("sharrow.omx") +def _initialize_omx_file(source: h5py.File, target: h5py.File) -> None: + """Copy OMX metadata and ensure the standard HDF5 groups exist.""" + for name, value in source.attrs.items(): + target.attrs[name] = value + for group_name in ("data", "lookup"): + source_group = source[f"/{group_name}"] + target_group = target["/"].require_group(group_name) + for name, value in source_group.attrs.items(): + target_group.attrs[name] = value + + +def _copy_dataset( + source_group: h5py.Group, target_group: h5py.Group, name: str +) -> None: + """Copy one HDF5 dataset, replacing a same-named target if present.""" + if name in target_group: + del target_group[name] + source_group.copy(name, target_group, name=name) + + def split_omx(source_file, dest_directory, global_lookups=False, n_chunks=None): - """ - Split an OMX file into separate files for each element. + """Split the matrices in an OMX file across smaller OMX files. Parameters ---------- - source_file : str - dest_directory : str - global_lookups : bool - + source_file : str or path-like + OMX-format HDF5 file to split. + dest_directory : str or path-like + Directory in which to write the split files. + global_lookups : bool, default False + If true, copy all lookups into every matrix output. Otherwise, write + each lookup to its own OMX file. + n_chunks : int, optional + Number of output matrix files. By default, write one file per matrix. """ - try: - from larch import OMX - except ImportError: - raise ImportError("larch is required to split OMX files") from None - - source = OMX(source_file, mode="r") - os.makedirs(dest_directory, exist_ok=True) - - if n_chunks is not None: - general_name = os.path.splitext(source_file)[0] - chunkfiles = [f"{general_name}-chunk{n}.omx" for n in range(n_chunks)] - else: - chunkfiles = [f"{k}.omx" for k in list(source.data._v_children)] - - n = 0 - for k in list(source.data._v_children): - newfile = os.path.join( - dest_directory, - chunkfiles[n], - ) - logger.info(f"writing {k} to {newfile}") - b = OMX(newfile, mode="a") - b.add_matrix(k, source.data[k], overwrite=True) + if n_chunks is not None and n_chunks < 1: + raise ValueError("n_chunks must be a positive integer") + + source_path = Path(source_file) + destination = Path(dest_directory) + destination.mkdir(parents=True, exist_ok=True) + + with openmatrix.open_file(source_path, "r") as source: + matrix_names = list(source["/data"].keys()) + if n_chunks is not None: + chunk_names = [ + f"{source_path.stem}-chunk{number}.omx" for number in range(n_chunks) + ] + else: + chunk_names = [f"{name}.omx" for name in matrix_names] + + output_paths = [] + initialized_outputs = set() + for number, matrix_name in enumerate(matrix_names): + output_path = destination / chunk_names[number % len(chunk_names)] + output_paths.append(output_path) + logger.info(f"writing {matrix_name} to {output_path}") + # Truncate each output the first time it is used so rerunning a + # split cannot preserve stale matrices or lookups. Later matrices + # assigned to the same chunk append to the freshly initialized file. + mode = "a" if output_path in initialized_outputs else "w" + initialized_outputs.add(output_path) + with openmatrix.open_file(output_path, mode) as target: + _initialize_omx_file(source, target) + _copy_dataset(source["/data"], target["/data"], matrix_name) + if global_lookups: - # todo only write once per file - for j in list(source.lookup._v_children): - b.add_lookup(j, source.lookup[j], overwrite=True) - b.close() - n += 1 - if n >= len(chunkfiles): - n = 0 - if not global_lookups: - for k in list(source.lookup._v_children): - newfile = os.path.join( - dest_directory, - f"_{k}.omx", - ) - logger.info(f"writing {newfile}") - b = OMX(newfile, mode="w") - b.shape = source.shape - b.add_lookup(k, source.lookup[k]) - b.close() + for output_path in dict.fromkeys(output_paths): + with openmatrix.open_file(output_path, "a") as target: + for lookup_name in source["/lookup"]: + _copy_dataset(source["/lookup"], target["/lookup"], lookup_name) + else: + for lookup_name in source["/lookup"]: + output_path = destination / f"_{lookup_name}.omx" + logger.info(f"writing {output_path}") + with openmatrix.open_file(output_path, "w") as target: + _initialize_omx_file(source, target) + _copy_dataset(source["/lookup"], target["/lookup"], lookup_name) diff --git a/sharrow/omx_reader.py b/sharrow/omx_reader.py index e9321b3..f951742 100644 --- a/sharrow/omx_reader.py +++ b/sharrow/omx_reader.py @@ -9,7 +9,8 @@ If a dataset uses a filter this module does not know how to invert, reading transparently falls back to h5py, which will use the registered HDF5 filter -plugins instead. +plugins instead. Standard gzip/shuffle OMX data needs no optional plugin; +install ``sharrow[hdf5-plugins]`` for Blosc and other HDF5 filter families. This module is adapted from the ``omx_fast_reader`` module of the `wring `_ project. @@ -19,14 +20,24 @@ import concurrent.futures import os +import threading import zlib from collections.abc import Sequence -import blosc2 import h5py -import hdf5plugin # noqa: F401 (registers blosc/blosc2/zstd/etc HDF5 filters) import numpy as np +try: + import blosc2 +except ImportError: # Blosc-compressed OMX files require the optional extra. + blosc2 = None + +try: + # Optional plugins let h5py handle filters outside the native fast path. + import hdf5plugin # noqa: F401 +except ImportError: # Standard gzip/shuffle OMX files do not need plugins. + hdf5plugin = None + __all__ = [ "read_dataset", "h5_filename", @@ -44,20 +55,18 @@ H5Z_FILTER_DEFLATE, H5Z_FILTER_SHUFFLE, H5Z_FILTER_FLETCHER32, - H5Z_FILTER_BLOSC, - H5Z_FILTER_BLOSC2, + *((H5Z_FILTER_BLOSC, H5Z_FILTER_BLOSC2) if blosc2 is not None else ()), } ) -def h5_filename(omx) -> str | None: - """Resolve the on-disk filename of an OMX-like object. +def h5_filename(omx: h5py.File | str | os.PathLike) -> str | None: + """Resolve the on-disk filename of an OMX HDF5 file. Parameters ---------- - omx : str, os.PathLike, h5py.File, tables.File, or larch.OMX - An OMX file reference: a path, an h5py file, a pytables file (which - includes ``openmatrix.File``), or a larch OMX object. + omx : str, os.PathLike, or h5py.File + An OMX file path or open HDF5 file. Returns ------- @@ -66,8 +75,8 @@ def h5_filename(omx) -> str | None: """ if isinstance(omx, (str, os.PathLike)): return os.fspath(omx) - # h5py.File, tables.File (and openmatrix.File), and larch.OMX all expose - # the underlying file path as a `filename` attribute. + if not isinstance(omx, h5py.File): + return None filename = getattr(omx, "filename", None) if isinstance(filename, (str, os.PathLike)): filename = os.fspath(filename) @@ -148,6 +157,8 @@ def _decode_blosc2_cframe(data: bytes): bytes or numpy.ndarray The decompressed payload. """ + if blosc2 is None: + raise ImportError("Blosc-compressed OMX data requires sharrow[hdf5-plugins]") try: return blosc2.ndarray_from_cframe(data)[:] except (RuntimeError, ValueError): @@ -231,6 +242,7 @@ def _write_chunk( filter_mask: int, pipeline: Sequence, chunk_shape: tuple, + source_dtype: np.dtype, ) -> None: """Decode one raw chunk and write it into its place in `out`. @@ -251,8 +263,13 @@ def _write_chunk( The dataset filter pipeline, in write order. chunk_shape : tuple[int, ...] Shape of a full chunk. + source_dtype : numpy.dtype + Element type encoded in the source chunk. This may differ from the + destination dtype; assignment performs a chunk-local conversion. """ - chunk = _decompress_chunk(raw_bytes, filter_mask, pipeline, out.dtype, chunk_shape) + chunk = _decompress_chunk( + raw_bytes, filter_mask, pipeline, source_dtype, chunk_shape + ) # Edge chunks are stored padded out to the full chunk shape, so the part # that actually lands in the dataset may be smaller than the chunk itself. @@ -267,9 +284,21 @@ def _write_chunk( def _fallback_read(dset: h5py.Dataset, out: np.ndarray) -> None: """Read `dset` into `out` using h5py's ordinary (serial) read path.""" if out.flags.c_contiguous: + # HDF5 converts to the destination dtype during a direct read. dset.read_direct(out) else: - out[...] = dset[()] + out[...] = dset.astype(out.dtype)[()] + + +def _write_chunk_and_release( + pending_semaphore: threading.Semaphore, + *args, +) -> None: + """Write a decoded chunk and return its slot to the global memory budget.""" + try: + _write_chunk(*args) + finally: + pending_semaphore.release() def _load_dataset_into( @@ -277,6 +306,7 @@ def _load_dataset_into( out: np.ndarray, executor: concurrent.futures.ThreadPoolExecutor, max_pending: int, + pending_semaphore: threading.Semaphore | None = None, ) -> None: """Fill `out` with the contents of `dset`, decoding chunks in parallel. @@ -288,11 +318,14 @@ def _load_dataset_into( dset : h5py.Dataset Source dataset. out : numpy.ndarray - Destination array; must have the same shape and dtype as `dset`. + Destination array; must have the same shape as `dset`. Source chunks + are converted to the destination dtype as they are written. executor : concurrent.futures.ThreadPoolExecutor Pool used to decode chunks. max_pending : int Maximum number of raw chunks held in memory awaiting decoding. + pending_semaphore : threading.Semaphore, optional + Process-wide limit on chunks queued by concurrent dataset reads. """ chunk_shape = dset.chunks if chunk_shape is None or dset.size == 0: @@ -321,9 +354,6 @@ def _load_dataset_into( pending = set() for i in range(num_chunks): - offset = dset_id.get_chunk_info(i).chunk_offset - filter_mask, raw_bytes = dset_id.read_direct_chunk(offset) - if len(pending) >= max_pending: done, pending = concurrent.futures.wait( pending, return_when=concurrent.futures.FIRST_COMPLETED @@ -331,17 +361,39 @@ def _load_dataset_into( for future in done: future.result() - pending.add( - executor.submit( - _write_chunk, - out, - offset, - raw_bytes, - filter_mask, - pipeline, - chunk_shape, - ) - ) + if pending_semaphore is not None: + pending_semaphore.acquire() + try: + offset = dset_id.get_chunk_info(i).chunk_offset + filter_mask, raw_bytes = dset_id.read_direct_chunk(offset) + if pending_semaphore is None: + future = executor.submit( + _write_chunk, + out, + offset, + raw_bytes, + filter_mask, + pipeline, + chunk_shape, + dset.dtype, + ) + else: + future = executor.submit( + _write_chunk_and_release, + pending_semaphore, + out, + offset, + raw_bytes, + filter_mask, + pipeline, + chunk_shape, + dset.dtype, + ) + except BaseException: + if pending_semaphore is not None: + pending_semaphore.release() + raise + pending.add(future) for future in concurrent.futures.as_completed(pending): future.result() @@ -359,6 +411,7 @@ def read_dataset( out: np.ndarray | None = None, executor: concurrent.futures.ThreadPoolExecutor | None = None, max_workers: int | None = None, + pending_semaphore: threading.Semaphore | None = None, ) -> np.ndarray: """Read an already-open HDF5 dataset, decoding its chunks in parallel. @@ -367,14 +420,17 @@ def read_dataset( dset : h5py.Dataset The dataset to read. out : numpy.ndarray, optional - Destination array, which must have the same shape and dtype as - `dset`. It may be a non-contiguous view (e.g. a slice of a larger - array). If not given, a new array is allocated. + Destination array, which must have the same shape as `dset`. It may + use a different dtype and may be a non-contiguous view (e.g. a slice + of a larger array). If not given, a source-dtype array is allocated. executor : concurrent.futures.ThreadPoolExecutor, optional Pool used to decode chunks. If not given, a temporary pool is created and shut down before returning. max_workers : int, optional Size of the temporary thread pool. Ignored when `executor` is given. + pending_semaphore : threading.Semaphore, optional + Shared bound on in-flight chunks when several calls use the same + executor concurrently. Returns ------- @@ -386,12 +442,22 @@ def read_dataset( else: if tuple(out.shape) != tuple(dset.shape): raise ValueError(f"out has shape {out.shape}, expected {dset.shape}") - if out.dtype != dset.dtype: - raise ValueError(f"out has dtype {out.dtype}, expected {dset.dtype}") if executor is not None: workers = getattr(executor, "_max_workers", None) - _load_dataset_into(dset, out, executor, _max_pending(workers)) + _load_dataset_into( + dset, + out, + executor, + _max_pending(workers), + pending_semaphore=pending_semaphore, + ) else: with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as pool: - _load_dataset_into(dset, out, pool, _max_pending(max_workers)) + _load_dataset_into( + dset, + out, + pool, + _max_pending(max_workers), + pending_semaphore=pending_semaphore, + ) return out diff --git a/sharrow/shared_memory.py b/sharrow/shared_memory.py index d93b38f..ce803ea 100644 --- a/sharrow/shared_memory.py +++ b/sharrow/shared_memory.py @@ -1,4 +1,5 @@ import atexit +import contextlib import hashlib import logging import os @@ -224,6 +225,20 @@ def delete_shared_memory_files(key): os.unlink(mmap_filename + ".meta.pkl") +def _close_memmap_objects(memory_objects, buffer=None): + """Flush and close memmap resources so Windows can unlink their files.""" + if buffer is not None: + buffer.release() + for memory_object in memory_objects: + if isinstance(memory_object, np.memmap): + try: + memory_object.flush() + finally: + mmap = getattr(memory_object, "_mmap", None) + if mmap is not None and not mmap.closed: + mmap.close() + + @xr.register_dataset_accessor("shm") class SharedMemDatasetAccessor: _parent_class = xr.Dataset @@ -245,8 +260,27 @@ def __repr__(self): return r def release_shared_memory(self): - """Release shared memory allocated to this Dataset.""" - release_shared_memory(self._shared_memory_key_) + """Release this accessor's ownership of shared storage. + + Memmap views keep their mapping alive until the last array reference is + discarded. On Windows, discard the Dataset and any retained views before + calling ``delete_shared_memory_files``; release does not invalidate them. + """ + key = self._shared_memory_key_ + if key and key.startswith("memmap:"): + # NumPy views retain the memmap through their base chain. Never + # close its mmap explicitly here: callers may still hold those views. + for memory_object in self._shared_memory_objs_: + if isinstance(memory_object, np.memmap): + memory_object.flush() + buffer = getattr(self, "_buffer", None) + if buffer is not None: + buffer.release() + del self._buffer + self._shared_memory_objs_.clear() + self._shared_memory_owned_ = False + else: + release_shared_memory(key) @staticmethod def delete_shared_memory_files(key): @@ -260,6 +294,7 @@ def to_shared_memory( dask_scheduler="threads", pre_init=False, load=True, + array_order=None, ): """ Load this Dataset into shared memory. @@ -291,6 +326,11 @@ def to_shared_memory( the `shm.tasks` attribute of the resulting Dataset object, but do not necessarily need to be run if data can be loaded using alternative methods (e.g. `sharrow.dataset.reload_from_omx_3d`). + array_order : mapping, optional + Optional storage order overrides keyed by variable name. The + ``"last-axis-first"`` order stores each page along the last axis + contiguously. This is useful when a three-dimensional array is + populated from a collection of two-dimensional source arrays. Returns ------- @@ -336,18 +376,33 @@ def emit(k, a, is_coord): a_nbytes = a.data.nbytes else: logger.debug(f"preparing dense array {a.name}") - wrappers.append( - { - "dims": a.dims, - "name": a.name, - "attrs": a.attrs, - "dtype": a.dtype, - "shape": a.shape, - "coord": is_coord, - "nbytes": a.nbytes, - "position": position, - } - ) + wrapper = { + "dims": a.dims, + "name": a.name, + "attrs": a.attrs, + "dtype": a.dtype, + "shape": a.shape, + "coord": is_coord, + "nbytes": a.nbytes, + "position": position, + } + order = None if array_order is None else array_order.get(k) + if order == "last-axis-first": + if a.ndim < 2: + raise ValueError( + "last-axis-first storage requires at least two dimensions" + ) + # The logical last axis is physically outermost, making + # each ``[..., page]`` view C contiguous for direct I/O. + itemsize = a.dtype.itemsize + strides = [itemsize] + for size in reversed(a.shape[1:-1]): + strides.insert(0, strides[0] * size) + strides.append(int(np.prod(a.shape[:-1])) * itemsize) + wrapper["strides"] = tuple(strides) + elif order is not None: + raise ValueError(f"unknown shared-memory array order {order!r}") + wrappers.append(wrapper) a_nbytes = a.nbytes sizes.append(a_nbytes) @@ -362,83 +417,106 @@ def emit(k, a, is_coord): a = self._obj[k] emit(k, a, False) - mem = create_shared_memory_array(key, size=position) - - logger.debug("declaring shared memory buffer") - if key.startswith("memmap:"): - buffer = memoryview(mem) - else: - buffer = mem.buf - - if pre_init: - logger.debug("pre-initializing shared memory buffer") - # gross init with all zeros - buffer[:] = b"\0" * len(buffer) - - tasks = [] - task_names = [] - for w in wrappers: - _is_sparse = w.get("sparse", False) - _size = w["nbytes"] - _name = w["name"] - _pos = w["position"] - a = self._obj[_name] - if _is_sparse: - logger.info(f"running load task: {_name} ({si_units(_size)})") - ad = a.data - _size_d = w["data.nbytes"] - _size_i = w["indices.nbytes"] - _size_p = w["indptr.nbytes"] - mem_arr_d = np.ndarray( - shape=(_size_d // ad.data.dtype.itemsize,), - dtype=ad.data.dtype, - buffer=buffer[_pos : _pos + _size_d], - ) - mem_arr_i = np.ndarray( - shape=(_size_i // ad.indices.dtype.itemsize,), - dtype=ad.indices.dtype, - buffer=buffer[_pos + _size_d : _pos + _size_d + _size_i], - ) - mem_arr_p = np.ndarray( - shape=(_size_p // ad.indptr.dtype.itemsize,), - dtype=ad.indptr.dtype, - buffer=buffer[ - _pos + _size_d + _size_i : _pos + _size_d + _size_i + _size_p - ], - ) - mem_arr_d[:] = ad.data[:] - mem_arr_i[:] = ad.indices[:] - mem_arr_p[:] = ad.indptr[:] - else: - logger.debug(f"preparing load task: {_name} ({si_units(_size)})") - mem_arr = np.ndarray( - shape=a.shape, dtype=a.dtype, buffer=buffer[_pos : _pos + _size] - ) - if isinstance(a, xr.DataArray) and isinstance(a.data, da.Array): - tasks.append(da.store(a.data, mem_arr, lock=False, compute=False)) - task_names.append(_name) + is_memmap = key.startswith("memmap:") + memmap_created = is_memmap and not os.path.isfile(key[7:]) + mem = None + buffer = None + try: + mem = create_shared_memory_array(key, size=position) + logger.debug("declaring shared memory buffer") + buffer = memoryview(mem) if is_memmap else mem.buf + + if pre_init: + logger.debug("pre-initializing shared memory buffer") + # Fill in place. Constructing a same-sized bytes object can + # temporarily double memory use for multi-gigabyte skim datasets. + np.ndarray(len(buffer), dtype=np.uint8, buffer=buffer).fill(0) + + tasks = [] + task_names = [] + for w in wrappers: + _is_sparse = w.get("sparse", False) + _size = w["nbytes"] + _name = w["name"] + _pos = w["position"] + a = self._obj[_name] + if _is_sparse: + logger.info(f"running load task: {_name} ({si_units(_size)})") + ad = a.data + _size_d = w["data.nbytes"] + _size_i = w["indices.nbytes"] + _size_p = w["indptr.nbytes"] + mem_arr_d = np.ndarray( + shape=(_size_d // ad.data.dtype.itemsize,), + dtype=ad.data.dtype, + buffer=buffer[_pos : _pos + _size_d], + ) + mem_arr_i = np.ndarray( + shape=(_size_i // ad.indices.dtype.itemsize,), + dtype=ad.indices.dtype, + buffer=buffer[_pos + _size_d : _pos + _size_d + _size_i], + ) + mem_arr_p = np.ndarray( + shape=(_size_p // ad.indptr.dtype.itemsize,), + dtype=ad.indptr.dtype, + buffer=buffer[ + _pos + _size_d + _size_i : _pos + + _size_d + + _size_i + + _size_p + ], + ) + mem_arr_d[:] = ad.data[:] + mem_arr_i[:] = ad.indices[:] + mem_arr_p[:] = ad.indptr[:] else: - mem_arr[:] = a[:] - if tasks and load: - self.tasks = tasks - self.task_names = task_names - self.run_tasks(dask_scheduler=dask_scheduler) - - if key.startswith("memmap:"): - mem.flush() - - logger.info("storing metadata in shared memory") - create_shared_list( - [pickle.dumps(self._obj.attrs)] + [pickle.dumps(i) for i in wrappers], key - ) - result = type(self).from_shared_memory(key, own_data=mem, mode=mode) - if tasks and not load: - # attach incompleted tasks to the result - result.shm.tasks = tasks - result.shm.task_names = task_names - result.shm._buffer = buffer - result.shm._position = position - return result + logger.debug(f"preparing load task: {_name} ({si_units(_size)})") + mem_arr = np.ndarray( + shape=a.shape, + dtype=a.dtype, + buffer=buffer[_pos : _pos + _size], + strides=w.get("strides"), + ) + if isinstance(a, xr.DataArray) and isinstance(a.data, da.Array): + tasks.append( + da.store(a.data, mem_arr, lock=False, compute=False) + ) + task_names.append(_name) + else: + mem_arr[:] = a[:] + if tasks and load: + self.tasks = tasks + self.task_names = task_names + self.run_tasks(dask_scheduler=dask_scheduler) + + if is_memmap: + mem.flush() + + logger.info("storing metadata in shared memory") + create_shared_list( + [pickle.dumps(self._obj.attrs)] + [pickle.dumps(i) for i in wrappers], + key, + ) + result = type(self).from_shared_memory(key, own_data=mem, mode=mode) + if tasks and not load: + # attach incompleted tasks to the result + result.shm.tasks = tasks + result.shm.task_names = task_names + result.shm._buffer = buffer + result.shm._position = position + return result + except BaseException: + if is_memmap: + if mem is not None: + with contextlib.suppress(Exception): + _close_memmap_objects([mem], buffer=buffer) + if memmap_created: + with contextlib.suppress(Exception): + delete_shared_memory_files(key) + elif mem is not None: + with contextlib.suppress(Exception): + release_shared_memory(key) + raise def run_tasks(self, dask_scheduler="threads"): """Run any deferred dask tasks.""" @@ -500,7 +578,7 @@ def from_shared_memory(cls, key, own_data=False, mode="r+"): # for memmap, list is loaded from pickle, not shared ram pass - if own_data and own_data is not True: + if own_data is not None and not isinstance(own_data, (bool, np.bool_)): mem = own_data own_data = True else: @@ -523,6 +601,7 @@ def from_shared_memory(cls, key, own_data=False, mode="r+"): coord = t.pop("coord", False) # noqa: F841 position = t.pop("position") nbytes = t.pop("nbytes") + strides = t.pop("strides", None) is_sparse = t.pop("sparse", False) if is_sparse: if sparse is None: @@ -565,7 +644,10 @@ def from_shared_memory(cls, key, own_data=False, mode="r+"): ) else: mem_arr = np.ndarray( - shape, dtype=dtype, buffer=buffer[position : position + nbytes] + shape, + dtype=dtype, + buffer=buffer[position : position + nbytes], + strides=strides, ) content[name] = DataArray(mem_arr, **t) diff --git a/sharrow/tests/test_datasets.py b/sharrow/tests/test_datasets.py index a48b3e2..7b50159 100644 --- a/sharrow/tests/test_datasets.py +++ b/sharrow/tests/test_datasets.py @@ -1,7 +1,9 @@ +import gc import secrets import tempfile from pathlib import Path +import h5py import numpy as np import openmatrix import pandas as pd @@ -10,21 +12,20 @@ from pytest import approx import sharrow as sh +from sharrow import omx_reader +from sharrow.shared_memory import SharedMemDatasetAccessor def test_dataset_construct_with_zoneids(): tempdir = tempfile.TemporaryDirectory() t = Path(tempdir.name) - with openmatrix.open_file(t.joinpath("dummy5.omx"), mode="w") as out: - out.create_carray("/data", "Eye", obj=np.eye(5, dtype=np.float32)) - out.create_carray("/lookup", "Zone", obj=np.asarray([11, 22, 33, 44, 55])) - shp = np.empty(2, dtype=int) - shp[0] = 5 - shp[1] = 5 - out.root._v_attrs.SHAPE = shp + with h5py.File(t.joinpath("dummy5.omx"), mode="w") as out: + out.create_dataset("data/Eye", data=np.eye(5, dtype=np.float32)) + out.create_dataset("lookup/Zone", data=np.asarray([11, 22, 33, 44, 55])) + out.attrs["SHAPE"] = np.asarray([5, 5], dtype=int) - with openmatrix.open_file(t.joinpath("dummy5.omx"), mode="r") as back: + with h5py.File(t.joinpath("dummy5.omx"), mode="r") as back: ds = sh.dataset.from_omx(back, indexes="Zone") assert sorted(ds.coords) == ["dtaz", "otaz"] @@ -32,11 +33,11 @@ def test_dataset_construct_with_zoneids(): assert sorted(ds.variables) == ["Eye", "dtaz", "otaz"] assert ds["Eye"].data == approx(np.eye(5, dtype=np.float32)) - with openmatrix.open_file(t.joinpath("dummy5.omx"), mode="r") as back: + with h5py.File(t.joinpath("dummy5.omx"), mode="r") as back: ds0 = sh.dataset.from_omx(back, indexes="zero-based") assert ds0.coords["otaz"].values == approx(np.asarray([0, 1, 2, 3, 4])) - with openmatrix.open_file(t.joinpath("dummy5.omx"), mode="r") as back: + with h5py.File(t.joinpath("dummy5.omx"), mode="r") as back: ds1 = sh.dataset.from_omx(back, indexes="one-based") assert ds1.coords["otaz"].values == approx(np.asarray([1, 2, 3, 4, 5])) @@ -72,7 +73,7 @@ def income_cat(i): def test_load_with_ignore(): filename = sh.example_data.get_skims_filename() - with openmatrix.open_file(filename) as f: + with h5py.File(filename) as f: skims = sh.dataset.from_omx_3d( f, index_names=("otaz", "dtaz", "time_period"), @@ -83,7 +84,7 @@ def test_load_with_ignore(): ) assert "DRV_COM_WLK_FAR" in skims.variables - with openmatrix.open_file(filename) as f: + with h5py.File(filename) as f: skims1 = sh.dataset.from_omx_3d( f, index_names=("otaz", "dtaz", "time_period"), @@ -95,7 +96,7 @@ def test_load_with_ignore(): ) assert "DRV_COM_WLK_FAR" not in skims1.variables - with openmatrix.open_file(filename) as f: + with h5py.File(filename) as f: skims2 = sh.dataset.from_omx_3d( f, index_names=("otaz", "dtaz", "time_period"), @@ -121,7 +122,7 @@ def test_deferred_load_to_shared_memory(): from sharrow.example_data import get_skims_filename skims_filename = get_skims_filename() - with openmatrix.open_file(skims_filename) as f: + with h5py.File(skims_filename) as f: d0 = sh.dataset.from_omx_3d( f, index_names=("otaz", "dtaz", "time_period"), @@ -315,27 +316,23 @@ def test_from_parquet_3d_ignore(): assert "DIST" in skims.variables -def _write_compressed_omx(path, matrices, complib="zlib", complevel=7, shuffle=True): +def _write_compressed_omx(path, matrices, compression="gzip", compression_opts=7): """Write an OMX file with compressed, oddly-chunked matrix tables.""" - import tables - - filters = tables.Filters(complevel=complevel, complib=complib, shuffle=shuffle) n1, n2 = next(iter(matrices.values())).shape - with openmatrix.open_file(path, mode="w") as out: + with h5py.File(path, mode="w") as out: for name, arr in matrices.items(): - out.create_carray( - "/data", - name, - obj=arr, - filters=filters, + out.create_dataset( + f"data/{name}", + data=arr, + compression=compression, + compression_opts=compression_opts, + shuffle=True, # chunk shape that does not evenly divide the array, # to exercise edge-chunk handling - chunkshape=(7, 7), + chunks=(7, 7), ) - out.create_carray("/lookup", "taz", obj=np.arange(11, 11 + n1)) - shp = np.empty(2, dtype=int) - shp[0], shp[1] = n1, n2 - out.root._v_attrs.SHAPE = shp + out.create_dataset("lookup/taz", data=np.arange(11, 11 + n1)) + out.attrs["SHAPE"] = np.asarray([n1, n2], dtype=int) def _random_matrices(n=25, seed=42): @@ -348,12 +345,73 @@ def _random_matrices(n=25, seed=42): } +def test_parallel_omx_reader_converts_into_noncontiguous_output(tmp_path): + """Parallel chunk decoding casts directly into a final array view.""" + source = np.arange(625, dtype=np.float64).reshape(25, 25) / 7 + path = tmp_path / "direct-conversion.h5" + with h5py.File(path, "w") as handle: + handle.create_dataset( + "data", + data=source, + compression="gzip", + shuffle=True, + chunks=(7, 7), + ) + + backing = np.empty((25, 25, 2), dtype=np.float32) + destination = backing[..., 1] + assert not destination.flags.c_contiguous + with h5py.File(path, "r") as handle: + result = omx_reader.read_dataset(handle["data"], out=destination, max_workers=2) + + assert result is destination + np.testing.assert_array_equal(destination, source.astype(np.float32)) + + +def test_parallel_omx_reader_fallback_converts_dtype(tmp_path): + """The native h5py fallback also converts into the requested dtype.""" + source = np.arange(36, dtype=np.float64).reshape(6, 6) / 7 + path = tmp_path / "uncompressed.h5" + with h5py.File(path, "w") as handle: + handle.create_dataset("data", data=source) + + destination = np.empty(source.shape, dtype=np.float32) + with h5py.File(path, "r") as handle: + omx_reader.read_dataset(handle["data"], out=destination, max_workers=2) + np.testing.assert_array_equal(destination, source.astype(np.float32)) + + +def test_omx_shape_is_inferred_when_attribute_is_missing(tmp_path): + """Files written without SHAPE retain OpenMatrix-compatible behavior.""" + matrices = _random_matrices() + path = tmp_path / "missing-shape.omx" + _write_compressed_omx(path, matrices) + with h5py.File(path, "r+") as handle: + del handle.attrs["SHAPE"] + + two_dimensional = sh.dataset.from_omx(path, indexes="taz") + three_dimensional = sh.dataset.from_omx_3d( + path, + indexes="taz", + time_periods=["AM", "PM"], + load="eager", + ) + + np.testing.assert_array_equal(two_dimensional["DIST"], matrices["DIST"]) + np.testing.assert_array_equal( + three_dimensional["DIST"], matrices["DIST"].astype(np.float32) + ) + np.testing.assert_array_equal( + three_dimensional["TIME"].sel(time_period="PM"), matrices["TIME__PM"] + ) + + def test_from_omx_compressed_zlib(): matrices = _random_matrices() with tempfile.TemporaryDirectory() as tempdir: f = Path(tempdir).joinpath("skims.omx") _write_compressed_omx(f, matrices) - with openmatrix.open_file(f, mode="r") as back: + with h5py.File(f, mode="r") as back: ds = sh.dataset.from_omx(back, indexes="taz") ds_renamed = sh.dataset.from_omx( back, indexes="taz", renames={"distance": "DIST"} @@ -367,20 +425,55 @@ def test_from_omx_compressed_zlib(): assert sorted(ds_limited.data_vars) == ["COUNTS"] -def test_from_omx_compressed_h5py_handle(): - import h5py - +def test_from_omx_compressed_path(): matrices = _random_matrices() with tempfile.TemporaryDirectory() as tempdir: f = Path(tempdir).joinpath("skims.omx") _write_compressed_omx(f, matrices) - with h5py.File(f, mode="r") as back: - ds = sh.dataset.from_omx(back, indexes="taz") + ds = sh.dataset.from_omx(f, indexes="taz") for name, arr in matrices.items(): np.testing.assert_array_equal(ds[name].values, arr) assert ds.coords["otaz"].values == approx(np.arange(11, 36)) +def test_from_omx_filename_bearing_legacy_handle(): + """Legacy OMX handles are reopened by filename without being imported.""" + + class LegacyOMXHandle: + """Minimal protocol exposed by openmatrix.File and PyTables File.""" + + def __init__(self, filename): + self.filename = filename + self.close_called = False + + def close(self): + self.close_called = True + + matrices = _random_matrices() + with tempfile.TemporaryDirectory() as tempdir: + f = Path(tempdir).joinpath("skims.omx") + _write_compressed_omx(f, matrices) + legacy_handle = LegacyOMXHandle(f) + + expected_2d = sh.dataset.from_omx(f, indexes="taz") + actual_2d = sh.dataset.from_omx(legacy_handle, indexes="taz") + xr.testing.assert_equal(actual_2d, expected_2d) + + expected_3d = sh.dataset.from_omx_3d(f, time_periods=["AM", "PM"], load="eager") + actual_3d = sh.dataset.from_omx_3d( + [legacy_handle], time_periods=["AM", "PM"] + ).compute() + xr.testing.assert_equal(actual_3d, expected_3d) + + reloaded = xr.zeros_like(expected_3d) + sh.dataset.reload_from_omx_3d(reloaded, legacy_handle) + xr.testing.assert_equal(reloaded, expected_3d) + + # Sharrow owns only the temporary h5py handle it creates. The caller's + # compatibility handle remains open and under caller control. + assert not legacy_handle.close_called + + def test_from_omx_3d_compressed_zlib(): matrices = _random_matrices() with tempfile.TemporaryDirectory() as tempdir: @@ -402,7 +495,7 @@ def test_from_omx_3d_compressed_zlib(): assert skims.coords["otaz"].values == approx(np.arange(11, 36)) # also via an already-open file handle - with openmatrix.open_file(f, mode="r") as back: + with h5py.File(f, mode="r") as back: skims2 = sh.dataset.from_omx_3d( back, time_periods=["AM", "PM"], @@ -412,6 +505,232 @@ def test_from_omx_3d_compressed_zlib(): xr.testing.assert_equal(skims, skims2.compute()) +def test_from_omx_3d_loading_modes(): + """Lazy batching and direct eager loading produce identical skim arrays.""" + matrices = _random_matrices() + with tempfile.TemporaryDirectory() as tempdir: + f = Path(tempdir).joinpath("skims.omx") + _write_compressed_omx(f, matrices) + lazy_variable = sh.dataset.from_omx_3d( + f, + time_periods=["EA", "AM", "PM"], + task_granularity="variable", + ) + lazy_matrix = sh.dataset.from_omx_3d( + f, + time_periods=["EA", "AM", "PM"], + task_granularity="matrix", + ) + eager = sh.dataset.from_omx_3d(f, time_periods=["EA", "AM", "PM"], load="eager") + + # One task per logical variable substantially reduces scheduler work. + assert len(lazy_variable.__dask_graph__()) < len(lazy_matrix.__dask_graph__()) + xr.testing.assert_equal(lazy_variable.compute(), eager) + xr.testing.assert_equal(lazy_matrix.compute(), eager) + assert (eager["TIME"].sel(time_period="EA").values == 0).all() + assert eager["TIME"].data[..., 0].flags.c_contiguous + assert eager["DIST"].dtype == np.float32 + + +def test_from_omx_3d_shared_parallel(): + """Separate OMX files can load concurrently into final shared pages.""" + matrices = _random_matrices() + with tempfile.TemporaryDirectory() as tempdir: + first = Path(tempdir).joinpath("first.omx") + second = Path(tempdir).joinpath("second.omx") + _write_compressed_omx( + first, {name: data for name, data in matrices.items() if name != "TIME__PM"} + ) + _write_compressed_omx(second, {"TIME__PM": matrices["TIME__PM"]}) + expected = sh.dataset.from_omx_3d( + [first, second], time_periods=["EA", "AM", "PM"], load="eager" + ) + token = "parallel-skims-" + secrets.token_hex(5) + shared = sh.dataset.from_omx_3d( + [first, second], + time_periods=["EA", "AM", "PM"], + load="shared", + workers=2, + shared_memory_key=token, + ) + try: + xr.testing.assert_equal(shared, expected) + assert shared.shm.is_shared_memory + assert shared["TIME"].data[..., 1].flags.c_contiguous + finally: + shared.shm.release_shared_memory() + + +def test_from_omx_3d_memmap_low_memory(): + """The low-memory mode loads directly into a new disk-backed array.""" + matrices = _random_matrices() + with tempfile.TemporaryDirectory() as tempdir: + f = Path(tempdir).joinpath("skims.omx") + backing = Path(tempdir).joinpath("skims-memory.dat") + _write_compressed_omx(f, matrices) + expected = sh.dataset.from_omx_3d( + f, time_periods=["EA", "AM", "PM"], load="eager" + ) + mapped = sh.dataset.from_omx_3d( + f, + time_periods=["EA", "AM", "PM"], + load="memmap", + memory_path=backing, + workers=2, + ) + key = mapped.shm.shared_memory_key + try: + xr.testing.assert_equal(mapped, expected) + assert backing.exists() + assert Path(f"{backing}.meta.pkl").exists() + assert isinstance(mapped.shm._shared_memory_objs_[-1], np.memmap) + + with pytest.raises(FileExistsError): + sh.dataset.from_omx_3d( + f, + time_periods=["EA", "AM", "PM"], + load="memmap", + memory_path=backing, + ) + finally: + mapped.shm.release_shared_memory() + del mapped + gc.collect() + SharedMemDatasetAccessor.delete_shared_memory_files(key) + + assert not backing.exists() + assert not Path(f"{backing}.meta.pkl").exists() + + +def test_windows_parallel_backend_uses_threads(monkeypatch, tmp_path): + """The Windows backend avoids spawn and caps workers to active sources.""" + matrices = _random_matrices() + first = tmp_path / "first.omx" + second = tmp_path / "second.omx" + backing = tmp_path / "skims-memory.dat" + _write_compressed_omx( + first, {name: data for name, data in matrices.items() if name != "TIME__PM"} + ) + _write_compressed_omx(second, {"TIME__PM": matrices["TIME__PM"]}) + expected = sh.dataset.from_omx_3d( + [first, second], time_periods=["EA", "AM", "PM"], load="eager" + ) + + def forbid_process_pool(*args, **kwargs): + pytest.fail("the Windows OMX backend must not create a process pool") + + monkeypatch.setattr(sh.dataset, "_omx_uses_processes", lambda: False) + monkeypatch.setattr( + sh.dataset.concurrent.futures, "ProcessPoolExecutor", forbid_process_pool + ) + mapped = sh.dataset.from_omx_3d( + [first, second], + time_periods=["EA", "AM", "PM"], + load="memmap", + memory_path=backing, + workers=100, + ) + key = mapped.shm.shared_memory_key + try: + xr.testing.assert_equal(mapped, expected) + assert sh.dataset._omx_worker_count(100, 2) == 2 + finally: + mapped.shm.release_shared_memory() + del mapped + gc.collect() + SharedMemDatasetAccessor.delete_shared_memory_files(key) + + +def test_memmap_metadata_failure_removes_new_backing_file(monkeypatch, tmp_path): + """Partial memmap construction is transactional, including on Windows.""" + source = tmp_path / "skims.omx" + backing = tmp_path / "skims-memory.dat" + _write_compressed_omx(source, _random_matrices()) + + def fail_metadata(*args, **kwargs): + raise RuntimeError("injected metadata failure") + + monkeypatch.setattr(sh.shared_memory, "create_shared_list", fail_metadata) + with pytest.raises(RuntimeError, match="injected metadata failure"): + sh.dataset.from_omx_3d( + source, + time_periods=["EA", "AM", "PM"], + load="memmap", + memory_path=backing, + workers=1, + ) + + assert not backing.exists() + assert not Path(f"{backing}.meta.pkl").exists() + + +def test_from_omx_3d_memmap_failure_releases_before_delete(monkeypatch, tmp_path): + """A failed memmap load closes its mapping before deleting backing files.""" + source = tmp_path / "skims.omx" + backing = tmp_path / "skims-memory.dat" + _write_compressed_omx(source, _random_matrices()) + + def fail_load(*args, **kwargs): + raise RuntimeError("injected load failure") + + cleanup_steps = [] + original_release = SharedMemDatasetAccessor.release_shared_memory + original_delete = SharedMemDatasetAccessor.delete_shared_memory_files + + def record_release(accessor): + cleanup_steps.append("release") + original_release(accessor) + + def record_delete(key): + # Model Windows' requirement that a file cannot be unlinked while its + # memory mapping remains open. + assert cleanup_steps == ["release"] + cleanup_steps.append("delete") + original_delete(key) + + monkeypatch.setattr(sh.dataset, "_load_omx_assignments", fail_load) + monkeypatch.setattr( + SharedMemDatasetAccessor, "release_shared_memory", record_release + ) + monkeypatch.setattr( + SharedMemDatasetAccessor, + "delete_shared_memory_files", + staticmethod(record_delete), + ) + + with pytest.raises(RuntimeError, match="injected load failure"): + sh.dataset.from_omx_3d( + source, + time_periods=["EA", "AM", "PM"], + load="memmap", + memory_path=backing, + workers=1, + ) + + assert cleanup_steps == ["release", "delete"] + assert not backing.exists() + assert not Path(f"{backing}.meta.pkl").exists() + + +@pytest.mark.parametrize( + ("kwargs", "message"), + [ + ({"load": "invalid"}, "load must be"), + ({"task_granularity": "invalid"}, "task_granularity"), + ({"load": "shared", "workers": 0}, "positive integer"), + ({"load": "memmap"}, "memory_path is required"), + ({"load": "eager", "workers": 2}, "load='shared'"), + ], +) +def test_from_omx_3d_loading_mode_validation(kwargs, message): + with pytest.raises(ValueError, match=message): + sh.dataset.from_omx_3d( + sh.example_data.get_skims_filename(), + time_periods=["EA", "AM", "MD", "PM", "EV"], + **kwargs, + ) + + def test_reload_from_omx_3d_compressed(): matrices = _random_matrices() with tempfile.TemporaryDirectory() as tempdir: @@ -429,11 +748,66 @@ def test_reload_from_omx_3d_compressed(): sh.dataset.reload_from_omx_3d(blank2, [str(f)], ignore=["COUNTS"]) assert (blank2["COUNTS"].values == 0).all() np.testing.assert_array_equal(blank2["DIST"].values, expected["DIST"].values) + # An h5py handle is accepted directly and remains owned by the caller. + blank3 = xr.zeros_like(expected) + with h5py.File(f, "r") as handle: + sh.dataset.reload_from_omx_3d(blank3, handle) + assert handle.id.valid + xr.testing.assert_equal(blank3, expected) + + # The time-period dimension need not use Sharrow's default name. + custom_dims = xr.zeros_like(expected.rename(time_period="period")) + sh.dataset.reload_from_omx_3d(custom_dims, f) + xr.testing.assert_equal(custom_dims, expected.rename(time_period="period")) + + +def test_reload_from_omx_3d_shared_parallel(): + """Parallel reload fills only selected arrays and preserves source precedence.""" + matrices = _random_matrices() + with tempfile.TemporaryDirectory() as tempdir: + first = Path(tempdir).joinpath("first.omx") + second = Path(tempdir).joinpath("second.omx") + _write_compressed_omx( + first, {name: data for name, data in matrices.items() if name != "TIME__PM"} + ) + _write_compressed_omx( + second, + { + "DIST": matrices["DIST"] + 1, + "TIME__PM": matrices["TIME__PM"], + }, + ) + + expected = sh.dataset.from_omx_3d( + [first, second], time_periods=["AM", "PM"], load="eager" + ).drop_vars("COUNTS") + blank = xr.zeros_like(expected) + token = "parallel-reload-" + secrets.token_hex(5) + shared = blank.shm.to_shared_memory( + token, + mode="r+", + load=False, + array_order={ + name: "last-axis-first" + for name, variable in blank.data_vars.items() + if variable.ndim == 3 + }, + ) + try: + sh.dataset.reload_from_omx_3d(shared, [first, second], workers=2) + xr.testing.assert_equal(shared, expected) + assert shared["TIME"].data[..., 0].flags.c_contiguous + finally: + shared.shm.release_shared_memory() + + with pytest.raises(ValueError, match="shared-memory-backed"): + sh.dataset.reload_from_omx_3d(blank, [first, second], workers=2) + with pytest.raises(ValueError, match="positive integer"): + sh.dataset.reload_from_omx_3d(blank, [first, second], workers=0) def test_from_omx_compressed_blosc(): - import h5py - import hdf5plugin + hdf5plugin = pytest.importorskip("hdf5plugin") rng = np.random.default_rng(7) arr = rng.random((25, 25)) @@ -450,7 +824,20 @@ def test_from_omx_compressed_blosc(): out.attrs["SHAPE"] = np.asarray([25, 25], dtype=int) with h5py.File(f, mode="r") as back: ds = sh.dataset.from_omx(back, indexes="taz") + token = "blosc-skims-" + secrets.token_hex(5) + shared = sh.dataset.from_omx_3d( + f, + indexes="taz", + time_periods=["AM"], + load="shared", + workers=2, + shared_memory_key=token, + ) np.testing.assert_array_equal(ds["DIST"].values, arr) + try: + np.testing.assert_array_equal(shared["DIST"].values, arr.astype(np.float32)) + finally: + shared.shm.release_shared_memory() def test_from_omx_3d_to_zarr(): @@ -474,7 +861,7 @@ def test_from_omx_3d_writable_handle(): with tempfile.TemporaryDirectory() as tempdir: f = Path(tempdir).joinpath("skims.omx") _write_compressed_omx(f, matrices) - with openmatrix.open_file(f, mode="a") as back: + with h5py.File(f, mode="a") as back: skims = sh.dataset.from_omx_3d( back, time_periods=["AM", "PM"], max_float_precision=64 ) @@ -483,3 +870,319 @@ def test_from_omx_3d_writable_handle(): np.testing.assert_array_equal( computed["TIME"].sel(time_period="PM").values, matrices["TIME__PM"] ) + + +@pytest.mark.parametrize("load", ["lazy", "eager", "shared", "memmap"]) +@pytest.mark.parametrize("source_kind", ["path", "openmatrix", "h5py"]) +def test_openmatrix_loading_modes(tmp_path, load, source_kind): + """OpenMatrix-created files support all loaders and caller-owned handles.""" + import contextlib + + matrices = _random_matrices() + path = tmp_path / "openmatrix.omx" + with openmatrix.open_file(path, "w") as handle: + for name, values in matrices.items(): + handle.create_matrix(name, obj=values, chunks=(7, 7)) + handle.create_mapping("taz", np.arange(11, 36)) + kwargs = {"memory_path": tmp_path / "skims.dat"} if load == "memmap" else {} + opener = {"openmatrix": openmatrix.open_file, "h5py": h5py.File} + context = ( + opener[source_kind](path, "r") + if source_kind != "path" + else contextlib.nullcontext(path) + ) + with context as source: + two_d = sh.dataset.from_omx(source, indexes="taz") + np.testing.assert_array_equal(two_d["DIST"], matrices["DIST"]) + dataset = sh.dataset.from_omx_3d( + source, time_periods=["AM", "PM"], load=load, **kwargs + ) + if source_kind != "path": + assert source.id.valid + # Lazy arrays must still work after the caller closes the source. + key = dataset.shm.shared_memory_key if load in {"shared", "memmap"} else None + try: + dataset.load() + np.testing.assert_array_equal(dataset.coords["otaz"], np.arange(11, 36)) + np.testing.assert_array_equal( + dataset["DIST"], matrices["DIST"].astype("float32") + ) + for period in ["AM", "PM"]: + np.testing.assert_array_equal( + dataset["TIME"].sel(time_period=period), matrices[f"TIME__{period}"] + ) + # Reload also accepts the real OpenMatrix handle, without closing it. + dataset["DIST"].data[...] = 0 + with openmatrix.open_file(path, "r") as handle: + sh.dataset.reload_from_omx_3d(dataset, handle) + assert handle.id.valid + np.testing.assert_array_equal( + dataset["DIST"], matrices["DIST"].astype("float32") + ) + finally: + if key is not None: + dataset.shm.release_shared_memory() + del dataset + gc.collect() + SharedMemDatasetAccessor.delete_shared_memory_files(key) + + +def test_openmatrix_in_memory_handle(): + """Non-reopenable OpenMatrix files load directly and stay caller-owned.""" + with openmatrix.open_file( + "in-memory.omx", "w", driver="core", backing_store=False + ) as handle: + handle.create_matrix("DIST", obj=np.eye(3)) + handle.create_mapping("taz", [11, 22, 33]) + two_d = sh.dataset.from_omx(handle, indexes="taz") + three_d = sh.dataset.from_omx_3d(handle, time_periods=["AM"]) + assert handle.id.valid + np.testing.assert_array_equal(two_d["DIST"], np.eye(3)) + np.testing.assert_array_equal(three_d["DIST"], np.eye(3, dtype="float32")) + + +def test_memmap_release_keeps_retained_views_alive(tmp_path): + """Releasing a Dataset must not unmap storage still used by NumPy views.""" + import subprocess + import sys + + source = tmp_path / "skims.omx" + _write_compressed_omx(source, _random_matrices()) + code = """ +import gc +import sys +import weakref +import numpy as np +import sharrow as sh + +ds = sh.dataset.from_omx_3d( + sys.argv[1], time_periods=['AM', 'PM'], load='memmap', + memory_path=sys.argv[2], workers=1, +) +key = ds.shm.shared_memory_key +mapping = weakref.ref(ds.shm._shared_memory_objs_[0]) +view = ds['DIST'].values +expected = view.copy() +ds.shm.release_shared_memory() +ds.shm.release_shared_memory() +np.testing.assert_array_equal(ds['DIST'].values, expected) +np.testing.assert_array_equal(view, expected) +del ds +gc.collect() +np.testing.assert_array_equal(view, expected) +assert mapping() is not None +del view +gc.collect() +assert mapping() is None +sh.shared_memory.delete_shared_memory_files(key) +""" + subprocess.run( + [sys.executable, "-c", code, str(source), str(tmp_path / "mapped.dat")], + check=True, + capture_output=True, + text=True, + timeout=30, + ) + assert not (tmp_path / "mapped.dat").exists() + assert not (tmp_path / "mapped.dat.meta.pkl").exists() + + +@pytest.mark.parametrize("mode", ["r", "a", "w", "memory"]) +def test_reload_preserves_caller_handle(tmp_path, mode): + """Serial reload uses the live handle even when no backing file exists.""" + path = tmp_path / "source.omx" + values = np.arange(9, dtype=np.float64).reshape(3, 3) + if mode in {"r", "a"}: + with openmatrix.open_file(path, "w") as handle: + handle.create_matrix("DIST", obj=values) + options = {"driver": "core", "backing_store": False} if mode == "memory" else {} + with openmatrix.open_file( + path, "w" if mode == "memory" else mode, **options + ) as handle: + if mode in {"w", "memory"}: + handle.create_matrix("DIST", obj=values) + target = xr.Dataset({"DIST": (("o", "d"), np.zeros((3, 3), dtype="float32"))}) + sh.dataset.reload_from_omx_3d(target, handle, workers=1) + np.testing.assert_array_equal(target.DIST, values) + assert handle.id.valid + if mode == "memory": + assert not path.exists() + + +def test_lazy_decoder_restarts_after_fork(tmp_path): + """A child can decode OMX after inheriting a warmed parent thread pool.""" + import os + import subprocess + import sys + + if not hasattr(os, "fork"): + pytest.skip("fork is unavailable") + source = tmp_path / "source.omx" + _write_compressed_omx(source, _random_matrices()) + code = """ +import os +import signal +import sys +import numpy as np +import sharrow as sh + +expected = sh.dataset._fast_load_omx_array(sys.argv[1], 'DIST') +reader = sh.dataset._lazy_omx_reader +sh.dataset._lazy_omx_reader_lock.acquire() +pid = os.fork() +if pid == 0: + signal.alarm(10) + assert sh.dataset._lazy_omx_reader is None + assert sh.dataset._lazy_omx_reader_lock.acquire(blocking=False) + sh.dataset._lazy_omx_reader_lock.release() + actual = sh.dataset._fast_load_omx_array(sys.argv[1], 'DIST') + np.testing.assert_array_equal(actual, expected) + sh.dataset._shutdown_lazy_omx_reader() + os._exit(0) +sh.dataset._lazy_omx_reader_lock.release() +_, status = os.waitpid(pid, 0) +assert os.waitstatus_to_exitcode(status) == 0, status +assert sh.dataset._lazy_omx_reader is reader +np.testing.assert_array_equal( + sh.dataset._fast_load_omx_array(sys.argv[1], 'DIST'), expected +) +""" + subprocess.run( + [sys.executable, "-c", code, str(source)], + check=True, + capture_output=True, + text=True, + timeout=20, + ) + + +def test_default_lazy_selection_reads_only_selected_period(tmp_path, monkeypatch): + """A selective read must not decode unrequested time-period matrices.""" + path = tmp_path / "skims.omx" + matrices = _random_matrices() + _write_compressed_omx(path, matrices) + reads = [] + original = sh.dataset._read_omx_dataset + + def record_read(dataset, *args, **kwargs): + reads.append(dataset.name) + return original(dataset, *args, **kwargs) + + monkeypatch.setattr(sh.dataset, "_read_omx_dataset", record_read) + lazy = sh.dataset.from_omx_3d(path, time_periods=["AM", "PM"]) + actual = lazy.TIME.sel(time_period="AM").compute(scheduler="synchronous") + np.testing.assert_array_equal(actual, matrices["TIME__AM"]) + assert reads == ["/data/TIME__AM"] + assert lazy.TIME.chunks[-1] == (1, 1) + + +def test_reload_default_does_not_spawn(tmp_path, monkeypatch): + """Legacy reload calls remain in-process even with several shared sources.""" + paths = [tmp_path / "one.omx", tmp_path / "two.omx"] + for path in paths: + _write_compressed_omx(path, _random_matrices()) + expected = sh.dataset.from_omx_3d(paths, time_periods=["AM", "PM"], load="eager") + shared = xr.zeros_like(expected).shm.to_shared_memory( + "default-reload-" + secrets.token_hex(5), load=False + ) + + def forbid_pool(*args, **kwargs): + pytest.fail("default reload must not spawn processes") + + monkeypatch.setattr( + sh.dataset.concurrent.futures, "ProcessPoolExecutor", forbid_pool + ) + try: + sh.dataset.reload_from_omx_3d(shared, paths) + xr.testing.assert_equal(shared, expected) + finally: + shared.shm.release_shared_memory() + + +def test_parallel_file_workers_share_decoder_budget(tmp_path, monkeypatch): + """The file-worker boundary passes a divided CPU budget to each decoder.""" + paths = [tmp_path / "one.omx", tmp_path / "two.omx"] + for path in paths: + _write_compressed_omx(path, _random_matrices()) + expected = sh.dataset.from_omx_3d(paths, time_periods=["AM", "PM"], load="eager") + budgets = [] + original_pool = sh.dataset.concurrent.futures.ProcessPoolExecutor + + def recording_pool(*args, **kwargs): + pool = original_pool(*args, **kwargs) + submit = pool.submit + + def record_submit(fn, *args, **kwargs): + budgets.append(args[-1]) + return submit(fn, *args, **kwargs) + + pool.submit = record_submit + return pool + + monkeypatch.setattr(sh.dataset.os, "cpu_count", lambda: 8) + monkeypatch.setattr(sh.dataset, "_omx_uses_processes", lambda: True) + monkeypatch.setattr( + sh.dataset.concurrent.futures, "ProcessPoolExecutor", recording_pool + ) + shared = sh.dataset.from_omx_3d( + paths, time_periods=["AM", "PM"], load="shared", workers=2 + ) + try: + xr.testing.assert_equal(shared, expected) + # Only the second source has assignments when names overlap. Use the + # actual active worker count, not the requested upper limit. + assert budgets == [] + finally: + shared.shm.release_shared_memory() + + with openmatrix.open_file(paths[0], "w") as handle: + handle.create_matrix("DIST", obj=np.eye(25)) + with openmatrix.open_file(paths[1], "w") as handle: + handle.create_matrix("TIME__AM", obj=np.eye(25)) + shared = sh.dataset.from_omx_3d( + paths, time_periods=["AM"], load="shared", workers=2 + ) + try: + np.testing.assert_array_equal(shared.DIST, np.eye(25)) + np.testing.assert_array_equal(shared.TIME[..., 0], np.eye(25)) + assert budgets == [4, 4] + finally: + shared.shm.release_shared_memory() + + +@pytest.mark.parametrize("load", ["lazy", "eager", "shared", "memmap"]) +def test_omx_shape_error_identifies_source_and_exclusion(tmp_path, load): + """Reject inconsistent matrices before allocation, with actionable context.""" + first, second = tmp_path / "first.omx", tmp_path / "second.omx" + with openmatrix.open_file(first, "w") as handle: + handle.create_matrix("DIST", obj=np.eye(3)) + with openmatrix.open_file(second, "w") as handle: + handle.create_matrix("UNRELATED", obj=np.eye(4)) + options = {"memory_path": tmp_path / "mapped.dat"} if load == "memmap" else {} + with pytest.raises(ValueError) as error: + sh.dataset.from_omx_3d( + [first, second], time_periods=["AM"], load=load, **options + ) + message = str(error.value) + # Diagnostics quote paths with repr, which escapes Windows backslashes. + assert repr(str(first)) in message and repr(str(second)) in message + assert "UNRELATED" in message + assert "(4, 4), expected (3, 3)" in message + assert "SHAPE metadata" in message and "ignore" in message + assert not (tmp_path / "mapped.dat").exists() + # Exclusion is appropriate for an unrelated matrix, not bad root metadata. + filtered = sh.dataset.from_omx_3d( + [first, second], time_periods=["AM"], ignore="^UNRELATED$", load="eager" + ) + np.testing.assert_array_equal(filtered.DIST, np.eye(3)) + + +def test_omx_stale_shape_metadata_is_rejected(tmp_path): + """A named coordinate does not silently override inconsistent SHAPE metadata.""" + path = tmp_path / "stale.omx" + with openmatrix.open_file(path, "w") as handle: + handle.create_matrix("DIST", obj=np.eye(3)) + handle.create_mapping("taz", [11, 22, 33]) + handle.attrs["SHAPE"] = [4, 4] + with pytest.raises(ValueError, match=r"expected \(4, 4\).*SHAPE metadata"): + sh.dataset.from_omx_3d(path, time_periods=["AM"], indexes="taz") diff --git a/sharrow/tests/test_omx.py b/sharrow/tests/test_omx.py new file mode 100644 index 0000000..b3bf647 --- /dev/null +++ b/sharrow/tests/test_omx.py @@ -0,0 +1,177 @@ +import subprocess +import sys +from pathlib import Path + +import h5py +import numpy as np +import openmatrix +import pytest +import xarray as xr + +from sharrow.omx import split_omx +from sharrow.translate import omx_to_zarr + + +@pytest.fixture +def omx_file(tmp_path: Path) -> tuple[Path, dict[str, np.ndarray]]: + """Create a representative OMX-format HDF5 file for utility tests.""" + matrices = { + "DIST": np.arange(9, dtype=np.float32).reshape(3, 3), + "TIME__AM": np.arange(10, 19, dtype=np.float32).reshape(3, 3), + "TIME__PM": np.arange(20, 29, dtype=np.float32).reshape(3, 3), + } + path = tmp_path / "skims.omx" + with openmatrix.open_file(path, "w") as handle: + for name, values in matrices.items(): + handle.create_matrix(name, obj=values, chunks=(2, 2)) + handle.create_mapping("taz", [101, 102, 103]) + return path, matrices + + +def test_split_omx_with_global_lookups(omx_file, tmp_path): + """Matrix chunks retain datasets, compression, lookups, and OMX metadata.""" + source_path, matrices = omx_file + destination = tmp_path / "global-lookups" + + split_omx(source_path, destination, global_lookups=True, n_chunks=2) + + matrix_names = set() + for chunk_number in range(2): + with h5py.File(destination / f"skims-chunk{chunk_number}.omx") as handle: + assert tuple(handle.attrs["SHAPE"]) == (3, 3) + np.testing.assert_array_equal(handle["lookup/taz"], [101, 102, 103]) + for name, dataset in handle["data"].items(): + matrix_names.add(name) + assert dataset.compression == "gzip" + np.testing.assert_array_equal(dataset, matrices[name]) + assert matrix_names == set(matrices) + + +def test_gzip_omx_load_without_optional_filter_packages(omx_file): + """The base installation reads standard HDF5 filters without plugins.""" + source_path, _ = omx_file + code = """ +import builtins +import sys + +real_import = builtins.__import__ + + +def import_without_optional_filters(name, *args, **kwargs): + if name.split('.', 1)[0] in {'blosc2', 'hdf5plugin'}: + raise ImportError(f'blocked optional dependency: {name}') + return real_import(name, *args, **kwargs) + + +builtins.__import__ = import_without_optional_filters +import sharrow as sh +from sharrow import omx_reader + +assert omx_reader.blosc2 is None +assert omx_reader.hdf5plugin is None +dataset = sh.dataset.from_omx_3d( + sys.argv[1], time_periods=['AM', 'PM'], load='eager' +) +assert dataset['TIME'].shape == (3, 3, 2) +""" + subprocess.run( + [sys.executable, "-c", code, str(source_path)], + check=True, + capture_output=True, + text=True, + ) + + +def test_split_omx_with_separate_lookups(omx_file, tmp_path): + """The default split writes each lookup to a standalone OMX file.""" + source_path, matrices = omx_file + destination = tmp_path / "separate-lookups" + + split_omx(source_path, destination) + + for name, values in matrices.items(): + with h5py.File(destination / f"{name}.omx") as handle: + assert list(handle["data"]) == [name] + assert list(handle["lookup"]) == [] + np.testing.assert_array_equal(handle[f"data/{name}"], values) + with h5py.File(destination / "_taz.omx") as handle: + assert list(handle["data"]) == [] + np.testing.assert_array_equal(handle["lookup/taz"], [101, 102, 103]) + + +def test_split_omx_replaces_existing_matrix_outputs(omx_file, tmp_path): + """Rerunning a split removes stale matrices and previously copied lookups.""" + source_path, matrices = omx_file + destination = tmp_path / "rerun" + + split_omx(source_path, destination, global_lookups=True, n_chunks=2) + with h5py.File(destination / "skims-chunk0.omx", "a") as handle: + handle.create_dataset("data/STALE", data=np.zeros((3, 3))) + + split_omx(source_path, destination, global_lookups=False, n_chunks=2) + + for chunk_number in range(2): + expected_names = { + name for number, name in enumerate(matrices) if number % 2 == chunk_number + } + with h5py.File(destination / f"skims-chunk{chunk_number}.omx") as handle: + assert set(handle["data"]) == expected_names + assert list(handle["lookup"]) == [] + + +def test_omx_to_zarr(omx_file, tmp_path): + """OMX conversion reads HDF5 matrices into the expected Zarr dimensions.""" + source_path, matrices = omx_file + destination = tmp_path / "skims.zarr" + + omx_to_zarr(source_path, destination, time_periods=["AM", "PM"]) + + with xr.open_zarr(destination) as dataset: + np.testing.assert_array_equal(dataset["DIST"], matrices["DIST"]) + np.testing.assert_array_equal( + dataset["TIME"].sel(time_period="AM"), matrices["TIME__AM"] + ) + np.testing.assert_array_equal( + dataset["TIME"].sel(time_period="PM"), matrices["TIME__PM"] + ) + np.testing.assert_array_equal(dataset.coords["otaz"], [101, 102, 103]) + + +@pytest.mark.parametrize("precision", [None, 32]) +def test_omx_to_zarr_precision_and_storage_chunks(tmp_path, precision): + """Conversion preserves float64 by default and bounds spatial/period chunks.""" + import json + + source = tmp_path / "precise.omx" + destination = tmp_path / "precise.zarr" + values = np.full((300, 270), 1.0000000001, dtype="float64") + with openmatrix.open_file(source, "w") as handle: + handle.create_matrix("DIST", obj=values) + handle.create_matrix("TIME__AM", obj=values) + handle.create_matrix("TIME__PM", obj=values * 2) + handle.create_matrix( + "COUNTS", obj=np.full(values.shape, 2**54 + 1, dtype="int64") + ) + kwargs = {} if precision is None else {"max_float_precision": precision} + omx_to_zarr( + source, + destination, + time_periods=["AM", "PM"], + index_names=("origin", "destination", "period"), + **kwargs, + ) + dtype = "float64" if precision is None else "float32" + with xr.open_zarr(destination) as result: + assert result.DIST.dtype == np.dtype(dtype) + assert result.TIME.dtype == np.dtype(dtype) + np.testing.assert_array_equal(result.DIST, values.astype(dtype)) + np.testing.assert_array_equal( + result.TIME.sel(period="AM"), values.astype(dtype) + ) + np.testing.assert_array_equal( + result.TIME.sel(period="PM"), (values * 2).astype(dtype) + ) + np.testing.assert_array_equal(result.COUNTS, 2**54 + 1) + for name, chunks in [("DIST", [256, 256]), ("TIME", [256, 256, 1])]: + metadata = json.loads((destination / name / ".zarray").read_text()) + assert metadata["chunks"] == chunks diff --git a/sharrow/translate.py b/sharrow/translate.py index 5cf6e00..e159468 100644 --- a/sharrow/translate.py +++ b/sharrow/translate.py @@ -1,12 +1,6 @@ import logging -import numpy as np -import pandas as pd -import xarray as xr - -from sharrow.dataset import Dataset - -from .dataset import one_based, zero_based +from .dataset import from_omx_3d logger = logging.getLogger("sharrow.translate") @@ -19,78 +13,35 @@ def omx_to_zarr( *, time_periods=None, time_period_sep="__", + max_float_precision=64, ): - try: - from larch import OMX - except ImportError: - raise ImportError("larch is required to read OMX files") from None - - bucket = {} - - r1 = r2 = None - - for omx_filename in omx_filenames: - logger.info(f"reading metadata from {omx_filename}") - - omx = OMX(omx_filename) - omx_data = omx.data - omx_shape = omx.shape - omx_lookup = omx.lookup - - data_names = list(omx_data._v_children.keys()) - n1, n2 = omx_shape - if indexes is None: - # default reads mapping if only one lookup is included, otherwise one-based - if len(omx_lookup._v_children) == 1: - ranger = None - else: - ranger = one_based - elif indexes == "one-based": - ranger = one_based - elif indexes == "zero-based": - ranger = zero_based - elif indexes in set(omx_lookup._v_children): - ranger = None - else: - raise NotImplementedError( - "only one-based, zero-based, and named indexes are implemented" - ) - if ranger is not None: - r1 = ranger(n1) - r2 = ranger(n2) - else: - r1 = r2 = pd.Index(omx_lookup[indexes]) - - if time_periods is None: - raise ValueError("must give time periods explicitly") - - bucket.update({i: omx.data[i] for i in omx.data._v_children}) - - data_names = list(bucket.keys()) + """Convert OMX matrices to format-2 Zarr without implicit float64 loss. + + ``max_float_precision`` defaults to 64 for conversion; pass 32 explicitly + to reduce floating-point precision. Stored chunks contain at most 256 rows, + 256 columns, and one time period, so slice reads do not fetch whole variables. + Input decoding still operates on one source matrix at a time. + """ + logger.info(f"reading metadata from {omx_filenames}") + dataset = from_omx_3d( + omx_filenames, + index_names=index_names, + indexes=indexes, + time_periods=time_periods, + time_period_sep=time_period_sep, + max_float_precision=max_float_precision, + task_granularity="matrix", + ) + # Set storage chunks independently of the loader's Dask task granularity. + # Period-only chunking would still leave huge matrices as oversized chunks. + dataset = dataset.chunk( + { + dim: 1 if dim == index_names[2] else max(1, min(256, size)) + for dim, size in dataset.sizes.items() + } + ) logger.info(f"writing to {zarr_directory}") - for k in data_names: - logger.info(f" - {k}") - ds = Dataset() - if time_period_sep in k: - base_k, time_k = k.split(time_period_sep, 1) - ds[base_k] = xr.DataArray( - np.float32(0), - dims=index_names, - coords={ - index_names[0]: r1, - index_names[1]: r2, - index_names[2]: time_periods, - }, - ) - ds[base_k].loc[:, :, time_k] = bucket[k][:] - else: - ds[k] = xr.DataArray( - bucket[k][:], - dims=index_names[:2], - coords={ - index_names[0]: r1, - index_names[1]: r2, - }, - ) - ds.to_zarr(zarr_directory, mode="a", zarr_format=2) + for name in dataset.data_vars: + logger.info(f" - {name}") + dataset.to_zarr(zarr_directory, mode="a", zarr_format=2) diff --git a/uv.lock b/uv.lock index 8e2a2eb..b7b3a06 100644 --- a/uv.lock +++ b/uv.lock @@ -1405,16 +1405,16 @@ wheels = [ [[package]] name = "openmatrix" -version = "0.3.5.0" +version = "0.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "h5py" }, { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "numpy", version = "2.5.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "tables" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/39/a4/c890e3af726ab91e496017fb108f6335663d6936cf6a26b17e66776e1b4c/OpenMatrix-0.3.5.0.tar.gz", hash = "sha256:1c10155bbe65eb4f542c388813d8dad8f317dfe642fb4f616276c95e6654bdee", size = 11626, upload-time = "2019-12-21T17:52:58.945Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/8d/101f54e964bf4e4254ea56152eff38e1a7fcdf001ebdcf5fc7c59c59f46d/openmatrix-0.4.0.tar.gz", hash = "sha256:d3779f4549ae7a1aae42d52ac1c9e2e7e2aa1f8a01899f88f6c46b1c6eaa68b6", size = 20868, upload-time = "2026-10-29T22:12:08.247Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/05/e8/6a8227e97127a2a1db98f5a22ca565bfe942e89de0644837627920101213/OpenMatrix-0.3.5.0-py3-none-any.whl", hash = "sha256:eeeb1a628b5eeefb572361537df5ad6e4d12c3cda1233023c4c367815cab6b5f", size = 13003, upload-time = "2019-12-21T17:52:57.777Z" }, + { url = "https://files.pythonhosted.org/packages/ec/f3/d85fd647398569a8d910851da06769328872209c60e053cef5bf22af1939/openmatrix-0.4.0-py3-none-any.whl", hash = "sha256:19e1d23958d9df3f289b1f9a14b31dd86d1b81b1653983ad44ef5b4b93ef42ec", size = 19166, upload-time = "2026-10-29T22:12:07.145Z" }, ] [[package]] @@ -1607,15 +1607,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6d/18/83376915176eb058cb86470eb7396388a13350b09a8f233a79303bbcbc5b/pure_eval-0.2.4-py3-none-any.whl", hash = "sha256:96cae060a313cfaad51bb761278bfb0e62dc0248d9315a81173752dc546cd37a", size = 11893, upload-time = "2026-09-10T21:41:21.532Z" }, ] -[[package]] -name = "py-cpuinfo" -version = "9.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, -] - [[package]] name = "pyarrow" version = "25.0.1" @@ -2319,27 +2310,33 @@ wheels = [ name = "sharrow" source = { editable = "." } dependencies = [ - { name = "blosc2" }, { name = "dask" }, { name = "filelock" }, { name = "h5py" }, - { name = "hdf5plugin" }, { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "networkx", version = "3.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "numba" }, { name = "numexpr" }, { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "numpy", version = "2.5.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "openmatrix" }, { name = "pandas" }, { name = "pyarrow" }, { name = "xarray" }, ] +[package.optional-dependencies] +hdf5-plugins = [ + { name = "blosc2", marker = "platform_machine != 'ARM64' or sys_platform != 'win32'" }, + { name = "hdf5plugin", marker = "platform_machine != 'ARM64' or sys_platform != 'win32'" }, +] + [package.dev-dependencies] dev = [ + { name = "blosc2", marker = "platform_machine != 'ARM64' or sys_platform != 'win32'" }, + { name = "hdf5plugin", marker = "platform_machine != 'ARM64' or sys_platform != 'win32'" }, { name = "ipykernel" }, { name = "nbmake" }, - { name = "openmatrix" }, { name = "pytest" }, { name = "pytest-regressions" }, { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, @@ -2351,25 +2348,28 @@ dev = [ [package.metadata] requires-dist = [ - { name = "blosc2" }, + { name = "blosc2", marker = "(platform_machine != 'ARM64' and extra == 'hdf5-plugins') or (sys_platform != 'win32' and extra == 'hdf5-plugins')" }, { name = "dask" }, { name = "filelock" }, { name = "h5py" }, - { name = "hdf5plugin" }, + { name = "hdf5plugin", marker = "(platform_machine != 'ARM64' and extra == 'hdf5-plugins') or (sys_platform != 'win32' and extra == 'hdf5-plugins')" }, { name = "networkx" }, { name = "numba", specifier = ">=0.57" }, { name = "numexpr" }, { name = "numpy", specifier = ">=1.19" }, + { name = "openmatrix", specifier = ">=0.4.0" }, { name = "pandas", specifier = ">=1.2" }, { name = "pyarrow" }, { name = "xarray", specifier = ">=2025.3.1" }, ] +provides-extras = ["hdf5-plugins"] [package.metadata.requires-dev] dev = [ + { name = "blosc2", marker = "platform_machine != 'ARM64' or sys_platform != 'win32'" }, + { name = "hdf5plugin", marker = "platform_machine != 'ARM64' or sys_platform != 'win32'" }, { name = "ipykernel", specifier = ">=6.31.0" }, { name = "nbmake", specifier = ">=1.5.5" }, - { name = "openmatrix", specifier = ">=0.3.5.0" }, { name = "pytest", specifier = ">=8.4.2" }, { name = "pytest-regressions", specifier = ">=2.8.3" }, { name = "scipy", specifier = ">=1.13.1" }, @@ -2414,32 +2414,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521, upload-time = "2023-09-30T13:58:03.53Z" }, ] -[[package]] -name = "tables" -version = "3.11.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "blosc2" }, - { name = "numexpr" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "numpy", version = "2.5.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "packaging" }, - { name = "py-cpuinfo" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cc/a3/d213ebe7376d48055bd55a29cd9f99061afa0dcece608f94a5025d797b0a/tables-3.11.1.tar.gz", hash = "sha256:78abcf413091bc7c1e4e8c10fbbb438d1ac0b5a87436c5b972c3e8253871b6fb", size = 4790533, upload-time = "2026-03-01T11:43:36.036Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/bb/4a9cde6628563388db26fa86c64adb0f2475a757e72af0ec185fd520b72f/tables-3.11.1-cp311-abi3-macosx_10_9_x86_64.whl", hash = "sha256:eb30684c42a77bbecdef2b9c763c4372b0ddc9cc5bd8b2a2055f2042eee67217", size = 7045977, upload-time = "2026-03-01T11:42:48.605Z" }, - { url = "https://files.pythonhosted.org/packages/78/74/6568c8d3aabf9982ab89fe3e378afbd7aad4894bde4570991a3246169ef4/tables-3.11.1-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:f0367d2e3df0f10ea63ccf4279f3fe58e32ec481767320301a483e2b3cd83efc", size = 6264947, upload-time = "2026-03-01T11:42:53.192Z" }, - { url = "https://files.pythonhosted.org/packages/cc/a3/ec228901fca4c996306b17f5c60a4105144df0bbd07b3a4a816f91f37b4a/tables-3.11.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:56bf6fb9132ead989b7e76695d7613d6d08f071a8019038d6565ba90c66b9f3e", size = 6903733, upload-time = "2026-03-01T11:42:58.349Z" }, - { url = "https://files.pythonhosted.org/packages/99/29/c2dc674ea70fa9a4819417289a9c0d3e4780835beeed573eb66964cfb763/tables-3.11.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1e78fe190fdeb4afe430b79651bae2a4f341904eb85aa8dbafe5f1caee1c7f67", size = 7241357, upload-time = "2026-03-01T11:43:03.938Z" }, - { url = "https://files.pythonhosted.org/packages/60/b5/a59b62af4127790c618eb11c06c106706e07509a3fb9e346b2a3ffa74419/tables-3.11.1-cp311-abi3-win_amd64.whl", hash = "sha256:7fa6cb03f6fe55ae4f85e89ec5450e5c40cc4c52d8c3b60eb157a445c2219e89", size = 6526565, upload-time = "2026-03-01T11:43:08.58Z" }, - { url = "https://files.pythonhosted.org/packages/1e/ce/561c82496e7c8c15ebf19b53b12c0ef91b322a66869db762db9711102764/tables-3.11.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a4bbd95036a4d0cc5c86c1f87fbb490b4c53cd70982f1c01b3ed6dcb3085cbb9", size = 7111409, upload-time = "2026-03-01T11:43:13.424Z" }, - { url = "https://files.pythonhosted.org/packages/84/18/bac920aee8239b572c506459607c6dd8742bc6275a43d51d2dd6ae1a1541/tables-3.11.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e3cfe79484351f7216eb8f3767bfa1217bfd271b04428f79cfa7ef6d7491919d", size = 6380142, upload-time = "2026-03-01T11:43:17.213Z" }, - { url = "https://files.pythonhosted.org/packages/59/3c/f4a694aa744d2b14d536e172c28dd70c84445f4787083a82d6d44a39e39f/tables-3.11.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a9c35f87fcb6a48c79fbc4e3ab15ca8f6053c4ce13063d6ca2ec36cbb58f40f", size = 7014135, upload-time = "2026-03-01T11:43:22.359Z" }, - { url = "https://files.pythonhosted.org/packages/45/82/94d4320d6c0fe5bd55230eec90cd142d58cda37b7cce00a318ac2a6abd93/tables-3.11.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4cf3218b76ba78d156d6ee75c19fb757d50682f6c7b4905370441afbfc9d77f3", size = 7349293, upload-time = "2026-03-01T11:43:27.569Z" }, - { url = "https://files.pythonhosted.org/packages/f7/02/a0f61a602ce2f2be8cc2e6146cc51acdaa8a1bb9b823b3863e70d3e0505d/tables-3.11.1-cp314-cp314t-win_amd64.whl", hash = "sha256:a6f7a3b82dbf0ae0f30de635ca88bb42dd87938b0950369d0ee4289c52ae6de2", size = 6854713, upload-time = "2026-03-01T11:43:31.934Z" }, -] - [[package]] name = "threadpoolctl" version = "3.7.0"