From 7c9e6f6d4798e858c6fb595d2ca1f52a259ce933 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 7 Nov 2026 16:11:22 +0900 Subject: [PATCH 1/3] Windows parity: test_dev_engine_pin.py in WINDOWS_NOT_APPLICABLE, and a guard that the two exclusions agree (closes #307) Co-Authored-By: Claude Sonnet 5.5 --- tests/conftest.py | 3 +- tests/test_conftest_windows_table.py | 84 ++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 tests/test_conftest_windows_table.py diff --git a/tests/conftest.py b/tests/conftest.py index 129dbbf..1e479b7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -35,10 +35,11 @@ # the release preflight is a maintainer tool for a POSIX shell: its check (f) runs the workflows' bash step code, # and on Windows it reports a SKIP, which its own tests cover on Linux and macOS (ArcadeData/arcadedb-python#208) "test_release_preflight.py": "runs the workflows' bash step code", + # runs the bash code of the plan step of test-native.yml (scripts/dev_engine_pin.py, #304) + "test_dev_engine_pin.py": "runs the bash code of a workflow step", } if sys.platform == "win32": collect_ignore.extend(WINDOWS_NOT_APPLICABLE) - collect_ignore.append("test_dev_engine_pin.py") # runs the bash code of a workflow step if BACKEND != "native": # Java -> Python upcalls (Python functions in SQL) exist on the native build only collect_ignore.append("test_native_udf.py") diff --git a/tests/test_conftest_windows_table.py b/tests/test_conftest_windows_table.py new file mode 100644 index 0000000..694e737 --- /dev/null +++ b/tests/test_conftest_windows_table.py @@ -0,0 +1,84 @@ +"""The Windows exclusions of tests/conftest.py cannot diverge from the table that native/wheel/parity_wheel.py reads. + +conftest.py ignores a file on Windows with `collect_ignore`, but the native parity runner passes every test file to pytest by +name, and a file named on the command line is collected anyway. The runner therefore reads the table WINDOWS_NOT_APPLICABLE. +A file added to the `if sys.platform == "win32":` block only (test_dev_engine_pin.py once was) runs on the Windows parity leg +and fails there. Both files are parsed, not imported, as parity_wheel.py itself does. +""" + +import ast +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +CONFTEST = ROOT / "tests" / "conftest.py" +PARITY = ROOT / "native" / "wheel" / "parity_wheel.py" + + +def _tree(path): + return ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + + +def _table(tree): + for node in tree.body: + if ( + isinstance(node, ast.Assign) + and getattr(node.targets[0], "id", "") == "WINDOWS_NOT_APPLICABLE" + ): + return ast.literal_eval(node.value) + raise AssertionError("conftest.py has no WINDOWS_NOT_APPLICABLE table") + + +def _is_win32_test(test): + return isinstance(test, ast.Compare) and "win32" in [ + getattr(c, "value", None) for c in test.comparators + ] + + +def _win32_ignored(tree): + """The file names that the `if sys.platform == "win32":` block appends to collect_ignore.""" + names = [] + blocks = [n for n in tree.body if isinstance(n, ast.If) and _is_win32_test(n.test)] + assert blocks, "conftest.py has no `if sys.platform == 'win32'` block" + for block in blocks: + for node in ast.walk(block): + if ( + isinstance(node, ast.Call) + and getattr(node.func, "attr", "") == "append" + ): + names += [ + a.value + for a in node.args + if isinstance(a, ast.Constant) and isinstance(a.value, str) + ] + return names + + +def test_every_file_ignored_on_windows_is_in_the_table_the_parity_runner_reads(): + tree = _tree(CONFTEST) + table = _table(tree) + missing = [n for n in _win32_ignored(tree) if n not in table] + assert ( + not missing + ), f"{missing}: add them to WINDOWS_NOT_APPLICABLE, not only to collect_ignore (native/wheel/parity_wheel.py)" + + +def test_every_file_of_the_table_exists(): + table = _table(_tree(CONFTEST)) + assert all((ROOT / "tests" / name).is_file() for name in table), [ + n for n in table if not (ROOT / "tests" / n).is_file() + ] + + +def test_the_dev_engine_pin_test_is_in_the_table(): + assert "test_dev_engine_pin.py" in _table(_tree(CONFTEST)) + + +def test_the_parity_runner_still_reads_the_table_by_that_name_and_applies_it(): + src = PARITY.read_text(encoding="utf-8") + assert '"WINDOWS_NOT_APPLICABLE"' in src and "ast.literal_eval" in src + assert ( + "win_files" in src + and 'sys.platform == "win32"' + in src.replace("\n ", " ").replace("(", "").replace(")", "") + or "win_files" in src + ) From ce319c81285b7a5a2b804f1cadde6ef57a50ca6e Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 7 Nov 2026 16:11:39 +0900 Subject: [PATCH 2/3] Guard test: assert the parity runner's own filter line Co-Authored-By: Claude Sonnet 5.5 --- tests/test_conftest_windows_table.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/test_conftest_windows_table.py b/tests/test_conftest_windows_table.py index 694e737..89a21e1 100644 --- a/tests/test_conftest_windows_table.py +++ b/tests/test_conftest_windows_table.py @@ -77,8 +77,5 @@ def test_the_parity_runner_still_reads_the_table_by_that_name_and_applies_it(): src = PARITY.read_text(encoding="utf-8") assert '"WINDOWS_NOT_APPLICABLE"' in src and "ast.literal_eval" in src assert ( - "win_files" in src - and 'sys.platform == "win32"' - in src.replace("\n ", " ").replace("(", "").replace(")", "") - or "win_files" in src - ) + "not in win_files" in src + ), "the parity runner no longer filters its files by the Windows table" From 3fa3b7766da33ac494f46e560a8d641409f0ec6f Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 7 Nov 2026 16:13:06 +0900 Subject: [PATCH 3/3] arcadedb.ENGINE_SNAPSHOT: the native wheel's recorded value was overwritten by a later fallback; release workflows refuse a wheel that does not state its engine Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/release-jvm.yml | 7 +++ .github/workflows/release-native.yml | 7 +++ scripts/verify_wheel_engine_pin.py | 76 ++++++++++++++++++++++++++ src/arcadedb/__init__.py | 15 +++-- tests/test_engine_snapshot_recorded.py | 24 ++++++++ tests/test_verify_wheel_engine_pin.py | 63 +++++++++++++++++++++ 6 files changed, 188 insertions(+), 4 deletions(-) create mode 100644 scripts/verify_wheel_engine_pin.py create mode 100644 tests/test_engine_snapshot_recorded.py create mode 100644 tests/test_verify_wheel_engine_pin.py diff --git a/.github/workflows/release-jvm.yml b/.github/workflows/release-jvm.yml index 9b6bb24..84f8ed6 100644 --- a/.github/workflows/release-jvm.yml +++ b/.github/workflows/release-jvm.yml @@ -253,6 +253,13 @@ jobs: echo '```' } >> "$GITHUB_STEP_SUMMARY" + # A published wheel states what it embeds (scripts/verify_wheel_engine_pin.py): a .devN wheel records the pinned snapshot and image + # digest, a final or .postN wheel records none. + - name: Every wheel states the engine it embeds + env: + TAG: ${{ needs.validate-version.outputs.version }} + run: python3 -I scripts/verify_wheel_engine_pin.py "$TAG" dist/*.whl + - name: Upload the checked wheels uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: diff --git a/.github/workflows/release-native.yml b/.github/workflows/release-native.yml index 0e52fb4..a4fcf29 100644 --- a/.github/workflows/release-native.yml +++ b/.github/workflows/release-native.yml @@ -169,6 +169,13 @@ jobs: ls -l dist echo "ok=true" >> "$GITHUB_OUTPUT" + # A published wheel states what it embeds: a .devN wheel records the pinned snapshot and image digest (scripts/dev_engine_pin.json), + # a final or .postN wheel records none. The wheel that was built and tested is the one checked; a mismatch stops the release here. + - name: Every wheel states the engine it embeds + env: + VERSION: ${{ needs.validate-version.outputs.version }} + run: python3 -I scripts/verify_wheel_engine_pin.py "$VERSION" dist/*.whl + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: native-release-dist diff --git a/scripts/verify_wheel_engine_pin.py b/scripts/verify_wheel_engine_pin.py new file mode 100644 index 0000000..ec0b55f --- /dev/null +++ b/scripts/verify_wheel_engine_pin.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""A wheel states which engine snapshot it embeds, and a release publishes no wheel that does not. + + verify_wheel_engine_pin.py VERSION WHEEL... + +VERSION is the tag. For `A.B.C.devN` every wheel's recording (arcadedb/_build_info.py of the native build, arcadedb/_version.py of the +JVM build) must carry ENGINE_SNAPSHOT and ENGINE_IMAGE_DIGEST of scripts/dev_engine_pin.json. For a final or `.postN` version they must +be None, because such a wheel embeds its release engine. Exit 1 on any difference. Reads only. Tests: tests/test_verify_wheel_engine_pin.py. +""" + +from __future__ import annotations + +import ast +import re +import sys +import zipfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import dev_engine_pin # noqa: E402 + +RECORD = re.compile(r"^[^/]+/(_build_info|_version)\.py$") + + +def recorded(wheel: Path) -> dict[str, object]: + """ENGINE_SNAPSHOT and ENGINE_IMAGE_DIGEST as the wheel's recording file assigns them.""" + with zipfile.ZipFile(wheel) as zf: + names = [n for n in zf.namelist() if RECORD.match(n)] + found: dict[str, object] = {} + for name in names: + text = zf.read(name).decode("utf-8") + for key in ("ENGINE_SNAPSHOT", "ENGINE_IMAGE_DIGEST"): + m = re.search(rf"^{key} = (.+?)(?:\s+#.*)?\s*$", text, re.M) + if m: + found[key] = ast.literal_eval(m.group(1)) + return found + + +def problems(version: str, wheels: list[Path]) -> list[str]: + pin = dev_engine_pin.load_pin() + if dev_engine_pin.DEV_TAG.fullmatch(version): + bad = dev_engine_pin.tag_problem(version, pin) + if bad: + return [bad] + want = { + "ENGINE_SNAPSHOT": pin["snapshot"], + "ENGINE_IMAGE_DIGEST": pin["image_digest"], + } + else: + want = {"ENGINE_SNAPSHOT": None, "ENGINE_IMAGE_DIGEST": None} + out = [] + for w in wheels: + got = recorded(w) + if got != want: + out.append( + f"{w.name}: the wheel records {got or 'nothing'}, a {version} wheel must record {want}" + ) + if not wheels: + out.append("no wheels given") + return out + + +def main(argv: list[str]) -> int: + if len(argv) < 2: + print(__doc__) + return 2 + bad = problems(argv[0], [Path(a) for a in argv[1:]]) + for line in bad: + print(f"::error::{line}") + if not bad: + print(f"OK: {len(argv) - 1} wheels state the engine they embed ({argv[0]})") + return 1 if bad else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/src/arcadedb/__init__.py b/src/arcadedb/__init__.py index a3fbc0d..d208741 100644 --- a/src/arcadedb/__init__.py +++ b/src/arcadedb/__init__.py @@ -96,11 +96,18 @@ to_python_array, ) +# The native wheel records them in _build_info.py (and build_wheel.py imports them near the top of this file), the JVM wheel in +# _version.py. Look in _build_info first: a fallback that ran last and set None hid the native wheel's value (26.12.1.dev0). try: - from ._version import ENGINE_IMAGE_DIGEST, ENGINE_SNAPSHOT -except ImportError: # a source checkout without a generated _version.py, or an older build - ENGINE_SNAPSHOT = None - ENGINE_IMAGE_DIGEST = None + from ._build_info import ENGINE_IMAGE_DIGEST, ENGINE_SNAPSHOT +except ImportError: + try: + from ._version import ENGINE_IMAGE_DIGEST, ENGINE_SNAPSHOT + except ( + ImportError + ): # a source checkout without a generated _version.py, or an older build + ENGINE_SNAPSHOT = None + ENGINE_IMAGE_DIGEST = None __all__ = [ "__version__", diff --git a/tests/test_engine_snapshot_recorded.py b/tests/test_engine_snapshot_recorded.py new file mode 100644 index 0000000..4b65dbd --- /dev/null +++ b/tests/test_engine_snapshot_recorded.py @@ -0,0 +1,24 @@ +"""The package attributes say what the build recorded (the 26.12.1.dev0 native wheel recorded the snapshot but printed None). + +native/wheel/build_wheel.py inserts its `from ._build_info import ...` near the top of arcadedb/__init__.py; a later fallback +in the same file once overwrote it. The attribute must be the recorded value on both builds. +""" + +import importlib + +import arcadedb + + +def test_the_package_attributes_are_what_the_build_recorded(): + module = ( + "arcadedb._build_info" + if arcadedb.__backend__ == "native" + else "arcadedb._version" + ) + try: + rec = importlib.import_module(module) + except ImportError: + assert arcadedb.ENGINE_SNAPSHOT is None and arcadedb.ENGINE_IMAGE_DIGEST is None + return + assert arcadedb.ENGINE_SNAPSHOT == getattr(rec, "ENGINE_SNAPSHOT", None) + assert arcadedb.ENGINE_IMAGE_DIGEST == getattr(rec, "ENGINE_IMAGE_DIGEST", None) diff --git a/tests/test_verify_wheel_engine_pin.py b/tests/test_verify_wheel_engine_pin.py new file mode 100644 index 0000000..e7b5138 --- /dev/null +++ b/tests/test_verify_wheel_engine_pin.py @@ -0,0 +1,63 @@ +"""scripts/verify_wheel_engine_pin.py: a dev wheel must state the pinned snapshot, a final wheel must not state one.""" + +import sys +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "scripts")) +import dev_engine_pin # noqa: E402 +import verify_wheel_engine_pin as v # noqa: E402 + +PIN = dev_engine_pin.load_pin() + + +def _wheel(tmp_path, record, name="_build_info.py", text=None): + path = tmp_path / "arcadedb-26.12.1.dev0-py3-none-any.whl" + body = text + if body is None: + body = f'ENGINE_VERSION = "x"\nENGINE_SNAPSHOT = {record[0]!r} # note\nENGINE_IMAGE_DIGEST = {record[1]!r} # note\n' + with zipfile.ZipFile(path, "w") as zf: + zf.writestr(f"arcadedb/{name}", body) + return path + + +def test_a_dev_wheel_with_the_pinned_snapshot_passes(tmp_path): + w = _wheel(tmp_path, (PIN["snapshot"], PIN["image_digest"])) + assert v.problems("26.12.1.dev0", [w]) == [] + + +def test_a_dev_wheel_of_the_jvm_build_records_in_version_py(tmp_path): + w = _wheel(tmp_path, (PIN["snapshot"], PIN["image_digest"]), name="_version.py") + assert v.problems("26.12.1.dev1", [w]) == [] + + +def test_a_dev_wheel_that_records_none_fails(tmp_path): + w = _wheel(tmp_path, (None, None)) + assert len(v.problems("26.12.1.dev0", [w])) == 1 + + +def test_a_dev_wheel_with_another_snapshot_fails(tmp_path): + w = _wheel(tmp_path, ("26.12.1-20261201.000000-1", PIN["image_digest"])) + assert len(v.problems("26.12.1.dev0", [w])) == 1 + + +def test_a_dev_wheel_without_a_recording_fails(tmp_path): + w = _wheel(tmp_path, None, text="ENGINE_VERSION = 'x'\n") + assert "records nothing" in v.problems("26.12.1.dev0", [w])[0] + + +def test_a_final_wheel_must_not_state_a_snapshot(tmp_path): + assert v.problems("26.12.1", [_wheel(tmp_path, (None, None))]) == [] + assert ( + len( + v.problems( + "26.12.1", [_wheel(tmp_path, (PIN["snapshot"], PIN["image_digest"]))] + ) + ) + == 1 + ) + + +def test_no_wheels_fail(): + assert v.problems("26.12.1.dev0", []) == ["no wheels given"]