From c53e90100a7949f202121e599facf4d053e1c4cf Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:44:35 +0400 Subject: [PATCH 01/25] Test that the Stage 12 stores use the service library's run-table models (failing) The Stage 12 library will import its models of `report_runs`, `simulation_runs`, `budget_window_runs` and `household_jobs` from `policyengine-simulation-persistence`, whose models the CI comparison checks against the Public API's tables. It now depends on that library, so the entry's and executor's locks follow. The service library's models must store a Python `None` in `simulation_runs.household_result`, `household_jobs.result` and `report_runs.spm_provenance` as SQL NULL, as the Stage 12 models do: the Modal worker writes `None` to them. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_tables.py | 15 ++++++++ .../pyproject.toml | 4 +++ .../tests/test_shared_table_models.py | 35 +++++++++++++++++++ libs/policyengine-stage12-persistence/uv.lock | 22 ++++++++++++ .../policyengine-simulation-entry/uv.lock | 2 ++ .../policyengine-simulation-executor/uv.lock | 22 ++++++++++++ 6 files changed, 100 insertions(+) create mode 100644 libs/policyengine-stage12-persistence/tests/test_shared_table_models.py diff --git a/libs/policyengine-simulation-persistence/tests/test_tables.py b/libs/policyengine-simulation-persistence/tests/test_tables.py index 2d7ade37f..651dfb00a 100644 --- a/libs/policyengine-simulation-persistence/tests/test_tables.py +++ b/libs/policyengine-simulation-persistence/tests/test_tables.py @@ -182,6 +182,21 @@ def test_a_budget_window_has_one_report_run_per_year_of_one_report() -> None: ) +def test_a_python_none_in_a_runs_json_result_is_stored_as_sql_null() -> None: + """The Modal worker role's ``None`` results are SQL ``NULL``, not JSON ``null``. + + A household job failed outside its calculation has no result, and a report + run whose output has no SPM receipts has no ``spm_provenance``; the stores + write ``None`` for both, and a query for ``IS NULL`` must find them. + """ + for result_column in ( + simulation_runs.c.household_result, + household_jobs.c.result, + report_runs.c.spm_provenance, + ): + assert result_column.type.none_as_null is True, result_column + + def test_the_input_models_are_tables_the_service_reads() -> None: """Each input table is one the service's role may SELECT and never writes.""" readable_only = { diff --git a/libs/policyengine-stage12-persistence/pyproject.toml b/libs/policyengine-stage12-persistence/pyproject.toml index 87587670c..261eaa635 100644 --- a/libs/policyengine-stage12-persistence/pyproject.toml +++ b/libs/policyengine-stage12-persistence/pyproject.toml @@ -9,6 +9,9 @@ license = {file = "../../LICENSE"} requires-python = ">=3.13,<3.14" dependencies = [ "policyengine-simulation-contract", + # The models of the Public API's run and result tables, which the CI + # comparison checks against the tables the Public API's chain builds. + "policyengine-simulation-persistence", "psycopg[binary]>=3.2,<4", "sqlalchemy>=2,<3", ] @@ -34,6 +37,7 @@ packages = ["src/policyengine_stage12_persistence"] [tool.uv.sources] policyengine-simulation-contract = { path = "../policyengine-simulation-contract", editable = true } +policyengine-simulation-persistence = { path = "../policyengine-simulation-persistence", editable = true } [tool.pytest.ini_options] pythonpath = ["src"] diff --git a/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py b/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py new file mode 100644 index 000000000..54e3331ec --- /dev/null +++ b/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py @@ -0,0 +1,35 @@ +"""The Modal worker role's stores use the simulation service's table models. + +``policyengine-simulation-persistence`` holds the one set of models of the +Public API's run tables, which the CI comparison checks against the tables the +Public API's migration chain builds. This library keeps only the comparison +path's own tables (``tables.py``) until that path is removed. +""" + +from __future__ import annotations + +from policyengine_simulation_persistence import tables as shared_tables +from policyengine_stage12_persistence import ( + budget_window_store, + household_report_run_store, + report_run_store, +) +from policyengine_stage12_persistence.report_run_store import RUN_TABLES, RunKind + + +def test_the_run_stores_write_the_shared_run_table_models() -> None: + assert RUN_TABLES[RunKind.REPORT_RUN] is shared_tables.report_runs + assert RUN_TABLES[RunKind.SIMULATION_RUN] is shared_tables.simulation_runs + assert RUN_TABLES[RunKind.HOUSEHOLD_JOB] is shared_tables.household_jobs + assert report_run_store.budget_window_runs is shared_tables.budget_window_runs + + +def test_the_budget_window_store_writes_the_shared_run_table_models() -> None: + assert budget_window_store.budget_window_runs is shared_tables.budget_window_runs + assert budget_window_store.report_runs is shared_tables.report_runs + assert budget_window_store.simulation_runs is shared_tables.simulation_runs + + +def test_the_household_report_store_writes_the_shared_run_table_models() -> None: + assert household_report_run_store.report_runs is shared_tables.report_runs + assert household_report_run_store.simulation_runs is shared_tables.simulation_runs diff --git a/libs/policyengine-stage12-persistence/uv.lock b/libs/policyengine-stage12-persistence/uv.lock index a1d9b1fab..e0607ad62 100644 --- a/libs/policyengine-stage12-persistence/uv.lock +++ b/libs/policyengine-stage12-persistence/uv.lock @@ -636,12 +636,33 @@ requires-dist = [ ] provides-extras = ["test", "build"] +[[package]] +name = "policyengine-simulation-persistence" +version = "0.1.0" +source = { editable = "../policyengine-simulation-persistence" } +dependencies = [ + { name = "psycopg", extra = ["binary"] }, + { name = "sqlalchemy" }, +] + +[package.metadata] +requires-dist = [ + { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" }, + { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, + { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, + { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=6.1.1" }, + { name = "pyyaml", marker = "extra == 'test'", specifier = ">=6.0.2" }, + { name = "sqlalchemy", specifier = ">=2,<2.1" }, +] +provides-extras = ["test", "build"] + [[package]] name = "policyengine-stage12-persistence" version = "0.1.0" source = { editable = "." } dependencies = [ { name = "policyengine-simulation-contract" }, + { name = "policyengine-simulation-persistence" }, { name = "psycopg", extra = ["binary"] }, { name = "sqlalchemy" }, ] @@ -660,6 +681,7 @@ test = [ requires-dist = [ { name = "black", marker = "extra == 'build'", specifier = ">=25.1.0" }, { name = "policyengine-simulation-contract", editable = "../policyengine-simulation-contract" }, + { name = "policyengine-simulation-persistence", editable = "../policyengine-simulation-persistence" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" }, { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, diff --git a/projects/policyengine-simulation-entry/uv.lock b/projects/policyengine-simulation-entry/uv.lock index 7bef95438..f5cb9787a 100644 --- a/projects/policyengine-simulation-entry/uv.lock +++ b/projects/policyengine-simulation-entry/uv.lock @@ -991,6 +991,7 @@ version = "0.1.0" source = { editable = "../../libs/policyengine-stage12-persistence" } dependencies = [ { name = "policyengine-simulation-contract" }, + { name = "policyengine-simulation-persistence" }, { name = "psycopg", extra = ["binary"] }, { name = "sqlalchemy" }, ] @@ -999,6 +1000,7 @@ dependencies = [ requires-dist = [ { name = "black", marker = "extra == 'build'", specifier = ">=25.1.0" }, { name = "policyengine-simulation-contract", editable = "../../libs/policyengine-simulation-contract" }, + { name = "policyengine-simulation-persistence", editable = "../../libs/policyengine-simulation-persistence" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" }, { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, diff --git a/projects/policyengine-simulation-executor/uv.lock b/projects/policyengine-simulation-executor/uv.lock index b46aa9520..dbb6996da 100644 --- a/projects/policyengine-simulation-executor/uv.lock +++ b/projects/policyengine-simulation-executor/uv.lock @@ -2049,12 +2049,33 @@ requires-dist = [ ] provides-extras = ["test", "build"] +[[package]] +name = "policyengine-simulation-persistence" +version = "0.1.0" +source = { editable = "../../libs/policyengine-simulation-persistence" } +dependencies = [ + { name = "psycopg", extra = ["binary"] }, + { name = "sqlalchemy" }, +] + +[package.metadata] +requires-dist = [ + { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" }, + { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, + { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, + { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=6.1.1" }, + { name = "pyyaml", marker = "extra == 'test'", specifier = ">=6.0.2" }, + { name = "sqlalchemy", specifier = ">=2,<2.1" }, +] +provides-extras = ["test", "build"] + [[package]] name = "policyengine-stage12-persistence" version = "0.1.0" source = { editable = "../../libs/policyengine-stage12-persistence" } dependencies = [ { name = "policyengine-simulation-contract" }, + { name = "policyengine-simulation-persistence" }, { name = "psycopg", extra = ["binary"] }, { name = "sqlalchemy" }, ] @@ -2063,6 +2084,7 @@ dependencies = [ requires-dist = [ { name = "black", marker = "extra == 'build'", specifier = ">=25.1.0" }, { name = "policyengine-simulation-contract", editable = "../../libs/policyengine-simulation-contract" }, + { name = "policyengine-simulation-persistence", editable = "../../libs/policyengine-simulation-persistence" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" }, { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, From 8921122b7a780d8dd26f611a04d430d13df49353 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:46:09 +0400 Subject: [PATCH 02/25] Make the Stage 12 stores use the service library's run-table models `ReportRunStore`, `BudgetWindowStore` and `HouseholdReportRunStore` now import `report_runs`, `simulation_runs`, `budget_window_runs` and `household_jobs` from `policyengine_simulation_persistence.tables`, the complete models the CI comparison checks, and `report_run_tables.py` is gone. Every statement names the same columns with the same casts and parameters as before; only the order of an UPDATE's SET columns follows the complete models' column order. The complete models store a Python `None` in `household_result`, `result` and `spm_provenance` as SQL NULL, as the Stage 12 models did. The service only reads those columns. The result-table models keep their own metadata in `report_run_results.py` until they move too. Co-Authored-By: Claude Opus 5.5 --- .../tables.py | 9 +- .../budget_window_store.py | 10 +- .../household_report_run_store.py | 8 +- .../report_run_results.py | 5 +- .../report_run_store.py | 8 +- .../report_run_tables.py | 118 ------------------ .../tests/test_report_run_store.py | 2 +- 7 files changed, 22 insertions(+), 138 deletions(-) delete mode 100644 libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_tables.py diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/tables.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/tables.py index ac6caf05f..c1128c88b 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/tables.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/tables.py @@ -366,7 +366,10 @@ def _public_api_key( Column("error_message", String, nullable=True), _timestamp("started_at"), _timestamp("completed_at"), - Column("household_result", JSON, nullable=True), + # The Modal worker writes None as SQL NULL here and in the two columns + # below (household_jobs.result, report_runs.spm_provenance), so a query + # for IS NULL finds an empty result. + Column("household_result", JSON(none_as_null=True), nullable=True), Column("output_uri", String(OUTPUT_URI_LENGTH), nullable=True), Column("output_sha256", String(SHA256_LENGTH), nullable=True), PrimaryKeyConstraint("id", name="pk_simulation_runs"), @@ -509,7 +512,7 @@ def _public_api_key( Column("baseline_cliff_share", Float, nullable=True), Column("reform_cliff_gap", Float, nullable=True), Column("reform_cliff_share", Float, nullable=True), - Column("spm_provenance", JSONB, nullable=True), + Column("spm_provenance", JSONB(none_as_null=True), nullable=True), Column("data_version", String(VERSION_LENGTH), nullable=True), PrimaryKeyConstraint("id", name="pk_report_runs"), ForeignKeyConstraint( @@ -595,7 +598,7 @@ def _public_api_key( nullable=False, ), Column("error_message", String, nullable=True), - Column("result", JSON, nullable=True), + Column("result", JSON(none_as_null=True), nullable=True), _timestamp("started_at"), _timestamp("completed_at"), _timestamp("updated_at", server_now=True), diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/budget_window_store.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/budget_window_store.py index 4a025babc..3ff656684 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/budget_window_store.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/budget_window_store.py @@ -28,6 +28,11 @@ from typing import TYPE_CHECKING, Protocol from policyengine_simulation_contract.stage12_execution import ReportRunIds +from policyengine_simulation_persistence.tables import ( + budget_window_runs, + report_runs, + simulation_runs, +) from sqlalchemy import and_, exists, func, or_, select, update from policyengine_stage12_persistence.report_run_store import ( @@ -35,11 +40,6 @@ RunStatus, no_active_report_run_names, ) -from policyengine_stage12_persistence.report_run_tables import ( - budget_window_runs, - report_runs, - simulation_runs, -) from policyengine_stage12_persistence.store import ( ConnectionOpener, create_stage12_engine, diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/household_report_run_store.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/household_report_run_store.py index 1526f1b57..b3515e167 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/household_report_run_store.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/household_report_run_store.py @@ -20,6 +20,10 @@ from typing import TYPE_CHECKING, Any +from policyengine_simulation_persistence.tables import ( + report_runs, + simulation_runs, +) from sqlalchemy import select, update from policyengine_stage12_persistence.report_run_store import ( @@ -27,10 +31,6 @@ RunStatus, running_under_call, ) -from policyengine_stage12_persistence.report_run_tables import ( - report_runs, - simulation_runs, -) if TYPE_CHECKING: from collections.abc import Mapping diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_results.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_results.py index ed3b61c78..f71ef4cbd 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_results.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_results.py @@ -15,11 +15,9 @@ from uuid import uuid4 from policyengine_simulation_contract.macro_output import SingleYearMacroOutput -from sqlalchemy import Boolean, Column, Float, Integer, String, Table +from sqlalchemy import Boolean, Column, Float, Integer, MetaData, String, Table from sqlalchemy.dialects.postgresql import ENUM, UUID -from policyengine_stage12_persistence.report_run_tables import metadata - if TYPE_CHECKING: from collections.abc import Mapping @@ -37,6 +35,7 @@ ) from policyengine_simulation_contract.stage12_execution import ReportRunIds +metadata = MetaData(schema="public") decile_type_type = ENUM("income", "wealth", name="v2_decile_type", create_type=False) uk_area_type_type = ENUM( "constituency", "local_authority", name="v2_uk_area_type", create_type=False diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py index 520a32544..8dc6548d5 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py @@ -16,15 +16,15 @@ from enum import StrEnum from typing import TYPE_CHECKING, Any -from sqlalchemy import and_, case, exists, func, insert, or_, select, true, update -from sqlalchemy.dialects.postgresql import insert as postgresql_insert - -from policyengine_stage12_persistence.report_run_tables import ( +from policyengine_simulation_persistence.tables import ( budget_window_runs, household_jobs, report_runs, simulation_runs, ) +from sqlalchemy import and_, case, exists, func, insert, or_, select, true, update +from sqlalchemy.dialects.postgresql import insert as postgresql_insert + from policyengine_stage12_persistence.store import ( ConnectionOpener, create_stage12_engine, diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_tables.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_tables.py deleted file mode 100644 index 68c574b40..000000000 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_tables.py +++ /dev/null @@ -1,118 +0,0 @@ -"""SQLAlchemy mappings of the Public API's run tables that v2 Modal calls update. - -``policyengine-api`` owns these tables and every schema migration. The -mappings declare only the columns the coordinator, the workers, the household -functions, the budget-window scheduling and the stuck-run sweep read or write -as the Modal worker role, so they can issue typed DML; runtime code never -emits DDL. -""" - -from __future__ import annotations - -from sqlalchemy import JSON, Column, DateTime, Float, Integer, MetaData, String, Table -from sqlalchemy.dialects.postgresql import ENUM, JSONB, UUID - -RUN_STATUS_VALUES = ("pending", "running", "succeeded", "failed") - -report_run_status_type = ENUM( - *RUN_STATUS_VALUES, - name="v2_report_run_status", - create_type=False, -) -simulation_run_status_type = ENUM( - *RUN_STATUS_VALUES, - name="v2_simulation_status", - create_type=False, -) -household_job_status_type = ENUM( - *RUN_STATUS_VALUES, - name="v2_household_job_status", - create_type=False, -) - -metadata = MetaData(schema="public") - -# A report run compares one pair of simulation runs; a report with only a -# baseline has no reform run. The cliff impact, SPM receipts and data version -# are the report run's own results. A budget window's years are report runs -# naming its budget-window run and their year. -report_runs = Table( - "report_runs", - metadata, - Column("id", UUID(as_uuid=True), primary_key=True), - Column("created_at", DateTime(timezone=True), nullable=False), - Column("baseline_simulation_run_id", UUID(as_uuid=True), nullable=False), - Column("reform_simulation_run_id", UUID(as_uuid=True)), - Column("budget_window_run_id", UUID(as_uuid=True)), - Column("year", Integer), - Column("status", report_run_status_type, nullable=False), - Column("modal_call_id", String(255)), - Column("error_message", String), - Column("started_at", DateTime(timezone=True)), - Column("completed_at", DateTime(timezone=True)), - Column("updated_at", DateTime(timezone=True), nullable=False), - Column("baseline_cliff_gap", Float), - Column("baseline_cliff_share", Float), - Column("reform_cliff_gap", Float), - Column("reform_cliff_share", Float), - Column("spm_provenance", JSONB(none_as_null=True)), - Column("data_version", String(128)), -) - -# One calculation of a budget-window report: its years are the report runs -# naming it. It has no Modal call of its own; the coordinators of its years -# mark it running, succeeded or failed. -budget_window_runs = Table( - "budget_window_runs", - metadata, - Column("id", UUID(as_uuid=True), primary_key=True), - Column("created_at", DateTime(timezone=True), nullable=False), - Column("status", report_run_status_type, nullable=False), - Column("error_message", String), - Column("started_at", DateTime(timezone=True)), - Column("completed_at", DateTime(timezone=True)), - Column("updated_at", DateTime(timezone=True), nullable=False), -) - -# A simulation run's year, versions, dataset and region say what it -# calculated; the coordinator checks them on a succeeded run that a report -# run reuses. -simulation_runs = Table( - "simulation_runs", - metadata, - Column("id", UUID(as_uuid=True), primary_key=True), - Column("created_at", DateTime(timezone=True), nullable=False), - Column("year", Integer, nullable=False), - Column("country_package_version", String(128), nullable=False), - Column("policyengine_version", String(128), nullable=False), - Column("dataset_id", UUID(as_uuid=True)), - Column("region_id", UUID(as_uuid=True)), - Column("status", simulation_run_status_type, nullable=False), - Column("modal_call_id", String(255)), - Column("error_message", String), - Column("started_at", DateTime(timezone=True)), - Column("completed_at", DateTime(timezone=True)), - Column("updated_at", DateTime(timezone=True), nullable=False), - Column("output_uri", String(2048)), - Column("output_sha256", String(64)), - # A household simulation run's ``HouseholdExecutionResult`` or - # ``HouseholdExecutionFailure`` document, set by the household function. - Column("household_result", JSON(none_as_null=True)), -) - -# A household job is one household calculation, which a household function -# claims under its Modal call. ``result`` holds the calculation's -# ``HouseholdExecutionResult`` or ``HouseholdExecutionFailure`` document. -household_jobs = Table( - "household_jobs", - metadata, - Column("id", UUID(as_uuid=True), primary_key=True), - Column("created_at", DateTime(timezone=True), nullable=False), - Column("status", household_job_status_type, nullable=False), - Column("modal_call_id", String(255)), - Column("error_message", String), - Column("result", JSON(none_as_null=True)), - Column("started_at", DateTime(timezone=True)), - Column("completed_at", DateTime(timezone=True)), - Column("updated_at", DateTime(timezone=True), nullable=False), -) diff --git a/libs/policyengine-stage12-persistence/tests/test_report_run_store.py b/libs/policyengine-stage12-persistence/tests/test_report_run_store.py index f29f450fa..036880e45 100644 --- a/libs/policyengine-stage12-persistence/tests/test_report_run_store.py +++ b/libs/policyengine-stage12-persistence/tests/test_report_run_store.py @@ -44,7 +44,7 @@ def test_the_store_connects_as_the_modal_worker_role(store) -> None: def test_the_modal_worker_role_cannot_create_a_report_run(store, seeded_run) -> None: - from policyengine_stage12_persistence.report_run_tables import report_runs + from policyengine_simulation_persistence.tables import report_runs with ( pytest.raises(ProgrammingError, match="permission denied"), From d3b98aa5324dfabfe4848a14539b918826737f1c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:47:22 +0400 Subject: [PATCH 03/25] Test that the result-table models are the service library's, and match (failing) The twelve result tables the Modal worker role writes are to be modelled completely in `policyengine-simulation-persistence` (`result_metadata`), so the `Public API schema` job compares them exactly with the tables the Public API's chain builds. The Stage 12 library then declares no table of its own but the comparison path's, and every result row it builds goes in a shared model. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_public_api_schema.py | 33 ++++++++ .../tests/test_result_tables.py | 63 +++++++++++++++ .../tests/test_shared_table_models.py | 77 +++++++++++++++++++ 3 files changed, 173 insertions(+) create mode 100644 libs/policyengine-simulation-persistence/tests/test_result_tables.py diff --git a/libs/policyengine-simulation-persistence/tests/test_public_api_schema.py b/libs/policyengine-simulation-persistence/tests/test_public_api_schema.py index 728c54232..b163dd170 100644 --- a/libs/policyengine-simulation-persistence/tests/test_public_api_schema.py +++ b/libs/policyengine-simulation-persistence/tests/test_public_api_schema.py @@ -20,6 +20,7 @@ from policyengine_simulation_persistence import ( input_metadata, referenced_metadata, + result_metadata, service_metadata, ) from public_api_keys import KEYS_TO_PUBLIC_API_TABLES @@ -76,6 +77,38 @@ def test_the_models_match_the_public_api_tables( ) +@pytest.mark.public_api_schema +def test_the_result_models_match_the_public_api_tables( + disposable_database, public_api_database_url +) -> None: + """The result tables the Modal worker role writes match, as the service's do. + + Their keys name the service's run tables, so those are built first, after + the Public API's tables the run tables' own keys name. + """ + table_names = [table.name for table in result_metadata.tables.values()] + models_engine = _engine(disposable_database.superuser_url) + real_engine = _engine(public_api_database_url) + try: + # Tests may build the models' tables; runtime code never does. + referenced_metadata.create_all(models_engine) + service_metadata.create_all(models_engine) + result_metadata.create_all(models_engine) + with models_engine.connect() as connection: + models = describe_tables(connection, table_names) + with real_engine.connect() as connection: + real = describe_tables(connection, table_names) + finally: + models_engine.dispose() + real_engine.dispose() + + assert set(models.tables) == set(table_names) + compared_with = os.environ.get(COMMIT_VARIABLE) or "an unrecorded commit" + assert schema_differences(models, real) == [], ( + f"compared with the Public API at {compared_with}" + ) + + @pytest.mark.public_api_schema def test_the_input_models_match_the_public_api_columns( disposable_database, public_api_database_url diff --git a/libs/policyengine-simulation-persistence/tests/test_result_tables.py b/libs/policyengine-simulation-persistence/tests/test_result_tables.py new file mode 100644 index 000000000..7b6e008f5 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_result_tables.py @@ -0,0 +1,63 @@ +"""Sim's models of the Public API's result tables, which the Modal worker writes.""" + +from __future__ import annotations + +from policyengine_simulation_persistence import ( + report_runs, + result_metadata, + service_metadata, + simulation_runs, +) +from sqlalchemy import Enum + +# Results that compare a report run's two simulation runs belong to the report +# run; poverty and inequality, which describe one simulation, to a simulation +# run. +REPORT_RUN_RESULT_TABLES = { + "budget_summary", + "congressional_district_impacts", + "constituency_impacts", + "decile_impacts", + "intra_decile_impacts", + "labor_supply_decile_responses", + "labor_supply_responses", + "local_authority_impacts", + "program_statistics", + "withheld_area_codes", +} +SIMULATION_RUN_RESULT_TABLES = {"inequality", "poverty"} + + +def test_the_result_models_are_the_twelve_result_tables() -> None: + modelled = {table.name for table in result_metadata.tables.values()} + + assert modelled == REPORT_RUN_RESULT_TABLES | SIMULATION_RUN_RESULT_TABLES + + +def test_the_result_models_share_no_table_with_the_service_models() -> None: + assert not set(result_metadata.tables) & set(service_metadata.tables) + + +def test_each_result_names_the_service_models_run_it_belongs_to() -> None: + for table in result_metadata.tables.values(): + referred_tables = { + constraint.column_keys[0]: constraint.referred_table + for constraint in table.foreign_key_constraints + } + if table.name in SIMULATION_RUN_RESULT_TABLES: + assert referred_tables == {"simulation_run_id": simulation_runs}, table + continue + assert referred_tables["report_run_id"] is report_runs, table + if table.name.startswith("labor_supply_"): + assert set(referred_tables) == {"report_run_id"}, table + else: + assert referred_tables["baseline_simulation_run_id"] is simulation_runs + assert referred_tables["reform_simulation_run_id"] is simulation_runs + + +def test_every_result_table_and_enum_type_names_the_public_schema() -> None: + for table in result_metadata.tables.values(): + assert table.schema == "public", table + for column in table.columns: + if isinstance(column.type, Enum): + assert column.type.schema == "public", column diff --git a/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py b/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py index 54e3331ec..d96035c9f 100644 --- a/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py +++ b/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py @@ -8,13 +8,42 @@ from __future__ import annotations +import importlib +import pkgutil +from typing import TYPE_CHECKING +from uuid import uuid4 + +import policyengine_stage12_persistence +from economy_output import ( + US_AT_LARGE_STATES, + US_STATE_FIPS, + uk_output_with_withheld_codes, + us_output_with_every_part, +) +from policyengine_simulation_contract.stage12_execution import ReportRunIds +from policyengine_simulation_persistence import ( + result_metadata, + service_metadata, +) from policyengine_simulation_persistence import tables as shared_tables from policyengine_stage12_persistence import ( + DistrictCodes, budget_window_store, household_report_run_store, + report_run_result, report_run_store, ) from policyengine_stage12_persistence.report_run_store import RUN_TABLES, RunKind +from policyengine_stage12_persistence.tables import metadata as comparison_metadata +from sqlalchemy import Table + +if TYPE_CHECKING: + from collections.abc import Iterable + + +def _is_one_of(table: Table, tables: Iterable[Table]) -> bool: + """Return whether ``table`` is one of ``tables`` itself, not just alike.""" + return any(table is candidate for candidate in tables) def test_the_run_stores_write_the_shared_run_table_models() -> None: @@ -33,3 +62,51 @@ def test_the_budget_window_store_writes_the_shared_run_table_models() -> None: def test_the_household_report_store_writes_the_shared_run_table_models() -> None: assert household_report_run_store.report_runs is shared_tables.report_runs assert household_report_run_store.simulation_runs is shared_tables.simulation_runs + + +def test_every_table_the_library_names_is_shared_or_a_comparison_table() -> None: + """The library declares no model of a Public API run or result table itself. + + Its only tables of its own are the comparison path's, in ``tables.py``. + """ + shared_models = ( + *service_metadata.tables.values(), + *result_metadata.tables.values(), + ) + comparison_tables = tuple(comparison_metadata.tables.values()) + for module_info in pkgutil.iter_modules(policyengine_stage12_persistence.__path__): + module = importlib.import_module( + f"{policyengine_stage12_persistence.__name__}.{module_info.name}" + ) + for name, value in vars(module).items(): + if not isinstance(value, Table): + continue + assert _is_one_of(value, shared_models) or _is_one_of( + value, comparison_tables + ), f"{module_info.name}.{name} models {value.name} itself" + + +def test_every_result_row_goes_in_a_shared_result_table_model() -> None: + run_ids = ReportRunIds( + report_run_id=uuid4(), + baseline_simulation_run_id=uuid4(), + reform_simulation_run_id=uuid4(), + ) + results = ( + report_run_result( + us_output_with_every_part(), + run_ids=run_ids, + country="us", + district_codes=DistrictCodes( + state_fips=US_STATE_FIPS, at_large_states=US_AT_LARGE_STATES + ), + ), + report_run_result( + uk_output_with_withheld_codes(), run_ids=run_ids, country="uk" + ), + ) + result_models = tuple(result_metadata.tables.values()) + + for result in results: + for result_rows in result.rows: + assert _is_one_of(result_rows.table, result_models), result_rows.table.name From 66e0ae92bc3f27641afb8b499bd73da0f3e48bd3 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:50:41 +0400 Subject: [PATCH 04/25] Model the twelve result tables completely in the service library `policyengine_simulation_persistence.result_tables` models the result tables the Modal worker role writes, in `result_metadata`: every column, the primary and unique keys, checks and foreign keys under the Public API's names, and the `v2_decile_type` and `v2_uk_area_type` enums. Their keys name the service library's `report_runs` and `simulation_runs`. The comparison now covers them, as it covers the service's own tables. The Stage 12 library's `report_run_results.py` keeps the code that builds result rows from the economy output, and imports the tables it inserts into from the service library instead of declaring partial models of its own. Co-Authored-By: Claude Opus 5.5 --- .../__init__.py | 8 + .../result_tables.py | 455 ++++++++++++++++++ .../report_run_results.py | 202 +------- 3 files changed, 482 insertions(+), 183 deletions(-) create mode 100644 libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/result_tables.py diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py index 5af229675..d5220ba19 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py @@ -128,6 +128,11 @@ read_dynamic_reform, read_policy_reform, ) +from policyengine_simulation_persistence.result_tables import ( + DecileType, + UKAreaType, + result_metadata, +) from policyengine_simulation_persistence.tables import ( LATEST_YEAR, MAX_BUDGET_WINDOW_YEARS, @@ -156,6 +161,7 @@ "BudgetWindowYear", "CatalogDataset", "CatalogRegion", + "DecileType", "EconomyCatalogRows", "EconomyPopulation", "EconomyRecordOutcomeUnknownError", @@ -210,6 +216,7 @@ "SimulationStatus", "SimulationType", "StoredHousehold", + "UKAreaType", "UnknownCatalogVersionError", "UnknownDatasetError", "UnknownDynamicError", @@ -263,6 +270,7 @@ "referenced_metadata", "report_runs", "reports", + "result_metadata", "service_metadata", "simulation_runs", "simulations", diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/result_tables.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/result_tables.py new file mode 100644 index 000000000..e3fb9ccd4 --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/result_tables.py @@ -0,0 +1,455 @@ +"""Models of the Public API's result tables, which the Modal worker role writes. + +The Public API's v2 migration chain creates these tables; this module only +describes them, so the v2 Modal runtime can insert typed rows. Each table +matches what the chain builds, column for column, with its constraints under +the same names. They are in ``result_metadata``, apart from the tables the +simulation service creates rows in (``service_metadata``), whose run tables +their keys name. + +Results that compare a report run's baseline and reform belong to the report +run, and name its two simulation runs; poverty and inequality, which describe +one simulation, belong to a simulation run. Deleting a run deletes its +results; a simulation run that a report run's results name can't be deleted. +""" + +from __future__ import annotations + +from enum import StrEnum +from typing import TYPE_CHECKING, Final + +from sqlalchemy import ( + Boolean, + CheckConstraint, + Column, + Float, + ForeignKeyConstraint, + Integer, + MetaData, + PrimaryKeyConstraint, + String, + Table, + UniqueConstraint, + Uuid, +) + +from policyengine_simulation_persistence.tables import ( + TABLE_SCHEMA, + _enum_type, + _identifier, + _timestamp, + report_runs, + simulation_runs, +) + +if TYPE_CHECKING: + from sqlalchemy.sql.schema import SchemaItem + +NAME_LENGTH: Final = 512 +ENTITY_LENGTH: Final = 128 +AREA_CODE_LENGTH: Final = 64 +AREA_NAME_LENGTH: Final = 255 + + +class DecileType(StrEnum): + """Whether deciles rank households by income or by wealth.""" + + INCOME = "income" + WEALTH = "wealth" + + +class UKAreaType(StrEnum): + """The level of UK area a local-area breakdown reports.""" + + CONSTITUENCY = "constituency" + LOCAL_AUTHORITY = "local_authority" + + +result_metadata = MetaData(schema=TABLE_SCHEMA) + + +def _result_columns(table_name: str) -> tuple[SchemaItem, ...]: + """Return every result's ``id`` and ``created_at``, and its primary key.""" + return ( + _identifier(), + _timestamp("created_at", server_now=True), + PrimaryKeyConstraint("id", name=f"pk_{table_name}"), + ) + + +def _of_report_run( + table_name: str, + *, + baseline_key_name: str | None = None, + reform_key_name: str | None = None, +) -> tuple[SchemaItem, ...]: + """Return the keys of a result of a report run that compares its two runs. + + The key names are the Public API's, which its chain shortens past + PostgreSQL's 63 characters and ends in four characters of their hash; a + shortened name is passed in. + """ + return ( + *_of_report_run_alone(table_name), + Column("baseline_simulation_run_id", Uuid, nullable=False), + Column("reform_simulation_run_id", Uuid, nullable=False), + ForeignKeyConstraint( + ["baseline_simulation_run_id"], + [simulation_runs.c.id], + name=baseline_key_name + or f"fk_{table_name}_baseline_simulation_run_id_simulation_runs", + ondelete="RESTRICT", + ), + ForeignKeyConstraint( + ["reform_simulation_run_id"], + [simulation_runs.c.id], + name=reform_key_name + or f"fk_{table_name}_reform_simulation_run_id_simulation_runs", + ondelete="RESTRICT", + ), + ) + + +def _of_report_run_alone(table_name: str) -> tuple[SchemaItem, ...]: + """Return the key of a result of a report run that names no simulation run.""" + return ( + *_result_columns(table_name), + Column("report_run_id", Uuid, nullable=False), + ForeignKeyConstraint( + ["report_run_id"], + [report_runs.c.id], + name=f"fk_{table_name}_report_run_id_report_runs", + ondelete="CASCADE", + ), + ) + + +def _of_simulation_run(table_name: str) -> tuple[SchemaItem, ...]: + """Return the key of a result of one simulation run.""" + return ( + *_result_columns(table_name), + Column("simulation_run_id", Uuid, nullable=False), + ForeignKeyConstraint( + ["simulation_run_id"], + [simulation_runs.c.id], + name=f"fk_{table_name}_simulation_run_id_simulation_runs", + ondelete="CASCADE", + ), + ) + + +def _values(*names: str, nullable: bool) -> tuple[Column[float], ...]: + return tuple(Column(name, Float, nullable=nullable) for name in names) + + +def _area_values() -> tuple[Column[float], ...]: + """Return a geographic impact's household income changes and population.""" + return _values( + "average_household_income_change", + "relative_household_income_change", + "population", + nullable=False, + ) + + +def _disclosure(table_name: str) -> tuple[SchemaItem, ...]: + """Return a UK area's disclosure tier and the bounds of its income changes. + + The tier is ``reported``, or ``approximate`` for a thin sample; the bounds + are each income change's 90% interval. All are null without disclosure + control. + """ + return ( + Column("disclosure_tier", String(16), nullable=True), + *_values( + "average_household_income_change_lower", + "average_household_income_change_upper", + "relative_household_income_change_lower", + "relative_household_income_change_upper", + nullable=True, + ), + CheckConstraint( + "disclosure_tier IS NULL OR disclosure_tier IN ('reported', 'approximate')", + name=f"ck_{table_name}_disclosure_tier", + ), + ) + + +labor_supply_responses = Table( + "labor_supply_responses", + result_metadata, + *_of_report_run_alone("labor_supply_responses"), + *_values( + "income_effect_earnings_change", + "substitution_effect_earnings_change", + "total_earnings_change", + "revenue_change", + "relative_income_effect", + "relative_substitution_effect", + "baseline_weekly_hours", + "reform_weekly_hours", + "weekly_hours_change", + "weekly_hours_income_effect", + "weekly_hours_substitution_effect", + nullable=False, + ), + UniqueConstraint("report_run_id", name="uq_labor_supply_responses_run"), +) + +labor_supply_decile_responses = Table( + "labor_supply_decile_responses", + result_metadata, + *_of_report_run_alone("labor_supply_decile_responses"), + Column("decile", Integer, nullable=False), + *_values( + "average_income_effect", + "average_substitution_effect", + "relative_income_effect", + "relative_substitution_effect", + nullable=True, + ), + UniqueConstraint( + "report_run_id", + "decile", + name="uq_labor_supply_decile_responses_run_decile", + ), +) + +budget_summary = Table( + "budget_summary", + result_metadata, + *_of_report_run("budget_summary"), + Column("variable_name", String(NAME_LENGTH), nullable=False), + Column("entity", String(ENTITY_LENGTH), nullable=False), + *_values("baseline_total", "reform_total", "change", nullable=True), + UniqueConstraint( + "report_run_id", + "variable_name", + "entity", + name="uq_budget_summary_run_variable_entity", + ), +) + +# ``decile`` runs from 1 to ``quantiles``; rows without a decile variable +# count as equal. +decile_impacts = Table( + "decile_impacts", + result_metadata, + *_of_report_run("decile_impacts"), + Column("income_variable", String(NAME_LENGTH), nullable=False), + Column("decile_variable", String(NAME_LENGTH), nullable=True), + Column("entity", String(ENTITY_LENGTH), nullable=False), + Column("decile", Integer, nullable=False), + Column("quantiles", Integer, nullable=False), + *_values( + "baseline_mean", + "reform_mean", + "absolute_change", + "relative_change", + "count_better_off", + "count_worse_off", + "count_no_change", + nullable=True, + ), + UniqueConstraint( + "report_run_id", + "income_variable", + "decile_variable", + "entity", + "decile", + "quantiles", + name="uq_decile_impacts_run_measure", + postgresql_nulls_not_distinct=True, + ), + CheckConstraint("quantiles > 0", name="ck_decile_impacts_quantiles"), + CheckConstraint("decile BETWEEN 1 AND quantiles", name="ck_decile_impacts_decile"), +) + +intra_decile_impacts = Table( + "intra_decile_impacts", + result_metadata, + *_of_report_run( + "intra_decile_impacts", + baseline_key_name="fk_intra_decile_impacts_baseline_simulation_run_id_simu_7cf0", + reform_key_name="fk_intra_decile_impacts_reform_simulation_run_id_simula_467e", + ), + Column( + "decile_type", + _enum_type(DecileType, "v2_decile_type"), + nullable=False, + ), + Column("decile", Integer, nullable=False), + *_values( + "lose_more_than_5pct", + "lose_less_than_5pct", + "no_change", + "gain_less_than_5pct", + "gain_more_than_5pct", + nullable=True, + ), + UniqueConstraint( + "report_run_id", + "decile_type", + "decile", + name="uq_intra_decile_impacts_run_type_decile", + ), + CheckConstraint( + "decile BETWEEN 0 AND 10", + name="ck_intra_decile_impacts_decile", + ), +) + +# Rows without an entity count as equal. +program_statistics = Table( + "program_statistics", + result_metadata, + *_of_report_run( + "program_statistics", + baseline_key_name="fk_program_statistics_baseline_simulation_run_id_simula_faf9", + ), + Column("program_name", String(NAME_LENGTH), nullable=False), + Column("entity", String(ENTITY_LENGTH), nullable=True), + Column("is_tax", Boolean, nullable=True), + *_values( + "baseline_total", + "reform_total", + "change", + "baseline_count", + "reform_count", + "winners", + "losers", + nullable=True, + ), + UniqueConstraint( + "report_run_id", + "program_name", + "entity", + name="uq_program_statistics_run_program_entity", + postgresql_nulls_not_distinct=True, + ), +) + +congressional_district_impacts = Table( + "congressional_district_impacts", + result_metadata, + *_of_report_run( + "congressional_district_impacts", + baseline_key_name="fk_congressional_district_impacts_baseline_simulation_r_9f79", + reform_key_name="fk_congressional_district_impacts_reform_simulation_run_80f5", + ), + *_area_values(), + Column("district_geoid", Integer, nullable=False), + Column("state_fips", Integer, nullable=False), + Column("district_number", Integer, nullable=False), + *_values( + "winner_percentage", + "loser_percentage", + "no_change_percentage", + nullable=False, + ), + UniqueConstraint( + "report_run_id", + "district_geoid", + name="uq_congressional_district_impacts_run_geoid", + ), +) + +constituency_impacts = Table( + "constituency_impacts", + result_metadata, + *_of_report_run( + "constituency_impacts", + baseline_key_name="fk_constituency_impacts_baseline_simulation_run_id_simu_d2cd", + reform_key_name="fk_constituency_impacts_reform_simulation_run_id_simula_aaa1", + ), + *_area_values(), + *_disclosure("constituency_impacts"), + Column("constituency_code", String(AREA_CODE_LENGTH), nullable=False), + Column("constituency_name", String(AREA_NAME_LENGTH), nullable=False), + UniqueConstraint( + "report_run_id", + "constituency_code", + name="uq_constituency_impacts_run_code", + ), +) + +local_authority_impacts = Table( + "local_authority_impacts", + result_metadata, + *_of_report_run( + "local_authority_impacts", + baseline_key_name="fk_local_authority_impacts_baseline_simulation_run_id_s_0d31", + reform_key_name="fk_local_authority_impacts_reform_simulation_run_id_sim_0dc2", + ), + *_area_values(), + *_disclosure("local_authority_impacts"), + Column("local_authority_code", String(AREA_CODE_LENGTH), nullable=False), + Column("local_authority_name", String(AREA_NAME_LENGTH), nullable=False), + UniqueConstraint( + "report_run_id", + "local_authority_code", + name="uq_local_authority_impacts_run_code", + ), +) + +# The codes of the UK areas a local-area run estimated but disclosure control +# withheld, one row per area; none means none were withheld. +withheld_area_codes = Table( + "withheld_area_codes", + result_metadata, + *_of_report_run( + "withheld_area_codes", + baseline_key_name="fk_withheld_area_codes_baseline_simulation_run_id_simul_a169", + ), + Column("area_type", _enum_type(UKAreaType, "v2_uk_area_type"), nullable=False), + Column("code", String(AREA_CODE_LENGTH), nullable=False), + UniqueConstraint( + "report_run_id", + "area_type", + "code", + name="uq_withheld_area_codes_run_type_code", + ), +) + +# Rows without a filter variable or group count as equal, so a run has one +# unfiltered row per poverty type and entity. +poverty = Table( + "poverty", + result_metadata, + *_of_simulation_run("poverty"), + Column("poverty_type", String(ENTITY_LENGTH), nullable=False), + Column("entity", String(ENTITY_LENGTH), nullable=False), + Column("filter_variable", String(NAME_LENGTH), nullable=True), + Column("filter_group", String(64), nullable=True), + *_values("headcount", "total_population", "rate", nullable=True), + UniqueConstraint( + "simulation_run_id", + "poverty_type", + "entity", + "filter_variable", + "filter_group", + name="uq_poverty_simulation_run_measure", + postgresql_nulls_not_distinct=True, + ), +) + +inequality = Table( + "inequality", + result_metadata, + *_of_simulation_run("inequality"), + Column("income_variable", String(NAME_LENGTH), nullable=False), + Column("entity", String(ENTITY_LENGTH), nullable=False), + *_values( + "gini", + "top_10_share", + "top_1_share", + "bottom_50_share", + nullable=True, + ), + UniqueConstraint( + "simulation_run_id", + "income_variable", + "entity", + name="uq_inequality_simulation_run_measure", + ), +) diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_results.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_results.py index f71ef4cbd..6b2592362 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_results.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_results.py @@ -6,6 +6,10 @@ two simulation runs where the table names them, and poverty and inequality keyed by each simulation run. The cliff impact, SPM receipts and data version are columns of the report run itself, which its completion sets. + +The result tables' models are the simulation service library's +(``policyengine_simulation_persistence.result_tables``), which the CI +comparison checks against the Public API's tables. """ from __future__ import annotations @@ -15,8 +19,20 @@ from uuid import uuid4 from policyengine_simulation_contract.macro_output import SingleYearMacroOutput -from sqlalchemy import Boolean, Column, Float, Integer, MetaData, String, Table -from sqlalchemy.dialects.postgresql import ENUM, UUID +from policyengine_simulation_persistence.result_tables import ( + budget_summary, + congressional_district_impacts, + constituency_impacts, + decile_impacts, + inequality, + intra_decile_impacts, + labor_supply_decile_responses, + labor_supply_responses, + local_authority_impacts, + poverty, + program_statistics, + withheld_area_codes, +) if TYPE_CHECKING: from collections.abc import Mapping @@ -34,187 +50,7 @@ RacePovertyOutput, ) from policyengine_simulation_contract.stage12_execution import ReportRunIds - -metadata = MetaData(schema="public") -decile_type_type = ENUM("income", "wealth", name="v2_decile_type", create_type=False) -uk_area_type_type = ENUM( - "constituency", "local_authority", name="v2_uk_area_type", create_type=False -) - - -def _report_run_keys() -> tuple[Column[Any], ...]: - return ( - Column("id", UUID(as_uuid=True), primary_key=True), - Column("report_run_id", UUID(as_uuid=True), nullable=False), - Column("baseline_simulation_run_id", UUID(as_uuid=True), nullable=False), - Column("reform_simulation_run_id", UUID(as_uuid=True), nullable=False), - ) - - -def _simulation_run_keys() -> tuple[Column[Any], ...]: - return ( - Column("id", UUID(as_uuid=True), primary_key=True), - Column("simulation_run_id", UUID(as_uuid=True), nullable=False), - ) - - -labor_supply_responses = Table( - "labor_supply_responses", - metadata, - Column("id", UUID(as_uuid=True), primary_key=True), - Column("report_run_id", UUID(as_uuid=True), nullable=False), - *( - Column(name, Float, nullable=False) - for name in ( - "income_effect_earnings_change", - "substitution_effect_earnings_change", - "total_earnings_change", - "revenue_change", - "relative_income_effect", - "relative_substitution_effect", - "baseline_weekly_hours", - "reform_weekly_hours", - "weekly_hours_change", - "weekly_hours_income_effect", - "weekly_hours_substitution_effect", - ) - ), -) -labor_supply_decile_responses = Table( - "labor_supply_decile_responses", - metadata, - Column("id", UUID(as_uuid=True), primary_key=True), - Column("report_run_id", UUID(as_uuid=True), nullable=False), - Column("decile", Integer, nullable=False), - Column("average_income_effect", Float), - Column("average_substitution_effect", Float), - Column("relative_income_effect", Float), - Column("relative_substitution_effect", Float), -) -program_statistics = Table( - "program_statistics", - metadata, - *_report_run_keys(), - Column("program_name", String(512), nullable=False), - Column("entity", String(128)), - Column("is_tax", Boolean), - Column("baseline_total", Float), - Column("reform_total", Float), - Column("change", Float), -) - - -def _area_values() -> tuple[Column[Any], ...]: - return ( - Column("average_household_income_change", Float, nullable=False), - Column("relative_household_income_change", Float, nullable=False), - Column("population", Float, nullable=False), - ) - - -def _disclosure_values() -> tuple[Column[Any], ...]: - return ( - Column("disclosure_tier", String(16)), - Column("average_household_income_change_lower", Float), - Column("average_household_income_change_upper", Float), - Column("relative_household_income_change_lower", Float), - Column("relative_household_income_change_upper", Float), - ) - - -congressional_district_impacts = Table( - "congressional_district_impacts", - metadata, - *_report_run_keys(), - *_area_values(), - Column("district_geoid", Integer, nullable=False), - Column("state_fips", Integer, nullable=False), - Column("district_number", Integer, nullable=False), - Column("winner_percentage", Float, nullable=False), - Column("loser_percentage", Float, nullable=False), - Column("no_change_percentage", Float, nullable=False), -) -constituency_impacts = Table( - "constituency_impacts", - metadata, - *_report_run_keys(), - *_area_values(), - *_disclosure_values(), - Column("constituency_code", String(64), nullable=False), - Column("constituency_name", String(255), nullable=False), -) -local_authority_impacts = Table( - "local_authority_impacts", - metadata, - *_report_run_keys(), - *_area_values(), - *_disclosure_values(), - Column("local_authority_code", String(64), nullable=False), - Column("local_authority_name", String(255), nullable=False), -) -# The codes of the areas a local-area run estimated but disclosure control did -# not publish, one row per area: none means none were withheld. -withheld_area_codes = Table( - "withheld_area_codes", - metadata, - *_report_run_keys(), - Column("area_type", uk_area_type_type, nullable=False), - Column("code", String(64), nullable=False), -) -budget_summary = Table( - "budget_summary", - metadata, - *_report_run_keys(), - Column("variable_name", String(512), nullable=False), - Column("entity", String(128), nullable=False), - Column("baseline_total", Float), - Column("reform_total", Float), - Column("change", Float), -) -decile_impacts = Table( - "decile_impacts", - metadata, - *_report_run_keys(), - Column("income_variable", String(512), nullable=False), - Column("decile_variable", String(512)), - Column("entity", String(128), nullable=False), - Column("decile", Integer, nullable=False), - Column("quantiles", Integer, nullable=False), - Column("absolute_change", Float), - Column("relative_change", Float), -) -intra_decile_impacts = Table( - "intra_decile_impacts", - metadata, - *_report_run_keys(), - Column("decile_type", decile_type_type, nullable=False), - Column("decile", Integer, nullable=False), - Column("lose_more_than_5pct", Float), - Column("lose_less_than_5pct", Float), - Column("no_change", Float), - Column("gain_less_than_5pct", Float), - Column("gain_more_than_5pct", Float), -) -poverty = Table( - "poverty", - metadata, - *_simulation_run_keys(), - Column("poverty_type", String(128), nullable=False), - Column("entity", String(128), nullable=False), - Column("filter_variable", String(512)), - Column("filter_group", String(64)), - Column("rate", Float), -) -inequality = Table( - "inequality", - metadata, - *_simulation_run_keys(), - Column("income_variable", String(512), nullable=False), - Column("entity", String(128), nullable=False), - Column("gini", Float), - Column("top_10_share", Float), - Column("top_1_share", Float), -) + from sqlalchemy import Table # The household variable each headline budget figure totals, and the column # its figure goes in. ``budgetary_impact`` is tax minus benefits, as the From e7672c67807fbef583daf813b8adf7a60c251ec3 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:51:38 +0400 Subject: [PATCH 05/25] Test that one engine and URL helper serve both libraries (failing) The service library is to hold the one URL check and the unpooled engine the Modal calls' stores and the comparison path use: an engine with no pool and a 10-second connect timeout, so a long calculation holds no idle database session. The Stage 12 library's tests of its own copies move to the service library's engine tests. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_engine.py | 66 ++++++++++++++++++- .../tests/test_shared_table_models.py | 10 +++ .../tests/test_store.py | 58 ---------------- 3 files changed, 75 insertions(+), 59 deletions(-) diff --git a/libs/policyengine-simulation-persistence/tests/test_engine.py b/libs/policyengine-simulation-persistence/tests/test_engine.py index 6e3121b8a..8d89249f9 100644 --- a/libs/policyengine-simulation-persistence/tests/test_engine.py +++ b/libs/policyengine-simulation-persistence/tests/test_engine.py @@ -5,13 +5,18 @@ from __future__ import annotations +import sqlite3 + import pytest from policyengine_simulation_persistence import ( bounded_transaction, create_simulation_engine, + create_unpooled_engine, normalized_database_url, ) -from sqlalchemy import text +from policyengine_simulation_persistence import engine as engine_module +from sqlalchemy import create_engine, text +from sqlalchemy.pool import NullPool, QueuePool def test_database_url_selects_psycopg_without_changing_the_secret_format() -> None: @@ -70,6 +75,65 @@ def capture_engine(url, **options): assert captured_options["pool_size"] >= 1 +def test_the_unpooled_engine_holds_no_connection_and_selects_psycopg( + monkeypatch, +) -> None: + captured_options = {} + + def capture_engine(url, **options): + captured_options.update(options, url=url) + return object() + + monkeypatch.setattr(engine_module, "create_engine", capture_engine) + + create_unpooled_engine("postgresql://modal_worker:secret@db.example/postgres") + + assert captured_options["url"].drivername == "postgresql+psycopg" + assert captured_options["poolclass"] is NullPool + assert captured_options["connect_args"] == {"connect_timeout": 10} + + +@pytest.mark.parametrize("fail_transaction", [False, True]) +def test_the_unpooled_engine_closes_connections_between_transactions( + monkeypatch, fail_transaction +) -> None: + """A long calculation holds no idle database session after its operation.""" + + def local_engine(_url, **options): + # Exercise the unpooled configuration with a real local DBAPI. + # PostgreSQL's default is QueuePool; SQLite's memory default differs. + options.pop("connect_args") + options.setdefault("poolclass", QueuePool) + return create_engine("sqlite://", **options) + + monkeypatch.setattr(engine_module, "create_engine", local_engine) + engine = create_unpooled_engine("postgresql://runtime:secret@db.example/postgres") + try: + try: + with engine.begin() as connection: + physical_connection = connection.connection.driver_connection + assert connection.exec_driver_sql("SELECT 1").scalar_one() == 1 + if fail_transaction: + _fail_the_calculation() + except RuntimeError: + assert fail_transaction + + with pytest.raises(sqlite3.ProgrammingError, match="closed database"): + physical_connection.execute("SELECT 1") + + with engine.begin() as connection: + assert connection.connection.driver_connection is not physical_connection + assert connection.exec_driver_sql("SELECT 1").scalar_one() == 1 + finally: + engine.dispose() + + +def _fail_the_calculation() -> None: + """Raise the failure a calculation inside the transaction would.""" + msg = "calculation failed" + raise RuntimeError(msg) + + @pytest.mark.postgres def test_a_bounded_transaction_limits_its_own_statements_and_commits( disposable_database, diff --git a/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py b/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py index d96035c9f..30e392f52 100644 --- a/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py +++ b/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py @@ -22,6 +22,7 @@ ) from policyengine_simulation_contract.stage12_execution import ReportRunIds from policyengine_simulation_persistence import ( + engine, result_metadata, service_metadata, ) @@ -32,6 +33,7 @@ household_report_run_store, report_run_result, report_run_store, + store, ) from policyengine_stage12_persistence.report_run_store import RUN_TABLES, RunKind from policyengine_stage12_persistence.tables import metadata as comparison_metadata @@ -110,3 +112,11 @@ def test_every_result_row_goes_in_a_shared_result_table_model() -> None: for result in results: for result_rows in result.rows: assert _is_one_of(result_rows.table, result_models), result_rows.table.name + + +def test_the_stores_connect_with_the_service_librarys_one_engine_helper() -> None: + """One URL check and one unpooled engine serve every Modal call's store.""" + assert store.create_stage12_engine is engine.create_unpooled_engine + assert store.normalized_database_url is engine.normalized_database_url + assert report_run_store.create_stage12_engine is engine.create_unpooled_engine + assert budget_window_store.create_stage12_engine is engine.create_unpooled_engine diff --git a/libs/policyengine-stage12-persistence/tests/test_store.py b/libs/policyengine-stage12-persistence/tests/test_store.py index 811045120..cc9119402 100644 --- a/libs/policyengine-stage12-persistence/tests/test_store.py +++ b/libs/policyengine-stage12-persistence/tests/test_store.py @@ -14,11 +14,8 @@ SimulationRole, ) from policyengine_stage12_persistence import Stage12PersistenceStore -from policyengine_stage12_persistence import store as store_module -from policyengine_stage12_persistence.store import normalized_database_url from sqlalchemy import create_engine from sqlalchemy.exc import OperationalError -from sqlalchemy.pool import QueuePool from sqlalchemy.sql import ClauseElement NOW = datetime(2026, 9, 22, tzinfo=UTC) @@ -27,48 +24,6 @@ OBSERVABILITY_ID = "00000000-0000-4000-8000-000000000012" -@pytest.mark.parametrize("fail_transaction", [False, True]) -def test_runtime_engine_closes_connections_between_transactions( - monkeypatch, fail_transaction -) -> None: - def local_engine(_url, **options): - # Exercise the runtime pooling configuration with a real local DBAPI. - # PostgreSQL's default is QueuePool; SQLite's memory default differs. - options.pop("connect_args") - options.setdefault("poolclass", QueuePool) - return create_engine("sqlite://", **options) - - monkeypatch.setattr(store_module, "create_engine", local_engine) - engine = store_module.create_stage12_engine( - "postgresql://runtime:secret@db.example/postgres" - ) - try: - try: - with engine.begin() as connection: - physical_connection = connection.connection.driver_connection - assert connection.exec_driver_sql("SELECT 1").scalar_one() == 1 - if fail_transaction: - _fail_the_calculation() - except RuntimeError: - assert fail_transaction - - # Long calculations must not hold an idle session after DB work ends. - with pytest.raises(sqlite3.ProgrammingError, match="closed database"): - physical_connection.execute("SELECT 1") - - with engine.begin() as connection: - assert connection.connection.driver_connection is not physical_connection - assert connection.exec_driver_sql("SELECT 1").scalar_one() == 1 - finally: - engine.dispose() - - -def _fail_the_calculation() -> None: - """Raise the failure a calculation inside the transaction would.""" - msg = "calculation failed" - raise RuntimeError(msg) - - def _report() -> ComparisonReportRecord: return ComparisonReportRecord( evaluation_id=REPORT_ID, @@ -442,16 +397,3 @@ def test_simulation_status_replacement_preserves_a_concurrent_cancellation() -> assert replaced is False assert result == cancelled assert len(engine.connection.statements) == 1 - - -def test_database_url_uses_one_canonical_secret_format() -> None: - parsed = normalized_database_url( - "postgresql://policyengine_v2_runtime:secret@db.example/postgres" - ) - assert parsed.drivername == "postgresql+psycopg" - assert parsed.username == "policyengine_v2_runtime" - - with pytest.raises(ValueError, match="postgresql://"): - normalized_database_url( - "postgresql+psycopg://policyengine_v2_runtime:secret@db.example/postgres" - ) From 3f655bdddf61546682d325a41cf52f5a4db4bf33 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:52:40 +0400 Subject: [PATCH 06/25] Use the service library's one engine and URL helper in the Stage 12 library `create_unpooled_engine` in `policyengine_simulation_persistence.engine` builds the engine the Stage 12 stores used: no pool, so each operation's session ends with it, and a 10-second connect timeout. The Stage 12 store binds it as `create_stage12_engine`, the name the stores and the comparison path already use, and its own copies of the engine and URL helpers are gone. A malformed Modal worker or comparison-path URL now raises the service's "Simulation database URL ..." error without the parser's message, which may quote the URL, as its cause. Co-Authored-By: Claude Opus 5.5 --- .../__init__.py | 2 + .../engine.py | 20 ++++++++- .../__init__.py | 2 - .../policyengine_stage12_persistence/store.py | 42 ++++--------------- .../tests/test_shared_table_models.py | 2 +- 5 files changed, 30 insertions(+), 38 deletions(-) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py index d5220ba19..dbeae927d 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py @@ -87,6 +87,7 @@ from policyengine_simulation_persistence.engine import ( bounded_transaction, create_simulation_engine, + create_unpooled_engine, normalized_database_url, ) from policyengine_simulation_persistence.household_job_records import ( @@ -231,6 +232,7 @@ "budget_window_runs", "combine_reforms", "create_simulation_engine", + "create_unpooled_engine", "fail_unstarted_household_job", "fail_unstarted_report_run", "fail_unstarted_simulation_run", diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/engine.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/engine.py index b97fb9af1..ef01257be 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/engine.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/engine.py @@ -1,4 +1,4 @@ -"""Database connections for the simulation service's own role.""" +"""Database connections for the simulation service and the v2 Modal calls.""" from __future__ import annotations @@ -8,6 +8,7 @@ from sqlalchemy import create_engine, func, select from sqlalchemy.engine import URL, make_url from sqlalchemy.exc import ArgumentError +from sqlalchemy.pool import NullPool if TYPE_CHECKING: from collections.abc import Iterator @@ -20,6 +21,8 @@ # enough. POOL_SIZE = 2 MAX_OVERFLOW = 3 +# A v2 Modal call connects across the internet to the database's pooler. +UNPOOLED_CONNECT_TIMEOUT_SECONDS = 10 def normalized_database_url(database_url: str) -> URL: @@ -63,6 +66,21 @@ def create_simulation_engine(database_url: str) -> Engine: ) +def create_unpooled_engine(database_url: str) -> Engine: + """Build an engine that opens a connection per operation and closes it after. + + The v2 Modal coordinators, workers and household functions, and the + comparison path, spend most of their lifetime calculating or waiting, so + each database session ends with its short operation instead of holding a + pooler slot through that idle time. It creates no objects. + """ + return create_engine( + normalized_database_url(database_url), + poolclass=NullPool, + connect_args={"connect_timeout": UNPOOLED_CONNECT_TIMEOUT_SECONDS}, + ) + + @contextmanager def bounded_transaction( engine: Engine, *, statement_timeout_milliseconds: int diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/__init__.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/__init__.py index 7e86cb336..bf45c8159 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/__init__.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/__init__.py @@ -28,7 +28,6 @@ ConnectionOpener, Stage12PersistenceStore, create_stage12_engine, - normalized_database_url, ) from policyengine_stage12_persistence.validation import ( Stage12RuntimeAccessError, @@ -55,7 +54,6 @@ "YearStarter", "YearToStart", "create_stage12_engine", - "normalized_database_url", "report_run_result", "validate_runtime_database", ] diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/store.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/store.py index 43476ee6c..91caaa169 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/store.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/store.py @@ -14,10 +14,14 @@ ComparisonSimulationRecord, ResultComparisonStatus, ) -from sqlalchemy import Engine, create_engine, select, update + +# Every Stage 12 store builds its engine with the simulation service library's +# unpooled engine, which checks the URL with that library's one URL check. +from policyengine_simulation_persistence.engine import ( + create_unpooled_engine as create_stage12_engine, +) +from sqlalchemy import Engine, select, update from sqlalchemy.dialects.postgresql import insert -from sqlalchemy.engine import URL, Connection, make_url -from sqlalchemy.pool import NullPool from policyengine_stage12_persistence.tables import ( REPORT_IDENTITY_CONSTRAINT, @@ -46,6 +50,7 @@ from uuid import UUID from sqlalchemy import ColumnElement, RowMapping, Table + from sqlalchemy.engine import Connection REPORT_IDENTITY_COLUMNS = ( "environment", @@ -101,37 +106,6 @@ ) -def normalized_database_url(database_url: str) -> URL: - """Select psycopg 3 without changing the stored secret's URL format.""" - if not database_url or database_url != database_url.strip(): - msg = "Stage 12 database URL is missing or malformed" - raise ValueError(msg) - try: - parsed = make_url(database_url) - except Exception as error: - msg = "Stage 12 database URL is malformed" - raise ValueError(msg) from error - if parsed.drivername != "postgresql": - msg = "Stage 12 database URL must use postgresql://" - raise ValueError(msg) - if not parsed.username or not parsed.host or not parsed.database: - msg = "Stage 12 database URL is incomplete" - raise ValueError(msg) - return parsed.set(drivername="postgresql+psycopg") - - -def create_stage12_engine(database_url: str) -> Engine: - """Build the DML-only runtime engine; this never creates database objects.""" - return create_engine( - normalized_database_url(database_url), - # Coordinators and workers spend most of their lifetime calculating - # or waiting. Release each database session after its short operation - # instead of reserving a pooler slot throughout that idle interval. - poolclass=NullPool, - connect_args={"connect_timeout": 10}, - ) - - class ConnectionOpener(Protocol): """Open the database connection that one store operation uses. diff --git a/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py b/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py index 30e392f52..403dd1696 100644 --- a/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py +++ b/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py @@ -117,6 +117,6 @@ def test_every_result_row_goes_in_a_shared_result_table_model() -> None: def test_the_stores_connect_with_the_service_librarys_one_engine_helper() -> None: """One URL check and one unpooled engine serve every Modal call's store.""" assert store.create_stage12_engine is engine.create_unpooled_engine - assert store.normalized_database_url is engine.normalized_database_url + assert not hasattr(store, "normalized_database_url") assert report_run_store.create_stage12_engine is engine.create_unpooled_engine assert budget_window_store.create_stage12_engine is engine.create_unpooled_engine From 04c03b9313d20dd63eaad30757855a9a92ea29f6 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:55:40 +0400 Subject: [PATCH 07/25] Test the Modal worker role's privileges and writes (failing) The service library is to list the privileges the v2 Modal calls need as `policyengine_v2_modal_worker`: SELECT on the four run tables, column UPDATE of each run's state and the results its row holds, and INSERT on the twelve result tables. `missing_table_privileges` is to check any role's lists. In the `Public API schema` job, a new test checks that the Public API's grants give the role those privileges, and, as the role, records a run's state in each run table and inserts a row in every result table through the shared models; a Python None in a JSON result is stored as SQL NULL. The service write test's row helpers move to `tests/service_rows.py`, so both roles' tests insert the same rows. Co-Authored-By: Claude Opus 5.5 --- .../tests/conftest.py | 11 + .../tests/service_rows.py | 233 ++++++++++++++++++ .../tests/test_access.py | 95 +++++++ .../tests/test_modal_worker_writes.py | 190 ++++++++++++++ .../tests/test_service_writes.py | 192 +-------------- 5 files changed, 537 insertions(+), 184 deletions(-) create mode 100644 libs/policyengine-simulation-persistence/tests/service_rows.py create mode 100644 libs/policyengine-simulation-persistence/tests/test_modal_worker_writes.py diff --git a/libs/policyengine-simulation-persistence/tests/conftest.py b/libs/policyengine-simulation-persistence/tests/conftest.py index 328c731b3..4baa96233 100644 --- a/libs/policyengine-simulation-persistence/tests/conftest.py +++ b/libs/policyengine-simulation-persistence/tests/conftest.py @@ -43,6 +43,7 @@ # The role the Public API's privilege manifest manages for the simulation # service; the deployed SIMULATION_SERVICE_DATABASE_URL logs in as it. SERVICE_ROLE = "policyengine_v2_simulation_service" +MODAL_WORKER_ROLE = "policyengine_v2_modal_worker" @dataclass(frozen=True) @@ -140,6 +141,16 @@ def act_as_the_service(connection: Connection) -> None: connection.execute(text(f"SET LOCAL ROLE {SERVICE_ROLE}")) +def act_as_the_modal_worker(connection: Connection) -> None: + """Take the Modal worker role for the rest of the connection's transaction. + + The superuser's own role comes back first, so a test that acted as the + service may act as the worker next. + """ + connection.execute(text("RESET ROLE")) + connection.execute(text(f"SET LOCAL ROLE {MODAL_WORKER_ROLE}")) + + @contextmanager def connection_as_the_service(database_url: str) -> Iterator[Connection]: """Yield a connection acting as the service's role, rolled back afterwards.""" diff --git a/libs/policyengine-simulation-persistence/tests/service_rows.py b/libs/policyengine-simulation-persistence/tests/service_rows.py new file mode 100644 index 000000000..6179023ef --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/service_rows.py @@ -0,0 +1,233 @@ +"""Rows the simulation service creates, for tests of the roles' writes. + +The tests run against a database the Public API's own v2 migration chain +built (``PUBLIC_API_SCHEMA_DATABASE_URL``; see ``conftest``), inside a +transaction they roll back. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from hashlib import sha256 +from typing import TYPE_CHECKING +from uuid import UUID, uuid4 + +from policyengine_simulation_persistence import ( + HouseholdJobStatus, + ReportRunStatus, + ReportRunTrigger, + SimulationStatus, + SimulationType, + budget_window_runs, + household_jobs, + report_runs, + reports, + simulation_runs, + simulations, +) +from sqlalchemy import insert, text + +if TYPE_CHECKING: + from sqlalchemy import Connection + +COUNTRY_PACKAGE_VERSION = "1.402.0" +POLICYENGINE_VERSION = "3.4.0" +YEAR = 2026 + + +@dataclass(frozen=True) +class PublicApiRows: + """Rows of the Public API's own tables that the service's rows name.""" + + tax_benefit_model_id: UUID + model_version_id: UUID + dataset_id: UUID + household_id: UUID + + +@dataclass(frozen=True) +class ServiceRows: + """A budget-window report's rows, one year's report run and a household job.""" + + baseline_simulation_id: UUID + reform_simulation_id: UUID + report_id: UUID + baseline_run_id: UUID + reform_run_id: UUID + budget_window_run_id: UUID + report_run_id: UUID + household_job_id: UUID + + +def insert_public_api_rows(connection: Connection) -> PublicApiRows: + """Insert, as the superuser, the catalog, dataset and household rows a run names. + + The Public API writes these rows, and the database checks on insert that + each row the service writes names ones that exist. + """ + rows = PublicApiRows( + tax_benefit_model_id=uuid4(), + model_version_id=uuid4(), + dataset_id=uuid4(), + household_id=uuid4(), + ) + connection.execute( + text("INSERT INTO public.tax_benefit_models (id, name) VALUES (:id, :name)"), + { + "id": rows.tax_benefit_model_id, + "name": f"test-{rows.tax_benefit_model_id.hex[:12]}", + }, + ) + connection.execute( + text( + "INSERT INTO public.tax_benefit_model_versions " + "(id, model_id, version, current_law_id, metadata_time_periods) " + "VALUES (:id, :model_id, :version, 1, '{}')" + ), + { + "id": rows.model_version_id, + "model_id": rows.tax_benefit_model_id, + "version": COUNTRY_PACKAGE_VERSION, + }, + ) + connection.execute( + text( + "INSERT INTO public.datasets " + "(id, name, year, is_output_dataset, tax_benefit_model_version_id) " + "VALUES (:id, 'test dataset', :year, false, :model_version_id)" + ), + { + "id": rows.dataset_id, + "year": YEAR, + "model_version_id": rows.model_version_id, + }, + ) + connection.execute( + text( + "INSERT INTO public.households " + "(id, household_data, country_id, canonicalization_version, content_hash) " + "VALUES (:id, '{}', 'us', 1, :content_hash)" + ), + { + "id": rows.household_id, + "content_hash": sha256(rows.household_id.bytes).hexdigest(), + }, + ) + return rows + + +def insert_economy_simulation( + connection: Connection, public_api_rows: PublicApiRows +) -> UUID: + return connection.execute( + insert(simulations) + .values( + simulation_type=SimulationType.ECONOMY, + dataset_id=public_api_rows.dataset_id, + tax_benefit_model_version_id=public_api_rows.model_version_id, + year=YEAR, + ) + .returning(simulations.c.id) + ).scalar_one() + + +def insert_simulation_run( + connection: Connection, simulation_id: UUID, public_api_rows: PublicApiRows +) -> UUID: + return connection.execute( + insert(simulation_runs) + .values( + simulation_id=simulation_id, + year=YEAR, + tax_benefit_model_version_id=public_api_rows.model_version_id, + country_package_version=COUNTRY_PACKAGE_VERSION, + policyengine_version=POLICYENGINE_VERSION, + dataset_id=public_api_rows.dataset_id, + status=SimulationStatus.PENDING, + ) + .returning(simulation_runs.c.id) + ).scalar_one() + + +def insert_service_rows( + connection: Connection, public_api_rows: PublicApiRows +) -> ServiceRows: + """Insert a row in every table the service writes, as a submission would. + + That is a two-year budget-window report's simulations, the report, the + first year's simulation runs, the budget-window run and that year's report + run, and a household job, every run pending. + """ + baseline_simulation_id = insert_economy_simulation(connection, public_api_rows) + reform_simulation_id = insert_economy_simulation(connection, public_api_rows) + report_id = connection.execute( + insert(reports) + .values( + label="Test report", + country="us", + tax_benefit_model_id=public_api_rows.tax_benefit_model_id, + baseline_simulation_id=baseline_simulation_id, + reform_simulation_id=reform_simulation_id, + dataset_id=public_api_rows.dataset_id, + year=YEAR, + budget_window_size=2, + inputs={}, + ) + .returning(reports.c.id) + ).scalar_one() + baseline_run_id = insert_simulation_run( + connection, baseline_simulation_id, public_api_rows + ) + reform_run_id = insert_simulation_run( + connection, reform_simulation_id, public_api_rows + ) + budget_window_run_id = connection.execute( + insert(budget_window_runs) + .values( + report_id=report_id, + tax_benefit_model_version_id=public_api_rows.model_version_id, + trigger=ReportRunTrigger.INITIAL, + recalculate=False, + status=ReportRunStatus.PENDING, + ) + .returning(budget_window_runs.c.id) + ).scalar_one() + report_run_id = connection.execute( + insert(report_runs) + .values( + report_id=report_id, + baseline_simulation_run_id=baseline_run_id, + reform_simulation_run_id=reform_run_id, + budget_window_run_id=budget_window_run_id, + year=YEAR, + country_package_version=COUNTRY_PACKAGE_VERSION, + policyengine_version=POLICYENGINE_VERSION, + status=ReportRunStatus.PENDING, + trigger=ReportRunTrigger.INITIAL, + ) + .returning(report_runs.c.id) + ).scalar_one() + household_job_id = connection.execute( + insert(household_jobs) + .values( + country="us", + request_data={}, + status=HouseholdJobStatus.PENDING, + household_id=public_api_rows.household_id, + year=YEAR, + tax_benefit_model_version_id=public_api_rows.model_version_id, + country_package_version=COUNTRY_PACKAGE_VERSION, + policyengine_version=POLICYENGINE_VERSION, + ) + .returning(household_jobs.c.id) + ).scalar_one() + return ServiceRows( + baseline_simulation_id=baseline_simulation_id, + reform_simulation_id=reform_simulation_id, + report_id=report_id, + baseline_run_id=baseline_run_id, + reform_run_id=reform_run_id, + budget_window_run_id=budget_window_run_id, + report_run_id=report_run_id, + household_job_id=household_job_id, + ) diff --git a/libs/policyengine-simulation-persistence/tests/test_access.py b/libs/policyengine-simulation-persistence/tests/test_access.py index a1be6d2ca..f8a9def23 100644 --- a/libs/policyengine-simulation-persistence/tests/test_access.py +++ b/libs/policyengine-simulation-persistence/tests/test_access.py @@ -7,11 +7,14 @@ import pytest from conftest import SERVICE_TEST_ROLE from policyengine_simulation_persistence import ( + MODAL_WORKER_TABLE_PRIVILEGES, + MODAL_WORKER_UPDATABLE_COLUMNS, SERVICE_TABLE_PRIVILEGES, SERVICE_UPDATABLE_COLUMNS, create_simulation_engine, missing_table_privileges, read_missing_table_privileges, + result_metadata, service_metadata, ) from policyengine_simulation_persistence import access as access_module @@ -86,6 +89,65 @@ def test_every_column_the_service_updates_is_modelled() -> None: assert column_names <= set(tables[table_name].columns.keys()), table_name +# The run tables whose status, call and results the Modal worker role records. +MODAL_WORKER_RUN_TABLES = ( + "report_runs", + "simulation_runs", + "household_jobs", + "budget_window_runs", +) +# A run's state, which the Modal call holding it records. +RUN_STATE_COLUMNS = frozenset( + { + "status", + "modal_call_id", + "error_message", + "started_at", + "completed_at", + "updated_at", + } +) + + +def test_the_modal_worker_role_reads_its_runs_and_inserts_their_results() -> None: + result_tables = {table.name for table in result_metadata.tables.values()} + + assert dict(MODAL_WORKER_TABLE_PRIVILEGES) == { + **dict.fromkeys(MODAL_WORKER_RUN_TABLES, frozenset({"SELECT"})), + **dict.fromkeys(result_tables, frozenset({"INSERT"})), + } + + +def test_the_modal_worker_role_updates_its_runs_state_and_results() -> None: + """A call records its run's state and the results the run row holds. + + A budget-window run has no call of its own: the years' coordinators record + its status, failure and times. + """ + assert dict(MODAL_WORKER_UPDATABLE_COLUMNS) == { + "report_runs": RUN_STATE_COLUMNS + | { + "baseline_cliff_gap", + "baseline_cliff_share", + "reform_cliff_gap", + "reform_cliff_share", + "spm_provenance", + "data_version", + }, + "simulation_runs": RUN_STATE_COLUMNS + | {"output_uri", "output_sha256", "household_result"}, + "household_jobs": RUN_STATE_COLUMNS | {"result"}, + "budget_window_runs": RUN_STATE_COLUMNS - {"modal_call_id"}, + } + + +def test_every_column_the_modal_worker_role_updates_is_modelled() -> None: + tables = {table.name: table for table in service_metadata.tables.values()} + + for table_name, column_names in MODAL_WORKER_UPDATABLE_COLUMNS.items(): + assert column_names <= set(tables[table_name].columns.keys()), table_name + + def _required_grants() -> list[tuple[str, ...]]: """Return every grant the service needs, each a tuple. @@ -258,6 +320,39 @@ def test_a_missing_column_lacks_its_privilege(disposable_database): ] +@pytest.mark.postgres +def test_the_check_names_what_a_role_lacks_of_another_roles_lists( + disposable_database, +): + """Given the Modal worker role's lists, the check asks for them instead.""" + _grant( + disposable_database, + [ + "CREATE TABLE public.report_runs " + f"(id uuid PRIMARY KEY, {', '.join(sorted(RUN_STATE_COLUMNS))} text)", + "CREATE TABLE public.poverty (id uuid PRIMARY KEY)", + f"GRANT SELECT ON public.report_runs TO {SERVICE_TEST_ROLE}", + "GRANT UPDATE (status, modal_call_id) ON public.report_runs " + f"TO {SERVICE_TEST_ROLE}", + ], + ) + engine = create_simulation_engine(disposable_database.service_url) + try: + with engine.connect() as connection: + missing = missing_table_privileges( + connection, + table_privileges={ + "report_runs": frozenset({"SELECT"}), + "poverty": frozenset({"INSERT"}), + }, + updatable_columns={"report_runs": frozenset({"status", "started_at"})}, + ) + finally: + engine.dispose() + + assert missing == ["INSERT on poverty", "UPDATE (started_at) on report_runs"] + + def _put_a_decoy_schema_first(database): """Create schema ``decoy``, usable by the role and first on its search_path.""" _grant( diff --git a/libs/policyengine-simulation-persistence/tests/test_modal_worker_writes.py b/libs/policyengine-simulation-persistence/tests/test_modal_worker_writes.py new file mode 100644 index 000000000..44de60c0e --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_modal_worker_writes.py @@ -0,0 +1,190 @@ +"""The Modal worker role writes its runs and results in the Public API's database. + +The v2 Modal coordinators, workers, household functions and stuck-run sweep +connect as ``policyengine_v2_modal_worker`` and write through this library's +models. These tests run against a database the Public API's own v2 migration +chain built, with the privileges its manifest declares granted +(``PUBLIC_API_SCHEMA_DATABASE_URL``; see ``conftest``). Each works in one +transaction and rolls it back. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any +from uuid import uuid4 + +import pytest +from conftest import ( + act_as_the_modal_worker, + act_as_the_service, + rolled_back_transaction, +) +from policyengine_simulation_persistence import ( + MODAL_WORKER_TABLE_PRIVILEGES, + MODAL_WORKER_UPDATABLE_COLUMNS, + budget_window_runs, + household_jobs, + missing_table_privileges, + report_runs, + result_metadata, + simulation_runs, +) +from service_rows import ServiceRows, insert_public_api_rows, insert_service_rows +from sqlalchemy import Enum, Float, Integer, String, func, insert, select, text, update + +if TYPE_CHECKING: + from sqlalchemy import Connection, Table + +pytestmark = pytest.mark.public_api_schema + +NOW = datetime(2026, 11, 16, tzinfo=UTC) +MODAL_CALL_ID = "fc-modal-worker-test" + + +def test_the_modal_worker_role_holds_every_privilege_the_modal_calls_need( + public_api_database_url, +) -> None: + with rolled_back_transaction(public_api_database_url) as connection: + act_as_the_modal_worker(connection) + + assert ( + missing_table_privileges( + connection, + table_privileges=MODAL_WORKER_TABLE_PRIVILEGES, + updatable_columns=MODAL_WORKER_UPDATABLE_COLUMNS, + ) + == [] + ) + + +def _run_state(service_rows: ServiceRows) -> dict[Table, tuple[Any, dict[str, Any]]]: + """Return, by run table, a run's ID and a value for every column the role updates. + + A Python ``None`` in a JSON result is written as one, so the test can read + back that it is stored as SQL NULL. + """ + finished = { + "modal_call_id": MODAL_CALL_ID, + "started_at": NOW, + "completed_at": NOW, + "updated_at": NOW, + } + return { + report_runs: ( + service_rows.report_run_id, + { + **finished, + "status": "succeeded", + "error_message": None, + "baseline_cliff_gap": 1.0, + "baseline_cliff_share": 0.1, + "reform_cliff_gap": 2.0, + "reform_cliff_share": 0.2, + "spm_provenance": None, + "data_version": "test-data-version", + }, + ), + simulation_runs: ( + service_rows.baseline_run_id, + { + **finished, + "status": "succeeded", + "error_message": None, + "output_uri": "gs://test-bucket/simulation-run.parquet", + "output_sha256": "0" * 64, + "household_result": None, + }, + ), + household_jobs: ( + service_rows.household_job_id, + { + **finished, + "status": "failed", + "error_message": "The calculation failed.", + "result": None, + }, + ), + budget_window_runs: ( + service_rows.budget_window_run_id, + { + "status": "running", + "error_message": None, + "started_at": NOW, + "completed_at": None, + "updated_at": NOW, + }, + ), + } + + +def _result_row(table: Table, service_rows: ServiceRows) -> dict[str, Any]: + """Return a row with a value for each of ``table``'s required columns. + + A result of a report run names it and its two simulation runs; a result + of one simulation run names the baseline's. + """ + keys = { + "report_run_id": service_rows.report_run_id, + "baseline_simulation_run_id": service_rows.baseline_run_id, + "reform_simulation_run_id": service_rows.reform_run_id, + "simulation_run_id": service_rows.baseline_run_id, + } + row: dict[str, Any] = {"id": uuid4()} + for column in table.columns: + if column.name in row or column.server_default is not None: + continue + if column.name in keys: + row[column.name] = keys[column.name] + elif column.nullable: + continue + elif isinstance(column.type, Enum): + row[column.name] = column.type.enums[0] + elif isinstance(column.type, String): + row[column.name] = f"test-{column.name}"[: column.type.length] + elif isinstance(column.type, Integer): + # A decile within every table's checks: 1, of 1 quantile. + row[column.name] = 1 + elif isinstance(column.type, Float): + row[column.name] = 0.5 + else: + msg = f"no test value for {table.name}.{column.name}" + raise AssertionError(msg) + return row + + +def _stored_as_sql_null(connection: Connection, table: Table, column_name: str) -> int: + return connection.execute( + select(func.count()) + .select_from(table) + .where(table.c[column_name].is_(None), table.c.modal_call_id == MODAL_CALL_ID) + ).scalar_one() + + +def test_the_modal_worker_role_records_its_runs_and_inserts_every_result( + public_api_database_url, +) -> None: + with rolled_back_transaction(public_api_database_url) as connection: + public_api_rows = insert_public_api_rows(connection) + act_as_the_service(connection) + service_rows = insert_service_rows(connection, public_api_rows) + act_as_the_modal_worker(connection) + + for table, (run_id, values) in _run_state(service_rows).items(): + assert set(values) == MODAL_WORKER_UPDATABLE_COLUMNS[table.name], table + updated = connection.execute( + update(table).where(table.c.id == run_id).values(**values) + ) + assert updated.rowcount == 1, table.name + for table in result_metadata.tables.values(): + connection.execute(insert(table).values(_result_row(table, service_rows))) + + connection.execute(text("RESET ROLE")) + for table in result_metadata.tables.values(): + stored = connection.execute( + select(func.count()).select_from(table) + ).scalar_one() + assert stored >= 1, table.name + assert _stored_as_sql_null(connection, report_runs, "spm_provenance") == 1 + assert _stored_as_sql_null(connection, simulation_runs, "household_result") == 1 + assert _stored_as_sql_null(connection, household_jobs, "result") == 1 diff --git a/libs/policyengine-simulation-persistence/tests/test_service_writes.py b/libs/policyengine-simulation-persistence/tests/test_service_writes.py index fc4c274ce..ec72684a4 100644 --- a/libs/policyengine-simulation-persistence/tests/test_service_writes.py +++ b/libs/policyengine-simulation-persistence/tests/test_service_writes.py @@ -9,11 +9,6 @@ from __future__ import annotations -from dataclasses import dataclass -from hashlib import sha256 -from typing import TYPE_CHECKING -from uuid import UUID, uuid4 - import pytest from conftest import ( act_as_the_service, @@ -21,11 +16,6 @@ rolled_back_transaction, ) from policyengine_simulation_persistence import ( - HouseholdJobStatus, - ReportRunStatus, - ReportRunTrigger, - SimulationStatus, - SimulationType, budget_window_runs, household_jobs, missing_table_privileges, @@ -34,117 +24,11 @@ simulation_runs, simulations, ) -from sqlalchemy import func, insert, select, text - -if TYPE_CHECKING: - from sqlalchemy import Connection +from service_rows import insert_public_api_rows, insert_service_rows +from sqlalchemy import func, select pytestmark = pytest.mark.public_api_schema -COUNTRY_PACKAGE_VERSION = "1.402.0" -POLICYENGINE_VERSION = "3.4.0" -YEAR = 2026 - - -@dataclass(frozen=True) -class PublicApiRows: - """Rows of the Public API's own tables that the service's rows name.""" - - tax_benefit_model_id: UUID - model_version_id: UUID - dataset_id: UUID - household_id: UUID - - -def _insert_public_api_rows(connection: Connection) -> PublicApiRows: - """Insert, as the superuser, the catalog, dataset and household rows a run names. - - The Public API writes these rows, and the database checks on insert that - each row the service writes names ones that exist. - """ - rows = PublicApiRows( - tax_benefit_model_id=uuid4(), - model_version_id=uuid4(), - dataset_id=uuid4(), - household_id=uuid4(), - ) - connection.execute( - text("INSERT INTO public.tax_benefit_models (id, name) VALUES (:id, :name)"), - { - "id": rows.tax_benefit_model_id, - "name": f"test-{rows.tax_benefit_model_id.hex[:12]}", - }, - ) - connection.execute( - text( - "INSERT INTO public.tax_benefit_model_versions " - "(id, model_id, version, current_law_id, metadata_time_periods) " - "VALUES (:id, :model_id, :version, 1, '{}')" - ), - { - "id": rows.model_version_id, - "model_id": rows.tax_benefit_model_id, - "version": COUNTRY_PACKAGE_VERSION, - }, - ) - connection.execute( - text( - "INSERT INTO public.datasets " - "(id, name, year, is_output_dataset, tax_benefit_model_version_id) " - "VALUES (:id, 'test dataset', :year, false, :model_version_id)" - ), - { - "id": rows.dataset_id, - "year": YEAR, - "model_version_id": rows.model_version_id, - }, - ) - connection.execute( - text( - "INSERT INTO public.households " - "(id, household_data, country_id, canonicalization_version, content_hash) " - "VALUES (:id, '{}', 'us', 1, :content_hash)" - ), - { - "id": rows.household_id, - "content_hash": sha256(rows.household_id.bytes).hexdigest(), - }, - ) - return rows - - -def _insert_economy_simulation( - connection: Connection, public_api_rows: PublicApiRows -) -> UUID: - return connection.execute( - insert(simulations) - .values( - simulation_type=SimulationType.ECONOMY, - dataset_id=public_api_rows.dataset_id, - tax_benefit_model_version_id=public_api_rows.model_version_id, - year=YEAR, - ) - .returning(simulations.c.id) - ).scalar_one() - - -def _insert_simulation_run( - connection: Connection, simulation_id: UUID, public_api_rows: PublicApiRows -) -> UUID: - return connection.execute( - insert(simulation_runs) - .values( - simulation_id=simulation_id, - year=YEAR, - tax_benefit_model_version_id=public_api_rows.model_version_id, - country_package_version=COUNTRY_PACKAGE_VERSION, - policyengine_version=POLICYENGINE_VERSION, - dataset_id=public_api_rows.dataset_id, - status=SimulationStatus.PENDING, - ) - .returning(simulation_runs.c.id) - ).scalar_one() - def test_the_service_role_holds_every_privilege_the_service_needs( public_api_database_url, @@ -157,81 +41,21 @@ def test_the_service_role_creates_and_reads_a_row_in_every_table_it_writes( public_api_database_url, ) -> None: with rolled_back_transaction(public_api_database_url) as connection: - public_api_rows = _insert_public_api_rows(connection) + public_api_rows = insert_public_api_rows(connection) act_as_the_service(connection) - baseline_simulation_id = _insert_economy_simulation(connection, public_api_rows) - reform_simulation_id = _insert_economy_simulation(connection, public_api_rows) - report_id = connection.execute( - insert(reports) - .values( - label="Test report", - country="us", - tax_benefit_model_id=public_api_rows.tax_benefit_model_id, - baseline_simulation_id=baseline_simulation_id, - reform_simulation_id=reform_simulation_id, - dataset_id=public_api_rows.dataset_id, - year=YEAR, - budget_window_size=2, - inputs={}, - ) - .returning(reports.c.id) - ).scalar_one() - baseline_run_id = _insert_simulation_run( - connection, baseline_simulation_id, public_api_rows - ) - reform_run_id = _insert_simulation_run( - connection, reform_simulation_id, public_api_rows - ) - budget_window_run_id = connection.execute( - insert(budget_window_runs) - .values( - report_id=report_id, - tax_benefit_model_version_id=public_api_rows.model_version_id, - trigger=ReportRunTrigger.INITIAL, - recalculate=False, - status=ReportRunStatus.PENDING, - ) - .returning(budget_window_runs.c.id) - ).scalar_one() - report_run_id = connection.execute( - insert(report_runs) - .values( - report_id=report_id, - baseline_simulation_run_id=baseline_run_id, - reform_simulation_run_id=reform_run_id, - budget_window_run_id=budget_window_run_id, - year=YEAR, - country_package_version=COUNTRY_PACKAGE_VERSION, - policyengine_version=POLICYENGINE_VERSION, - status=ReportRunStatus.PENDING, - trigger=ReportRunTrigger.INITIAL, - ) - .returning(report_runs.c.id) - ).scalar_one() - connection.execute( - insert(household_jobs).values( - country="us", - request_data={}, - status=HouseholdJobStatus.PENDING, - household_id=public_api_rows.household_id, - year=YEAR, - tax_benefit_model_version_id=public_api_rows.model_version_id, - country_package_version=COUNTRY_PACKAGE_VERSION, - policyengine_version=POLICYENGINE_VERSION, - ) - ) + service_rows = insert_service_rows(connection, public_api_rows) compared_runs = connection.execute( select( report_runs.c.baseline_simulation_run_id, report_runs.c.reform_simulation_run_id, report_runs.c.budget_window_run_id, - ).where(report_runs.c.id == report_run_id) + ).where(report_runs.c.id == service_rows.report_run_id) ).one() assert tuple(compared_runs) == ( - baseline_run_id, - reform_run_id, - budget_window_run_id, + service_rows.baseline_run_id, + service_rows.reform_run_id, + service_rows.budget_window_run_id, ) for table in ( simulations, From 9789b21179b63d5bfaf1eb5d226f461cc4490f14 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:56:44 +0400 Subject: [PATCH 08/25] List the Modal worker role's privileges and check them against the grants `MODAL_WORKER_TABLE_PRIVILEGES` and `MODAL_WORKER_UPDATABLE_COLUMNS` list what the v2 Modal calls need as `policyengine_v2_modal_worker`: SELECT on `report_runs`, `simulation_runs`, `household_jobs` and `budget_window_runs`; UPDATE of each run's state (status, call, failure and times) and of the results its own row holds; and INSERT on every result table. `missing_table_privileges` takes another role's lists, and still checks the simulation service's by default, as the readiness check uses it. Co-Authored-By: Claude Opus 5.5 --- .../__init__.py | 4 + .../access.py | 98 +++++++++++++++---- .../tests/test_access.py | 7 +- 3 files changed, 90 insertions(+), 19 deletions(-) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py index dbeae927d..0be095120 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py @@ -1,6 +1,8 @@ """Connections to the Public API's database for the simulation service.""" from policyengine_simulation_persistence.access import ( + MODAL_WORKER_TABLE_PRIVILEGES, + MODAL_WORKER_UPDATABLE_COLUMNS, SERVICE_TABLE_PRIVILEGES, SERVICE_UPDATABLE_COLUMNS, missing_table_privileges, @@ -156,6 +158,8 @@ "HOUSEHOLD_REPORT_TYPE", "LATEST_YEAR", "MAX_BUDGET_WINDOW_YEARS", + "MODAL_WORKER_TABLE_PRIVILEGES", + "MODAL_WORKER_UPDATABLE_COLUMNS", "SERVICE_TABLE_PRIVILEGES", "SERVICE_UPDATABLE_COLUMNS", "BudgetWindowInputs", diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/access.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/access.py index bafb7799e..6b2216ac8 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/access.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/access.py @@ -1,9 +1,11 @@ -"""The privileges the simulation service's role needs on the Public API's tables. +"""The privileges sim's two roles need on the Public API's tables. -The Public API's privilege manifest grants the role these privileges, and -its migration chain creates the tables. The readiness check reports any the -connected role lacks, which shows that the deployment's database URL and the -role's grants both reached the running service. +The Public API's privilege manifest grants each role these privileges, and +its migration chain creates the tables. The simulation service's readiness +check reports any its role lacks, which shows that the deployment's database +URL and the role's grants both reached the running service. The v2 Modal +calls' role, ``policyengine_v2_modal_worker``, has lists of its own, which +CI checks against the manifest's grants. """ from __future__ import annotations @@ -23,6 +25,7 @@ values, ) +from policyengine_simulation_persistence.result_tables import result_metadata from policyengine_simulation_persistence.tables import TABLE_SCHEMA if TYPE_CHECKING: @@ -98,6 +101,53 @@ } ) +# A run's state, which the Modal call holding it records: its claim, its +# outcome and when each happened. +_RUN_STATE: Final = frozenset( + { + "status", + "modal_call_id", + "error_message", + "started_at", + "completed_at", + "updated_at", + } +) +# The v2 Modal coordinators, workers, household functions and stuck-run sweep +# read the runs they were given and insert their results, once each. +MODAL_WORKER_TABLE_PRIVILEGES: Final[Mapping[str, frozenset[str]]] = MappingProxyType( + { + "report_runs": _READ, + "simulation_runs": _READ, + "household_jobs": _READ, + "budget_window_runs": _READ, + **{ + table.name: frozenset({"INSERT"}) + for table in result_metadata.tables.values() + }, + } +) +# The columns the Modal worker role may UPDATE, by table: each run's state and +# the results its own row holds. A budget-window run has no Modal call of its +# own; its years' coordinators record its status, failure and times. +MODAL_WORKER_UPDATABLE_COLUMNS: Final[Mapping[str, frozenset[str]]] = MappingProxyType( + { + "report_runs": _RUN_STATE + | { + "baseline_cliff_gap", + "baseline_cliff_share", + "reform_cliff_gap", + "reform_cliff_share", + "spm_provenance", + "data_version", + }, + "simulation_runs": _RUN_STATE + | {"output_uri", "output_sha256", "household_result"}, + "household_jobs": _RUN_STATE | {"result"}, + "budget_window_runs": _RUN_STATE - {"modal_call_id"}, + } +) + def _public_tables() -> Subquery: """Select the oid and name of each table in the ``public`` schema.""" @@ -114,7 +164,9 @@ def _public_tables() -> Subquery: ) -def _missing_table_grants(connection: Connection) -> list[tuple[str, str, str]]: +def _missing_table_grants( + connection: Connection, table_privileges: Mapping[str, frozenset[str]] +) -> list[tuple[str, str, str]]: """Return ``(table, privilege, "")`` for each table privilege the role lacks.""" required = values( column("table_name", String), @@ -123,7 +175,7 @@ def _missing_table_grants(connection: Connection) -> list[tuple[str, str, str]]: ).data( [ (table_name, privilege) - for table_name, privileges in SERVICE_TABLE_PRIVILEGES.items() + for table_name, privileges in table_privileges.items() for privilege in sorted(privileges) ] ) @@ -146,7 +198,9 @@ def _missing_table_grants(connection: Connection) -> list[tuple[str, str, str]]: return [(table_name, privilege, "") for table_name, privilege in rows] -def _missing_column_grants(connection: Connection) -> list[tuple[str, str, str]]: +def _missing_column_grants( + connection: Connection, updatable_columns: Mapping[str, frozenset[str]] +) -> list[tuple[str, str, str]]: """Return ``(table, "UPDATE", column)`` for each column the role can't update.""" required = values( column("table_name", String), @@ -155,7 +209,7 @@ def _missing_column_grants(connection: Connection) -> list[tuple[str, str, str]] ).data( [ (table_name, column_name) - for table_name, column_names in SERVICE_UPDATABLE_COLUMNS.items() + for table_name, column_names in updatable_columns.items() for column_name in sorted(column_names) ] ) @@ -189,18 +243,28 @@ def _missing_column_grants(connection: Connection) -> list[tuple[str, str, str]] return [(table_name, "UPDATE", column_name) for table_name, column_name in rows] -def missing_table_privileges(connection: Connection) -> list[str]: +def missing_table_privileges( + connection: Connection, + *, + table_privileges: Mapping[str, frozenset[str]] = SERVICE_TABLE_PRIVILEGES, + updatable_columns: Mapping[str, frozenset[str]] = SERVICE_UPDATABLE_COLUMNS, +) -> list[str]: """Return each required privilege the connected role lacks, ordered by table. - A table privilege is named as ``" on "`` and a column - privilege as ``"UPDATE () on
"``. A table missing from the - ``public`` schema lacks every privilege the service needs on it, and a - missing column lacks its own. A role without the USAGE privilege on that - schema can't use any of its grants, so ``"USAGE on schema public"`` comes - first. + The required privileges are the simulation service's unless another + role's lists are given, such as ``MODAL_WORKER_TABLE_PRIVILEGES`` and + ``MODAL_WORKER_UPDATABLE_COLUMNS``. A table privilege is named as + ``" on
"`` and a column privilege as + ``"UPDATE () on
"``. A table missing from the ``public`` + schema lacks every privilege required on it, and a missing column lacks + its own. A role without the USAGE privilege on that schema can't use any + of its grants, so ``"USAGE on schema public"`` comes first. """ missing_grants = sorted( - [*_missing_table_grants(connection), *_missing_column_grants(connection)] + [ + *_missing_table_grants(connection, table_privileges), + *_missing_column_grants(connection, updatable_columns), + ] ) missing = [ f"{privilege} ({column_name}) on {table_name}" diff --git a/libs/policyengine-simulation-persistence/tests/test_access.py b/libs/policyengine-simulation-persistence/tests/test_access.py index f8a9def23..e6740422a 100644 --- a/libs/policyengine-simulation-persistence/tests/test_access.py +++ b/libs/policyengine-simulation-persistence/tests/test_access.py @@ -328,8 +328,11 @@ def test_the_check_names_what_a_role_lacks_of_another_roles_lists( _grant( disposable_database, [ - "CREATE TABLE public.report_runs " - f"(id uuid PRIMARY KEY, {', '.join(sorted(RUN_STATE_COLUMNS))} text)", + "CREATE TABLE public.report_runs (id uuid PRIMARY KEY, " + + ", ".join( + f"{column_name} text" for column_name in sorted(RUN_STATE_COLUMNS) + ) + + ")", "CREATE TABLE public.poverty (id uuid PRIMARY KEY)", f"GRANT SELECT ON public.report_runs TO {SERVICE_TEST_ROLE}", "GRANT UPDATE (status, modal_call_id) ON public.report_runs " From 8fd1b18c7e919cc9d8aaaae3105dc0b91849eb58 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:58:15 +0400 Subject: [PATCH 09/25] Test that the v2 Modal images carry the service library and v1 code doesn't (failing) The Stage 12 library's stores now import the service library's table models, so every v2 Modal image (the workers', the coordinator's, the household functions', the stuck-run sweep's and the Stage 12 precompute's) must copy `policyengine_simulation_persistence` with the other project sources. The v1 `run_simulation` code stays apart from both persistence libraries: a new test follows `src/modal/app.py`'s imports through this project's modules, and reads the old gateway's source, and finds neither library imported. Co-Authored-By: Claude Opus 5.5 --- .../test_runtime_evaluated_annotations.py | 1 + .../tests/test_stage12_modal_app.py | 9 +- .../tests/test_stage12_precompute_app.py | 1 + .../tests/test_stage12_run_sweep_app.py | 1 + .../test_v1_code_imports_no_table_models.py | 131 ++++++++++++++++++ 5 files changed, 141 insertions(+), 2 deletions(-) create mode 100644 projects/policyengine-simulation-executor/tests/test_v1_code_imports_no_table_models.py diff --git a/projects/policyengine-simulation-executor/tests/test_runtime_evaluated_annotations.py b/projects/policyengine-simulation-executor/tests/test_runtime_evaluated_annotations.py index 0a4864937..a6af2feb5 100644 --- a/projects/policyengine-simulation-executor/tests/test_runtime_evaluated_annotations.py +++ b/projects/policyengine-simulation-executor/tests/test_runtime_evaluated_annotations.py @@ -43,6 +43,7 @@ "policyengine_simulation_contract", "policyengine_simulation_executor", "policyengine_simulation_observability", + "policyengine_simulation_persistence", "policyengine_stage12_persistence", ) # src/modal holds the Modal entry files. They are not part of the installed diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py b/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py index e3f6ef998..e0b4e3a15 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py @@ -146,19 +146,24 @@ def test_v2_app_name_and_images_are_separate_and_bundle_derived(monkeypatch) -> module.us_worker_image, module.uk_worker_image, module.coordinator_image, + module.household_image, ): local_source = next( call for call in image.calls if call[0] == "add_local_python_source" ) - bundle_install = next(call for call in image.calls if call[0] == "run_commands") - assert bundle_install[2]["secrets"] == [module.hf_secret] + # The Stage 12 library's stores use the service library's table models. assert local_source[1] == ( "src.modal", "policyengine_simulation_executor", "policyengine_simulation_observability", "policyengine_simulation_contract", + "policyengine_simulation_persistence", "policyengine_stage12_persistence", ) + if image is module.household_image: + continue + bundle_install = next(call for call in image.calls if call[0] == "run_commands") + assert bundle_install[2]["secrets"] == [module.hf_secret] static_runtime_files = next( call for call in image.calls if call[0] == "add_local_dir" ) diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_precompute_app.py b/projects/policyengine-simulation-executor/tests/test_stage12_precompute_app.py index c260afda0..44a95cd53 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_precompute_app.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_precompute_app.py @@ -58,6 +58,7 @@ def test_precompute_image_installs_only_the_us_bundle(monkeypatch) -> None: ) assert "policyengine_simulation_executor" in local_source[1] assert "policyengine_stage12_persistence" in local_source[1] + assert "policyengine_simulation_persistence" in local_source[1] def test_precompute_image_refreshes_ca_certificates_before_installing_packages( diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep_app.py b/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep_app.py index db32d645a..a8a25e038 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep_app.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep_app.py @@ -116,6 +116,7 @@ def test_the_sweep_image_installs_the_locked_group_and_the_runtime_sources( "policyengine_simulation_executor", "policyengine_simulation_observability", "policyengine_simulation_contract", + "policyengine_simulation_persistence", "policyengine_stage12_persistence", ) diff --git a/projects/policyengine-simulation-executor/tests/test_v1_code_imports_no_table_models.py b/projects/policyengine-simulation-executor/tests/test_v1_code_imports_no_table_models.py new file mode 100644 index 000000000..5229968a9 --- /dev/null +++ b/projects/policyengine-simulation-executor/tests/test_v1_code_imports_no_table_models.py @@ -0,0 +1,131 @@ +"""The v1 ``run_simulation`` code imports neither persistence library. + +The simulation service and the v2 Modal runtime share one library of the +Public API's table models, ``policyengine-simulation-persistence``, which the +Stage 12 library builds on. The v1 code, which will be removed, stays apart +from both: the old gateway, and the v1 Modal app (``src/modal/app.py``) with +every module of this project it imports, however indirectly and wherever in +the module the import is. +""" + +from __future__ import annotations + +import ast +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +SOURCE_ROOT = PROJECT_ROOT / "src" +GATEWAY_SOURCE_ROOT = PROJECT_ROOT.parent / "policyengine-simulation-gateway" / "src" +V1_MODAL_APP = SOURCE_ROOT / "modal" / "app.py" +# Modules of this project, found as files under src. +PROJECT_PACKAGES = ("src.modal", "policyengine_simulation_executor") +TABLE_MODEL_LIBRARIES = ( + "policyengine_simulation_persistence", + "policyengine_stage12_persistence", +) + + +def _module_name(path: Path) -> str: + parts = path.relative_to(SOURCE_ROOT).with_suffix("").parts + if parts[0] == "modal": + parts = ("src", *parts) + if parts[-1] == "__init__": + parts = parts[:-1] + return ".".join(parts) + + +def _module_path(module_name: str) -> Path | None: + """Return the file of one of this project's modules, or None for another's.""" + if not module_name.startswith(PROJECT_PACKAGES): + return None + parts = module_name.split(".") + if parts[0] == "src": + parts = parts[1:] + candidate = SOURCE_ROOT.joinpath(*parts) + for path in (candidate.with_suffix(".py"), candidate / "__init__.py"): + if path.is_file(): + return path + return None + + +def _imported_modules(path: Path) -> set[str]: + """Return every module ``path`` imports, and each package it imports from.""" + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + package = _module_name(path) + if path.name != "__init__.py": + package = package.rpartition(".")[0] + imported: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + base = node.module or "" + if node.level: + anchor = ( + package.rsplit(".", node.level - 1)[0] + if node.level > 1 + else package + ) + base = f"{anchor}.{base}" if base else anchor + imported.add(base) + # ``from package import module`` imports a submodule too. + imported.update(f"{base}.{alias.name}" for alias in node.names) + return imported + + +def _v1_modal_imports() -> dict[str, set[str]]: + """Return, by module, what the v1 Modal app and its project modules import.""" + pending = [V1_MODAL_APP] + imports_by_module: dict[str, set[str]] = {} + while pending: + path = pending.pop() + module_name = _module_name(path) + if module_name in imports_by_module: + continue + imported = _imported_modules(path) + imports_by_module[module_name] = imported + pending.extend( + module_path + for name in imported + if (module_path := _module_path(name)) is not None + ) + return imports_by_module + + +def _table_model_imports(imports_by_module: dict[str, set[str]]) -> list[str]: + return sorted( + f"{module_name}: {name}" + for module_name, imported in imports_by_module.items() + for name in imported + if name.split(".")[0] in TABLE_MODEL_LIBRARIES + ) + + +def test_the_v1_modal_app_and_its_modules_import_no_table_model_library() -> None: + imports_by_module = _v1_modal_imports() + + # The walk follows the app's own imports, not only the file itself. + assert "src.modal._image_setup" in imports_by_module + assert _table_model_imports(imports_by_module) == [] + + +def test_the_old_gateway_imports_no_table_model_library() -> None: + gateway_files = sorted(GATEWAY_SOURCE_ROOT.rglob("*.py")) + imports_by_module = { + str(path.relative_to(GATEWAY_SOURCE_ROOT)): _gateway_imports(path) + for path in gateway_files + } + + assert gateway_files + assert _table_model_imports(imports_by_module) == [] + + +def _gateway_imports(path: Path) -> set[str]: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + imported: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and not node.level: + imported.add(node.module or "") + return imported From f566202abd866a65b4807e577c46d4e7844d06c7 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:59:29 +0400 Subject: [PATCH 10/25] Copy the service library into every v2 Modal image The executor depends on `policyengine-simulation-persistence` directly, and the v2 app, the stuck-run sweep and the Stage 12 precompute copy it with the other project sources, since the Stage 12 library's stores use its table models. The images install no new third-party package: SQLAlchemy and psycopg are already in the `modal-simulation-image` group, and the executor's lock keeps SQLAlchemy 2.0.49, within the service library's `<2.1`. Co-Authored-By: Claude Opus 5.5 --- projects/policyengine-simulation-executor/pyproject.toml | 4 ++++ .../src/modal/run_sweep_app.py | 1 + .../src/modal/stage12_precompute_app.py | 1 + projects/policyengine-simulation-executor/src/modal/v2_app.py | 1 + projects/policyengine-simulation-executor/uv.lock | 2 ++ 5 files changed, 9 insertions(+) diff --git a/projects/policyengine-simulation-executor/pyproject.toml b/projects/policyengine-simulation-executor/pyproject.toml index 2b4f42ac6..7f4dfbbc1 100644 --- a/projects/policyengine-simulation-executor/pyproject.toml +++ b/projects/policyengine-simulation-executor/pyproject.toml @@ -18,6 +18,9 @@ dependencies = [ "policyengine-fastapi", "policyengine-simulation-observability", "policyengine-simulation-contract[modal]", + # The models of the Public API's tables the v2 Modal calls write; the + # v2 images copy it with the Stage 12 library, whose stores use them. + "policyengine-simulation-persistence", "policyengine-stage12-persistence", # PolicyEngine.py's models extra is the only selector for core, country # models, and the SPM calculator. Do not duplicate those versions here. @@ -79,6 +82,7 @@ dev = [ policyengine-fastapi = { path = "../../libs/policyengine-fastapi", editable = true } policyengine-simulation-observability = { path = "../../libs/policyengine-simulation-observability", editable = true } policyengine-simulation-contract = { path = "../../libs/policyengine-simulation-contract", editable = true } +policyengine-simulation-persistence = { path = "../../libs/policyengine-simulation-persistence", editable = true } policyengine-stage12-persistence = { path = "../../libs/policyengine-stage12-persistence", editable = true } policyengine-simulation-gateway = { path = "../policyengine-simulation-gateway", editable = true } diff --git a/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py b/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py index bb566ea51..04381c48c 100644 --- a/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py +++ b/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py @@ -65,6 +65,7 @@ "policyengine_simulation_executor", "policyengine_simulation_observability", "policyengine_simulation_contract", + "policyengine_simulation_persistence", "policyengine_stage12_persistence", copy=True, ) diff --git a/projects/policyengine-simulation-executor/src/modal/stage12_precompute_app.py b/projects/policyengine-simulation-executor/src/modal/stage12_precompute_app.py index c8332e576..b0182e537 100644 --- a/projects/policyengine-simulation-executor/src/modal/stage12_precompute_app.py +++ b/projects/policyengine-simulation-executor/src/modal/stage12_precompute_app.py @@ -76,6 +76,7 @@ "policyengine_simulation_executor", "policyengine_simulation_observability", "policyengine_simulation_contract", + "policyengine_simulation_persistence", "policyengine_stage12_persistence", copy=True, ) diff --git a/projects/policyengine-simulation-executor/src/modal/v2_app.py b/projects/policyengine-simulation-executor/src/modal/v2_app.py index ea7b4b5be..b7f824ad1 100644 --- a/projects/policyengine-simulation-executor/src/modal/v2_app.py +++ b/projects/policyengine-simulation-executor/src/modal/v2_app.py @@ -196,6 +196,7 @@ def _add_v2_sources(image: modal.Image) -> modal.Image: "policyengine_simulation_executor", "policyengine_simulation_observability", "policyengine_simulation_contract", + "policyengine_simulation_persistence", "policyengine_stage12_persistence", copy=True, ) diff --git a/projects/policyengine-simulation-executor/uv.lock b/projects/policyengine-simulation-executor/uv.lock index dbb6996da..3323e149b 100644 --- a/projects/policyengine-simulation-executor/uv.lock +++ b/projects/policyengine-simulation-executor/uv.lock @@ -1906,6 +1906,7 @@ dependencies = [ { name = "policyengine-observability", extra = ["fastapi", "google", "otlp-grpc"] }, { name = "policyengine-simulation-contract", extra = ["modal"] }, { name = "policyengine-simulation-observability" }, + { name = "policyengine-simulation-persistence" }, { name = "policyengine-stage12-persistence" }, { name = "pyarrow" }, { name = "pydantic-settings" }, @@ -1958,6 +1959,7 @@ requires-dist = [ { name = "policyengine-observability", extras = ["fastapi", "google", "otlp-grpc"], specifier = ">=3.0.7,<4" }, { name = "policyengine-simulation-contract", extras = ["modal"], editable = "../../libs/policyengine-simulation-contract" }, { name = "policyengine-simulation-observability", editable = "../../libs/policyengine-simulation-observability" }, + { name = "policyengine-simulation-persistence", editable = "../../libs/policyengine-simulation-persistence" }, { name = "policyengine-stage12-persistence", editable = "../../libs/policyengine-stage12-persistence" }, { name = "pyarrow", specifier = ">=20,<24" }, { name = "pydantic-settings", specifier = ">=2.7.1,<3.0.0" }, From 44d8945b37713eee609132c4942ca59f6c838bce Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:59:29 +0400 Subject: [PATCH 11/25] Test that the image smoke runs when the service library changes (failing) The v2 images copy `policyengine-simulation-persistence` now, so a change to it must build and import them again, as a change to the other libraries they copy does. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_modal_scripts.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/projects/policyengine-simulation-executor/tests/test_modal_scripts.py b/projects/policyengine-simulation-executor/tests/test_modal_scripts.py index 4980507e7..c2c4bdd7e 100644 --- a/projects/policyengine-simulation-executor/tests/test_modal_scripts.py +++ b/projects/policyengine-simulation-executor/tests/test_modal_scripts.py @@ -309,6 +309,24 @@ def test_workflow_supplies_hf_token(self): workflow.index("Run image smokes") ) + def test_workflow_runs_when_a_library_the_v2_images_copy_changes(self): + """A change to a library the v2 images copy builds and imports them again. + + The Stage 12 library's stores use the simulation service library's + table models, so the images copy both. + """ + workflow = ( + REPO_ROOT / ".github" / "workflows" / "pr-image-smoke.yml" + ).read_text(encoding="utf-8") + + for library in ( + "policyengine-simulation-contract", + "policyengine-simulation-observability", + "policyengine-simulation-persistence", + "policyengine-stage12-persistence", + ): + assert f"- 'libs/{library}/**'" in workflow, library + def test_deployment_sources_exclude_legacy_hf_secret_names(self): paths = ( REPO_ROOT / ".github/scripts/modal-image-smoke.sh", From 0f36f49cf8daab54c207556ffbf952edc6f3fc4b Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:59:47 +0400 Subject: [PATCH 12/25] Run the image smoke when the service library changes The v2 Modal images copy `policyengine-simulation-persistence`, so a pull request that changes it builds them and imports their entry points again. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/pr-image-smoke.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-image-smoke.yml b/.github/workflows/pr-image-smoke.yml index 7755e947b..7dccf79b3 100644 --- a/.github/workflows/pr-image-smoke.yml +++ b/.github/workflows/pr-image-smoke.yml @@ -18,6 +18,7 @@ on: - 'libs/policyengine-simulation-contract/**' - 'libs/policyengine-simulation-observability/**' - 'libs/policyengine-fastapi/**' + - 'libs/policyengine-simulation-persistence/**' - 'libs/policyengine-stage12-persistence/**' - '.github/workflows/pr-image-smoke.yml' - '.github/scripts/modal-image-smoke.sh' From 7044bfae9276e8159d842dd64b30d7385133a034 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:01:39 +0400 Subject: [PATCH 13/25] Test that the Public API schema job runs the Stage 12 tests as the worker (failing) After the comparison, the job is to give `policyengine_v2_modal_worker` a login on the database it built, with a password that exists only there, and run the Stage 12 library's tests against it as that role, with the superuser seeding the service's rows. A new script gives the login; the Stage 12 tests are to fail instead of skipping when the job requires their database (`REPORT_RUN_REQUIRE_DATABASE_TESTS=1`). Co-Authored-By: Claude Opus 5.5 --- .../tests/test_public_api_schema_job.py | 50 +++++++++++++ .../tests/test_public_api_schema_scripts.py | 74 ++++++++++++++++++- .../tests/test_database_requirement.py | 36 +++++++++ 3 files changed, 158 insertions(+), 2 deletions(-) create mode 100644 libs/policyengine-stage12-persistence/tests/test_database_requirement.py diff --git a/libs/policyengine-simulation-persistence/tests/test_public_api_schema_job.py b/libs/policyengine-simulation-persistence/tests/test_public_api_schema_job.py index 17153f30b..f3cb68a4a 100644 --- a/libs/policyengine-simulation-persistence/tests/test_public_api_schema_job.py +++ b/libs/policyengine-simulation-persistence/tests/test_public_api_schema_job.py @@ -95,6 +95,56 @@ def test_the_job_builds_the_reference_and_compares_without_skipping() -> None: ) +STAGE12_LIBRARY = "libs/policyengine-stage12-persistence" +MODAL_WORKER_ROLE = "policyengine_v2_modal_worker" + + +def test_the_job_runs_the_stage12_tests_as_the_modal_worker_role() -> None: + """The Stage 12 stores' tests run on the same tables, as the role they use. + + The Public API's manifest creates the role without a login, so a step + gives it one, with a password that exists only in the job's database. + """ + job = _jobs()[JOB_ID] + password = job["services"]["postgres"]["env"]["POSTGRES_PASSWORD"] + names = [step.get("name") for step in job["steps"]] + compare = _step(job, "Compare sim's table models with the Public API's tables") + login = _step(job, "Give the Modal worker role a login on the job's database") + install = _step(job, "Install the Stage 12 library's dependencies") + stage12_tests = _step( + job, "Run the Stage 12 library's tests as the Modal worker role" + ) + on_default_branch = _step( + job, "Require the pinned commit on the Public API's default branch" + ) + worker_password = login["env"]["MODAL_WORKER_TEST_PASSWORD"] + + assert login["run"] == ".github/scripts/public-api-schema-worker-login.sh" + assert install["working-directory"] == STAGE12_LIBRARY + assert install["run"] == "uv sync --extra test --frozen" + assert stage12_tests["working-directory"] == STAGE12_LIBRARY + assert stage12_tests["run"] == "uv run --frozen pytest tests/ -v" + assert stage12_tests["env"] == { + "REPORT_RUN_TEST_DATABASE_URL": ( + f"postgresql://{MODAL_WORKER_ROLE}:{worker_password}@127.0.0.1:5432/" + "policyengine_v2_alembic_test" + ), + "REPORT_RUN_TEST_SEED_DATABASE_URL": ( + f"postgresql+psycopg://postgres:{password}@127.0.0.1:5432/" + "policyengine_v2_alembic_test" + ), + # A missing database fails the tests instead of skipping them. + "REPORT_RUN_REQUIRE_DATABASE_TESTS": "1", + } + assert ( + names.index(compare["name"]) + < names.index(login["name"]) + < names.index(install["name"]) + < names.index(stage12_tests["name"]) + < names.index(on_default_branch["name"]) + ) + + def test_the_job_reads_no_secret() -> None: job_text = yaml.safe_dump(_jobs()[JOB_ID]) diff --git a/libs/policyengine-simulation-persistence/tests/test_public_api_schema_scripts.py b/libs/policyengine-simulation-persistence/tests/test_public_api_schema_scripts.py index d973f4464..392a903b5 100644 --- a/libs/policyengine-simulation-persistence/tests/test_public_api_schema_scripts.py +++ b/libs/policyengine-simulation-persistence/tests/test_public_api_schema_scripts.py @@ -2,8 +2,10 @@ ``public-api-schema-commit.sh`` reads the pinned Public API commit, ``public-api-commit-on-default-branch.sh`` requires it to be on the Public -API's default branch, and ``public-api-schema-reference.sh`` builds the -Public API's tables at that commit and grants its declared privileges. +API's default branch, ``public-api-schema-reference.sh`` builds the +Public API's tables at that commit and grants its declared privileges, and +``public-api-schema-worker-login.sh`` gives the Modal worker role a login there, +for the Stage 12 library's tests. """ from __future__ import annotations @@ -20,6 +22,7 @@ COMMIT_SCRIPT = SCRIPTS / "public-api-schema-commit.sh" DEFAULT_BRANCH_SCRIPT = SCRIPTS / "public-api-commit-on-default-branch.sh" REFERENCE_SCRIPT = SCRIPTS / "public-api-schema-reference.sh" +WORKER_LOGIN_SCRIPT = SCRIPTS / "public-api-schema-worker-login.sh" PINNED_COMMIT = "0123456789abcdef0123456789abcdef01234567" DATABASE_URL = ( "postgresql://postgres:simulation_ci@127.0.0.1:5432/policyengine_v2_alembic_test" @@ -237,3 +240,70 @@ def test_the_reference_needs_a_database_to_migrate(tmp_path): assert result.returncode != 0 assert "PUBLIC_API_SCHEMA_DATABASE_URL" in result.stderr assert not [line for line in commands if line.startswith("uv|")] + + +# --- The Modal worker role's login ---------------------------------------- + + +def _fake_psql(tmp_path, *, exit_status: int = 0) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir(exist_ok=True) + # The script sends its statement on standard input; the fake logs it. + _fake_command( + fake_bin, + "psql", + f'cat >> "${{FAKE_COMMAND_LOG}}"\nexit {exit_status}', + ) + + +def test_the_worker_role_is_given_a_login_with_the_tests_password(tmp_path): + _fake_psql(tmp_path) + + result, log = _run( + WORKER_LOGIN_SCRIPT, + [], + tmp_path, + PUBLIC_API_SCHEMA_DATABASE_URL=DATABASE_URL, + MODAL_WORKER_TEST_PASSWORD="modal_worker_ci", + ) + + assert result.returncode == 0, result.stderr + assert log == [ + f"psql|{DATABASE_URL} --no-psqlrc --set=ON_ERROR_STOP=1 " + "--set=password=modal_worker_ci", + "ALTER ROLE policyengine_v2_modal_worker LOGIN PASSWORD :'password';", + ] + # The password is passed as a psql variable, never written into the SQL. + assert "modal_worker_ci" not in log[1] + + +@pytest.mark.parametrize( + "missing", ["PUBLIC_API_SCHEMA_DATABASE_URL", "MODAL_WORKER_TEST_PASSWORD"] +) +def test_the_login_needs_the_database_and_the_password(tmp_path, missing): + _fake_psql(tmp_path) + environment = { + "PUBLIC_API_SCHEMA_DATABASE_URL": DATABASE_URL, + "MODAL_WORKER_TEST_PASSWORD": "modal_worker_ci", + } + del environment[missing] + + result, log = _run(WORKER_LOGIN_SCRIPT, [], tmp_path, **environment) + + assert result.returncode != 0 + assert missing in result.stderr + assert log == [] + + +def test_a_refused_login_fails_the_step(tmp_path): + _fake_psql(tmp_path, exit_status=3) + + result, _ = _run( + WORKER_LOGIN_SCRIPT, + [], + tmp_path, + PUBLIC_API_SCHEMA_DATABASE_URL=DATABASE_URL, + MODAL_WORKER_TEST_PASSWORD="modal_worker_ci", + ) + + assert result.returncode == 3 diff --git a/libs/policyengine-stage12-persistence/tests/test_database_requirement.py b/libs/policyengine-stage12-persistence/tests/test_database_requirement.py new file mode 100644 index 000000000..5867bc24a --- /dev/null +++ b/libs/policyengine-stage12-persistence/tests/test_database_requirement.py @@ -0,0 +1,36 @@ +"""The report run tests skip without their database, unless a job requires it. + +The Public API schema job sets ``REPORT_RUN_REQUIRE_DATABASE_TESTS=1``, so a +database it failed to name fails the tests there instead of skipping them. +""" + +from __future__ import annotations + +import pytest +from report_run_database import ( + REQUIRE_DATABASE_ENV, + SEED_URL_ENV, + WORKER_URL_ENV, + worker_database_url, +) + + +def _without_test_databases(monkeypatch) -> None: + monkeypatch.delenv(WORKER_URL_ENV, raising=False) + monkeypatch.delenv(SEED_URL_ENV, raising=False) + + +def test_the_tests_skip_without_a_database(monkeypatch) -> None: + _without_test_databases(monkeypatch) + monkeypatch.delenv(REQUIRE_DATABASE_ENV, raising=False) + + with pytest.raises(pytest.skip.Exception, match=WORKER_URL_ENV): + worker_database_url() + + +def test_a_job_that_requires_the_database_fails_without_one(monkeypatch) -> None: + _without_test_databases(monkeypatch) + monkeypatch.setenv(REQUIRE_DATABASE_ENV, "1") + + with pytest.raises(pytest.fail.Exception, match=WORKER_URL_ENV): + worker_database_url() From ac0e723546978078b3481bae646ea6e6f58dac49 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:02:36 +0400 Subject: [PATCH 14/25] Run the Stage 12 library's tests in the Public API schema job After the comparison, the job gives `policyengine_v2_modal_worker` a login on the database it built (`public-api-schema-worker-login.sh`, with a password that exists only in that database), then runs the Stage 12 library's tests as that role, the superuser seeding the service's rows. Until now no workflow set their database URLs, so they skipped in CI; here a missing database fails them (`REPORT_RUN_REQUIRE_DATABASE_TESTS=1`). The new step variables are literals in `pr.yml`, naming the job's own PostgreSQL service; the job still reads no secret. Co-Authored-By: Claude Opus 5.5 --- .../scripts/public-api-schema-worker-login.sh | 21 +++++++++++++++++++ .github/workflows/pr.yml | 21 +++++++++++++++++++ .../tests/report_run_database.py | 16 +++++++++++--- 3 files changed, 55 insertions(+), 3 deletions(-) create mode 100755 .github/scripts/public-api-schema-worker-login.sh diff --git a/.github/scripts/public-api-schema-worker-login.sh b/.github/scripts/public-api-schema-worker-login.sh new file mode 100755 index 000000000..6e86c604e --- /dev/null +++ b/.github/scripts/public-api-schema-worker-login.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Give the Public API's Modal worker role a login on the disposable database +# the Public API schema job built, so the Stage 12 library's tests connect as +# that role (rule sim.persistence.models-match-public-api). +# +# Usage: public-api-schema-worker-login.sh +# +# PUBLIC_API_SCHEMA_DATABASE_URL names that database, as the superuser's +# postgresql:// URL; the Public API's privilege manifest creates its roles +# without a login. MODAL_WORKER_TEST_PASSWORD is the password the tests' URL +# gives, which exists only in the job's database. +set -euo pipefail + +database_url="${PUBLIC_API_SCHEMA_DATABASE_URL:?PUBLIC_API_SCHEMA_DATABASE_URL must name the database the job built}" +password="${MODAL_WORKER_TEST_PASSWORD:?MODAL_WORKER_TEST_PASSWORD must give the password the Modal worker role logs in with}" + +# psql quotes the password variable as a literal (:'password'), so it never +# appears in the statement itself. +psql "${database_url}" --no-psqlrc --set=ON_ERROR_STOP=1 "--set=password=${password}" <<'SQL' +ALTER ROLE policyengine_v2_modal_worker LOGIN PASSWORD :'password'; +SQL diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 068690334..90347c0cc 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -213,6 +213,27 @@ jobs: working-directory: libs/policyengine-simulation-persistence run: uv run --frozen pytest tests/ -v -m public_api_schema + # The Stage 12 library's stores write the same tables as the Modal worker + # role, which the manifest creates without a login: the password exists + # only in this job's database. + - name: Give the Modal worker role a login on the job's database + env: + MODAL_WORKER_TEST_PASSWORD: modal_worker_ci + run: .github/scripts/public-api-schema-worker-login.sh + + - name: Install the Stage 12 library's dependencies + working-directory: libs/policyengine-stage12-persistence + run: uv sync --extra test --frozen + + - name: Run the Stage 12 library's tests as the Modal worker role + working-directory: libs/policyengine-stage12-persistence + env: + REPORT_RUN_TEST_DATABASE_URL: postgresql://policyengine_v2_modal_worker:modal_worker_ci@127.0.0.1:5432/policyengine_v2_alembic_test + REPORT_RUN_TEST_SEED_DATABASE_URL: postgresql+psycopg://postgres:simulation_ci@127.0.0.1:5432/policyengine_v2_alembic_test + # A missing database fails the tests instead of skipping them. + REPORT_RUN_REQUIRE_DATABASE_TESTS: "1" + run: uv run --frozen pytest tests/ -v + - name: Require the pinned commit on the Public API's default branch run: .github/scripts/public-api-commit-on-default-branch.sh public-api master diff --git a/libs/policyengine-stage12-persistence/tests/report_run_database.py b/libs/policyengine-stage12-persistence/tests/report_run_database.py index 280186a54..42fb5c18b 100644 --- a/libs/policyengine-stage12-persistence/tests/report_run_database.py +++ b/libs/policyengine-stage12-persistence/tests/report_run_database.py @@ -1,7 +1,8 @@ """A disposable Postgres holding the Public API's v2 tables, for report run tests. The tests run only when both URLs name a disposable database that the Public -API's v2 migrations created and whose declared privileges were applied: +API's v2 migrations created and whose declared privileges were applied, as the +Public API schema job's does: - ``REPORT_RUN_TEST_DATABASE_URL`` logs in as the Modal worker role, which the store under test uses; @@ -23,6 +24,9 @@ WORKER_URL_ENV = "REPORT_RUN_TEST_DATABASE_URL" SEED_URL_ENV = "REPORT_RUN_TEST_SEED_DATABASE_URL" +# Set to "1" by the Public API schema job, where a missing database fails the +# tests instead of skipping them. +REQUIRE_DATABASE_ENV = "REPORT_RUN_REQUIRE_DATABASE_TESTS" # The foreign keys between the Public API's tables and the simulation API's # stay, so every catalog row a seeded run names exists. SEEDED_TABLES = ( @@ -53,10 +57,16 @@ def worker_database_url() -> str: - """Return the Modal worker role's test URL, or skip when none is set.""" + """Return the Modal worker role's test URL, or skip when none is set. + + A job that sets ``REPORT_RUN_REQUIRE_DATABASE_TESTS=1`` fails instead. + """ url = os.environ.get(WORKER_URL_ENV) if not url or not os.environ.get(SEED_URL_ENV): - pytest.skip(f"{WORKER_URL_ENV} and {SEED_URL_ENV} name no test database") + message = f"{WORKER_URL_ENV} and {SEED_URL_ENV} name no test database" + if os.environ.get(REQUIRE_DATABASE_ENV) == "1": + pytest.fail(message) + pytest.skip(message) return url From a6afe391847339f0524399ddf72a97c1c9c82bd7 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:05:22 +0400 Subject: [PATCH 15/25] Describe the one table-model library in the rules, docs and changelog Rule `sim.persistence.models-match-public-api` now covers the result-table models and the Modal worker role's writes, with their tests as enforcers, and `sim.persistence.missing-privileges-reported` covers the check run for another role's lists. The two libraries' READMEs and the database-access skill say where the models live, who imports them, and where their tests run. Co-Authored-By: Claude Opus 5.5 --- .../c1-32-one-table-model-library.changed.md | 1 + docs/engineering/skills/database-access.md | 15 +-- docs/generated/rules.md | 12 +-- .../README.md | 100 +++++++++++++----- .../README.md | 26 +++-- rules/registry.toml | 9 +- 6 files changed, 116 insertions(+), 47 deletions(-) create mode 100644 changelog.d/c1-32-one-table-model-library.changed.md diff --git a/changelog.d/c1-32-one-table-model-library.changed.md b/changelog.d/c1-32-one-table-model-library.changed.md new file mode 100644 index 000000000..c5e177648 --- /dev/null +++ b/changelog.d/c1-32-one-table-model-library.changed.md @@ -0,0 +1 @@ +The simulation service and the v2 Modal runtime use one library of the Public API's table models, `policyengine-simulation-persistence`. The Stage 12 library's stores, which the v2 coordinators, workers, household functions and stuck-run sweep use, now import its models of `report_runs`, `simulation_runs`, `budget_window_runs` and `household_jobs`, and its new complete models of the twelve result tables, instead of partial models of their own, and connect with its engine and URL check. They write the same columns as before. The v2 Modal images copy the library. CI's comparison with the Public API's tables now covers the result tables, a new test checks the Modal worker role's grants and writes against the Public API's, and the Stage 12 library's database tests, which skipped in CI, run in the `Public API schema` job as that role. A malformed Modal worker database URL now raises the service's "Simulation database URL" error. diff --git a/docs/engineering/skills/database-access.md b/docs/engineering/skills/database-access.md index f6325820e..84572177a 100644 --- a/docs/engineering/skills/database-access.md +++ b/docs/engineering/skills/database-access.md @@ -46,13 +46,14 @@ and this repository has no migration chain of its own Their read-only models in that library's `input_tables.py` name only the columns the service reads, apart from the models of the tables it writes. - the run tables the v2 Modal coordinator, workers, household functions and - stuck-run sweep write as the Modal worker role: `report_runs`, `simulation_runs`, - `budget_window_runs`, the result tables and `household_jobs`. Their mappings in - `policyengine_stage12_persistence.report_run_tables` and - `report_run_results`, separate from the simulation service's models, - declare only the columns the coordinator's, workers', household functions' - and stuck-run sweep's statements read or write; deployment validation - doesn't compare them with the live schema. + stuck-run sweep write as the Modal worker role: `report_runs`, + `simulation_runs`, `budget_window_runs`, the result tables and + `household_jobs`. They use the same models, in + `libs/policyengine-simulation-persistence` (`tables.py` and + `result_tables.py`, which `sim.persistence.models-match-public-api` + compares with the Public API's tables), through the stores in + `libs/policyengine-stage12-persistence`. The v1 `run_simulation` code + imports neither library. This repository's code reads and writes their rows; nothing here changes them. A change to one of those tables starts in policyengine-api, and the diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 0b43a4a3f..c7c4f6a75 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -142,8 +142,8 @@ flowchart LR | Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | | --- | --- | --- | --- | --- | --- | --- | --- | -| `sim.persistence.missing-privileges-reported` | The simulation service's privilege check, which its readiness check uses, names every table privilege and every column UPDATE privilege its role needs on the Public API's tables and lacks, as PostgreSQL reports it, counts a table missing from the public schema as lacking all of them and a missing column as lacking its own, and names a missing USAGE privilege on that schema. | mechanical | enforced | yes | pytest `libs/policyengine-simulation-persistence/tests/test_access.py` | `pr.yml:simulation-database` | yes | -| `sim.persistence.models-match-public-api` | Sim's models of the Public API's tables match the tables the Public API's v2 migration chain builds at the pinned Public API commit, which is on the Public API's default branch: every column's type, nullability and default, every constraint and index by name and definition, and every enum's values; every column its read-only models of the Public API's input tables name exists there with the same type and nullability; and the simulation service's role, with the privileges the Public API's manifest declares, writes every modelled table. | mechanical | enforced | yes | pytest `libs/policyengine-simulation-persistence/tests/test_public_api_schema.py`, `libs/policyengine-simulation-persistence/tests/test_service_writes.py` | `pr.yml:public-api-schema`, `public-api-schema-drift.yml:public-api-schema-drift` | yes | +| `sim.persistence.missing-privileges-reported` | The privilege check, which the simulation service's readiness check uses and CI runs for the Modal worker role too, names every table privilege and every column UPDATE privilege the connected role needs on the Public API's tables and lacks, as PostgreSQL reports it, counts a table missing from the public schema as lacking all of them and a missing column as lacking its own, and names a missing USAGE privilege on that schema. | mechanical | enforced | yes | pytest `libs/policyengine-simulation-persistence/tests/test_access.py` | `pr.yml:simulation-database` | yes | +| `sim.persistence.models-match-public-api` | Sim's one set of models of the Public API's tables, those the simulation service writes and the result tables the v2 Modal calls write, match the tables the Public API's v2 migration chain builds at the pinned Public API commit, which is on the Public API's default branch: every column's type, nullability and default, every constraint and index by name and definition, and every enum's values; every column its read-only models of the Public API's input tables name exists there with the same type and nullability; and, with the privileges the Public API's manifest declares, the simulation service's role writes every table the service writes, and the Modal worker role records its runs and inserts a row in every result table. | mechanical | enforced | yes | pytest `libs/policyengine-simulation-persistence/tests/test_modal_worker_writes.py`, `libs/policyengine-simulation-persistence/tests/test_public_api_schema.py`, `libs/policyengine-simulation-persistence/tests/test_service_writes.py` | `pr.yml:public-api-schema`, `public-api-schema-drift.yml:public-api-schema-drift` | yes | ```mermaid flowchart LR @@ -151,14 +151,14 @@ flowchart LR e_pytest_libs_policyengine_simulation_persistence_tests_test_access_py["pytest libs/policyengine-simulation-persistence/tests/test_access.py"] j_CI_pr_yml_simulation_database["CI pr.yml:simulation-database"] r_sim_persistence_models_match_public_api("sim.persistence.models-match-public-api (enforced)") - e_pytest_libs_policyengine_simulation_persistence_tests_test_public_api_schema_py__libs_policyengine_simulation_persistence_tests_test_service_writes_py["pytest libs/policyengine-simulation-persistence/tests/test_public_api_schema.py, libs/policyengine-simulation-persistence/tests/test_service_writes.py"] + e_pytest_libs_policyengine_simulation_persistence_tests_test_modal_worker_writes_py__libs_policyengine_simulation_persistence_tests_test_public_api_schema_py__libs_policyengine_simulation_persistence_tests_test_service_writes_py["pytest libs/policyengine-simulation-persistence/tests/test_modal_worker_writes.py, libs/policyengine-simulation-persistence/tests/test_public_api_schema.py, libs/policyengine-simulation-persistence/tests/test_service_writes.py"] j_CI_pr_yml_public_api_schema["CI pr.yml:public-api-schema"] j_CI_public_api_schema_drift_yml_public_api_schema_drift["CI public-api-schema-drift.yml:public-api-schema-drift"] r_sim_persistence_missing_privileges_reported --> e_pytest_libs_policyengine_simulation_persistence_tests_test_access_py e_pytest_libs_policyengine_simulation_persistence_tests_test_access_py --> j_CI_pr_yml_simulation_database - r_sim_persistence_models_match_public_api --> e_pytest_libs_policyengine_simulation_persistence_tests_test_public_api_schema_py__libs_policyengine_simulation_persistence_tests_test_service_writes_py - e_pytest_libs_policyengine_simulation_persistence_tests_test_public_api_schema_py__libs_policyengine_simulation_persistence_tests_test_service_writes_py --> j_CI_pr_yml_public_api_schema - e_pytest_libs_policyengine_simulation_persistence_tests_test_public_api_schema_py__libs_policyengine_simulation_persistence_tests_test_service_writes_py --> j_CI_public_api_schema_drift_yml_public_api_schema_drift + r_sim_persistence_models_match_public_api --> e_pytest_libs_policyengine_simulation_persistence_tests_test_modal_worker_writes_py__libs_policyengine_simulation_persistence_tests_test_public_api_schema_py__libs_policyengine_simulation_persistence_tests_test_service_writes_py + e_pytest_libs_policyengine_simulation_persistence_tests_test_modal_worker_writes_py__libs_policyengine_simulation_persistence_tests_test_public_api_schema_py__libs_policyengine_simulation_persistence_tests_test_service_writes_py --> j_CI_pr_yml_public_api_schema + e_pytest_libs_policyengine_simulation_persistence_tests_test_modal_worker_writes_py__libs_policyengine_simulation_persistence_tests_test_public_api_schema_py__libs_policyengine_simulation_persistence_tests_test_service_writes_py --> j_CI_public_api_schema_drift_yml_public_api_schema_drift ``` ## Deployment pipeline diff --git a/libs/policyengine-simulation-persistence/README.md b/libs/policyengine-simulation-persistence/README.md index b3ca22046..78d3d140d 100644 --- a/libs/policyengine-simulation-persistence/README.md +++ b/libs/policyengine-simulation-persistence/README.md @@ -1,12 +1,20 @@ # PolicyEngine simulation persistence -Database connections for the simulation service (the Simulation Entrypoint on -Cloud Run). The service connects to the Public API's Postgres database with -its own role, and writes rows in tables the Public API's migration chain -creates. This repository has no schema or migration chain of its own. - -The temporary Stage 12 comparison tables are not part of this library: -`libs/policyengine-stage12-persistence` writes them with its own connection. +Database connections and table models for the simulation service (the +Simulation Entrypoint on Cloud Run) and the v2 Modal runtime. The service +connects to the Public API's Postgres database with its own role, and writes +rows in tables the Public API's migration chain creates. This repository has +no schema or migration chain of its own. + +This library holds sim's one set of models of the Public API's tables. The v2 +Modal coordinators, workers, household functions and stuck-run sweep write +the run tables and the result tables through +`libs/policyengine-stage12-persistence`, whose stores import these models and +connect as the Modal worker role (`policyengine_v2_modal_worker`). The v1 +`run_simulation` code (the old gateway and `src/modal/app.py` with its +modules) imports neither library. The temporary Stage 12 comparison tables +are not modelled here: the Stage 12 library keeps them until the comparison +path is removed. ## Connections @@ -17,6 +25,12 @@ server-side statements, so the same URL works through Supabase's session and transaction poolers. A malformed URL raises `ValueError` without repeating the URL, which holds a password. +`create_unpooled_engine` builds the engine the v2 Modal calls and the +comparison path use, from the same URL check: it keeps no pool, so each +operation's session ends with it and a call that spends most of its life +calculating holds no pooler slot, and it waits 10 seconds to connect. The +Stage 12 library binds it as `create_stage12_engine`. + ## Tables `tables.py` models, with SQLAlchemy Core, the six tables the service creates @@ -29,6 +43,9 @@ every column with its type, nullability and server default, and every constraint and index under the chain's name. The models only describe the tables; runtime code never creates or changes them. Status and other enum columns use the `StrEnum` classes beside them and store each member's value. +A Python `None` in `simulation_runs.household_result`, `household_jobs.result` +or `report_runs.spm_provenance` is stored as SQL `NULL`, not JSON `null`, so +a query for `IS NULL` finds a run with no result. Each service writes only its own tables, and the foreign keys between them stay: 20 keys name rows the Public API writes in its users, policies, @@ -47,6 +64,21 @@ household report with only a baseline). A report with a `budget_window_size` is calculated as a `budget_window_runs` row with one report run per year; each year's report run names that row and its year, and belongs to the same report. +## Result tables + +`result_tables.py` models the twelve result tables the Modal worker role +inserts into, in `result_metadata`, as completely as `tables.py` models the +service's: `budget_summary`, `decile_impacts`, `intra_decile_impacts`, +`program_statistics`, `congressional_district_impacts`, +`constituency_impacts`, `local_authority_impacts`, `withheld_area_codes`, +`labor_supply_responses` and `labor_supply_decile_responses` belong to a +report run (all but the labour supply tables also name its two simulation +runs), and `poverty` and `inequality` to one simulation run. Their keys name +the models of `report_runs` and `simulation_runs` in `service_metadata`, so +code that builds the tables in a test database builds those first. The +Stage 12 library's `report_run_results.py` builds their rows from a report +run's economy output. + ## Input tables `input_tables.py` holds read-only models of the Public API's tables a @@ -245,24 +277,31 @@ Rule `sim.persistence.models-match-public-api`. The PR workflow's grants the privileges its manifest declares (`.github/scripts/public-api-schema-reference.sh`), then runs the tests marked `public_api_schema`. `test_public_api_schema` builds the models' tables -from `service_metadata` in a disposable database, after the Public API's -tables their keys name (`referenced_metadata`, by `id` alone), and compares -the models' tables with the Public API's through PostgreSQL's catalog -(`tests/schema_comparison.py`): every column's type, nullability and default, -every constraint and index by name and definition, and every enum's values -must match. `test_the_input_models_match_the_public_api_columns` builds +from `service_metadata`, then `result_metadata`, in a disposable database, +after the Public API's tables their keys name (`referenced_metadata`, by +`id` alone), and compares the models' tables with the Public API's through +PostgreSQL's catalog (`tests/schema_comparison.py`): every column's type, +nullability and default, every constraint and index by name and definition, +and every enum's values must match, for the service's tables and the result +tables alike. `test_the_input_models_match_the_public_api_columns` builds the read-only input models' tables in another disposable database and requires every column they name to exist in the Public API's table with the same type and nullability; their other columns, defaults, constraints and -indexes are the Public API's alone. `test_service_writes` creates a row in every modelled -table as the service's role. Another test, outside the rule, requires the modelled +indexes are the Public API's alone. `test_service_writes` creates a row in every table +the service writes, as the service's role; `test_modal_worker_writes`, as the +Modal worker role, checks the role's privileges, records a run's state in +each run table, inserts a row in every result table and reads back that a +`None` JSON result is SQL `NULL`. Another test, outside the rule, requires the modelled tables' foreign keys to the Public API's users, policies, dynamics, households, datasets, regions and model catalog tables to be exactly the 20 in `tests/public_api_keys.py`, each by its own table, its name and the table it names, so a key dropped, added, renamed, moved or pointed at another table -shows even if the models follow it. Last, the job requires the -pinned commit to be on the Public API's `master` branch, so the models only -ever describe a chain the Public API has merged. +shows even if the models follow it. The job then gives the Modal worker role +a login on its database (`.github/scripts/public-api-schema-worker-login.sh`) +and runs the Stage 12 library's tests, whose stores write these tables as +that role. Last, it requires the pinned commit to be on the Public API's +`master` branch, so the models only ever describe a chain the Public API has +merged. **Choosing the pin.** Pin the Public API commit whose migration chain the models describe: the merge commit of the Public API change that last @@ -271,9 +310,9 @@ head commit. Until that pull request merges, the job's last step fails and says so. A merge commit keeps the pull request's head on `master`; if the Public API squashes or rebases the pull request instead, its head never reaches `master`, so move the pin to the commit that landed. The pin may -also move to a later merged commit that alters no modelled table, such as -one adding a table another package writes; the job still checks only this -package's tables, so moving the pin is no check of that other table. +also move to a later merged commit that alters no modelled table. Every +table sim writes is modelled here, the result tables included, so a change +to any of them moves the pin with this library's models. **Keeping it current.** A sim pull request that changes the models moves the pin in the same change. The `Public API schema drift` workflow runs the same @@ -296,7 +335,7 @@ PUBLIC_API_SCHEMA_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/pol uv run pytest tests/ -m public_api_schema ``` -## The role's privileges +## The roles' privileges `SERVICE_TABLE_PRIVILEGES` lists the table privileges the service's role needs: `SELECT` and `INSERT` on the tables it creates rows in @@ -313,6 +352,16 @@ when Modal refuses one of the first years the service starts. The role holds refuses any other change to a run. The Public API's privilege manifest grants them all. +`MODAL_WORKER_TABLE_PRIVILEGES` and `MODAL_WORKER_UPDATABLE_COLUMNS` list what +the Modal worker role needs: `SELECT` on the four run tables, `INSERT` on +every result table, and `UPDATE` of each run's state (`status`, +`modal_call_id`, `error_message`, `started_at`, `completed_at` and +`updated_at`; a budget-window run has no call of its own) and of the results +its own row holds: a report run's cliff impact, `spm_provenance` and +`data_version`, a simulation run's output file and `household_result`, and a +household job's `result`. The Public API's manifest grants them; +`test_modal_worker_writes` checks that in CI. + `missing_table_privileges` returns each one the connected role lacks, ordered by table, as `"INSERT on reports"` or `"UPDATE (modal_call_id) on report_runs"`; it asks PostgreSQL with `has_table_privilege` and @@ -342,8 +391,11 @@ rule's proof among them needs no database, and the others need one the Public API's own v2 migration chain built, with the privileges its manifest declares granted: set `PUBLIC_API_SCHEMA_DATABASE_URL` to that database's superuser URL. -`test_service_writes` creates a row in every modelled table as the service's -role and rolls it back; the comparison in `test_public_api_schema` also needs +`test_service_writes` creates a row in every table the service writes as the +service's role, and `test_modal_worker_writes` writes the Modal worker role's +runs and results as that role, each rolling its transaction back; neither +role needs a login there, since the superuser takes each role for the +transaction. The comparison in `test_public_api_schema` also needs `SIMULATION_TEST_DATABASE_URL`, to build the models' own tables. Without the variables they skip. diff --git a/libs/policyengine-stage12-persistence/README.md b/libs/policyengine-stage12-persistence/README.md index 814bc1a18..8229d40d3 100644 --- a/libs/policyengine-stage12-persistence/README.md +++ b/libs/policyengine-stage12-persistence/README.md @@ -1,8 +1,15 @@ # PolicyEngine Stage 12 persistence -SQLAlchemy Core mappings and transactional DML for two sets of tables, both -owned by `policyengine-api`, which keeps the SQLModel definitions, Alembic -revisions and live schema. This package never changes a schema. +Transactional DML for two sets of tables, both owned by `policyengine-api`, +which keeps the SQLModel definitions, Alembic revisions and live schema. This +package never changes a schema. + +The models of the Public API's run and result tables are the simulation +service library's (`libs/policyengine-simulation-persistence`: `tables.py` +and `result_tables.py`), which CI compares with the tables the Public API's +chain builds; this package's stores import them, and its engines come from +that library's `create_unpooled_engine`. Its only models of its own are the +comparison path's tables, in `tables.py`. - **The Stage 12 comparison tables** (`Stage12PersistenceStore`): temporary state of the comparison path, removed with it. @@ -18,9 +25,9 @@ revisions and live schema. This package never changes a schema. The coordinator also reads a simulation run's status, year, versions, dataset, region, output file, recorded call and failure, so a report run can use a run that already succeeded instead of calculating it again, or - wait for one another call holds. The mappings declare only the columns - those statements read or write, and each `UPDATE` sets only columns the - role may update. Every status write is one `UPDATE` conditional on the + wait for one another call holds. Every statement names the columns it + reads or writes, so the complete models send no other, and each `UPDATE` + sets only columns the role may update. Every status write is one `UPDATE` conditional on the row's status and, for a call's claim, completion or failure, on its recorded Modal call: a call claims a pending or running row that records no call or its own, and completes or fails it only while it holds it @@ -55,8 +62,11 @@ revisions and live schema. This package never changes a schema. ## Tests The report run and budget-window tests need a disposable Postgres migrated -by the Public API's v2 chain, with its declared privileges applied. They skip -unless both URLs are set: +by the Public API's v2 chain, with its declared privileges applied. The +`Public API schema` job in `pr.yml` runs them on the database it builds at the +pinned Public API commit, after giving the Modal worker role a login there, +and sets `REPORT_RUN_REQUIRE_DATABASE_TESTS=1`, so a missing database fails +them. Elsewhere they skip unless both URLs are set: - `REPORT_RUN_TEST_DATABASE_URL`: logs in as `policyengine_v2_modal_worker`; - `REPORT_RUN_TEST_SEED_DATABASE_URL`: logs in as a role that may insert the diff --git a/rules/registry.toml b/rules/registry.toml index b0100b932..7314d8237 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -141,7 +141,7 @@ test = "tests/test_database_access_boundaries.py::test_rule_proof_sqlalchemy_and [[rule]] id = "sim.persistence.missing-privileges-reported" category = "persistence" -statement = "The simulation service's privilege check, which its readiness check uses, names every table privilege and every column UPDATE privilege its role needs on the Public API's tables and lacks, as PostgreSQL reports it, counts a table missing from the public schema as lacking all of them and a missing column as lacking its own, and names a missing USAGE privilege on that schema." +statement = "The privilege check, which the simulation service's readiness check uses and CI runs for the Modal worker role too, names every table privilege and every column UPDATE privilege the connected role needs on the Public API's tables and lacks, as PostgreSQL reports it, counts a table missing from the public schema as lacking all of them and a missing column as lacking its own, and names a missing USAGE privilege on that schema." rationale = "libs/policyengine-simulation-persistence/README.md#the-roles-privileges" scope = ["libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/access.py"] tier = "mechanical" @@ -159,6 +159,7 @@ nodes = [ "tests/test_access.py::test_only_the_public_schema_counts", "tests/test_access.py::test_only_the_public_schemas_columns_count", "tests/test_access.py::test_a_role_without_usage_on_the_schema_is_told_so", + "tests/test_access.py::test_the_check_names_what_a_role_lacks_of_another_roles_lists", ] [[rule.ci]] workflow = ".github/workflows/pr.yml" @@ -173,10 +174,11 @@ test = "tests/test_access.py::test_rule_proof_missing_privileges_reported" [[rule]] id = "sim.persistence.models-match-public-api" category = "persistence" -statement = "Sim's models of the Public API's tables match the tables the Public API's v2 migration chain builds at the pinned Public API commit, which is on the Public API's default branch: every column's type, nullability and default, every constraint and index by name and definition, and every enum's values; every column its read-only models of the Public API's input tables name exists there with the same type and nullability; and the simulation service's role, with the privileges the Public API's manifest declares, writes every modelled table." +statement = "Sim's one set of models of the Public API's tables, those the simulation service writes and the result tables the v2 Modal calls write, match the tables the Public API's v2 migration chain builds at the pinned Public API commit, which is on the Public API's default branch: every column's type, nullability and default, every constraint and index by name and definition, and every enum's values; every column its read-only models of the Public API's input tables name exists there with the same type and nullability; and, with the privileges the Public API's manifest declares, the simulation service's role writes every table the service writes, and the Modal worker role records its runs and inserts a row in every result table." rationale = "libs/policyengine-simulation-persistence/README.md#checking-the-models-against-the-public-api" scope = [ "libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/tables.py", + "libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/result_tables.py", "libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/input_tables.py", "libs/policyengine-simulation-persistence/public-api-commit", ".github/scripts/public-api-schema-commit.sh", @@ -191,9 +193,12 @@ kind = "pytest" project = "libs/policyengine-simulation-persistence" nodes = [ "tests/test_public_api_schema.py::test_the_models_match_the_public_api_tables", + "tests/test_public_api_schema.py::test_the_result_models_match_the_public_api_tables", "tests/test_public_api_schema.py::test_the_input_models_match_the_public_api_columns", "tests/test_service_writes.py::test_the_service_role_holds_every_privilege_the_service_needs", "tests/test_service_writes.py::test_the_service_role_creates_and_reads_a_row_in_every_table_it_writes", + "tests/test_modal_worker_writes.py::test_the_modal_worker_role_holds_every_privilege_the_modal_calls_need", + "tests/test_modal_worker_writes.py::test_the_modal_worker_role_records_its_runs_and_inserts_every_result", ] [[rule.ci]] workflow = ".github/workflows/pr.yml" From 826b4b6dc952b59e1b61c40a05bd410a9ef916b6 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:29:07 +0400 Subject: [PATCH 16/25] Test that the v1 import walk counts a module's packages and sim's libraries (failing) Review round 1 found that the walk from `src/modal/app.py` resolved each imported module to its own file only. Python runs every parent package's `__init__` first, so an import of a `stage12_runtime` submodule would load the Stage 12 library through the package and pass unseen. The walk now takes its source folders as arguments, follows sim's own libraries too, and two tests on throwaway source trees check both paths; the package one fails. Co-Authored-By: Claude Opus 5.5 --- .../test_v1_code_imports_no_table_models.py | 131 ++++++++++++++---- 1 file changed, 104 insertions(+), 27 deletions(-) diff --git a/projects/policyengine-simulation-executor/tests/test_v1_code_imports_no_table_models.py b/projects/policyengine-simulation-executor/tests/test_v1_code_imports_no_table_models.py index 5229968a9..71196a440 100644 --- a/projects/policyengine-simulation-executor/tests/test_v1_code_imports_no_table_models.py +++ b/projects/policyengine-simulation-executor/tests/test_v1_code_imports_no_table_models.py @@ -4,8 +4,8 @@ Public API's table models, ``policyengine-simulation-persistence``, which the Stage 12 library builds on. The v1 code, which will be removed, stays apart from both: the old gateway, and the v1 Modal app (``src/modal/app.py``) with -every module of this project it imports, however indirectly and wherever in -the module the import is. +every module of this repository it imports, however indirectly and wherever +in the module the import is. """ from __future__ import annotations @@ -14,19 +14,31 @@ from pathlib import Path PROJECT_ROOT = Path(__file__).resolve().parents[1] +REPOSITORY_ROOT = PROJECT_ROOT.parents[1] SOURCE_ROOT = PROJECT_ROOT / "src" GATEWAY_SOURCE_ROOT = PROJECT_ROOT.parent / "policyengine-simulation-gateway" / "src" V1_MODAL_APP = SOURCE_ROOT / "modal" / "app.py" -# Modules of this project, found as files under src. -PROJECT_PACKAGES = ("src.modal", "policyengine_simulation_executor") TABLE_MODEL_LIBRARIES = ( "policyengine_simulation_persistence", "policyengine_stage12_persistence", ) -def _module_name(path: Path) -> str: - parts = path.relative_to(SOURCE_ROOT).with_suffix("").parts +def _first_party_packages(source_roots: tuple[Path, ...]) -> frozenset[str]: + """Return the packages under ``source_roots``, but the table-model libraries.""" + packages = { + path.name + for root in source_roots + for path in root.iterdir() + if (path / "__init__.py").is_file() and path.name != "modal" + } + return frozenset(packages - set(TABLE_MODEL_LIBRARIES)) + + +def _module_name(path: Path, source_roots: tuple[Path, ...]) -> str: + root = next(root for root in source_roots if path.is_relative_to(root)) + parts = path.relative_to(root).with_suffix("").parts + # The Modal entry files are imported as src.modal, from the project. if parts[0] == "modal": parts = ("src", *parts) if parts[-1] == "__init__": @@ -34,24 +46,29 @@ def _module_name(path: Path) -> str: return ".".join(parts) -def _module_path(module_name: str) -> Path | None: - """Return the file of one of this project's modules, or None for another's.""" - if not module_name.startswith(PROJECT_PACKAGES): - return None +def _module_path( + module_name: str, + source_roots: tuple[Path, ...], + first_party: frozenset[str], +) -> Path | None: + """Return the file of a module of this repository, or None for another's.""" parts = module_name.split(".") - if parts[0] == "src": + if parts[:2] == ["src", "modal"]: parts = parts[1:] - candidate = SOURCE_ROOT.joinpath(*parts) - for path in (candidate.with_suffix(".py"), candidate / "__init__.py"): - if path.is_file(): - return path + elif parts[0] not in first_party: + return None + for root in source_roots: + candidate = root.joinpath(*parts) + for path in (candidate.with_suffix(".py"), candidate / "__init__.py"): + if path.is_file(): + return path return None -def _imported_modules(path: Path) -> set[str]: +def _imported_modules(path: Path, source_roots: tuple[Path, ...]) -> set[str]: """Return every module ``path`` imports, and each package it imports from.""" tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - package = _module_name(path) + package = _module_name(path, source_roots) if path.name != "__init__.py": package = package.rpartition(".")[0] imported: set[str] = set() @@ -73,26 +90,32 @@ def _imported_modules(path: Path) -> set[str]: return imported -def _v1_modal_imports() -> dict[str, set[str]]: - """Return, by module, what the v1 Modal app and its project modules import.""" - pending = [V1_MODAL_APP] +def imports_reached(entry: Path, source_roots: tuple[Path, ...]) -> dict[str, set[str]]: + """Return, by module, what ``entry`` and every module it reaches import. + + It follows only modules under ``source_roots``, never into the table-model + libraries, whose imports are what the tests look for. + """ + first_party = _first_party_packages(source_roots) + pending = [entry] imports_by_module: dict[str, set[str]] = {} while pending: path = pending.pop() - module_name = _module_name(path) + module_name = _module_name(path, source_roots) if module_name in imports_by_module: continue - imported = _imported_modules(path) + imported = _imported_modules(path, source_roots) imports_by_module[module_name] = imported pending.extend( module_path for name in imported - if (module_path := _module_path(name)) is not None + if (module_path := _module_path(name, source_roots, first_party)) + is not None ) return imports_by_module -def _table_model_imports(imports_by_module: dict[str, set[str]]) -> list[str]: +def table_model_imports(imports_by_module: dict[str, set[str]]) -> list[str]: return sorted( f"{module_name}: {name}" for module_name, imported in imports_by_module.items() @@ -101,12 +124,66 @@ def _table_model_imports(imports_by_module: dict[str, set[str]]) -> list[str]: ) +def _write_module(root: Path, relative_path: str, source: str = "") -> None: + path = root / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(source, encoding="utf-8") + + +def test_an_import_through_a_package_counts_its_package_too(tmp_path) -> None: + """Python runs a package's ``__init__`` before any module in it.""" + project = tmp_path / "src" + _write_module(project, "modal/__init__.py") + _write_module( + project, + "modal/app.py", + "from policyengine_simulation_executor.runtime.leaf import value\n", + ) + _write_module(project, "policyengine_simulation_executor/__init__.py") + _write_module( + project, + "policyengine_simulation_executor/runtime/__init__.py", + "import policyengine_stage12_persistence\n", + ) + _write_module(project, "policyengine_simulation_executor/runtime/leaf.py") + + reached = imports_reached(project / "modal" / "app.py", (project,)) + + assert table_model_imports(reached) == [ + "policyengine_simulation_executor.runtime: policyengine_stage12_persistence" + ] + + +def test_an_import_through_a_first_party_library_counts(tmp_path) -> None: + project = tmp_path / "project" / "src" + library = tmp_path / "library" / "src" + _write_module(project, "modal/__init__.py") + _write_module(project, "modal/app.py", "import policyengine_shared_helpers\n") + _write_module( + library, + "policyengine_shared_helpers/__init__.py", + "from policyengine_simulation_persistence import tables\n", + ) + + reached = imports_reached(project / "modal" / "app.py", (project, library)) + + assert table_model_imports(reached) == [ + "policyengine_shared_helpers: policyengine_simulation_persistence", + "policyengine_shared_helpers: policyengine_simulation_persistence.tables", + ] + + +def _repository_source_roots() -> tuple[Path, ...]: + """Return this project's source and every library's, for the walk.""" + return (SOURCE_ROOT, *sorted((REPOSITORY_ROOT / "libs").glob("*/src"))) + + def test_the_v1_modal_app_and_its_modules_import_no_table_model_library() -> None: - imports_by_module = _v1_modal_imports() + imports_by_module = imports_reached(V1_MODAL_APP, _repository_source_roots()) # The walk follows the app's own imports, not only the file itself. assert "src.modal._image_setup" in imports_by_module - assert _table_model_imports(imports_by_module) == [] + assert table_model_imports(imports_by_module) == [] def test_the_old_gateway_imports_no_table_model_library() -> None: @@ -117,7 +194,7 @@ def test_the_old_gateway_imports_no_table_model_library() -> None: } assert gateway_files - assert _table_model_imports(imports_by_module) == [] + assert table_model_imports(imports_by_module) == [] def _gateway_imports(path: Path) -> set[str]: From e9fcb60e7bc4285bf76075b2981c6e4492577043 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:29:27 +0400 Subject: [PATCH 17/25] Follow every parent package in the v1 import walk Importing a module runs its parent packages' `__init__` first, so the walk from `src/modal/app.py` now reaches each of them too. It walks 66 modules, 18 of them in sim's libraries, and finds no import of either table-model library. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_v1_code_imports_no_table_models.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/projects/policyengine-simulation-executor/tests/test_v1_code_imports_no_table_models.py b/projects/policyengine-simulation-executor/tests/test_v1_code_imports_no_table_models.py index 71196a440..52df78351 100644 --- a/projects/policyengine-simulation-executor/tests/test_v1_code_imports_no_table_models.py +++ b/projects/policyengine-simulation-executor/tests/test_v1_code_imports_no_table_models.py @@ -90,11 +90,19 @@ def _imported_modules(path: Path, source_roots: tuple[Path, ...]) -> set[str]: return imported +def _with_parent_packages(module_name: str) -> list[str]: + """Return ``module_name`` and each package Python imports before it.""" + parts = module_name.split(".") + return [".".join(parts[:count]) for count in range(1, len(parts) + 1)] + + def imports_reached(entry: Path, source_roots: tuple[Path, ...]) -> dict[str, set[str]]: """Return, by module, what ``entry`` and every module it reaches import. - It follows only modules under ``source_roots``, never into the table-model - libraries, whose imports are what the tests look for. + Importing a module runs each of its parent packages' ``__init__`` first, + so those are reached too. It follows only modules under ``source_roots``, + never into the table-model libraries, whose imports are what the tests + look for. """ first_party = _first_party_packages(source_roots) pending = [entry] @@ -109,7 +117,8 @@ def imports_reached(entry: Path, source_roots: tuple[Path, ...]) -> dict[str, se pending.extend( module_path for name in imported - if (module_path := _module_path(name, source_roots, first_party)) + for reached_name in _with_parent_packages(name) + if (module_path := _module_path(reached_name, source_roots, first_party)) is not None ) return imports_by_module From 262a176167a512c0b65092246c0cda9d1f93d9d3 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:30:25 +0400 Subject: [PATCH 18/25] Pin RunStatus to each run table's status values `RunStatus` stays the Stage 12 code's one status for any run kind; the service library's `SimulationStatus`, `ReportRunStatus` and `HouseholdJobStatus`, which name the entry's OpenAPI schemas, stay separate. A read through the shared models returns the table's own member, which the stores convert with `RunStatus(...)`, so the test requires the same values in the same order and the conversion of each member. It fails when `RunStatus` gains a value the tables don't have. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_shared_table_models.py | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py b/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py index 403dd1696..f0aa0d6c1 100644 --- a/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py +++ b/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py @@ -14,6 +14,7 @@ from uuid import uuid4 import policyengine_stage12_persistence +import pytest from economy_output import ( US_AT_LARGE_STATES, US_STATE_FIPS, @@ -22,6 +23,9 @@ ) from policyengine_simulation_contract.stage12_execution import ReportRunIds from policyengine_simulation_persistence import ( + HouseholdJobStatus, + ReportRunStatus, + SimulationStatus, engine, result_metadata, service_metadata, @@ -35,7 +39,11 @@ report_run_store, store, ) -from policyengine_stage12_persistence.report_run_store import RUN_TABLES, RunKind +from policyengine_stage12_persistence.report_run_store import ( + RUN_TABLES, + RunKind, + RunStatus, +) from policyengine_stage12_persistence.tables import metadata as comparison_metadata from sqlalchemy import Table @@ -120,3 +128,19 @@ def test_the_stores_connect_with_the_service_librarys_one_engine_helper() -> Non assert not hasattr(store, "normalized_database_url") assert report_run_store.create_stage12_engine is engine.create_unpooled_engine assert budget_window_store.create_stage12_engine is engine.create_unpooled_engine + + +@pytest.mark.parametrize( + "table_status", [SimulationStatus, ReportRunStatus, HouseholdJobStatus] +) +def test_run_status_has_each_run_tables_status_values(table_status) -> None: + """``RunStatus`` is a status of any run kind, so it has each table's values. + + A read through the shared models returns the table's own enum member, + which the stores convert with ``RunStatus(...)`` before comparing it. + """ + assert [status.value for status in RunStatus] == [ + status.value for status in table_status + ] + for status in table_status: + assert RunStatus(status) is RunStatus(status.value) From 98853c5180bf74c80f4f6bd887e13bcfffe44f8b Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:30:27 +0400 Subject: [PATCH 19/25] Say which tables the comparison leaves out, and what the schema job runs The service library README's pin guidance now names the comparison path's `stage12_evaluation_*` tables as the only tables sim writes that this comparison doesn't cover, and the `Public API schema` job's comment in the PR workflow describes the Modal worker role's tests and the Stage 12 tests it runs. Review round 1's documentation findings. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/pr.yml | 6 ++++-- libs/policyengine-simulation-persistence/README.md | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 90347c0cc..e5aad93e4 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -136,8 +136,10 @@ jobs: # commit pinned in libs/policyengine-simulation-persistence/public-api-commit, # grants the privileges its manifest declares, and runs the persistence # tests marked public_api_schema against them: sim's table models must - # match those tables, and the simulation service's role must write them - # (rule sim.persistence.models-match-public-api). The pinned commit must + # match those tables, and the simulation service's role and the Modal + # worker role must write the tables each writes (rule + # sim.persistence.models-match-public-api). It then runs the Stage 12 + # library's tests there, as the Modal worker role. The pinned commit must # also be on the Public API's default branch. The Public API's repository # is public, so the job reads no secret. Full CI runs only on the lowest # unmerged pull request in a stack. diff --git a/libs/policyengine-simulation-persistence/README.md b/libs/policyengine-simulation-persistence/README.md index 78d3d140d..e44fb3ced 100644 --- a/libs/policyengine-simulation-persistence/README.md +++ b/libs/policyengine-simulation-persistence/README.md @@ -311,8 +311,10 @@ says so. A merge commit keeps the pull request's head on `master`; if the Public API squashes or rebases the pull request instead, its head never reaches `master`, so move the pin to the commit that landed. The pin may also move to a later merged commit that alters no modelled table. Every -table sim writes is modelled here, the result tables included, so a change -to any of them moves the pin with this library's models. +run and result table sim writes is modelled here, so a change to any of them +moves the pin with this library's models; only the temporary comparison +path's `stage12_evaluation_*` tables, which the Stage 12 library models and +checks before each deploy, are outside this comparison. **Keeping it current.** A sim pull request that changes the models moves the pin in the same change. The `Public API schema drift` workflow runs the same From 969fad8e0f18bb664d7d249b29ba0fb9cf263f92 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:11:45 +0400 Subject: [PATCH 20/25] Test that the Stage 12 library and the Modal image group use the shared ranges (failing) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The migration plan decided (task C1 ยท 31, decision 4) that the shared table-model library's `sqlalchemy>=2,<2.1` and `psycopg[binary]>=3.2,<4` are the one range: the Stage 12 library and the executor's `modal-simulation-image` group, which still allow `sqlalchemy<3`, are to declare the same, so no declaration names a version the locks can't take. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_database_dependency_ranges.py | 43 +++++++++++++++++ .../tests/test_database_dependency_ranges.py | 46 +++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 libs/policyengine-stage12-persistence/tests/test_database_dependency_ranges.py create mode 100644 projects/policyengine-simulation-executor/tests/test_database_dependency_ranges.py diff --git a/libs/policyengine-stage12-persistence/tests/test_database_dependency_ranges.py b/libs/policyengine-stage12-persistence/tests/test_database_dependency_ranges.py new file mode 100644 index 000000000..e2b52f80d --- /dev/null +++ b/libs/policyengine-stage12-persistence/tests/test_database_dependency_ranges.py @@ -0,0 +1,43 @@ +"""The library declares the shared table-model library's database ranges. + +Its stores use ``policyengine-simulation-persistence``'s models, so they run on +the SQLAlchemy and psycopg releases that library declares, and declaring a +wider range would name versions the locks can't take. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import tomllib + +LIBRARY_ROOT = Path(__file__).resolve().parents[1] +SHARED_LIBRARY_ROOT = LIBRARY_ROOT.parent / "policyengine-simulation-persistence" +DATABASE_PACKAGES = ("sqlalchemy", "psycopg") + + +def _requirements(pyproject: Path) -> list[str]: + return tomllib.loads(pyproject.read_text(encoding="utf-8"))["project"][ + "dependencies" + ] + + +def _requirement(requirements: list[str], package_name: str) -> str: + """Return the one requirement naming ``package_name``, extras included.""" + matching = [ + requirement + for requirement in requirements + if (name := re.match(r"[A-Za-z0-9_.-]+", requirement)) is not None + and name.group(0).lower() == package_name + ] + assert len(matching) == 1, (package_name, matching) + return matching[0] + + +def test_the_library_declares_the_shared_librarys_database_ranges() -> None: + own = _requirements(LIBRARY_ROOT / "pyproject.toml") + shared = _requirements(SHARED_LIBRARY_ROOT / "pyproject.toml") + + for package_name in DATABASE_PACKAGES: + assert _requirement(own, package_name) == _requirement(shared, package_name) diff --git a/projects/policyengine-simulation-executor/tests/test_database_dependency_ranges.py b/projects/policyengine-simulation-executor/tests/test_database_dependency_ranges.py new file mode 100644 index 000000000..8b4e04a38 --- /dev/null +++ b/projects/policyengine-simulation-executor/tests/test_database_dependency_ranges.py @@ -0,0 +1,46 @@ +"""The Modal images install the shared table-model library's database ranges. + +The images install third-party packages only from the ``modal-simulation-image`` +group, and the Stage 12 stores in them use ``policyengine-simulation-persistence``'s +models, so the group declares that library's SQLAlchemy and psycopg ranges. +""" + +from __future__ import annotations + +import re +import tomllib +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +SHARED_LIBRARY_PYPROJECT = ( + PROJECT_ROOT.parents[1] + / "libs" + / "policyengine-simulation-persistence" + / "pyproject.toml" +) +DATABASE_PACKAGES = ("sqlalchemy", "psycopg") + + +def _requirement(requirements: list[str], package_name: str) -> str: + """Return the one requirement naming ``package_name``, extras included.""" + matching = [ + requirement + for requirement in requirements + if (name := re.match(r"[A-Za-z0-9_.-]+", requirement)) is not None + and name.group(0).lower() == package_name + ] + assert len(matching) == 1, (package_name, matching) + return matching[0] + + +def test_the_image_group_declares_the_shared_librarys_database_ranges() -> None: + project = tomllib.loads((PROJECT_ROOT / "pyproject.toml").read_text("utf-8")) + image_group = project["dependency-groups"]["modal-simulation-image"] + shared = tomllib.loads(SHARED_LIBRARY_PYPROJECT.read_text("utf-8"))["project"][ + "dependencies" + ] + + for package_name in DATABASE_PACKAGES: + assert _requirement(image_group, package_name) == _requirement( + shared, package_name + ) From 282a838064d1f37ddcdc14259d0363318ce674e6 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:12:17 +0400 Subject: [PATCH 21/25] Narrow the Stage 12 library and the Modal image group to the shared ranges Both now declare `sqlalchemy>=2,<2.1`, the shared table-model library's range, beside its `psycopg[binary]>=3.2,<4`. The locks keep their own patch releases: 2.0.49 in the executor's, which the Modal images install, and 2.0.54 in the Stage 12 library's and the entry's, which Cloud Run installs; only their recorded ranges change. Co-Authored-By: Claude Opus 5.5 --- libs/policyengine-stage12-persistence/pyproject.toml | 3 ++- libs/policyengine-stage12-persistence/uv.lock | 2 +- projects/policyengine-simulation-entry/uv.lock | 2 +- projects/policyengine-simulation-executor/pyproject.toml | 9 +++++---- projects/policyengine-simulation-executor/uv.lock | 4 ++-- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/libs/policyengine-stage12-persistence/pyproject.toml b/libs/policyengine-stage12-persistence/pyproject.toml index 261eaa635..da21f32e8 100644 --- a/libs/policyengine-stage12-persistence/pyproject.toml +++ b/libs/policyengine-stage12-persistence/pyproject.toml @@ -12,8 +12,9 @@ dependencies = [ # The models of the Public API's run and result tables, which the CI # comparison checks against the tables the Public API's chain builds. "policyengine-simulation-persistence", + # The shared library's ranges: the stores use its table models. "psycopg[binary]>=3.2,<4", - "sqlalchemy>=2,<3", + "sqlalchemy>=2,<2.1", ] [project.optional-dependencies] diff --git a/libs/policyengine-stage12-persistence/uv.lock b/libs/policyengine-stage12-persistence/uv.lock index e0607ad62..2752c838c 100644 --- a/libs/policyengine-stage12-persistence/uv.lock +++ b/libs/policyengine-stage12-persistence/uv.lock @@ -686,7 +686,7 @@ requires-dist = [ { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=6.1.1" }, - { name = "sqlalchemy", specifier = ">=2,<3" }, + { name = "sqlalchemy", specifier = ">=2,<2.1" }, ] provides-extras = ["test", "build"] diff --git a/projects/policyengine-simulation-entry/uv.lock b/projects/policyengine-simulation-entry/uv.lock index f5cb9787a..f9d13ff46 100644 --- a/projects/policyengine-simulation-entry/uv.lock +++ b/projects/policyengine-simulation-entry/uv.lock @@ -1005,7 +1005,7 @@ requires-dist = [ { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=6.1.1" }, - { name = "sqlalchemy", specifier = ">=2,<3" }, + { name = "sqlalchemy", specifier = ">=2,<2.1" }, ] provides-extras = ["test", "build"] diff --git a/projects/policyengine-simulation-executor/pyproject.toml b/projects/policyengine-simulation-executor/pyproject.toml index 7f4dfbbc1..23180cd2a 100644 --- a/projects/policyengine-simulation-executor/pyproject.toml +++ b/projects/policyengine-simulation-executor/pyproject.toml @@ -66,10 +66,11 @@ modal-simulation-image = [ # this directly; it is also a transitive dependency of policyengine. "requests>=2.32,<3", "pyarrow>=20,<24", - # The Stage 12 coordinator and child functions use typed SQLAlchemy Core - # DML against the API-owned temporary tables. These dependencies are - # explicit because only this group is installed in Modal. - "sqlalchemy>=2,<3", + # The Stage 12 stores use typed SQLAlchemy Core DML through the shared + # table-model library (policyengine-simulation-persistence), whose ranges + # these are. They are explicit because only this group is installed in + # Modal. + "sqlalchemy>=2,<2.1", "psycopg[binary]>=3.2,<4", ] # The executor's budget-window scheduler tests exercise the gateway app diff --git a/projects/policyengine-simulation-executor/uv.lock b/projects/policyengine-simulation-executor/uv.lock index 3323e149b..061a5437f 100644 --- a/projects/policyengine-simulation-executor/uv.lock +++ b/projects/policyengine-simulation-executor/uv.lock @@ -1983,7 +1983,7 @@ modal-simulation-image = [ { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" }, { name = "pyarrow", specifier = ">=20,<24" }, { name = "requests", specifier = ">=2.32,<3" }, - { name = "sqlalchemy", specifier = ">=2,<3" }, + { name = "sqlalchemy", specifier = ">=2,<2.1" }, { name = "tables", specifier = ">=3.10.2" }, ] @@ -2091,7 +2091,7 @@ requires-dist = [ { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=6.1.1" }, - { name = "sqlalchemy", specifier = ">=2,<3" }, + { name = "sqlalchemy", specifier = ">=2,<2.1" }, ] provides-extras = ["test", "build"] From 20ff954acbfc4fa8b2874006efc390900f5e1c12 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:26:13 +0400 Subject: [PATCH 22/25] Expect the shared SQLAlchemy range in the Modal image group's test `test_v2_image_retains_required_runtime_dependencies` still required `sqlalchemy>=2,<3` in the `modal-simulation-image` group, which now declares the shared library's `sqlalchemy>=2,<2.1`, so it failed. Review round 2's bug. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_stage12_modal_app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py b/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py index e0b4e3a15..e26731168 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py @@ -208,7 +208,9 @@ def test_v2_image_retains_required_runtime_dependencies() -> None: f"policyengine[models]=={get_bundled_package_version('policyengine')}" in group ) assert not any(isinstance(item, dict) for item in group) - assert "sqlalchemy>=2,<3" in group + # The shared table-model library's ranges, which the Stage 12 stores use + # (test_database_dependency_ranges compares them with that library's). + assert "sqlalchemy>=2,<2.1" in group assert "psycopg[binary]>=3.2,<4" in group From 42b20710b27342c48649982dc58a0dda294d44d7 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:26:35 +0400 Subject: [PATCH 23/25] Test that the v1 import walk counts the entry module's own package (failing) Importing `src.modal.app` runs `src.modal`'s `__init__` first, but the walk reached that package only because the app happens to import another module in it. The real v1 test also now requires the walk to reach sim's libraries. Review round 2's quality findings. Co-Authored-By: Claude Opus 5.5 --- .../test_v1_code_imports_no_table_models.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/projects/policyengine-simulation-executor/tests/test_v1_code_imports_no_table_models.py b/projects/policyengine-simulation-executor/tests/test_v1_code_imports_no_table_models.py index 52df78351..3c7de25a5 100644 --- a/projects/policyengine-simulation-executor/tests/test_v1_code_imports_no_table_models.py +++ b/projects/policyengine-simulation-executor/tests/test_v1_code_imports_no_table_models.py @@ -163,6 +163,21 @@ def test_an_import_through_a_package_counts_its_package_too(tmp_path) -> None: ] +def test_an_import_in_the_entry_modules_own_package_counts(tmp_path) -> None: + """Importing ``src.modal.app`` runs ``src.modal``'s ``__init__`` first.""" + project = tmp_path / "src" + _write_module( + project, "modal/__init__.py", "import policyengine_simulation_persistence\n" + ) + _write_module(project, "modal/app.py") + + reached = imports_reached(project / "modal" / "app.py", (project,)) + + assert table_model_imports(reached) == [ + "src.modal: policyengine_simulation_persistence" + ] + + def test_an_import_through_a_first_party_library_counts(tmp_path) -> None: project = tmp_path / "project" / "src" library = tmp_path / "library" / "src" @@ -190,8 +205,10 @@ def _repository_source_roots() -> tuple[Path, ...]: def test_the_v1_modal_app_and_its_modules_import_no_table_model_library() -> None: imports_by_module = imports_reached(V1_MODAL_APP, _repository_source_roots()) - # The walk follows the app's own imports, not only the file itself. + # The walk follows the app's own imports, not only the file itself, and + # into sim's libraries. assert "src.modal._image_setup" in imports_by_module + assert "policyengine_simulation_contract" in imports_by_module assert table_model_imports(imports_by_module) == [] From aafa2749c83ac959c7118fec69d3145c58dcbe0b Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:26:51 +0400 Subject: [PATCH 24/25] Start the v1 import walk from the entry module's own packages too `src.modal`'s `__init__` is now reached because `src.modal.app` is in it, not only because the app imports another module of that package. Co-Authored-By: Claude Opus 5.5 --- .../test_v1_code_imports_no_table_models.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/projects/policyengine-simulation-executor/tests/test_v1_code_imports_no_table_models.py b/projects/policyengine-simulation-executor/tests/test_v1_code_imports_no_table_models.py index 3c7de25a5..2614ec14e 100644 --- a/projects/policyengine-simulation-executor/tests/test_v1_code_imports_no_table_models.py +++ b/projects/policyengine-simulation-executor/tests/test_v1_code_imports_no_table_models.py @@ -100,12 +100,21 @@ def imports_reached(entry: Path, source_roots: tuple[Path, ...]) -> dict[str, se """Return, by module, what ``entry`` and every module it reaches import. Importing a module runs each of its parent packages' ``__init__`` first, - so those are reached too. It follows only modules under ``source_roots``, - never into the table-model libraries, whose imports are what the tests - look for. + so those are reached too, the entry's own included. It follows only + modules under ``source_roots``, never into the table-model libraries, + whose imports are what the tests look for. """ first_party = _first_party_packages(source_roots) - pending = [entry] + entry_packages = _with_parent_packages(_module_name(entry, source_roots))[:-1] + pending = [ + entry, + *( + module_path + for package_name in entry_packages + if (module_path := _module_path(package_name, source_roots, first_party)) + is not None + ), + ] imports_by_module: dict[str, set[str]] = {} while pending: path = pending.pop() From 79ab78c63d1b8acea43e412d89c33cd191a71ecc Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:33:19 +0400 Subject: [PATCH 25/25] Leave the database ranges to their comparison test in the image-group test `test_v2_image_retains_required_runtime_dependencies` now checks only that the `modal-simulation-image` group installs SQLAlchemy and psycopg; `test_database_dependency_ranges` compares their ranges with the shared table-model library's, so a change to that range edits one test. Review round 3's quality finding. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_stage12_modal_app.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py b/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py index e26731168..f7c1beb60 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py @@ -3,6 +3,7 @@ from __future__ import annotations import importlib +import re import sys import tomllib from contextlib import contextmanager @@ -208,10 +209,14 @@ def test_v2_image_retains_required_runtime_dependencies() -> None: f"policyengine[models]=={get_bundled_package_version('policyengine')}" in group ) assert not any(isinstance(item, dict) for item in group) - # The shared table-model library's ranges, which the Stage 12 stores use - # (test_database_dependency_ranges compares them with that library's). - assert "sqlalchemy>=2,<2.1" in group - assert "psycopg[binary]>=3.2,<4" in group + # The Stage 12 stores need the database packages; their ranges are the + # shared table-model library's, which test_database_dependency_ranges + # compares, so they aren't repeated here. + installed_packages = { + re.split(r"[\[<>=!~;\s]", requirement, maxsplit=1)[0].lower() + for requirement in group + } + assert {"sqlalchemy", "psycopg"} <= installed_packages def test_v2_static_runtime_file_smoke_uses_exact_uk_worker_image(monkeypatch) -> None: