diff --git a/.github/scripts/public-api-schema-worker-login.sh b/.github/scripts/public-api-schema-worker-login.sh new file mode 100755 index 000000000..6e86c604e --- /dev/null +++ b/.github/scripts/public-api-schema-worker-login.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Give the Public API's Modal worker role a login on the disposable database +# the Public API schema job built, so the Stage 12 library's tests connect as +# that role (rule sim.persistence.models-match-public-api). +# +# Usage: public-api-schema-worker-login.sh +# +# PUBLIC_API_SCHEMA_DATABASE_URL names that database, as the superuser's +# postgresql:// URL; the Public API's privilege manifest creates its roles +# without a login. MODAL_WORKER_TEST_PASSWORD is the password the tests' URL +# gives, which exists only in the job's database. +set -euo pipefail + +database_url="${PUBLIC_API_SCHEMA_DATABASE_URL:?PUBLIC_API_SCHEMA_DATABASE_URL must name the database the job built}" +password="${MODAL_WORKER_TEST_PASSWORD:?MODAL_WORKER_TEST_PASSWORD must give the password the Modal worker role logs in with}" + +# psql quotes the password variable as a literal (:'password'), so it never +# appears in the statement itself. +psql "${database_url}" --no-psqlrc --set=ON_ERROR_STOP=1 "--set=password=${password}" <<'SQL' +ALTER ROLE policyengine_v2_modal_worker LOGIN PASSWORD :'password'; +SQL diff --git a/.github/workflows/pr-image-smoke.yml b/.github/workflows/pr-image-smoke.yml index 7755e947b..7dccf79b3 100644 --- a/.github/workflows/pr-image-smoke.yml +++ b/.github/workflows/pr-image-smoke.yml @@ -18,6 +18,7 @@ on: - 'libs/policyengine-simulation-contract/**' - 'libs/policyengine-simulation-observability/**' - 'libs/policyengine-fastapi/**' + - 'libs/policyengine-simulation-persistence/**' - 'libs/policyengine-stage12-persistence/**' - '.github/workflows/pr-image-smoke.yml' - '.github/scripts/modal-image-smoke.sh' diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 068690334..e5aad93e4 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -136,8 +136,10 @@ jobs: # commit pinned in libs/policyengine-simulation-persistence/public-api-commit, # grants the privileges its manifest declares, and runs the persistence # tests marked public_api_schema against them: sim's table models must - # match those tables, and the simulation service's role must write them - # (rule sim.persistence.models-match-public-api). The pinned commit must + # match those tables, and the simulation service's role and the Modal + # worker role must write the tables each writes (rule + # sim.persistence.models-match-public-api). It then runs the Stage 12 + # library's tests there, as the Modal worker role. The pinned commit must # also be on the Public API's default branch. The Public API's repository # is public, so the job reads no secret. Full CI runs only on the lowest # unmerged pull request in a stack. @@ -213,6 +215,27 @@ jobs: working-directory: libs/policyengine-simulation-persistence run: uv run --frozen pytest tests/ -v -m public_api_schema + # The Stage 12 library's stores write the same tables as the Modal worker + # role, which the manifest creates without a login: the password exists + # only in this job's database. + - name: Give the Modal worker role a login on the job's database + env: + MODAL_WORKER_TEST_PASSWORD: modal_worker_ci + run: .github/scripts/public-api-schema-worker-login.sh + + - name: Install the Stage 12 library's dependencies + working-directory: libs/policyengine-stage12-persistence + run: uv sync --extra test --frozen + + - name: Run the Stage 12 library's tests as the Modal worker role + working-directory: libs/policyengine-stage12-persistence + env: + REPORT_RUN_TEST_DATABASE_URL: postgresql://policyengine_v2_modal_worker:modal_worker_ci@127.0.0.1:5432/policyengine_v2_alembic_test + REPORT_RUN_TEST_SEED_DATABASE_URL: postgresql+psycopg://postgres:simulation_ci@127.0.0.1:5432/policyengine_v2_alembic_test + # A missing database fails the tests instead of skipping them. + REPORT_RUN_REQUIRE_DATABASE_TESTS: "1" + run: uv run --frozen pytest tests/ -v + - name: Require the pinned commit on the Public API's default branch run: .github/scripts/public-api-commit-on-default-branch.sh public-api master diff --git a/changelog.d/c1-32-one-table-model-library.changed.md b/changelog.d/c1-32-one-table-model-library.changed.md new file mode 100644 index 000000000..c5e177648 --- /dev/null +++ b/changelog.d/c1-32-one-table-model-library.changed.md @@ -0,0 +1 @@ +The simulation service and the v2 Modal runtime use one library of the Public API's table models, `policyengine-simulation-persistence`. The Stage 12 library's stores, which the v2 coordinators, workers, household functions and stuck-run sweep use, now import its models of `report_runs`, `simulation_runs`, `budget_window_runs` and `household_jobs`, and its new complete models of the twelve result tables, instead of partial models of their own, and connect with its engine and URL check. They write the same columns as before. The v2 Modal images copy the library. CI's comparison with the Public API's tables now covers the result tables, a new test checks the Modal worker role's grants and writes against the Public API's, and the Stage 12 library's database tests, which skipped in CI, run in the `Public API schema` job as that role. A malformed Modal worker database URL now raises the service's "Simulation database URL" error. diff --git a/docs/engineering/skills/database-access.md b/docs/engineering/skills/database-access.md index f6325820e..84572177a 100644 --- a/docs/engineering/skills/database-access.md +++ b/docs/engineering/skills/database-access.md @@ -46,13 +46,14 @@ and this repository has no migration chain of its own Their read-only models in that library's `input_tables.py` name only the columns the service reads, apart from the models of the tables it writes. - the run tables the v2 Modal coordinator, workers, household functions and - stuck-run sweep write as the Modal worker role: `report_runs`, `simulation_runs`, - `budget_window_runs`, the result tables and `household_jobs`. Their mappings in - `policyengine_stage12_persistence.report_run_tables` and - `report_run_results`, separate from the simulation service's models, - declare only the columns the coordinator's, workers', household functions' - and stuck-run sweep's statements read or write; deployment validation - doesn't compare them with the live schema. + stuck-run sweep write as the Modal worker role: `report_runs`, + `simulation_runs`, `budget_window_runs`, the result tables and + `household_jobs`. They use the same models, in + `libs/policyengine-simulation-persistence` (`tables.py` and + `result_tables.py`, which `sim.persistence.models-match-public-api` + compares with the Public API's tables), through the stores in + `libs/policyengine-stage12-persistence`. The v1 `run_simulation` code + imports neither library. This repository's code reads and writes their rows; nothing here changes them. A change to one of those tables starts in policyengine-api, and the diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 0b43a4a3f..c7c4f6a75 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -142,8 +142,8 @@ flowchart LR | Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | | --- | --- | --- | --- | --- | --- | --- | --- | -| `sim.persistence.missing-privileges-reported` | The simulation service's privilege check, which its readiness check uses, names every table privilege and every column UPDATE privilege its role needs on the Public API's tables and lacks, as PostgreSQL reports it, counts a table missing from the public schema as lacking all of them and a missing column as lacking its own, and names a missing USAGE privilege on that schema. | mechanical | enforced | yes | pytest `libs/policyengine-simulation-persistence/tests/test_access.py` | `pr.yml:simulation-database` | yes | -| `sim.persistence.models-match-public-api` | Sim's models of the Public API's tables match the tables the Public API's v2 migration chain builds at the pinned Public API commit, which is on the Public API's default branch: every column's type, nullability and default, every constraint and index by name and definition, and every enum's values; every column its read-only models of the Public API's input tables name exists there with the same type and nullability; and the simulation service's role, with the privileges the Public API's manifest declares, writes every modelled table. | mechanical | enforced | yes | pytest `libs/policyengine-simulation-persistence/tests/test_public_api_schema.py`, `libs/policyengine-simulation-persistence/tests/test_service_writes.py` | `pr.yml:public-api-schema`, `public-api-schema-drift.yml:public-api-schema-drift` | yes | +| `sim.persistence.missing-privileges-reported` | The privilege check, which the simulation service's readiness check uses and CI runs for the Modal worker role too, names every table privilege and every column UPDATE privilege the connected role needs on the Public API's tables and lacks, as PostgreSQL reports it, counts a table missing from the public schema as lacking all of them and a missing column as lacking its own, and names a missing USAGE privilege on that schema. | mechanical | enforced | yes | pytest `libs/policyengine-simulation-persistence/tests/test_access.py` | `pr.yml:simulation-database` | yes | +| `sim.persistence.models-match-public-api` | Sim's one set of models of the Public API's tables, those the simulation service writes and the result tables the v2 Modal calls write, match the tables the Public API's v2 migration chain builds at the pinned Public API commit, which is on the Public API's default branch: every column's type, nullability and default, every constraint and index by name and definition, and every enum's values; every column its read-only models of the Public API's input tables name exists there with the same type and nullability; and, with the privileges the Public API's manifest declares, the simulation service's role writes every table the service writes, and the Modal worker role records its runs and inserts a row in every result table. | mechanical | enforced | yes | pytest `libs/policyengine-simulation-persistence/tests/test_modal_worker_writes.py`, `libs/policyengine-simulation-persistence/tests/test_public_api_schema.py`, `libs/policyengine-simulation-persistence/tests/test_service_writes.py` | `pr.yml:public-api-schema`, `public-api-schema-drift.yml:public-api-schema-drift` | yes | ```mermaid flowchart LR @@ -151,14 +151,14 @@ flowchart LR e_pytest_libs_policyengine_simulation_persistence_tests_test_access_py["pytest libs/policyengine-simulation-persistence/tests/test_access.py"] j_CI_pr_yml_simulation_database["CI pr.yml:simulation-database"] r_sim_persistence_models_match_public_api("sim.persistence.models-match-public-api (enforced)") - e_pytest_libs_policyengine_simulation_persistence_tests_test_public_api_schema_py__libs_policyengine_simulation_persistence_tests_test_service_writes_py["pytest libs/policyengine-simulation-persistence/tests/test_public_api_schema.py, libs/policyengine-simulation-persistence/tests/test_service_writes.py"] + e_pytest_libs_policyengine_simulation_persistence_tests_test_modal_worker_writes_py__libs_policyengine_simulation_persistence_tests_test_public_api_schema_py__libs_policyengine_simulation_persistence_tests_test_service_writes_py["pytest libs/policyengine-simulation-persistence/tests/test_modal_worker_writes.py, libs/policyengine-simulation-persistence/tests/test_public_api_schema.py, libs/policyengine-simulation-persistence/tests/test_service_writes.py"] j_CI_pr_yml_public_api_schema["CI pr.yml:public-api-schema"] j_CI_public_api_schema_drift_yml_public_api_schema_drift["CI public-api-schema-drift.yml:public-api-schema-drift"] r_sim_persistence_missing_privileges_reported --> e_pytest_libs_policyengine_simulation_persistence_tests_test_access_py e_pytest_libs_policyengine_simulation_persistence_tests_test_access_py --> j_CI_pr_yml_simulation_database - r_sim_persistence_models_match_public_api --> e_pytest_libs_policyengine_simulation_persistence_tests_test_public_api_schema_py__libs_policyengine_simulation_persistence_tests_test_service_writes_py - e_pytest_libs_policyengine_simulation_persistence_tests_test_public_api_schema_py__libs_policyengine_simulation_persistence_tests_test_service_writes_py --> j_CI_pr_yml_public_api_schema - e_pytest_libs_policyengine_simulation_persistence_tests_test_public_api_schema_py__libs_policyengine_simulation_persistence_tests_test_service_writes_py --> j_CI_public_api_schema_drift_yml_public_api_schema_drift + r_sim_persistence_models_match_public_api --> e_pytest_libs_policyengine_simulation_persistence_tests_test_modal_worker_writes_py__libs_policyengine_simulation_persistence_tests_test_public_api_schema_py__libs_policyengine_simulation_persistence_tests_test_service_writes_py + e_pytest_libs_policyengine_simulation_persistence_tests_test_modal_worker_writes_py__libs_policyengine_simulation_persistence_tests_test_public_api_schema_py__libs_policyengine_simulation_persistence_tests_test_service_writes_py --> j_CI_pr_yml_public_api_schema + e_pytest_libs_policyengine_simulation_persistence_tests_test_modal_worker_writes_py__libs_policyengine_simulation_persistence_tests_test_public_api_schema_py__libs_policyengine_simulation_persistence_tests_test_service_writes_py --> j_CI_public_api_schema_drift_yml_public_api_schema_drift ``` ## Deployment pipeline diff --git a/libs/policyengine-simulation-persistence/README.md b/libs/policyengine-simulation-persistence/README.md index cdd29d74a..a25ad7771 100644 --- a/libs/policyengine-simulation-persistence/README.md +++ b/libs/policyengine-simulation-persistence/README.md @@ -1,12 +1,20 @@ # PolicyEngine simulation persistence -Database connections for the simulation service (the Simulation Entrypoint on -Cloud Run). The service connects to the Public API's Postgres database with -its own role, and writes rows in tables the Public API's migration chain -creates. This repository has no schema or migration chain of its own. - -The temporary Stage 12 comparison tables are not part of this library: -`libs/policyengine-stage12-persistence` writes them with its own connection. +Database connections and table models for the simulation service (the +Simulation Entrypoint on Cloud Run) and the v2 Modal runtime. The service +connects to the Public API's Postgres database with its own role, and writes +rows in tables the Public API's migration chain creates. This repository has +no schema or migration chain of its own. + +This library holds sim's one set of models of the Public API's tables. The v2 +Modal coordinators, workers, household functions and stuck-run sweep write +the run tables and the result tables through +`libs/policyengine-stage12-persistence`, whose stores import these models and +connect as the Modal worker role (`policyengine_v2_modal_worker`). The v1 +`run_simulation` code (the old gateway and `src/modal/app.py` with its +modules) imports neither library. The temporary Stage 12 comparison tables +are not modelled here: the Stage 12 library keeps them until the comparison +path is removed. ## Connections @@ -17,6 +25,12 @@ server-side statements, so the same URL works through Supabase's session and transaction poolers. A malformed URL raises `ValueError` without repeating the URL, which holds a password. +`create_unpooled_engine` builds the engine the v2 Modal calls and the +comparison path use, from the same URL check: it keeps no pool, so each +operation's session ends with it and a call that spends most of its life +calculating holds no pooler slot, and it waits 10 seconds to connect. The +Stage 12 library binds it as `create_stage12_engine`. + ## Tables `tables.py` models, with SQLAlchemy Core, the six tables the service creates @@ -29,6 +43,9 @@ every column with its type, nullability and server default, and every constraint and index under the chain's name. The models only describe the tables; runtime code never creates or changes them. Status and other enum columns use the `StrEnum` classes beside them and store each member's value. +A Python `None` in `simulation_runs.household_result`, `household_jobs.result` +or `report_runs.spm_provenance` is stored as SQL `NULL`, not JSON `null`, so +a query for `IS NULL` finds a run with no result. Each service writes only its own tables, and the foreign keys between them stay: 20 keys name rows the Public API writes in its users, policies, @@ -47,6 +64,21 @@ household report with only a baseline). A report with a `budget_window_size` is calculated as a `budget_window_runs` row with one report run per year; each year's report run names that row and its year, and belongs to the same report. +## Result tables + +`result_tables.py` models the twelve result tables the Modal worker role +inserts into, in `result_metadata`, as completely as `tables.py` models the +service's: `budget_summary`, `decile_impacts`, `intra_decile_impacts`, +`program_statistics`, `congressional_district_impacts`, +`constituency_impacts`, `local_authority_impacts`, `withheld_area_codes`, +`labor_supply_responses` and `labor_supply_decile_responses` belong to a +report run (all but the labour supply tables also name its two simulation +runs), and `poverty` and `inequality` to one simulation run. Their keys name +the models of `report_runs` and `simulation_runs` in `service_metadata`, so +code that builds the tables in a test database builds those first. The +Stage 12 library's `report_run_results.py` builds their rows from a report +run's economy output. + ## Input tables `input_tables.py` holds read-only models of the Public API's tables a @@ -312,24 +344,31 @@ Rule `sim.persistence.models-match-public-api`. The PR workflow's grants the privileges its manifest declares (`.github/scripts/public-api-schema-reference.sh`), then runs the tests marked `public_api_schema`. `test_public_api_schema` builds the models' tables -from `service_metadata` in a disposable database, after the Public API's -tables their keys name (`referenced_metadata`, by `id` alone), and compares -the models' tables with the Public API's through PostgreSQL's catalog -(`tests/schema_comparison.py`): every column's type, nullability and default, -every constraint and index by name and definition, and every enum's values -must match. `test_the_input_models_match_the_public_api_columns` builds +from `service_metadata`, then `result_metadata`, in a disposable database, +after the Public API's tables their keys name (`referenced_metadata`, by +`id` alone), and compares the models' tables with the Public API's through +PostgreSQL's catalog (`tests/schema_comparison.py`): every column's type, +nullability and default, every constraint and index by name and definition, +and every enum's values must match, for the service's tables and the result +tables alike. `test_the_input_models_match_the_public_api_columns` builds the read-only input models' tables in another disposable database and requires every column they name to exist in the Public API's table with the same type and nullability; their other columns, defaults, constraints and -indexes are the Public API's alone. `test_service_writes` creates a row in every modelled -table as the service's role. Another test, outside the rule, requires the modelled +indexes are the Public API's alone. `test_service_writes` creates a row in every table +the service writes, as the service's role; `test_modal_worker_writes`, as the +Modal worker role, checks the role's privileges, records a run's state in +each run table, inserts a row in every result table and reads back that a +`None` JSON result is SQL `NULL`. Another test, outside the rule, requires the modelled tables' foreign keys to the Public API's users, policies, dynamics, households, datasets, regions and model catalog tables to be exactly the 20 in `tests/public_api_keys.py`, each by its own table, its name and the table it names, so a key dropped, added, renamed, moved or pointed at another table -shows even if the models follow it. Last, the job requires the -pinned commit to be on the Public API's `master` branch, so the models only -ever describe a chain the Public API has merged. +shows even if the models follow it. The job then gives the Modal worker role +a login on its database (`.github/scripts/public-api-schema-worker-login.sh`) +and runs the Stage 12 library's tests, whose stores write these tables as +that role. Last, it requires the pinned commit to be on the Public API's +`master` branch, so the models only ever describe a chain the Public API has +merged. **Choosing the pin.** Pin the Public API commit whose migration chain the models describe: the merge commit of the Public API change that last @@ -338,9 +377,11 @@ head commit. Until that pull request merges, the job's last step fails and says so. A merge commit keeps the pull request's head on `master`; if the Public API squashes or rebases the pull request instead, its head never reaches `master`, so move the pin to the commit that landed. The pin may -also move to a later merged commit that alters no modelled table, such as -one adding a table another package writes; the job still checks only this -package's tables, so moving the pin is no check of that other table. +also move to a later merged commit that alters no modelled table. Every +run and result table sim writes is modelled here, so a change to any of them +moves the pin with this library's models; only the temporary comparison +path's `stage12_evaluation_*` tables, which the Stage 12 library models and +checks before each deploy, are outside this comparison. **Keeping it current.** A sim pull request that changes the models moves the pin in the same change. The `Public API schema drift` workflow runs the same @@ -363,7 +404,7 @@ PUBLIC_API_SCHEMA_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/pol uv run pytest tests/ -m public_api_schema ``` -## The role's privileges +## The roles' privileges `SERVICE_TABLE_PRIVILEGES` lists the table privileges the service's role needs: `SELECT` and `INSERT` on the tables it creates rows in @@ -380,6 +421,16 @@ when Modal refuses one of the first years the service starts. The role holds refuses any other change to a run. The Public API's privilege manifest grants them all. +`MODAL_WORKER_TABLE_PRIVILEGES` and `MODAL_WORKER_UPDATABLE_COLUMNS` list what +the Modal worker role needs: `SELECT` on the four run tables, `INSERT` on +every result table, and `UPDATE` of each run's state (`status`, +`modal_call_id`, `error_message`, `started_at`, `completed_at` and +`updated_at`; a budget-window run has no call of its own) and of the results +its own row holds: a report run's cliff impact, `spm_provenance` and +`data_version`, a simulation run's output file and `household_result`, and a +household job's `result`. The Public API's manifest grants them; +`test_modal_worker_writes` checks that in CI. + `missing_table_privileges` returns each one the connected role lacks, ordered by table, as `"INSERT on reports"` or `"UPDATE (modal_call_id) on report_runs"`; it asks PostgreSQL with `has_table_privilege` and @@ -409,8 +460,11 @@ rule's proof among them needs no database, and the others need one the Public API's own v2 migration chain built, with the privileges its manifest declares granted: set `PUBLIC_API_SCHEMA_DATABASE_URL` to that database's superuser URL. -`test_service_writes` creates a row in every modelled table as the service's -role and rolls it back; the comparison in `test_public_api_schema` also needs +`test_service_writes` creates a row in every table the service writes as the +service's role, and `test_modal_worker_writes` writes the Modal worker role's +runs and results as that role, each rolling its transaction back; neither +role needs a login there, since the superuser takes each role for the +transaction. The comparison in `test_public_api_schema` also needs `SIMULATION_TEST_DATABASE_URL`, to build the models' own tables. Without the variables they skip. diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py index 620dfdbc7..8d05ea589 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py @@ -1,6 +1,8 @@ """Connections to the Public API's database for the simulation service.""" from policyengine_simulation_persistence.access import ( + MODAL_WORKER_TABLE_PRIVILEGES, + MODAL_WORKER_UPDATABLE_COLUMNS, SERVICE_TABLE_PRIVILEGES, SERVICE_UPDATABLE_COLUMNS, missing_table_privileges, @@ -85,6 +87,7 @@ from policyengine_simulation_persistence.engine import ( bounded_transaction, create_simulation_engine, + create_unpooled_engine, normalized_database_url, ) from policyengine_simulation_persistence.household_job_records import ( @@ -145,6 +148,11 @@ read_dynamic_reform, read_policy_reform, ) +from policyengine_simulation_persistence.result_tables import ( + DecileType, + UKAreaType, + result_metadata, +) from policyengine_simulation_persistence.shared_rows import ( FoundReportRun, FoundRow, @@ -181,11 +189,14 @@ "KEY_VERSION", "LATEST_YEAR", "MAX_BUDGET_WINDOW_YEARS", + "MODAL_WORKER_TABLE_PRIVILEGES", + "MODAL_WORKER_UPDATABLE_COLUMNS", "SERVICE_TABLE_PRIVILEGES", "SERVICE_UPDATABLE_COLUMNS", "BudgetWindowInputs", "CatalogDataset", "CatalogRegion", + "DecileType", "EconomyCatalogRows", "EconomyPopulation", "EconomyRecordOutcomeUnknownError", @@ -253,6 +264,7 @@ "SimulationStatus", "SimulationType", "StoredHousehold", + "UKAreaType", "UnknownCatalogVersionError", "UnknownDatasetError", "UnknownDynamicError", @@ -268,6 +280,7 @@ "canonical_json", "combine_reforms", "create_simulation_engine", + "create_unpooled_engine", "definition_key", "fail_unstarted_household_job", "fail_unstarted_report_run", @@ -313,6 +326,7 @@ "referenced_metadata", "report_runs", "reports", + "result_metadata", "service_metadata", "simulation_runs", "simulations", diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/access.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/access.py index bafb7799e..6b2216ac8 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/access.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/access.py @@ -1,9 +1,11 @@ -"""The privileges the simulation service's role needs on the Public API's tables. +"""The privileges sim's two roles need on the Public API's tables. -The Public API's privilege manifest grants the role these privileges, and -its migration chain creates the tables. The readiness check reports any the -connected role lacks, which shows that the deployment's database URL and the -role's grants both reached the running service. +The Public API's privilege manifest grants each role these privileges, and +its migration chain creates the tables. The simulation service's readiness +check reports any its role lacks, which shows that the deployment's database +URL and the role's grants both reached the running service. The v2 Modal +calls' role, ``policyengine_v2_modal_worker``, has lists of its own, which +CI checks against the manifest's grants. """ from __future__ import annotations @@ -23,6 +25,7 @@ values, ) +from policyengine_simulation_persistence.result_tables import result_metadata from policyengine_simulation_persistence.tables import TABLE_SCHEMA if TYPE_CHECKING: @@ -98,6 +101,53 @@ } ) +# A run's state, which the Modal call holding it records: its claim, its +# outcome and when each happened. +_RUN_STATE: Final = frozenset( + { + "status", + "modal_call_id", + "error_message", + "started_at", + "completed_at", + "updated_at", + } +) +# The v2 Modal coordinators, workers, household functions and stuck-run sweep +# read the runs they were given and insert their results, once each. +MODAL_WORKER_TABLE_PRIVILEGES: Final[Mapping[str, frozenset[str]]] = MappingProxyType( + { + "report_runs": _READ, + "simulation_runs": _READ, + "household_jobs": _READ, + "budget_window_runs": _READ, + **{ + table.name: frozenset({"INSERT"}) + for table in result_metadata.tables.values() + }, + } +) +# The columns the Modal worker role may UPDATE, by table: each run's state and +# the results its own row holds. A budget-window run has no Modal call of its +# own; its years' coordinators record its status, failure and times. +MODAL_WORKER_UPDATABLE_COLUMNS: Final[Mapping[str, frozenset[str]]] = MappingProxyType( + { + "report_runs": _RUN_STATE + | { + "baseline_cliff_gap", + "baseline_cliff_share", + "reform_cliff_gap", + "reform_cliff_share", + "spm_provenance", + "data_version", + }, + "simulation_runs": _RUN_STATE + | {"output_uri", "output_sha256", "household_result"}, + "household_jobs": _RUN_STATE | {"result"}, + "budget_window_runs": _RUN_STATE - {"modal_call_id"}, + } +) + def _public_tables() -> Subquery: """Select the oid and name of each table in the ``public`` schema.""" @@ -114,7 +164,9 @@ def _public_tables() -> Subquery: ) -def _missing_table_grants(connection: Connection) -> list[tuple[str, str, str]]: +def _missing_table_grants( + connection: Connection, table_privileges: Mapping[str, frozenset[str]] +) -> list[tuple[str, str, str]]: """Return ``(table, privilege, "")`` for each table privilege the role lacks.""" required = values( column("table_name", String), @@ -123,7 +175,7 @@ def _missing_table_grants(connection: Connection) -> list[tuple[str, str, str]]: ).data( [ (table_name, privilege) - for table_name, privileges in SERVICE_TABLE_PRIVILEGES.items() + for table_name, privileges in table_privileges.items() for privilege in sorted(privileges) ] ) @@ -146,7 +198,9 @@ def _missing_table_grants(connection: Connection) -> list[tuple[str, str, str]]: return [(table_name, privilege, "") for table_name, privilege in rows] -def _missing_column_grants(connection: Connection) -> list[tuple[str, str, str]]: +def _missing_column_grants( + connection: Connection, updatable_columns: Mapping[str, frozenset[str]] +) -> list[tuple[str, str, str]]: """Return ``(table, "UPDATE", column)`` for each column the role can't update.""" required = values( column("table_name", String), @@ -155,7 +209,7 @@ def _missing_column_grants(connection: Connection) -> list[tuple[str, str, str]] ).data( [ (table_name, column_name) - for table_name, column_names in SERVICE_UPDATABLE_COLUMNS.items() + for table_name, column_names in updatable_columns.items() for column_name in sorted(column_names) ] ) @@ -189,18 +243,28 @@ def _missing_column_grants(connection: Connection) -> list[tuple[str, str, str]] return [(table_name, "UPDATE", column_name) for table_name, column_name in rows] -def missing_table_privileges(connection: Connection) -> list[str]: +def missing_table_privileges( + connection: Connection, + *, + table_privileges: Mapping[str, frozenset[str]] = SERVICE_TABLE_PRIVILEGES, + updatable_columns: Mapping[str, frozenset[str]] = SERVICE_UPDATABLE_COLUMNS, +) -> list[str]: """Return each required privilege the connected role lacks, ordered by table. - A table privilege is named as ``" on "`` and a column - privilege as ``"UPDATE () on
"``. A table missing from the - ``public`` schema lacks every privilege the service needs on it, and a - missing column lacks its own. A role without the USAGE privilege on that - schema can't use any of its grants, so ``"USAGE on schema public"`` comes - first. + The required privileges are the simulation service's unless another + role's lists are given, such as ``MODAL_WORKER_TABLE_PRIVILEGES`` and + ``MODAL_WORKER_UPDATABLE_COLUMNS``. A table privilege is named as + ``" on
"`` and a column privilege as + ``"UPDATE () on
"``. A table missing from the ``public`` + schema lacks every privilege required on it, and a missing column lacks + its own. A role without the USAGE privilege on that schema can't use any + of its grants, so ``"USAGE on schema public"`` comes first. """ missing_grants = sorted( - [*_missing_table_grants(connection), *_missing_column_grants(connection)] + [ + *_missing_table_grants(connection, table_privileges), + *_missing_column_grants(connection, updatable_columns), + ] ) missing = [ f"{privilege} ({column_name}) on {table_name}" diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/engine.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/engine.py index b97fb9af1..ef01257be 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/engine.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/engine.py @@ -1,4 +1,4 @@ -"""Database connections for the simulation service's own role.""" +"""Database connections for the simulation service and the v2 Modal calls.""" from __future__ import annotations @@ -8,6 +8,7 @@ from sqlalchemy import create_engine, func, select from sqlalchemy.engine import URL, make_url from sqlalchemy.exc import ArgumentError +from sqlalchemy.pool import NullPool if TYPE_CHECKING: from collections.abc import Iterator @@ -20,6 +21,8 @@ # enough. POOL_SIZE = 2 MAX_OVERFLOW = 3 +# A v2 Modal call connects across the internet to the database's pooler. +UNPOOLED_CONNECT_TIMEOUT_SECONDS = 10 def normalized_database_url(database_url: str) -> URL: @@ -63,6 +66,21 @@ def create_simulation_engine(database_url: str) -> Engine: ) +def create_unpooled_engine(database_url: str) -> Engine: + """Build an engine that opens a connection per operation and closes it after. + + The v2 Modal coordinators, workers and household functions, and the + comparison path, spend most of their lifetime calculating or waiting, so + each database session ends with its short operation instead of holding a + pooler slot through that idle time. It creates no objects. + """ + return create_engine( + normalized_database_url(database_url), + poolclass=NullPool, + connect_args={"connect_timeout": UNPOOLED_CONNECT_TIMEOUT_SECONDS}, + ) + + @contextmanager def bounded_transaction( engine: Engine, *, statement_timeout_milliseconds: int diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/result_tables.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/result_tables.py new file mode 100644 index 000000000..e3fb9ccd4 --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/result_tables.py @@ -0,0 +1,455 @@ +"""Models of the Public API's result tables, which the Modal worker role writes. + +The Public API's v2 migration chain creates these tables; this module only +describes them, so the v2 Modal runtime can insert typed rows. Each table +matches what the chain builds, column for column, with its constraints under +the same names. They are in ``result_metadata``, apart from the tables the +simulation service creates rows in (``service_metadata``), whose run tables +their keys name. + +Results that compare a report run's baseline and reform belong to the report +run, and name its two simulation runs; poverty and inequality, which describe +one simulation, belong to a simulation run. Deleting a run deletes its +results; a simulation run that a report run's results name can't be deleted. +""" + +from __future__ import annotations + +from enum import StrEnum +from typing import TYPE_CHECKING, Final + +from sqlalchemy import ( + Boolean, + CheckConstraint, + Column, + Float, + ForeignKeyConstraint, + Integer, + MetaData, + PrimaryKeyConstraint, + String, + Table, + UniqueConstraint, + Uuid, +) + +from policyengine_simulation_persistence.tables import ( + TABLE_SCHEMA, + _enum_type, + _identifier, + _timestamp, + report_runs, + simulation_runs, +) + +if TYPE_CHECKING: + from sqlalchemy.sql.schema import SchemaItem + +NAME_LENGTH: Final = 512 +ENTITY_LENGTH: Final = 128 +AREA_CODE_LENGTH: Final = 64 +AREA_NAME_LENGTH: Final = 255 + + +class DecileType(StrEnum): + """Whether deciles rank households by income or by wealth.""" + + INCOME = "income" + WEALTH = "wealth" + + +class UKAreaType(StrEnum): + """The level of UK area a local-area breakdown reports.""" + + CONSTITUENCY = "constituency" + LOCAL_AUTHORITY = "local_authority" + + +result_metadata = MetaData(schema=TABLE_SCHEMA) + + +def _result_columns(table_name: str) -> tuple[SchemaItem, ...]: + """Return every result's ``id`` and ``created_at``, and its primary key.""" + return ( + _identifier(), + _timestamp("created_at", server_now=True), + PrimaryKeyConstraint("id", name=f"pk_{table_name}"), + ) + + +def _of_report_run( + table_name: str, + *, + baseline_key_name: str | None = None, + reform_key_name: str | None = None, +) -> tuple[SchemaItem, ...]: + """Return the keys of a result of a report run that compares its two runs. + + The key names are the Public API's, which its chain shortens past + PostgreSQL's 63 characters and ends in four characters of their hash; a + shortened name is passed in. + """ + return ( + *_of_report_run_alone(table_name), + Column("baseline_simulation_run_id", Uuid, nullable=False), + Column("reform_simulation_run_id", Uuid, nullable=False), + ForeignKeyConstraint( + ["baseline_simulation_run_id"], + [simulation_runs.c.id], + name=baseline_key_name + or f"fk_{table_name}_baseline_simulation_run_id_simulation_runs", + ondelete="RESTRICT", + ), + ForeignKeyConstraint( + ["reform_simulation_run_id"], + [simulation_runs.c.id], + name=reform_key_name + or f"fk_{table_name}_reform_simulation_run_id_simulation_runs", + ondelete="RESTRICT", + ), + ) + + +def _of_report_run_alone(table_name: str) -> tuple[SchemaItem, ...]: + """Return the key of a result of a report run that names no simulation run.""" + return ( + *_result_columns(table_name), + Column("report_run_id", Uuid, nullable=False), + ForeignKeyConstraint( + ["report_run_id"], + [report_runs.c.id], + name=f"fk_{table_name}_report_run_id_report_runs", + ondelete="CASCADE", + ), + ) + + +def _of_simulation_run(table_name: str) -> tuple[SchemaItem, ...]: + """Return the key of a result of one simulation run.""" + return ( + *_result_columns(table_name), + Column("simulation_run_id", Uuid, nullable=False), + ForeignKeyConstraint( + ["simulation_run_id"], + [simulation_runs.c.id], + name=f"fk_{table_name}_simulation_run_id_simulation_runs", + ondelete="CASCADE", + ), + ) + + +def _values(*names: str, nullable: bool) -> tuple[Column[float], ...]: + return tuple(Column(name, Float, nullable=nullable) for name in names) + + +def _area_values() -> tuple[Column[float], ...]: + """Return a geographic impact's household income changes and population.""" + return _values( + "average_household_income_change", + "relative_household_income_change", + "population", + nullable=False, + ) + + +def _disclosure(table_name: str) -> tuple[SchemaItem, ...]: + """Return a UK area's disclosure tier and the bounds of its income changes. + + The tier is ``reported``, or ``approximate`` for a thin sample; the bounds + are each income change's 90% interval. All are null without disclosure + control. + """ + return ( + Column("disclosure_tier", String(16), nullable=True), + *_values( + "average_household_income_change_lower", + "average_household_income_change_upper", + "relative_household_income_change_lower", + "relative_household_income_change_upper", + nullable=True, + ), + CheckConstraint( + "disclosure_tier IS NULL OR disclosure_tier IN ('reported', 'approximate')", + name=f"ck_{table_name}_disclosure_tier", + ), + ) + + +labor_supply_responses = Table( + "labor_supply_responses", + result_metadata, + *_of_report_run_alone("labor_supply_responses"), + *_values( + "income_effect_earnings_change", + "substitution_effect_earnings_change", + "total_earnings_change", + "revenue_change", + "relative_income_effect", + "relative_substitution_effect", + "baseline_weekly_hours", + "reform_weekly_hours", + "weekly_hours_change", + "weekly_hours_income_effect", + "weekly_hours_substitution_effect", + nullable=False, + ), + UniqueConstraint("report_run_id", name="uq_labor_supply_responses_run"), +) + +labor_supply_decile_responses = Table( + "labor_supply_decile_responses", + result_metadata, + *_of_report_run_alone("labor_supply_decile_responses"), + Column("decile", Integer, nullable=False), + *_values( + "average_income_effect", + "average_substitution_effect", + "relative_income_effect", + "relative_substitution_effect", + nullable=True, + ), + UniqueConstraint( + "report_run_id", + "decile", + name="uq_labor_supply_decile_responses_run_decile", + ), +) + +budget_summary = Table( + "budget_summary", + result_metadata, + *_of_report_run("budget_summary"), + Column("variable_name", String(NAME_LENGTH), nullable=False), + Column("entity", String(ENTITY_LENGTH), nullable=False), + *_values("baseline_total", "reform_total", "change", nullable=True), + UniqueConstraint( + "report_run_id", + "variable_name", + "entity", + name="uq_budget_summary_run_variable_entity", + ), +) + +# ``decile`` runs from 1 to ``quantiles``; rows without a decile variable +# count as equal. +decile_impacts = Table( + "decile_impacts", + result_metadata, + *_of_report_run("decile_impacts"), + Column("income_variable", String(NAME_LENGTH), nullable=False), + Column("decile_variable", String(NAME_LENGTH), nullable=True), + Column("entity", String(ENTITY_LENGTH), nullable=False), + Column("decile", Integer, nullable=False), + Column("quantiles", Integer, nullable=False), + *_values( + "baseline_mean", + "reform_mean", + "absolute_change", + "relative_change", + "count_better_off", + "count_worse_off", + "count_no_change", + nullable=True, + ), + UniqueConstraint( + "report_run_id", + "income_variable", + "decile_variable", + "entity", + "decile", + "quantiles", + name="uq_decile_impacts_run_measure", + postgresql_nulls_not_distinct=True, + ), + CheckConstraint("quantiles > 0", name="ck_decile_impacts_quantiles"), + CheckConstraint("decile BETWEEN 1 AND quantiles", name="ck_decile_impacts_decile"), +) + +intra_decile_impacts = Table( + "intra_decile_impacts", + result_metadata, + *_of_report_run( + "intra_decile_impacts", + baseline_key_name="fk_intra_decile_impacts_baseline_simulation_run_id_simu_7cf0", + reform_key_name="fk_intra_decile_impacts_reform_simulation_run_id_simula_467e", + ), + Column( + "decile_type", + _enum_type(DecileType, "v2_decile_type"), + nullable=False, + ), + Column("decile", Integer, nullable=False), + *_values( + "lose_more_than_5pct", + "lose_less_than_5pct", + "no_change", + "gain_less_than_5pct", + "gain_more_than_5pct", + nullable=True, + ), + UniqueConstraint( + "report_run_id", + "decile_type", + "decile", + name="uq_intra_decile_impacts_run_type_decile", + ), + CheckConstraint( + "decile BETWEEN 0 AND 10", + name="ck_intra_decile_impacts_decile", + ), +) + +# Rows without an entity count as equal. +program_statistics = Table( + "program_statistics", + result_metadata, + *_of_report_run( + "program_statistics", + baseline_key_name="fk_program_statistics_baseline_simulation_run_id_simula_faf9", + ), + Column("program_name", String(NAME_LENGTH), nullable=False), + Column("entity", String(ENTITY_LENGTH), nullable=True), + Column("is_tax", Boolean, nullable=True), + *_values( + "baseline_total", + "reform_total", + "change", + "baseline_count", + "reform_count", + "winners", + "losers", + nullable=True, + ), + UniqueConstraint( + "report_run_id", + "program_name", + "entity", + name="uq_program_statistics_run_program_entity", + postgresql_nulls_not_distinct=True, + ), +) + +congressional_district_impacts = Table( + "congressional_district_impacts", + result_metadata, + *_of_report_run( + "congressional_district_impacts", + baseline_key_name="fk_congressional_district_impacts_baseline_simulation_r_9f79", + reform_key_name="fk_congressional_district_impacts_reform_simulation_run_80f5", + ), + *_area_values(), + Column("district_geoid", Integer, nullable=False), + Column("state_fips", Integer, nullable=False), + Column("district_number", Integer, nullable=False), + *_values( + "winner_percentage", + "loser_percentage", + "no_change_percentage", + nullable=False, + ), + UniqueConstraint( + "report_run_id", + "district_geoid", + name="uq_congressional_district_impacts_run_geoid", + ), +) + +constituency_impacts = Table( + "constituency_impacts", + result_metadata, + *_of_report_run( + "constituency_impacts", + baseline_key_name="fk_constituency_impacts_baseline_simulation_run_id_simu_d2cd", + reform_key_name="fk_constituency_impacts_reform_simulation_run_id_simula_aaa1", + ), + *_area_values(), + *_disclosure("constituency_impacts"), + Column("constituency_code", String(AREA_CODE_LENGTH), nullable=False), + Column("constituency_name", String(AREA_NAME_LENGTH), nullable=False), + UniqueConstraint( + "report_run_id", + "constituency_code", + name="uq_constituency_impacts_run_code", + ), +) + +local_authority_impacts = Table( + "local_authority_impacts", + result_metadata, + *_of_report_run( + "local_authority_impacts", + baseline_key_name="fk_local_authority_impacts_baseline_simulation_run_id_s_0d31", + reform_key_name="fk_local_authority_impacts_reform_simulation_run_id_sim_0dc2", + ), + *_area_values(), + *_disclosure("local_authority_impacts"), + Column("local_authority_code", String(AREA_CODE_LENGTH), nullable=False), + Column("local_authority_name", String(AREA_NAME_LENGTH), nullable=False), + UniqueConstraint( + "report_run_id", + "local_authority_code", + name="uq_local_authority_impacts_run_code", + ), +) + +# The codes of the UK areas a local-area run estimated but disclosure control +# withheld, one row per area; none means none were withheld. +withheld_area_codes = Table( + "withheld_area_codes", + result_metadata, + *_of_report_run( + "withheld_area_codes", + baseline_key_name="fk_withheld_area_codes_baseline_simulation_run_id_simul_a169", + ), + Column("area_type", _enum_type(UKAreaType, "v2_uk_area_type"), nullable=False), + Column("code", String(AREA_CODE_LENGTH), nullable=False), + UniqueConstraint( + "report_run_id", + "area_type", + "code", + name="uq_withheld_area_codes_run_type_code", + ), +) + +# Rows without a filter variable or group count as equal, so a run has one +# unfiltered row per poverty type and entity. +poverty = Table( + "poverty", + result_metadata, + *_of_simulation_run("poverty"), + Column("poverty_type", String(ENTITY_LENGTH), nullable=False), + Column("entity", String(ENTITY_LENGTH), nullable=False), + Column("filter_variable", String(NAME_LENGTH), nullable=True), + Column("filter_group", String(64), nullable=True), + *_values("headcount", "total_population", "rate", nullable=True), + UniqueConstraint( + "simulation_run_id", + "poverty_type", + "entity", + "filter_variable", + "filter_group", + name="uq_poverty_simulation_run_measure", + postgresql_nulls_not_distinct=True, + ), +) + +inequality = Table( + "inequality", + result_metadata, + *_of_simulation_run("inequality"), + Column("income_variable", String(NAME_LENGTH), nullable=False), + Column("entity", String(ENTITY_LENGTH), nullable=False), + *_values( + "gini", + "top_10_share", + "top_1_share", + "bottom_50_share", + nullable=True, + ), + UniqueConstraint( + "simulation_run_id", + "income_variable", + "entity", + name="uq_inequality_simulation_run_measure", + ), +) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/tables.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/tables.py index 7e5822411..b06417d16 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/tables.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/tables.py @@ -435,7 +435,10 @@ def _public_api_key( Column("error_message", String, nullable=True), _timestamp("started_at"), _timestamp("completed_at"), - Column("household_result", JSON, nullable=True), + # The Modal worker writes None as SQL NULL here and in the two columns + # below (household_jobs.result, report_runs.spm_provenance), so a query + # for IS NULL finds an empty result. + Column("household_result", JSON(none_as_null=True), nullable=True), Column("output_uri", String(OUTPUT_URI_LENGTH), nullable=True), Column("output_sha256", String(SHA256_LENGTH), nullable=True), _forced_column(), @@ -608,7 +611,7 @@ def _public_api_key( Column("baseline_cliff_share", Float, nullable=True), Column("reform_cliff_gap", Float, nullable=True), Column("reform_cliff_share", Float, nullable=True), - Column("spm_provenance", JSONB, nullable=True), + Column("spm_provenance", JSONB(none_as_null=True), nullable=True), Column("data_version", String(VERSION_LENGTH), nullable=True), _forced_column(), PrimaryKeyConstraint("id", name="pk_report_runs"), @@ -708,7 +711,7 @@ def _public_api_key( nullable=False, ), Column("error_message", String, nullable=True), - Column("result", JSON, nullable=True), + Column("result", JSON(none_as_null=True), nullable=True), _timestamp("started_at"), _timestamp("completed_at"), _timestamp("updated_at", server_now=True), diff --git a/libs/policyengine-simulation-persistence/tests/conftest.py b/libs/policyengine-simulation-persistence/tests/conftest.py index 5a0fd71da..e9bfe6431 100644 --- a/libs/policyengine-simulation-persistence/tests/conftest.py +++ b/libs/policyengine-simulation-persistence/tests/conftest.py @@ -46,6 +46,7 @@ # The role the Public API's privilege manifest manages for the simulation # service; the deployed SIMULATION_SERVICE_DATABASE_URL logs in as it. SERVICE_ROLE = "policyengine_v2_simulation_service" +MODAL_WORKER_ROLE = "policyengine_v2_modal_worker" @dataclass(frozen=True) @@ -143,6 +144,16 @@ def act_as_the_service(connection: Connection) -> None: connection.execute(text(f"SET LOCAL ROLE {SERVICE_ROLE}")) +def act_as_the_modal_worker(connection: Connection) -> None: + """Take the Modal worker role for the rest of the connection's transaction. + + The superuser's own role comes back first, so a test that acted as the + service may act as the worker next. + """ + connection.execute(text("RESET ROLE")) + connection.execute(text(f"SET LOCAL ROLE {MODAL_WORKER_ROLE}")) + + @contextmanager def connection_as_the_service(database_url: str) -> Iterator[Connection]: """Yield a connection acting as the service's role, rolled back afterwards.""" diff --git a/libs/policyengine-simulation-persistence/tests/service_rows.py b/libs/policyengine-simulation-persistence/tests/service_rows.py new file mode 100644 index 000000000..6179023ef --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/service_rows.py @@ -0,0 +1,233 @@ +"""Rows the simulation service creates, for tests of the roles' writes. + +The tests run against a database the Public API's own v2 migration chain +built (``PUBLIC_API_SCHEMA_DATABASE_URL``; see ``conftest``), inside a +transaction they roll back. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from hashlib import sha256 +from typing import TYPE_CHECKING +from uuid import UUID, uuid4 + +from policyengine_simulation_persistence import ( + HouseholdJobStatus, + ReportRunStatus, + ReportRunTrigger, + SimulationStatus, + SimulationType, + budget_window_runs, + household_jobs, + report_runs, + reports, + simulation_runs, + simulations, +) +from sqlalchemy import insert, text + +if TYPE_CHECKING: + from sqlalchemy import Connection + +COUNTRY_PACKAGE_VERSION = "1.402.0" +POLICYENGINE_VERSION = "3.4.0" +YEAR = 2026 + + +@dataclass(frozen=True) +class PublicApiRows: + """Rows of the Public API's own tables that the service's rows name.""" + + tax_benefit_model_id: UUID + model_version_id: UUID + dataset_id: UUID + household_id: UUID + + +@dataclass(frozen=True) +class ServiceRows: + """A budget-window report's rows, one year's report run and a household job.""" + + baseline_simulation_id: UUID + reform_simulation_id: UUID + report_id: UUID + baseline_run_id: UUID + reform_run_id: UUID + budget_window_run_id: UUID + report_run_id: UUID + household_job_id: UUID + + +def insert_public_api_rows(connection: Connection) -> PublicApiRows: + """Insert, as the superuser, the catalog, dataset and household rows a run names. + + The Public API writes these rows, and the database checks on insert that + each row the service writes names ones that exist. + """ + rows = PublicApiRows( + tax_benefit_model_id=uuid4(), + model_version_id=uuid4(), + dataset_id=uuid4(), + household_id=uuid4(), + ) + connection.execute( + text("INSERT INTO public.tax_benefit_models (id, name) VALUES (:id, :name)"), + { + "id": rows.tax_benefit_model_id, + "name": f"test-{rows.tax_benefit_model_id.hex[:12]}", + }, + ) + connection.execute( + text( + "INSERT INTO public.tax_benefit_model_versions " + "(id, model_id, version, current_law_id, metadata_time_periods) " + "VALUES (:id, :model_id, :version, 1, '{}')" + ), + { + "id": rows.model_version_id, + "model_id": rows.tax_benefit_model_id, + "version": COUNTRY_PACKAGE_VERSION, + }, + ) + connection.execute( + text( + "INSERT INTO public.datasets " + "(id, name, year, is_output_dataset, tax_benefit_model_version_id) " + "VALUES (:id, 'test dataset', :year, false, :model_version_id)" + ), + { + "id": rows.dataset_id, + "year": YEAR, + "model_version_id": rows.model_version_id, + }, + ) + connection.execute( + text( + "INSERT INTO public.households " + "(id, household_data, country_id, canonicalization_version, content_hash) " + "VALUES (:id, '{}', 'us', 1, :content_hash)" + ), + { + "id": rows.household_id, + "content_hash": sha256(rows.household_id.bytes).hexdigest(), + }, + ) + return rows + + +def insert_economy_simulation( + connection: Connection, public_api_rows: PublicApiRows +) -> UUID: + return connection.execute( + insert(simulations) + .values( + simulation_type=SimulationType.ECONOMY, + dataset_id=public_api_rows.dataset_id, + tax_benefit_model_version_id=public_api_rows.model_version_id, + year=YEAR, + ) + .returning(simulations.c.id) + ).scalar_one() + + +def insert_simulation_run( + connection: Connection, simulation_id: UUID, public_api_rows: PublicApiRows +) -> UUID: + return connection.execute( + insert(simulation_runs) + .values( + simulation_id=simulation_id, + year=YEAR, + tax_benefit_model_version_id=public_api_rows.model_version_id, + country_package_version=COUNTRY_PACKAGE_VERSION, + policyengine_version=POLICYENGINE_VERSION, + dataset_id=public_api_rows.dataset_id, + status=SimulationStatus.PENDING, + ) + .returning(simulation_runs.c.id) + ).scalar_one() + + +def insert_service_rows( + connection: Connection, public_api_rows: PublicApiRows +) -> ServiceRows: + """Insert a row in every table the service writes, as a submission would. + + That is a two-year budget-window report's simulations, the report, the + first year's simulation runs, the budget-window run and that year's report + run, and a household job, every run pending. + """ + baseline_simulation_id = insert_economy_simulation(connection, public_api_rows) + reform_simulation_id = insert_economy_simulation(connection, public_api_rows) + report_id = connection.execute( + insert(reports) + .values( + label="Test report", + country="us", + tax_benefit_model_id=public_api_rows.tax_benefit_model_id, + baseline_simulation_id=baseline_simulation_id, + reform_simulation_id=reform_simulation_id, + dataset_id=public_api_rows.dataset_id, + year=YEAR, + budget_window_size=2, + inputs={}, + ) + .returning(reports.c.id) + ).scalar_one() + baseline_run_id = insert_simulation_run( + connection, baseline_simulation_id, public_api_rows + ) + reform_run_id = insert_simulation_run( + connection, reform_simulation_id, public_api_rows + ) + budget_window_run_id = connection.execute( + insert(budget_window_runs) + .values( + report_id=report_id, + tax_benefit_model_version_id=public_api_rows.model_version_id, + trigger=ReportRunTrigger.INITIAL, + recalculate=False, + status=ReportRunStatus.PENDING, + ) + .returning(budget_window_runs.c.id) + ).scalar_one() + report_run_id = connection.execute( + insert(report_runs) + .values( + report_id=report_id, + baseline_simulation_run_id=baseline_run_id, + reform_simulation_run_id=reform_run_id, + budget_window_run_id=budget_window_run_id, + year=YEAR, + country_package_version=COUNTRY_PACKAGE_VERSION, + policyengine_version=POLICYENGINE_VERSION, + status=ReportRunStatus.PENDING, + trigger=ReportRunTrigger.INITIAL, + ) + .returning(report_runs.c.id) + ).scalar_one() + household_job_id = connection.execute( + insert(household_jobs) + .values( + country="us", + request_data={}, + status=HouseholdJobStatus.PENDING, + household_id=public_api_rows.household_id, + year=YEAR, + tax_benefit_model_version_id=public_api_rows.model_version_id, + country_package_version=COUNTRY_PACKAGE_VERSION, + policyengine_version=POLICYENGINE_VERSION, + ) + .returning(household_jobs.c.id) + ).scalar_one() + return ServiceRows( + baseline_simulation_id=baseline_simulation_id, + reform_simulation_id=reform_simulation_id, + report_id=report_id, + baseline_run_id=baseline_run_id, + reform_run_id=reform_run_id, + budget_window_run_id=budget_window_run_id, + report_run_id=report_run_id, + household_job_id=household_job_id, + ) diff --git a/libs/policyengine-simulation-persistence/tests/test_access.py b/libs/policyengine-simulation-persistence/tests/test_access.py index a1be6d2ca..e6740422a 100644 --- a/libs/policyengine-simulation-persistence/tests/test_access.py +++ b/libs/policyengine-simulation-persistence/tests/test_access.py @@ -7,11 +7,14 @@ import pytest from conftest import SERVICE_TEST_ROLE from policyengine_simulation_persistence import ( + MODAL_WORKER_TABLE_PRIVILEGES, + MODAL_WORKER_UPDATABLE_COLUMNS, SERVICE_TABLE_PRIVILEGES, SERVICE_UPDATABLE_COLUMNS, create_simulation_engine, missing_table_privileges, read_missing_table_privileges, + result_metadata, service_metadata, ) from policyengine_simulation_persistence import access as access_module @@ -86,6 +89,65 @@ def test_every_column_the_service_updates_is_modelled() -> None: assert column_names <= set(tables[table_name].columns.keys()), table_name +# The run tables whose status, call and results the Modal worker role records. +MODAL_WORKER_RUN_TABLES = ( + "report_runs", + "simulation_runs", + "household_jobs", + "budget_window_runs", +) +# A run's state, which the Modal call holding it records. +RUN_STATE_COLUMNS = frozenset( + { + "status", + "modal_call_id", + "error_message", + "started_at", + "completed_at", + "updated_at", + } +) + + +def test_the_modal_worker_role_reads_its_runs_and_inserts_their_results() -> None: + result_tables = {table.name for table in result_metadata.tables.values()} + + assert dict(MODAL_WORKER_TABLE_PRIVILEGES) == { + **dict.fromkeys(MODAL_WORKER_RUN_TABLES, frozenset({"SELECT"})), + **dict.fromkeys(result_tables, frozenset({"INSERT"})), + } + + +def test_the_modal_worker_role_updates_its_runs_state_and_results() -> None: + """A call records its run's state and the results the run row holds. + + A budget-window run has no call of its own: the years' coordinators record + its status, failure and times. + """ + assert dict(MODAL_WORKER_UPDATABLE_COLUMNS) == { + "report_runs": RUN_STATE_COLUMNS + | { + "baseline_cliff_gap", + "baseline_cliff_share", + "reform_cliff_gap", + "reform_cliff_share", + "spm_provenance", + "data_version", + }, + "simulation_runs": RUN_STATE_COLUMNS + | {"output_uri", "output_sha256", "household_result"}, + "household_jobs": RUN_STATE_COLUMNS | {"result"}, + "budget_window_runs": RUN_STATE_COLUMNS - {"modal_call_id"}, + } + + +def test_every_column_the_modal_worker_role_updates_is_modelled() -> None: + tables = {table.name: table for table in service_metadata.tables.values()} + + for table_name, column_names in MODAL_WORKER_UPDATABLE_COLUMNS.items(): + assert column_names <= set(tables[table_name].columns.keys()), table_name + + def _required_grants() -> list[tuple[str, ...]]: """Return every grant the service needs, each a tuple. @@ -258,6 +320,42 @@ def test_a_missing_column_lacks_its_privilege(disposable_database): ] +@pytest.mark.postgres +def test_the_check_names_what_a_role_lacks_of_another_roles_lists( + disposable_database, +): + """Given the Modal worker role's lists, the check asks for them instead.""" + _grant( + disposable_database, + [ + "CREATE TABLE public.report_runs (id uuid PRIMARY KEY, " + + ", ".join( + f"{column_name} text" for column_name in sorted(RUN_STATE_COLUMNS) + ) + + ")", + "CREATE TABLE public.poverty (id uuid PRIMARY KEY)", + f"GRANT SELECT ON public.report_runs TO {SERVICE_TEST_ROLE}", + "GRANT UPDATE (status, modal_call_id) ON public.report_runs " + f"TO {SERVICE_TEST_ROLE}", + ], + ) + engine = create_simulation_engine(disposable_database.service_url) + try: + with engine.connect() as connection: + missing = missing_table_privileges( + connection, + table_privileges={ + "report_runs": frozenset({"SELECT"}), + "poverty": frozenset({"INSERT"}), + }, + updatable_columns={"report_runs": frozenset({"status", "started_at"})}, + ) + finally: + engine.dispose() + + assert missing == ["INSERT on poverty", "UPDATE (started_at) on report_runs"] + + def _put_a_decoy_schema_first(database): """Create schema ``decoy``, usable by the role and first on its search_path.""" _grant( diff --git a/libs/policyengine-simulation-persistence/tests/test_engine.py b/libs/policyengine-simulation-persistence/tests/test_engine.py index 6e3121b8a..8d89249f9 100644 --- a/libs/policyengine-simulation-persistence/tests/test_engine.py +++ b/libs/policyengine-simulation-persistence/tests/test_engine.py @@ -5,13 +5,18 @@ from __future__ import annotations +import sqlite3 + import pytest from policyengine_simulation_persistence import ( bounded_transaction, create_simulation_engine, + create_unpooled_engine, normalized_database_url, ) -from sqlalchemy import text +from policyengine_simulation_persistence import engine as engine_module +from sqlalchemy import create_engine, text +from sqlalchemy.pool import NullPool, QueuePool def test_database_url_selects_psycopg_without_changing_the_secret_format() -> None: @@ -70,6 +75,65 @@ def capture_engine(url, **options): assert captured_options["pool_size"] >= 1 +def test_the_unpooled_engine_holds_no_connection_and_selects_psycopg( + monkeypatch, +) -> None: + captured_options = {} + + def capture_engine(url, **options): + captured_options.update(options, url=url) + return object() + + monkeypatch.setattr(engine_module, "create_engine", capture_engine) + + create_unpooled_engine("postgresql://modal_worker:secret@db.example/postgres") + + assert captured_options["url"].drivername == "postgresql+psycopg" + assert captured_options["poolclass"] is NullPool + assert captured_options["connect_args"] == {"connect_timeout": 10} + + +@pytest.mark.parametrize("fail_transaction", [False, True]) +def test_the_unpooled_engine_closes_connections_between_transactions( + monkeypatch, fail_transaction +) -> None: + """A long calculation holds no idle database session after its operation.""" + + def local_engine(_url, **options): + # Exercise the unpooled configuration with a real local DBAPI. + # PostgreSQL's default is QueuePool; SQLite's memory default differs. + options.pop("connect_args") + options.setdefault("poolclass", QueuePool) + return create_engine("sqlite://", **options) + + monkeypatch.setattr(engine_module, "create_engine", local_engine) + engine = create_unpooled_engine("postgresql://runtime:secret@db.example/postgres") + try: + try: + with engine.begin() as connection: + physical_connection = connection.connection.driver_connection + assert connection.exec_driver_sql("SELECT 1").scalar_one() == 1 + if fail_transaction: + _fail_the_calculation() + except RuntimeError: + assert fail_transaction + + with pytest.raises(sqlite3.ProgrammingError, match="closed database"): + physical_connection.execute("SELECT 1") + + with engine.begin() as connection: + assert connection.connection.driver_connection is not physical_connection + assert connection.exec_driver_sql("SELECT 1").scalar_one() == 1 + finally: + engine.dispose() + + +def _fail_the_calculation() -> None: + """Raise the failure a calculation inside the transaction would.""" + msg = "calculation failed" + raise RuntimeError(msg) + + @pytest.mark.postgres def test_a_bounded_transaction_limits_its_own_statements_and_commits( disposable_database, diff --git a/libs/policyengine-simulation-persistence/tests/test_modal_worker_writes.py b/libs/policyengine-simulation-persistence/tests/test_modal_worker_writes.py new file mode 100644 index 000000000..44de60c0e --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_modal_worker_writes.py @@ -0,0 +1,190 @@ +"""The Modal worker role writes its runs and results in the Public API's database. + +The v2 Modal coordinators, workers, household functions and stuck-run sweep +connect as ``policyengine_v2_modal_worker`` and write through this library's +models. These tests run against a database the Public API's own v2 migration +chain built, with the privileges its manifest declares granted +(``PUBLIC_API_SCHEMA_DATABASE_URL``; see ``conftest``). Each works in one +transaction and rolls it back. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any +from uuid import uuid4 + +import pytest +from conftest import ( + act_as_the_modal_worker, + act_as_the_service, + rolled_back_transaction, +) +from policyengine_simulation_persistence import ( + MODAL_WORKER_TABLE_PRIVILEGES, + MODAL_WORKER_UPDATABLE_COLUMNS, + budget_window_runs, + household_jobs, + missing_table_privileges, + report_runs, + result_metadata, + simulation_runs, +) +from service_rows import ServiceRows, insert_public_api_rows, insert_service_rows +from sqlalchemy import Enum, Float, Integer, String, func, insert, select, text, update + +if TYPE_CHECKING: + from sqlalchemy import Connection, Table + +pytestmark = pytest.mark.public_api_schema + +NOW = datetime(2026, 11, 16, tzinfo=UTC) +MODAL_CALL_ID = "fc-modal-worker-test" + + +def test_the_modal_worker_role_holds_every_privilege_the_modal_calls_need( + public_api_database_url, +) -> None: + with rolled_back_transaction(public_api_database_url) as connection: + act_as_the_modal_worker(connection) + + assert ( + missing_table_privileges( + connection, + table_privileges=MODAL_WORKER_TABLE_PRIVILEGES, + updatable_columns=MODAL_WORKER_UPDATABLE_COLUMNS, + ) + == [] + ) + + +def _run_state(service_rows: ServiceRows) -> dict[Table, tuple[Any, dict[str, Any]]]: + """Return, by run table, a run's ID and a value for every column the role updates. + + A Python ``None`` in a JSON result is written as one, so the test can read + back that it is stored as SQL NULL. + """ + finished = { + "modal_call_id": MODAL_CALL_ID, + "started_at": NOW, + "completed_at": NOW, + "updated_at": NOW, + } + return { + report_runs: ( + service_rows.report_run_id, + { + **finished, + "status": "succeeded", + "error_message": None, + "baseline_cliff_gap": 1.0, + "baseline_cliff_share": 0.1, + "reform_cliff_gap": 2.0, + "reform_cliff_share": 0.2, + "spm_provenance": None, + "data_version": "test-data-version", + }, + ), + simulation_runs: ( + service_rows.baseline_run_id, + { + **finished, + "status": "succeeded", + "error_message": None, + "output_uri": "gs://test-bucket/simulation-run.parquet", + "output_sha256": "0" * 64, + "household_result": None, + }, + ), + household_jobs: ( + service_rows.household_job_id, + { + **finished, + "status": "failed", + "error_message": "The calculation failed.", + "result": None, + }, + ), + budget_window_runs: ( + service_rows.budget_window_run_id, + { + "status": "running", + "error_message": None, + "started_at": NOW, + "completed_at": None, + "updated_at": NOW, + }, + ), + } + + +def _result_row(table: Table, service_rows: ServiceRows) -> dict[str, Any]: + """Return a row with a value for each of ``table``'s required columns. + + A result of a report run names it and its two simulation runs; a result + of one simulation run names the baseline's. + """ + keys = { + "report_run_id": service_rows.report_run_id, + "baseline_simulation_run_id": service_rows.baseline_run_id, + "reform_simulation_run_id": service_rows.reform_run_id, + "simulation_run_id": service_rows.baseline_run_id, + } + row: dict[str, Any] = {"id": uuid4()} + for column in table.columns: + if column.name in row or column.server_default is not None: + continue + if column.name in keys: + row[column.name] = keys[column.name] + elif column.nullable: + continue + elif isinstance(column.type, Enum): + row[column.name] = column.type.enums[0] + elif isinstance(column.type, String): + row[column.name] = f"test-{column.name}"[: column.type.length] + elif isinstance(column.type, Integer): + # A decile within every table's checks: 1, of 1 quantile. + row[column.name] = 1 + elif isinstance(column.type, Float): + row[column.name] = 0.5 + else: + msg = f"no test value for {table.name}.{column.name}" + raise AssertionError(msg) + return row + + +def _stored_as_sql_null(connection: Connection, table: Table, column_name: str) -> int: + return connection.execute( + select(func.count()) + .select_from(table) + .where(table.c[column_name].is_(None), table.c.modal_call_id == MODAL_CALL_ID) + ).scalar_one() + + +def test_the_modal_worker_role_records_its_runs_and_inserts_every_result( + public_api_database_url, +) -> None: + with rolled_back_transaction(public_api_database_url) as connection: + public_api_rows = insert_public_api_rows(connection) + act_as_the_service(connection) + service_rows = insert_service_rows(connection, public_api_rows) + act_as_the_modal_worker(connection) + + for table, (run_id, values) in _run_state(service_rows).items(): + assert set(values) == MODAL_WORKER_UPDATABLE_COLUMNS[table.name], table + updated = connection.execute( + update(table).where(table.c.id == run_id).values(**values) + ) + assert updated.rowcount == 1, table.name + for table in result_metadata.tables.values(): + connection.execute(insert(table).values(_result_row(table, service_rows))) + + connection.execute(text("RESET ROLE")) + for table in result_metadata.tables.values(): + stored = connection.execute( + select(func.count()).select_from(table) + ).scalar_one() + assert stored >= 1, table.name + assert _stored_as_sql_null(connection, report_runs, "spm_provenance") == 1 + assert _stored_as_sql_null(connection, simulation_runs, "household_result") == 1 + assert _stored_as_sql_null(connection, household_jobs, "result") == 1 diff --git a/libs/policyengine-simulation-persistence/tests/test_public_api_schema.py b/libs/policyengine-simulation-persistence/tests/test_public_api_schema.py index 728c54232..b163dd170 100644 --- a/libs/policyengine-simulation-persistence/tests/test_public_api_schema.py +++ b/libs/policyengine-simulation-persistence/tests/test_public_api_schema.py @@ -20,6 +20,7 @@ from policyengine_simulation_persistence import ( input_metadata, referenced_metadata, + result_metadata, service_metadata, ) from public_api_keys import KEYS_TO_PUBLIC_API_TABLES @@ -76,6 +77,38 @@ def test_the_models_match_the_public_api_tables( ) +@pytest.mark.public_api_schema +def test_the_result_models_match_the_public_api_tables( + disposable_database, public_api_database_url +) -> None: + """The result tables the Modal worker role writes match, as the service's do. + + Their keys name the service's run tables, so those are built first, after + the Public API's tables the run tables' own keys name. + """ + table_names = [table.name for table in result_metadata.tables.values()] + models_engine = _engine(disposable_database.superuser_url) + real_engine = _engine(public_api_database_url) + try: + # Tests may build the models' tables; runtime code never does. + referenced_metadata.create_all(models_engine) + service_metadata.create_all(models_engine) + result_metadata.create_all(models_engine) + with models_engine.connect() as connection: + models = describe_tables(connection, table_names) + with real_engine.connect() as connection: + real = describe_tables(connection, table_names) + finally: + models_engine.dispose() + real_engine.dispose() + + assert set(models.tables) == set(table_names) + compared_with = os.environ.get(COMMIT_VARIABLE) or "an unrecorded commit" + assert schema_differences(models, real) == [], ( + f"compared with the Public API at {compared_with}" + ) + + @pytest.mark.public_api_schema def test_the_input_models_match_the_public_api_columns( disposable_database, public_api_database_url diff --git a/libs/policyengine-simulation-persistence/tests/test_public_api_schema_job.py b/libs/policyengine-simulation-persistence/tests/test_public_api_schema_job.py index 17153f30b..f3cb68a4a 100644 --- a/libs/policyengine-simulation-persistence/tests/test_public_api_schema_job.py +++ b/libs/policyengine-simulation-persistence/tests/test_public_api_schema_job.py @@ -95,6 +95,56 @@ def test_the_job_builds_the_reference_and_compares_without_skipping() -> None: ) +STAGE12_LIBRARY = "libs/policyengine-stage12-persistence" +MODAL_WORKER_ROLE = "policyengine_v2_modal_worker" + + +def test_the_job_runs_the_stage12_tests_as_the_modal_worker_role() -> None: + """The Stage 12 stores' tests run on the same tables, as the role they use. + + The Public API's manifest creates the role without a login, so a step + gives it one, with a password that exists only in the job's database. + """ + job = _jobs()[JOB_ID] + password = job["services"]["postgres"]["env"]["POSTGRES_PASSWORD"] + names = [step.get("name") for step in job["steps"]] + compare = _step(job, "Compare sim's table models with the Public API's tables") + login = _step(job, "Give the Modal worker role a login on the job's database") + install = _step(job, "Install the Stage 12 library's dependencies") + stage12_tests = _step( + job, "Run the Stage 12 library's tests as the Modal worker role" + ) + on_default_branch = _step( + job, "Require the pinned commit on the Public API's default branch" + ) + worker_password = login["env"]["MODAL_WORKER_TEST_PASSWORD"] + + assert login["run"] == ".github/scripts/public-api-schema-worker-login.sh" + assert install["working-directory"] == STAGE12_LIBRARY + assert install["run"] == "uv sync --extra test --frozen" + assert stage12_tests["working-directory"] == STAGE12_LIBRARY + assert stage12_tests["run"] == "uv run --frozen pytest tests/ -v" + assert stage12_tests["env"] == { + "REPORT_RUN_TEST_DATABASE_URL": ( + f"postgresql://{MODAL_WORKER_ROLE}:{worker_password}@127.0.0.1:5432/" + "policyengine_v2_alembic_test" + ), + "REPORT_RUN_TEST_SEED_DATABASE_URL": ( + f"postgresql+psycopg://postgres:{password}@127.0.0.1:5432/" + "policyengine_v2_alembic_test" + ), + # A missing database fails the tests instead of skipping them. + "REPORT_RUN_REQUIRE_DATABASE_TESTS": "1", + } + assert ( + names.index(compare["name"]) + < names.index(login["name"]) + < names.index(install["name"]) + < names.index(stage12_tests["name"]) + < names.index(on_default_branch["name"]) + ) + + def test_the_job_reads_no_secret() -> None: job_text = yaml.safe_dump(_jobs()[JOB_ID]) diff --git a/libs/policyengine-simulation-persistence/tests/test_public_api_schema_scripts.py b/libs/policyengine-simulation-persistence/tests/test_public_api_schema_scripts.py index d973f4464..392a903b5 100644 --- a/libs/policyengine-simulation-persistence/tests/test_public_api_schema_scripts.py +++ b/libs/policyengine-simulation-persistence/tests/test_public_api_schema_scripts.py @@ -2,8 +2,10 @@ ``public-api-schema-commit.sh`` reads the pinned Public API commit, ``public-api-commit-on-default-branch.sh`` requires it to be on the Public -API's default branch, and ``public-api-schema-reference.sh`` builds the -Public API's tables at that commit and grants its declared privileges. +API's default branch, ``public-api-schema-reference.sh`` builds the +Public API's tables at that commit and grants its declared privileges, and +``public-api-schema-worker-login.sh`` gives the Modal worker role a login there, +for the Stage 12 library's tests. """ from __future__ import annotations @@ -20,6 +22,7 @@ COMMIT_SCRIPT = SCRIPTS / "public-api-schema-commit.sh" DEFAULT_BRANCH_SCRIPT = SCRIPTS / "public-api-commit-on-default-branch.sh" REFERENCE_SCRIPT = SCRIPTS / "public-api-schema-reference.sh" +WORKER_LOGIN_SCRIPT = SCRIPTS / "public-api-schema-worker-login.sh" PINNED_COMMIT = "0123456789abcdef0123456789abcdef01234567" DATABASE_URL = ( "postgresql://postgres:simulation_ci@127.0.0.1:5432/policyengine_v2_alembic_test" @@ -237,3 +240,70 @@ def test_the_reference_needs_a_database_to_migrate(tmp_path): assert result.returncode != 0 assert "PUBLIC_API_SCHEMA_DATABASE_URL" in result.stderr assert not [line for line in commands if line.startswith("uv|")] + + +# --- The Modal worker role's login ---------------------------------------- + + +def _fake_psql(tmp_path, *, exit_status: int = 0) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir(exist_ok=True) + # The script sends its statement on standard input; the fake logs it. + _fake_command( + fake_bin, + "psql", + f'cat >> "${{FAKE_COMMAND_LOG}}"\nexit {exit_status}', + ) + + +def test_the_worker_role_is_given_a_login_with_the_tests_password(tmp_path): + _fake_psql(tmp_path) + + result, log = _run( + WORKER_LOGIN_SCRIPT, + [], + tmp_path, + PUBLIC_API_SCHEMA_DATABASE_URL=DATABASE_URL, + MODAL_WORKER_TEST_PASSWORD="modal_worker_ci", + ) + + assert result.returncode == 0, result.stderr + assert log == [ + f"psql|{DATABASE_URL} --no-psqlrc --set=ON_ERROR_STOP=1 " + "--set=password=modal_worker_ci", + "ALTER ROLE policyengine_v2_modal_worker LOGIN PASSWORD :'password';", + ] + # The password is passed as a psql variable, never written into the SQL. + assert "modal_worker_ci" not in log[1] + + +@pytest.mark.parametrize( + "missing", ["PUBLIC_API_SCHEMA_DATABASE_URL", "MODAL_WORKER_TEST_PASSWORD"] +) +def test_the_login_needs_the_database_and_the_password(tmp_path, missing): + _fake_psql(tmp_path) + environment = { + "PUBLIC_API_SCHEMA_DATABASE_URL": DATABASE_URL, + "MODAL_WORKER_TEST_PASSWORD": "modal_worker_ci", + } + del environment[missing] + + result, log = _run(WORKER_LOGIN_SCRIPT, [], tmp_path, **environment) + + assert result.returncode != 0 + assert missing in result.stderr + assert log == [] + + +def test_a_refused_login_fails_the_step(tmp_path): + _fake_psql(tmp_path, exit_status=3) + + result, _ = _run( + WORKER_LOGIN_SCRIPT, + [], + tmp_path, + PUBLIC_API_SCHEMA_DATABASE_URL=DATABASE_URL, + MODAL_WORKER_TEST_PASSWORD="modal_worker_ci", + ) + + assert result.returncode == 3 diff --git a/libs/policyengine-simulation-persistence/tests/test_result_tables.py b/libs/policyengine-simulation-persistence/tests/test_result_tables.py new file mode 100644 index 000000000..7b6e008f5 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_result_tables.py @@ -0,0 +1,63 @@ +"""Sim's models of the Public API's result tables, which the Modal worker writes.""" + +from __future__ import annotations + +from policyengine_simulation_persistence import ( + report_runs, + result_metadata, + service_metadata, + simulation_runs, +) +from sqlalchemy import Enum + +# Results that compare a report run's two simulation runs belong to the report +# run; poverty and inequality, which describe one simulation, to a simulation +# run. +REPORT_RUN_RESULT_TABLES = { + "budget_summary", + "congressional_district_impacts", + "constituency_impacts", + "decile_impacts", + "intra_decile_impacts", + "labor_supply_decile_responses", + "labor_supply_responses", + "local_authority_impacts", + "program_statistics", + "withheld_area_codes", +} +SIMULATION_RUN_RESULT_TABLES = {"inequality", "poverty"} + + +def test_the_result_models_are_the_twelve_result_tables() -> None: + modelled = {table.name for table in result_metadata.tables.values()} + + assert modelled == REPORT_RUN_RESULT_TABLES | SIMULATION_RUN_RESULT_TABLES + + +def test_the_result_models_share_no_table_with_the_service_models() -> None: + assert not set(result_metadata.tables) & set(service_metadata.tables) + + +def test_each_result_names_the_service_models_run_it_belongs_to() -> None: + for table in result_metadata.tables.values(): + referred_tables = { + constraint.column_keys[0]: constraint.referred_table + for constraint in table.foreign_key_constraints + } + if table.name in SIMULATION_RUN_RESULT_TABLES: + assert referred_tables == {"simulation_run_id": simulation_runs}, table + continue + assert referred_tables["report_run_id"] is report_runs, table + if table.name.startswith("labor_supply_"): + assert set(referred_tables) == {"report_run_id"}, table + else: + assert referred_tables["baseline_simulation_run_id"] is simulation_runs + assert referred_tables["reform_simulation_run_id"] is simulation_runs + + +def test_every_result_table_and_enum_type_names_the_public_schema() -> None: + for table in result_metadata.tables.values(): + assert table.schema == "public", table + for column in table.columns: + if isinstance(column.type, Enum): + assert column.type.schema == "public", column diff --git a/libs/policyengine-simulation-persistence/tests/test_service_writes.py b/libs/policyengine-simulation-persistence/tests/test_service_writes.py index fc4c274ce..ec72684a4 100644 --- a/libs/policyengine-simulation-persistence/tests/test_service_writes.py +++ b/libs/policyengine-simulation-persistence/tests/test_service_writes.py @@ -9,11 +9,6 @@ from __future__ import annotations -from dataclasses import dataclass -from hashlib import sha256 -from typing import TYPE_CHECKING -from uuid import UUID, uuid4 - import pytest from conftest import ( act_as_the_service, @@ -21,11 +16,6 @@ rolled_back_transaction, ) from policyengine_simulation_persistence import ( - HouseholdJobStatus, - ReportRunStatus, - ReportRunTrigger, - SimulationStatus, - SimulationType, budget_window_runs, household_jobs, missing_table_privileges, @@ -34,117 +24,11 @@ simulation_runs, simulations, ) -from sqlalchemy import func, insert, select, text - -if TYPE_CHECKING: - from sqlalchemy import Connection +from service_rows import insert_public_api_rows, insert_service_rows +from sqlalchemy import func, select pytestmark = pytest.mark.public_api_schema -COUNTRY_PACKAGE_VERSION = "1.402.0" -POLICYENGINE_VERSION = "3.4.0" -YEAR = 2026 - - -@dataclass(frozen=True) -class PublicApiRows: - """Rows of the Public API's own tables that the service's rows name.""" - - tax_benefit_model_id: UUID - model_version_id: UUID - dataset_id: UUID - household_id: UUID - - -def _insert_public_api_rows(connection: Connection) -> PublicApiRows: - """Insert, as the superuser, the catalog, dataset and household rows a run names. - - The Public API writes these rows, and the database checks on insert that - each row the service writes names ones that exist. - """ - rows = PublicApiRows( - tax_benefit_model_id=uuid4(), - model_version_id=uuid4(), - dataset_id=uuid4(), - household_id=uuid4(), - ) - connection.execute( - text("INSERT INTO public.tax_benefit_models (id, name) VALUES (:id, :name)"), - { - "id": rows.tax_benefit_model_id, - "name": f"test-{rows.tax_benefit_model_id.hex[:12]}", - }, - ) - connection.execute( - text( - "INSERT INTO public.tax_benefit_model_versions " - "(id, model_id, version, current_law_id, metadata_time_periods) " - "VALUES (:id, :model_id, :version, 1, '{}')" - ), - { - "id": rows.model_version_id, - "model_id": rows.tax_benefit_model_id, - "version": COUNTRY_PACKAGE_VERSION, - }, - ) - connection.execute( - text( - "INSERT INTO public.datasets " - "(id, name, year, is_output_dataset, tax_benefit_model_version_id) " - "VALUES (:id, 'test dataset', :year, false, :model_version_id)" - ), - { - "id": rows.dataset_id, - "year": YEAR, - "model_version_id": rows.model_version_id, - }, - ) - connection.execute( - text( - "INSERT INTO public.households " - "(id, household_data, country_id, canonicalization_version, content_hash) " - "VALUES (:id, '{}', 'us', 1, :content_hash)" - ), - { - "id": rows.household_id, - "content_hash": sha256(rows.household_id.bytes).hexdigest(), - }, - ) - return rows - - -def _insert_economy_simulation( - connection: Connection, public_api_rows: PublicApiRows -) -> UUID: - return connection.execute( - insert(simulations) - .values( - simulation_type=SimulationType.ECONOMY, - dataset_id=public_api_rows.dataset_id, - tax_benefit_model_version_id=public_api_rows.model_version_id, - year=YEAR, - ) - .returning(simulations.c.id) - ).scalar_one() - - -def _insert_simulation_run( - connection: Connection, simulation_id: UUID, public_api_rows: PublicApiRows -) -> UUID: - return connection.execute( - insert(simulation_runs) - .values( - simulation_id=simulation_id, - year=YEAR, - tax_benefit_model_version_id=public_api_rows.model_version_id, - country_package_version=COUNTRY_PACKAGE_VERSION, - policyengine_version=POLICYENGINE_VERSION, - dataset_id=public_api_rows.dataset_id, - status=SimulationStatus.PENDING, - ) - .returning(simulation_runs.c.id) - ).scalar_one() - def test_the_service_role_holds_every_privilege_the_service_needs( public_api_database_url, @@ -157,81 +41,21 @@ def test_the_service_role_creates_and_reads_a_row_in_every_table_it_writes( public_api_database_url, ) -> None: with rolled_back_transaction(public_api_database_url) as connection: - public_api_rows = _insert_public_api_rows(connection) + public_api_rows = insert_public_api_rows(connection) act_as_the_service(connection) - baseline_simulation_id = _insert_economy_simulation(connection, public_api_rows) - reform_simulation_id = _insert_economy_simulation(connection, public_api_rows) - report_id = connection.execute( - insert(reports) - .values( - label="Test report", - country="us", - tax_benefit_model_id=public_api_rows.tax_benefit_model_id, - baseline_simulation_id=baseline_simulation_id, - reform_simulation_id=reform_simulation_id, - dataset_id=public_api_rows.dataset_id, - year=YEAR, - budget_window_size=2, - inputs={}, - ) - .returning(reports.c.id) - ).scalar_one() - baseline_run_id = _insert_simulation_run( - connection, baseline_simulation_id, public_api_rows - ) - reform_run_id = _insert_simulation_run( - connection, reform_simulation_id, public_api_rows - ) - budget_window_run_id = connection.execute( - insert(budget_window_runs) - .values( - report_id=report_id, - tax_benefit_model_version_id=public_api_rows.model_version_id, - trigger=ReportRunTrigger.INITIAL, - recalculate=False, - status=ReportRunStatus.PENDING, - ) - .returning(budget_window_runs.c.id) - ).scalar_one() - report_run_id = connection.execute( - insert(report_runs) - .values( - report_id=report_id, - baseline_simulation_run_id=baseline_run_id, - reform_simulation_run_id=reform_run_id, - budget_window_run_id=budget_window_run_id, - year=YEAR, - country_package_version=COUNTRY_PACKAGE_VERSION, - policyengine_version=POLICYENGINE_VERSION, - status=ReportRunStatus.PENDING, - trigger=ReportRunTrigger.INITIAL, - ) - .returning(report_runs.c.id) - ).scalar_one() - connection.execute( - insert(household_jobs).values( - country="us", - request_data={}, - status=HouseholdJobStatus.PENDING, - household_id=public_api_rows.household_id, - year=YEAR, - tax_benefit_model_version_id=public_api_rows.model_version_id, - country_package_version=COUNTRY_PACKAGE_VERSION, - policyengine_version=POLICYENGINE_VERSION, - ) - ) + service_rows = insert_service_rows(connection, public_api_rows) compared_runs = connection.execute( select( report_runs.c.baseline_simulation_run_id, report_runs.c.reform_simulation_run_id, report_runs.c.budget_window_run_id, - ).where(report_runs.c.id == report_run_id) + ).where(report_runs.c.id == service_rows.report_run_id) ).one() assert tuple(compared_runs) == ( - baseline_run_id, - reform_run_id, - budget_window_run_id, + service_rows.baseline_run_id, + service_rows.reform_run_id, + service_rows.budget_window_run_id, ) for table in ( simulations, diff --git a/libs/policyengine-simulation-persistence/tests/test_tables.py b/libs/policyengine-simulation-persistence/tests/test_tables.py index 40b7181ca..6c3d7685b 100644 --- a/libs/policyengine-simulation-persistence/tests/test_tables.py +++ b/libs/policyengine-simulation-persistence/tests/test_tables.py @@ -184,6 +184,21 @@ def test_a_budget_window_has_one_report_run_per_year_of_one_report() -> None: ) +def test_a_python_none_in_a_runs_json_result_is_stored_as_sql_null() -> None: + """The Modal worker role's ``None`` results are SQL ``NULL``, not JSON ``null``. + + A household job failed outside its calculation has no result, and a report + run whose output has no SPM receipts has no ``spm_provenance``; the stores + write ``None`` for both, and a query for ``IS NULL`` must find them. + """ + for result_column in ( + simulation_runs.c.household_result, + household_jobs.c.result, + report_runs.c.spm_provenance, + ): + assert result_column.type.none_as_null is True, result_column + + def test_the_input_models_are_tables_the_service_reads() -> None: """Each input table is one the service's role may SELECT and never writes.""" readable_only = { diff --git a/libs/policyengine-stage12-persistence/README.md b/libs/policyengine-stage12-persistence/README.md index 814bc1a18..8229d40d3 100644 --- a/libs/policyengine-stage12-persistence/README.md +++ b/libs/policyengine-stage12-persistence/README.md @@ -1,8 +1,15 @@ # PolicyEngine Stage 12 persistence -SQLAlchemy Core mappings and transactional DML for two sets of tables, both -owned by `policyengine-api`, which keeps the SQLModel definitions, Alembic -revisions and live schema. This package never changes a schema. +Transactional DML for two sets of tables, both owned by `policyengine-api`, +which keeps the SQLModel definitions, Alembic revisions and live schema. This +package never changes a schema. + +The models of the Public API's run and result tables are the simulation +service library's (`libs/policyengine-simulation-persistence`: `tables.py` +and `result_tables.py`), which CI compares with the tables the Public API's +chain builds; this package's stores import them, and its engines come from +that library's `create_unpooled_engine`. Its only models of its own are the +comparison path's tables, in `tables.py`. - **The Stage 12 comparison tables** (`Stage12PersistenceStore`): temporary state of the comparison path, removed with it. @@ -18,9 +25,9 @@ revisions and live schema. This package never changes a schema. The coordinator also reads a simulation run's status, year, versions, dataset, region, output file, recorded call and failure, so a report run can use a run that already succeeded instead of calculating it again, or - wait for one another call holds. The mappings declare only the columns - those statements read or write, and each `UPDATE` sets only columns the - role may update. Every status write is one `UPDATE` conditional on the + wait for one another call holds. Every statement names the columns it + reads or writes, so the complete models send no other, and each `UPDATE` + sets only columns the role may update. Every status write is one `UPDATE` conditional on the row's status and, for a call's claim, completion or failure, on its recorded Modal call: a call claims a pending or running row that records no call or its own, and completes or fails it only while it holds it @@ -55,8 +62,11 @@ revisions and live schema. This package never changes a schema. ## Tests The report run and budget-window tests need a disposable Postgres migrated -by the Public API's v2 chain, with its declared privileges applied. They skip -unless both URLs are set: +by the Public API's v2 chain, with its declared privileges applied. The +`Public API schema` job in `pr.yml` runs them on the database it builds at the +pinned Public API commit, after giving the Modal worker role a login there, +and sets `REPORT_RUN_REQUIRE_DATABASE_TESTS=1`, so a missing database fails +them. Elsewhere they skip unless both URLs are set: - `REPORT_RUN_TEST_DATABASE_URL`: logs in as `policyengine_v2_modal_worker`; - `REPORT_RUN_TEST_SEED_DATABASE_URL`: logs in as a role that may insert the diff --git a/libs/policyengine-stage12-persistence/pyproject.toml b/libs/policyengine-stage12-persistence/pyproject.toml index 87587670c..da21f32e8 100644 --- a/libs/policyengine-stage12-persistence/pyproject.toml +++ b/libs/policyengine-stage12-persistence/pyproject.toml @@ -9,8 +9,12 @@ license = {file = "../../LICENSE"} requires-python = ">=3.13,<3.14" dependencies = [ "policyengine-simulation-contract", + # The models of the Public API's run and result tables, which the CI + # comparison checks against the tables the Public API's chain builds. + "policyengine-simulation-persistence", + # The shared library's ranges: the stores use its table models. "psycopg[binary]>=3.2,<4", - "sqlalchemy>=2,<3", + "sqlalchemy>=2,<2.1", ] [project.optional-dependencies] @@ -34,6 +38,7 @@ packages = ["src/policyengine_stage12_persistence"] [tool.uv.sources] policyengine-simulation-contract = { path = "../policyengine-simulation-contract", editable = true } +policyengine-simulation-persistence = { path = "../policyengine-simulation-persistence", editable = true } [tool.pytest.ini_options] pythonpath = ["src"] diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/__init__.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/__init__.py index 7e86cb336..bf45c8159 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/__init__.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/__init__.py @@ -28,7 +28,6 @@ ConnectionOpener, Stage12PersistenceStore, create_stage12_engine, - normalized_database_url, ) from policyengine_stage12_persistence.validation import ( Stage12RuntimeAccessError, @@ -55,7 +54,6 @@ "YearStarter", "YearToStart", "create_stage12_engine", - "normalized_database_url", "report_run_result", "validate_runtime_database", ] diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/budget_window_store.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/budget_window_store.py index 4a025babc..3ff656684 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/budget_window_store.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/budget_window_store.py @@ -28,6 +28,11 @@ from typing import TYPE_CHECKING, Protocol from policyengine_simulation_contract.stage12_execution import ReportRunIds +from policyengine_simulation_persistence.tables import ( + budget_window_runs, + report_runs, + simulation_runs, +) from sqlalchemy import and_, exists, func, or_, select, update from policyengine_stage12_persistence.report_run_store import ( @@ -35,11 +40,6 @@ RunStatus, no_active_report_run_names, ) -from policyengine_stage12_persistence.report_run_tables import ( - budget_window_runs, - report_runs, - simulation_runs, -) from policyengine_stage12_persistence.store import ( ConnectionOpener, create_stage12_engine, diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/household_report_run_store.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/household_report_run_store.py index 1526f1b57..b3515e167 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/household_report_run_store.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/household_report_run_store.py @@ -20,6 +20,10 @@ from typing import TYPE_CHECKING, Any +from policyengine_simulation_persistence.tables import ( + report_runs, + simulation_runs, +) from sqlalchemy import select, update from policyengine_stage12_persistence.report_run_store import ( @@ -27,10 +31,6 @@ RunStatus, running_under_call, ) -from policyengine_stage12_persistence.report_run_tables import ( - report_runs, - simulation_runs, -) if TYPE_CHECKING: from collections.abc import Mapping diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_results.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_results.py index ed3b61c78..6b2592362 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_results.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_results.py @@ -6,6 +6,10 @@ two simulation runs where the table names them, and poverty and inequality keyed by each simulation run. The cliff impact, SPM receipts and data version are columns of the report run itself, which its completion sets. + +The result tables' models are the simulation service library's +(``policyengine_simulation_persistence.result_tables``), which the CI +comparison checks against the Public API's tables. """ from __future__ import annotations @@ -15,10 +19,20 @@ from uuid import uuid4 from policyengine_simulation_contract.macro_output import SingleYearMacroOutput -from sqlalchemy import Boolean, Column, Float, Integer, String, Table -from sqlalchemy.dialects.postgresql import ENUM, UUID - -from policyengine_stage12_persistence.report_run_tables import metadata +from policyengine_simulation_persistence.result_tables import ( + budget_summary, + congressional_district_impacts, + constituency_impacts, + decile_impacts, + inequality, + intra_decile_impacts, + labor_supply_decile_responses, + labor_supply_responses, + local_authority_impacts, + poverty, + program_statistics, + withheld_area_codes, +) if TYPE_CHECKING: from collections.abc import Mapping @@ -36,186 +50,7 @@ RacePovertyOutput, ) from policyengine_simulation_contract.stage12_execution import ReportRunIds - -decile_type_type = ENUM("income", "wealth", name="v2_decile_type", create_type=False) -uk_area_type_type = ENUM( - "constituency", "local_authority", name="v2_uk_area_type", create_type=False -) - - -def _report_run_keys() -> tuple[Column[Any], ...]: - return ( - Column("id", UUID(as_uuid=True), primary_key=True), - Column("report_run_id", UUID(as_uuid=True), nullable=False), - Column("baseline_simulation_run_id", UUID(as_uuid=True), nullable=False), - Column("reform_simulation_run_id", UUID(as_uuid=True), nullable=False), - ) - - -def _simulation_run_keys() -> tuple[Column[Any], ...]: - return ( - Column("id", UUID(as_uuid=True), primary_key=True), - Column("simulation_run_id", UUID(as_uuid=True), nullable=False), - ) - - -labor_supply_responses = Table( - "labor_supply_responses", - metadata, - Column("id", UUID(as_uuid=True), primary_key=True), - Column("report_run_id", UUID(as_uuid=True), nullable=False), - *( - Column(name, Float, nullable=False) - for name in ( - "income_effect_earnings_change", - "substitution_effect_earnings_change", - "total_earnings_change", - "revenue_change", - "relative_income_effect", - "relative_substitution_effect", - "baseline_weekly_hours", - "reform_weekly_hours", - "weekly_hours_change", - "weekly_hours_income_effect", - "weekly_hours_substitution_effect", - ) - ), -) -labor_supply_decile_responses = Table( - "labor_supply_decile_responses", - metadata, - Column("id", UUID(as_uuid=True), primary_key=True), - Column("report_run_id", UUID(as_uuid=True), nullable=False), - Column("decile", Integer, nullable=False), - Column("average_income_effect", Float), - Column("average_substitution_effect", Float), - Column("relative_income_effect", Float), - Column("relative_substitution_effect", Float), -) -program_statistics = Table( - "program_statistics", - metadata, - *_report_run_keys(), - Column("program_name", String(512), nullable=False), - Column("entity", String(128)), - Column("is_tax", Boolean), - Column("baseline_total", Float), - Column("reform_total", Float), - Column("change", Float), -) - - -def _area_values() -> tuple[Column[Any], ...]: - return ( - Column("average_household_income_change", Float, nullable=False), - Column("relative_household_income_change", Float, nullable=False), - Column("population", Float, nullable=False), - ) - - -def _disclosure_values() -> tuple[Column[Any], ...]: - return ( - Column("disclosure_tier", String(16)), - Column("average_household_income_change_lower", Float), - Column("average_household_income_change_upper", Float), - Column("relative_household_income_change_lower", Float), - Column("relative_household_income_change_upper", Float), - ) - - -congressional_district_impacts = Table( - "congressional_district_impacts", - metadata, - *_report_run_keys(), - *_area_values(), - Column("district_geoid", Integer, nullable=False), - Column("state_fips", Integer, nullable=False), - Column("district_number", Integer, nullable=False), - Column("winner_percentage", Float, nullable=False), - Column("loser_percentage", Float, nullable=False), - Column("no_change_percentage", Float, nullable=False), -) -constituency_impacts = Table( - "constituency_impacts", - metadata, - *_report_run_keys(), - *_area_values(), - *_disclosure_values(), - Column("constituency_code", String(64), nullable=False), - Column("constituency_name", String(255), nullable=False), -) -local_authority_impacts = Table( - "local_authority_impacts", - metadata, - *_report_run_keys(), - *_area_values(), - *_disclosure_values(), - Column("local_authority_code", String(64), nullable=False), - Column("local_authority_name", String(255), nullable=False), -) -# The codes of the areas a local-area run estimated but disclosure control did -# not publish, one row per area: none means none were withheld. -withheld_area_codes = Table( - "withheld_area_codes", - metadata, - *_report_run_keys(), - Column("area_type", uk_area_type_type, nullable=False), - Column("code", String(64), nullable=False), -) -budget_summary = Table( - "budget_summary", - metadata, - *_report_run_keys(), - Column("variable_name", String(512), nullable=False), - Column("entity", String(128), nullable=False), - Column("baseline_total", Float), - Column("reform_total", Float), - Column("change", Float), -) -decile_impacts = Table( - "decile_impacts", - metadata, - *_report_run_keys(), - Column("income_variable", String(512), nullable=False), - Column("decile_variable", String(512)), - Column("entity", String(128), nullable=False), - Column("decile", Integer, nullable=False), - Column("quantiles", Integer, nullable=False), - Column("absolute_change", Float), - Column("relative_change", Float), -) -intra_decile_impacts = Table( - "intra_decile_impacts", - metadata, - *_report_run_keys(), - Column("decile_type", decile_type_type, nullable=False), - Column("decile", Integer, nullable=False), - Column("lose_more_than_5pct", Float), - Column("lose_less_than_5pct", Float), - Column("no_change", Float), - Column("gain_less_than_5pct", Float), - Column("gain_more_than_5pct", Float), -) -poverty = Table( - "poverty", - metadata, - *_simulation_run_keys(), - Column("poverty_type", String(128), nullable=False), - Column("entity", String(128), nullable=False), - Column("filter_variable", String(512)), - Column("filter_group", String(64)), - Column("rate", Float), -) -inequality = Table( - "inequality", - metadata, - *_simulation_run_keys(), - Column("income_variable", String(512), nullable=False), - Column("entity", String(128), nullable=False), - Column("gini", Float), - Column("top_10_share", Float), - Column("top_1_share", Float), -) + from sqlalchemy import Table # The household variable each headline budget figure totals, and the column # its figure goes in. ``budgetary_impact`` is tax minus benefits, as the diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py index 520a32544..8dc6548d5 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py @@ -16,15 +16,15 @@ from enum import StrEnum from typing import TYPE_CHECKING, Any -from sqlalchemy import and_, case, exists, func, insert, or_, select, true, update -from sqlalchemy.dialects.postgresql import insert as postgresql_insert - -from policyengine_stage12_persistence.report_run_tables import ( +from policyengine_simulation_persistence.tables import ( budget_window_runs, household_jobs, report_runs, simulation_runs, ) +from sqlalchemy import and_, case, exists, func, insert, or_, select, true, update +from sqlalchemy.dialects.postgresql import insert as postgresql_insert + from policyengine_stage12_persistence.store import ( ConnectionOpener, create_stage12_engine, diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_tables.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_tables.py deleted file mode 100644 index 68c574b40..000000000 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_tables.py +++ /dev/null @@ -1,118 +0,0 @@ -"""SQLAlchemy mappings of the Public API's run tables that v2 Modal calls update. - -``policyengine-api`` owns these tables and every schema migration. The -mappings declare only the columns the coordinator, the workers, the household -functions, the budget-window scheduling and the stuck-run sweep read or write -as the Modal worker role, so they can issue typed DML; runtime code never -emits DDL. -""" - -from __future__ import annotations - -from sqlalchemy import JSON, Column, DateTime, Float, Integer, MetaData, String, Table -from sqlalchemy.dialects.postgresql import ENUM, JSONB, UUID - -RUN_STATUS_VALUES = ("pending", "running", "succeeded", "failed") - -report_run_status_type = ENUM( - *RUN_STATUS_VALUES, - name="v2_report_run_status", - create_type=False, -) -simulation_run_status_type = ENUM( - *RUN_STATUS_VALUES, - name="v2_simulation_status", - create_type=False, -) -household_job_status_type = ENUM( - *RUN_STATUS_VALUES, - name="v2_household_job_status", - create_type=False, -) - -metadata = MetaData(schema="public") - -# A report run compares one pair of simulation runs; a report with only a -# baseline has no reform run. The cliff impact, SPM receipts and data version -# are the report run's own results. A budget window's years are report runs -# naming its budget-window run and their year. -report_runs = Table( - "report_runs", - metadata, - Column("id", UUID(as_uuid=True), primary_key=True), - Column("created_at", DateTime(timezone=True), nullable=False), - Column("baseline_simulation_run_id", UUID(as_uuid=True), nullable=False), - Column("reform_simulation_run_id", UUID(as_uuid=True)), - Column("budget_window_run_id", UUID(as_uuid=True)), - Column("year", Integer), - Column("status", report_run_status_type, nullable=False), - Column("modal_call_id", String(255)), - Column("error_message", String), - Column("started_at", DateTime(timezone=True)), - Column("completed_at", DateTime(timezone=True)), - Column("updated_at", DateTime(timezone=True), nullable=False), - Column("baseline_cliff_gap", Float), - Column("baseline_cliff_share", Float), - Column("reform_cliff_gap", Float), - Column("reform_cliff_share", Float), - Column("spm_provenance", JSONB(none_as_null=True)), - Column("data_version", String(128)), -) - -# One calculation of a budget-window report: its years are the report runs -# naming it. It has no Modal call of its own; the coordinators of its years -# mark it running, succeeded or failed. -budget_window_runs = Table( - "budget_window_runs", - metadata, - Column("id", UUID(as_uuid=True), primary_key=True), - Column("created_at", DateTime(timezone=True), nullable=False), - Column("status", report_run_status_type, nullable=False), - Column("error_message", String), - Column("started_at", DateTime(timezone=True)), - Column("completed_at", DateTime(timezone=True)), - Column("updated_at", DateTime(timezone=True), nullable=False), -) - -# A simulation run's year, versions, dataset and region say what it -# calculated; the coordinator checks them on a succeeded run that a report -# run reuses. -simulation_runs = Table( - "simulation_runs", - metadata, - Column("id", UUID(as_uuid=True), primary_key=True), - Column("created_at", DateTime(timezone=True), nullable=False), - Column("year", Integer, nullable=False), - Column("country_package_version", String(128), nullable=False), - Column("policyengine_version", String(128), nullable=False), - Column("dataset_id", UUID(as_uuid=True)), - Column("region_id", UUID(as_uuid=True)), - Column("status", simulation_run_status_type, nullable=False), - Column("modal_call_id", String(255)), - Column("error_message", String), - Column("started_at", DateTime(timezone=True)), - Column("completed_at", DateTime(timezone=True)), - Column("updated_at", DateTime(timezone=True), nullable=False), - Column("output_uri", String(2048)), - Column("output_sha256", String(64)), - # A household simulation run's ``HouseholdExecutionResult`` or - # ``HouseholdExecutionFailure`` document, set by the household function. - Column("household_result", JSON(none_as_null=True)), -) - -# A household job is one household calculation, which a household function -# claims under its Modal call. ``result`` holds the calculation's -# ``HouseholdExecutionResult`` or ``HouseholdExecutionFailure`` document. -household_jobs = Table( - "household_jobs", - metadata, - Column("id", UUID(as_uuid=True), primary_key=True), - Column("created_at", DateTime(timezone=True), nullable=False), - Column("status", household_job_status_type, nullable=False), - Column("modal_call_id", String(255)), - Column("error_message", String), - Column("result", JSON(none_as_null=True)), - Column("started_at", DateTime(timezone=True)), - Column("completed_at", DateTime(timezone=True)), - Column("updated_at", DateTime(timezone=True), nullable=False), -) diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/store.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/store.py index 43476ee6c..91caaa169 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/store.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/store.py @@ -14,10 +14,14 @@ ComparisonSimulationRecord, ResultComparisonStatus, ) -from sqlalchemy import Engine, create_engine, select, update + +# Every Stage 12 store builds its engine with the simulation service library's +# unpooled engine, which checks the URL with that library's one URL check. +from policyengine_simulation_persistence.engine import ( + create_unpooled_engine as create_stage12_engine, +) +from sqlalchemy import Engine, select, update from sqlalchemy.dialects.postgresql import insert -from sqlalchemy.engine import URL, Connection, make_url -from sqlalchemy.pool import NullPool from policyengine_stage12_persistence.tables import ( REPORT_IDENTITY_CONSTRAINT, @@ -46,6 +50,7 @@ from uuid import UUID from sqlalchemy import ColumnElement, RowMapping, Table + from sqlalchemy.engine import Connection REPORT_IDENTITY_COLUMNS = ( "environment", @@ -101,37 +106,6 @@ ) -def normalized_database_url(database_url: str) -> URL: - """Select psycopg 3 without changing the stored secret's URL format.""" - if not database_url or database_url != database_url.strip(): - msg = "Stage 12 database URL is missing or malformed" - raise ValueError(msg) - try: - parsed = make_url(database_url) - except Exception as error: - msg = "Stage 12 database URL is malformed" - raise ValueError(msg) from error - if parsed.drivername != "postgresql": - msg = "Stage 12 database URL must use postgresql://" - raise ValueError(msg) - if not parsed.username or not parsed.host or not parsed.database: - msg = "Stage 12 database URL is incomplete" - raise ValueError(msg) - return parsed.set(drivername="postgresql+psycopg") - - -def create_stage12_engine(database_url: str) -> Engine: - """Build the DML-only runtime engine; this never creates database objects.""" - return create_engine( - normalized_database_url(database_url), - # Coordinators and workers spend most of their lifetime calculating - # or waiting. Release each database session after its short operation - # instead of reserving a pooler slot throughout that idle interval. - poolclass=NullPool, - connect_args={"connect_timeout": 10}, - ) - - class ConnectionOpener(Protocol): """Open the database connection that one store operation uses. diff --git a/libs/policyengine-stage12-persistence/tests/report_run_database.py b/libs/policyengine-stage12-persistence/tests/report_run_database.py index 280186a54..42fb5c18b 100644 --- a/libs/policyengine-stage12-persistence/tests/report_run_database.py +++ b/libs/policyengine-stage12-persistence/tests/report_run_database.py @@ -1,7 +1,8 @@ """A disposable Postgres holding the Public API's v2 tables, for report run tests. The tests run only when both URLs name a disposable database that the Public -API's v2 migrations created and whose declared privileges were applied: +API's v2 migrations created and whose declared privileges were applied, as the +Public API schema job's does: - ``REPORT_RUN_TEST_DATABASE_URL`` logs in as the Modal worker role, which the store under test uses; @@ -23,6 +24,9 @@ WORKER_URL_ENV = "REPORT_RUN_TEST_DATABASE_URL" SEED_URL_ENV = "REPORT_RUN_TEST_SEED_DATABASE_URL" +# Set to "1" by the Public API schema job, where a missing database fails the +# tests instead of skipping them. +REQUIRE_DATABASE_ENV = "REPORT_RUN_REQUIRE_DATABASE_TESTS" # The foreign keys between the Public API's tables and the simulation API's # stay, so every catalog row a seeded run names exists. SEEDED_TABLES = ( @@ -53,10 +57,16 @@ def worker_database_url() -> str: - """Return the Modal worker role's test URL, or skip when none is set.""" + """Return the Modal worker role's test URL, or skip when none is set. + + A job that sets ``REPORT_RUN_REQUIRE_DATABASE_TESTS=1`` fails instead. + """ url = os.environ.get(WORKER_URL_ENV) if not url or not os.environ.get(SEED_URL_ENV): - pytest.skip(f"{WORKER_URL_ENV} and {SEED_URL_ENV} name no test database") + message = f"{WORKER_URL_ENV} and {SEED_URL_ENV} name no test database" + if os.environ.get(REQUIRE_DATABASE_ENV) == "1": + pytest.fail(message) + pytest.skip(message) return url diff --git a/libs/policyengine-stage12-persistence/tests/test_database_dependency_ranges.py b/libs/policyengine-stage12-persistence/tests/test_database_dependency_ranges.py new file mode 100644 index 000000000..e2b52f80d --- /dev/null +++ b/libs/policyengine-stage12-persistence/tests/test_database_dependency_ranges.py @@ -0,0 +1,43 @@ +"""The library declares the shared table-model library's database ranges. + +Its stores use ``policyengine-simulation-persistence``'s models, so they run on +the SQLAlchemy and psycopg releases that library declares, and declaring a +wider range would name versions the locks can't take. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import tomllib + +LIBRARY_ROOT = Path(__file__).resolve().parents[1] +SHARED_LIBRARY_ROOT = LIBRARY_ROOT.parent / "policyengine-simulation-persistence" +DATABASE_PACKAGES = ("sqlalchemy", "psycopg") + + +def _requirements(pyproject: Path) -> list[str]: + return tomllib.loads(pyproject.read_text(encoding="utf-8"))["project"][ + "dependencies" + ] + + +def _requirement(requirements: list[str], package_name: str) -> str: + """Return the one requirement naming ``package_name``, extras included.""" + matching = [ + requirement + for requirement in requirements + if (name := re.match(r"[A-Za-z0-9_.-]+", requirement)) is not None + and name.group(0).lower() == package_name + ] + assert len(matching) == 1, (package_name, matching) + return matching[0] + + +def test_the_library_declares_the_shared_librarys_database_ranges() -> None: + own = _requirements(LIBRARY_ROOT / "pyproject.toml") + shared = _requirements(SHARED_LIBRARY_ROOT / "pyproject.toml") + + for package_name in DATABASE_PACKAGES: + assert _requirement(own, package_name) == _requirement(shared, package_name) diff --git a/libs/policyengine-stage12-persistence/tests/test_database_requirement.py b/libs/policyengine-stage12-persistence/tests/test_database_requirement.py new file mode 100644 index 000000000..5867bc24a --- /dev/null +++ b/libs/policyengine-stage12-persistence/tests/test_database_requirement.py @@ -0,0 +1,36 @@ +"""The report run tests skip without their database, unless a job requires it. + +The Public API schema job sets ``REPORT_RUN_REQUIRE_DATABASE_TESTS=1``, so a +database it failed to name fails the tests there instead of skipping them. +""" + +from __future__ import annotations + +import pytest +from report_run_database import ( + REQUIRE_DATABASE_ENV, + SEED_URL_ENV, + WORKER_URL_ENV, + worker_database_url, +) + + +def _without_test_databases(monkeypatch) -> None: + monkeypatch.delenv(WORKER_URL_ENV, raising=False) + monkeypatch.delenv(SEED_URL_ENV, raising=False) + + +def test_the_tests_skip_without_a_database(monkeypatch) -> None: + _without_test_databases(monkeypatch) + monkeypatch.delenv(REQUIRE_DATABASE_ENV, raising=False) + + with pytest.raises(pytest.skip.Exception, match=WORKER_URL_ENV): + worker_database_url() + + +def test_a_job_that_requires_the_database_fails_without_one(monkeypatch) -> None: + _without_test_databases(monkeypatch) + monkeypatch.setenv(REQUIRE_DATABASE_ENV, "1") + + with pytest.raises(pytest.fail.Exception, match=WORKER_URL_ENV): + worker_database_url() diff --git a/libs/policyengine-stage12-persistence/tests/test_report_run_store.py b/libs/policyengine-stage12-persistence/tests/test_report_run_store.py index f29f450fa..036880e45 100644 --- a/libs/policyengine-stage12-persistence/tests/test_report_run_store.py +++ b/libs/policyengine-stage12-persistence/tests/test_report_run_store.py @@ -44,7 +44,7 @@ def test_the_store_connects_as_the_modal_worker_role(store) -> None: def test_the_modal_worker_role_cannot_create_a_report_run(store, seeded_run) -> None: - from policyengine_stage12_persistence.report_run_tables import report_runs + from policyengine_simulation_persistence.tables import report_runs with ( pytest.raises(ProgrammingError, match="permission denied"), diff --git a/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py b/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py new file mode 100644 index 000000000..f0aa0d6c1 --- /dev/null +++ b/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py @@ -0,0 +1,146 @@ +"""The Modal worker role's stores use the simulation service's table models. + +``policyengine-simulation-persistence`` holds the one set of models of the +Public API's run tables, which the CI comparison checks against the tables the +Public API's migration chain builds. This library keeps only the comparison +path's own tables (``tables.py``) until that path is removed. +""" + +from __future__ import annotations + +import importlib +import pkgutil +from typing import TYPE_CHECKING +from uuid import uuid4 + +import policyengine_stage12_persistence +import pytest +from economy_output import ( + US_AT_LARGE_STATES, + US_STATE_FIPS, + uk_output_with_withheld_codes, + us_output_with_every_part, +) +from policyengine_simulation_contract.stage12_execution import ReportRunIds +from policyengine_simulation_persistence import ( + HouseholdJobStatus, + ReportRunStatus, + SimulationStatus, + engine, + result_metadata, + service_metadata, +) +from policyengine_simulation_persistence import tables as shared_tables +from policyengine_stage12_persistence import ( + DistrictCodes, + budget_window_store, + household_report_run_store, + report_run_result, + report_run_store, + store, +) +from policyengine_stage12_persistence.report_run_store import ( + RUN_TABLES, + RunKind, + RunStatus, +) +from policyengine_stage12_persistence.tables import metadata as comparison_metadata +from sqlalchemy import Table + +if TYPE_CHECKING: + from collections.abc import Iterable + + +def _is_one_of(table: Table, tables: Iterable[Table]) -> bool: + """Return whether ``table`` is one of ``tables`` itself, not just alike.""" + return any(table is candidate for candidate in tables) + + +def test_the_run_stores_write_the_shared_run_table_models() -> None: + assert RUN_TABLES[RunKind.REPORT_RUN] is shared_tables.report_runs + assert RUN_TABLES[RunKind.SIMULATION_RUN] is shared_tables.simulation_runs + assert RUN_TABLES[RunKind.HOUSEHOLD_JOB] is shared_tables.household_jobs + assert report_run_store.budget_window_runs is shared_tables.budget_window_runs + + +def test_the_budget_window_store_writes_the_shared_run_table_models() -> None: + assert budget_window_store.budget_window_runs is shared_tables.budget_window_runs + assert budget_window_store.report_runs is shared_tables.report_runs + assert budget_window_store.simulation_runs is shared_tables.simulation_runs + + +def test_the_household_report_store_writes_the_shared_run_table_models() -> None: + assert household_report_run_store.report_runs is shared_tables.report_runs + assert household_report_run_store.simulation_runs is shared_tables.simulation_runs + + +def test_every_table_the_library_names_is_shared_or_a_comparison_table() -> None: + """The library declares no model of a Public API run or result table itself. + + Its only tables of its own are the comparison path's, in ``tables.py``. + """ + shared_models = ( + *service_metadata.tables.values(), + *result_metadata.tables.values(), + ) + comparison_tables = tuple(comparison_metadata.tables.values()) + for module_info in pkgutil.iter_modules(policyengine_stage12_persistence.__path__): + module = importlib.import_module( + f"{policyengine_stage12_persistence.__name__}.{module_info.name}" + ) + for name, value in vars(module).items(): + if not isinstance(value, Table): + continue + assert _is_one_of(value, shared_models) or _is_one_of( + value, comparison_tables + ), f"{module_info.name}.{name} models {value.name} itself" + + +def test_every_result_row_goes_in_a_shared_result_table_model() -> None: + run_ids = ReportRunIds( + report_run_id=uuid4(), + baseline_simulation_run_id=uuid4(), + reform_simulation_run_id=uuid4(), + ) + results = ( + report_run_result( + us_output_with_every_part(), + run_ids=run_ids, + country="us", + district_codes=DistrictCodes( + state_fips=US_STATE_FIPS, at_large_states=US_AT_LARGE_STATES + ), + ), + report_run_result( + uk_output_with_withheld_codes(), run_ids=run_ids, country="uk" + ), + ) + result_models = tuple(result_metadata.tables.values()) + + for result in results: + for result_rows in result.rows: + assert _is_one_of(result_rows.table, result_models), result_rows.table.name + + +def test_the_stores_connect_with_the_service_librarys_one_engine_helper() -> None: + """One URL check and one unpooled engine serve every Modal call's store.""" + assert store.create_stage12_engine is engine.create_unpooled_engine + assert not hasattr(store, "normalized_database_url") + assert report_run_store.create_stage12_engine is engine.create_unpooled_engine + assert budget_window_store.create_stage12_engine is engine.create_unpooled_engine + + +@pytest.mark.parametrize( + "table_status", [SimulationStatus, ReportRunStatus, HouseholdJobStatus] +) +def test_run_status_has_each_run_tables_status_values(table_status) -> None: + """``RunStatus`` is a status of any run kind, so it has each table's values. + + A read through the shared models returns the table's own enum member, + which the stores convert with ``RunStatus(...)`` before comparing it. + """ + assert [status.value for status in RunStatus] == [ + status.value for status in table_status + ] + for status in table_status: + assert RunStatus(status) is RunStatus(status.value) diff --git a/libs/policyengine-stage12-persistence/tests/test_store.py b/libs/policyengine-stage12-persistence/tests/test_store.py index 811045120..cc9119402 100644 --- a/libs/policyengine-stage12-persistence/tests/test_store.py +++ b/libs/policyengine-stage12-persistence/tests/test_store.py @@ -14,11 +14,8 @@ SimulationRole, ) from policyengine_stage12_persistence import Stage12PersistenceStore -from policyengine_stage12_persistence import store as store_module -from policyengine_stage12_persistence.store import normalized_database_url from sqlalchemy import create_engine from sqlalchemy.exc import OperationalError -from sqlalchemy.pool import QueuePool from sqlalchemy.sql import ClauseElement NOW = datetime(2026, 9, 22, tzinfo=UTC) @@ -27,48 +24,6 @@ OBSERVABILITY_ID = "00000000-0000-4000-8000-000000000012" -@pytest.mark.parametrize("fail_transaction", [False, True]) -def test_runtime_engine_closes_connections_between_transactions( - monkeypatch, fail_transaction -) -> None: - def local_engine(_url, **options): - # Exercise the runtime pooling configuration with a real local DBAPI. - # PostgreSQL's default is QueuePool; SQLite's memory default differs. - options.pop("connect_args") - options.setdefault("poolclass", QueuePool) - return create_engine("sqlite://", **options) - - monkeypatch.setattr(store_module, "create_engine", local_engine) - engine = store_module.create_stage12_engine( - "postgresql://runtime:secret@db.example/postgres" - ) - try: - try: - with engine.begin() as connection: - physical_connection = connection.connection.driver_connection - assert connection.exec_driver_sql("SELECT 1").scalar_one() == 1 - if fail_transaction: - _fail_the_calculation() - except RuntimeError: - assert fail_transaction - - # Long calculations must not hold an idle session after DB work ends. - with pytest.raises(sqlite3.ProgrammingError, match="closed database"): - physical_connection.execute("SELECT 1") - - with engine.begin() as connection: - assert connection.connection.driver_connection is not physical_connection - assert connection.exec_driver_sql("SELECT 1").scalar_one() == 1 - finally: - engine.dispose() - - -def _fail_the_calculation() -> None: - """Raise the failure a calculation inside the transaction would.""" - msg = "calculation failed" - raise RuntimeError(msg) - - def _report() -> ComparisonReportRecord: return ComparisonReportRecord( evaluation_id=REPORT_ID, @@ -442,16 +397,3 @@ def test_simulation_status_replacement_preserves_a_concurrent_cancellation() -> assert replaced is False assert result == cancelled assert len(engine.connection.statements) == 1 - - -def test_database_url_uses_one_canonical_secret_format() -> None: - parsed = normalized_database_url( - "postgresql://policyengine_v2_runtime:secret@db.example/postgres" - ) - assert parsed.drivername == "postgresql+psycopg" - assert parsed.username == "policyengine_v2_runtime" - - with pytest.raises(ValueError, match="postgresql://"): - normalized_database_url( - "postgresql+psycopg://policyengine_v2_runtime:secret@db.example/postgres" - ) diff --git a/libs/policyengine-stage12-persistence/uv.lock b/libs/policyengine-stage12-persistence/uv.lock index a1d9b1fab..2752c838c 100644 --- a/libs/policyengine-stage12-persistence/uv.lock +++ b/libs/policyengine-stage12-persistence/uv.lock @@ -636,12 +636,33 @@ requires-dist = [ ] provides-extras = ["test", "build"] +[[package]] +name = "policyengine-simulation-persistence" +version = "0.1.0" +source = { editable = "../policyengine-simulation-persistence" } +dependencies = [ + { name = "psycopg", extra = ["binary"] }, + { name = "sqlalchemy" }, +] + +[package.metadata] +requires-dist = [ + { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" }, + { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, + { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, + { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=6.1.1" }, + { name = "pyyaml", marker = "extra == 'test'", specifier = ">=6.0.2" }, + { name = "sqlalchemy", specifier = ">=2,<2.1" }, +] +provides-extras = ["test", "build"] + [[package]] name = "policyengine-stage12-persistence" version = "0.1.0" source = { editable = "." } dependencies = [ { name = "policyengine-simulation-contract" }, + { name = "policyengine-simulation-persistence" }, { name = "psycopg", extra = ["binary"] }, { name = "sqlalchemy" }, ] @@ -660,11 +681,12 @@ test = [ requires-dist = [ { name = "black", marker = "extra == 'build'", specifier = ">=25.1.0" }, { name = "policyengine-simulation-contract", editable = "../policyengine-simulation-contract" }, + { name = "policyengine-simulation-persistence", editable = "../policyengine-simulation-persistence" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" }, { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=6.1.1" }, - { name = "sqlalchemy", specifier = ">=2,<3" }, + { name = "sqlalchemy", specifier = ">=2,<2.1" }, ] provides-extras = ["test", "build"] diff --git a/projects/policyengine-simulation-entry/uv.lock b/projects/policyengine-simulation-entry/uv.lock index 7bef95438..f9d13ff46 100644 --- a/projects/policyengine-simulation-entry/uv.lock +++ b/projects/policyengine-simulation-entry/uv.lock @@ -991,6 +991,7 @@ version = "0.1.0" source = { editable = "../../libs/policyengine-stage12-persistence" } dependencies = [ { name = "policyengine-simulation-contract" }, + { name = "policyengine-simulation-persistence" }, { name = "psycopg", extra = ["binary"] }, { name = "sqlalchemy" }, ] @@ -999,11 +1000,12 @@ dependencies = [ requires-dist = [ { name = "black", marker = "extra == 'build'", specifier = ">=25.1.0" }, { name = "policyengine-simulation-contract", editable = "../../libs/policyengine-simulation-contract" }, + { name = "policyengine-simulation-persistence", editable = "../../libs/policyengine-simulation-persistence" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" }, { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=6.1.1" }, - { name = "sqlalchemy", specifier = ">=2,<3" }, + { name = "sqlalchemy", specifier = ">=2,<2.1" }, ] provides-extras = ["test", "build"] diff --git a/projects/policyengine-simulation-executor/pyproject.toml b/projects/policyengine-simulation-executor/pyproject.toml index 0352f41cd..c76ebc2bc 100644 --- a/projects/policyengine-simulation-executor/pyproject.toml +++ b/projects/policyengine-simulation-executor/pyproject.toml @@ -18,6 +18,9 @@ dependencies = [ "policyengine-fastapi", "policyengine-simulation-observability", "policyengine-simulation-contract[modal]", + # The models of the Public API's tables the v2 Modal calls write; the + # v2 images copy it with the Stage 12 library, whose stores use them. + "policyengine-simulation-persistence", "policyengine-stage12-persistence", # PolicyEngine.py's models extra is the only selector for core, country # models, and the SPM calculator. Do not duplicate those versions here. @@ -63,10 +66,11 @@ modal-simulation-image = [ # this directly; it is also a transitive dependency of policyengine. "requests>=2.32,<3", "pyarrow>=20,<24", - # The Stage 12 coordinator and child functions use typed SQLAlchemy Core - # DML against the API-owned temporary tables. These dependencies are - # explicit because only this group is installed in Modal. - "sqlalchemy>=2,<3", + # The Stage 12 stores use typed SQLAlchemy Core DML through the shared + # table-model library (policyengine-simulation-persistence), whose ranges + # these are. They are explicit because only this group is installed in + # Modal. + "sqlalchemy>=2,<2.1", "psycopg[binary]>=3.2,<4", ] # The executor's budget-window scheduler tests exercise the gateway app @@ -79,6 +83,7 @@ dev = [ policyengine-fastapi = { path = "../../libs/policyengine-fastapi", editable = true } policyengine-simulation-observability = { path = "../../libs/policyengine-simulation-observability", editable = true } policyengine-simulation-contract = { path = "../../libs/policyengine-simulation-contract", editable = true } +policyengine-simulation-persistence = { path = "../../libs/policyengine-simulation-persistence", editable = true } policyengine-stage12-persistence = { path = "../../libs/policyengine-stage12-persistence", editable = true } policyengine-simulation-gateway = { path = "../policyengine-simulation-gateway", editable = true } diff --git a/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py b/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py index bb566ea51..04381c48c 100644 --- a/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py +++ b/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py @@ -65,6 +65,7 @@ "policyengine_simulation_executor", "policyengine_simulation_observability", "policyengine_simulation_contract", + "policyengine_simulation_persistence", "policyengine_stage12_persistence", copy=True, ) diff --git a/projects/policyengine-simulation-executor/src/modal/stage12_precompute_app.py b/projects/policyengine-simulation-executor/src/modal/stage12_precompute_app.py index c8332e576..b0182e537 100644 --- a/projects/policyengine-simulation-executor/src/modal/stage12_precompute_app.py +++ b/projects/policyengine-simulation-executor/src/modal/stage12_precompute_app.py @@ -76,6 +76,7 @@ "policyengine_simulation_executor", "policyengine_simulation_observability", "policyengine_simulation_contract", + "policyengine_simulation_persistence", "policyengine_stage12_persistence", copy=True, ) diff --git a/projects/policyengine-simulation-executor/src/modal/v2_app.py b/projects/policyengine-simulation-executor/src/modal/v2_app.py index ea7b4b5be..b7f824ad1 100644 --- a/projects/policyengine-simulation-executor/src/modal/v2_app.py +++ b/projects/policyengine-simulation-executor/src/modal/v2_app.py @@ -196,6 +196,7 @@ def _add_v2_sources(image: modal.Image) -> modal.Image: "policyengine_simulation_executor", "policyengine_simulation_observability", "policyengine_simulation_contract", + "policyengine_simulation_persistence", "policyengine_stage12_persistence", copy=True, ) diff --git a/projects/policyengine-simulation-executor/tests/test_database_dependency_ranges.py b/projects/policyengine-simulation-executor/tests/test_database_dependency_ranges.py new file mode 100644 index 000000000..8b4e04a38 --- /dev/null +++ b/projects/policyengine-simulation-executor/tests/test_database_dependency_ranges.py @@ -0,0 +1,46 @@ +"""The Modal images install the shared table-model library's database ranges. + +The images install third-party packages only from the ``modal-simulation-image`` +group, and the Stage 12 stores in them use ``policyengine-simulation-persistence``'s +models, so the group declares that library's SQLAlchemy and psycopg ranges. +""" + +from __future__ import annotations + +import re +import tomllib +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +SHARED_LIBRARY_PYPROJECT = ( + PROJECT_ROOT.parents[1] + / "libs" + / "policyengine-simulation-persistence" + / "pyproject.toml" +) +DATABASE_PACKAGES = ("sqlalchemy", "psycopg") + + +def _requirement(requirements: list[str], package_name: str) -> str: + """Return the one requirement naming ``package_name``, extras included.""" + matching = [ + requirement + for requirement in requirements + if (name := re.match(r"[A-Za-z0-9_.-]+", requirement)) is not None + and name.group(0).lower() == package_name + ] + assert len(matching) == 1, (package_name, matching) + return matching[0] + + +def test_the_image_group_declares_the_shared_librarys_database_ranges() -> None: + project = tomllib.loads((PROJECT_ROOT / "pyproject.toml").read_text("utf-8")) + image_group = project["dependency-groups"]["modal-simulation-image"] + shared = tomllib.loads(SHARED_LIBRARY_PYPROJECT.read_text("utf-8"))["project"][ + "dependencies" + ] + + for package_name in DATABASE_PACKAGES: + assert _requirement(image_group, package_name) == _requirement( + shared, package_name + ) diff --git a/projects/policyengine-simulation-executor/tests/test_modal_scripts.py b/projects/policyengine-simulation-executor/tests/test_modal_scripts.py index 4980507e7..c2c4bdd7e 100644 --- a/projects/policyengine-simulation-executor/tests/test_modal_scripts.py +++ b/projects/policyengine-simulation-executor/tests/test_modal_scripts.py @@ -309,6 +309,24 @@ def test_workflow_supplies_hf_token(self): workflow.index("Run image smokes") ) + def test_workflow_runs_when_a_library_the_v2_images_copy_changes(self): + """A change to a library the v2 images copy builds and imports them again. + + The Stage 12 library's stores use the simulation service library's + table models, so the images copy both. + """ + workflow = ( + REPO_ROOT / ".github" / "workflows" / "pr-image-smoke.yml" + ).read_text(encoding="utf-8") + + for library in ( + "policyengine-simulation-contract", + "policyengine-simulation-observability", + "policyengine-simulation-persistence", + "policyengine-stage12-persistence", + ): + assert f"- 'libs/{library}/**'" in workflow, library + def test_deployment_sources_exclude_legacy_hf_secret_names(self): paths = ( REPO_ROOT / ".github/scripts/modal-image-smoke.sh", diff --git a/projects/policyengine-simulation-executor/tests/test_runtime_evaluated_annotations.py b/projects/policyengine-simulation-executor/tests/test_runtime_evaluated_annotations.py index 0a4864937..a6af2feb5 100644 --- a/projects/policyengine-simulation-executor/tests/test_runtime_evaluated_annotations.py +++ b/projects/policyengine-simulation-executor/tests/test_runtime_evaluated_annotations.py @@ -43,6 +43,7 @@ "policyengine_simulation_contract", "policyengine_simulation_executor", "policyengine_simulation_observability", + "policyengine_simulation_persistence", "policyengine_stage12_persistence", ) # src/modal holds the Modal entry files. They are not part of the installed diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py b/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py index e3f6ef998..f7c1beb60 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py @@ -3,6 +3,7 @@ from __future__ import annotations import importlib +import re import sys import tomllib from contextlib import contextmanager @@ -146,19 +147,24 @@ def test_v2_app_name_and_images_are_separate_and_bundle_derived(monkeypatch) -> module.us_worker_image, module.uk_worker_image, module.coordinator_image, + module.household_image, ): local_source = next( call for call in image.calls if call[0] == "add_local_python_source" ) - bundle_install = next(call for call in image.calls if call[0] == "run_commands") - assert bundle_install[2]["secrets"] == [module.hf_secret] + # The Stage 12 library's stores use the service library's table models. assert local_source[1] == ( "src.modal", "policyengine_simulation_executor", "policyengine_simulation_observability", "policyengine_simulation_contract", + "policyengine_simulation_persistence", "policyengine_stage12_persistence", ) + if image is module.household_image: + continue + bundle_install = next(call for call in image.calls if call[0] == "run_commands") + assert bundle_install[2]["secrets"] == [module.hf_secret] static_runtime_files = next( call for call in image.calls if call[0] == "add_local_dir" ) @@ -203,8 +209,14 @@ def test_v2_image_retains_required_runtime_dependencies() -> None: f"policyengine[models]=={get_bundled_package_version('policyengine')}" in group ) assert not any(isinstance(item, dict) for item in group) - assert "sqlalchemy>=2,<3" in group - assert "psycopg[binary]>=3.2,<4" in group + # The Stage 12 stores need the database packages; their ranges are the + # shared table-model library's, which test_database_dependency_ranges + # compares, so they aren't repeated here. + installed_packages = { + re.split(r"[\[<>=!~;\s]", requirement, maxsplit=1)[0].lower() + for requirement in group + } + assert {"sqlalchemy", "psycopg"} <= installed_packages def test_v2_static_runtime_file_smoke_uses_exact_uk_worker_image(monkeypatch) -> None: diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_precompute_app.py b/projects/policyengine-simulation-executor/tests/test_stage12_precompute_app.py index c260afda0..44a95cd53 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_precompute_app.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_precompute_app.py @@ -58,6 +58,7 @@ def test_precompute_image_installs_only_the_us_bundle(monkeypatch) -> None: ) assert "policyengine_simulation_executor" in local_source[1] assert "policyengine_stage12_persistence" in local_source[1] + assert "policyengine_simulation_persistence" in local_source[1] def test_precompute_image_refreshes_ca_certificates_before_installing_packages( diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep_app.py b/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep_app.py index db32d645a..a8a25e038 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep_app.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep_app.py @@ -116,6 +116,7 @@ def test_the_sweep_image_installs_the_locked_group_and_the_runtime_sources( "policyengine_simulation_executor", "policyengine_simulation_observability", "policyengine_simulation_contract", + "policyengine_simulation_persistence", "policyengine_stage12_persistence", ) diff --git a/projects/policyengine-simulation-executor/tests/test_v1_code_imports_no_table_models.py b/projects/policyengine-simulation-executor/tests/test_v1_code_imports_no_table_models.py new file mode 100644 index 000000000..2614ec14e --- /dev/null +++ b/projects/policyengine-simulation-executor/tests/test_v1_code_imports_no_table_models.py @@ -0,0 +1,243 @@ +"""The v1 ``run_simulation`` code imports neither persistence library. + +The simulation service and the v2 Modal runtime share one library of the +Public API's table models, ``policyengine-simulation-persistence``, which the +Stage 12 library builds on. The v1 code, which will be removed, stays apart +from both: the old gateway, and the v1 Modal app (``src/modal/app.py``) with +every module of this repository it imports, however indirectly and wherever +in the module the import is. +""" + +from __future__ import annotations + +import ast +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +REPOSITORY_ROOT = PROJECT_ROOT.parents[1] +SOURCE_ROOT = PROJECT_ROOT / "src" +GATEWAY_SOURCE_ROOT = PROJECT_ROOT.parent / "policyengine-simulation-gateway" / "src" +V1_MODAL_APP = SOURCE_ROOT / "modal" / "app.py" +TABLE_MODEL_LIBRARIES = ( + "policyengine_simulation_persistence", + "policyengine_stage12_persistence", +) + + +def _first_party_packages(source_roots: tuple[Path, ...]) -> frozenset[str]: + """Return the packages under ``source_roots``, but the table-model libraries.""" + packages = { + path.name + for root in source_roots + for path in root.iterdir() + if (path / "__init__.py").is_file() and path.name != "modal" + } + return frozenset(packages - set(TABLE_MODEL_LIBRARIES)) + + +def _module_name(path: Path, source_roots: tuple[Path, ...]) -> str: + root = next(root for root in source_roots if path.is_relative_to(root)) + parts = path.relative_to(root).with_suffix("").parts + # The Modal entry files are imported as src.modal, from the project. + if parts[0] == "modal": + parts = ("src", *parts) + if parts[-1] == "__init__": + parts = parts[:-1] + return ".".join(parts) + + +def _module_path( + module_name: str, + source_roots: tuple[Path, ...], + first_party: frozenset[str], +) -> Path | None: + """Return the file of a module of this repository, or None for another's.""" + parts = module_name.split(".") + if parts[:2] == ["src", "modal"]: + parts = parts[1:] + elif parts[0] not in first_party: + return None + for root in source_roots: + candidate = root.joinpath(*parts) + for path in (candidate.with_suffix(".py"), candidate / "__init__.py"): + if path.is_file(): + return path + return None + + +def _imported_modules(path: Path, source_roots: tuple[Path, ...]) -> set[str]: + """Return every module ``path`` imports, and each package it imports from.""" + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + package = _module_name(path, source_roots) + if path.name != "__init__.py": + package = package.rpartition(".")[0] + imported: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + base = node.module or "" + if node.level: + anchor = ( + package.rsplit(".", node.level - 1)[0] + if node.level > 1 + else package + ) + base = f"{anchor}.{base}" if base else anchor + imported.add(base) + # ``from package import module`` imports a submodule too. + imported.update(f"{base}.{alias.name}" for alias in node.names) + return imported + + +def _with_parent_packages(module_name: str) -> list[str]: + """Return ``module_name`` and each package Python imports before it.""" + parts = module_name.split(".") + return [".".join(parts[:count]) for count in range(1, len(parts) + 1)] + + +def imports_reached(entry: Path, source_roots: tuple[Path, ...]) -> dict[str, set[str]]: + """Return, by module, what ``entry`` and every module it reaches import. + + Importing a module runs each of its parent packages' ``__init__`` first, + so those are reached too, the entry's own included. It follows only + modules under ``source_roots``, never into the table-model libraries, + whose imports are what the tests look for. + """ + first_party = _first_party_packages(source_roots) + entry_packages = _with_parent_packages(_module_name(entry, source_roots))[:-1] + pending = [ + entry, + *( + module_path + for package_name in entry_packages + if (module_path := _module_path(package_name, source_roots, first_party)) + is not None + ), + ] + imports_by_module: dict[str, set[str]] = {} + while pending: + path = pending.pop() + module_name = _module_name(path, source_roots) + if module_name in imports_by_module: + continue + imported = _imported_modules(path, source_roots) + imports_by_module[module_name] = imported + pending.extend( + module_path + for name in imported + for reached_name in _with_parent_packages(name) + if (module_path := _module_path(reached_name, source_roots, first_party)) + is not None + ) + return imports_by_module + + +def table_model_imports(imports_by_module: dict[str, set[str]]) -> list[str]: + return sorted( + f"{module_name}: {name}" + for module_name, imported in imports_by_module.items() + for name in imported + if name.split(".")[0] in TABLE_MODEL_LIBRARIES + ) + + +def _write_module(root: Path, relative_path: str, source: str = "") -> None: + path = root / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(source, encoding="utf-8") + + +def test_an_import_through_a_package_counts_its_package_too(tmp_path) -> None: + """Python runs a package's ``__init__`` before any module in it.""" + project = tmp_path / "src" + _write_module(project, "modal/__init__.py") + _write_module( + project, + "modal/app.py", + "from policyengine_simulation_executor.runtime.leaf import value\n", + ) + _write_module(project, "policyengine_simulation_executor/__init__.py") + _write_module( + project, + "policyengine_simulation_executor/runtime/__init__.py", + "import policyengine_stage12_persistence\n", + ) + _write_module(project, "policyengine_simulation_executor/runtime/leaf.py") + + reached = imports_reached(project / "modal" / "app.py", (project,)) + + assert table_model_imports(reached) == [ + "policyengine_simulation_executor.runtime: policyengine_stage12_persistence" + ] + + +def test_an_import_in_the_entry_modules_own_package_counts(tmp_path) -> None: + """Importing ``src.modal.app`` runs ``src.modal``'s ``__init__`` first.""" + project = tmp_path / "src" + _write_module( + project, "modal/__init__.py", "import policyengine_simulation_persistence\n" + ) + _write_module(project, "modal/app.py") + + reached = imports_reached(project / "modal" / "app.py", (project,)) + + assert table_model_imports(reached) == [ + "src.modal: policyengine_simulation_persistence" + ] + + +def test_an_import_through_a_first_party_library_counts(tmp_path) -> None: + project = tmp_path / "project" / "src" + library = tmp_path / "library" / "src" + _write_module(project, "modal/__init__.py") + _write_module(project, "modal/app.py", "import policyengine_shared_helpers\n") + _write_module( + library, + "policyengine_shared_helpers/__init__.py", + "from policyengine_simulation_persistence import tables\n", + ) + + reached = imports_reached(project / "modal" / "app.py", (project, library)) + + assert table_model_imports(reached) == [ + "policyengine_shared_helpers: policyengine_simulation_persistence", + "policyengine_shared_helpers: policyengine_simulation_persistence.tables", + ] + + +def _repository_source_roots() -> tuple[Path, ...]: + """Return this project's source and every library's, for the walk.""" + return (SOURCE_ROOT, *sorted((REPOSITORY_ROOT / "libs").glob("*/src"))) + + +def test_the_v1_modal_app_and_its_modules_import_no_table_model_library() -> None: + imports_by_module = imports_reached(V1_MODAL_APP, _repository_source_roots()) + + # The walk follows the app's own imports, not only the file itself, and + # into sim's libraries. + assert "src.modal._image_setup" in imports_by_module + assert "policyengine_simulation_contract" in imports_by_module + assert table_model_imports(imports_by_module) == [] + + +def test_the_old_gateway_imports_no_table_model_library() -> None: + gateway_files = sorted(GATEWAY_SOURCE_ROOT.rglob("*.py")) + imports_by_module = { + str(path.relative_to(GATEWAY_SOURCE_ROOT)): _gateway_imports(path) + for path in gateway_files + } + + assert gateway_files + assert table_model_imports(imports_by_module) == [] + + +def _gateway_imports(path: Path) -> set[str]: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + imported: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and not node.level: + imported.add(node.module or "") + return imported diff --git a/projects/policyengine-simulation-executor/uv.lock b/projects/policyengine-simulation-executor/uv.lock index 66aff8e33..a129f4d4c 100644 --- a/projects/policyengine-simulation-executor/uv.lock +++ b/projects/policyengine-simulation-executor/uv.lock @@ -1906,6 +1906,7 @@ dependencies = [ { name = "policyengine-observability", extra = ["fastapi", "google", "otlp-grpc"] }, { name = "policyengine-simulation-contract", extra = ["modal"] }, { name = "policyengine-simulation-observability" }, + { name = "policyengine-simulation-persistence" }, { name = "policyengine-stage12-persistence" }, { name = "pyarrow" }, { name = "pydantic-settings" }, @@ -1958,6 +1959,7 @@ requires-dist = [ { name = "policyengine-observability", extras = ["fastapi", "google", "otlp-grpc"], specifier = ">=3.0.7,<4" }, { name = "policyengine-simulation-contract", extras = ["modal"], editable = "../../libs/policyengine-simulation-contract" }, { name = "policyengine-simulation-observability", editable = "../../libs/policyengine-simulation-observability" }, + { name = "policyengine-simulation-persistence", editable = "../../libs/policyengine-simulation-persistence" }, { name = "policyengine-stage12-persistence", editable = "../../libs/policyengine-stage12-persistence" }, { name = "pyarrow", specifier = ">=20,<24" }, { name = "pydantic-settings", specifier = ">=2.7.1,<3.0.0" }, @@ -1981,7 +1983,7 @@ modal-simulation-image = [ { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" }, { name = "pyarrow", specifier = ">=20,<24" }, { name = "requests", specifier = ">=2.32,<3" }, - { name = "sqlalchemy", specifier = ">=2,<3" }, + { name = "sqlalchemy", specifier = ">=2,<2.1" }, { name = "tables", specifier = ">=3.10.2" }, ] @@ -2049,12 +2051,33 @@ requires-dist = [ ] provides-extras = ["test", "build"] +[[package]] +name = "policyengine-simulation-persistence" +version = "0.1.0" +source = { editable = "../../libs/policyengine-simulation-persistence" } +dependencies = [ + { name = "psycopg", extra = ["binary"] }, + { name = "sqlalchemy" }, +] + +[package.metadata] +requires-dist = [ + { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" }, + { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, + { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, + { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=6.1.1" }, + { name = "pyyaml", marker = "extra == 'test'", specifier = ">=6.0.2" }, + { name = "sqlalchemy", specifier = ">=2,<2.1" }, +] +provides-extras = ["test", "build"] + [[package]] name = "policyengine-stage12-persistence" version = "0.1.0" source = { editable = "../../libs/policyengine-stage12-persistence" } dependencies = [ { name = "policyengine-simulation-contract" }, + { name = "policyengine-simulation-persistence" }, { name = "psycopg", extra = ["binary"] }, { name = "sqlalchemy" }, ] @@ -2063,11 +2086,12 @@ dependencies = [ requires-dist = [ { name = "black", marker = "extra == 'build'", specifier = ">=25.1.0" }, { name = "policyengine-simulation-contract", editable = "../../libs/policyengine-simulation-contract" }, + { name = "policyengine-simulation-persistence", editable = "../../libs/policyengine-simulation-persistence" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" }, { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=6.1.1" }, - { name = "sqlalchemy", specifier = ">=2,<3" }, + { name = "sqlalchemy", specifier = ">=2,<2.1" }, ] provides-extras = ["test", "build"] diff --git a/rules/registry.toml b/rules/registry.toml index b0100b932..7314d8237 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -141,7 +141,7 @@ test = "tests/test_database_access_boundaries.py::test_rule_proof_sqlalchemy_and [[rule]] id = "sim.persistence.missing-privileges-reported" category = "persistence" -statement = "The simulation service's privilege check, which its readiness check uses, names every table privilege and every column UPDATE privilege its role needs on the Public API's tables and lacks, as PostgreSQL reports it, counts a table missing from the public schema as lacking all of them and a missing column as lacking its own, and names a missing USAGE privilege on that schema." +statement = "The privilege check, which the simulation service's readiness check uses and CI runs for the Modal worker role too, names every table privilege and every column UPDATE privilege the connected role needs on the Public API's tables and lacks, as PostgreSQL reports it, counts a table missing from the public schema as lacking all of them and a missing column as lacking its own, and names a missing USAGE privilege on that schema." rationale = "libs/policyengine-simulation-persistence/README.md#the-roles-privileges" scope = ["libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/access.py"] tier = "mechanical" @@ -159,6 +159,7 @@ nodes = [ "tests/test_access.py::test_only_the_public_schema_counts", "tests/test_access.py::test_only_the_public_schemas_columns_count", "tests/test_access.py::test_a_role_without_usage_on_the_schema_is_told_so", + "tests/test_access.py::test_the_check_names_what_a_role_lacks_of_another_roles_lists", ] [[rule.ci]] workflow = ".github/workflows/pr.yml" @@ -173,10 +174,11 @@ test = "tests/test_access.py::test_rule_proof_missing_privileges_reported" [[rule]] id = "sim.persistence.models-match-public-api" category = "persistence" -statement = "Sim's models of the Public API's tables match the tables the Public API's v2 migration chain builds at the pinned Public API commit, which is on the Public API's default branch: every column's type, nullability and default, every constraint and index by name and definition, and every enum's values; every column its read-only models of the Public API's input tables name exists there with the same type and nullability; and the simulation service's role, with the privileges the Public API's manifest declares, writes every modelled table." +statement = "Sim's one set of models of the Public API's tables, those the simulation service writes and the result tables the v2 Modal calls write, match the tables the Public API's v2 migration chain builds at the pinned Public API commit, which is on the Public API's default branch: every column's type, nullability and default, every constraint and index by name and definition, and every enum's values; every column its read-only models of the Public API's input tables name exists there with the same type and nullability; and, with the privileges the Public API's manifest declares, the simulation service's role writes every table the service writes, and the Modal worker role records its runs and inserts a row in every result table." rationale = "libs/policyengine-simulation-persistence/README.md#checking-the-models-against-the-public-api" scope = [ "libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/tables.py", + "libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/result_tables.py", "libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/input_tables.py", "libs/policyengine-simulation-persistence/public-api-commit", ".github/scripts/public-api-schema-commit.sh", @@ -191,9 +193,12 @@ kind = "pytest" project = "libs/policyengine-simulation-persistence" nodes = [ "tests/test_public_api_schema.py::test_the_models_match_the_public_api_tables", + "tests/test_public_api_schema.py::test_the_result_models_match_the_public_api_tables", "tests/test_public_api_schema.py::test_the_input_models_match_the_public_api_columns", "tests/test_service_writes.py::test_the_service_role_holds_every_privilege_the_service_needs", "tests/test_service_writes.py::test_the_service_role_creates_and_reads_a_row_in_every_table_it_writes", + "tests/test_modal_worker_writes.py::test_the_modal_worker_role_holds_every_privilege_the_modal_calls_need", + "tests/test_modal_worker_writes.py::test_the_modal_worker_role_records_its_runs_and_inserts_every_result", ] [[rule.ci]] workflow = ".github/workflows/pr.yml"