diff --git a/AGENTS.md b/AGENTS.md index 12ce348df..3142ba7dc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -126,7 +126,7 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`. ### Service and package boundaries -- Checked: `sim.boundaries.sqlalchemy-and-alembic-only` +- Checked: `sim.boundaries.sqlalchemy-and-alembic-only`, `sim.persistence.orm-only` - `sim.boundaries.gateway-executor-decoupled`: The gateway and executor never import each other; they communicate only through the models and state helpers in policyengine-simulation-contract. (doc-only, not checked: follow it yourself) - `sim.boundaries.unit-tests-need-no-services`: Unit tests never require real network credentials, Modal, Hugging Face, GCP or deployed services; those seams are mocked. (doc-only, not checked: follow it yourself) diff --git a/changelog.d/1103.changed.md b/changelog.d/1103.changed.md new file mode 100644 index 000000000..94f7e026f --- /dev/null +++ b/changelog.d/1103.changed.md @@ -0,0 +1 @@ +The v2 Modal runtime's database stores now run on SQLAlchemy ORM sessions. They send the same statements as before, plus one `SELECT true` that checks out each operation's connection. Only that opening step is retried, so a write is never repeated. The Stage 12 deployment check now repeats a step when the database refuses or loses a connection. diff --git a/changelog.d/c1-33-household-create-outcome-unknown.fixed.md b/changelog.d/c1-33-household-create-outcome-unknown.fixed.md new file mode 100644 index 000000000..c5dab2468 --- /dev/null +++ b/changelog.d/c1-33-household-create-outcome-unknown.fixed.md @@ -0,0 +1 @@ +`POST /household-calculations` answers 500 `internal_error`, not 503 `database_unavailable`, when the database fails after the household job's row was inserted, since PostgreSQL may have applied the commit and the job may exist. It logs the failure as `household_job_outcome_unknown` with the commit error's type, and still answers 503 `database_unavailable` when no job was written. diff --git a/changelog.d/c1-33-orm-only-rule.added.md b/changelog.d/c1-33-orm-only-rule.added.md new file mode 100644 index 000000000..530868126 --- /dev/null +++ b/changelog.d/c1-33-orm-only-rule.added.md @@ -0,0 +1 @@ +The rule `sim.persistence.orm-only` requires database code to declare tables as SQLAlchemy ORM mapped classes and to run every statement through an ORM Session. The unused `policyengine_fastapi.database` module is removed. diff --git a/changelog.d/deployment-check-one-connection.changed.md b/changelog.d/deployment-check-one-connection.changed.md new file mode 100644 index 000000000..3f712e3f2 --- /dev/null +++ b/changelog.d/deployment-check-one-connection.changed.md @@ -0,0 +1 @@ +The Stage 12 deployment check now uses one database connection for its role check, schema reads and rolled-back test writes, so it needs one free client slot on the database pooler. The check now fails after 15 minutes if the database stops responding. diff --git a/changelog.d/orm-economy-stores.changed.md b/changelog.d/orm-economy-stores.changed.md new file mode 100644 index 000000000..147a52000 --- /dev/null +++ b/changelog.d/orm-economy-stores.changed.md @@ -0,0 +1 @@ +Run the simulation service's economy reports, simulations, re-runs and budget windows on SQLAlchemy ORM sessions and mapped classes. diff --git a/changelog.d/orm-household-stores.changed.md b/changelog.d/orm-household-stores.changed.md new file mode 100644 index 000000000..37627f59a --- /dev/null +++ b/changelog.d/orm-household-stores.changed.md @@ -0,0 +1 @@ +Run the simulation service's household job and household report stores on SQLAlchemy ORM sessions and mapped classes. diff --git a/changelog.d/orm-only-enforced.changed.md b/changelog.d/orm-only-enforced.changed.md new file mode 100644 index 000000000..af4da948e --- /dev/null +++ b/changelog.d/orm-only-enforced.changed.md @@ -0,0 +1 @@ +Require SQLAlchemy ORM sessions and mapped classes in all of sim's database code, run the database privilege check on them, and remove the unused SQLModel dependency. diff --git a/changelog.d/orm-table-models.changed.md b/changelog.d/orm-table-models.changed.md new file mode 100644 index 000000000..cfc4a036a --- /dev/null +++ b/changelog.d/orm-table-models.changed.md @@ -0,0 +1 @@ +Described the database tables as SQLAlchemy ORM mapped classes and added ORM sessions with a per-transaction statement timeout. The schema and the service's behaviour don't change. diff --git a/docs/engineering/skills/database-access.md b/docs/engineering/skills/database-access.md index 84572177a..25f9ce793 100644 --- a/docs/engineering/skills/database-access.md +++ b/docs/engineering/skills/database-access.md @@ -25,7 +25,141 @@ It builds on the Stage 12 rules: of its own; policyengine-api migrates the Stage 12 tables. - `sim.stage12.persistence-typed-dml-only` and `sim.stage12.deployment-uses-api-schema`: the Stage 12 persistence - library uses typed SQLAlchemy Core DML only, with no raw SQL text. + library uses typed SQLAlchemy statements only, with no raw SQL text. + +## ORM + +`sim.persistence.orm-only` requires the SQLAlchemy ORM, not Core: code that +reads or writes database rows declares tables only as mapped classes and runs +every statement through an ORM `Session` against them. The rule is enforced +with no baseline, so any finding fails. + +What the ORM means here: + +- **Models** are SQLAlchemy 2.0 typed declarative classes (`Mapped[...]`, + `mapped_column`). Each derives from a `DeclarativeBase` in its library's + `orm_base.py`, the only module that builds a `MetaData`, with its schema. A + library has one base per `MetaData` it needs. A `MetaData` holds one table of + each name, so `policyengine-simulation-persistence` has `ServiceBase`, + `ReferencedBase`, `InputBase` and `ResultBase`, because its read-only models + and its key targets describe some of the same tables, and `CatalogBase` for + the PostgreSQL catalog tables the privilege check reads. A key to another + base's table names that class's attribute, such as `ReportRun.id`. + `policyengine-stage12-persistence` has `ComparisonBase`. Constraints and + indexes go in `__table_args__` under API v1's names. Use SQLAlchemy's own + ORM, not SQLModel. +- **Sessions:** only a library's `engine.py` creates or holds an `Engine`, and + no public function there returns one or a `Connection`. It hands out a + `sessionmaker` with `expire_on_commit=False`: + `create_simulation_session_factory` for the simulation service's connection + pool, `create_unpooled_session_factory` for the v2 Modal calls and + `create_single_connection_session_factory` for a short task that should take + one pooler slot, all from a database URL. `dispose_session_factory` closes a + factory's connections. + Everything else receives a `sessionmaker` or a `Session`, and runs a + transaction as `with session_factory.begin() as session:`, or as `with + bounded_session(session_factory, statement_timeout_milliseconds=...) as + session:` to limit its statements. + The classes declare no relationships, so a flush doesn't order inserts by + foreign key: flush the referenced row first. An attribute set to a SQL + expression, such as `func.now()`, must be refreshed before the session closes + if it will be read afterwards. To retry a refused connection without any risk + of repeating a write, retry only a first statement that changes nothing, which + makes the session check out its connection, as `StoreSessions` in + `policyengine_stage12_persistence.engine` does. +- **Statements** name mapped classes or their attributes: `session.get`, + `session.add`, `session.scalars(select(Model))`, and ORM-enabled + `update(Model)`, `insert(Model)` or + `postgresql.insert(Model).on_conflict_do_nothing()` run with + `session.execute`. The single-statement forms keep conditional updates, such + as claiming a pending job only while it has no Modal call ID, and + find-or-create inserts atomic. +- **SQL text** appears only as a model's server default, a partial index's + condition or a check constraint's condition. +- **Names:** an attribute or field that holds a mapped class is named after + what it holds, such as `model`, never `table`: every `.table` read in a + module that imports sqlalchemy is reported, because that is how a column + reaches its Core table. +- **PostgreSQL's catalog** is read through mapped classes too: `PgClass`, + `PgNamespace` and `PgAttribute` in `catalog_tables.py`. +- **Schema inspection**, for the deployment check that compares the live + schema with the models, reads the live schema through + `inspect_live_schema(session_factory)` in the Stage 12 library's `engine.py`, + which needs no Connection. An inspector on an engine takes a connection for + each read, so the check's factory has a one-connection pool + (`create_single_connection_session_factory`). It reads the models' columns + and keys from their mappers (`inspect(Model).columns`, `.primary_key`) and + `__table_args__`, none of which is reported. + +`projects/policyengine-simulation-entry/tests/test_orm_only.py` runs the checks +in `orm_access_support.py` over the same files as the guard below. They look +for Core rather than for the ORM, so a statement on mapped classes passes, +including one on expressions, labels, subqueries and aliases. + +Each file is checked on its own, through its own imports. A Core table is +reported where it starts, not at each later use, because the first finding +already fails the rule: where it is declared, where it is read from a mapped +class, mapper or `MetaData` or as the bare name `__table__`, and where an +annotation names `Table` or `TableClause`. + +Each finding is one of: + +- **Core table declaration:** a `Table(...)`, `Column(...)`, `table(...)`, + `column(...)`, `TableClause(...)`, `ColumnClause(...)` or `MetaData(...)` + call, apart from `MetaData` in an `orm_base.py`. +- **Core table access:** a parameter, variable, attribute or return annotated + `Table` or `TableClause`, unquoted or wholly quoted, apart from a class's bare + `__table__` annotation in an `orm_base.py`; + `__table__`, `local_table`, `persist_selectable` or `sorted_tables` anywhere, + and a read of the bare name `__table__`; + `tables` or + `selectable` of a mapper (`__mapper__`, `.mapper`, or `inspect(...)`, + `class_mapper(...)` or `object_mapper(...)`, called or held in a name, or a + name assigned `__mapper__` or `.mapper`) or of a MetaData (a name bound to + `MetaData(...)` or ending in `metadata`); and any `.table` read, which is + how a column or mapped attribute reaches its table, as in + `HouseholdJob.id.table`. A called `.table`, such as `sa.table(...)`, isn't + a read. +- **Core engine or connection:** `Connection` or `AsyncConnection` from + SQLAlchemy in any module; `Engine`, `create_engine` and the like outside a + library's `engine.py`, imported or referenced as `sa.Engine`; any + `.connect()`, `.connection()`, `.raw_connection()` or `.get_bind()` call; + `.begin()` on an engine (a name annotated as `Engine`, bound to + `create_engine(...)`, or named `engine` or ending in `_engine`); `.execute` + on anything not named `session` or ending in `_session`; `.scalar`, + `.scalars`, `.stream` or `.stream_scalars` on a name that is `connection` or + `conn` or ends in `_connection` or `_conn` (a query result has `.scalar` and + `.scalars` too); a public function of a library's `engine.py` whose return annotation + names an `Engine` or `Connection`; and `factory.kw["bind"]`, the engine + behind a `sessionmaker`, outside a library's `engine.py`. A library's + `engine.py` opens sessions, not connections. +- **Raw SQL:** `text(...)`, `TextClause(...)` or `literal_column(...)` + anywhere but a server default, an index's `postgresql_where` or a + `CheckConstraint`'s condition, or `exec_driver_sql` or `.cursor()`. +- **sqlmodel import.** + +Those that go by a method or attribute name alone apply only in modules that +import sqlalchemy: any `.table` read, `kw["bind"]`, the statement methods above, +and the `.connect()`, `.connection()`, `.raw_connection()`, +`.get_bind()`, `.begin()`, `.cursor()` and `.exec_driver_sql()` calls. + +The checks read source, not types, and guard against ordinary Core use, not +against deliberate evasion. They miss imports through `importlib`; engines and +connections passed around without an `Engine` or `Connection` annotation, so a +connection held under a name such as `session` isn't reported; a statement method +passed without being called, such as `run(connection.execute, statement)`; a +function that returns an engine with no return annotation; the engine an +`Inspector` holds in `bind` or `engine`; a quoted name inside an annotation, such +as `Optional["Table"]`; `Table` under another local name, such as `Jobs = Table`; +names from `from sqlalchemy import *`; a `.table` read in a module that doesn't +import sqlalchemy; and a Session's `bind` attribute. `.tables` of a `MetaData` +isn't reported when the name holding it doesn't end in `metadata` and wasn't +assigned `MetaData(...)`. + +`test_every_table_is_mapped_by_a_class` also imports both persistence libraries +and reports every table they hold that no mapped class maps, and +`test_no_project_installs_sqlmodel` fails if the `uv.lock` of any project with +a `pyproject.toml` holds SQLModel. ## Who owns the schema @@ -44,7 +178,8 @@ and this repository has no migration chain of its own `policies`, `parameters`, `parameter_values`, `tax_benefit_models` and `tax_benefit_model_versions`), which the simulation service only reads. Their read-only models in that library's `input_tables.py` name only the - columns the service reads, apart from the models of the tables it writes. + columns the service reads and each table's primary key, apart from the + models of the tables it writes. - the run tables the v2 Modal coordinator, workers, household functions and stuck-run sweep write as the Modal worker role: `report_runs`, `simulation_runs`, `budget_window_runs`, the result tables and diff --git a/docs/generated/rules.md b/docs/generated/rules.md index c7c4f6a75..dd13645b2 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -8,11 +8,11 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf | Tier | enforced | ratcheted | advisory | doc-only | Total | | --- | ---: | ---: | ---: | ---: | ---: | -| mechanical | 64 | 0 | 0 | 14 | 78 | +| mechanical | 65 | 0 | 0 | 14 | 79 | | metric | 1 | 0 | 0 | 0 | 1 | | judgment | 0 | 0 | 0 | 0 | 0 | -**Blocking:** 65 of 79 rules. +**Blocking:** 66 of 80 rules. ## Gaps @@ -82,6 +82,7 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf | `sim.boundaries.gateway-executor-decoupled` | The gateway and executor never import each other; they communicate only through the models and state helpers in policyengine-simulation-contract. | mechanical | doc-only | no | nothing | — | — | | `sim.boundaries.sqlalchemy-and-alembic-only` | Code connects to databases only through SQLAlchemy engines and connections, never by importing a database driver directly, and runtime code never changes the schema, which changes only through Alembic migrations. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_database_access_boundaries.py` | `pr.yml:test` | yes | | `sim.boundaries.unit-tests-need-no-services` | Unit tests never require real network credentials, Modal, Hugging Face, GCP or deployed services; those seams are mocked. | mechanical | doc-only | no | nothing | — | — | +| `sim.persistence.orm-only` | Code that reads or writes database rows declares tables only as SQLAlchemy ORM mapped classes and runs every statement through an ORM Session against those classes, never through Core Table objects or Connections. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_database_access_boundaries.py`, `projects/policyengine-simulation-entry/tests/test_orm_only.py` | `pr.yml:test` | yes | ```mermaid flowchart LR @@ -91,10 +92,14 @@ flowchart LR e_pytest_projects_policyengine_simulation_entry_tests_test_database_access_boundaries_py["pytest projects/policyengine-simulation-entry/tests/test_database_access_boundaries.py"] j_CI_pr_yml_test["CI pr.yml:test"] r_sim_boundaries_unit_tests_need_no_services("sim.boundaries.unit-tests-need-no-services (doc-only)") + r_sim_persistence_orm_only("sim.persistence.orm-only (enforced)") + e_pytest_projects_policyengine_simulation_entry_tests_test_database_access_boundaries_py__projects_policyengine_simulation_entry_tests_test_orm_only_py["pytest projects/policyengine-simulation-entry/tests/test_database_access_boundaries.py, projects/policyengine-simulation-entry/tests/test_orm_only.py"] r_sim_boundaries_gateway_executor_decoupled -.-> gap_unenforced r_sim_boundaries_sqlalchemy_and_alembic_only --> e_pytest_projects_policyengine_simulation_entry_tests_test_database_access_boundaries_py e_pytest_projects_policyengine_simulation_entry_tests_test_database_access_boundaries_py --> j_CI_pr_yml_test r_sim_boundaries_unit_tests_need_no_services -.-> gap_unenforced + r_sim_persistence_orm_only --> e_pytest_projects_policyengine_simulation_entry_tests_test_database_access_boundaries_py__projects_policyengine_simulation_entry_tests_test_orm_only_py + e_pytest_projects_policyengine_simulation_entry_tests_test_database_access_boundaries_py__projects_policyengine_simulation_entry_tests_test_orm_only_py --> j_CI_pr_yml_test ``` ## Stage 12 persistence @@ -106,7 +111,7 @@ flowchart LR | `sim.stage12.no-migration-chain` | This repository has no schema migration chain (no Alembic); Stage 12 tables are migrated by policyengine-api. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py` | `pr.yml:test` | **unproven** | | `sim.stage12.no-production-user-associations` | Stage 12 runtime modules never create production user associations (user_simulations or user_reports). | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py` | `pr.yml:test` | **unproven** | | `sim.stage12.no-schema-ddl` | Simulation runtimes and the Stage 12 persistence library contain no schema DDL (CREATE, ALTER, DROP or TRUNCATE of tables, types, schemas or databases, or create_all); policyengine-api owns the schema. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py` | `pr.yml:test` | yes | -| `sim.stage12.persistence-typed-dml-only` | The Stage 12 persistence library and runtime use typed SQLAlchemy Core DML only: no raw SQL text, driver-level SQL or schema operations. | mechanical | enforced | yes | pytest `libs/policyengine-stage12-persistence/tests/test_source_boundaries.py` | `pr.yml:test` | **unproven** | +| `sim.stage12.persistence-typed-dml-only` | The Stage 12 persistence library and runtime use typed SQLAlchemy statements only: no raw SQL text, driver-level SQL or schema operations. | mechanical | enforced | yes | pytest `libs/policyengine-stage12-persistence/tests/test_source_boundaries.py` | `pr.yml:test` | **unproven** | | `sim.stage12.production-lock-has-db-runtime` | The production lock file includes the direct Stage 12 database runtime dependencies. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py` | `pr.yml:test` | **unproven** | ```mermaid diff --git a/libs/policyengine-fastapi/pyproject.toml b/libs/policyengine-fastapi/pyproject.toml index 0eeed269d..9a11ca3f2 100644 --- a/libs/policyengine-fastapi/pyproject.toml +++ b/libs/policyengine-fastapi/pyproject.toml @@ -11,7 +11,6 @@ dependencies = [ "fastapi[standard] >=0.115.8,<0.116.0", "pyjwt >=2.10.1,<3.0.0", "opentelemetry-sdk >=1.44.0,<2.0.0", - "sqlmodel >=0.0.22,<0.0.23", "python-json-logger >=3.2.1,<4.0.0", "opentelemetry-instrumentation-logging >=0.65b0,<0.66", "opentelemetry-exporter-gcp-trace >=1.9.0,<2.0.0", diff --git a/libs/policyengine-fastapi/src/policyengine_fastapi/__init__.py b/libs/policyengine-fastapi/src/policyengine_fastapi/__init__.py index 77965ebbf..ae1b3d03d 100644 --- a/libs/policyengine-fastapi/src/policyengine_fastapi/__init__.py +++ b/libs/policyengine-fastapi/src/policyengine_fastapi/__init__.py @@ -1,4 +1,4 @@ -"""Reusable FastAPI and SQLModel building blocks that are independent of any one app.""" +"""Reusable FastAPI building blocks that are independent of any one app.""" from .observability import ( ObservabilityConfig as ObservabilityConfig, diff --git a/libs/policyengine-fastapi/src/policyengine_fastapi/database.py b/libs/policyengine-fastapi/src/policyengine_fastapi/database.py deleted file mode 100644 index 6a8341fe7..000000000 --- a/libs/policyengine-fastapi/src/policyengine_fastapi/database.py +++ /dev/null @@ -1,41 +0,0 @@ -"""SQLModel engine and per-request session dependency helpers.""" - -from collections.abc import Callable, Generator -from typing import Any - -from sqlalchemy import Engine, StaticPool, create_engine -from sqlmodel import Session - -# Generally following the guidance for SQL in fastAPI here -# https://fastapi.tiangolo.com/tutorial/sql-databases/ - -# NOTE: if you don't use StaticPool with in-memory sqlite you will get -# errors. -# https://sqlmodel.tiangolo.com/tutorial/fastapi/tests/#configure-the-in-memory-database - - -def create_sqlite_engine(filename: str | None = None) -> Engine: - """Create a SQLite engine for desktop use and tests. - - Without ``filename`` the database is in memory; StaticPool keeps that single - in-memory connection shared across threads. - """ - sqlite_url = f"sqlite:///{filename}" if filename else "sqlite:///:memory:" - connect_args = {"check_same_thread": False} - return create_engine(sqlite_url, connect_args=connect_args, poolclass=StaticPool) - - -SessionGeneratorFactory = Callable[[], Generator[Session, Any, None]] - - -def create_session_dep(engine: Engine) -> SessionGeneratorFactory: - """Return a FastAPI dependency that yields one SQLModel session per request. - - The session is bound to ``engine`` and closed when the request finishes. - """ - - def session_dep() -> Generator[Session]: - with Session(engine) as session: - yield session - - return session_dep diff --git a/libs/policyengine-fastapi/uv.lock b/libs/policyengine-fastapi/uv.lock index ea2d69a1e..117bba60e 100644 --- a/libs/policyengine-fastapi/uv.lock +++ b/libs/policyengine-fastapi/uv.lock @@ -546,37 +546,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b6/b0/be5d3329badb9230b765de6eea66b73abd5944bdeb5afb3562ddcd80ae84/googleapis_common_protos-1.74.0-py3-none-any.whl", hash = "sha256:702216f78610bb510e3f12ac3cafd281b7ac45cc5d86e90ad87e4d301a3426b5", size = 300743, upload-time = "2026-04-02T21:22:49.108Z" }, ] -[[package]] -name = "greenlet" -version = "3.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/86/94/a5935717b307d7c71fe877b52b884c6af707d2d2090db118a03fbd799369/greenlet-3.4.0.tar.gz", hash = "sha256:f50a96b64dafd6169e595a5c56c9146ef80333e67d4476a65a9c55f400fc22ff", size = 195913, upload-time = "2026-04-08T17:08:00.863Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7a/75/7e9cd1126a1e1f0cd67b0eda02e5221b28488d352684704a78ed505bd719/greenlet-3.4.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:43748988b097f9c6f09364f260741aa73c80747f63389824435c7a50bfdfd5c1", size = 285856, upload-time = "2026-04-08T15:52:45.82Z" }, - { url = "https://files.pythonhosted.org/packages/9d/c4/3e2df392e5cb199527c4d9dbcaa75c14edcc394b45040f0189f649631e3c/greenlet-3.4.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5566e4e2cd7a880e8c27618e3eab20f3494452d12fd5129edef7b2f7aa9a36d1", size = 610208, upload-time = "2026-04-08T16:24:39.674Z" }, - { url = "https://files.pythonhosted.org/packages/da/af/750cdfda1d1bd30a6c28080245be8d0346e669a98fdbae7f4102aa95fff3/greenlet-3.4.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1054c5a3c78e2ab599d452f23f7adafef55062a783a8e241d24f3b633ba6ff82", size = 621269, upload-time = "2026-04-08T16:30:59.767Z" }, - { url = "https://files.pythonhosted.org/packages/54/78/0cbc693622cd54ebe25207efbb3a0eb07c2639cb8594f6e3aaaa0bb077a8/greenlet-3.4.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f82cb6cddc27dd81c96b1506f4aa7def15070c3b2a67d4e46fd19016aacce6cf", size = 617549, upload-time = "2026-04-08T15:56:34.893Z" }, - { url = "https://files.pythonhosted.org/packages/ba/c0/8966767de01343c1ff47e8b855dc78e7d1a8ed2b7b9c83576a57e289f81d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:227a46251ecba4ff46ae742bc5ce95c91d5aceb4b02f885487aff269c127a729", size = 1575310, upload-time = "2026-04-08T16:26:21.671Z" }, - { url = "https://files.pythonhosted.org/packages/b8/38/bcdc71ba05e9a5fda87f63ffc2abcd1f15693b659346df994a48c968003d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5b99e87be7eba788dd5b75ba1cde5639edffdec5f91fe0d734a249535ec3408c", size = 1640435, upload-time = "2026-04-08T15:57:32.572Z" }, - { url = "https://files.pythonhosted.org/packages/a1/c2/19b664b7173b9e4ef5f77e8cef9f14c20ec7fce7920dc1ccd7afd955d093/greenlet-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:849f8bc17acd6295fcb5de8e46d55cc0e52381c56eaf50a2afd258e97bc65940", size = 238760, upload-time = "2026-04-08T17:04:03.878Z" }, - { url = "https://files.pythonhosted.org/packages/9b/96/795619651d39c7fbd809a522f881aa6f0ead504cc8201c3a5b789dfaef99/greenlet-3.4.0-cp313-cp313-win_arm64.whl", hash = "sha256:9390ad88b652b1903814eaabd629ca184db15e0eeb6fe8a390bbf8b9106ae15a", size = 235498, upload-time = "2026-04-08T17:05:00.584Z" }, - { url = "https://files.pythonhosted.org/packages/78/02/bde66806e8f169cf90b14d02c500c44cdbe02c8e224c9c67bafd1b8cadd1/greenlet-3.4.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:10a07aca6babdd18c16a3f4f8880acfffc2b88dfe431ad6aa5f5740759d7d75e", size = 286291, upload-time = "2026-04-08T17:09:34.307Z" }, - { url = "https://files.pythonhosted.org/packages/05/1f/39da1c336a87d47c58352fb8a78541ce63d63ae57c5b9dae1fe02801bbc2/greenlet-3.4.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:076e21040b3a917d3ce4ad68fb5c3c6b32f1405616c4a57aa83120979649bd3d", size = 656749, upload-time = "2026-04-08T16:24:41.721Z" }, - { url = "https://files.pythonhosted.org/packages/d3/6c/90ee29a4ee27af7aa2e2ec408799eeb69ee3fcc5abcecac6ddd07a5cd0f2/greenlet-3.4.0-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e82689eea4a237e530bb5cb41b180ef81fa2160e1f89422a67be7d90da67f615", size = 669084, upload-time = "2026-04-08T16:31:01.372Z" }, - { url = "https://files.pythonhosted.org/packages/07/49/d4cad6e5381a50947bb973d2f6cf6592621451b09368b8c20d9b8af49c5b/greenlet-3.4.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df3b0b2289ec686d3c821a5fee44259c05cfe824dd5e6e12c8e5f5df23085cf", size = 665621, upload-time = "2026-04-08T15:56:35.995Z" }, - { url = "https://files.pythonhosted.org/packages/37/31/d1edd54f424761b5d47718822f506b435b6aab2f3f93b465441143ea5119/greenlet-3.4.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8bff29d586ea415688f4cec96a591fcc3bf762d046a796cdadc1fdb6e7f2d5bf", size = 1622259, upload-time = "2026-04-08T16:26:23.201Z" }, - { url = "https://files.pythonhosted.org/packages/b0/c6/6d3f9cdcb21c4e12a79cb332579f1c6aa1af78eb68059c5a957c7812d95e/greenlet-3.4.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8a569c2fb840c53c13a2b8967c63621fafbd1a0e015b9c82f408c33d626a2fda", size = 1686916, upload-time = "2026-04-08T15:57:34.282Z" }, - { url = "https://files.pythonhosted.org/packages/63/45/c1ca4a1ad975de4727e52d3ffe641ae23e1d7a8ffaa8ff7a0477e1827b92/greenlet-3.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:207ba5b97ea8b0b60eb43ffcacf26969dd83726095161d676aac03ff913ee50d", size = 239821, upload-time = "2026-04-08T17:03:48.423Z" }, - { url = "https://files.pythonhosted.org/packages/71/c4/6f621023364d7e85a4769c014c8982f98053246d142420e0328980933ceb/greenlet-3.4.0-cp314-cp314-win_arm64.whl", hash = "sha256:f8296d4e2b92af34ebde81085a01690f26a51eb9ac09a0fcadb331eb36dbc802", size = 236932, upload-time = "2026-04-08T17:04:33.551Z" }, - { url = "https://files.pythonhosted.org/packages/d4/8f/18d72b629783f5e8d045a76f5325c1e938e659a9e4da79c7dcd10169a48d/greenlet-3.4.0-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:d70012e51df2dbbccfaf63a40aaf9b40c8bed37c3e3a38751c926301ce538ece", size = 294681, upload-time = "2026-04-08T15:52:35.778Z" }, - { url = "https://files.pythonhosted.org/packages/9e/ad/5fa86ec46769c4153820d58a04062285b3b9e10ba3d461ee257b68dcbf53/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a58bec0751f43068cd40cff31bb3ca02ad6000b3a51ca81367af4eb5abc480c8", size = 658899, upload-time = "2026-04-08T16:24:43.32Z" }, - { url = "https://files.pythonhosted.org/packages/43/f0/4e8174ca0e87ae748c409f055a1ba161038c43cc0a5a6f1433a26ac2e5bf/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:05fa0803561028f4b2e3b490ee41216a842eaee11aed004cc343a996d9523aa2", size = 665284, upload-time = "2026-04-08T16:31:02.833Z" }, - { url = "https://files.pythonhosted.org/packages/19/da/991cf7cd33662e2df92a1274b7eb4d61769294d38a1bba8a45f31364845e/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e60d38719cb80b3ab5e85f9f1aed4960acfde09868af6762ccb27b260d68f4ed", size = 661861, upload-time = "2026-04-08T15:56:37.269Z" }, - { url = "https://files.pythonhosted.org/packages/36/c5/6c2c708e14db3d9caea4b459d8464f58c32047451142fe2cfd90e7458f41/greenlet-3.4.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7f50c804733b43eded05ae694691c9aa68bca7d0a867d67d4a3f514742a2d53f", size = 1622182, upload-time = "2026-04-08T16:26:24.777Z" }, - { url = "https://files.pythonhosted.org/packages/7a/4c/50c5fed19378e11a29fabab1f6be39ea95358f4a0a07e115a51ca93385d8/greenlet-3.4.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2d4f0635dc4aa638cda4b2f5a07ae9a2cff9280327b581a3fcb6f317b4fbc38a", size = 1685050, upload-time = "2026-04-08T15:57:36.453Z" }, - { url = "https://files.pythonhosted.org/packages/db/72/85ae954d734703ab48e622c59d4ce35d77ce840c265814af9c078cacc7aa/greenlet-3.4.0-cp314-cp314t-win_amd64.whl", hash = "sha256:1a4a48f24681300c640f143ba7c404270e1ebbbcf34331d7104a4ff40f8ea705", size = 245554, upload-time = "2026-04-08T17:03:50.044Z" }, -] - [[package]] name = "grpcio" version = "1.80.0" @@ -1022,7 +991,6 @@ dependencies = [ { name = "opentelemetry-sdk" }, { name = "pyjwt" }, { name = "python-json-logger" }, - { name = "sqlmodel" }, { name = "uvicorn" }, ] @@ -1053,7 +1021,6 @@ requires-dist = [ { name = "pytest-asyncio", marker = "extra == 'test'", specifier = ">=0.25.3" }, { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=6.1.1" }, { name = "python-json-logger", specifier = ">=3.2.1,<4.0.0" }, - { name = "sqlmodel", specifier = ">=0.0.22,<0.0.23" }, { name = "uvicorn", specifier = ">=0.35.0" }, ] provides-extras = ["test", "build"] @@ -1468,58 +1435,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, ] -[[package]] -name = "sqlalchemy" -version = "2.0.49" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/09/45/461788f35e0364a8da7bda51a1fe1b09762d0c32f12f63727998d85a873b/sqlalchemy-2.0.49.tar.gz", hash = "sha256:d15950a57a210e36dd4cec1aac22787e2a4d57ba9318233e2ef8b2daf9ff2d5f", size = 9898221, upload-time = "2026-04-03T16:38:11.704Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ae/81/81755f50eb2478eaf2049728491d4ea4f416c1eb013338682173259efa09/sqlalchemy-2.0.49-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:df2d441bacf97022e81ad047e1597552eb3f83ca8a8f1a1fdd43cd7fe3898120", size = 2154547, upload-time = "2026-04-03T16:53:08.64Z" }, - { url = "https://files.pythonhosted.org/packages/a2/bc/3494270da80811d08bcfa247404292428c4fe16294932bce5593f215cad9/sqlalchemy-2.0.49-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8e20e511dc15265fb433571391ba313e10dd8ea7e509d51686a51313b4ac01a2", size = 3280782, upload-time = "2026-04-03T17:07:43.508Z" }, - { url = "https://files.pythonhosted.org/packages/cd/f5/038741f5e747a5f6ea3e72487211579d8cbea5eb9827a9cbd61d0108c4bd/sqlalchemy-2.0.49-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47604cb2159f8bbd5a1ab48a714557156320f20871ee64d550d8bf2683d980d3", size = 3297156, upload-time = "2026-04-03T17:12:27.697Z" }, - { url = "https://files.pythonhosted.org/packages/88/50/a6af0ff9dc954b43a65ca9b5367334e45d99684c90a3d3413fc19a02d43c/sqlalchemy-2.0.49-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:22d8798819f86720bc646ab015baff5ea4c971d68121cb36e2ebc2ee43ead2b7", size = 3228832, upload-time = "2026-04-03T17:07:45.38Z" }, - { url = "https://files.pythonhosted.org/packages/bc/d1/5f6bdad8de0bf546fc74370939621396515e0cdb9067402d6ba1b8afbe9a/sqlalchemy-2.0.49-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9b1c058c171b739e7c330760044803099c7fff11511e3ab3573e5327116a9c33", size = 3267000, upload-time = "2026-04-03T17:12:29.657Z" }, - { url = "https://files.pythonhosted.org/packages/f7/30/ad62227b4a9819a5e1c6abff77c0f614fa7c9326e5a3bdbee90f7139382b/sqlalchemy-2.0.49-cp313-cp313-win32.whl", hash = "sha256:a143af2ea6672f2af3f44ed8f9cd020e9cc34c56f0e8db12019d5d9ecf41cb3b", size = 2115641, upload-time = "2026-04-03T17:05:43.989Z" }, - { url = "https://files.pythonhosted.org/packages/17/3a/7215b1b7d6d49dc9a87211be44562077f5f04f9bb5a59552c1c8e2d98173/sqlalchemy-2.0.49-cp313-cp313-win_amd64.whl", hash = "sha256:12b04d1db2663b421fe072d638a138460a51d5a862403295671c4f3987fb9148", size = 2141498, upload-time = "2026-04-03T17:05:45.7Z" }, - { url = "https://files.pythonhosted.org/packages/28/4b/52a0cb2687a9cd1648252bb257be5a1ba2c2ded20ba695c65756a55a15a4/sqlalchemy-2.0.49-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:24bd94bb301ec672d8f0623eba9226cc90d775d25a0c92b5f8e4965d7f3a1518", size = 3560807, upload-time = "2026-04-03T16:58:31.666Z" }, - { url = "https://files.pythonhosted.org/packages/8c/d8/fda95459204877eed0458550d6c7c64c98cc50c2d8d618026737de9ed41a/sqlalchemy-2.0.49-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a51d3db74ba489266ef55c7a4534eb0b8db9a326553df481c11e5d7660c8364d", size = 3527481, upload-time = "2026-04-03T17:06:00.155Z" }, - { url = "https://files.pythonhosted.org/packages/ff/0a/2aac8b78ac6487240cf7afef8f203ca783e8796002dc0cf65c4ee99ff8bb/sqlalchemy-2.0.49-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:55250fe61d6ebfd6934a272ee16ef1244e0f16b7af6cd18ab5b1fc9f08631db0", size = 3468565, upload-time = "2026-04-03T16:58:33.414Z" }, - { url = "https://files.pythonhosted.org/packages/a5/3d/ce71cfa82c50a373fd2148b3c870be05027155ce791dc9a5dcf439790b8b/sqlalchemy-2.0.49-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:46796877b47034b559a593d7e4b549aba151dae73f9e78212a3478161c12ab08", size = 3477769, upload-time = "2026-04-03T17:06:02.787Z" }, - { url = "https://files.pythonhosted.org/packages/d5/e8/0a9f5c1f7c6f9ca480319bf57c2d7423f08d31445974167a27d14483c948/sqlalchemy-2.0.49-cp313-cp313t-win32.whl", hash = "sha256:9c4969a86e41454f2858256c39bdfb966a20961e9b58bf8749b65abf447e9a8d", size = 2143319, upload-time = "2026-04-03T17:02:04.328Z" }, - { url = "https://files.pythonhosted.org/packages/0e/51/fb5240729fbec73006e137c4f7a7918ffd583ab08921e6ff81a999d6517a/sqlalchemy-2.0.49-cp313-cp313t-win_amd64.whl", hash = "sha256:b9870d15ef00e4d0559ae10ee5bc71b654d1f20076dbe8bc7ed19b4c0625ceba", size = 2175104, upload-time = "2026-04-03T17:02:05.989Z" }, - { url = "https://files.pythonhosted.org/packages/55/33/bf28f618c0a9597d14e0b9ee7d1e0622faff738d44fe986ee287cdf1b8d0/sqlalchemy-2.0.49-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:233088b4b99ebcbc5258c755a097aa52fbf90727a03a5a80781c4b9c54347a2e", size = 2156356, upload-time = "2026-04-03T16:53:09.914Z" }, - { url = "https://files.pythonhosted.org/packages/d1/a7/5f476227576cb8644650eff68cc35fa837d3802b997465c96b8340ced1e2/sqlalchemy-2.0.49-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57ca426a48eb2c682dae8204cd89ea8ab7031e2675120a47924fabc7caacbc2a", size = 3276486, upload-time = "2026-04-03T17:07:46.9Z" }, - { url = "https://files.pythonhosted.org/packages/2e/84/efc7c0bf3a1c5eef81d397f6fddac855becdbb11cb38ff957888603014a7/sqlalchemy-2.0.49-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:685e93e9c8f399b0c96a624799820176312f5ceef958c0f88215af4013d29066", size = 3281479, upload-time = "2026-04-03T17:12:32.226Z" }, - { url = "https://files.pythonhosted.org/packages/91/68/bb406fa4257099c67bd75f3f2261b129c63204b9155de0d450b37f004698/sqlalchemy-2.0.49-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9e0400fa22f79acc334d9a6b185dc00a44a8e6578aa7e12d0ddcd8434152b187", size = 3226269, upload-time = "2026-04-03T17:07:48.678Z" }, - { url = "https://files.pythonhosted.org/packages/67/84/acb56c00cca9f251f437cb49e718e14f7687505749ea9255d7bd8158a6df/sqlalchemy-2.0.49-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a05977bffe9bffd2229f477fa75eabe3192b1b05f408961d1bebff8d1cd4d401", size = 3248260, upload-time = "2026-04-03T17:12:34.381Z" }, - { url = "https://files.pythonhosted.org/packages/56/19/6a20ea25606d1efd7bd1862149bb2a22d1451c3f851d23d887969201633f/sqlalchemy-2.0.49-cp314-cp314-win32.whl", hash = "sha256:0f2fa354ba106eafff2c14b0cc51f22801d1e8b2e4149342023bd6f0955de5f5", size = 2118463, upload-time = "2026-04-03T17:05:47.093Z" }, - { url = "https://files.pythonhosted.org/packages/cf/4f/8297e4ed88e80baa1f5aa3c484a0ee29ef3c69c7582f206c916973b75057/sqlalchemy-2.0.49-cp314-cp314-win_amd64.whl", hash = "sha256:77641d299179c37b89cf2343ca9972c88bb6eef0d5fc504a2f86afd15cd5adf5", size = 2144204, upload-time = "2026-04-03T17:05:48.694Z" }, - { url = "https://files.pythonhosted.org/packages/1f/33/95e7216df810c706e0cd3655a778604bbd319ed4f43333127d465a46862d/sqlalchemy-2.0.49-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c1dc3368794d522f43914e03312202523cc89692f5389c32bea0233924f8d977", size = 3565474, upload-time = "2026-04-03T16:58:35.128Z" }, - { url = "https://files.pythonhosted.org/packages/0c/a4/ed7b18d8ccf7f954a83af6bb73866f5bc6f5636f44c7731fbb741f72cc4f/sqlalchemy-2.0.49-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7c821c47ecfe05cc32140dcf8dc6fd5d21971c86dbd56eabfe5ba07a64910c01", size = 3530567, upload-time = "2026-04-03T17:06:04.587Z" }, - { url = "https://files.pythonhosted.org/packages/73/a3/20faa869c7e21a827c4a2a42b41353a54b0f9f5e96df5087629c306df71e/sqlalchemy-2.0.49-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9c04bff9a5335eb95c6ecf1c117576a0aa560def274876fd156cfe5510fccc61", size = 3474282, upload-time = "2026-04-03T16:58:37.131Z" }, - { url = "https://files.pythonhosted.org/packages/b7/50/276b9a007aa0764304ad467eceb70b04822dc32092492ee5f322d559a4dc/sqlalchemy-2.0.49-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7f605a456948c35260e7b2a39f8952a26f077fd25653c37740ed186b90aaa68a", size = 3480406, upload-time = "2026-04-03T17:06:07.176Z" }, - { url = "https://files.pythonhosted.org/packages/e5/c3/c80fcdb41905a2df650c2a3e0337198b6848876e63d66fe9188ef9003d24/sqlalchemy-2.0.49-cp314-cp314t-win32.whl", hash = "sha256:6270d717b11c5476b0cbb21eedc8d4dbb7d1a956fd6c15a23e96f197a6193158", size = 2149151, upload-time = "2026-04-03T17:02:07.281Z" }, - { url = "https://files.pythonhosted.org/packages/05/52/9f1a62feab6ed368aff068524ff414f26a6daebc7361861035ae00b05530/sqlalchemy-2.0.49-cp314-cp314t-win_amd64.whl", hash = "sha256:275424295f4256fd301744b8f335cff367825d270f155d522b30c7bf49903ee7", size = 2184178, upload-time = "2026-04-03T17:02:08.623Z" }, - { url = "https://files.pythonhosted.org/packages/e5/30/8519fdde58a7bdf155b714359791ad1dc018b47d60269d5d160d311fdc36/sqlalchemy-2.0.49-py3-none-any.whl", hash = "sha256:ec44cfa7ef1a728e88ad41674de50f6db8cfdb3e2af84af86e0041aaf02d43d0", size = 1942158, upload-time = "2026-04-03T16:53:44.135Z" }, -] - -[[package]] -name = "sqlmodel" -version = "0.0.22" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "sqlalchemy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b5/39/8641040ab0d5e1d8a1c2325ae89a01ae659fc96c61a43d158fb71c9a0bf0/sqlmodel-0.0.22.tar.gz", hash = "sha256:7d37c882a30c43464d143e35e9ecaf945d88035e20117bf5ec2834a23cbe505e", size = 116392, upload-time = "2024-08-31T09:43:24.088Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dd/b1/3af5104b716c420e40a6ea1b09886cae3a1b9f4538343875f637755cae5b/sqlmodel-0.0.22-py3-none-any.whl", hash = "sha256:a1ed13e28a1f4057cbf4ff6cdb4fc09e85702621d3259ba17b3c230bfb2f941b", size = 28276, upload-time = "2024-08-31T09:43:22.358Z" }, -] - [[package]] name = "starlette" version = "0.46.2" diff --git a/libs/policyengine-simulation-persistence/README.md b/libs/policyengine-simulation-persistence/README.md index a25ad7771..2c4115e6f 100644 --- a/libs/policyengine-simulation-persistence/README.md +++ b/libs/policyengine-simulation-persistence/README.md @@ -1,6 +1,6 @@ # PolicyEngine simulation persistence -Database connections and table models for the simulation service (the +Database sessions and table models for the simulation service (the Simulation Entrypoint on Cloud Run) and the v2 Modal runtime. The service connects to the Public API's Postgres database with its own role, and writes rows in tables the Public API's migration chain creates. This repository has @@ -16,28 +16,44 @@ modules) imports neither library. The temporary Stage 12 comparison tables are not modelled here: the Stage 12 library keeps them until the comparison path is removed. -## Connections - -`create_simulation_engine` builds the runtime engine from the canonical -`postgresql://` URL the deployment stores, selecting the psycopg 3 driver. It -keeps a small pool, checks connections before use, and never prepares -server-side statements, so the same URL works through Supabase's session and -transaction poolers. A malformed URL raises `ValueError` without repeating the -URL, which holds a password. - -`create_unpooled_engine` builds the engine the v2 Modal calls and the -comparison path use, from the same URL check: it keeps no pool, so each -operation's session ends with it and a call that spends most of its life -calculating holds no pooler slot, and it waits 10 seconds to connect. The -Stage 12 library binds it as `create_stage12_engine`. +## Sessions + +`engine.py` keeps each engine to itself and hands out `sessionmaker`s. + +`create_simulation_session_factory` builds the runtime sessions from the +canonical `postgresql://` URL the deployment stores, selecting the psycopg 3 +driver. They share a small pool, check connections before use, and never +prepare server-side statements, so the same URL works through Supabase's +session and transaction poolers. A malformed URL raises `ValueError` without +repeating the URL, which holds a password. + +`create_unpooled_session_factory` builds the sessions the v2 Modal calls and +the comparison path use, from the same URL check: it keeps no pool, so each +operation's session ends with its connection and a call that spends most of +its life calculating holds no pooler slot, and it waits 10 seconds to connect. +The Stage 12 library's `create_stage12_session_factory` returns these. +`create_single_connection_session_factory` keeps one connection that its +sessions and schema reads take turns on, for a short task such as the Stage 12 +deployment check. `dispose_session_factory` closes a factory's pooled +connections. + +`create_session_factory` turns an engine into a `sessionmaker`, for tests that +build their own engine. Attributes keep their values after the transaction +commits, apart from one set to a SQL expression such as `func.now()`, which +must be refreshed before the session closes. `bounded_session` runs one +transaction on such a factory with a `statement_timeout` local to it. The +classes declare no relationships, so a session doesn't order inserts by foreign key: flush a +referenced row before adding the row that names it. ## Tables -`tables.py` models, with SQLAlchemy Core, the six tables the service creates -rows in: `simulations`, `reports`, `simulation_runs`, `report_runs`, -`budget_window_runs` and `household_jobs`, all in `service_metadata`, -which names the `public` schema so writes reach those tables whatever the -role's `search_path`. +`tables.py` models, as SQLAlchemy ORM mapped classes, the six tables the +service creates rows in: `Simulation`, `Report`, `SimulationRun`, `ReportRun`, +`BudgetWindowRun` and `HouseholdJob`, for `simulations`, `reports`, +`simulation_runs`, `report_runs`, `budget_window_runs` and `household_jobs`. +They derive from `ServiceBase` in `orm_base.py`, and their tables are in +`service_metadata`, which names the `public` schema so writes reach those +tables whatever the role's `search_path`. Each model matches the table the Public API's v2 migration chain builds: every column with its type, nullability and server default, and every constraint and index under the chain's name. The models only describe the @@ -52,9 +68,10 @@ stay: 20 keys name rows the Public API writes in its users, policies, dynamics, households, datasets, regions and model catalog tables. PostgreSQL checks each of them on insert, as the referenced table's owner, so the service's role needs no privilege on those tables for it, and a row naming -one that doesn't exist fails with SQLSTATE `23503`. Those tables are modelled -in `referenced_metadata` by their `id` alone, the column every key names, so -SQLAlchemy can declare the keys; nothing reads or writes through them. Code +one that doesn't exist fails with SQLSTATE `23503`. `referenced_tables.py` +models those tables, in `referenced_metadata`, by their `id` alone, the column +every key names, so SQLAlchemy can declare the keys; nothing reads or writes +through them. Code that builds the models' tables in a test database creates `referenced_metadata`'s tables first. @@ -85,7 +102,8 @@ run's economy output. calculation is read from: `households`, `policies`, `dynamics`, `parameters`, `parameter_values`, `regions`, `datasets`, `tax_benefit_models` and `tax_benefit_model_versions`, in `input_metadata`. The service only reads them, with `SELECT`. Each names only -the columns the service reads, so they are not models of the whole tables, +the columns the service reads and its primary key, `id`, which a mapped class +needs, so they are not models of the whole tables, and they are kept apart from the models of the tables the service writes (`service_metadata`) and from the tables those name by `id` alone (`referenced_metadata`). @@ -103,12 +121,17 @@ call (`fail_unstarted_household_job`). Both updates apply only while the job is still `pending` with no call ID, and set `updated_at` themselves. A job naming a household, policy or catalog version that does not exist is refused by its foreign key with SQLSTATE `23503`; `HouseholdJobReferenceError` names -the column. Each function takes a connection, so the caller chooses the +the column. Each function takes an ORM session, so the caller chooses the transactions. `HouseholdJobStore` runs them as the Simulation Entrypoint -does, each step in a transaction of its own: `read_inputs` reads the catalog -version, the household and the policy's reform values together, and -`create_job`, `record_call`, `fail_unstarted` and `read` each write or read -the job. `HouseholdJobStore.from_engine` opens each with `bounded_transaction`, +does, each step in a session and transaction of its own: `read_inputs` reads +the catalog version, the household and the policy's reform values together, +and `create_job`, `record_call`, `fail_unstarted` and `read` each write or +read the job. `insert_household_job` flushes the new row, so its INSERT runs +before the commit. A `create_job` whose commit fails after that flush raises +`EconomyRecordOutcomeUnknownError`, as the economy and household report +creates do: PostgreSQL may have applied the commit, so the job may exist. A +failure during the flush leaves nothing written and is raised as it is. +`HouseholdJobStore.from_session_factory` opens each with `bounded_session`, which commits when its block ends and sets `statement_timeout` with `set_config`, local to the transaction, so the database ends a statement that runs too long and the limit never stays on a pooled connection. @@ -137,7 +160,8 @@ runs created for it. `economy_run_reads.py` reads a report, or a simulation, with its newest run's state. `EconomyRunStore` runs them as the Simulation Entrypoint does, each step in -a transaction of its own: `read_report_inputs` reads the catalog version, +an ORM session and transaction of its own (`from_session_factory` opens each +with `bounded_session`): `read_report_inputs` reads the catalog version, model, region and dataset, and each side's policy and dynamic values together; `read_simulation_inputs` does the same for one simulation, and `read_budget_window_inputs` for a budget window's two sides; the writes and @@ -227,7 +251,8 @@ Modal call ID (`household_simulation_run_row`, `insert_household_simulation_run`). A row naming one that does not exist is refused as `EconomyRecordReferenceError`. -`HouseholdReportStore` runs the steps in transactions of their own: +`HouseholdReportStore` runs the steps in ORM sessions and transactions of +their own, opened by `from_session_factory` with `bounded_session`: `read_inputs` reads the catalog version for the .py version, the country's model, the stored household and each side's policy values together, raising `UnknownCatalogVersionError`, `UnknownHouseholdError` or @@ -422,8 +447,9 @@ refuses any other change to a run. The Public API's privilege manifest grants them all. `MODAL_WORKER_TABLE_PRIVILEGES` and `MODAL_WORKER_UPDATABLE_COLUMNS` list what -the Modal worker role needs: `SELECT` on the four run tables, `INSERT` on -every result table, and `UPDATE` of each run's state (`status`, +the Modal worker role needs: `SELECT` on the four run tables, `SELECT` and +`INSERT` on every result table, because a session's insert reads the row's +server defaults back, and `UPDATE` of each run's state (`status`, `modal_call_id`, `error_message`, `started_at`, `completed_at` and `updated_at`; a budget-window run has no call of its own) and of the results its own row holds: a report run's cliff impact, `spm_provenance` and @@ -441,11 +467,12 @@ public"`), since it can use none of its grants. The check doesn't look for privileges the role holds beyond these: `UPDATE` on a whole table satisfies each column's check, and the Public API's own privilege verification fails when the role holds it. Tables and columns are found through PostgreSQL's -catalog, which needs no privilege. `read_missing_table_privileges` runs that -check in a transaction whose `statement_timeout` it sets with `set_config`, -local to the transaction, so the database ends any statement of the check -that runs too long; -the Simulation Entrypoint's readiness check uses it, on a thread of its own, +catalog, which needs no privilege, through the mapped classes in +`catalog_tables.py`. `missing_table_privileges` takes a `Session`, and +`read_missing_table_privileges` runs it from a session factory, in a +transaction whose `statement_timeout` it sets with `set_config`, local to the +transaction, so the database ends any statement of the check that runs too +long; the Simulation Entrypoint's readiness check uses it, on a thread of its own, reports whether any privilege is missing, and logs the names. ## Tests diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py index 8d05ea589..eb6663fdb 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py @@ -85,9 +85,12 @@ UnknownRegionError, ) from policyengine_simulation_persistence.engine import ( - bounded_transaction, - create_simulation_engine, - create_unpooled_engine, + bounded_session, + create_session_factory, + create_simulation_session_factory, + create_single_connection_session_factory, + create_unpooled_session_factory, + dispose_session_factory, normalized_database_url, ) from policyengine_simulation_persistence.household_job_records import ( @@ -115,7 +118,18 @@ HouseholdReportInputs, HouseholdReportStore, ) -from policyengine_simulation_persistence.input_tables import input_metadata +from policyengine_simulation_persistence.input_tables import ( + Dataset, + Dynamic, + Household, + Parameter, + ParameterValue, + Policy, + Region, + TaxBenefitModel, + TaxBenefitModelVersion, + input_metadata, +) from policyengine_simulation_persistence.keyed_economy_records import ( FoundEconomyReport, FoundEconomySimulation, @@ -138,6 +152,12 @@ KeyedWindowYear, find_or_create_budget_window, ) +from policyengine_simulation_persistence.orm_base import ( + InputBase, + ReferencedBase, + ResultBase, + ServiceBase, +) from policyengine_simulation_persistence.policy_reform import ( MissingPolicyParametersError, OverlappingReformError, @@ -148,9 +168,22 @@ read_dynamic_reform, read_policy_reform, ) +from policyengine_simulation_persistence.referenced_tables import referenced_metadata from policyengine_simulation_persistence.result_tables import ( + BudgetSummary, + CongressionalDistrictImpact, + ConstituencyImpact, + DecileImpact, DecileType, + Inequality, + IntraDecileImpact, + LaborSupplyDecileResponse, + LaborSupplyResponse, + LocalAuthorityImpact, + Poverty, + ProgramStatistics, UKAreaType, + WithheldAreaCode, result_metadata, ) from policyengine_simulation_persistence.shared_rows import ( @@ -169,19 +202,18 @@ from policyengine_simulation_persistence.tables import ( LATEST_YEAR, MAX_BUDGET_WINDOW_YEARS, + BudgetWindowRun, + HouseholdJob, HouseholdJobStatus, + Report, + ReportRun, ReportRunStatus, ReportRunTrigger, + Simulation, + SimulationRun, SimulationStatus, SimulationType, - budget_window_runs, - household_jobs, - referenced_metadata, - report_runs, - reports, service_metadata, - simulation_runs, - simulations, ) __all__ = [ @@ -193,10 +225,17 @@ "MODAL_WORKER_UPDATABLE_COLUMNS", "SERVICE_TABLE_PRIVILEGES", "SERVICE_UPDATABLE_COLUMNS", + "BudgetSummary", "BudgetWindowInputs", + "BudgetWindowRun", "CatalogDataset", "CatalogRegion", + "CongressionalDistrictImpact", + "ConstituencyImpact", + "Dataset", + "DecileImpact", "DecileType", + "Dynamic", "EconomyCatalogRows", "EconomyPopulation", "EconomyRecordOutcomeUnknownError", @@ -219,7 +258,9 @@ "FoundSimulationRun", "FoundWindowRun", "FoundWindowYear", + "Household", "HouseholdCalculationInputs", + "HouseholdJob", "HouseholdJobRecord", "HouseholdJobReferenceError", "HouseholdJobStatus", @@ -229,12 +270,18 @@ "HouseholdReportStore", "HouseholdRerunSide", "HouseholdSimulationDefinition", + "Inequality", + "InputBase", + "IntraDecileImpact", "KeyedBudgetWindow", "KeyedEconomyReport", "KeyedEconomySimulation", "KeyedHouseholdReport", "KeyedWindowYear", + "LaborSupplyDecileResponse", + "LaborSupplyResponse", "LocalAreaDatasetMismatchError", + "LocalAuthorityImpact", "MissingPolicyParametersError", "NewHouseholdJob", "NewHouseholdReportRerun", @@ -242,10 +289,19 @@ "NewSimulationRerun", "NewWindowRerun", "OverlappingReformError", + "Parameter", + "ParameterValue", + "Policy", "PolicyReform", + "Poverty", + "ProgramStatistics", "ProposedSide", "RecordKind", + "ReferencedBase", + "Region", + "Report", "ReportDefinition", + "ReportRun", "ReportRunState", "ReportRunStatus", "ReportRunTrigger", @@ -253,17 +309,23 @@ "ReportSideError", "RerunRecordDeletedError", "RerunSide", + "ResultBase", "RunVersions", + "ServiceBase", "SharedRowNotSettledError", "SideDefinition", "SideRefusal", + "Simulation", "SimulationChoice", "SimulationDefinition", "SimulationInputs", + "SimulationRun", "SimulationRunState", "SimulationStatus", "SimulationType", "StoredHousehold", + "TaxBenefitModel", + "TaxBenefitModelVersion", "UKAreaType", "UnknownCatalogVersionError", "UnknownDatasetError", @@ -275,13 +337,16 @@ "WindowSide", "WindowYearDefinition", "WindowYearRuns", - "bounded_transaction", - "budget_window_runs", + "WithheldAreaCode", + "bounded_session", "canonical_json", "combine_reforms", - "create_simulation_engine", - "create_unpooled_engine", + "create_session_factory", + "create_simulation_session_factory", + "create_single_connection_session_factory", + "create_unpooled_session_factory", "definition_key", + "dispose_session_factory", "fail_unstarted_household_job", "fail_unstarted_report_run", "fail_unstarted_simulation_run", @@ -295,7 +360,6 @@ "find_or_insert_simulation", "find_or_insert_simulation_run", "find_or_insert_simulation_runs", - "household_jobs", "input_metadata", "insert_household_job", "insert_household_report_rerun", @@ -324,10 +388,6 @@ "record_report_run_call", "record_simulation_run_call", "referenced_metadata", - "report_runs", - "reports", "result_metadata", "service_metadata", - "simulation_runs", - "simulations", ] diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/access.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/access.py index 6b2216ac8..1ab50e183 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/access.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/access.py @@ -13,47 +13,24 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Final -from sqlalchemy import ( - String, - Subquery, - and_, - column, - false, - func, - select, - table, - values, -) +from sqlalchemy import String, Subquery, and_, false, func, literal, select +from sqlalchemy.dialects.postgresql import ARRAY -from policyengine_simulation_persistence.result_tables import result_metadata -from policyengine_simulation_persistence.tables import TABLE_SCHEMA +from policyengine_simulation_persistence.catalog_tables import ( + PgAttribute, + PgClass, + PgNamespace, +) +from policyengine_simulation_persistence.engine import bounded_session +from policyengine_simulation_persistence.orm_base import TABLE_SCHEMA +from policyengine_simulation_persistence.result_tables import RESULT_TABLE_NAMES if TYPE_CHECKING: - from collections.abc import Mapping + from collections.abc import Mapping, Sequence - from sqlalchemy import Connection, Engine + from sqlalchemy.orm import Session, sessionmaker + from sqlalchemy.sql.selectable import TableValuedAlias -# PostgreSQL's catalog, which any role may read: a table is found by name -# without the USAGE privilege on its schema that to_regclass would need. -_pg_class = table( - "pg_class", - column("oid"), - column("relname"), - column("relnamespace"), - column("relkind"), - schema="pg_catalog", -) -_pg_namespace = table( - "pg_namespace", column("oid"), column("nspname"), schema="pg_catalog" -) -_pg_attribute = table( - "pg_attribute", - column("attrelid"), - column("attname"), - column("attnum"), - column("attisdropped"), - schema="pg_catalog", -) _CREATE: Final = frozenset({"SELECT", "INSERT"}) _READ: Final = frozenset({"SELECT"}) @@ -114,17 +91,15 @@ } ) # The v2 Modal coordinators, workers, household functions and stuck-run sweep -# read the runs they were given and insert their results, once each. +# read the runs they were given and insert their results, once each. A session's +# insert of a result also reads the row's server defaults back. MODAL_WORKER_TABLE_PRIVILEGES: Final[Mapping[str, frozenset[str]]] = MappingProxyType( { "report_runs": _READ, "simulation_runs": _READ, "household_jobs": _READ, "budget_window_runs": _READ, - **{ - table.name: frozenset({"INSERT"}) - for table in result_metadata.tables.values() - }, + **dict.fromkeys(RESULT_TABLE_NAMES, _CREATE), } ) # The columns the Modal worker role may UPDATE, by table: each run's state and @@ -152,32 +127,47 @@ def _public_tables() -> Subquery: """Select the oid and name of each table in the ``public`` schema.""" return ( - select(_pg_class.c.oid, _pg_class.c.relname) - .join(_pg_namespace, _pg_namespace.c.oid == _pg_class.c.relnamespace) + select(PgClass.oid, PgClass.relname) + .join(PgNamespace, PgNamespace.oid == PgClass.relnamespace) .where( - and_( - _pg_namespace.c.nspname == TABLE_SCHEMA, - _pg_class.c.relkind.in_(["r", "p"]), - ) + PgNamespace.nspname == TABLE_SCHEMA, + PgClass.relkind.in_(["r", "p"]), ) .subquery("public_tables") ) +def _required_rows( + pairs: Sequence[tuple[str, str]], + *, + name: str, + first_column: str, + second_column: str, +) -> TableValuedAlias: + """Return ``pairs`` as a table of two text columns, one row for each pair.""" + return ( + func.unnest( + literal([first for first, _ in pairs], ARRAY(String)), + literal([second for _, second in pairs], ARRAY(String)), + ) + .table_valued(first_column, second_column) + .render_derived(name=name) + ) + + def _missing_table_grants( - connection: Connection, table_privileges: Mapping[str, frozenset[str]] + session: Session, table_privileges: Mapping[str, frozenset[str]] ) -> list[tuple[str, str, str]]: """Return ``(table, privilege, "")`` for each table privilege the role lacks.""" - required = values( - column("table_name", String), - column("privilege", String), - name="required_privileges", - ).data( + required = _required_rows( [ (table_name, privilege) for table_name, privileges in table_privileges.items() for privilege in sorted(privileges) - ] + ], + name="required_privileges", + first_column="table_name", + second_column="privilege", ) public_tables = _public_tables() # A missing table joins to a null oid, and has_table_privilege returns @@ -186,57 +176,49 @@ def _missing_table_grants( func.has_table_privilege(public_tables.c.oid, required.c.privilege), false(), ) - rows = connection.execute( + rows = session.execute( select(required.c.table_name, required.c.privilege) - .select_from( - required.outerjoin( - public_tables, public_tables.c.relname == required.c.table_name - ) - ) + .select_from(required) + .outerjoin(public_tables, public_tables.c.relname == required.c.table_name) .where(~granted) ) return [(table_name, privilege, "") for table_name, privilege in rows] def _missing_column_grants( - connection: Connection, updatable_columns: Mapping[str, frozenset[str]] + session: Session, updatable_columns: Mapping[str, frozenset[str]] ) -> list[tuple[str, str, str]]: """Return ``(table, "UPDATE", column)`` for each column the role can't update.""" - required = values( - column("table_name", String), - column("column_name", String), - name="required_column_updates", - ).data( + required = _required_rows( [ (table_name, column_name) for table_name, column_names in updatable_columns.items() for column_name in sorted(column_names) - ] + ], + name="required_column_updates", + first_column="table_name", + second_column="column_name", ) public_tables = _public_tables() # A missing table or column joins to a null column number, which # has_column_privilege answers with null, so it counts as not granted. # Asked by number, not name: by name, a missing column raises an error. granted = func.coalesce( - func.has_column_privilege( - public_tables.c.oid, _pg_attribute.c.attnum, "UPDATE" - ), + func.has_column_privilege(public_tables.c.oid, PgAttribute.attnum, "UPDATE"), false(), ) - rows = connection.execute( + rows = session.execute( select(required.c.table_name, required.c.column_name) - .select_from( - required.outerjoin( - public_tables, public_tables.c.relname == required.c.table_name - ).outerjoin( - _pg_attribute, - and_( - _pg_attribute.c.attrelid == public_tables.c.oid, - _pg_attribute.c.attname == required.c.column_name, - _pg_attribute.c.attnum > 0, - _pg_attribute.c.attisdropped.is_(false()), - ), - ) + .select_from(required) + .outerjoin(public_tables, public_tables.c.relname == required.c.table_name) + .outerjoin( + PgAttribute, + and_( + PgAttribute.attrelid == public_tables.c.oid, + PgAttribute.attname == required.c.column_name, + PgAttribute.attnum > 0, + PgAttribute.attisdropped.is_(false()), + ), ) .where(~granted) ) @@ -244,12 +226,12 @@ def _missing_column_grants( def missing_table_privileges( - connection: Connection, + session: Session, *, table_privileges: Mapping[str, frozenset[str]] = SERVICE_TABLE_PRIVILEGES, updatable_columns: Mapping[str, frozenset[str]] = SERVICE_UPDATABLE_COLUMNS, ) -> list[str]: - """Return each required privilege the connected role lacks, ordered by table. + """Return each required privilege the session's role lacks, ordered by table. The required privileges are the simulation service's unless another role's lists are given, such as ``MODAL_WORKER_TABLE_PRIVILEGES`` and @@ -262,8 +244,8 @@ def missing_table_privileges( """ missing_grants = sorted( [ - *_missing_table_grants(connection, table_privileges), - *_missing_column_grants(connection, updatable_columns), + *_missing_table_grants(session, table_privileges), + *_missing_column_grants(session, updatable_columns), ] ) missing = [ @@ -272,7 +254,7 @@ def missing_table_privileges( else f"{privilege} on {table_name}" for table_name, privilege, column_name in missing_grants ] - schema_usage = connection.execute( + schema_usage = session.execute( select(func.has_schema_privilege(TABLE_SCHEMA, "USAGE")) ).scalar_one() if not schema_usage: @@ -281,7 +263,7 @@ def missing_table_privileges( def read_missing_table_privileges( - engine: Engine, *, statement_timeout_milliseconds: int + session_factory: sessionmaker[Session], *, statement_timeout_milliseconds: int ) -> list[str]: """Run ``missing_table_privileges`` in a transaction the database bounds. @@ -290,14 +272,8 @@ def read_missing_table_privileges( pooled connection; a statement that runs longer raises the database's cancellation as an OperationalError. """ - with engine.connect() as connection, connection.begin(): - connection.execute( - select( - func.set_config( - "statement_timeout", - str(statement_timeout_milliseconds), - True, # noqa: FBT003 # set_config's is_local argument - ) - ) - ) - return missing_table_privileges(connection) + with bounded_session( + session_factory, + statement_timeout_milliseconds=statement_timeout_milliseconds, + ) as session: + return missing_table_privileges(session) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/catalog_tables.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/catalog_tables.py new file mode 100644 index 000000000..8c9f3c8c3 --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/catalog_tables.py @@ -0,0 +1,46 @@ +"""Mapped classes of the PostgreSQL catalog tables the privilege check reads. + +Any role may read the catalog, so a table is found by name without the USAGE +privilege on its schema that ``to_regclass`` would need. PostgreSQL owns these +tables; each class names only the columns the check uses and the table's +primary key, and is only ever read. +""" + +from __future__ import annotations + +from sqlalchemy import Boolean, SmallInteger, String +from sqlalchemy.dialects.postgresql import OID +from sqlalchemy.orm import Mapped, mapped_column + +from policyengine_simulation_persistence.orm_base import CatalogBase + + +class PgNamespace(CatalogBase): + """A schema.""" + + __tablename__ = "pg_namespace" + + oid: Mapped[int] = mapped_column(OID, primary_key=True) + nspname: Mapped[str] = mapped_column(String) + + +class PgClass(CatalogBase): + """A table or a table-like object, such as a view or an index.""" + + __tablename__ = "pg_class" + + oid: Mapped[int] = mapped_column(OID, primary_key=True) + relname: Mapped[str] = mapped_column(String) + relnamespace: Mapped[int] = mapped_column(OID) + relkind: Mapped[str] = mapped_column(String) + + +class PgAttribute(CatalogBase): + """A column of a table, numbered from 1; system columns are below zero.""" + + __tablename__ = "pg_attribute" + + attrelid: Mapped[int] = mapped_column(OID, primary_key=True) + attnum: Mapped[int] = mapped_column(SmallInteger, primary_key=True) + attname: Mapped[str] = mapped_column(String) + attisdropped: Mapped[bool] = mapped_column(Boolean) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_inputs.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_inputs.py index 431abef80..f2b505ad0 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_inputs.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_inputs.py @@ -44,7 +44,7 @@ if TYPE_CHECKING: from uuid import UUID - from sqlalchemy import Connection + from sqlalchemy.orm import Session from policyengine_simulation_persistence.policy_reform import PolicyReform @@ -175,7 +175,7 @@ class BudgetWindowInputs: def read_catalog_rows( - connection: Connection, + session: Session, request: EconomyRequest, *, local_area_dataset_name: str | None = None, @@ -197,16 +197,16 @@ def read_catalog_rows( raise ValueError(msg) country = request.country catalog_version_id = read_catalog_version_id( - connection, + session, country=country, policyengine_version=request.policyengine_version, ) - model_id = read_model_id(connection, country=country) + model_id = read_model_id(session, country=country) if catalog_version_id is None or model_id is None: msg = f"{country} has no catalog version for .py {request.policyengine_version}" raise UnknownCatalogVersionError(msg) region = read_region( - connection, + session, catalog_version_id=catalog_version_id, region_code=request.region_code, ) @@ -218,19 +218,19 @@ def read_catalog_rows( raise UnknownRegionError(msg) if local_area_dataset_name is not None: dataset = read_local_area_dataset( - connection, catalog_version_id=catalog_version_id + session, catalog_version_id=catalog_version_id ) if dataset is None or dataset.name != local_area_dataset_name: raise LocalAreaDatasetMismatchError(local_area_dataset_name) elif request.dataset_name is None: dataset = read_dataset_by_id( - connection, + session, catalog_version_id=catalog_version_id, dataset_id=region.default_dataset_id, ) else: dataset = read_dataset( - connection, catalog_version_id=catalog_version_id, name=request.dataset_name + session, catalog_version_id=catalog_version_id, name=request.dataset_name ) if dataset is None: msg = ( @@ -247,7 +247,7 @@ def read_catalog_rows( def read_simulation_values( - connection: Connection, + session: Session, country: str, catalog_version_id: UUID, choice: SimulationChoice, @@ -257,7 +257,7 @@ def read_simulation_values( {} if choice.policy_id is None else read_policy_reform( - connection, + session, country=country, policy_id=choice.policy_id, catalog_version_id=catalog_version_id, @@ -267,7 +267,7 @@ def read_simulation_values( {} if choice.dynamic_id is None else read_dynamic_reform( - connection, + session, dynamic_id=choice.dynamic_id, catalog_version_id=catalog_version_id, ) @@ -278,7 +278,7 @@ def read_simulation_values( def read_side_values( - connection: Connection, + session: Session, request: EconomyRequest, catalog: EconomyCatalogRows, side: ReportSide, @@ -287,14 +287,14 @@ def read_side_values( """Read one report side's values; raise ReportSideError naming a refused side.""" try: return read_simulation_values( - connection, request.country, catalog.catalog_version_id, choice + session, request.country, catalog.catalog_version_id, choice ) except SIDE_ERRORS as error: raise ReportSideError(side, error) from error def read_report_sides( - connection: Connection, + session: Session, request: EconomyRequest, catalog: EconomyCatalogRows, *, @@ -304,13 +304,13 @@ def read_report_sides( """Read both sides' values, the baseline's first.""" return EconomyReportInputs( catalog=catalog, - baseline=read_side_values(connection, request, catalog, "baseline", baseline), - reform=read_side_values(connection, request, catalog, "reform", reform), + baseline=read_side_values(session, request, catalog, "baseline", baseline), + reform=read_side_values(session, request, catalog, "reform", reform), ) def read_window_sides( - connection: Connection, + session: Session, request: EconomyRequest, catalog: EconomyCatalogRows, *, @@ -320,6 +320,6 @@ def read_window_sides( """Read both sides' values, the baseline's first, for every year of a window.""" return BudgetWindowInputs( catalog=catalog, - baseline=read_side_values(connection, request, catalog, "baseline", baseline), - reform=read_side_values(connection, request, catalog, "reform", reform), + baseline=read_side_values(session, request, catalog, "baseline", baseline), + reform=read_side_values(session, request, catalog, "reform", reform), ) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_rerun_records.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_rerun_records.py index b1bdee38f..7bb49df2e 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_rerun_records.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_rerun_records.py @@ -34,7 +34,7 @@ A forced re-run (C1 · 24's answer 3) is an extra run instead: its report run, or each year's report run of a budget window, and the simulation runs -it calculates are marked ``forced``, which the Public API's one-unfailed-run +it calculates are marked ``forced``, which API v1's one-unfailed-run indexes leave out, so it is created beside any run on the same versions, even one under way, and nothing ever joins it. @@ -91,17 +91,17 @@ find_or_insert_window_run, ) from policyengine_simulation_persistence.tables import ( + BudgetWindowRun, + ReportRun, ReportRunStatus, - budget_window_runs, - report_runs, - simulation_runs, + SimulationRun, ) if TYPE_CHECKING: from collections.abc import Iterator, Mapping from uuid import UUID - from sqlalchemy import Column, Connection + from sqlalchemy.orm import InstrumentedAttribute, Session from policyengine_simulation_persistence.economy_run_records import ( EconomyPopulation, @@ -261,18 +261,18 @@ class RerunRecordDeletedError(LookupError): """ -def lock_reruns_of(connection: Connection, record_id: UUID) -> None: +def lock_reruns_of(session: Session, record_id: UUID) -> None: """Wait for, then hold until the transaction ends, the record's re-run lock. It is a transaction-level advisory lock keyed on the record's ID, which any role may take: a second re-run of the same report waits here until the first one's transaction ends, then sees its run, which an economy or household report's re-run, or a budget window's, joins. A - simulation's re-run takes none, as the Public API's index on runs + simulation's re-run takes none, as API v1's index on runs lets one joinable run exist, and neither does a forced re-run, which nothing joins. """ - connection.execute( + session.execute( select( func.pg_advisory_xact_lock( func.hashtextextended(literal(f"rerun:{record_id}"), 0) @@ -281,9 +281,7 @@ def lock_reruns_of(connection: Connection, record_id: UUID) -> None: ) -def insert_report_rerun( - connection: Connection, rerun: NewReportRerun -) -> FoundReportRun: +def insert_report_rerun(session: Session, rerun: NewReportRerun) -> FoundReportRun: """Join the report's run on the re-run's versions, or create it, pending. A single-year report's re-run (the service re-runs a budget window with @@ -306,9 +304,9 @@ def insert_report_rerun( """ versions = rerun.versions if not rerun.forced: - lock_reruns_of(connection, rerun.report_id) + lock_reruns_of(session, rerun.report_id) joined = find_joinable_report_run( - connection, + session, report_id=rerun.report_id, country_package_version=versions.country_package_version, policyengine_version=versions.policyengine_version, @@ -319,20 +317,20 @@ def insert_report_rerun( run_id=rerun.run_id, report_id=rerun.report_id, versions=versions ) sides: dict[Side, RerunSide] = {"baseline": rerun.baseline, "reform": rerun.reform} - with _record_checked(report_runs.c.report_id), references_checked(): + with _record_checked(ReportRun.report_id), references_checked(): if rerun.forced: for side in sides.values(): insert_simulation_run( - connection, + session, run_id=side.simulation_run_id, simulation_id=side.simulation_id, population=rerun.population, versions=versions, forced=True, ) - return _insert_forced_report_run(connection, row, sides) + return _insert_forced_report_run(session, row, sides) found, _ = find_or_insert_report_run_on_sides( - connection, + session, row, { name: simulation_run_row( @@ -367,7 +365,7 @@ def _rerun_report_run_row( def _insert_forced_report_run( - connection: Connection, + session: Session, row: dict[str, Any], side_runs: Mapping[Side, RerunSide | HouseholdRerunSide], ) -> FoundReportRun: @@ -375,8 +373,8 @@ def _insert_forced_report_run( reform = side_runs.get("reform") baseline_run_id = side_runs["baseline"].simulation_run_id reform_run_id = None if reform is None else reform.simulation_run_id - connection.execute( - insert(report_runs).values( + session.execute( + insert(ReportRun).values( { **row, "baseline_simulation_run_id": baseline_run_id, @@ -394,9 +392,7 @@ def _insert_forced_report_run( ) -def insert_window_rerun( - connection: Connection, rerun: NewWindowRerun -) -> FoundBudgetWindow: +def insert_window_rerun(session: Session, rerun: NewWindowRerun) -> FoundBudgetWindow: """Join the window's run on the re-run's versions, or create it, pending. Unforced (a keyed window's re-run), take the report's re-run lock and @@ -437,20 +433,20 @@ def insert_window_rerun( recalculate=rerun.recalculate, ) with ( - _record_checked(budget_window_runs.c.report_id, report_runs.c.report_id), + _record_checked(BudgetWindowRun.report_id, ReportRun.report_id), references_checked(), ): if rerun.recalculate: # First, so a report deleted after its reads is told apart from # a year's simulation deleted with it. - connection.execute(insert(budget_window_runs).values(row)) + session.execute(insert(BudgetWindowRun).values(row)) window_run = FoundWindowRun( id=rerun.window_run_id, inserted=True, status=ReportRunStatus.PENDING ) else: - lock_reruns_of(connection, rerun.report_id) + lock_reruns_of(session, rerun.report_id) joined = find_joinable_window_run( - connection, + session, report_id=rerun.report_id, tax_benefit_model_version_id=( rerun.population.tax_benefit_model_version_id @@ -460,7 +456,7 @@ def insert_window_rerun( window_run = ( joined if joined is not None - else find_or_insert_window_run(connection, row) + else find_or_insert_window_run(session, row) ) if not window_run.inserted: return FoundBudgetWindow( @@ -468,7 +464,7 @@ def insert_window_rerun( report_inserted=False, window_run=window_run, years=stored_window_years( - connection, + session, window_run.id, { (year.year, side): year.simulation_ids[side] @@ -478,7 +474,7 @@ def insert_window_rerun( ), ) found_years = insert_window_years( - connection, + session, years, report_id=rerun.report_id, window_run_id=rerun.window_run_id, @@ -495,7 +491,7 @@ def insert_window_rerun( def insert_household_report_rerun( - connection: Connection, rerun: NewHouseholdReportRerun + session: Session, rerun: NewHouseholdReportRerun ) -> FoundReportRun: """Join the household report's run on the re-run's versions, or create it. @@ -521,20 +517,20 @@ def insert_household_report_rerun( ) sides = rerun.named_sides if not rerun.forced: - lock_reruns_of(connection, rerun.report_id) + lock_reruns_of(session, rerun.report_id) joined = find_joinable_report_run( - connection, + session, report_id=rerun.report_id, country_package_version=versions.country_package_version, policyengine_version=versions.policyengine_version, ) if joined is not None: return joined - with _record_checked(report_runs.c.report_id), references_checked(): + with _record_checked(ReportRun.report_id), references_checked(): if rerun.forced: for side in sides.values(): insert_household_simulation_run( - connection, + session, run_id=side.simulation_run_id, simulation_id=side.simulation_id, year=rerun.year, @@ -542,9 +538,9 @@ def insert_household_report_rerun( versions=versions, forced=True, ) - return _insert_forced_report_run(connection, row, sides) + return _insert_forced_report_run(session, row, sides) found, _ = find_or_insert_report_run_on_sides( - connection, + session, row, { name: household_simulation_run_row( @@ -561,7 +557,7 @@ def insert_household_report_rerun( def insert_simulation_rerun( - connection: Connection, rerun: NewSimulationRerun + session: Session, rerun: NewSimulationRerun ) -> FoundSimulationRun: """Join the simulation's run on the re-run's versions, or create it, pending. @@ -572,9 +568,9 @@ def insert_simulation_rerun( Raise RerunRecordDeletedError or EconomyRecordReferenceError as ``insert_report_rerun`` does. """ - with _record_checked(simulation_runs.c.simulation_id), references_checked(): + with _record_checked(SimulationRun.simulation_id), references_checked(): return find_or_insert_simulation_run( - connection, + session, simulation_run_row( run_id=rerun.run_id, simulation_id=rerun.simulation_id, @@ -585,17 +581,17 @@ def insert_simulation_rerun( @contextmanager -def _record_checked(*own_keys: Column[Any]) -> Iterator[None]: +def _record_checked(*own_keys: InstrumentedAttribute[Any]) -> Iterator[None]: """Turn a refusal of a new run's key to the record into RerunRecordDeletedError. - ``own_keys`` are those keys' columns; a refusal of any other row - propagates unchanged. + ``own_keys`` are those keys' mapped attributes; a refusal of any other + row propagates unchanged. """ try: yield except EconomyRecordReferenceError as error: refused = (error.table_name, error.column_name) - if refused in {(key.table.name, key.key) for key in own_keys}: + if refused in {(key.class_.__tablename__, key.key) for key in own_keys}: raise RerunRecordDeletedError from None raise diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_run_reads.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_run_reads.py index 1cde162ad..eda5ecf00 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_run_reads.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_run_reads.py @@ -1,8 +1,10 @@ """Read a report or a simulation as it stands, with its newest run. The newest run is the one created last, with ties broken by the larger ID, -whatever its status and versions, as the Public API reads it. The run's -results are the Public API's to read: these reads give only its state. +whatever its status and versions, as API v1 reads it. The run's +results are API v1's to read: these reads give only its state. Each read +selects the mapped classes' attributes, so it returns what the database holds +now, whatever the session has loaded. """ from __future__ import annotations @@ -13,19 +15,20 @@ from sqlalchemy import select from policyengine_simulation_persistence.tables import ( + Report, + ReportRun, ReportRunStatus, + Simulation, + SimulationRun, SimulationStatus, - report_runs, - reports, - simulation_runs, - simulations, ) if TYPE_CHECKING: from datetime import datetime from uuid import UUID - from sqlalchemy import Connection, RowMapping + from sqlalchemy import RowMapping + from sqlalchemy.orm import Session @dataclass(frozen=True) @@ -77,36 +80,34 @@ class EconomySimulationState: run: SimulationRunState | None -def read_economy_report( - connection: Connection, report_id: UUID -) -> EconomyReportState | None: +def read_economy_report(session: Session, report_id: UUID) -> EconomyReportState | None: """Return the report with its newest run, or None when there is no such report.""" - report = connection.execute( + report = session.execute( select( - reports.c.id, - reports.c.country, - reports.c.baseline_simulation_id, - reports.c.reform_simulation_id, - ).where(reports.c.id == report_id) + Report.id, + Report.country, + Report.baseline_simulation_id, + Report.reform_simulation_id, + ).where(Report.id == report_id) ).one_or_none() if report is None: return None run = ( - connection.execute( + session.execute( select( - report_runs.c.id, - report_runs.c.status, - report_runs.c.country_package_version, - report_runs.c.policyengine_version, - report_runs.c.error_message, - report_runs.c.created_at, - report_runs.c.started_at, - report_runs.c.completed_at, - report_runs.c.baseline_simulation_run_id, - report_runs.c.reform_simulation_run_id, + ReportRun.id, + ReportRun.status, + ReportRun.country_package_version, + ReportRun.policyengine_version, + ReportRun.error_message, + ReportRun.created_at, + ReportRun.started_at, + ReportRun.completed_at, + ReportRun.baseline_simulation_run_id, + ReportRun.reform_simulation_run_id, ) - .where(report_runs.c.report_id == report_id) - .order_by(report_runs.c.created_at.desc(), report_runs.c.id.desc()) + .where(ReportRun.report_id == report_id) + .order_by(ReportRun.created_at.desc(), ReportRun.id.desc()) .limit(1) ) .mappings() @@ -122,28 +123,28 @@ def read_economy_report( def read_economy_simulation( - connection: Connection, simulation_id: UUID + session: Session, simulation_id: UUID ) -> EconomySimulationState | None: """Return the simulation with its newest run, or None when there is none.""" - found = connection.execute( - select(simulations.c.id).where(simulations.c.id == simulation_id) + found = session.execute( + select(Simulation.id).where(Simulation.id == simulation_id) ).scalar_one_or_none() if found is None: return None run = ( - connection.execute( + session.execute( select( - simulation_runs.c.id, - simulation_runs.c.status, - simulation_runs.c.country_package_version, - simulation_runs.c.policyengine_version, - simulation_runs.c.error_message, - simulation_runs.c.created_at, - simulation_runs.c.started_at, - simulation_runs.c.completed_at, + SimulationRun.id, + SimulationRun.status, + SimulationRun.country_package_version, + SimulationRun.policyengine_version, + SimulationRun.error_message, + SimulationRun.created_at, + SimulationRun.started_at, + SimulationRun.completed_at, ) - .where(simulation_runs.c.simulation_id == simulation_id) - .order_by(simulation_runs.c.created_at.desc(), simulation_runs.c.id.desc()) + .where(SimulationRun.simulation_id == simulation_id) + .order_by(SimulationRun.created_at.desc(), SimulationRun.id.desc()) .limit(1) ) .mappings() diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_run_records.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_run_records.py index 501c1d8c2..719b9450a 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_run_records.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_run_records.py @@ -2,7 +2,7 @@ An economy report compares a baseline and a reform simulation of one region's data for one year. The service reads the catalog rows a request -names from the Public API's tables (``read_model_id``, ``read_region``, +names from API v1's tables (``read_model_id``, ``read_region``, ``read_dataset``), then finds or creates, by definition key, each simulation, the report and their runs (``keyed_economy_records``). A run it creates is ``pending`` with no Modal call ID; this module builds and @@ -17,11 +17,14 @@ for it, under the same condition. The coordinator and workers write the rest: ``running``, the results, and ``succeeded`` or ``failed``. -Every function takes a connection, so the caller chooses the transactions. -Each update sets only the columns the service's role may update, with -``updated_at``, whose automatic update the Public API's models make on their -own side only. ``references_checked`` turns the database's refusal of a row -naming one that does not exist into ``EconomyRecordReferenceError``. +Every function takes an ORM session, so the caller chooses the transactions. +Reads select the mapped classes' attributes, so they return what the database +holds now, whatever the session has loaded. Each update is one conditional +statement that leaves the session's objects alone, and sets only the columns +the service's role may update, with ``updated_at``, whose automatic update +API v1's models make on their own side only. ``references_checked`` turns the +database's refusal of a row naming one that does not exist into +``EconomyRecordReferenceError``. """ from __future__ import annotations @@ -31,6 +34,7 @@ from typing import TYPE_CHECKING, Any, Final from sqlalchemy import ( + ForeignKeyConstraint, String, cast, func, @@ -43,29 +47,33 @@ from policyengine_simulation_persistence.input_tables import ( MODEL_NAME_PREFIX, UK_AREA_REGION_TYPES, - datasets, - regions, - tax_benefit_models, + Dataset, + Region, + TaxBenefitModel, ) from policyengine_simulation_persistence.tables import ( + BudgetWindowRun, + Report, + ReportRun, ReportRunStatus, + Simulation, + SimulationRun, SimulationStatus, - budget_window_runs, - report_runs, - reports, - simulation_runs, - simulations, ) if TYPE_CHECKING: from collections.abc import Iterator, Sequence from uuid import UUID - from sqlalchemy import ColumnElement, Connection, Table + from sqlalchemy import ColumnElement + from sqlalchemy.orm import Session + +# The two classes whose runs a Modal call calculates. +type RunModel = type[SimulationRun | ReportRun] # PostgreSQL's SQLSTATE for a row naming one that does not exist. FOREIGN_KEY_VIOLATION: Final = "23503" -# A report's type, as the Public API reads it. +# A report's type, as API v1 reads it. ECONOMY_REPORT_TYPE: Final = "economy" # Both run tables' statuses use these values. _PENDING: Final = SimulationStatus.PENDING.value @@ -73,15 +81,10 @@ # Each foreign key of the tables the service writes, by name: its table and # the first column it constrains. _KEY_COLUMNS: Final = { - str(constraint.name): (table.name, constraint.column_keys[0]) - for table in ( - simulations, - simulation_runs, - reports, - report_runs, - budget_window_runs, - ) - for constraint in table.foreign_key_constraints + str(constraint.name): (model.__tablename__, constraint.column_keys[0]) + for model in (Simulation, SimulationRun, Report, ReportRun, BudgetWindowRun) + for constraint in model.__table_args__ + if isinstance(constraint, ForeignKeyConstraint) } @@ -150,23 +153,23 @@ def __init__(self, table_name: str | None, column_name: str | None) -> None: super().__init__(f"{named} names no stored row") -def read_model_id(connection: Connection, *, country: str) -> UUID | None: +def read_model_id(session: Session, *, country: str) -> UUID | None: """Return the id of the country's tax-benefit model, or None.""" - return connection.execute( - select(tax_benefit_models.c.id).where( - tax_benefit_models.c.name == f"{MODEL_NAME_PREFIX}{country}" + return session.execute( + select(TaxBenefitModel.id).where( + TaxBenefitModel.name == f"{MODEL_NAME_PREFIX}{country}" ) ).scalar_one_or_none() def read_region( - connection: Connection, *, catalog_version_id: UUID, region_code: str + session: Session, *, catalog_version_id: UUID, region_code: str ) -> CatalogRegion | None: """Return the catalog version's region with this code, or None.""" - row = connection.execute( - select(regions.c.id, regions.c.code, regions.c.default_dataset_id).where( - regions.c.tax_benefit_model_version_id == catalog_version_id, - regions.c.code == region_code, + row = session.execute( + select(Region.id, Region.code, Region.default_dataset_id).where( + Region.tax_benefit_model_version_id == catalog_version_id, + Region.code == region_code, ) ).one_or_none() if row is None: @@ -177,26 +180,26 @@ def read_region( def read_dataset( - connection: Connection, *, catalog_version_id: UUID, name: str + session: Session, *, catalog_version_id: UUID, name: str ) -> CatalogDataset | None: """Return the catalog version's dataset with this name, or None.""" - return _dataset(connection, catalog_version_id, datasets.c.name == name) + return _dataset(session, catalog_version_id, Dataset.name == name) def read_dataset_by_id( - connection: Connection, *, catalog_version_id: UUID, dataset_id: UUID + session: Session, *, catalog_version_id: UUID, dataset_id: UUID ) -> CatalogDataset | None: """Return the catalog version's dataset with this id, or None.""" - return _dataset(connection, catalog_version_id, datasets.c.id == dataset_id) + return _dataset(session, catalog_version_id, Dataset.id == dataset_id) def read_local_area_dataset( - connection: Connection, *, catalog_version_id: UUID + session: Session, *, catalog_version_id: UUID ) -> CatalogDataset | None: """Return the catalog version's local-area dataset, or None when it has none. It is the one default dataset every constituency and local-authority - region of the version uses. The Public API's catalog lists those regions + region of the version uses. API v1's catalog lists those regions only when the .py release routes them to a dataset of their own, and gives them that dataset, so a version with no such regions, with only one of the two types, or with a dataset for each type has none: its @@ -206,20 +209,21 @@ def read_local_area_dataset( the release routes both types to one dataset (``uk_area_breakdowns_omitted``). """ - region_type = cast(regions.c.region_type, String) - rows = connection.execute( - select(region_type.label("region_type"), datasets.c.id, datasets.c.name) + region_type = cast(Region.region_type, String) + rows = session.execute( + select(region_type.label("region_type"), Dataset.id, Dataset.name) + .select_from(Region) .distinct() .join( - datasets, - (datasets.c.id == regions.c.default_dataset_id) + Dataset, + (Dataset.id == Region.default_dataset_id) & ( - datasets.c.tax_benefit_model_version_id - == regions.c.tax_benefit_model_version_id + Dataset.tax_benefit_model_version_id + == Region.tax_benefit_model_version_id ), ) .where( - regions.c.tax_benefit_model_version_id == catalog_version_id, + Region.tax_benefit_model_version_id == catalog_version_id, region_type.in_(UK_AREA_REGION_TYPES), ) ).all() @@ -231,18 +235,18 @@ def read_local_area_dataset( def _dataset( - connection: Connection, catalog_version_id: UUID, condition: ColumnElement[bool] + session: Session, catalog_version_id: UUID, condition: ColumnElement[bool] ) -> CatalogDataset | None: - row = connection.execute( - select(datasets.c.id, datasets.c.name).where( - datasets.c.tax_benefit_model_version_id == catalog_version_id, condition + row = session.execute( + select(Dataset.id, Dataset.name).where( + Dataset.tax_benefit_model_version_id == catalog_version_id, condition ) ).one_or_none() return None if row is None else CatalogDataset(id=row.id, name=row.name) def insert_simulation_run( # noqa: PLR0913 # the run, what it calculates on, and its marker - connection: Connection, + session: Session, *, run_id: UUID, simulation_id: UUID, @@ -254,10 +258,11 @@ def insert_simulation_run( # noqa: PLR0913 # the run, what it calculates on, a The run records the catalog version, dataset and region it calculates on, which a re-run's may differ from its simulation's. ``forced`` marks - a forced re-run's run, which nothing joins. + a forced re-run's run, which nothing joins. The statement runs at once, + so a report run naming the new run finds it. """ - connection.execute( - insert(simulation_runs).values( + session.execute( + insert(SimulationRun).values( simulation_run_row( run_id=run_id, simulation_id=simulation_id, @@ -313,58 +318,59 @@ def references_checked() -> Iterator[None]: def _pending_without_call( - table: Table, run_id: UUID + model: RunModel, run_id: UUID ) -> tuple[ColumnElement[bool], ...]: return ( - table.c.id == run_id, - table.c.status == _PENDING, - table.c.modal_call_id.is_(None), + model.id == run_id, + model.status == _PENDING, + model.modal_call_id.is_(None), ) def record_report_run_call( - connection: Connection, *, report_run_id: UUID, modal_call_id: str + session: Session, *, report_run_id: UUID, modal_call_id: str ) -> ReportRunStatus | None: """Record the report run's Modal call ID while it is pending with none. Return the run's status afterwards, or None when there is no such run. A run the coordinator has claimed already keeps its claim. """ - status = _record_call(connection, report_runs, report_run_id, modal_call_id) + status = _record_call(session, ReportRun, report_run_id, modal_call_id) return None if status is None else ReportRunStatus(status) def record_simulation_run_call( - connection: Connection, *, simulation_run_id: UUID, modal_call_id: str + session: Session, *, simulation_run_id: UUID, modal_call_id: str ) -> SimulationStatus | None: """Record the simulation run's Modal call ID while it is pending with none. Return the run's status afterwards, or None when there is no such run. A run the worker has claimed already keeps its claim. """ - status = _record_call(connection, simulation_runs, simulation_run_id, modal_call_id) + status = _record_call(session, SimulationRun, simulation_run_id, modal_call_id) return None if status is None else SimulationStatus(status) def _record_call( - connection: Connection, table: Table, run_id: UUID, modal_call_id: str + session: Session, model: RunModel, run_id: UUID, modal_call_id: str ) -> str | None: - recorded = connection.execute( - update(table) - .where(*_pending_without_call(table, run_id)) + recorded = session.execute( + update(model) + .where(*_pending_without_call(model, run_id)) .values(modal_call_id=modal_call_id, updated_at=func.now()) - .returning(table.c.status) + .returning(model.status) + .execution_options(synchronize_session=False) ).scalar_one_or_none() if recorded is not None: return str(recorded) - status = connection.execute( - select(table.c.status).where(table.c.id == run_id) + status = session.execute( + select(model.status).where(model.id == run_id) ).scalar_one_or_none() return None if status is None else str(status) def fail_unstarted_report_run( - connection: Connection, + session: Session, *, report_run_id: UUID, simulation_run_ids: Sequence[UUID], @@ -378,38 +384,37 @@ def fail_unstarted_report_run( for, or one the coordinator has claimed, is left as it is, and so are its simulation runs. """ - if not _fail_unstarted(connection, report_runs, report_run_id, error_message): + if not _fail_unstarted(session, ReportRun, report_run_id, error_message): return False for simulation_run_id in simulation_run_ids: - _fail_unstarted(connection, simulation_runs, simulation_run_id, error_message) + _fail_unstarted(session, SimulationRun, simulation_run_id, error_message) return True def fail_unstarted_simulation_run( - connection: Connection, *, simulation_run_id: UUID, error_message: str + session: Session, *, simulation_run_id: UUID, error_message: str ) -> bool: """Fail the simulation run at once while it is pending with no Modal call ID. Return whether it was failed: a run a call has been recorded for, or one a worker has claimed, is left as it is. """ - return _fail_unstarted( - connection, simulation_runs, simulation_run_id, error_message - ) + return _fail_unstarted(session, SimulationRun, simulation_run_id, error_message) def _fail_unstarted( - connection: Connection, table: Table, run_id: UUID, error_message: str + session: Session, model: RunModel, run_id: UUID, error_message: str ) -> bool: - failed = connection.execute( - update(table) - .where(*_pending_without_call(table, run_id)) + failed = session.execute( + update(model) + .where(*_pending_without_call(model, run_id)) .values( status=_FAILED, error_message=error_message, completed_at=func.now(), updated_at=func.now(), ) - .returning(table.c.id) + .returning(model.id) + .execution_options(synchronize_session=False) ).scalar_one_or_none() return failed is not None diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_run_store.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_run_store.py index 2c75e31cb..dec3009ab 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_run_store.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_run_store.py @@ -7,15 +7,14 @@ A stored record's re-run takes the same steps, joining or creating its new runs instead. ``EconomyRunStore`` runs the reads and writes of ``economy_inputs``, ``keyed_economy_records``, ``keyed_window_records``, -``economy_rerun_records`` and ``economy_run_records`` that way. +``economy_rerun_records`` and ``economy_run_records`` that way, each in an ORM +session of its own. """ from __future__ import annotations from typing import TYPE_CHECKING -from sqlalchemy.exc import SQLAlchemyError - from policyengine_simulation_persistence.economy_inputs import ( BudgetWindowInputs, EconomyCatalogRows, @@ -52,7 +51,6 @@ record_report_run_call, record_simulation_run_call, ) -from policyengine_simulation_persistence.engine import bounded_transaction from policyengine_simulation_persistence.keyed_economy_records import ( find_or_create_economy_report, find_or_create_economy_simulation, @@ -60,13 +58,18 @@ from policyengine_simulation_persistence.keyed_window_records import ( find_or_create_budget_window, ) +from policyengine_simulation_persistence.store_transactions import ( + EconomyRecordOutcomeUnknownError, + TransactionFactory, + bounded_transactions, + create_in_transaction, +) if TYPE_CHECKING: from collections.abc import Callable, Sequence - from contextlib import AbstractContextManager from uuid import UUID - from sqlalchemy import Connection, Engine + from sqlalchemy.orm import Session, sessionmaker from policyengine_simulation_persistence.economy_rerun_records import ( NewReportRerun, @@ -99,36 +102,28 @@ SimulationStatus, ) -type TransactionFactory = Callable[[], AbstractContextManager[Connection]] - - -class EconomyRecordOutcomeUnknownError(RuntimeError): - """A create's statements ran, but its commit failed: the rows may exist. - - PostgreSQL may have applied the commit before the connection failed, so - the caller must not answer that nothing was created. - """ - class EconomyRunStore: """Read and write economy simulations and reports, each in its own transaction.""" def __init__(self, transaction: TransactionFactory) -> None: - """Open each transaction with ``transaction``, which commits on success.""" + """Open each session with ``transaction``, which commits on success.""" self._transaction = transaction @classmethod - def from_engine( - cls, engine: Engine, *, statement_timeout_milliseconds: int + def from_session_factory( + cls, + session_factory: sessionmaker[Session], + *, + statement_timeout_milliseconds: int, ) -> EconomyRunStore: - """Use ``engine``, bounding each statement to the given time.""" - - def transaction() -> AbstractContextManager[Connection]: - return bounded_transaction( - engine, statement_timeout_milliseconds=statement_timeout_milliseconds + """Use ``session_factory``, bounding each statement to the given time.""" + return cls( + bounded_transactions( + session_factory, + statement_timeout_milliseconds=statement_timeout_milliseconds, ) - - return cls(transaction) + ) def read_simulation_inputs( self, request: EconomyRequest, choice: SimulationChoice @@ -140,10 +135,10 @@ def read_simulation_inputs( UnknownDynamicError or MissingPolicyParametersError, or OverlappingReformError. """ - with self._transaction() as connection: - catalog = read_catalog_rows(connection, request) + with self._transaction() as session: + catalog = read_catalog_rows(session, request) simulation = read_simulation_values( - connection, request.country, catalog.catalog_version_id, choice + session, request.country, catalog.catalog_version_id, choice ) return EconomySimulationInputs(catalog=catalog, simulation=simulation) @@ -166,25 +161,25 @@ def read_report_inputs( or dynamic is refused, or LocalAreaDatasetMismatchError when the catalog version doesn't give both area types that dataset. """ - with self._transaction() as connection: + with self._transaction() as session: catalog = read_catalog_rows( - connection, request, local_area_dataset_name=local_area_dataset_name + session, request, local_area_dataset_name=local_area_dataset_name ) return read_report_sides( - connection, request, catalog, baseline=baseline, reform=reform + session, request, catalog, baseline=baseline, reform=reform ) def read_report_definition(self, report_id: UUID) -> ReportDefinition | None: """Return what the stored report calculates, or None when there is none.""" - with self._transaction() as connection: - return read_report_definition(connection, report_id) + with self._transaction() as session: + return read_report_definition(session, report_id) def read_simulation_definition( self, simulation_id: UUID ) -> SimulationDefinition | None: """Return what the stored simulation calculates, or None when there is none.""" - with self._transaction() as connection: - return read_simulation_definition(connection, simulation_id) + with self._transaction() as session: + return read_simulation_definition(session, simulation_id) def read_report_rerun_inputs( self, @@ -199,10 +194,10 @@ def read_report_rerun_inputs( found in the catalog version of the current .py version. Raise the errors ``read_report_inputs`` raises. """ - with self._transaction() as connection: - catalog = read_catalog_rows(connection, request) + with self._transaction() as session: + catalog = read_catalog_rows(session, request) return read_report_sides( - connection, + session, request, catalog, baseline=baseline.choice, @@ -223,10 +218,10 @@ def read_window_rerun_inputs( catalog version of the current .py version. Raise the errors ``read_report_inputs`` raises. """ - with self._transaction() as connection: - catalog = read_catalog_rows(connection, request) + with self._transaction() as session: + catalog = read_catalog_rows(session, request) return read_window_sides( - connection, request, catalog, baseline=baseline, reform=reform + session, request, catalog, baseline=baseline, reform=reform ) def create_window_rerun(self, rerun: NewWindowRerun) -> FoundBudgetWindow: @@ -234,7 +229,7 @@ def create_window_rerun(self, rerun: NewWindowRerun) -> FoundBudgetWindow: Raise EconomyRecordOutcomeUnknownError as ``find_or_create_report`` does. """ - return self._create(lambda connection: insert_window_rerun(connection, rerun)) + return self._create(lambda session: insert_window_rerun(session, rerun)) def create_report_rerun(self, rerun: NewReportRerun) -> FoundReportRun: """Join or create the report's run; see ``insert_report_rerun``. @@ -242,16 +237,14 @@ def create_report_rerun(self, rerun: NewReportRerun) -> FoundReportRun: Raise ``insert_report_rerun``'s errors, and EconomyRecordOutcomeUnknownError as ``find_or_create_report`` does. """ - return self._create(lambda connection: insert_report_rerun(connection, rerun)) + return self._create(lambda session: insert_report_rerun(session, rerun)) def create_simulation_rerun(self, rerun: NewSimulationRerun) -> FoundSimulationRun: """Join or create the simulation's run; see ``insert_simulation_rerun``. Raise EconomyRecordOutcomeUnknownError as ``find_or_create_report`` does. """ - return self._create( - lambda connection: insert_simulation_rerun(connection, rerun) - ) + return self._create(lambda session: insert_simulation_rerun(session, rerun)) def read_budget_window_inputs( self, @@ -265,10 +258,10 @@ def read_budget_window_inputs( ``request.year`` is the window's first year. Raise the errors ``read_report_inputs`` raises. """ - with self._transaction() as connection: - catalog = read_catalog_rows(connection, request) + with self._transaction() as session: + catalog = read_catalog_rows(session, request) return read_window_sides( - connection, request, catalog, baseline=baseline, reform=reform + session, request, catalog, baseline=baseline, reform=reform ) def find_or_create_report(self, report: KeyedEconomyReport) -> FoundEconomyReport: @@ -279,7 +272,7 @@ def find_or_create_report(self, report: KeyedEconomyReport) -> FoundEconomyRepor rows were inserted. """ return self._create( - lambda connection: find_or_create_economy_report(connection, report) + lambda session: find_or_create_economy_report(session, report) ) def find_or_create_budget_window( @@ -291,7 +284,7 @@ def find_or_create_budget_window( ``find_or_create_report`` does. """ return self._create( - lambda connection: find_or_create_budget_window(connection, window) + lambda session: find_or_create_budget_window(session, window) ) def find_or_create_simulation( @@ -302,36 +295,24 @@ def find_or_create_simulation( Raise as ``find_or_create_report`` does. """ return self._create( - lambda connection: find_or_create_economy_simulation(connection, simulation) + lambda session: find_or_create_economy_simulation(session, simulation) ) - def _create[Created](self, insert: Callable[[Connection], Created]) -> Created: - """Run ``insert`` in a transaction, telling a failed commit apart. - - Return what ``insert`` returns. An error from ``insert`` leaves - nothing written and propagates. An error once it has run is the - commit's, after which the rows may exist: - EconomyRecordOutcomeUnknownError. - """ - inserted = False - try: - with self._transaction() as connection: - created = insert(connection) - inserted = True - except SQLAlchemyError as error: - if not inserted: - raise - msg = "the create's commit failed after its rows were inserted" - raise EconomyRecordOutcomeUnknownError(msg) from error - return created + def _create[Created](self, insert: Callable[[Session], Created]) -> Created: + """Run ``insert`` in a session, telling a failed commit apart.""" + return create_in_transaction( + self._transaction, + insert, + commit_failure="the create's commit failed after its rows were inserted", + ) def record_report_call( self, report_run_id: UUID, modal_call_id: str ) -> ReportRunStatus | None: """Record the report run's call ID while it is pending with none.""" - with self._transaction() as connection: + with self._transaction() as session: return record_report_run_call( - connection, report_run_id=report_run_id, modal_call_id=modal_call_id + session, report_run_id=report_run_id, modal_call_id=modal_call_id ) def fail_unstarted_report( @@ -341,9 +322,9 @@ def fail_unstarted_report( error_message: str, ) -> bool: """Fail the report run, and its new simulation runs, while it is unstarted.""" - with self._transaction() as connection: + with self._transaction() as session: return fail_unstarted_report_run( - connection, + session, report_run_id=report_run_id, simulation_run_ids=simulation_run_ids, error_message=error_message, @@ -353,9 +334,9 @@ def record_simulation_call( self, simulation_run_id: UUID, modal_call_id: str ) -> SimulationStatus | None: """Record the simulation run's call ID while it is pending with none.""" - with self._transaction() as connection: + with self._transaction() as session: return record_simulation_run_call( - connection, + session, simulation_run_id=simulation_run_id, modal_call_id=modal_call_id, ) @@ -364,22 +345,22 @@ def fail_unstarted_simulation( self, simulation_run_id: UUID, error_message: str ) -> bool: """Fail the simulation run while it is pending with no call ID.""" - with self._transaction() as connection: + with self._transaction() as session: return fail_unstarted_simulation_run( - connection, + session, simulation_run_id=simulation_run_id, error_message=error_message, ) def read_report(self, report_id: UUID) -> EconomyReportState | None: """Return the report with its newest run, or None when there is none.""" - with self._transaction() as connection: - return read_economy_report(connection, report_id) + with self._transaction() as session: + return read_economy_report(session, report_id) def read_simulation(self, simulation_id: UUID) -> EconomySimulationState | None: """Return the simulation with its newest run, or None when there is none.""" - with self._transaction() as connection: - return read_economy_simulation(connection, simulation_id) + with self._transaction() as session: + return read_economy_simulation(session, simulation_id) __all__ = [ diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/engine.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/engine.py index ef01257be..2b71795ce 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/engine.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/engine.py @@ -1,4 +1,7 @@ -"""Database connections for the simulation service and the v2 Modal calls.""" +"""ORM sessions for the simulation service and the v2 Modal calls. + +Each engine stays in this module; everything else receives a ``sessionmaker``. +""" from __future__ import annotations @@ -8,12 +11,14 @@ from sqlalchemy import create_engine, func, select from sqlalchemy.engine import URL, make_url from sqlalchemy.exc import ArgumentError -from sqlalchemy.pool import NullPool +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import NullPool, QueuePool if TYPE_CHECKING: from collections.abc import Iterator - from sqlalchemy import Connection, Engine + from sqlalchemy import Engine, Select + from sqlalchemy.orm import Session CONNECT_TIMEOUT_SECONDS = 5 # Each Cloud Run instance serves up to 80 concurrent requests. Each database @@ -23,6 +28,10 @@ MAX_OVERFLOW = 3 # A v2 Modal call connects across the internet to the database's pooler. UNPOOLED_CONNECT_TIMEOUT_SECONDS = 10 +SINGLE_CONNECTION_CONNECT_TIMEOUT_SECONDS = 10 +# How long a single-connection engine's user waits for the connection. Its users +# take turns, so waiting at all means one of them kept it. +SINGLE_CONNECTION_WAIT_SECONDS = 5 def normalized_database_url(database_url: str) -> URL: @@ -50,7 +59,7 @@ def normalized_database_url(database_url: str) -> URL: return parsed.set(drivername="postgresql+psycopg") -def create_simulation_engine(database_url: str) -> Engine: +def _create_pooled_engine(database_url: str) -> Engine: """Build the engine the simulation service uses; it creates no objects.""" return create_engine( normalized_database_url(database_url), @@ -66,7 +75,7 @@ def create_simulation_engine(database_url: str) -> Engine: ) -def create_unpooled_engine(database_url: str) -> Engine: +def _create_unpooled_engine(database_url: str) -> Engine: """Build an engine that opens a connection per operation and closes it after. The v2 Modal coordinators, workers and household functions, and the @@ -81,26 +90,100 @@ def create_unpooled_engine(database_url: str) -> Engine: ) -@contextmanager -def bounded_transaction( - engine: Engine, *, statement_timeout_milliseconds: int -) -> Iterator[Connection]: - """Yield a connection in a transaction whose statements the database bounds. - - The transaction commits when the block ends and rolls back when it - raises. ``statement_timeout`` is set with ``set_config`` local to the - transaction, so it works through Supabase's transaction pooler and never - stays on the pooled connection; a statement that runs longer raises the - database's cancellation as an OperationalError. +def _create_single_connection_engine(database_url: str) -> Engine: + """Build an engine whose sessions and schema reads share one connection. + + A short task that opens many sessions or reads the schema piece by piece + then takes one client slot of the database's pooler. Each user must return + the connection before the next takes it. A connection checked out after it + was lost is replaced, and no statement is prepared on the server, so the + engine works through Supabase's session and transaction poolers. + """ + return create_engine( + normalized_database_url(database_url), + poolclass=QueuePool, + pool_size=1, + max_overflow=0, + pool_timeout=SINGLE_CONNECTION_WAIT_SECONDS, + pool_pre_ping=True, + connect_args={ + "connect_timeout": SINGLE_CONNECTION_CONNECT_TIMEOUT_SECONDS, + "prepare_threshold": None, + }, + ) + + +def create_session_factory(engine: Engine) -> sessionmaker[Session]: + """Return the factory of ORM sessions on ``engine``. + + Attributes keep their values after the transaction commits, so a function + can return what it wrote once its session has closed. An attribute set to a + SQL expression, such as ``func.now()``, is the exception: only the database + knows its value, so refresh the object before the session closes. Run a + transaction as ``with session_factory.begin() as session:``. + """ + return sessionmaker(engine, expire_on_commit=False) + + +def create_simulation_session_factory(database_url: str) -> sessionmaker[Session]: + """Return the session factory of the simulation service's connection pool. + + Raises ValueError, without echoing the URL, when it is malformed. Nothing + connects until a session runs its first statement. """ - with engine.begin() as connection: - connection.execute( - select( - func.set_config( - "statement_timeout", - str(statement_timeout_milliseconds), - True, # noqa: FBT003 # set_config's is_local argument - ) - ) + return create_session_factory(_create_pooled_engine(database_url)) + + +def create_unpooled_session_factory(database_url: str) -> sessionmaker[Session]: + """Return a factory of sessions that each open and close their connection. + + Raises ValueError, without echoing the URL, when it is malformed. Nothing + connects until a session runs its first statement. + """ + return create_session_factory(_create_unpooled_engine(database_url)) + + +def create_single_connection_session_factory( + database_url: str, +) -> sessionmaker[Session]: + """Return a factory of sessions that take turns on one kept connection. + + Raises ValueError, without echoing the URL, when it is malformed. Nothing + connects until a session runs its first statement. Close the connection + with ``dispose_session_factory``. + """ + return create_session_factory(_create_single_connection_engine(database_url)) + + +def dispose_session_factory(session_factory: sessionmaker[Session]) -> None: + """Close every pooled connection of the engine ``session_factory`` uses.""" + session_factory.kw["bind"].dispose() + + +def _set_statement_timeout(statement_timeout_milliseconds: int) -> Select[tuple[str]]: + """Return the statement that bounds the rest of the current transaction.""" + return select( + func.set_config( + "statement_timeout", + str(statement_timeout_milliseconds), + True, # noqa: FBT003 # set_config's is_local argument ) - yield connection + ) + + +@contextmanager +def bounded_session( + session_factory: sessionmaker[Session], *, statement_timeout_milliseconds: int +) -> Iterator[Session]: + """Yield a session in a transaction whose statements the database bounds. + + The transaction commits when the block ends, flushing what the session + holds under the same limit, and rolls back when it raises. + ``statement_timeout`` is set with ``set_config`` local to the transaction, + so it works through Supabase's transaction pooler and never stays on the + pooled connection; a statement that runs longer raises the database's + cancellation as an OperationalError. + """ + with session_factory.begin() as session: + session.execute(_set_statement_timeout(statement_timeout_milliseconds)) + yield session diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/household_job_records.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/household_job_records.py index 168f317aa..3d5be32a7 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/household_job_records.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/household_job_records.py @@ -2,7 +2,7 @@ A household calculation is one ``household_jobs`` row, which the service creates and a household function on Modal calculates. The service reads the -stored household and the catalog version from the Public API's tables, creates +stored household and the catalog version from API v1's tables, creates the job ``pending`` with no Modal call ID, starts the call, then records the call's ID, or fails the job at once when Modal refuses the call. Both updates apply only while the job is still ``pending`` with no call ID: the household @@ -10,10 +10,12 @@ job no call was recorded for. The household function writes the rest: ``running``, then the result and ``succeeded`` or ``failed``. -Every function takes a connection, so the caller chooses the transactions. -Each update sets only the columns the service's role may update, with -``updated_at``, whose automatic update the Public API's models make on their -own side only. +Every function takes an ORM session, so the caller chooses the transactions. +Reads select the mapped classes' attributes, so they return what the database +holds now, whatever the session has loaded. Each update is one conditional +statement that leaves the session's objects alone, and sets only the columns +the service's role may update, with ``updated_at``, whose automatic update +API v1's models make on their own side only. """ from __future__ import annotations @@ -21,38 +23,40 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any -from sqlalchemy import func, insert, select, update +from sqlalchemy import ForeignKeyConstraint, func, select, update from sqlalchemy.exc import IntegrityError from policyengine_simulation_persistence.input_tables import ( MODEL_NAME_PREFIX, - households, - tax_benefit_model_versions, - tax_benefit_models, + Household, + TaxBenefitModel, + TaxBenefitModelVersion, ) from policyengine_simulation_persistence.tables import ( + HouseholdJob, HouseholdJobStatus, - household_jobs, ) if TYPE_CHECKING: from datetime import datetime from uuid import UUID - from sqlalchemy import ColumnElement, Connection + from sqlalchemy import ColumnElement + from sqlalchemy.orm import Session # PostgreSQL's SQLSTATE for a row naming one that does not exist. FOREIGN_KEY_VIOLATION = "23503" # Each of the job's foreign keys, by name, and the column it constrains. _KEY_COLUMNS = { str(constraint.name): constraint.column_keys[0] - for constraint in household_jobs.foreign_key_constraints + for constraint in HouseholdJob.__table_args__ + if isinstance(constraint, ForeignKeyConstraint) } @dataclass(frozen=True) class StoredHousehold: - """A household as the Public API stores it. + """A household as API v1 stores it. ``document`` lists the people and each group entity as records with an ``id`` and ``values``, each person naming its groups, and may hold a saved @@ -127,29 +131,26 @@ def __init__(self, column_name: str | None) -> None: def read_catalog_version_id( - connection: Connection, *, country: str, policyengine_version: str + session: Session, *, country: str, policyengine_version: str ) -> UUID | None: """Return the id of the country's catalog version for a .py version, or None.""" - return connection.execute( - select(tax_benefit_model_versions.c.id) - .join( - tax_benefit_models, - tax_benefit_models.c.id == tax_benefit_model_versions.c.model_id, - ) + return session.execute( + select(TaxBenefitModelVersion.id) + .join(TaxBenefitModel, TaxBenefitModel.id == TaxBenefitModelVersion.model_id) .where( - tax_benefit_models.c.name == f"{MODEL_NAME_PREFIX}{country}", - tax_benefit_model_versions.c.version == policyengine_version, + TaxBenefitModel.name == f"{MODEL_NAME_PREFIX}{country}", + TaxBenefitModelVersion.version == policyengine_version, ) ).scalar_one_or_none() def read_stored_household( - connection: Connection, *, country: str, household_id: UUID + session: Session, *, country: str, household_id: UUID ) -> StoredHousehold | None: """Return the country's stored household, or None when it has no such one.""" - row = connection.execute( - select(households.c.default_year, households.c.household_data).where( - households.c.id == household_id, households.c.country_id == country + row = session.execute( + select(Household.default_year, Household.household_data).where( + Household.id == household_id, Household.country_id == country ) ).one_or_none() if row is None: @@ -162,29 +163,32 @@ def read_stored_household( ) -def insert_household_job(connection: Connection, job: NewHouseholdJob) -> None: +def insert_household_job(session: Session, job: NewHouseholdJob) -> None: """Create the job ``pending``, with no Modal call ID and no dynamic. - Raise HouseholdJobReferenceError when the database refuses a row it - names. The refusal ends the caller's transaction, or its savepoint. + The row is flushed here, so a failure of its INSERT is raised here and + never at the commit. Raise HouseholdJobReferenceError when the database + refuses a row it names. The refusal ends the caller's transaction, or its + savepoint. """ - try: - connection.execute( - insert(household_jobs).values( - id=job.id, - country=job.country, - household_id=job.household_id, - policy_id=job.policy_id, - dynamic_id=None, - year=job.year, - request_data=job.request_data, - tax_benefit_model_version_id=job.tax_benefit_model_version_id, - country_package_version=job.country_package_version, - policyengine_version=job.policyengine_version, - status=HouseholdJobStatus.PENDING, - modal_call_id=None, - ) + session.add( + HouseholdJob( + id=job.id, + country=job.country, + household_id=job.household_id, + policy_id=job.policy_id, + dynamic_id=None, + year=job.year, + request_data=job.request_data, + tax_benefit_model_version_id=job.tax_benefit_model_version_id, + country_package_version=job.country_package_version, + policyengine_version=job.policyengine_version, + status=HouseholdJobStatus.PENDING, + modal_call_id=None, ) + ) + try: + session.flush() except IntegrityError as error: driver_error = error.orig if getattr(driver_error, "sqlstate", None) != FOREIGN_KEY_VIOLATION: @@ -199,44 +203,45 @@ def insert_household_job(connection: Connection, job: NewHouseholdJob) -> None: def _pending_without_call(job_id: UUID) -> tuple[ColumnElement[bool], ...]: return ( - household_jobs.c.id == job_id, - household_jobs.c.status == HouseholdJobStatus.PENDING, - household_jobs.c.modal_call_id.is_(None), + HouseholdJob.id == job_id, + HouseholdJob.status == HouseholdJobStatus.PENDING, + HouseholdJob.modal_call_id.is_(None), ) def record_household_call( - connection: Connection, *, job_id: UUID, modal_call_id: str + session: Session, *, job_id: UUID, modal_call_id: str ) -> HouseholdJobStatus | None: """Record the job's Modal call ID while it is pending with none. Return the job's status afterwards, or None when there is no such job. A job the household function has claimed already keeps its claim. """ - recorded = connection.execute( - update(household_jobs) + recorded = session.execute( + update(HouseholdJob) .where(*_pending_without_call(job_id)) .values(modal_call_id=modal_call_id, updated_at=func.now()) - .returning(household_jobs.c.status) + .returning(HouseholdJob.status) + .execution_options(synchronize_session=False) ).scalar_one_or_none() if recorded is not None: return HouseholdJobStatus(recorded) - status = connection.execute( - select(household_jobs.c.status).where(household_jobs.c.id == job_id) + status = session.execute( + select(HouseholdJob.status).where(HouseholdJob.id == job_id) ).scalar_one_or_none() return None if status is None else HouseholdJobStatus(status) def fail_unstarted_household_job( - connection: Connection, *, job_id: UUID, error_message: str + session: Session, *, job_id: UUID, error_message: str ) -> bool: """Fail the job at once while it is pending with no Modal call ID. Return whether it was failed: a job a call has been recorded for, or one the household function has claimed, is left as it is. """ - failed = connection.execute( - update(household_jobs) + failed = session.execute( + update(HouseholdJob) .where(*_pending_without_call(job_id)) .values( status=HouseholdJobStatus.FAILED, @@ -244,34 +249,33 @@ def fail_unstarted_household_job( completed_at=func.now(), updated_at=func.now(), ) - .returning(household_jobs.c.id) + .returning(HouseholdJob.id) + .execution_options(synchronize_session=False) ).scalar_one_or_none() return failed is not None -def read_household_job( - connection: Connection, job_id: UUID -) -> HouseholdJobRecord | None: +def read_household_job(session: Session, job_id: UUID) -> HouseholdJobRecord | None: """Return the household job as it stands, or None when there is no such job.""" - row = connection.execute( + row = session.execute( select( - household_jobs.c.id, - household_jobs.c.status, - household_jobs.c.country, - household_jobs.c.household_id, - household_jobs.c.policy_id, - household_jobs.c.year, - household_jobs.c.request_data, - household_jobs.c.tax_benefit_model_version_id, - household_jobs.c.country_package_version, - household_jobs.c.policyengine_version, - household_jobs.c.modal_call_id, - household_jobs.c.result, - household_jobs.c.error_message, - household_jobs.c.created_at, - household_jobs.c.started_at, - household_jobs.c.completed_at, - ).where(household_jobs.c.id == job_id) + HouseholdJob.id, + HouseholdJob.status, + HouseholdJob.country, + HouseholdJob.household_id, + HouseholdJob.policy_id, + HouseholdJob.year, + HouseholdJob.request_data, + HouseholdJob.tax_benefit_model_version_id, + HouseholdJob.country_package_version, + HouseholdJob.policyengine_version, + HouseholdJob.modal_call_id, + HouseholdJob.result, + HouseholdJob.error_message, + HouseholdJob.created_at, + HouseholdJob.started_at, + HouseholdJob.completed_at, + ).where(HouseholdJob.id == job_id) ).one_or_none() if row is None: return None diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/household_job_store.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/household_job_store.py index ab3c1a09c..71bc6f37f 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/household_job_store.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/household_job_store.py @@ -4,7 +4,8 @@ own transaction, with the Modal call started between them: it reads the inputs, creates the job, then records the call or fails the job. Later reads return the job as it stands. ``HouseholdJobStore`` runs the reads and writes -of ``household_job_records`` and ``policy_reform`` that way. +of ``household_job_records`` and ``policy_reform`` that way, each in an ORM +session of its own. """ from __future__ import annotations @@ -12,7 +13,6 @@ from dataclasses import dataclass from typing import TYPE_CHECKING -from policyengine_simulation_persistence.engine import bounded_transaction from policyengine_simulation_persistence.household_job_records import ( fail_unstarted_household_job, insert_household_job, @@ -22,13 +22,16 @@ record_household_call, ) from policyengine_simulation_persistence.policy_reform import read_policy_reform +from policyengine_simulation_persistence.store_transactions import ( + TransactionFactory, + bounded_transactions, + create_in_transaction, +) if TYPE_CHECKING: - from collections.abc import Callable - from contextlib import AbstractContextManager from uuid import UUID - from sqlalchemy import Connection, Engine + from sqlalchemy.orm import Session, sessionmaker from policyengine_simulation_persistence.household_job_records import ( HouseholdJobRecord, @@ -38,15 +41,13 @@ from policyengine_simulation_persistence.policy_reform import PolicyReform from policyengine_simulation_persistence.tables import HouseholdJobStatus -type TransactionFactory = Callable[[], AbstractContextManager[Connection]] - class UnknownCatalogVersionError(LookupError): """The country has no catalog version for the .py version.""" class UnknownHouseholdError(LookupError): - """The Public API has no such household in the country.""" + """API v1 has no such household in the country.""" @dataclass(frozen=True) @@ -67,21 +68,23 @@ class HouseholdJobStore: """Read and write household jobs, each operation in its own transaction.""" def __init__(self, transaction: TransactionFactory) -> None: - """Open each transaction with ``transaction``, which commits on success.""" + """Open each session with ``transaction``, which commits on success.""" self._transaction = transaction @classmethod - def from_engine( - cls, engine: Engine, *, statement_timeout_milliseconds: int + def from_session_factory( + cls, + session_factory: sessionmaker[Session], + *, + statement_timeout_milliseconds: int, ) -> HouseholdJobStore: - """Use ``engine``, bounding each statement to the given time.""" - - def transaction() -> AbstractContextManager[Connection]: - return bounded_transaction( - engine, statement_timeout_milliseconds=statement_timeout_milliseconds + """Use ``session_factory``, bounding each statement to the given time.""" + return cls( + bounded_transactions( + session_factory, + statement_timeout_milliseconds=statement_timeout_milliseconds, ) - - return cls(transaction) + ) def read_inputs( self, @@ -96,15 +99,15 @@ def read_inputs( Raise UnknownCatalogVersionError, UnknownHouseholdError, or the policy reader's UnknownPolicyError or MissingPolicyParametersError. """ - with self._transaction() as connection: + with self._transaction() as session: catalog_version_id = read_catalog_version_id( - connection, country=country, policyengine_version=policyengine_version + session, country=country, policyengine_version=policyengine_version ) if catalog_version_id is None: msg = f"{country} has no catalog version for .py {policyengine_version}" raise UnknownCatalogVersionError(msg) household = read_stored_household( - connection, country=country, household_id=household_id + session, country=country, household_id=household_id ) if household is None: msg = f"{country} has no household {household_id}" @@ -113,7 +116,7 @@ def read_inputs( {} if policy_id is None else read_policy_reform( - connection, + session, country=country, policy_id=policy_id, catalog_version_id=catalog_version_id, @@ -124,27 +127,38 @@ def read_inputs( ) def create_job(self, job: NewHouseholdJob) -> None: - """Create the job ``pending`` with no call; see ``insert_household_job``.""" - with self._transaction() as connection: - insert_household_job(connection, job) + """Create the job ``pending`` with no call; see ``insert_household_job``. + + An error while the row is flushed leaves nothing written and + propagates, a refused reference as HouseholdJobReferenceError. An + error once it is flushed is the commit's, after which the job may + exist: EconomyRecordOutcomeUnknownError, caused by the commit's error. + """ + create_in_transaction( + self._transaction, + lambda session: insert_household_job(session, job), + commit_failure=( + "the household job's commit failed after its row was inserted" + ), + ) def record_call( self, job_id: UUID, modal_call_id: str ) -> HouseholdJobStatus | None: """Record the job's call ID while it is pending with none; return its status.""" - with self._transaction() as connection: + with self._transaction() as session: return record_household_call( - connection, job_id=job_id, modal_call_id=modal_call_id + session, job_id=job_id, modal_call_id=modal_call_id ) def fail_unstarted(self, job_id: UUID, error_message: str) -> bool: """Fail the job while it is pending with no call ID; return whether it was.""" - with self._transaction() as connection: + with self._transaction() as session: return fail_unstarted_household_job( - connection, job_id=job_id, error_message=error_message + session, job_id=job_id, error_message=error_message ) def read(self, job_id: UUID) -> HouseholdJobRecord | None: """Return the job as it stands, or None when there is no such job.""" - with self._transaction() as connection: - return read_household_job(connection, job_id) + with self._transaction() as session: + return read_household_job(session, job_id) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/household_report_records.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/household_report_records.py index c4df16170..a5b66826c 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/household_report_records.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/household_report_records.py @@ -21,26 +21,24 @@ from typing import TYPE_CHECKING, Any, Final -from sqlalchemy import insert - from policyengine_simulation_persistence.tables import ( + SimulationRun, SimulationStatus, - simulation_runs, ) if TYPE_CHECKING: from uuid import UUID - from sqlalchemy import Connection + from sqlalchemy.orm import Session from policyengine_simulation_persistence.economy_run_records import RunVersions -# A report's type, as the Public API reads it. +# A report's type, as API v1 reads it. HOUSEHOLD_REPORT_TYPE: Final = "household" def insert_household_simulation_run( # noqa: PLR0913 # the run and what it calculates on - connection: Connection, + session: Session, *, run_id: UUID, simulation_id: UUID, @@ -53,22 +51,22 @@ def insert_household_simulation_run( # noqa: PLR0913 # the run and what it cal It records the year, catalog version and versions it calculates on; a household run names no dataset or region. A ``forced`` run is an extra - run that nothing joins. + run that nothing joins. The row is flushed here: no relationship orders + it before the report run that goes on to name it. """ - connection.execute( - insert(simulation_runs).values( - { - **household_simulation_run_row( - run_id=run_id, - simulation_id=simulation_id, - year=year, - tax_benefit_model_version_id=tax_benefit_model_version_id, - versions=versions, - ), - "forced": forced, - } + session.add( + SimulationRun( + **household_simulation_run_row( + run_id=run_id, + simulation_id=simulation_id, + year=year, + tax_benefit_model_version_id=tax_benefit_model_version_id, + versions=versions, + ), + forced=forced, ) ) + session.flush() def household_simulation_run_row( diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/household_report_store.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/household_report_store.py index e33d29231..99c8f2330 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/household_report_store.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/household_report_store.py @@ -8,7 +8,8 @@ runs instead. ``HouseholdReportStore`` runs the reads and writes of ``keyed_household_records``, ``household_job_records``, ``policy_reform``, ``economy_rerun_records``' household report re-run and -``economy_run_records``' report run writes that way. +``economy_run_records``' report run writes that way, each in an ORM session +of its own. """ from __future__ import annotations @@ -16,8 +17,6 @@ from dataclasses import dataclass from typing import TYPE_CHECKING -from sqlalchemy.exc import SQLAlchemyError - from policyengine_simulation_persistence.economy_rerun_records import ( insert_household_report_rerun, ) @@ -26,11 +25,7 @@ read_model_id, record_report_run_call, ) -from policyengine_simulation_persistence.economy_run_store import ( - EconomyRecordOutcomeUnknownError, - ReportSideError, -) -from policyengine_simulation_persistence.engine import bounded_transaction +from policyengine_simulation_persistence.economy_run_store import ReportSideError from policyengine_simulation_persistence.household_job_records import ( read_catalog_version_id, read_stored_household, @@ -47,13 +42,17 @@ UnknownPolicyError, read_policy_reform, ) +from policyengine_simulation_persistence.store_transactions import ( + TransactionFactory, + bounded_transactions, + create_in_transaction, +) if TYPE_CHECKING: from collections.abc import Callable, Mapping, Sequence - from contextlib import AbstractContextManager from uuid import UUID - from sqlalchemy import Connection, Engine + from sqlalchemy.orm import Session, sessionmaker from policyengine_simulation_persistence.economy_rerun_records import ( NewHouseholdReportRerun, @@ -70,8 +69,6 @@ from policyengine_simulation_persistence.shared_rows import FoundReportRun from policyengine_simulation_persistence.tables import ReportRunStatus -type TransactionFactory = Callable[[], AbstractContextManager[Connection]] - @dataclass(frozen=True) class HouseholdReportInputs: @@ -90,7 +87,7 @@ class HouseholdReportInputs: def _side_reform( - connection: Connection, + session: Session, side: ReportSide, *, country: str, @@ -102,7 +99,7 @@ def _side_reform( return {} try: return read_policy_reform( - connection, + session, country=country, policy_id=policy_id, catalog_version_id=catalog_version_id, @@ -115,21 +112,23 @@ class HouseholdReportStore: """Read and write household reports, each operation in its own transaction.""" def __init__(self, transaction: TransactionFactory) -> None: - """Open each transaction with ``transaction``, which commits on success.""" + """Open each session with ``transaction``, which commits on success.""" self._transaction = transaction @classmethod - def from_engine( - cls, engine: Engine, *, statement_timeout_milliseconds: int + def from_session_factory( + cls, + session_factory: sessionmaker[Session], + *, + statement_timeout_milliseconds: int, ) -> HouseholdReportStore: - """Use ``engine``, bounding each statement to the given time.""" - - def transaction() -> AbstractContextManager[Connection]: - return bounded_transaction( - engine, statement_timeout_milliseconds=statement_timeout_milliseconds + """Use ``session_factory``, bounding each statement to the given time.""" + return cls( + bounded_transactions( + session_factory, + statement_timeout_milliseconds=statement_timeout_milliseconds, ) - - return cls(transaction) + ) def read_inputs( self, @@ -147,23 +146,23 @@ def read_inputs( (UnknownPolicyError) or sets parameters the catalog version lacks (MissingPolicyParametersError). """ - with self._transaction() as connection: + with self._transaction() as session: catalog_version_id = read_catalog_version_id( - connection, country=country, policyengine_version=policyengine_version + session, country=country, policyengine_version=policyengine_version ) - model_id = read_model_id(connection, country=country) + model_id = read_model_id(session, country=country) if catalog_version_id is None or model_id is None: msg = f"{country} has no catalog version for .py {policyengine_version}" raise UnknownCatalogVersionError(msg) household = read_stored_household( - connection, country=country, household_id=household_id + session, country=country, household_id=household_id ) if household is None: msg = f"{country} has no household {household_id}" raise UnknownHouseholdError(msg) reforms: dict[ReportSide, PolicyReform] = { side: _side_reform( - connection, + session, side, country=country, policy_id=policy_id, @@ -188,7 +187,7 @@ def find_or_create_report( rows were inserted. """ return self._create( - lambda connection: find_or_create_household_report(connection, report) + lambda session: find_or_create_household_report(session, report) ) def create_report_rerun(self, rerun: NewHouseholdReportRerun) -> FoundReportRun: @@ -198,33 +197,24 @@ def create_report_rerun(self, rerun: NewHouseholdReportRerun) -> FoundReportRun: EconomyRecordOutcomeUnknownError as ``find_or_create_report`` does. """ return self._create( - lambda connection: insert_household_report_rerun(connection, rerun) + lambda session: insert_household_report_rerun(session, rerun) ) - def _create[Created](self, insert_rows: Callable[[Connection], Created]) -> Created: - """Insert the rows in a transaction; tell a lost commit from a refusal. - - Return what ``insert_rows`` returns. - """ - inserted = False - try: - with self._transaction() as connection: - created = insert_rows(connection) - inserted = True - except SQLAlchemyError as error: - if not inserted: - raise - msg = "the create's commit failed after its rows were inserted" - raise EconomyRecordOutcomeUnknownError(msg) from error - return created + def _create[Created](self, insert_rows: Callable[[Session], Created]) -> Created: + """Insert the rows in a transaction; tell a lost commit from a refusal.""" + return create_in_transaction( + self._transaction, + insert_rows, + commit_failure="the create's commit failed after its rows were inserted", + ) def record_report_call( self, report_run_id: UUID, modal_call_id: str ) -> ReportRunStatus | None: """Record the report run's call ID while it is pending with none.""" - with self._transaction() as connection: + with self._transaction() as session: return record_report_run_call( - connection, report_run_id=report_run_id, modal_call_id=modal_call_id + session, report_run_id=report_run_id, modal_call_id=modal_call_id ) def fail_unstarted_report( @@ -234,9 +224,9 @@ def fail_unstarted_report( error_message: str, ) -> bool: """Fail the report run, and its simulation runs, while it is unstarted.""" - with self._transaction() as connection: + with self._transaction() as session: return fail_unstarted_report_run( - connection, + session, report_run_id=report_run_id, simulation_run_ids=simulation_run_ids, error_message=error_message, diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/input_tables.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/input_tables.py index aef548d47..1a9d37c5b 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/input_tables.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/input_tables.py @@ -1,40 +1,32 @@ -"""Read-only models of the Public API's tables a calculation is read from. - -The Public API writes households, policies, dynamics and their parameter -values, and its model catalog; the simulation service only reads them, with -``SELECT``. Each table here names only the columns the service reads, so these -are not models of the whole tables: the Public API's other columns, -defaults, constraints and indexes are its own. The tables live in -``input_metadata``, apart from the models of the tables the service writes -(``service_metadata``) and from the tables those name by ``id`` alone -(``referenced_metadata``). ``tests/test_public_api_schema.py`` compares each -column with the Public API's table. +"""Read-only mapped classes of API v1's tables a calculation is read from. + +API v1 writes households, policies, dynamics and their parameter values, and +its model catalog; the simulation service only reads them, with ``SELECT``. +Each class names only the columns the service reads and the table's primary +key, so these are not models of the whole tables: API v1's other columns, +defaults, constraints and indexes are its own. The classes derive from +``InputBase``, apart from the models of the tables the service writes +(``ServiceBase``) and from the tables those name by ``id`` alone +(``ReferencedBase``). ``tests/test_public_api_schema.py`` compares each column +with API v1's table. """ from __future__ import annotations -from typing import Final - -from sqlalchemy import ( - Column, - DateTime, - Enum, - Integer, - MetaData, - String, - Table, - Uuid, -) -from sqlalchemy.dialects.postgresql import JSONB +from datetime import datetime # noqa: TC003 # SQLAlchemy reads Mapped annotations +from typing import Any, Final +from uuid import UUID # noqa: TC003 # SQLAlchemy reads Mapped annotations at run time -from policyengine_simulation_persistence.tables import TABLE_SCHEMA +from sqlalchemy import DateTime, Enum, Integer, String, Uuid +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column -input_metadata = MetaData(schema=TABLE_SCHEMA) +from policyengine_simulation_persistence.orm_base import TABLE_SCHEMA, InputBase # A country's tax-benefit model is named "policyengine-". MODEL_NAME_PREFIX: Final = "policyengine-" -# The kinds of geography a region covers, as the Public API's -# ``v2_region_type`` lists them. +# The kinds of geography a region covers, as API v1's ``v2_region_type`` lists +# them. REGION_TYPES: Final = ( "national", "country", @@ -48,94 +40,119 @@ # The UK region types a report on the local-area dataset calculates. UK_AREA_REGION_TYPES: Final = ("constituency", "local_authority") -tax_benefit_models = Table( - "tax_benefit_models", - input_metadata, - Column("id", Uuid, nullable=False), - Column("name", String(32), nullable=False), -) -# One published catalog version of a model; ``version`` is the .py -# (policyengine package) version it was published from. -tax_benefit_model_versions = Table( - "tax_benefit_model_versions", - input_metadata, - Column("id", Uuid, nullable=False), - Column("model_id", Uuid, nullable=False), - Column("version", String(128), nullable=False), -) +class TaxBenefitModel(InputBase): + """A country's tax-benefit model.""" + + __tablename__ = "tax_benefit_models" + + id: Mapped[UUID] = mapped_column(Uuid, primary_key=True) + name: Mapped[str] = mapped_column(String(32)) + + +class TaxBenefitModelVersion(InputBase): + """One published catalog version of a model. + + ``version`` is the .py (policyengine package) version it was published from. + """ + + __tablename__ = "tax_benefit_model_versions" + + id: Mapped[UUID] = mapped_column(Uuid, primary_key=True) + model_id: Mapped[UUID] = mapped_column(Uuid) + version: Mapped[str] = mapped_column(String(128)) -# A region of one catalog version, unique by code within it, and the dataset -# it uses unless a calculation names another. -regions = Table( - "regions", - input_metadata, - Column("id", Uuid, nullable=False), - Column("code", String(255), nullable=False), + +class Region(InputBase): + """A region of one catalog version, unique by code within it. + + ``default_dataset_id`` is the dataset it uses unless a calculation names + another. + """ + + __tablename__ = "regions" + + id: Mapped[UUID] = mapped_column(Uuid, primary_key=True) + code: Mapped[str] = mapped_column(String(255)) # The enum names its schema itself, as the service tables' enums do. - Column( - "region_type", - Enum(*REGION_TYPES, name="v2_region_type", schema=TABLE_SCHEMA), - nullable=False, - ), - Column("tax_benefit_model_version_id", Uuid, nullable=False), - Column("default_dataset_id", Uuid, nullable=False), -) + region_type: Mapped[str] = mapped_column( + Enum(*REGION_TYPES, name="v2_region_type", schema=TABLE_SCHEMA) + ) + tax_benefit_model_version_id: Mapped[UUID] = mapped_column(Uuid) + default_dataset_id: Mapped[UUID] = mapped_column(Uuid) -# A dataset of one catalog version, unique by name within it. -datasets = Table( - "datasets", - input_metadata, - Column("id", Uuid, nullable=False), - Column("name", String(255), nullable=False), - Column("tax_benefit_model_version_id", Uuid, nullable=False), -) -# ``household_data`` lists the people and each group entity as records, and -# may hold a saved SPM selection; ``default_year`` is the year it describes. -households = Table( - "households", - input_metadata, - Column("id", Uuid, nullable=False), - Column("country_id", String(2), nullable=False), - Column("default_year", Integer, nullable=True), - Column("household_data", JSONB, nullable=False), -) +class Dataset(InputBase): + """A dataset of one catalog version, unique by name within it.""" -policies = Table( - "policies", - input_metadata, - Column("id", Uuid, nullable=False), - Column("country_id", String(2), nullable=False), -) + __tablename__ = "datasets" -# A catalog version's parameters, each named once within it. -parameters = Table( - "parameters", - input_metadata, - Column("id", Uuid, nullable=False), - Column("name", String(512), nullable=False), - Column("tax_benefit_model_version_id", Uuid, nullable=False), -) + id: Mapped[UUID] = mapped_column(Uuid, primary_key=True) + name: Mapped[str] = mapped_column(String(255)) + tax_benefit_model_version_id: Mapped[UUID] = mapped_column(Uuid) -# A parameter's value from ``start_date`` until ``end_date``, or with no end -# date indefinitely; a policy's own values name it in ``policy_id``, and a -# dynamic's in ``dynamic_id``. -parameter_values = Table( - "parameter_values", - input_metadata, - Column("parameter_id", Uuid, nullable=False), - Column("value_json", JSONB, nullable=False), - Column("start_date", DateTime(timezone=True), nullable=False), - Column("end_date", DateTime(timezone=True), nullable=True), - Column("policy_id", Uuid, nullable=True), - Column("dynamic_id", Uuid, nullable=True), -) -# Named parameter values a simulation applies beside its policy, such as -# behavioural responses. A dynamic belongs to no country. -dynamics = Table( - "dynamics", - input_metadata, - Column("id", Uuid, nullable=False), -) +class Household(InputBase): + """A stored household. + + ``household_data`` lists the people and each group entity as records, and + may hold a saved SPM selection; ``default_year`` is the year it describes. + """ + + __tablename__ = "households" + + id: Mapped[UUID] = mapped_column(Uuid, primary_key=True) + country_id: Mapped[str] = mapped_column(String(2)) + default_year: Mapped[int | None] = mapped_column(Integer) + household_data: Mapped[dict[str, Any]] = mapped_column(JSONB) + + +class Policy(InputBase): + """A stored policy; its values are ``ParameterValue`` rows.""" + + __tablename__ = "policies" + + id: Mapped[UUID] = mapped_column(Uuid, primary_key=True) + country_id: Mapped[str] = mapped_column(String(2)) + + +class Parameter(InputBase): + """A catalog version's parameter, named once within it.""" + + __tablename__ = "parameters" + + id: Mapped[UUID] = mapped_column(Uuid, primary_key=True) + name: Mapped[str] = mapped_column(String(512)) + tax_benefit_model_version_id: Mapped[UUID] = mapped_column(Uuid) + + +class ParameterValue(InputBase): + """A parameter's value from ``start_date`` until ``end_date``. + + With no end date it applies indefinitely. A policy's own values name it in + ``policy_id``, and a dynamic's in ``dynamic_id``. + """ + + __tablename__ = "parameter_values" + + id: Mapped[UUID] = mapped_column(Uuid, primary_key=True) + parameter_id: Mapped[UUID] = mapped_column(Uuid) + value_json: Mapped[Any] = mapped_column(JSONB, nullable=False) + start_date: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + end_date: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + policy_id: Mapped[UUID | None] = mapped_column(Uuid) + dynamic_id: Mapped[UUID | None] = mapped_column(Uuid) + + +class Dynamic(InputBase): + """Named parameter values a simulation applies beside its policy. + + Behavioural responses are an example. A dynamic belongs to no country. + """ + + __tablename__ = "dynamics" + + id: Mapped[UUID] = mapped_column(Uuid, primary_key=True) + + +input_metadata = InputBase.metadata diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/keyed_economy_records.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/keyed_economy_records.py index e5691ad30..9e7b73741 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/keyed_economy_records.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/keyed_economy_records.py @@ -56,7 +56,7 @@ from collections.abc import Mapping from uuid import UUID - from sqlalchemy import Connection + from sqlalchemy.orm import Session from policyengine_simulation_persistence.definition_keys import ( EconomyReportDefinition, @@ -149,7 +149,7 @@ class FoundEconomySimulation: def find_or_create_economy_report( - connection: Connection, request: KeyedEconomyReport + session: Session, request: KeyedEconomyReport ) -> FoundEconomyReport: """Find the report the request names, and its run, or create them. @@ -159,9 +159,9 @@ def find_or_create_economy_report( transaction, or its savepoint. """ with references_checked(): - simulation_ids = _find_or_insert_simulations(connection, request) + simulation_ids = _find_or_insert_simulations(session, request) report = find_or_insert_report( - connection, + session, economy_report_row( request.definition, report_id=request.report_id, @@ -173,7 +173,7 @@ def find_or_create_economy_report( ) versions = request.versions joined = find_joinable_report_run( - connection, + session, report_id=report.id, country_package_version=versions.country_package_version, policyengine_version=versions.policyengine_version, @@ -182,7 +182,7 @@ def find_or_create_economy_report( return _found(report.id, report.inserted, joined, simulation_ids, set()) proposed = {"baseline": request.baseline, "reform": request.reform} run, runs = find_or_insert_report_run_on_sides( - connection, + session, _report_run_row(request, report.id), { side: simulation_run_row( @@ -199,7 +199,7 @@ def find_or_create_economy_report( def find_or_create_economy_simulation( - connection: Connection, request: KeyedEconomySimulation + session: Session, request: KeyedEconomySimulation ) -> FoundEconomySimulation: """Find the simulation the request names, and its run, or create them. @@ -207,13 +207,13 @@ def find_or_create_economy_simulation( """ with references_checked(): simulation = find_or_insert_simulation( - connection, + session, economy_simulation_row( request.definition, request.simulation_id, request.population ), ) run = find_or_insert_simulation_run( - connection, + session, simulation_run_row( run_id=request.run_id, simulation_id=simulation.id, @@ -229,7 +229,7 @@ def find_or_create_economy_simulation( def _find_or_insert_simulations( - connection: Connection, request: KeyedEconomyReport + session: Session, request: KeyedEconomyReport ) -> dict[Side, UUID]: """Return each side's simulation ID, found by key or inserted, in key order.""" proposed = {"baseline": request.baseline, "reform": request.reform} @@ -241,7 +241,7 @@ def _find_or_insert_simulations( for key in sorted(by_key): sides = by_key[key] found = find_or_insert_simulation( - connection, + session, economy_simulation_row( request.definition.simulation(sides[0]), proposed[sides[0]].simulation_id, @@ -254,7 +254,7 @@ def _find_or_insert_simulations( def find_or_insert_report_run_on_sides( - connection: Connection, + session: Session, report_run_row: Mapping[str, Any], side_rows: Mapping[Side, Mapping[str, Any]], ) -> tuple[FoundReportRun, dict[Side, FoundSimulationRun]]: @@ -269,12 +269,12 @@ def find_or_insert_report_run_on_sides( inserted is kept, so no side run is returned (the joined report run names its own). """ - savepoint = connection.begin_nested() + savepoint = session.begin_nested() try: - runs = find_or_insert_side_runs(connection, side_rows) + runs = find_or_insert_side_runs(session, side_rows) reform = runs.get("reform") run = find_or_insert_report_run( - connection, + session, { **report_run_row, "baseline_simulation_run_id": runs["baseline"].id, @@ -292,7 +292,7 @@ def find_or_insert_report_run_on_sides( def find_or_insert_side_runs( - connection: Connection, side_rows: Mapping[Side, Mapping[str, Any]] + session: Session, side_rows: Mapping[Side, Mapping[str, Any]] ) -> dict[Side, FoundSimulationRun]: """Return each side's run of its simulation, found or inserted. @@ -313,7 +313,7 @@ def find_or_insert_side_runs( for side in reversed(_SIDES) if side in side_rows } - found = find_or_insert_simulation_runs(connection, rows) + found = find_or_insert_simulation_runs(session, rows) return {side: found[row["simulation_id"]] for side, row in side_rows.items()} diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/keyed_household_records.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/keyed_household_records.py index f8ec4bc4b..fb3eb8dab 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/keyed_household_records.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/keyed_household_records.py @@ -59,7 +59,7 @@ if TYPE_CHECKING: from uuid import UUID - from sqlalchemy import Connection + from sqlalchemy.orm import Session from policyengine_simulation_persistence.definition_keys import ( HouseholdReportDefinition, @@ -116,7 +116,7 @@ class FoundHouseholdReport: def find_or_create_household_report( - connection: Connection, request: KeyedHouseholdReport + session: Session, request: KeyedHouseholdReport ) -> FoundHouseholdReport: """Find the household report the request names, and its run, or create them. @@ -126,11 +126,11 @@ def find_or_create_household_report( transaction, or its savepoint. """ with references_checked(): - simulation_ids = _find_or_insert_simulations(connection, request) - report = find_or_insert_report(connection, _report_row(request, simulation_ids)) + simulation_ids = _find_or_insert_simulations(session, request) + report = find_or_insert_report(session, _report_row(request, simulation_ids)) versions = request.versions joined = find_joinable_report_run( - connection, + session, report_id=report.id, country_package_version=versions.country_package_version, policyengine_version=versions.policyengine_version, @@ -138,7 +138,7 @@ def find_or_create_household_report( if joined is not None: return _found(report.id, report.inserted, joined, simulation_ids, set()) run, runs = find_or_insert_report_run_on_sides( - connection, + session, _report_run_row(request, report.id), { side: household_simulation_run_row( @@ -156,7 +156,7 @@ def find_or_create_household_report( def _find_or_insert_simulations( - connection: Connection, request: KeyedHouseholdReport + session: Session, request: KeyedHouseholdReport ) -> dict[Side, UUID]: """Return each side's simulation ID, found by key or inserted, in key order.""" definition = request.definition @@ -168,7 +168,7 @@ def _find_or_insert_simulations( for key in sorted(by_key): sides = by_key[key] found = find_or_insert_simulation( - connection, + session, _simulation_row(request, sides[0], proposed[sides[0]].simulation_id), ) for side in sides: diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/keyed_window_records.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/keyed_window_records.py index 9fbf98711..85dc19a6d 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/keyed_window_records.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/keyed_window_records.py @@ -15,9 +15,9 @@ 3. the report's window run on the current catalog version and .py version that hasn't failed and isn't a recalculation: when there is one, the request joins it, with its year runs, and inserts nothing more; -4. otherwise the window run, recording its .py version, against the Public - API's window run index (``find_or_insert_window_run``), so a request - inserting one another request has inserted waits for it and joins it; +4. otherwise the window run, recording its .py version, against API v1's + window run index (``find_or_insert_window_run``), so a request inserting + one another request has inserted waits for it and joins it; then each distinct simulation's run on those versions, catalog version and year that hasn't failed and wasn't forced, found or inserted in simulation ID order, and each year's report run naming them. @@ -59,15 +59,15 @@ find_or_insert_window_run, ) from policyengine_simulation_persistence.tables import ( + ReportRun, ReportRunStatus, - report_runs, ) if TYPE_CHECKING: from collections.abc import Sequence from uuid import UUID - from sqlalchemy import Connection + from sqlalchemy.orm import Session from policyengine_simulation_persistence.definition_keys import ( EconomyReportDefinition, @@ -166,7 +166,7 @@ class FoundBudgetWindow: def find_or_create_budget_window( - connection: Connection, request: KeyedBudgetWindow + session: Session, request: KeyedBudgetWindow ) -> FoundBudgetWindow: """Find the window the request names, and its window run, or create them. @@ -176,10 +176,10 @@ def find_or_create_budget_window( transaction, or its savepoint. """ with references_checked(): - simulation_ids = _find_or_insert_simulations(connection, request) + simulation_ids = _find_or_insert_simulations(session, request) first_year = request.years[0].year report = find_or_insert_report( - connection, + session, economy_report_row( request.definition, report_id=request.report_id, @@ -194,7 +194,7 @@ def find_or_create_budget_window( ) versions = request.versions joined = find_joinable_window_run( - connection, + session, report_id=report.id, tax_benefit_model_version_id=request.population.tax_benefit_model_version_id, policyengine_version=versions.policyengine_version, @@ -203,7 +203,7 @@ def find_or_create_budget_window( # The window run first: a request inserting one another request # has inserted waits for it, then joins it with its year runs. window_run = find_or_insert_window_run( - connection, + session, window_run_row( run_id=request.window_run_id, report_id=report.id, @@ -214,7 +214,7 @@ def find_or_create_budget_window( ) if window_run.inserted: years = insert_window_years( - connection, + session, _years_to_insert(request, simulation_ids), report_id=report.id, window_run_id=request.window_run_id, @@ -233,7 +233,7 @@ def find_or_create_budget_window( report_id=report.id, report_inserted=report.inserted, window_run=joined, - years=stored_window_years(connection, joined.id, simulation_ids), + years=stored_window_years(session, joined.id, simulation_ids), ) @@ -287,7 +287,7 @@ def window_year_run_row( # noqa: PLR0913 # the year's run, then what it calcul def _find_or_insert_simulations( - connection: Connection, request: KeyedBudgetWindow + session: Session, request: KeyedBudgetWindow ) -> dict[YearSide, UUID]: """Return each year's sides' simulation IDs, found by key or inserted. @@ -307,7 +307,7 @@ def _find_or_insert_simulations( year_sides = by_key[key] year, side = year_sides[0] found = find_or_insert_simulation( - connection, + session, economy_simulation_row( request.definition.simulation(side, year=year), proposed[year, side].simulation_id, @@ -320,7 +320,7 @@ def _find_or_insert_simulations( def insert_window_years( # noqa: PLR0913 # the years, then the window they belong to - connection: Connection, + session: Session, years: Sequence[WindowYearToInsert], *, report_id: UUID, @@ -342,7 +342,7 @@ def insert_window_years( # noqa: PLR0913 # the years, then the window they bel for window_year in years: for side in _SIDE_NAMES: insert_simulation_run( - connection, + session, run_id=window_year.proposed_run_ids[side], simulation_id=window_year.simulation_ids[side], population=replace(population, year=window_year.year), @@ -372,7 +372,7 @@ def insert_window_years( # noqa: PLR0913 # the years, then the window they bel population=replace(population, year=window_year.year), versions=versions, ) - runs = find_or_insert_simulation_runs(connection, rows) + runs = find_or_insert_simulation_runs(session, rows) found_years = tuple( _found_year( window_year, @@ -386,8 +386,9 @@ def insert_window_years( # noqa: PLR0913 # the years, then the window they bel ) for window_year in years ) - connection.execute( - insert(report_runs), + # A multi-row insert leaves out a None unless told to send it. + session.execute( + insert(ReportRun).execution_options(render_nulls=True), [ window_year_run_row( run_id=year.report_run_id, @@ -459,7 +460,7 @@ def _years_to_insert( def stored_window_years( - connection: Connection, + session: Session, window_run_id: UUID, simulation_ids: dict[YearSide, UUID], ) -> tuple[FoundWindowYear, ...]: @@ -468,15 +469,15 @@ def stored_window_years( Raise ValueError when its years aren't the request's: a window run of one report always has them. """ - rows = connection.execute( + rows = session.execute( select( - report_runs.c.id, - report_runs.c.year, - report_runs.c.baseline_simulation_run_id, - report_runs.c.reform_simulation_run_id, + ReportRun.id, + ReportRun.year, + ReportRun.baseline_simulation_run_id, + ReportRun.reform_simulation_run_id, ) - .where(report_runs.c.budget_window_run_id == window_run_id) - .order_by(report_runs.c.year) + .where(ReportRun.budget_window_run_id == window_run_id) + .order_by(ReportRun.year) ).all() requested = sorted({year for year, _ in simulation_ids}) if [row.year for row in rows] != requested: diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/orm_base.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/orm_base.py new file mode 100644 index 000000000..011677a3e --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/orm_base.py @@ -0,0 +1,105 @@ +"""The declarative bases of the library's mapped classes, and their shared columns. + +The library models API v1's tables in four groups, each with its own +declarative base and ``MetaData``. One base can't hold them all: the read-only +models in ``InputBase`` and the key targets in ``ReferencedBase`` describe some +of the same tables, and a ``MetaData`` holds one table of each name. Keys +between the groups name the other group's mapped attribute, such as +``ReportRun.id``. A fifth base, ``CatalogBase``, maps the PostgreSQL catalog +tables the privilege check reads. + +The four groups' ``MetaData`` name the ``public`` schema, so statements reach +API v1's tables whatever the role's ``search_path``. A ``MetaData`` holds a +group's tables once the module declaring them is imported, so each of those +modules exports its own: ``service_metadata`` from ``tables``, +``referenced_metadata`` from ``referenced_tables``, ``input_metadata`` from +``input_tables`` and ``result_metadata`` from ``result_tables``. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, ClassVar, Final +from uuid import uuid4 + +from sqlalchemy import DateTime, Enum, MetaData, Uuid, text +from sqlalchemy.orm import DeclarativeBase, mapped_column + +if TYPE_CHECKING: + from datetime import datetime + from enum import StrEnum + from uuid import UUID + + from sqlalchemy import Table + from sqlalchemy.orm import MappedColumn + +TABLE_SCHEMA: Final = "public" +CATALOG_SCHEMA: Final = "pg_catalog" + + +class _TableModel: + """Types ``__table__`` as what every class here maps: one ``Table``.""" + + __table__: ClassVar[Table] + + +class ServiceBase(_TableModel, DeclarativeBase): + """Base of the tables the simulation service creates rows in.""" + + metadata = MetaData(schema=TABLE_SCHEMA) + + +class ReferencedBase(_TableModel, DeclarativeBase): + """Base of API v1's tables that the service's keys name, by ``id`` alone.""" + + metadata = MetaData(schema=TABLE_SCHEMA) + + +class InputBase(_TableModel, DeclarativeBase): + """Base of the read-only models of the tables a calculation is read from.""" + + metadata = MetaData(schema=TABLE_SCHEMA) + + +class ResultBase(_TableModel, DeclarativeBase): + """Base of the result tables the Modal worker role inserts into.""" + + metadata = MetaData(schema=TABLE_SCHEMA) + + +class CatalogBase(_TableModel, DeclarativeBase): + """Base of the PostgreSQL catalog tables the privilege check reads.""" + + metadata = MetaData(schema=CATALOG_SCHEMA) + + +def enum_type(values: type[StrEnum], name: str) -> Enum: + """Store each member's value, under the Postgres type API v1's chain names. + + The type names its schema itself: SQLAlchemy 2.0 doesn't give an enum the + metadata's schema, so it would otherwise follow the role's search_path. + """ + return Enum( + values, + name=name, + schema=TABLE_SCHEMA, + values_callable=lambda members: [member.value for member in members], + ) + + +def identifier_column() -> MappedColumn[UUID]: + """Return a row's ``id``, generated here unless the row is given one.""" + return mapped_column(Uuid, nullable=False, default=uuid4) + + +def server_timestamp_column() -> MappedColumn[datetime]: + """Return a timezone-aware timestamp the database defaults to ``now()``.""" + return mapped_column( + DateTime(timezone=True), + nullable=False, + server_default=text("now()"), + ) + + +def optional_timestamp_column() -> MappedColumn[datetime | None]: + """Return a timezone-aware timestamp that is empty until it is set.""" + return mapped_column(DateTime(timezone=True), nullable=True) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/policy_reform.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/policy_reform.py index e1261160b..c2b130c9e 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/policy_reform.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/policy_reform.py @@ -1,10 +1,10 @@ """Read a stored policy, or dynamic, as the reform values a calculation applies. -The Public API stores a policy's parameter values against the parameters of +API v1 stores a policy's parameter values against the parameters of the catalog version it was saved on, each from a start date until an end date, or with no end date indefinitely. A calculation on any catalog version applies them by parameter name, as ``{parameter name: {period: value}}``, the -shape of the Public API's policy JSON that the household functions' contract +shape of API v1's policy JSON that the household functions' contract and the economy requests take. A dynamic's values are stored and read the same way. A simulation applies its policy's and its dynamic's values together, and is refused when both set the same parameter (``combine_reforms``). @@ -36,10 +36,10 @@ from sqlalchemy import select from policyengine_simulation_persistence.input_tables import ( - dynamics, - parameter_values, - parameters, - policies, + Dynamic, + Parameter, + ParameterValue, + Policy, ) from policyengine_simulation_persistence.tables import LATEST_YEAR @@ -48,7 +48,8 @@ from datetime import datetime from uuid import UUID - from sqlalchemy import ColumnElement, Connection + from sqlalchemy import ColumnElement + from sqlalchemy.orm import Session # The last day a value without an end date is applied. OPEN_ENDED_STOP: Final = date(LATEST_YEAR, 12, 31) @@ -60,11 +61,11 @@ class UnknownPolicyError(LookupError): - """The Public API has no such policy in the country.""" + """API v1 has no such policy in the country.""" class UnknownDynamicError(LookupError): - """The Public API has no such dynamic.""" + """API v1 has no such dynamic.""" class MissingPolicyParametersError(ValueError): @@ -127,7 +128,7 @@ def _period( def read_policy_reform( - connection: Connection, + session: Session, *, country: str, policy_id: UUID, @@ -140,24 +141,22 @@ def read_policy_reform( policy, and MissingPolicyParametersError when ``catalog_version_id`` lacks a parameter the policy sets. """ - found = connection.execute( - select(policies.c.id).where( - policies.c.id == policy_id, policies.c.country_id == country - ) + found = session.execute( + select(Policy.id).where(Policy.id == policy_id, Policy.country_id == country) ).scalar_one_or_none() if found is None: msg = f"{country} has no policy {policy_id}" raise UnknownPolicyError(msg) return _read_reform( - connection, - owner=parameter_values.c.policy_id == policy_id, + session, + owner=ParameterValue.policy_id == policy_id, catalog_version_id=catalog_version_id, source="policy", ) def read_dynamic_reform( - connection: Connection, *, dynamic_id: UUID, catalog_version_id: UUID + session: Session, *, dynamic_id: UUID, catalog_version_id: UUID ) -> PolicyReform: """Return the values the dynamic sets, by parameter name and period. @@ -166,15 +165,15 @@ def read_dynamic_reform( MissingPolicyParametersError, naming the dynamic as its source, when ``catalog_version_id`` lacks a parameter it sets. """ - found = connection.execute( - select(dynamics.c.id).where(dynamics.c.id == dynamic_id) + found = session.execute( + select(Dynamic.id).where(Dynamic.id == dynamic_id) ).scalar_one_or_none() if found is None: msg = f"there is no dynamic {dynamic_id}" raise UnknownDynamicError(msg) return _read_reform( - connection, - owner=parameter_values.c.dynamic_id == dynamic_id, + session, + owner=ParameterValue.dynamic_id == dynamic_id, catalog_version_id=catalog_version_id, source="dynamic", ) @@ -192,7 +191,7 @@ def combine_reforms(policy: PolicyReform, dynamic: PolicyReform) -> PolicyReform def _read_reform( - connection: Connection, + session: Session, *, owner: ColumnElement[bool], catalog_version_id: UUID, @@ -203,23 +202,24 @@ def _read_reform( Raise MissingPolicyParametersError, naming ``source``, when ``catalog_version_id`` lacks a parameter they set. """ - values = connection.execute( + values = session.execute( select( - parameters.c.name, - parameter_values.c.value_json, - parameter_values.c.start_date, - parameter_values.c.end_date, + Parameter.name, + ParameterValue.value_json, + ParameterValue.start_date, + ParameterValue.end_date, ) - .join(parameters, parameters.c.id == parameter_values.c.parameter_id) + .select_from(ParameterValue) + .join(Parameter, Parameter.id == ParameterValue.parameter_id) .where(owner) - .order_by(parameters.c.name, parameter_values.c.start_date) + .order_by(Parameter.name, ParameterValue.start_date) ).all() names = sorted({row.name for row in values}) available = set( - connection.execute( - select(parameters.c.name).where( - parameters.c.tax_benefit_model_version_id == catalog_version_id, - parameters.c.name.in_(names), + session.execute( + select(Parameter.name).where( + Parameter.tax_benefit_model_version_id == catalog_version_id, + Parameter.name.in_(names), ) ).scalars() ) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/referenced_tables.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/referenced_tables.py new file mode 100644 index 000000000..14f44eac7 --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/referenced_tables.py @@ -0,0 +1,98 @@ +"""API v1's own tables that the simulation service's rows name, by ``id`` alone. + +Some keys of the service's tables name rows API v1 writes: users, policies, +dynamics, households, datasets, regions and the model catalog. PostgreSQL checks +each of those keys when a row is inserted, as the referenced table's owner, so +the service's role needs no privilege for it, and a row naming one that doesn't +exist fails with SQLSTATE 23503. These classes give SQLAlchemy each key's +target. They aren't models of those tables: nothing reads or writes them here. +""" + +from __future__ import annotations + +from typing import ClassVar +from uuid import UUID # noqa: TC003 # SQLAlchemy reads Mapped annotations at run time + +from sqlalchemy import PrimaryKeyConstraint, Uuid +from sqlalchemy.orm import Mapped, mapped_column + +from policyengine_simulation_persistence.orm_base import ReferencedBase + + +class ReferencedDataset(ReferencedBase): + """The ``id`` of a row of ``datasets``.""" + + __tablename__ = "datasets" + __table_args__: ClassVar = (PrimaryKeyConstraint("id", name="pk_datasets"),) + + id: Mapped[UUID] = mapped_column(Uuid) + + +class ReferencedDynamic(ReferencedBase): + """The ``id`` of a row of ``dynamics``.""" + + __tablename__ = "dynamics" + __table_args__: ClassVar = (PrimaryKeyConstraint("id", name="pk_dynamics"),) + + id: Mapped[UUID] = mapped_column(Uuid) + + +class ReferencedHousehold(ReferencedBase): + """The ``id`` of a row of ``households``.""" + + __tablename__ = "households" + __table_args__: ClassVar = (PrimaryKeyConstraint("id", name="pk_households"),) + + id: Mapped[UUID] = mapped_column(Uuid) + + +class ReferencedPolicy(ReferencedBase): + """The ``id`` of a row of ``policies``.""" + + __tablename__ = "policies" + __table_args__: ClassVar = (PrimaryKeyConstraint("id", name="pk_policies"),) + + id: Mapped[UUID] = mapped_column(Uuid) + + +class ReferencedRegion(ReferencedBase): + """The ``id`` of a row of ``regions``.""" + + __tablename__ = "regions" + __table_args__: ClassVar = (PrimaryKeyConstraint("id", name="pk_regions"),) + + id: Mapped[UUID] = mapped_column(Uuid) + + +class ReferencedTaxBenefitModelVersion(ReferencedBase): + """The ``id`` of a row of ``tax_benefit_model_versions``.""" + + __tablename__ = "tax_benefit_model_versions" + __table_args__: ClassVar = ( + PrimaryKeyConstraint("id", name="pk_tax_benefit_model_versions"), + ) + + id: Mapped[UUID] = mapped_column(Uuid) + + +class ReferencedTaxBenefitModel(ReferencedBase): + """The ``id`` of a row of ``tax_benefit_models``.""" + + __tablename__ = "tax_benefit_models" + __table_args__: ClassVar = ( + PrimaryKeyConstraint("id", name="pk_tax_benefit_models"), + ) + + id: Mapped[UUID] = mapped_column(Uuid) + + +class ReferencedUser(ReferencedBase): + """The ``id`` of a row of ``users``.""" + + __tablename__ = "users" + __table_args__: ClassVar = (PrimaryKeyConstraint("id", name="pk_users"),) + + id: Mapped[UUID] = mapped_column(Uuid) + + +referenced_metadata = ReferencedBase.metadata diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/rerun_definitions.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/rerun_definitions.py index af0f332c4..113057b60 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/rerun_definitions.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/rerun_definitions.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any, Literal from sqlalchemy import select +from sqlalchemy.orm import aliased from policyengine_simulation_persistence.economy_run_records import ( ECONOMY_REPORT_TYPE, @@ -22,24 +23,25 @@ ) from policyengine_simulation_persistence.input_tables import ( MODEL_NAME_PREFIX, - datasets, - regions, - tax_benefit_model_versions, - tax_benefit_models, + Dataset, + Region, + TaxBenefitModel, + TaxBenefitModelVersion, ) from policyengine_simulation_persistence.tables import ( + BudgetWindowRun, + Report, + ReportRun, + Simulation, + SimulationRun, SimulationType, - budget_window_runs, - report_runs, - reports, - simulation_runs, - simulations, ) if TYPE_CHECKING: from uuid import UUID - from sqlalchemy import Connection, RowMapping + from sqlalchemy import RowMapping + from sqlalchemy.orm import Session # What a stored report or simulation calculates, as a re-run tells them apart: # a single-year economy report or simulation, a budget window, a household @@ -118,39 +120,39 @@ class SimulationDefinition: def read_report_definition( - connection: Connection, report_id: UUID + session: Session, report_id: UUID ) -> ReportDefinition | None: """Return what the stored report calculates, or None when there is none.""" - region = regions.alias("report_region") - dataset = datasets.alias("report_dataset") - baseline = simulations.alias("baseline_simulation") - reform = simulations.alias("reform_simulation") + region = aliased(Region, name="report_region") + dataset = aliased(Dataset, name="report_dataset") + baseline = aliased(Simulation, name="baseline_simulation") + reform = aliased(Simulation, name="reform_simulation") row = ( - connection.execute( + session.execute( select( - reports.c.id, - reports.c.country, - reports.c.type, - reports.c.household_id, - reports.c.budget_window_size, - reports.c.year, - reports.c.inputs, - reports.c.key_version, - region.c.code.label("region_code"), - dataset.c.name.label("dataset_name"), - baseline.c.id.label("baseline_id"), - baseline.c.policy_id.label("baseline_policy_id"), - baseline.c.dynamic_id.label("baseline_dynamic_id"), - reform.c.id.label("reform_id"), - reform.c.policy_id.label("reform_policy_id"), - reform.c.dynamic_id.label("reform_dynamic_id"), + Report.id, + Report.country, + Report.type, + Report.household_id, + Report.budget_window_size, + Report.year, + Report.inputs, + Report.key_version, + region.code.label("region_code"), + dataset.name.label("dataset_name"), + baseline.id.label("baseline_id"), + baseline.policy_id.label("baseline_policy_id"), + baseline.dynamic_id.label("baseline_dynamic_id"), + reform.id.label("reform_id"), + reform.policy_id.label("reform_policy_id"), + reform.dynamic_id.label("reform_dynamic_id"), ) - .select_from(reports) - .outerjoin(region, region.c.id == reports.c.region_id) - .outerjoin(dataset, dataset.c.id == reports.c.dataset_id) - .outerjoin(baseline, baseline.c.id == reports.c.baseline_simulation_id) - .outerjoin(reform, reform.c.id == reports.c.reform_simulation_id) - .where(reports.c.id == report_id) + .select_from(Report) + .outerjoin(region, region.id == Report.region_id) + .outerjoin(dataset, dataset.id == Report.dataset_id) + .outerjoin(baseline, baseline.id == Report.baseline_simulation_id) + .outerjoin(reform, reform.id == Report.reform_simulation_id) + .where(Report.id == report_id) ) .mappings() .one_or_none() @@ -170,7 +172,7 @@ def read_report_definition( reform=_side(row, "reform"), budget_window_size=row["budget_window_size"], window_years=( - _window_years(connection, report_id) if kind == "budget_window" else () + _window_years(session, report_id) if kind == "budget_window" else () ), household_id=row["household_id"], keyed=row["key_version"] is not None, @@ -178,35 +180,32 @@ def read_report_definition( def _window_years( - connection: Connection, report_id: UUID + session: Session, report_id: UUID ) -> tuple[WindowYearDefinition, ...]: """Return the years of the report's newest budget-window run, earliest first. A year whose report run names no reform is left out. """ newest_window_run = ( - select(budget_window_runs.c.id) - .where(budget_window_runs.c.report_id == report_id) - .order_by( - budget_window_runs.c.created_at.desc(), budget_window_runs.c.id.desc() - ) + select(BudgetWindowRun.id) + .where(BudgetWindowRun.report_id == report_id) + .order_by(BudgetWindowRun.created_at.desc(), BudgetWindowRun.id.desc()) .limit(1) .scalar_subquery() ) - baseline_run = simulation_runs.alias("baseline_run") - reform_run = simulation_runs.alias("reform_run") - rows = connection.execute( + baseline_run = aliased(SimulationRun, name="baseline_run") + reform_run = aliased(SimulationRun, name="reform_run") + rows = session.execute( select( - report_runs.c.year, - baseline_run.c.simulation_id.label("baseline_simulation_id"), - reform_run.c.simulation_id.label("reform_simulation_id"), - ) - .join( - baseline_run, baseline_run.c.id == report_runs.c.baseline_simulation_run_id + ReportRun.year, + baseline_run.simulation_id.label("baseline_simulation_id"), + reform_run.simulation_id.label("reform_simulation_id"), ) - .join(reform_run, reform_run.c.id == report_runs.c.reform_simulation_run_id) - .where(report_runs.c.budget_window_run_id == newest_window_run) - .order_by(report_runs.c.year) + .select_from(ReportRun) + .join(baseline_run, baseline_run.id == ReportRun.baseline_simulation_run_id) + .join(reform_run, reform_run.id == ReportRun.reform_simulation_run_id) + .where(ReportRun.budget_window_run_id == newest_window_run) + .order_by(ReportRun.year) ) return tuple( WindowYearDefinition( @@ -243,35 +242,34 @@ def _side( def read_simulation_definition( - connection: Connection, simulation_id: UUID + session: Session, simulation_id: UUID ) -> SimulationDefinition | None: """Return what the stored simulation calculates, or None when there is none.""" row = ( - connection.execute( + session.execute( select( - simulations.c.id, - simulations.c.simulation_type, - simulations.c.filter_field, - simulations.c.year, - simulations.c.policy_id, - simulations.c.dynamic_id, - tax_benefit_models.c.name.label("model_name"), - regions.c.code.label("region_code"), - datasets.c.name.label("dataset_name"), + Simulation.id, + Simulation.simulation_type, + Simulation.filter_field, + Simulation.year, + Simulation.policy_id, + Simulation.dynamic_id, + TaxBenefitModel.name.label("model_name"), + Region.code.label("region_code"), + Dataset.name.label("dataset_name"), ) - .select_from(simulations) + .select_from(Simulation) .join( - tax_benefit_model_versions, - tax_benefit_model_versions.c.id - == simulations.c.tax_benefit_model_version_id, + TaxBenefitModelVersion, + TaxBenefitModelVersion.id == Simulation.tax_benefit_model_version_id, ) .join( - tax_benefit_models, - tax_benefit_models.c.id == tax_benefit_model_versions.c.model_id, + TaxBenefitModel, + TaxBenefitModel.id == TaxBenefitModelVersion.model_id, ) - .outerjoin(regions, regions.c.id == simulations.c.region_id) - .outerjoin(datasets, datasets.c.id == simulations.c.dataset_id) - .where(simulations.c.id == simulation_id) + .outerjoin(Region, Region.id == Simulation.region_id) + .outerjoin(Dataset, Dataset.id == Simulation.dataset_id) + .where(Simulation.id == simulation_id) ) .mappings() .one_or_none() diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/result_tables.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/result_tables.py index e3fb9ccd4..7405e3847 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/result_tables.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/result_tables.py @@ -1,11 +1,10 @@ -"""Models of the Public API's result tables, which the Modal worker role writes. +"""Mapped classes of API v1's result tables, which the Modal worker role writes. -The Public API's v2 migration chain creates these tables; this module only -describes them, so the v2 Modal runtime can insert typed rows. Each table -matches what the chain builds, column for column, with its constraints under -the same names. They are in ``result_metadata``, apart from the tables the -simulation service creates rows in (``service_metadata``), whose run tables -their keys name. +API v1's v2 migration chain creates these tables; this module only describes +them, so the v2 Modal runtime can insert typed rows. Each class matches what +the chain builds, column for column, with its constraints under the same names. +They derive from ``ResultBase``, apart from the tables the simulation service +creates rows in (``ServiceBase``), whose run tables their keys name. Results that compare a report run's baseline and reform belong to the report run, and name its two simulation runs; poverty and inequality, which describe @@ -15,35 +14,31 @@ from __future__ import annotations +from datetime import datetime # noqa: TC003 # SQLAlchemy reads Mapped annotations from enum import StrEnum -from typing import TYPE_CHECKING, Final +from typing import ClassVar, Final +from uuid import UUID # noqa: TC003 # SQLAlchemy reads Mapped annotations at run time from sqlalchemy import ( Boolean, CheckConstraint, - Column, Float, ForeignKeyConstraint, Integer, - MetaData, PrimaryKeyConstraint, String, - Table, UniqueConstraint, Uuid, ) +from sqlalchemy.orm import Mapped, mapped_column -from policyengine_simulation_persistence.tables import ( - TABLE_SCHEMA, - _enum_type, - _identifier, - _timestamp, - report_runs, - simulation_runs, +from policyengine_simulation_persistence.orm_base import ( + ResultBase, + enum_type, + identifier_column, + server_timestamp_column, ) - -if TYPE_CHECKING: - from sqlalchemy.sql.schema import SchemaItem +from policyengine_simulation_persistence.tables import ReportRun, SimulationRun NAME_LENGTH: Final = 512 ENTITY_LENGTH: Final = 128 @@ -65,44 +60,43 @@ class UKAreaType(StrEnum): LOCAL_AUTHORITY = "local_authority" -result_metadata = MetaData(schema=TABLE_SCHEMA) +def _primary_key(table_name: str) -> PrimaryKeyConstraint: + return PrimaryKeyConstraint("id", name=f"pk_{table_name}") -def _result_columns(table_name: str) -> tuple[SchemaItem, ...]: - """Return every result's ``id`` and ``created_at``, and its primary key.""" - return ( - _identifier(), - _timestamp("created_at", server_now=True), - PrimaryKeyConstraint("id", name=f"pk_{table_name}"), +def _report_run_key(table_name: str) -> ForeignKeyConstraint: + """Return the key of a result to the report run it belongs to.""" + return ForeignKeyConstraint( + ["report_run_id"], + [ReportRun.id], + name=f"fk_{table_name}_report_run_id_report_runs", + ondelete="CASCADE", ) -def _of_report_run( +def _compared_run_keys( table_name: str, *, baseline_key_name: str | None = None, reform_key_name: str | None = None, -) -> tuple[SchemaItem, ...]: - """Return the keys of a result of a report run that compares its two runs. +) -> tuple[ForeignKeyConstraint, ForeignKeyConstraint]: + """Return the keys of a result to the two simulation runs it compares. - The key names are the Public API's, which its chain shortens past - PostgreSQL's 63 characters and ends in four characters of their hash; a - shortened name is passed in. + The key names are API v1's, which its chain shortens past PostgreSQL's 63 + characters and ends in four characters of their hash; a shortened name is + passed in. """ return ( - *_of_report_run_alone(table_name), - Column("baseline_simulation_run_id", Uuid, nullable=False), - Column("reform_simulation_run_id", Uuid, nullable=False), ForeignKeyConstraint( ["baseline_simulation_run_id"], - [simulation_runs.c.id], + [SimulationRun.id], name=baseline_key_name or f"fk_{table_name}_baseline_simulation_run_id_simulation_runs", ondelete="RESTRICT", ), ForeignKeyConstraint( ["reform_simulation_run_id"], - [simulation_runs.c.id], + [SimulationRun.id], name=reform_key_name or f"fk_{table_name}_reform_simulation_run_id_simulation_runs", ondelete="RESTRICT", @@ -110,346 +104,454 @@ def _of_report_run( ) -def _of_report_run_alone(table_name: str) -> tuple[SchemaItem, ...]: - """Return the key of a result of a report run that names no simulation run.""" - return ( - *_result_columns(table_name), - Column("report_run_id", Uuid, nullable=False), - ForeignKeyConstraint( - ["report_run_id"], - [report_runs.c.id], - name=f"fk_{table_name}_report_run_id_report_runs", - ondelete="CASCADE", - ), +def _simulation_run_key(table_name: str) -> ForeignKeyConstraint: + """Return the key of a result to the simulation run it belongs to.""" + return ForeignKeyConstraint( + ["simulation_run_id"], + [SimulationRun.id], + name=f"fk_{table_name}_simulation_run_id_simulation_runs", + ondelete="CASCADE", ) -def _of_simulation_run(table_name: str) -> tuple[SchemaItem, ...]: - """Return the key of a result of one simulation run.""" - return ( - *_result_columns(table_name), - Column("simulation_run_id", Uuid, nullable=False), - ForeignKeyConstraint( - ["simulation_run_id"], - [simulation_runs.c.id], - name=f"fk_{table_name}_simulation_run_id_simulation_runs", - ondelete="CASCADE", - ), +def _disclosure_tier_check(table_name: str) -> CheckConstraint: + """Allow a UK area's tier to be ``reported`` or ``approximate``, or empty. + + ``approximate`` marks a thin sample. The tier and the bounds of the area's + income changes, each change's 90% interval, are all null without disclosure + control. + """ + return CheckConstraint( + "disclosure_tier IS NULL OR disclosure_tier IN ('reported', 'approximate')", + name=f"ck_{table_name}_disclosure_tier", ) -def _values(*names: str, nullable: bool) -> tuple[Column[float], ...]: - return tuple(Column(name, Float, nullable=nullable) for name in names) +class LaborSupplyResponse(ResultBase): + """A report run's overall labour supply response.""" + __tablename__ = "labor_supply_responses" + __table_args__: ClassVar = ( + _primary_key("labor_supply_responses"), + _report_run_key("labor_supply_responses"), + UniqueConstraint("report_run_id", name="uq_labor_supply_responses_run"), + ) -def _area_values() -> tuple[Column[float], ...]: - """Return a geographic impact's household income changes and population.""" - return _values( - "average_household_income_change", - "relative_household_income_change", - "population", - nullable=False, + id: Mapped[UUID] = identifier_column() + created_at: Mapped[datetime] = server_timestamp_column() + report_run_id: Mapped[UUID] = mapped_column(Uuid) + income_effect_earnings_change: Mapped[float] = mapped_column(Float) + substitution_effect_earnings_change: Mapped[float] = mapped_column(Float) + total_earnings_change: Mapped[float] = mapped_column(Float) + revenue_change: Mapped[float] = mapped_column(Float) + relative_income_effect: Mapped[float] = mapped_column(Float) + relative_substitution_effect: Mapped[float] = mapped_column(Float) + baseline_weekly_hours: Mapped[float] = mapped_column(Float) + reform_weekly_hours: Mapped[float] = mapped_column(Float) + weekly_hours_change: Mapped[float] = mapped_column(Float) + weekly_hours_income_effect: Mapped[float] = mapped_column(Float) + weekly_hours_substitution_effect: Mapped[float] = mapped_column(Float) + + +class LaborSupplyDecileResponse(ResultBase): + """One decile's labour supply response in a report run.""" + + __tablename__ = "labor_supply_decile_responses" + __table_args__: ClassVar = ( + _primary_key("labor_supply_decile_responses"), + _report_run_key("labor_supply_decile_responses"), + UniqueConstraint( + "report_run_id", + "decile", + name="uq_labor_supply_decile_responses_run_decile", + ), ) + id: Mapped[UUID] = identifier_column() + created_at: Mapped[datetime] = server_timestamp_column() + report_run_id: Mapped[UUID] = mapped_column(Uuid) + decile: Mapped[int] = mapped_column(Integer) + average_income_effect: Mapped[float | None] = mapped_column(Float) + average_substitution_effect: Mapped[float | None] = mapped_column(Float) + relative_income_effect: Mapped[float | None] = mapped_column(Float) + relative_substitution_effect: Mapped[float | None] = mapped_column(Float) + + +class BudgetSummary(ResultBase): + """One variable's budget totals under a report run's baseline and reform.""" + + __tablename__ = "budget_summary" + __table_args__: ClassVar = ( + _primary_key("budget_summary"), + _report_run_key("budget_summary"), + *_compared_run_keys("budget_summary"), + UniqueConstraint( + "report_run_id", + "variable_name", + "entity", + name="uq_budget_summary_run_variable_entity", + ), + ) -def _disclosure(table_name: str) -> tuple[SchemaItem, ...]: - """Return a UK area's disclosure tier and the bounds of its income changes. + id: Mapped[UUID] = identifier_column() + created_at: Mapped[datetime] = server_timestamp_column() + report_run_id: Mapped[UUID] = mapped_column(Uuid) + baseline_simulation_run_id: Mapped[UUID] = mapped_column(Uuid) + reform_simulation_run_id: Mapped[UUID] = mapped_column(Uuid) + variable_name: Mapped[str] = mapped_column(String(NAME_LENGTH)) + entity: Mapped[str] = mapped_column(String(ENTITY_LENGTH)) + baseline_total: Mapped[float | None] = mapped_column(Float) + reform_total: Mapped[float | None] = mapped_column(Float) + change: Mapped[float | None] = mapped_column(Float) - The tier is ``reported``, or ``approximate`` for a thin sample; the bounds - are each income change's 90% interval. All are null without disclosure - control. + +class DecileImpact(ResultBase): + """One decile's change in a report run. + + ``decile`` runs from 1 to ``quantiles``; rows without a decile variable + count as equal. """ - return ( - Column("disclosure_tier", String(16), nullable=True), - *_values( - "average_household_income_change_lower", - "average_household_income_change_upper", - "relative_household_income_change_lower", - "relative_household_income_change_upper", - nullable=True, + + __tablename__ = "decile_impacts" + __table_args__: ClassVar = ( + _primary_key("decile_impacts"), + _report_run_key("decile_impacts"), + *_compared_run_keys("decile_impacts"), + UniqueConstraint( + "report_run_id", + "income_variable", + "decile_variable", + "entity", + "decile", + "quantiles", + name="uq_decile_impacts_run_measure", + postgresql_nulls_not_distinct=True, ), + CheckConstraint("quantiles > 0", name="ck_decile_impacts_quantiles"), CheckConstraint( - "disclosure_tier IS NULL OR disclosure_tier IN ('reported', 'approximate')", - name=f"ck_{table_name}_disclosure_tier", + "decile BETWEEN 1 AND quantiles", name="ck_decile_impacts_decile" ), ) + id: Mapped[UUID] = identifier_column() + created_at: Mapped[datetime] = server_timestamp_column() + report_run_id: Mapped[UUID] = mapped_column(Uuid) + baseline_simulation_run_id: Mapped[UUID] = mapped_column(Uuid) + reform_simulation_run_id: Mapped[UUID] = mapped_column(Uuid) + income_variable: Mapped[str] = mapped_column(String(NAME_LENGTH)) + decile_variable: Mapped[str | None] = mapped_column(String(NAME_LENGTH)) + entity: Mapped[str] = mapped_column(String(ENTITY_LENGTH)) + decile: Mapped[int] = mapped_column(Integer) + quantiles: Mapped[int] = mapped_column(Integer) + baseline_mean: Mapped[float | None] = mapped_column(Float) + reform_mean: Mapped[float | None] = mapped_column(Float) + absolute_change: Mapped[float | None] = mapped_column(Float) + relative_change: Mapped[float | None] = mapped_column(Float) + count_better_off: Mapped[float | None] = mapped_column(Float) + count_worse_off: Mapped[float | None] = mapped_column(Float) + count_no_change: Mapped[float | None] = mapped_column(Float) + + +class IntraDecileImpact(ResultBase): + """The shares of one decile that gain or lose, in a report run.""" + + __tablename__ = "intra_decile_impacts" + __table_args__: ClassVar = ( + _primary_key("intra_decile_impacts"), + _report_run_key("intra_decile_impacts"), + *_compared_run_keys( + "intra_decile_impacts", + baseline_key_name=( + "fk_intra_decile_impacts_baseline_simulation_run_id_simu_7cf0" + ), + reform_key_name=( + "fk_intra_decile_impacts_reform_simulation_run_id_simula_467e" + ), + ), + UniqueConstraint( + "report_run_id", + "decile_type", + "decile", + name="uq_intra_decile_impacts_run_type_decile", + ), + CheckConstraint( + "decile BETWEEN 0 AND 10", + name="ck_intra_decile_impacts_decile", + ), + ) -labor_supply_responses = Table( - "labor_supply_responses", - result_metadata, - *_of_report_run_alone("labor_supply_responses"), - *_values( - "income_effect_earnings_change", - "substitution_effect_earnings_change", - "total_earnings_change", - "revenue_change", - "relative_income_effect", - "relative_substitution_effect", - "baseline_weekly_hours", - "reform_weekly_hours", - "weekly_hours_change", - "weekly_hours_income_effect", - "weekly_hours_substitution_effect", - nullable=False, - ), - UniqueConstraint("report_run_id", name="uq_labor_supply_responses_run"), -) + id: Mapped[UUID] = identifier_column() + created_at: Mapped[datetime] = server_timestamp_column() + report_run_id: Mapped[UUID] = mapped_column(Uuid) + baseline_simulation_run_id: Mapped[UUID] = mapped_column(Uuid) + reform_simulation_run_id: Mapped[UUID] = mapped_column(Uuid) + decile_type: Mapped[DecileType] = mapped_column( + enum_type(DecileType, "v2_decile_type") + ) + decile: Mapped[int] = mapped_column(Integer) + lose_more_than_5pct: Mapped[float | None] = mapped_column(Float) + lose_less_than_5pct: Mapped[float | None] = mapped_column(Float) + no_change: Mapped[float | None] = mapped_column(Float) + gain_less_than_5pct: Mapped[float | None] = mapped_column(Float) + gain_more_than_5pct: Mapped[float | None] = mapped_column(Float) + + +class ProgramStatistics(ResultBase): + """One program's totals in a report run; rows without an entity count as equal.""" + + __tablename__ = "program_statistics" + __table_args__: ClassVar = ( + _primary_key("program_statistics"), + _report_run_key("program_statistics"), + *_compared_run_keys( + "program_statistics", + baseline_key_name=( + "fk_program_statistics_baseline_simulation_run_id_simula_faf9" + ), + ), + UniqueConstraint( + "report_run_id", + "program_name", + "entity", + name="uq_program_statistics_run_program_entity", + postgresql_nulls_not_distinct=True, + ), + ) -labor_supply_decile_responses = Table( - "labor_supply_decile_responses", - result_metadata, - *_of_report_run_alone("labor_supply_decile_responses"), - Column("decile", Integer, nullable=False), - *_values( - "average_income_effect", - "average_substitution_effect", - "relative_income_effect", - "relative_substitution_effect", - nullable=True, - ), - UniqueConstraint( - "report_run_id", - "decile", - name="uq_labor_supply_decile_responses_run_decile", - ), -) + id: Mapped[UUID] = identifier_column() + created_at: Mapped[datetime] = server_timestamp_column() + report_run_id: Mapped[UUID] = mapped_column(Uuid) + baseline_simulation_run_id: Mapped[UUID] = mapped_column(Uuid) + reform_simulation_run_id: Mapped[UUID] = mapped_column(Uuid) + program_name: Mapped[str] = mapped_column(String(NAME_LENGTH)) + entity: Mapped[str | None] = mapped_column(String(ENTITY_LENGTH)) + is_tax: Mapped[bool | None] = mapped_column(Boolean) + baseline_total: Mapped[float | None] = mapped_column(Float) + reform_total: Mapped[float | None] = mapped_column(Float) + change: Mapped[float | None] = mapped_column(Float) + baseline_count: Mapped[float | None] = mapped_column(Float) + reform_count: Mapped[float | None] = mapped_column(Float) + winners: Mapped[float | None] = mapped_column(Float) + losers: Mapped[float | None] = mapped_column(Float) + + +class CongressionalDistrictImpact(ResultBase): + """One US congressional district's change in a report run.""" + + __tablename__ = "congressional_district_impacts" + __table_args__: ClassVar = ( + _primary_key("congressional_district_impacts"), + _report_run_key("congressional_district_impacts"), + *_compared_run_keys( + "congressional_district_impacts", + baseline_key_name=( + "fk_congressional_district_impacts_baseline_simulation_r_9f79" + ), + reform_key_name=( + "fk_congressional_district_impacts_reform_simulation_run_80f5" + ), + ), + UniqueConstraint( + "report_run_id", + "district_geoid", + name="uq_congressional_district_impacts_run_geoid", + ), + ) -budget_summary = Table( - "budget_summary", - result_metadata, - *_of_report_run("budget_summary"), - Column("variable_name", String(NAME_LENGTH), nullable=False), - Column("entity", String(ENTITY_LENGTH), nullable=False), - *_values("baseline_total", "reform_total", "change", nullable=True), - UniqueConstraint( - "report_run_id", - "variable_name", - "entity", - name="uq_budget_summary_run_variable_entity", - ), -) + id: Mapped[UUID] = identifier_column() + created_at: Mapped[datetime] = server_timestamp_column() + report_run_id: Mapped[UUID] = mapped_column(Uuid) + baseline_simulation_run_id: Mapped[UUID] = mapped_column(Uuid) + reform_simulation_run_id: Mapped[UUID] = mapped_column(Uuid) + average_household_income_change: Mapped[float] = mapped_column(Float) + relative_household_income_change: Mapped[float] = mapped_column(Float) + population: Mapped[float] = mapped_column(Float) + district_geoid: Mapped[int] = mapped_column(Integer) + state_fips: Mapped[int] = mapped_column(Integer) + district_number: Mapped[int] = mapped_column(Integer) + winner_percentage: Mapped[float] = mapped_column(Float) + loser_percentage: Mapped[float] = mapped_column(Float) + no_change_percentage: Mapped[float] = mapped_column(Float) + + +class ConstituencyImpact(ResultBase): + """One UK constituency's change in a report run.""" + + __tablename__ = "constituency_impacts" + __table_args__: ClassVar = ( + _primary_key("constituency_impacts"), + _report_run_key("constituency_impacts"), + *_compared_run_keys( + "constituency_impacts", + baseline_key_name=( + "fk_constituency_impacts_baseline_simulation_run_id_simu_d2cd" + ), + reform_key_name=( + "fk_constituency_impacts_reform_simulation_run_id_simula_aaa1" + ), + ), + _disclosure_tier_check("constituency_impacts"), + UniqueConstraint( + "report_run_id", + "constituency_code", + name="uq_constituency_impacts_run_code", + ), + ) -# ``decile`` runs from 1 to ``quantiles``; rows without a decile variable -# count as equal. -decile_impacts = Table( - "decile_impacts", - result_metadata, - *_of_report_run("decile_impacts"), - Column("income_variable", String(NAME_LENGTH), nullable=False), - Column("decile_variable", String(NAME_LENGTH), nullable=True), - Column("entity", String(ENTITY_LENGTH), nullable=False), - Column("decile", Integer, nullable=False), - Column("quantiles", Integer, nullable=False), - *_values( - "baseline_mean", - "reform_mean", - "absolute_change", - "relative_change", - "count_better_off", - "count_worse_off", - "count_no_change", - nullable=True, - ), - UniqueConstraint( - "report_run_id", - "income_variable", - "decile_variable", - "entity", - "decile", - "quantiles", - name="uq_decile_impacts_run_measure", - postgresql_nulls_not_distinct=True, - ), - CheckConstraint("quantiles > 0", name="ck_decile_impacts_quantiles"), - CheckConstraint("decile BETWEEN 1 AND quantiles", name="ck_decile_impacts_decile"), -) + id: Mapped[UUID] = identifier_column() + created_at: Mapped[datetime] = server_timestamp_column() + report_run_id: Mapped[UUID] = mapped_column(Uuid) + baseline_simulation_run_id: Mapped[UUID] = mapped_column(Uuid) + reform_simulation_run_id: Mapped[UUID] = mapped_column(Uuid) + average_household_income_change: Mapped[float] = mapped_column(Float) + relative_household_income_change: Mapped[float] = mapped_column(Float) + population: Mapped[float] = mapped_column(Float) + disclosure_tier: Mapped[str | None] = mapped_column(String(16)) + average_household_income_change_lower: Mapped[float | None] = mapped_column(Float) + average_household_income_change_upper: Mapped[float | None] = mapped_column(Float) + relative_household_income_change_lower: Mapped[float | None] = mapped_column(Float) + relative_household_income_change_upper: Mapped[float | None] = mapped_column(Float) + constituency_code: Mapped[str] = mapped_column(String(AREA_CODE_LENGTH)) + constituency_name: Mapped[str] = mapped_column(String(AREA_NAME_LENGTH)) + + +class LocalAuthorityImpact(ResultBase): + """One UK local authority's change in a report run.""" + + __tablename__ = "local_authority_impacts" + __table_args__: ClassVar = ( + _primary_key("local_authority_impacts"), + _report_run_key("local_authority_impacts"), + *_compared_run_keys( + "local_authority_impacts", + baseline_key_name=( + "fk_local_authority_impacts_baseline_simulation_run_id_s_0d31" + ), + reform_key_name=( + "fk_local_authority_impacts_reform_simulation_run_id_sim_0dc2" + ), + ), + _disclosure_tier_check("local_authority_impacts"), + UniqueConstraint( + "report_run_id", + "local_authority_code", + name="uq_local_authority_impacts_run_code", + ), + ) -intra_decile_impacts = Table( - "intra_decile_impacts", - result_metadata, - *_of_report_run( - "intra_decile_impacts", - baseline_key_name="fk_intra_decile_impacts_baseline_simulation_run_id_simu_7cf0", - reform_key_name="fk_intra_decile_impacts_reform_simulation_run_id_simula_467e", - ), - Column( - "decile_type", - _enum_type(DecileType, "v2_decile_type"), - nullable=False, - ), - Column("decile", Integer, nullable=False), - *_values( - "lose_more_than_5pct", - "lose_less_than_5pct", - "no_change", - "gain_less_than_5pct", - "gain_more_than_5pct", - nullable=True, - ), - UniqueConstraint( - "report_run_id", - "decile_type", - "decile", - name="uq_intra_decile_impacts_run_type_decile", - ), - CheckConstraint( - "decile BETWEEN 0 AND 10", - name="ck_intra_decile_impacts_decile", - ), -) + id: Mapped[UUID] = identifier_column() + created_at: Mapped[datetime] = server_timestamp_column() + report_run_id: Mapped[UUID] = mapped_column(Uuid) + baseline_simulation_run_id: Mapped[UUID] = mapped_column(Uuid) + reform_simulation_run_id: Mapped[UUID] = mapped_column(Uuid) + average_household_income_change: Mapped[float] = mapped_column(Float) + relative_household_income_change: Mapped[float] = mapped_column(Float) + population: Mapped[float] = mapped_column(Float) + disclosure_tier: Mapped[str | None] = mapped_column(String(16)) + average_household_income_change_lower: Mapped[float | None] = mapped_column(Float) + average_household_income_change_upper: Mapped[float | None] = mapped_column(Float) + relative_household_income_change_lower: Mapped[float | None] = mapped_column(Float) + relative_household_income_change_upper: Mapped[float | None] = mapped_column(Float) + local_authority_code: Mapped[str] = mapped_column(String(AREA_CODE_LENGTH)) + local_authority_name: Mapped[str] = mapped_column(String(AREA_NAME_LENGTH)) + + +class WithheldAreaCode(ResultBase): + """A UK area a local-area run estimated but disclosure control withheld. + + There is one row per area; none means none were withheld. + """ -# Rows without an entity count as equal. -program_statistics = Table( - "program_statistics", - result_metadata, - *_of_report_run( - "program_statistics", - baseline_key_name="fk_program_statistics_baseline_simulation_run_id_simula_faf9", - ), - Column("program_name", String(NAME_LENGTH), nullable=False), - Column("entity", String(ENTITY_LENGTH), nullable=True), - Column("is_tax", Boolean, nullable=True), - *_values( - "baseline_total", - "reform_total", - "change", - "baseline_count", - "reform_count", - "winners", - "losers", - nullable=True, - ), - UniqueConstraint( - "report_run_id", - "program_name", - "entity", - name="uq_program_statistics_run_program_entity", - postgresql_nulls_not_distinct=True, - ), -) + __tablename__ = "withheld_area_codes" + __table_args__: ClassVar = ( + _primary_key("withheld_area_codes"), + _report_run_key("withheld_area_codes"), + *_compared_run_keys( + "withheld_area_codes", + baseline_key_name=( + "fk_withheld_area_codes_baseline_simulation_run_id_simul_a169" + ), + ), + UniqueConstraint( + "report_run_id", + "area_type", + "code", + name="uq_withheld_area_codes_run_type_code", + ), + ) -congressional_district_impacts = Table( - "congressional_district_impacts", - result_metadata, - *_of_report_run( - "congressional_district_impacts", - baseline_key_name="fk_congressional_district_impacts_baseline_simulation_r_9f79", - reform_key_name="fk_congressional_district_impacts_reform_simulation_run_80f5", - ), - *_area_values(), - Column("district_geoid", Integer, nullable=False), - Column("state_fips", Integer, nullable=False), - Column("district_number", Integer, nullable=False), - *_values( - "winner_percentage", - "loser_percentage", - "no_change_percentage", - nullable=False, - ), - UniqueConstraint( - "report_run_id", - "district_geoid", - name="uq_congressional_district_impacts_run_geoid", - ), -) + id: Mapped[UUID] = identifier_column() + created_at: Mapped[datetime] = server_timestamp_column() + report_run_id: Mapped[UUID] = mapped_column(Uuid) + baseline_simulation_run_id: Mapped[UUID] = mapped_column(Uuid) + reform_simulation_run_id: Mapped[UUID] = mapped_column(Uuid) + area_type: Mapped[UKAreaType] = mapped_column( + enum_type(UKAreaType, "v2_uk_area_type") + ) + code: Mapped[str] = mapped_column(String(AREA_CODE_LENGTH)) -constituency_impacts = Table( - "constituency_impacts", - result_metadata, - *_of_report_run( - "constituency_impacts", - baseline_key_name="fk_constituency_impacts_baseline_simulation_run_id_simu_d2cd", - reform_key_name="fk_constituency_impacts_reform_simulation_run_id_simula_aaa1", - ), - *_area_values(), - *_disclosure("constituency_impacts"), - Column("constituency_code", String(AREA_CODE_LENGTH), nullable=False), - Column("constituency_name", String(AREA_NAME_LENGTH), nullable=False), - UniqueConstraint( - "report_run_id", - "constituency_code", - name="uq_constituency_impacts_run_code", - ), -) -local_authority_impacts = Table( - "local_authority_impacts", - result_metadata, - *_of_report_run( - "local_authority_impacts", - baseline_key_name="fk_local_authority_impacts_baseline_simulation_run_id_s_0d31", - reform_key_name="fk_local_authority_impacts_reform_simulation_run_id_sim_0dc2", - ), - *_area_values(), - *_disclosure("local_authority_impacts"), - Column("local_authority_code", String(AREA_CODE_LENGTH), nullable=False), - Column("local_authority_name", String(AREA_NAME_LENGTH), nullable=False), - UniqueConstraint( - "report_run_id", - "local_authority_code", - name="uq_local_authority_impacts_run_code", - ), -) +class Poverty(ResultBase): + """One poverty measure of a simulation run. -# The codes of the UK areas a local-area run estimated but disclosure control -# withheld, one row per area; none means none were withheld. -withheld_area_codes = Table( - "withheld_area_codes", - result_metadata, - *_of_report_run( - "withheld_area_codes", - baseline_key_name="fk_withheld_area_codes_baseline_simulation_run_id_simul_a169", - ), - Column("area_type", _enum_type(UKAreaType, "v2_uk_area_type"), nullable=False), - Column("code", String(AREA_CODE_LENGTH), nullable=False), - UniqueConstraint( - "report_run_id", - "area_type", - "code", - name="uq_withheld_area_codes_run_type_code", - ), -) + Rows without a filter variable or group count as equal, so a run has one + unfiltered row per poverty type and entity. + """ + + __tablename__ = "poverty" + __table_args__: ClassVar = ( + _primary_key("poverty"), + _simulation_run_key("poverty"), + UniqueConstraint( + "simulation_run_id", + "poverty_type", + "entity", + "filter_variable", + "filter_group", + name="uq_poverty_simulation_run_measure", + postgresql_nulls_not_distinct=True, + ), + ) + + id: Mapped[UUID] = identifier_column() + created_at: Mapped[datetime] = server_timestamp_column() + simulation_run_id: Mapped[UUID] = mapped_column(Uuid) + poverty_type: Mapped[str] = mapped_column(String(ENTITY_LENGTH)) + entity: Mapped[str] = mapped_column(String(ENTITY_LENGTH)) + filter_variable: Mapped[str | None] = mapped_column(String(NAME_LENGTH)) + filter_group: Mapped[str | None] = mapped_column(String(64)) + headcount: Mapped[float | None] = mapped_column(Float) + total_population: Mapped[float | None] = mapped_column(Float) + rate: Mapped[float | None] = mapped_column(Float) + + +class Inequality(ResultBase): + """One income variable's inequality measures in a simulation run.""" + + __tablename__ = "inequality" + __table_args__: ClassVar = ( + _primary_key("inequality"), + _simulation_run_key("inequality"), + UniqueConstraint( + "simulation_run_id", + "income_variable", + "entity", + name="uq_inequality_simulation_run_measure", + ), + ) + + id: Mapped[UUID] = identifier_column() + created_at: Mapped[datetime] = server_timestamp_column() + simulation_run_id: Mapped[UUID] = mapped_column(Uuid) + income_variable: Mapped[str] = mapped_column(String(NAME_LENGTH)) + entity: Mapped[str] = mapped_column(String(ENTITY_LENGTH)) + gini: Mapped[float | None] = mapped_column(Float) + top_10_share: Mapped[float | None] = mapped_column(Float) + top_1_share: Mapped[float | None] = mapped_column(Float) + bottom_50_share: Mapped[float | None] = mapped_column(Float) -# Rows without a filter variable or group count as equal, so a run has one -# unfiltered row per poverty type and entity. -poverty = Table( - "poverty", - result_metadata, - *_of_simulation_run("poverty"), - Column("poverty_type", String(ENTITY_LENGTH), nullable=False), - Column("entity", String(ENTITY_LENGTH), nullable=False), - Column("filter_variable", String(NAME_LENGTH), nullable=True), - Column("filter_group", String(64), nullable=True), - *_values("headcount", "total_population", "rate", nullable=True), - UniqueConstraint( - "simulation_run_id", - "poverty_type", - "entity", - "filter_variable", - "filter_group", - name="uq_poverty_simulation_run_measure", - postgresql_nulls_not_distinct=True, - ), -) -inequality = Table( - "inequality", - result_metadata, - *_of_simulation_run("inequality"), - Column("income_variable", String(NAME_LENGTH), nullable=False), - Column("entity", String(ENTITY_LENGTH), nullable=False), - *_values( - "gini", - "top_10_share", - "top_1_share", - "bottom_50_share", - nullable=True, - ), - UniqueConstraint( - "simulation_run_id", - "income_variable", - "entity", - name="uq_inequality_simulation_run_measure", - ), +result_metadata = ResultBase.metadata +RESULT_TABLE_NAMES: Final = tuple( + sorted(mapper.class_.__tablename__ for mapper in ResultBase.registry.mappers) ) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/shared_rows.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/shared_rows.py index 7cf2ff7da..83133a357 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/shared_rows.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/shared_rows.py @@ -1,9 +1,9 @@ """Find a shared simulation, report or run, or insert it: one lookup each. Identical requests share simulations, reports and runs (C1 · 24). Each -lookup here is one ``INSERT … ON CONFLICT DO NOTHING`` against the Public -API's unique index for the row's definition or version set, followed, when -the insert did nothing, by a read of the row that won: +lookup here is one ``INSERT … ON CONFLICT DO NOTHING`` against API v1's +unique index for the row's definition or version set, followed, when the +insert did nothing, by a read of the row that won: - a simulation or a report by ``(key_version, definition_key)``; - a simulation run by simulation, year, catalog version, country package @@ -22,9 +22,11 @@ A forced re-run's runs are inserted directly, marked ``forced``: no index covers them, so they never conflict, never join and are never joined -(answer 3). Callers choose the transactions, and insert rows in one order -(definitions by key, runs by simulation ID), so two transactions never wait -for each other in opposite orders. +(answer 3). Callers choose the transactions by the ORM session they pass, and +insert rows in one order (definitions by key, runs by simulation ID), so two +transactions never wait for each other in opposite orders. Each lookup is one +statement on a mapped class, run at once, so it leaves the session nothing to +flush and no object loaded. """ from __future__ import annotations @@ -32,24 +34,30 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Final -from sqlalchemy import select +from sqlalchemy import Index, select from sqlalchemy.dialects.postgresql import insert as postgresql_insert from policyengine_simulation_persistence.tables import ( + BudgetWindowRun, + Report, + ReportRun, ReportRunStatus, + Simulation, + SimulationRun, SimulationStatus, - budget_window_runs, - report_runs, - reports, - simulation_runs, - simulations, ) if TYPE_CHECKING: from collections.abc import Mapping, Sequence from uuid import UUID - from sqlalchemy import ColumnElement, Connection, Row, Table + from sqlalchemy import ColumnElement, Row + from sqlalchemy.orm import InstrumentedAttribute, Session + +# The classes a lookup inserts into. +type SharedRowModel = type[ + Simulation | Report | SimulationRun | ReportRun | BudgetWindowRun +] # How many times a lookup inserts before giving up on a row that keeps # failing between its insert and its read. @@ -105,7 +113,7 @@ class FoundWindowRun: class _UniqueIndex: """A unique index a lookup inserts against: its columns and condition.""" - table: Table + model: SharedRowModel columns: tuple[str, ...] # The partial index's predicate, as the model declares it; None for a # unique constraint over the whole table. @@ -115,12 +123,16 @@ class _UniqueIndex: def _partial_index( - table: Table, name: str, read_conditions: tuple[ColumnElement[bool], ...] + model: SharedRowModel, name: str, read_conditions: tuple[ColumnElement[bool], ...] ) -> _UniqueIndex: """Return the model's partial unique index ``name``, with its own predicate.""" - (index,) = (index for index in table.indexes if index.name == name) + (index,) = ( + argument + for argument in model.__table_args__ + if isinstance(argument, Index) and argument.name == name + ) return _UniqueIndex( - table=table, + model=model, columns=tuple(column.name for column in index.columns), where=index.dialect_options["postgresql"]["where"], read_conditions=read_conditions, @@ -128,55 +140,51 @@ def _partial_index( _SIMULATION_DEFINITION: Final = _UniqueIndex( - table=simulations, columns=_DEFINITION_COLUMNS, where=None, read_conditions=() + model=Simulation, columns=_DEFINITION_COLUMNS, where=None, read_conditions=() ) _REPORT_DEFINITION: Final = _UniqueIndex( - table=reports, columns=_DEFINITION_COLUMNS, where=None, read_conditions=() + model=Report, columns=_DEFINITION_COLUMNS, where=None, read_conditions=() ) _JOINABLE_SIMULATION_RUN: Final = _partial_index( - simulation_runs, + SimulationRun, "uq_simulation_runs_unfailed_unforced_versions", ( - simulation_runs.c.status != SimulationStatus.FAILED, - simulation_runs.c.forced.is_(False), + SimulationRun.status != SimulationStatus.FAILED, + SimulationRun.forced.is_(False), ), ) _JOINABLE_REPORT_RUN: Final = _partial_index( - report_runs, + ReportRun, "uq_report_runs_unfailed_unforced_versions", ( - report_runs.c.budget_window_run_id.is_(None), - report_runs.c.status != ReportRunStatus.FAILED, - report_runs.c.forced.is_(False), + ReportRun.budget_window_run_id.is_(None), + ReportRun.status != ReportRunStatus.FAILED, + ReportRun.forced.is_(False), ), ) -def find_or_insert_simulation( - connection: Connection, values: Mapping[str, Any] -) -> FoundRow: +def find_or_insert_simulation(session: Session, values: Mapping[str, Any]) -> FoundRow: """Return the simulation with ``values``' key, inserting ``values`` if none. ``values`` is the whole row, its ``key_version`` and ``definition_key`` included. """ - row, inserted = _find_or_insert(connection, _SIMULATION_DEFINITION, values, ()) + row, inserted = _find_or_insert(session, _SIMULATION_DEFINITION, values, ()) return FoundRow(id=row.id, inserted=inserted) -def find_or_insert_report( - connection: Connection, values: Mapping[str, Any] -) -> FoundRow: +def find_or_insert_report(session: Session, values: Mapping[str, Any]) -> FoundRow: """Return the report with ``values``' key, inserting ``values`` if none. ``values`` is the whole row, its three key columns included. """ - row, inserted = _find_or_insert(connection, _REPORT_DEFINITION, values, ()) + row, inserted = _find_or_insert(session, _REPORT_DEFINITION, values, ()) return FoundRow(id=row.id, inserted=inserted) def find_or_insert_simulation_run( - connection: Connection, values: Mapping[str, Any] + session: Session, values: Mapping[str, Any] ) -> FoundSimulationRun: """Return the simulation's joinable run on ``values``' versions, or insert it. @@ -185,7 +193,7 @@ def find_or_insert_simulation_run( """ _refuse_forced(values) row, inserted = _find_or_insert( - connection, _JOINABLE_SIMULATION_RUN, values, (simulation_runs.c.status,) + session, _JOINABLE_SIMULATION_RUN, values, (SimulationRun.status,) ) return FoundSimulationRun( id=row.id, inserted=inserted, status=SimulationStatus(row.status) @@ -193,7 +201,7 @@ def find_or_insert_simulation_run( def find_or_insert_simulation_runs( - connection: Connection, rows: Mapping[UUID, Mapping[str, Any]] + session: Session, rows: Mapping[UUID, Mapping[str, Any]] ) -> dict[UUID, FoundSimulationRun]: """Return each simulation's joinable run, by simulation ID, or insert ``rows``. @@ -202,23 +210,23 @@ def find_or_insert_simulation_runs( for each other in opposite orders. """ return { - simulation_id: find_or_insert_simulation_run(connection, rows[simulation_id]) + simulation_id: find_or_insert_simulation_run(session, rows[simulation_id]) for simulation_id in sorted(rows) } _JOINABLE_WINDOW_RUN: Final = _partial_index( - budget_window_runs, + BudgetWindowRun, "uq_budget_window_runs_unfailed_unforced_versions", ( - budget_window_runs.c.status != ReportRunStatus.FAILED, - budget_window_runs.c.recalculate.is_(False), + BudgetWindowRun.status != ReportRunStatus.FAILED, + BudgetWindowRun.recalculate.is_(False), ), ) def find_or_insert_window_run( - connection: Connection, values: Mapping[str, Any] + session: Session, values: Mapping[str, Any] ) -> FoundWindowRun: """Return the report's joinable window run on ``values``' versions, or insert it. @@ -230,7 +238,7 @@ def find_or_insert_window_run( msg = "a recalculating window run is inserted directly: nothing joins it" raise ValueError(msg) row, inserted = _find_or_insert( - connection, _JOINABLE_WINDOW_RUN, values, (budget_window_runs.c.status,) + session, _JOINABLE_WINDOW_RUN, values, (BudgetWindowRun.status,) ) return FoundWindowRun( id=row.id, inserted=inserted, status=ReportRunStatus(row.status) @@ -238,7 +246,7 @@ def find_or_insert_window_run( def find_joinable_window_run( - connection: Connection, + session: Session, *, report_id: UUID, tax_benefit_model_version_id: UUID, @@ -249,12 +257,12 @@ def find_joinable_window_run( The one ``find_or_insert_window_run`` would join. A window run created before C1 · 27 records no .py version, so it is never joined. """ - row = connection.execute( - select(budget_window_runs.c.id, budget_window_runs.c.status).where( - budget_window_runs.c.report_id == report_id, - budget_window_runs.c.tax_benefit_model_version_id + row = session.execute( + select(BudgetWindowRun.id, BudgetWindowRun.status).where( + BudgetWindowRun.report_id == report_id, + BudgetWindowRun.tax_benefit_model_version_id == tax_benefit_model_version_id, - budget_window_runs.c.policyengine_version == policyengine_version, + BudgetWindowRun.policyengine_version == policyengine_version, *_JOINABLE_WINDOW_RUN.read_conditions, ) ).one_or_none() @@ -264,7 +272,7 @@ def find_joinable_window_run( def find_or_insert_report_run( - connection: Connection, values: Mapping[str, Any] + session: Session, values: Mapping[str, Any] ) -> FoundReportRun: """Return the report's joinable run on ``values``' versions, or insert it. @@ -277,13 +285,13 @@ def find_or_insert_report_run( msg = "a budget window's year runs are never looked up by versions" raise ValueError(msg) row, inserted = _find_or_insert( - connection, + session, _JOINABLE_REPORT_RUN, values, ( - report_runs.c.status, - report_runs.c.baseline_simulation_run_id, - report_runs.c.reform_simulation_run_id, + ReportRun.status, + ReportRun.baseline_simulation_run_id, + ReportRun.reform_simulation_run_id, ), ) return FoundReportRun( @@ -296,7 +304,7 @@ def find_or_insert_report_run( def find_joinable_report_run( - connection: Connection, + session: Session, *, report_id: UUID, country_package_version: str, @@ -307,16 +315,16 @@ def find_joinable_report_run( The run hasn't failed, wasn't forced, and is no budget window's year run: the one ``find_or_insert_report_run`` would join. """ - row = connection.execute( + row = session.execute( select( - report_runs.c.id, - report_runs.c.status, - report_runs.c.baseline_simulation_run_id, - report_runs.c.reform_simulation_run_id, + ReportRun.id, + ReportRun.status, + ReportRun.baseline_simulation_run_id, + ReportRun.reform_simulation_run_id, ).where( - report_runs.c.report_id == report_id, - report_runs.c.country_package_version == country_package_version, - report_runs.c.policyengine_version == policyengine_version, + ReportRun.report_id == report_id, + ReportRun.country_package_version == country_package_version, + ReportRun.policyengine_version == policyengine_version, *_JOINABLE_REPORT_RUN.read_conditions, ) ).one_or_none() @@ -338,10 +346,10 @@ def _refuse_forced(values: Mapping[str, Any]) -> None: def _find_or_insert( - connection: Connection, + session: Session, index: _UniqueIndex, values: Mapping[str, Any], - returned: Sequence[ColumnElement[Any]], + returned: Sequence[InstrumentedAttribute[Any]], ) -> tuple[Row[Any], bool]: """Insert ``values`` unless ``index`` holds a row like it; return the row. @@ -354,27 +362,27 @@ def _find_or_insert( if missing: msg = f"the row names no {', '.join(missing)}, which find it" raise ValueError(msg) - table = index.table - columns = (table.c.id, *returned) + model = index.model + columns = (model.id, *returned) statement = ( - postgresql_insert(table) + postgresql_insert(model) .values(dict(values)) .on_conflict_do_nothing(index_elements=index.columns, index_where=index.where) .returning(*columns) ) winner = select(*columns).where( - *(table.c[name] == values[name] for name in index.columns), + *(getattr(model, name) == values[name] for name in index.columns), *index.read_conditions, ) for _attempt in range(INSERT_ATTEMPTS): - inserted = connection.execute(statement).one_or_none() + inserted = session.execute(statement).one_or_none() if inserted is not None: return inserted, True - found = connection.execute(winner).one_or_none() + found = session.execute(winner).one_or_none() if found is not None: return found, False msg = ( - f"the {table.name} row that won kept failing before it was read, " + f"the {model.__tablename__} row that won kept failing before it was read, " f"{INSERT_ATTEMPTS} times" ) raise SharedRowNotSettledError(msg) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/store_transactions.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/store_transactions.py new file mode 100644 index 000000000..2b2b8966e --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/store_transactions.py @@ -0,0 +1,71 @@ +"""How the service's stores open sessions and tell a lost commit apart. + +Each store operation runs in an ORM session of its own, whose statements the +database bounds. A create's statements have all run by the time its function +returns, so a database error after that is the commit's: PostgreSQL may have +applied it before the connection failed, and the rows may exist. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sqlalchemy.exc import SQLAlchemyError + +from policyengine_simulation_persistence.engine import bounded_session + +if TYPE_CHECKING: + from collections.abc import Callable + from contextlib import AbstractContextManager + + from sqlalchemy.orm import Session, sessionmaker + +type TransactionFactory = Callable[[], AbstractContextManager[Session]] + + +class EconomyRecordOutcomeUnknownError(RuntimeError): + """A create's statements ran, but its commit failed: the rows may exist. + + PostgreSQL may have applied the commit before the connection failed, so + the caller must not answer that nothing was created. + """ + + +def bounded_transactions( + session_factory: sessionmaker[Session], *, statement_timeout_milliseconds: int +) -> TransactionFactory: + """Return a factory of sessions in transactions bounding each statement.""" + + def transaction() -> AbstractContextManager[Session]: + return bounded_session( + session_factory, + statement_timeout_milliseconds=statement_timeout_milliseconds, + ) + + return transaction + + +def create_in_transaction[Created]( + transaction: TransactionFactory, + insert_rows: Callable[[Session], Created], + *, + commit_failure: str, +) -> Created: + """Run ``insert_rows`` in a transaction, telling a failed commit apart. + + Return what it returns. It sends every row to the database before it + returns, so the commit has nothing to flush. An error from it leaves + nothing written and propagates. A database error once it has run is the + commit's: EconomyRecordOutcomeUnknownError, with ``commit_failure`` as its + message, caused by that error. + """ + inserted = False + try: + with transaction() as session: + created = insert_rows(session) + inserted = True + except SQLAlchemyError as error: + if not inserted: + raise + raise EconomyRecordOutcomeUnknownError(commit_failure) from error + return created diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/tables.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/tables.py index b06417d16..31823e2ba 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/tables.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/tables.py @@ -1,40 +1,29 @@ -"""Models of the Public API's tables the simulation service writes. - -The Public API's v2 migration chain creates and changes these tables; this -module only describes them, so the service can write typed rows. Each table -matches what the chain builds, column for column, with its constraints and -indexes under the same names, its foreign keys included. - -Some keys name rows of the Public API's own tables, which the Public API -writes: users, policies, dynamics, households, datasets, regions and the model -catalog. PostgreSQL checks each of those keys when a row is inserted, as the -referenced table's owner, so the service's role needs no privilege for it, -and a row naming one that doesn't exist fails with SQLSTATE 23503. Those -tables are modelled in ``referenced_metadata`` by their ``id`` alone, the -column every key names, so SQLAlchemy can declare the keys. +"""Models of API v1's tables the simulation service writes. + +API v1's v2 migration chain creates and changes these tables; this module only +describes them, as typed mapped classes, so the service can write typed rows. +Each class matches what the chain builds, column for column, with its +constraints and indexes under the same names, its foreign keys included. Keys +to rows API v1 itself writes name the classes in ``referenced_tables``. """ from __future__ import annotations +from datetime import datetime # noqa: TC003 # SQLAlchemy reads Mapped annotations from enum import StrEnum -from typing import TYPE_CHECKING, Final -from uuid import uuid4 +from typing import TYPE_CHECKING, Any, ClassVar, Final +from uuid import UUID # noqa: TC003 # SQLAlchemy reads Mapped annotations at run time from sqlalchemy import ( JSON, Boolean, CheckConstraint, - Column, - DateTime, - Enum, Float, ForeignKeyConstraint, Index, Integer, - MetaData, PrimaryKeyConstraint, String, - Table, Text, UniqueConstraint, Uuid, @@ -42,10 +31,29 @@ text, ) from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column + +from policyengine_simulation_persistence.orm_base import ( + TABLE_SCHEMA, + ServiceBase, + enum_type, + identifier_column, + optional_timestamp_column, + server_timestamp_column, +) +from policyengine_simulation_persistence.referenced_tables import ( + ReferencedDataset, + ReferencedDynamic, + ReferencedHousehold, + ReferencedPolicy, + ReferencedRegion, + ReferencedTaxBenefitModel, + ReferencedTaxBenefitModelVersion, + ReferencedUser, +) if TYPE_CHECKING: - from datetime import datetime - from uuid import UUID + from sqlalchemy.orm import InstrumentedAttribute, MappedColumn VERSION_LENGTH: Final = 128 # A report's budget window covers at most this many years. @@ -100,37 +108,7 @@ class HouseholdJobStatus(StrEnum): FAILED = "failed" -def _enum_type(values: type[StrEnum], name: str) -> Enum: - """Store each member's value, under the Postgres type the chain names. - - The type names its schema itself: SQLAlchemy 2.0 doesn't give an enum the - metadata's schema, so it would otherwise follow the role's search_path. - """ - return Enum( - values, - name=name, - schema=TABLE_SCHEMA, - values_callable=lambda members: [member.value for member in members], - ) - - -def _identifier() -> Column[UUID]: - return Column("id", Uuid, nullable=False, default=uuid4) - - -def _timestamp(name: str, *, server_now: bool = False) -> Column[datetime]: - """Return a timezone-aware timestamp column, defaulting to now() if asked.""" - if server_now: - return Column( - name, - DateTime(timezone=True), - nullable=False, - server_default=text("now()"), - ) - return Column(name, DateTime(timezone=True), nullable=True) - - -def _terminal_completion(table_name: str) -> CheckConstraint: +def _completion_required(table_name: str) -> CheckConstraint: """Require a succeeded or failed row to record when it completed.""" return CheckConstraint( "status NOT IN ('succeeded', 'failed') OR completed_at IS NOT NULL", @@ -138,23 +116,23 @@ def _terminal_completion(table_name: str) -> CheckConstraint: ) -def _key_columns(*names: str) -> tuple[Column[int] | Column[str], ...]: - """Return ``key_version`` and each named key column, nullable. +def _key_version_column() -> MappedColumn[int | None]: + """Return ``key_version``, empty on a row created before the keys existed. - A row created before the keys existed has none of them, and is never - found by key. + Such a row has none of the key columns, and is never found by key. """ - return ( - Column("key_version", Integer, nullable=True), - *(Column(name, String(DEFINITION_KEY_LENGTH), nullable=True) for name in names), - ) + return mapped_column(Integer, nullable=True) + + +def _key_column() -> MappedColumn[str | None]: + """Return a definition key column, empty on a row older than the keys.""" + return mapped_column(String(DEFINITION_KEY_LENGTH), nullable=True) def _key_checks(table_name: str, *names: str) -> tuple[CheckConstraint, ...]: """Require a positive key version, and each key in lowercase hexadecimal. - The checks are the Public API's, under its names; ``names`` are the key - columns. + The checks are API v1's, under its names; ``names`` are the key columns. """ return ( CheckConstraint("key_version > 0", name=f"ck_{table_name}_key_version"), @@ -169,9 +147,9 @@ def _key_checks(table_name: str, *names: str) -> tuple[CheckConstraint, ...]: ) -def _forced_column() -> Column[bool]: +def _forced_column() -> MappedColumn[bool]: """Return the marker of a forced re-run's run, which nothing joins.""" - return Column("forced", Boolean, nullable=False, server_default=false()) + return mapped_column(Boolean, nullable=False, server_default=false()) def _year_between(table_name: str, *, nullable: bool) -> CheckConstraint: @@ -181,577 +159,574 @@ def _year_between(table_name: str, *, nullable: bool) -> CheckConstraint: return CheckConstraint(condition, name=f"ck_{table_name}_year") -# Named explicitly, so writes reach the Public API's tables whatever the -# role's search_path. -TABLE_SCHEMA: Final = "public" -service_metadata = MetaData(schema=TABLE_SCHEMA) -# The Public API's own tables that the service's rows name, each by its id -# alone. They aren't models of those tables: nothing reads or writes them here. -referenced_metadata = MetaData(schema=TABLE_SCHEMA) -_PUBLIC_API_TABLES: Final = { - table_name: Table( - table_name, - referenced_metadata, - Column("id", Uuid, nullable=False), - PrimaryKeyConstraint("id", name=f"pk_{table_name}"), - ) - for table_name in ( - "datasets", - "dynamics", - "households", - "policies", - "regions", - "tax_benefit_model_versions", - "tax_benefit_models", - "users", - ) -} - - -def _public_api_key( - column_name: str, table_name: str, *, ondelete: str, name: str +def _api_v1_key( + column_name: str, + referenced_id: InstrumentedAttribute[UUID], + *, + ondelete: str, + name: str, ) -> ForeignKeyConstraint: - """Declare that ``column_name`` names a row of the Public API's ``table_name``. + """Declare that ``column_name`` names a row of one of API v1's own tables. - ``name`` is the key's name in the Public API's chain, which shortens a - name past PostgreSQL's 63 characters and ends it in four characters of - its hash. + ``name`` is the key's name in API v1's chain, which shortens a name past + PostgreSQL's 63 characters and ends it in four characters of its hash. """ return ForeignKeyConstraint( [column_name], - [_PUBLIC_API_TABLES[table_name].c.id], + [referenced_id], name=name, ondelete=ondelete, ) -simulations = Table( - "simulations", - service_metadata, - _identifier(), - _timestamp("created_at", server_now=True), - _timestamp("updated_at", server_now=True), - Column( - "simulation_type", - _enum_type(SimulationType, "v2_simulation_type"), - nullable=False, - ), - Column("dataset_id", Uuid, nullable=True), - Column("household_id", Uuid, nullable=True), - Column("policy_id", Uuid, nullable=True), - Column("dynamic_id", Uuid, nullable=True), - Column("tax_benefit_model_version_id", Uuid, nullable=False), - Column("region_id", Uuid, nullable=True), - Column("filter_field", String(128), nullable=True), - Column("filter_value", String(255), nullable=True), - Column("filter_strategy", String(64), nullable=True), - Column("year", Integer, nullable=True), - *_key_columns("definition_key"), - PrimaryKeyConstraint("id", name="pk_simulations"), - _public_api_key( - "dataset_id", - "datasets", - ondelete="RESTRICT", - name="fk_simulations_dataset_id_datasets", - ), - _public_api_key( - "household_id", - "households", - ondelete="RESTRICT", - name="fk_simulations_household_id_households", - ), - _public_api_key( - "policy_id", - "policies", - ondelete="SET NULL", - name="fk_simulations_policy_id_policies", - ), - _public_api_key( - "dynamic_id", - "dynamics", - ondelete="SET NULL", - name="fk_simulations_dynamic_id_dynamics", - ), - _public_api_key( - "tax_benefit_model_version_id", - "tax_benefit_model_versions", - ondelete="RESTRICT", - name="fk_simulations_tax_benefit_model_version_id_tax_benefit_84f8", - ), - _public_api_key( - "region_id", - "regions", - ondelete="SET NULL", - name="fk_simulations_region_id_regions", - ), - CheckConstraint( - "(filter_field IS NULL) = (filter_value IS NULL)", - name="ck_simulations_filter_pair", - ), - CheckConstraint( - "(simulation_type = 'household' AND household_id IS NOT NULL " - "AND dataset_id IS NULL) OR " - "(simulation_type = 'economy' AND dataset_id IS NOT NULL " - "AND household_id IS NULL)", - name="ck_simulations_type_input", - ), - _year_between("simulations", nullable=True), - *_key_checks("simulations", "definition_key"), - CheckConstraint( - "(key_version IS NULL) = (definition_key IS NULL)", - name="ck_simulations_keys_complete", - ), - # Find-or-create's conflict target: one simulation per definition. - UniqueConstraint( - "key_version", - "definition_key", - name="uq_simulations_key_version_definition_key", - ), - Index( - "ix_simulations_tax_benefit_model_version_id", - "tax_benefit_model_version_id", - ), -) +class Simulation(ServiceBase): + """What one simulation calculates; its calculations are ``SimulationRun`` rows.""" + + __tablename__ = "simulations" + __table_args__: ClassVar = ( + PrimaryKeyConstraint("id", name="pk_simulations"), + _api_v1_key( + "dataset_id", + ReferencedDataset.id, + ondelete="RESTRICT", + name="fk_simulations_dataset_id_datasets", + ), + _api_v1_key( + "household_id", + ReferencedHousehold.id, + ondelete="RESTRICT", + name="fk_simulations_household_id_households", + ), + _api_v1_key( + "policy_id", + ReferencedPolicy.id, + ondelete="SET NULL", + name="fk_simulations_policy_id_policies", + ), + _api_v1_key( + "dynamic_id", + ReferencedDynamic.id, + ondelete="SET NULL", + name="fk_simulations_dynamic_id_dynamics", + ), + _api_v1_key( + "tax_benefit_model_version_id", + ReferencedTaxBenefitModelVersion.id, + ondelete="RESTRICT", + name="fk_simulations_tax_benefit_model_version_id_tax_benefit_84f8", + ), + _api_v1_key( + "region_id", + ReferencedRegion.id, + ondelete="SET NULL", + name="fk_simulations_region_id_regions", + ), + CheckConstraint( + "(filter_field IS NULL) = (filter_value IS NULL)", + name="ck_simulations_filter_pair", + ), + CheckConstraint( + "(simulation_type = 'household' AND household_id IS NOT NULL " + "AND dataset_id IS NULL) OR " + "(simulation_type = 'economy' AND dataset_id IS NOT NULL " + "AND household_id IS NULL)", + name="ck_simulations_type_input", + ), + _year_between("simulations", nullable=True), + *_key_checks("simulations", "definition_key"), + CheckConstraint( + "(key_version IS NULL) = (definition_key IS NULL)", + name="ck_simulations_keys_complete", + ), + # Find-or-create's conflict target: one simulation per definition. + UniqueConstraint( + "key_version", + "definition_key", + name="uq_simulations_key_version_definition_key", + ), + Index( + "ix_simulations_tax_benefit_model_version_id", + "tax_benefit_model_version_id", + ), + ) -reports = Table( - "reports", - service_metadata, - _identifier(), - _timestamp("created_at", server_now=True), - _timestamp("updated_at", server_now=True), + id: Mapped[UUID] = identifier_column() + created_at: Mapped[datetime] = server_timestamp_column() + updated_at: Mapped[datetime] = server_timestamp_column() + simulation_type: Mapped[SimulationType] = mapped_column( + enum_type(SimulationType, "v2_simulation_type") + ) + dataset_id: Mapped[UUID | None] = mapped_column(Uuid) + household_id: Mapped[UUID | None] = mapped_column(Uuid) + policy_id: Mapped[UUID | None] = mapped_column(Uuid) + dynamic_id: Mapped[UUID | None] = mapped_column(Uuid) + tax_benefit_model_version_id: Mapped[UUID] = mapped_column(Uuid) + region_id: Mapped[UUID | None] = mapped_column(Uuid) + filter_field: Mapped[str | None] = mapped_column(String(128)) + filter_value: Mapped[str | None] = mapped_column(String(255)) + filter_strategy: Mapped[str | None] = mapped_column(String(64)) + year: Mapped[int | None] = mapped_column(Integer) + key_version: Mapped[int | None] = _key_version_column() + definition_key: Mapped[str | None] = _key_column() + + +class Report(ServiceBase): + """What one report compares; its calculations are ``ReportRun`` rows.""" + + __tablename__ = "reports" + __table_args__: ClassVar = ( + PrimaryKeyConstraint("id", name="pk_reports"), + _api_v1_key( + "user_id", + ReferencedUser.id, + ondelete="SET NULL", + name="fk_reports_user_id_users", + ), + _api_v1_key( + "tax_benefit_model_id", + ReferencedTaxBenefitModel.id, + ondelete="RESTRICT", + name="fk_reports_tax_benefit_model_id_tax_benefit_models", + ), + _api_v1_key( + "policy_id", + ReferencedPolicy.id, + ondelete="SET NULL", + name="fk_reports_policy_id_policies", + ), + _api_v1_key( + "household_id", + ReferencedHousehold.id, + ondelete="RESTRICT", + name="fk_reports_household_id_households", + ), + _api_v1_key( + "dataset_id", + ReferencedDataset.id, + ondelete="SET NULL", + name="fk_reports_dataset_id_datasets", + ), + _api_v1_key( + "region_id", + ReferencedRegion.id, + ondelete="SET NULL", + name="fk_reports_region_id_regions", + ), + ForeignKeyConstraint( + ["baseline_simulation_id"], + [f"{TABLE_SCHEMA}.simulations.id"], + name="fk_reports_baseline_simulation_id_simulations", + ondelete="SET NULL", + ), + ForeignKeyConstraint( + ["reform_simulation_id"], + [f"{TABLE_SCHEMA}.simulations.id"], + name="fk_reports_reform_simulation_id_simulations", + ondelete="SET NULL", + ), + CheckConstraint( + "budget_window_size IS NULL OR (household_id IS NULL AND " + f"budget_window_size BETWEEN 1 AND {MAX_BUDGET_WINDOW_YEARS} AND " + f"year IS NOT NULL AND year + budget_window_size - 1 <= {LATEST_YEAR})", + name="ck_reports_budget_window_size", + ), + _year_between("reports", nullable=True), + *_key_checks("reports", "definition_key", "dataset_independent_key"), + CheckConstraint( + "(key_version IS NULL) = (definition_key IS NULL) AND " + "(key_version IS NULL) = (dataset_independent_key IS NULL)", + name="ck_reports_keys_complete", + ), + # Find-or-create's conflict target: one report per definition. + UniqueConstraint( + "key_version", + "definition_key", + name="uq_reports_key_version_definition_key", + ), + # Reports that differ only in their dataset share this key, by which + # the app finds one from the other. + Index("ix_reports_dataset_independent_key", "dataset_independent_key"), + Index("ix_reports_country_type_created_at", "country", "type", "created_at"), + Index("ix_reports_tax_benefit_model_id", "tax_benefit_model_id"), + # Deleting a user sets its reports' user_id to null. + Index("ix_reports_user_id", "user_id"), + ) + + id: Mapped[UUID] = identifier_column() + created_at: Mapped[datetime] = server_timestamp_column() + updated_at: Mapped[datetime] = server_timestamp_column() # Per-user columns the service no longer writes (C1 · 24's answer 6): a # report's name and description live on each user's saved report. - Column("label", String(255), nullable=True), - Column("description", String, nullable=True), - Column("country", String(16), nullable=False), - Column("type", String(128), nullable=True), - Column("user_id", Uuid, nullable=True), - Column("tax_benefit_model_id", Uuid, nullable=False), - Column("policy_id", Uuid, nullable=True), - Column("baseline_simulation_id", Uuid, nullable=True), - Column("reform_simulation_id", Uuid, nullable=True), - Column("household_id", Uuid, nullable=True), - Column("dataset_id", Uuid, nullable=True), - Column("region_id", Uuid, nullable=True), - Column("year", Integer, nullable=True), - Column("inputs", JSON, nullable=False), + label: Mapped[str | None] = mapped_column(String(255)) + description: Mapped[str | None] = mapped_column(String) + country: Mapped[str] = mapped_column(String(16)) + type: Mapped[str | None] = mapped_column(String(128)) + user_id: Mapped[UUID | None] = mapped_column(Uuid) + tax_benefit_model_id: Mapped[UUID] = mapped_column(Uuid) + policy_id: Mapped[UUID | None] = mapped_column(Uuid) + baseline_simulation_id: Mapped[UUID | None] = mapped_column(Uuid) + reform_simulation_id: Mapped[UUID | None] = mapped_column(Uuid) + household_id: Mapped[UUID | None] = mapped_column(Uuid) + dataset_id: Mapped[UUID | None] = mapped_column(Uuid) + region_id: Mapped[UUID | None] = mapped_column(Uuid) + year: Mapped[int | None] = mapped_column(Integer) + inputs: Mapped[dict[str, Any]] = mapped_column(JSON) # The number of years from ``year`` an economy report's window covers. - Column("budget_window_size", Integer, nullable=True), - *_key_columns("definition_key", "dataset_independent_key"), - PrimaryKeyConstraint("id", name="pk_reports"), - _public_api_key( - "user_id", "users", ondelete="SET NULL", name="fk_reports_user_id_users" - ), - _public_api_key( - "tax_benefit_model_id", - "tax_benefit_models", - ondelete="RESTRICT", - name="fk_reports_tax_benefit_model_id_tax_benefit_models", - ), - _public_api_key( - "policy_id", - "policies", - ondelete="SET NULL", - name="fk_reports_policy_id_policies", - ), - _public_api_key( - "household_id", - "households", - ondelete="RESTRICT", - name="fk_reports_household_id_households", - ), - _public_api_key( - "dataset_id", - "datasets", - ondelete="SET NULL", - name="fk_reports_dataset_id_datasets", - ), - _public_api_key( - "region_id", - "regions", - ondelete="SET NULL", - name="fk_reports_region_id_regions", - ), - ForeignKeyConstraint( - ["baseline_simulation_id"], - [f"{TABLE_SCHEMA}.simulations.id"], - name="fk_reports_baseline_simulation_id_simulations", - ondelete="SET NULL", - ), - ForeignKeyConstraint( - ["reform_simulation_id"], - [f"{TABLE_SCHEMA}.simulations.id"], - name="fk_reports_reform_simulation_id_simulations", - ondelete="SET NULL", - ), - CheckConstraint( - "budget_window_size IS NULL OR (household_id IS NULL AND " - f"budget_window_size BETWEEN 1 AND {MAX_BUDGET_WINDOW_YEARS} AND " - f"year IS NOT NULL AND year + budget_window_size - 1 <= {LATEST_YEAR})", - name="ck_reports_budget_window_size", - ), - _year_between("reports", nullable=True), - *_key_checks("reports", "definition_key", "dataset_independent_key"), - CheckConstraint( - "(key_version IS NULL) = (definition_key IS NULL) AND " - "(key_version IS NULL) = (dataset_independent_key IS NULL)", - name="ck_reports_keys_complete", - ), - # Find-or-create's conflict target: one report per definition. - UniqueConstraint( - "key_version", - "definition_key", - name="uq_reports_key_version_definition_key", - ), - # Reports that differ only in their dataset share this key, by which - # the app finds one from the other. - Index("ix_reports_dataset_independent_key", "dataset_independent_key"), - Index("ix_reports_country_type_created_at", "country", "type", "created_at"), - Index("ix_reports_tax_benefit_model_id", "tax_benefit_model_id"), - # Deleting a user sets its reports' user_id to null. - Index("ix_reports_user_id", "user_id"), -) + budget_window_size: Mapped[int | None] = mapped_column(Integer) + key_version: Mapped[int | None] = _key_version_column() + definition_key: Mapped[str | None] = _key_column() + dataset_independent_key: Mapped[str | None] = _key_column() + + +class SimulationRun(ServiceBase): + """One calculation of a simulation for one year and set of versions.""" + + __tablename__ = "simulation_runs" + __table_args__: ClassVar = ( + PrimaryKeyConstraint("id", name="pk_simulation_runs"), + _api_v1_key( + "tax_benefit_model_version_id", + ReferencedTaxBenefitModelVersion.id, + ondelete="RESTRICT", + name="fk_simulation_runs_tax_benefit_model_version_id_tax_ben_1a94", + ), + _api_v1_key( + "dataset_id", + ReferencedDataset.id, + ondelete="RESTRICT", + name="fk_simulation_runs_dataset_id_datasets", + ), + _api_v1_key( + "region_id", + ReferencedRegion.id, + ondelete="SET NULL", + name="fk_simulation_runs_region_id_regions", + ), + ForeignKeyConstraint( + ["simulation_id"], + [f"{TABLE_SCHEMA}.simulations.id"], + name="fk_simulation_runs_simulation_id_simulations", + ondelete="CASCADE", + ), + CheckConstraint( + "(output_uri IS NULL) = (output_sha256 IS NULL)", + name="ck_simulation_runs_output_pair", + ), + CheckConstraint( + f"output_sha256 IS NULL OR length(output_sha256) = {SHA256_LENGTH}", + name="ck_simulation_runs_output_sha256_length", + ), + _completion_required("simulation_runs"), + _year_between("simulation_runs", nullable=False), + Index( + "ix_simulation_runs_current_output", + "simulation_id", + "year", + "status", + "country_package_version", + "policyengine_version", + "completed_at", + "id", + ), + Index("ix_simulation_runs_status_started_at", "status", "started_at"), + # One run that hasn't failed per simulation, year, catalog version and + # versions, apart from forced re-runs' runs: find-or-create's conflict + # target, so identical requests join one run. + Index( + "uq_simulation_runs_unfailed_unforced_versions", + "simulation_id", + "year", + "tax_benefit_model_version_id", + "country_package_version", + "policyengine_version", + unique=True, + postgresql_where=text("status <> 'failed' AND NOT forced"), + ), + Index( + "ix_simulation_runs_tax_benefit_model_version_id", + "tax_benefit_model_version_id", + ), + ) -simulation_runs = Table( - "simulation_runs", - service_metadata, - _identifier(), - _timestamp("created_at", server_now=True), - _timestamp("updated_at", server_now=True), - Column("simulation_id", Uuid, nullable=False), + id: Mapped[UUID] = identifier_column() + created_at: Mapped[datetime] = server_timestamp_column() + updated_at: Mapped[datetime] = server_timestamp_column() + simulation_id: Mapped[UUID] = mapped_column(Uuid) # The year it simulates; a budget window runs a simulation once per year. - Column("year", Integer, nullable=False), - Column("tax_benefit_model_version_id", Uuid, nullable=False), - Column("country_package_version", String(VERSION_LENGTH), nullable=False), - Column("policyengine_version", String(VERSION_LENGTH), nullable=False), - Column("dataset_id", Uuid, nullable=True), - Column("region_id", Uuid, nullable=True), - Column( - "status", - _enum_type(SimulationStatus, "v2_simulation_status"), - nullable=False, - ), - Column("modal_call_id", String(MODAL_CALL_ID_LENGTH), nullable=True), - Column("error_message", String, nullable=True), - _timestamp("started_at"), - _timestamp("completed_at"), + year: Mapped[int] = mapped_column(Integer) + tax_benefit_model_version_id: Mapped[UUID] = mapped_column(Uuid) + country_package_version: Mapped[str] = mapped_column(String(VERSION_LENGTH)) + policyengine_version: Mapped[str] = mapped_column(String(VERSION_LENGTH)) + dataset_id: Mapped[UUID | None] = mapped_column(Uuid) + region_id: Mapped[UUID | None] = mapped_column(Uuid) + status: Mapped[SimulationStatus] = mapped_column( + enum_type(SimulationStatus, "v2_simulation_status") + ) + modal_call_id: Mapped[str | None] = mapped_column(String(MODAL_CALL_ID_LENGTH)) + error_message: Mapped[str | None] = mapped_column(String) + started_at: Mapped[datetime | None] = optional_timestamp_column() + completed_at: Mapped[datetime | None] = optional_timestamp_column() # The Modal worker writes None as SQL NULL here and in the two columns - # below (household_jobs.result, report_runs.spm_provenance), so a query - # for IS NULL finds an empty result. - Column("household_result", JSON(none_as_null=True), nullable=True), - Column("output_uri", String(OUTPUT_URI_LENGTH), nullable=True), - Column("output_sha256", String(SHA256_LENGTH), nullable=True), - _forced_column(), - PrimaryKeyConstraint("id", name="pk_simulation_runs"), - _public_api_key( - "tax_benefit_model_version_id", - "tax_benefit_model_versions", - ondelete="RESTRICT", - name="fk_simulation_runs_tax_benefit_model_version_id_tax_ben_1a94", - ), - _public_api_key( - "dataset_id", - "datasets", - ondelete="RESTRICT", - name="fk_simulation_runs_dataset_id_datasets", - ), - _public_api_key( - "region_id", - "regions", - ondelete="SET NULL", - name="fk_simulation_runs_region_id_regions", - ), - ForeignKeyConstraint( - ["simulation_id"], - [f"{TABLE_SCHEMA}.simulations.id"], - name="fk_simulation_runs_simulation_id_simulations", - ondelete="CASCADE", - ), - CheckConstraint( - "(output_uri IS NULL) = (output_sha256 IS NULL)", - name="ck_simulation_runs_output_pair", - ), - CheckConstraint( - f"output_sha256 IS NULL OR length(output_sha256) = {SHA256_LENGTH}", - name="ck_simulation_runs_output_sha256_length", - ), - _terminal_completion("simulation_runs"), - _year_between("simulation_runs", nullable=False), - Index( - "ix_simulation_runs_current_output", - "simulation_id", - "year", - "status", - "country_package_version", - "policyengine_version", - "completed_at", - "id", - ), - Index("ix_simulation_runs_status_started_at", "status", "started_at"), - # One run that hasn't failed per simulation, year, catalog version and - # versions, apart from forced re-runs' runs: find-or-create's conflict - # target, so identical requests join one run. - Index( - "uq_simulation_runs_unfailed_unforced_versions", - "simulation_id", - "year", - "tax_benefit_model_version_id", - "country_package_version", - "policyengine_version", - unique=True, - postgresql_where=text("status <> 'failed' AND NOT forced"), - ), - Index( - "ix_simulation_runs_tax_benefit_model_version_id", - "tax_benefit_model_version_id", - ), -) + # below (HouseholdJob.result, ReportRun.spm_provenance), so a query for + # IS NULL finds an empty result. + household_result: Mapped[dict[str, Any] | None] = mapped_column( + JSON(none_as_null=True) + ) + output_uri: Mapped[str | None] = mapped_column(String(OUTPUT_URI_LENGTH)) + output_sha256: Mapped[str | None] = mapped_column(String(SHA256_LENGTH)) + forced: Mapped[bool] = _forced_column() + + +class BudgetWindowRun(ServiceBase): + """One calculation of a report's budget window: a report run for each year.""" + + __tablename__ = "budget_window_runs" + __table_args__: ClassVar = ( + PrimaryKeyConstraint("id", name="pk_budget_window_runs"), + _api_v1_key( + "tax_benefit_model_version_id", + ReferencedTaxBenefitModelVersion.id, + ondelete="RESTRICT", + name="fk_budget_window_runs_tax_benefit_model_version_id_tax__5106", + ), + ForeignKeyConstraint( + ["report_id"], + [f"{TABLE_SCHEMA}.reports.id"], + name="fk_budget_window_runs_report_id_reports", + ondelete="CASCADE", + ), + # The target of report_runs' key, which keeps a year run to its window + # run's report. + UniqueConstraint("id", "report_id", name="uq_budget_window_runs_id_report"), + _completion_required("budget_window_runs"), + Index( + "ix_budget_window_runs_report_id_created_at", + "report_id", + "created_at", + ), + Index( + "ix_budget_window_runs_tax_benefit_model_version_id", + "tax_benefit_model_version_id", + ), + # One window run that hasn't failed and wasn't forced per report and + # version set: find-or-create's conflict target for a window (answer 8). + Index( + "uq_budget_window_runs_unfailed_unforced_versions", + "report_id", + "tax_benefit_model_version_id", + "policyengine_version", + unique=True, + postgresql_where=text("status <> 'failed' AND NOT recalculate"), + ), + ) -# One calculation of a report's budget window: a report run for each year. -budget_window_runs = Table( - "budget_window_runs", - service_metadata, - _identifier(), - _timestamp("created_at", server_now=True), - _timestamp("updated_at", server_now=True), - Column("report_id", Uuid, nullable=False), - Column("tax_benefit_model_version_id", Uuid, nullable=False), + id: Mapped[UUID] = identifier_column() + created_at: Mapped[datetime] = server_timestamp_column() + updated_at: Mapped[datetime] = server_timestamp_column() + report_id: Mapped[UUID] = mapped_column(Uuid) + tax_benefit_model_version_id: Mapped[UUID] = mapped_column(Uuid) # The .py version the window calculates on, so a window run is found by # its report and version set (answer 8); null on runs before C1 · 27. - Column("policyengine_version", String(128), nullable=True), + policyengine_version: Mapped[str | None] = mapped_column(String(128)) # Per-request, so the service no longer writes it (answer 6). - Column( - "trigger", - _enum_type(ReportRunTrigger, "v2_report_run_trigger"), - nullable=True, - ), + trigger: Mapped[ReportRunTrigger | None] = mapped_column( + enum_type(ReportRunTrigger, "v2_report_run_trigger") + ) # The window's forced marker: a recalculating run is never joined. - Column("recalculate", Boolean, nullable=False), - Column( - "status", - _enum_type(ReportRunStatus, "v2_report_run_status"), - nullable=False, - ), - Column("error_message", String, nullable=True), - _timestamp("started_at"), - _timestamp("completed_at"), - PrimaryKeyConstraint("id", name="pk_budget_window_runs"), - _public_api_key( - "tax_benefit_model_version_id", - "tax_benefit_model_versions", - ondelete="RESTRICT", - name="fk_budget_window_runs_tax_benefit_model_version_id_tax__5106", - ), - ForeignKeyConstraint( - ["report_id"], - [f"{TABLE_SCHEMA}.reports.id"], - name="fk_budget_window_runs_report_id_reports", - ondelete="CASCADE", - ), - # The target of report_runs' key, which keeps a year run to its window - # run's report. - UniqueConstraint("id", "report_id", name="uq_budget_window_runs_id_report"), - _terminal_completion("budget_window_runs"), - Index( - "ix_budget_window_runs_report_id_created_at", - "report_id", - "created_at", - ), - Index( - "ix_budget_window_runs_tax_benefit_model_version_id", - "tax_benefit_model_version_id", - ), - # One window run that hasn't failed and wasn't forced per report and - # version set: find-or-create's conflict target for a window (answer 8). - Index( - "uq_budget_window_runs_unfailed_unforced_versions", - "report_id", - "tax_benefit_model_version_id", - "policyengine_version", - unique=True, - postgresql_where=text("status <> 'failed' AND NOT recalculate"), - ), -) + recalculate: Mapped[bool] = mapped_column(Boolean) + status: Mapped[ReportRunStatus] = mapped_column( + enum_type(ReportRunStatus, "v2_report_run_status") + ) + error_message: Mapped[str | None] = mapped_column(String) + started_at: Mapped[datetime | None] = optional_timestamp_column() + completed_at: Mapped[datetime | None] = optional_timestamp_column() + + +class ReportRun(ServiceBase): + """One calculation of a report, comparing a pair of simulation runs.""" + + __tablename__ = "report_runs" + __table_args__: ClassVar = ( + PrimaryKeyConstraint("id", name="pk_report_runs"), + ForeignKeyConstraint( + ["report_id"], + [f"{TABLE_SCHEMA}.reports.id"], + name="fk_report_runs_report_id_reports", + ondelete="CASCADE", + ), + ForeignKeyConstraint( + ["baseline_simulation_run_id"], + [f"{TABLE_SCHEMA}.simulation_runs.id"], + name="fk_report_runs_baseline_simulation_run_id_simulation_runs", + ondelete="RESTRICT", + ), + ForeignKeyConstraint( + ["reform_simulation_run_id"], + [f"{TABLE_SCHEMA}.simulation_runs.id"], + name="fk_report_runs_reform_simulation_run_id_simulation_runs", + ondelete="RESTRICT", + ), + # The window run belongs to the same report, and deleting it deletes + # its year runs. + ForeignKeyConstraint( + ["budget_window_run_id", "report_id"], + [ + f"{TABLE_SCHEMA}.budget_window_runs.id", + f"{TABLE_SCHEMA}.budget_window_runs.report_id", + ], + name="fk_report_runs_budget_window_run_id_budget_window_runs", + ondelete="CASCADE", + ), + UniqueConstraint( + "report_id", + "idempotency_key", + name="uq_report_runs_report_idempotency_key", + ), + UniqueConstraint( + "budget_window_run_id", + "year", + name="uq_report_runs_budget_window_run_id_year", + ), + CheckConstraint( + "budget_window_run_id IS NULL OR year IS NOT NULL", + name="ck_report_runs_budget_window_year", + ), + CheckConstraint( + "(baseline_cliff_gap IS NULL) = (baseline_cliff_share IS NULL) AND " + "(baseline_cliff_gap IS NULL) = (reform_cliff_gap IS NULL) AND " + "(baseline_cliff_gap IS NULL) = (reform_cliff_share IS NULL)", + name="ck_report_runs_cliff_impact_complete", + ), + _completion_required("report_runs"), + _year_between("report_runs", nullable=True), + Index( + "ix_report_runs_current_output", + "report_id", + "status", + "country_package_version", + "policyengine_version", + "completed_at", + "id", + ), + Index("ix_report_runs_status_started_at", "status", "started_at"), + # One run that hasn't failed per report and versions, apart from forced + # re-runs' runs and a budget window's years: find-or-create's conflict + # target. + Index( + "uq_report_runs_unfailed_unforced_versions", + "report_id", + "country_package_version", + "policyengine_version", + unique=True, + postgresql_where=text( + "budget_window_run_id IS NULL AND status <> 'failed' AND NOT forced" + ), + ), + Index( + "ix_report_runs_baseline_simulation_run_id", + "baseline_simulation_run_id", + ), + Index("ix_report_runs_reform_simulation_run_id", "reform_simulation_run_id"), + ) -report_runs = Table( - "report_runs", - service_metadata, - _identifier(), - _timestamp("created_at", server_now=True), - _timestamp("updated_at", server_now=True), - Column("report_id", Uuid, nullable=False), - Column("country_package_version", String(VERSION_LENGTH), nullable=False), - Column("policyengine_version", String(VERSION_LENGTH), nullable=False), - Column( - "status", - _enum_type(ReportRunStatus, "v2_report_run_status"), - nullable=False, - ), + id: Mapped[UUID] = identifier_column() + created_at: Mapped[datetime] = server_timestamp_column() + updated_at: Mapped[datetime] = server_timestamp_column() + report_id: Mapped[UUID] = mapped_column(Uuid) + country_package_version: Mapped[str] = mapped_column(String(VERSION_LENGTH)) + policyengine_version: Mapped[str] = mapped_column(String(VERSION_LENGTH)) + status: Mapped[ReportRunStatus] = mapped_column( + enum_type(ReportRunStatus, "v2_report_run_status") + ) # Per-request, so the service no longer writes it (answer 6). - Column( - "trigger", - _enum_type(ReportRunTrigger, "v2_report_run_trigger"), - nullable=True, - ), - Column("idempotency_key", Uuid, nullable=True), - _timestamp("started_at"), - _timestamp("completed_at"), - Column("error_message", String, nullable=True), - Column("markdown", Text, nullable=True), + trigger: Mapped[ReportRunTrigger | None] = mapped_column( + enum_type(ReportRunTrigger, "v2_report_run_trigger") + ) + idempotency_key: Mapped[UUID | None] = mapped_column(Uuid) + started_at: Mapped[datetime | None] = optional_timestamp_column() + completed_at: Mapped[datetime | None] = optional_timestamp_column() + error_message: Mapped[str | None] = mapped_column(String) + markdown: Mapped[str | None] = mapped_column(Text) # The pair it compares; a household report with only a baseline has no # reform run. - Column("baseline_simulation_run_id", Uuid, nullable=False), - Column("reform_simulation_run_id", Uuid, nullable=True), + baseline_simulation_run_id: Mapped[UUID] = mapped_column(Uuid) + reform_simulation_run_id: Mapped[UUID | None] = mapped_column(Uuid) # Set on each year's run of a budget window, with that year. - Column("budget_window_run_id", Uuid, nullable=True), - Column("year", Integer, nullable=True), - Column("modal_call_id", String(MODAL_CALL_ID_LENGTH), nullable=True), + budget_window_run_id: Mapped[UUID | None] = mapped_column(Uuid) + year: Mapped[int | None] = mapped_column(Integer) + modal_call_id: Mapped[str | None] = mapped_column(String(MODAL_CALL_ID_LENGTH)) # Cliff impact: all four values or none. - Column("baseline_cliff_gap", Float, nullable=True), - Column("baseline_cliff_share", Float, nullable=True), - Column("reform_cliff_gap", Float, nullable=True), - Column("reform_cliff_share", Float, nullable=True), - Column("spm_provenance", JSONB(none_as_null=True), nullable=True), - Column("data_version", String(VERSION_LENGTH), nullable=True), - _forced_column(), - PrimaryKeyConstraint("id", name="pk_report_runs"), - ForeignKeyConstraint( - ["report_id"], - [f"{TABLE_SCHEMA}.reports.id"], - name="fk_report_runs_report_id_reports", - ondelete="CASCADE", - ), - ForeignKeyConstraint( - ["baseline_simulation_run_id"], - [f"{TABLE_SCHEMA}.simulation_runs.id"], - name="fk_report_runs_baseline_simulation_run_id_simulation_runs", - ondelete="RESTRICT", - ), - ForeignKeyConstraint( - ["reform_simulation_run_id"], - [f"{TABLE_SCHEMA}.simulation_runs.id"], - name="fk_report_runs_reform_simulation_run_id_simulation_runs", - ondelete="RESTRICT", - ), - # The window run belongs to the same report, and deleting it deletes its - # year runs. - ForeignKeyConstraint( - ["budget_window_run_id", "report_id"], - [ - f"{TABLE_SCHEMA}.budget_window_runs.id", - f"{TABLE_SCHEMA}.budget_window_runs.report_id", - ], - name="fk_report_runs_budget_window_run_id_budget_window_runs", - ondelete="CASCADE", - ), - UniqueConstraint( - "report_id", - "idempotency_key", - name="uq_report_runs_report_idempotency_key", - ), - UniqueConstraint( - "budget_window_run_id", - "year", - name="uq_report_runs_budget_window_run_id_year", - ), - CheckConstraint( - "budget_window_run_id IS NULL OR year IS NOT NULL", - name="ck_report_runs_budget_window_year", - ), - CheckConstraint( - "(baseline_cliff_gap IS NULL) = (baseline_cliff_share IS NULL) AND " - "(baseline_cliff_gap IS NULL) = (reform_cliff_gap IS NULL) AND " - "(baseline_cliff_gap IS NULL) = (reform_cliff_share IS NULL)", - name="ck_report_runs_cliff_impact_complete", - ), - _terminal_completion("report_runs"), - _year_between("report_runs", nullable=True), - Index( - "ix_report_runs_current_output", - "report_id", - "status", - "country_package_version", - "policyengine_version", - "completed_at", - "id", - ), - Index("ix_report_runs_status_started_at", "status", "started_at"), - # One run that hasn't failed per report and versions, apart from forced - # re-runs' runs and a budget window's years: find-or-create's conflict - # target. - Index( - "uq_report_runs_unfailed_unforced_versions", - "report_id", - "country_package_version", - "policyengine_version", - unique=True, - postgresql_where=text( - "budget_window_run_id IS NULL AND status <> 'failed' AND NOT forced" - ), - ), - Index( - "ix_report_runs_baseline_simulation_run_id", - "baseline_simulation_run_id", - ), - Index("ix_report_runs_reform_simulation_run_id", "reform_simulation_run_id"), -) + baseline_cliff_gap: Mapped[float | None] = mapped_column(Float) + baseline_cliff_share: Mapped[float | None] = mapped_column(Float) + reform_cliff_gap: Mapped[float | None] = mapped_column(Float) + reform_cliff_share: Mapped[float | None] = mapped_column(Float) + spm_provenance: Mapped[dict[str, Any] | None] = mapped_column( + JSONB(none_as_null=True) + ) + data_version: Mapped[str | None] = mapped_column(String(VERSION_LENGTH)) + forced: Mapped[bool] = _forced_column() -household_jobs = Table( - "household_jobs", - service_metadata, - _identifier(), - _timestamp("created_at", server_now=True), - Column("country", String(16), nullable=False), - Column("request_data", JSON, nullable=False), - Column("policy_id", Uuid, nullable=True), - Column("dynamic_id", Uuid, nullable=True), - Column( - "status", - _enum_type(HouseholdJobStatus, "v2_household_job_status"), - nullable=False, - ), - Column("error_message", String, nullable=True), - Column("result", JSON(none_as_null=True), nullable=True), - _timestamp("started_at"), - _timestamp("completed_at"), - _timestamp("updated_at", server_now=True), - Column("household_id", Uuid, nullable=False), - Column("year", Integer, nullable=False), - Column("tax_benefit_model_version_id", Uuid, nullable=False), - Column("country_package_version", String(VERSION_LENGTH), nullable=False), - Column("policyengine_version", String(VERSION_LENGTH), nullable=False), - Column("modal_call_id", String(MODAL_CALL_ID_LENGTH), nullable=True), - PrimaryKeyConstraint("id", name="pk_household_jobs"), - _public_api_key( - "household_id", - "households", - ondelete="RESTRICT", - name="fk_household_jobs_household_id_households", - ), - _public_api_key( - "policy_id", - "policies", - ondelete="SET NULL", - name="fk_household_jobs_policy_id_policies", - ), - _public_api_key( - "dynamic_id", - "dynamics", - ondelete="SET NULL", - name="fk_household_jobs_dynamic_id_dynamics", - ), - _public_api_key( - "tax_benefit_model_version_id", - "tax_benefit_model_versions", - ondelete="RESTRICT", - name="fk_household_jobs_tax_benefit_model_version_id_tax_bene_82b2", - ), - _terminal_completion("household_jobs"), - _year_between("household_jobs", nullable=False), - Index("ix_household_jobs_household_id", "household_id"), - Index("ix_household_jobs_status_created_at", "status", "created_at"), - Index( - "ix_household_jobs_tax_benefit_model_version_id", - "tax_benefit_model_version_id", - ), -) + +class HouseholdJob(ServiceBase): + """One household calculation.""" + + __tablename__ = "household_jobs" + __table_args__: ClassVar = ( + PrimaryKeyConstraint("id", name="pk_household_jobs"), + _api_v1_key( + "household_id", + ReferencedHousehold.id, + ondelete="RESTRICT", + name="fk_household_jobs_household_id_households", + ), + _api_v1_key( + "policy_id", + ReferencedPolicy.id, + ondelete="SET NULL", + name="fk_household_jobs_policy_id_policies", + ), + _api_v1_key( + "dynamic_id", + ReferencedDynamic.id, + ondelete="SET NULL", + name="fk_household_jobs_dynamic_id_dynamics", + ), + _api_v1_key( + "tax_benefit_model_version_id", + ReferencedTaxBenefitModelVersion.id, + ondelete="RESTRICT", + name="fk_household_jobs_tax_benefit_model_version_id_tax_bene_82b2", + ), + _completion_required("household_jobs"), + _year_between("household_jobs", nullable=False), + Index("ix_household_jobs_household_id", "household_id"), + Index("ix_household_jobs_status_created_at", "status", "created_at"), + Index( + "ix_household_jobs_tax_benefit_model_version_id", + "tax_benefit_model_version_id", + ), + ) + + id: Mapped[UUID] = identifier_column() + created_at: Mapped[datetime] = server_timestamp_column() + country: Mapped[str] = mapped_column(String(16)) + request_data: Mapped[dict[str, Any]] = mapped_column(JSON) + policy_id: Mapped[UUID | None] = mapped_column(Uuid) + dynamic_id: Mapped[UUID | None] = mapped_column(Uuid) + status: Mapped[HouseholdJobStatus] = mapped_column( + enum_type(HouseholdJobStatus, "v2_household_job_status") + ) + error_message: Mapped[str | None] = mapped_column(String) + result: Mapped[dict[str, Any] | None] = mapped_column(JSON(none_as_null=True)) + started_at: Mapped[datetime | None] = optional_timestamp_column() + completed_at: Mapped[datetime | None] = optional_timestamp_column() + updated_at: Mapped[datetime] = server_timestamp_column() + household_id: Mapped[UUID] = mapped_column(Uuid) + year: Mapped[int] = mapped_column(Integer) + tax_benefit_model_version_id: Mapped[UUID] = mapped_column(Uuid) + country_package_version: Mapped[str] = mapped_column(String(VERSION_LENGTH)) + policyengine_version: Mapped[str] = mapped_column(String(VERSION_LENGTH)) + modal_call_id: Mapped[str | None] = mapped_column(String(MODAL_CALL_ID_LENGTH)) + + +service_metadata = ServiceBase.metadata diff --git a/libs/policyengine-simulation-persistence/tests/conftest.py b/libs/policyengine-simulation-persistence/tests/conftest.py index e9bfe6431..f0ba6f10e 100644 --- a/libs/policyengine-simulation-persistence/tests/conftest.py +++ b/libs/policyengine-simulation-persistence/tests/conftest.py @@ -26,12 +26,16 @@ import pytest from committed_rows import delete_catalog_rows from economy_rows import insert_economy_catalog -from policyengine_simulation_persistence import create_simulation_engine +from policyengine_simulation_persistence import ( + EconomyRunStore, + create_simulation_session_factory, +) from sqlalchemy import create_engine, text from sqlalchemy.engine import URL, make_url +from sqlalchemy.orm import Session, sessionmaker if TYPE_CHECKING: - from collections.abc import Iterator + from collections.abc import Callable, Iterator from economy_rows import EconomyCatalog from sqlalchemy import Connection, Engine @@ -47,6 +51,8 @@ # service; the deployed SIMULATION_SERVICE_DATABASE_URL logs in as it. SERVICE_ROLE = "policyengine_v2_simulation_service" MODAL_WORKER_ROLE = "policyengine_v2_modal_worker" +# The limit on each statement of a store a test builds on one connection. +STORE_STATEMENT_TIMEOUT_MILLISECONDS = 5_000 @dataclass(frozen=True) @@ -120,10 +126,15 @@ def public_api_database_url() -> str: return configured +def simulation_engine(database_url: str) -> Engine: + """Return the engine of the service's session factory, for test setup.""" + return create_simulation_session_factory(database_url).kw["bind"] + + @contextmanager def rolled_back_transaction(database_url: str) -> Iterator[Connection]: """Yield the superuser's connection in a transaction rolled back afterwards.""" - engine = create_simulation_engine(database_url) + engine = simulation_engine(database_url) try: with engine.connect() as connection: transaction = connection.begin() @@ -135,6 +146,44 @@ def rolled_back_transaction(database_url: str) -> Iterator[Connection]: engine.dispose() +def session_factory_on(connection: Connection) -> sessionmaker[Session]: + """Return a factory of sessions inside ``connection``'s transaction. + + Each session's transaction is a savepoint, so its commit releases the + savepoint and its rollback returns to it, and the test's own transaction + decides what is kept. + """ + return sessionmaker( + connection, expire_on_commit=False, join_transaction_mode="create_savepoint" + ) + + +@contextmanager +def session_in(connection: Connection) -> Iterator[Session]: + """Yield a session in a savepoint of ``connection``'s transaction.""" + with session_factory_on(connection).begin() as session: + yield session + + +def in_session[Result]( + connection: Connection, + operation: Callable[..., Result], + *arguments: object, + **keywords: object, +) -> Result: + """Run a function that takes a session, in a savepoint of ``connection``.""" + with session_in(connection) as session: + return operation(session, *arguments, **keywords) + + +def economy_store_on(connection: Connection) -> EconomyRunStore: + """Return an economy store whose sessions run in savepoints of ``connection``.""" + return EconomyRunStore.from_session_factory( + session_factory_on(connection), + statement_timeout_milliseconds=STORE_STATEMENT_TIMEOUT_MILLISECONDS, + ) + + def act_as_the_service(connection: Connection) -> None: """Take the service's role for the rest of the connection's transaction. @@ -165,7 +214,7 @@ def connection_as_the_service(database_url: str) -> Iterator[Connection]: @pytest.fixture def committed_engine(public_api_database_url) -> Iterator[Engine]: """Return an engine on the Public API's database, disposed afterwards.""" - engine = create_simulation_engine(public_api_database_url) + engine = simulation_engine(public_api_database_url) try: yield engine finally: diff --git a/libs/policyengine-simulation-persistence/tests/economy_rows.py b/libs/policyengine-simulation-persistence/tests/economy_rows.py index 4fc7ce6bc..c49e4ab65 100644 --- a/libs/policyengine-simulation-persistence/tests/economy_rows.py +++ b/libs/policyengine-simulation-persistence/tests/economy_rows.py @@ -16,16 +16,16 @@ from policyengine_simulation_persistence import ( EconomyPopulation, + Report, + ReportRun, ReportRunStatus, ReportRunTrigger, RunVersions, + Simulation, SimulationChoice, + SimulationRun, SimulationStatus, SimulationType, - report_runs, - reports, - simulation_runs, - simulations, ) from public_api_rows import ( CatalogVersion, @@ -191,7 +191,7 @@ def insert_simulation_run( shape = shape or RunShape() simulation_id = uuid4() connection.execute( - insert(simulations).values( + insert(Simulation).values( id=simulation_id, simulation_type=SimulationType.ECONOMY, dataset_id=population.dataset_id, @@ -220,7 +220,7 @@ def insert_run_of( finished = shape.status in {SimulationStatus.SUCCEEDED, SimulationStatus.FAILED} succeeded = shape.status is SimulationStatus.SUCCEEDED connection.execute( - insert(simulation_runs).values( + insert(SimulationRun).values( id=run_id, simulation_id=simulation_id, year=population.year, @@ -285,7 +285,7 @@ def insert_report_of( # noqa: PLR0913 # the report's sides and what else it ma baseline, reform = compared report_id = uuid4() connection.execute( - insert(reports).values( + insert(Report).values( id=report_id, label="An earlier report", country=economy.catalog.country, @@ -319,7 +319,7 @@ def insert_report_run_of( run_id = uuid4() finished = outcome.status in {ReportRunStatus.SUCCEEDED, ReportRunStatus.FAILED} connection.execute( - insert(report_runs).values( + insert(ReportRun).values( id=run_id, report_id=report_id, baseline_simulation_run_id=baseline.simulation_run_id, diff --git a/libs/policyengine-simulation-persistence/tests/household_report_rows.py b/libs/policyengine-simulation-persistence/tests/household_report_rows.py index 1ceb62eb5..4564b6f71 100644 --- a/libs/policyengine-simulation-persistence/tests/household_report_rows.py +++ b/libs/policyengine-simulation-persistence/tests/household_report_rows.py @@ -17,14 +17,14 @@ from policyengine_simulation_persistence import ( HOUSEHOLD_REPORT_TYPE, + Report, + ReportRun, ReportRunStatus, ReportRunTrigger, + Simulation, + SimulationRun, SimulationStatus, SimulationType, - report_runs, - reports, - simulation_runs, - simulations, ) from sqlalchemy import insert @@ -108,7 +108,7 @@ def insert_household_report_rows( # noqa: PLR0913 # the report's catalog, hous ) for side in report.sides: connection.execute( - insert(simulations).values( + insert(Simulation).values( id=side.simulation_id, simulation_type=SimulationType.HOUSEHOLD, dataset_id=None, @@ -121,13 +121,13 @@ def insert_household_report_rows( # noqa: PLR0913 # the report's catalog, hous ) ) connection.execute( - insert(simulation_runs).values( + insert(SimulationRun).values( _pending_run(side, catalog=catalog, year=year, versions=versions) ) ) reform = report.reform connection.execute( - insert(reports).values( + insert(Report).values( id=report.id, label="A larger child tax credit", description="For one family", @@ -147,7 +147,7 @@ def insert_household_report_rows( # noqa: PLR0913 # the report's catalog, hous ) ) connection.execute( - insert(report_runs).values( + insert(ReportRun).values( id=report.run_id, report_id=report.id, baseline_simulation_run_id=report.baseline.simulation_run_id, diff --git a/libs/policyengine-simulation-persistence/tests/service_rows.py b/libs/policyengine-simulation-persistence/tests/service_rows.py index 6179023ef..02d4b8f14 100644 --- a/libs/policyengine-simulation-persistence/tests/service_rows.py +++ b/libs/policyengine-simulation-persistence/tests/service_rows.py @@ -13,17 +13,17 @@ from uuid import UUID, uuid4 from policyengine_simulation_persistence import ( + BudgetWindowRun, + HouseholdJob, HouseholdJobStatus, + Report, + ReportRun, ReportRunStatus, ReportRunTrigger, + Simulation, + SimulationRun, SimulationStatus, SimulationType, - budget_window_runs, - household_jobs, - report_runs, - reports, - simulation_runs, - simulations, ) from sqlalchemy import insert, text @@ -120,14 +120,14 @@ def insert_economy_simulation( connection: Connection, public_api_rows: PublicApiRows ) -> UUID: return connection.execute( - insert(simulations) + insert(Simulation) .values( simulation_type=SimulationType.ECONOMY, dataset_id=public_api_rows.dataset_id, tax_benefit_model_version_id=public_api_rows.model_version_id, year=YEAR, ) - .returning(simulations.c.id) + .returning(Simulation.id) ).scalar_one() @@ -135,7 +135,7 @@ def insert_simulation_run( connection: Connection, simulation_id: UUID, public_api_rows: PublicApiRows ) -> UUID: return connection.execute( - insert(simulation_runs) + insert(SimulationRun) .values( simulation_id=simulation_id, year=YEAR, @@ -145,7 +145,7 @@ def insert_simulation_run( dataset_id=public_api_rows.dataset_id, status=SimulationStatus.PENDING, ) - .returning(simulation_runs.c.id) + .returning(SimulationRun.id) ).scalar_one() @@ -161,7 +161,7 @@ def insert_service_rows( baseline_simulation_id = insert_economy_simulation(connection, public_api_rows) reform_simulation_id = insert_economy_simulation(connection, public_api_rows) report_id = connection.execute( - insert(reports) + insert(Report) .values( label="Test report", country="us", @@ -173,7 +173,7 @@ def insert_service_rows( budget_window_size=2, inputs={}, ) - .returning(reports.c.id) + .returning(Report.id) ).scalar_one() baseline_run_id = insert_simulation_run( connection, baseline_simulation_id, public_api_rows @@ -182,7 +182,7 @@ def insert_service_rows( connection, reform_simulation_id, public_api_rows ) budget_window_run_id = connection.execute( - insert(budget_window_runs) + insert(BudgetWindowRun) .values( report_id=report_id, tax_benefit_model_version_id=public_api_rows.model_version_id, @@ -190,10 +190,10 @@ def insert_service_rows( recalculate=False, status=ReportRunStatus.PENDING, ) - .returning(budget_window_runs.c.id) + .returning(BudgetWindowRun.id) ).scalar_one() report_run_id = connection.execute( - insert(report_runs) + insert(ReportRun) .values( report_id=report_id, baseline_simulation_run_id=baseline_run_id, @@ -205,10 +205,10 @@ def insert_service_rows( status=ReportRunStatus.PENDING, trigger=ReportRunTrigger.INITIAL, ) - .returning(report_runs.c.id) + .returning(ReportRun.id) ).scalar_one() household_job_id = connection.execute( - insert(household_jobs) + insert(HouseholdJob) .values( country="us", request_data={}, @@ -219,7 +219,7 @@ def insert_service_rows( country_package_version=COUNTRY_PACKAGE_VERSION, policyengine_version=POLICYENGINE_VERSION, ) - .returning(household_jobs.c.id) + .returning(HouseholdJob.id) ).scalar_one() return ServiceRows( baseline_simulation_id=baseline_simulation_id, diff --git a/libs/policyengine-simulation-persistence/tests/session_probes.py b/libs/policyengine-simulation-persistence/tests/session_probes.py new file mode 100644 index 000000000..b7be096f3 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/session_probes.py @@ -0,0 +1,110 @@ +"""What tests of ORM sessions watch, and the real failures they cause. + +A session runs its statements through events a test can listen to, and a store's +create means different things depending on where it fails: at the commit the +rows may exist, before it nothing was written. These helpers record what a +session or connection does, and end a session's connection at a chosen moment. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sqlalchemy import event, func, select + +if TYPE_CHECKING: + from sqlalchemy import Connection, Engine + from sqlalchemy.orm import ORMExecuteState, Session, sessionmaker + +# How long PostgreSQL may take to end a backend it is told to end. +TERMINATE_TIMEOUT_MILLISECONDS = 5_000 + + +def terminate_backend(engine: Engine, backend_id: int) -> None: + """End the backend from another connection, and wait until it has gone.""" + with engine.connect() as connection: + ended = connection.execute( + select( + func.pg_terminate_backend(backend_id, TERMINATE_TIMEOUT_MILLISECONDS) + ) + ).scalar_one() + assert ended + + +def end_connection_at( + session_factory: sessionmaker[Session], session_event: str, engine: Engine +) -> list[int]: + """End each session's connection at ``session_event``, from another one. + + Return a list that gains, each time, how many added rows the session had + not yet flushed. A savepoint's release fires the session's commit events + too, so only the outermost transaction's count. + """ + unflushed_counts: list[int] = [] + + def end_the_connection(session: Session, *_event_details: object) -> None: + if session.in_nested_transaction(): + return + unflushed_counts.append(len(session.new)) + backend_id = session.execute(select(func.pg_backend_pid())).scalar_one() + terminate_backend(engine, backend_id) + + event.listen(session_factory, session_event, end_the_connection) + return unflushed_counts + + +def end_connection_before_statement(engine: Engine, statement_start: str) -> list[int]: + """End the connection about to send the first statement starting so. + + Return a list that gains the ended backend's ID. + """ + ended_backends: list[int] = [] + + def end_the_connection( + _connection, cursor, statement, _parameters, _context, _executemany + ) -> None: + if ended_backends or not statement.startswith(statement_start): + return + ended_backends.append(cursor.connection.info.backend_pid) + terminate_backend(engine, ended_backends[0]) + + event.listen(engine, "before_cursor_execute", end_the_connection) + return ended_backends + + +def statements_sent(engine_or_connection: Engine | Connection) -> list[str]: + """Return a list that gains each statement sent to the database from now on.""" + statements: list[str] = [] + + def record_statement( + _connection, _cursor, statement, _parameters, _context, _executemany + ) -> None: + statements.append(statement) + + event.listen(engine_or_connection, "before_cursor_execute", record_statement) + return statements + + +def orm_statement_flags(session: Session) -> list[bool]: + """Return a list that gains, per statement, whether it names mapped classes.""" + flags: list[bool] = [] + event.listen( + session, + "do_orm_execute", + lambda execute_state: flags.append(execute_state.is_orm_statement), + ) + return flags + + +def statements_naming_no_mapped_class( + session_or_factory: Session | sessionmaker[Session], +) -> list[str]: + """Return a list that gains each statement run without a mapped class.""" + statements: list[str] = [] + + def record_core_statement(execute_state: ORMExecuteState) -> None: + if not execute_state.is_orm_statement: + statements.append(str(execute_state.statement)) + + event.listen(session_or_factory, "do_orm_execute", record_core_statement) + return statements diff --git a/libs/policyengine-simulation-persistence/tests/test_access.py b/libs/policyengine-simulation-persistence/tests/test_access.py index e6740422a..5b349f064 100644 --- a/libs/policyengine-simulation-persistence/tests/test_access.py +++ b/libs/policyengine-simulation-persistence/tests/test_access.py @@ -5,22 +5,28 @@ from pathlib import Path import pytest -from conftest import SERVICE_TEST_ROLE +from conftest import SERVICE_TEST_ROLE, simulation_engine from policyengine_simulation_persistence import ( MODAL_WORKER_TABLE_PRIVILEGES, MODAL_WORKER_UPDATABLE_COLUMNS, SERVICE_TABLE_PRIVILEGES, SERVICE_UPDATABLE_COLUMNS, - create_simulation_engine, + create_session_factory, missing_table_privileges, read_missing_table_privileges, result_metadata, service_metadata, ) from policyengine_simulation_persistence import access as access_module +from policyengine_simulation_persistence.catalog_tables import ( + PgAttribute, + PgClass, + PgNamespace, +) from sqlalchemy import create_engine, func, select, text from sqlalchemy.engine import make_url from sqlalchemy.exc import OperationalError +from sqlalchemy.orm import Session PROOF_FIXTURES = ( Path(__file__).resolve().parents[3] @@ -114,7 +120,8 @@ def test_the_modal_worker_role_reads_its_runs_and_inserts_their_results() -> Non assert dict(MODAL_WORKER_TABLE_PRIVILEGES) == { **dict.fromkeys(MODAL_WORKER_RUN_TABLES, frozenset({"SELECT"})), - **dict.fromkeys(result_tables, frozenset({"INSERT"})), + # A session's insert also reads the row's server defaults back. + **dict.fromkeys(result_tables, frozenset({"SELECT", "INSERT"})), } @@ -228,10 +235,10 @@ def _grant_every_required_privilege(database, *, except_for=()): def _missing_privileges_as_the_service(database): - engine = create_simulation_engine(database.service_url) + engine = simulation_engine(database.service_url) try: - with engine.connect() as connection: - return missing_table_privileges(connection) + with create_session_factory(engine).begin() as session: + return missing_table_privileges(session) finally: engine.dispose() @@ -339,11 +346,11 @@ def test_the_check_names_what_a_role_lacks_of_another_roles_lists( f"TO {SERVICE_TEST_ROLE}", ], ) - engine = create_simulation_engine(disposable_database.service_url) + engine = simulation_engine(disposable_database.service_url) try: - with engine.connect() as connection: + with create_session_factory(engine).begin() as session: missing = missing_table_privileges( - connection, + session, table_privileges={ "report_runs": frozenset({"SELECT"}), "poverty": frozenset({"INSERT"}), @@ -356,6 +363,126 @@ def test_the_check_names_what_a_role_lacks_of_another_roles_lists( assert missing == ["INSERT on poverty", "UPDATE (started_at) on report_runs"] +@pytest.mark.postgres +def test_a_partitioned_table_counts_as_the_table(disposable_database): + """API v1 could partition a table; its grants still count.""" + other_tables = [name for name in SERVICE_TABLE_PRIVILEGES if name != "reports"] + _create_tables(disposable_database, other_tables) + _grant( + disposable_database, + ["CREATE TABLE public.reports (id uuid) PARTITION BY HASH (id)"], + ) + _grant_every_required_privilege(disposable_database) + + assert _missing_privileges_as_the_service(disposable_database) == [] + + +@pytest.mark.postgres +def test_a_view_of_a_required_tables_name_doesnt_count(disposable_database): + """Grants on a view say nothing about the table the role needs.""" + other_tables = [name for name in SERVICE_TABLE_PRIVILEGES if name != "reports"] + _create_tables(disposable_database, other_tables) + _grant( + disposable_database, + ["CREATE VIEW public.reports AS SELECT gen_random_uuid() AS id"], + ) + _grant_every_required_privilege(disposable_database) + + assert _missing_privileges_as_the_service(disposable_database) == [ + "INSERT on reports", + "SELECT on reports", + ] + + +@pytest.mark.postgres +def test_a_dropped_column_lacks_its_privilege(disposable_database): + _create_tables(disposable_database, SERVICE_TABLE_PRIVILEGES) + _grant_every_required_privilege(disposable_database) + _grant( + disposable_database, + ["ALTER TABLE public.report_runs DROP COLUMN modal_call_id"], + ) + + assert _missing_privileges_as_the_service(disposable_database) == [ + "UPDATE (modal_call_id) on report_runs", + ] + + +@pytest.mark.postgres +def test_a_system_column_isnt_a_column_the_role_can_update(disposable_database): + """PostgreSQL numbers system columns below zero; they can't be granted.""" + _grant( + disposable_database, + [ + "CREATE TABLE public.report_runs (id uuid PRIMARY KEY)", + f"GRANT UPDATE ON public.report_runs TO {SERVICE_TEST_ROLE}", + ], + ) + engine = simulation_engine(disposable_database.service_url) + try: + with create_session_factory(engine).begin() as session: + missing = missing_table_privileges( + session, + table_privileges={"report_runs": frozenset({"UPDATE"})}, + updatable_columns={"report_runs": frozenset({"id", "xmin"})}, + ) + finally: + engine.dispose() + + assert missing == ["UPDATE (xmin) on report_runs"] + + +@pytest.mark.postgres +def test_an_empty_list_requires_nothing(disposable_database): + engine = simulation_engine(disposable_database.service_url) + try: + with create_session_factory(engine).begin() as session: + missing = missing_table_privileges( + session, table_privileges={}, updatable_columns={} + ) + finally: + engine.dispose() + + assert missing == [] + + +@pytest.mark.postgres +def test_the_catalog_classes_read_postgresqls_catalog(disposable_database): + """Each mapped column of the catalog classes exists under its own name.""" + _grant( + disposable_database, + ["CREATE TABLE public.reports (id uuid PRIMARY KEY, label text)"], + ) + engine = simulation_engine(disposable_database.service_url) + try: + with create_session_factory(engine).begin() as session: + table_row = session.execute( + select(PgClass.relname, PgClass.relkind, PgNamespace.nspname) + .join(PgNamespace, PgNamespace.oid == PgClass.relnamespace) + .where(PgClass.relname == "reports") + ).one() + column_rows = session.execute( + select(PgAttribute.attname, PgAttribute.attisdropped) + .join(PgClass, PgClass.oid == PgAttribute.attrelid) + .where(PgClass.relname == "reports", PgAttribute.attnum > 0) + .order_by(PgAttribute.attnum) + ).all() + finally: + engine.dispose() + + assert tuple(table_row) == ("reports", "r", "public") + assert [tuple(row) for row in column_rows] == [("id", False), ("label", False)] + + +def test_the_catalog_classes_name_postgresqls_catalog_schema() -> None: + for model, table_name in ( + (PgClass, "pg_class"), + (PgNamespace, "pg_namespace"), + (PgAttribute, "pg_attribute"), + ): + assert model.__table__.fullname == f"pg_catalog.{table_name}" + + def _put_a_decoy_schema_first(database): """Create schema ``decoy``, usable by the role and first on its search_path.""" _grant( @@ -479,7 +606,7 @@ def test_the_bounded_check_reads_the_privileges_and_leaves_no_timeout_behind( engine = _service_engine_with_one_connection(disposable_database) try: missing = read_missing_table_privileges( - engine, statement_timeout_milliseconds=2000 + create_session_factory(engine), statement_timeout_milliseconds=2000 ) timeout_afterwards = _statement_timeout(engine) finally: @@ -489,19 +616,57 @@ def test_the_bounded_check_reads_the_privileges_and_leaves_no_timeout_behind( assert timeout_afterwards == "0" +@pytest.mark.postgres +def test_the_bounded_check_runs_every_statement_in_one_session( + disposable_database, monkeypatch +): + """The limit is set on the session the check itself then reads through.""" + sessions_used = [] + + def record_session(session, **keywords): + sessions_used.append( + ( + session, + session.execute( + select(func.current_setting("statement_timeout")) + ).scalar_one(), + ) + ) + return original_check(session, **keywords) + + original_check = access_module.missing_table_privileges + monkeypatch.setattr(access_module, "missing_table_privileges", record_session) + _create_tables(disposable_database, SERVICE_TABLE_PRIVILEGES) + _grant_every_required_privilege(disposable_database) + engine = _service_engine_with_one_connection(disposable_database) + try: + missing = read_missing_table_privileges( + create_session_factory(engine), statement_timeout_milliseconds=2000 + ) + finally: + engine.dispose() + + assert missing == [] + [(session, timeout)] = sessions_used + assert isinstance(session, Session) + assert timeout == "2s" + + @pytest.mark.postgres def test_the_database_ends_a_bounded_check_that_runs_too_long( disposable_database, monkeypatch ): - def sleep_instead(connection): - connection.execute(select(func.pg_sleep(2))) + def sleep_instead(session): + session.execute(select(func.pg_sleep(2))) return [] monkeypatch.setattr(access_module, "missing_table_privileges", sleep_instead) engine = _service_engine_with_one_connection(disposable_database) try: with pytest.raises(OperationalError, match="statement timeout"): - read_missing_table_privileges(engine, statement_timeout_milliseconds=100) + read_missing_table_privileges( + create_session_factory(engine), statement_timeout_milliseconds=100 + ) timeout_afterwards = _statement_timeout(engine) finally: engine.dispose() diff --git a/libs/policyengine-simulation-persistence/tests/test_budget_window_records.py b/libs/policyengine-simulation-persistence/tests/test_budget_window_records.py index 43f912a5a..2781100de 100644 --- a/libs/policyengine-simulation-persistence/tests/test_budget_window_records.py +++ b/libs/policyengine-simulation-persistence/tests/test_budget_window_records.py @@ -9,25 +9,17 @@ from __future__ import annotations -from contextlib import contextmanager from datetime import UTC, datetime -from typing import TYPE_CHECKING import pytest -from conftest import act_as_the_service, rolled_back_transaction +from conftest import act_as_the_service, economy_store_on, rolled_back_transaction from economy_rows import CURRENT_LAW, POLICYENGINE_VERSION, insert_economy_catalog from policyengine_simulation_persistence import ( EconomyRequest, - EconomyRunStore, SimulationChoice, ) from public_api_rows import StoredValue, insert_policy -if TYPE_CHECKING: - from collections.abc import Iterator - - from sqlalchemy import Connection - pytestmark = pytest.mark.public_api_schema START = datetime(2026, 1, 1, tzinfo=UTC) @@ -41,17 +33,6 @@ ) -def _store_on(connection: Connection) -> EconomyRunStore: - """Return a store whose transactions are savepoints of ``connection``.""" - - @contextmanager - def savepoint() -> Iterator[Connection]: - with connection.begin_nested(): - yield connection - - return EconomyRunStore(savepoint) - - def test_a_windows_catalog_rows_and_both_sides_values_are_read( public_api_database_url, ) -> None: @@ -63,7 +44,7 @@ def test_a_windows_catalog_rows_and_both_sides_values_are_read( act_as_the_service(connection) reform_choice = SimulationChoice(policy_id=policy_id, dynamic_id=None) - inputs = _store_on(connection).read_budget_window_inputs( + inputs = economy_store_on(connection).read_budget_window_inputs( REQUEST, baseline=CURRENT_LAW, reform=reform_choice ) diff --git a/libs/policyengine-simulation-persistence/tests/test_dynamic_reform.py b/libs/policyengine-simulation-persistence/tests/test_dynamic_reform.py index 0bce6c2ea..c4b5fe291 100644 --- a/libs/policyengine-simulation-persistence/tests/test_dynamic_reform.py +++ b/libs/policyengine-simulation-persistence/tests/test_dynamic_reform.py @@ -11,7 +11,7 @@ from uuid import uuid4 import pytest -from conftest import act_as_the_service, rolled_back_transaction +from conftest import act_as_the_service, in_session, rolled_back_transaction from policyengine_simulation_persistence import ( MissingPolicyParametersError, OverlappingReformError, @@ -55,8 +55,11 @@ def test_a_dynamic_is_read_as_its_values_by_parameter_name_and_period( ) act_as_the_service(connection) - reform = read_dynamic_reform( - connection, dynamic_id=dynamic_id, catalog_version_id=catalog.version_id + reform = in_session( + connection, + read_dynamic_reform, + dynamic_id=dynamic_id, + catalog_version_id=catalog.version_id, ) assert reform == { @@ -78,8 +81,9 @@ def test_an_unknown_dynamic_is_refused(public_api_database_url) -> None: missing_dynamic_id = uuid4() with pytest.raises(UnknownDynamicError, match=str(missing_dynamic_id)): - read_dynamic_reform( + in_session( connection, + read_dynamic_reform, dynamic_id=missing_dynamic_id, catalog_version_id=catalog.version_id, ) @@ -116,8 +120,9 @@ def test_a_dynamic_changing_parameters_the_run_lacks_names_each_of_them( act_as_the_service(connection) with pytest.raises(MissingPolicyParametersError) as refusal: - read_dynamic_reform( + in_session( connection, + read_dynamic_reform, dynamic_id=dynamic_id, catalog_version_id=running.version_id, ) diff --git a/libs/policyengine-simulation-persistence/tests/test_economy_reruns.py b/libs/policyengine-simulation-persistence/tests/test_economy_reruns.py index 83214bea3..7429997a4 100644 --- a/libs/policyengine-simulation-persistence/tests/test_economy_reruns.py +++ b/libs/policyengine-simulation-persistence/tests/test_economy_reruns.py @@ -12,14 +12,18 @@ from __future__ import annotations -from contextlib import contextmanager from dataclasses import dataclass, replace from datetime import UTC, datetime from typing import TYPE_CHECKING from uuid import UUID, uuid4 import pytest -from conftest import act_as_the_service, rolled_back_transaction +from conftest import ( + act_as_the_service, + economy_store_on, + in_session, + rolled_back_transaction, +) from economy_rows import ( CURRENT_LAW, LATER_POLICYENGINE_VERSION, @@ -40,7 +44,6 @@ from policyengine_simulation_persistence import ( EconomyRecordReferenceError, EconomyRequest, - EconomyRunStore, FoundReportRun, MissingPolicyParametersError, NewReportRerun, @@ -52,6 +55,7 @@ RerunRecordDeletedError, RerunSide, SideDefinition, + Simulation, SimulationChoice, SimulationDefinition, SimulationStatus, @@ -63,7 +67,6 @@ lock_reruns_of, read_report_definition, read_simulation_definition, - simulations, ) from public_api_rows import ( StoredValue, @@ -77,8 +80,6 @@ from sqlalchemy.exc import OperationalError if TYPE_CHECKING: - from collections.abc import Iterator - from sqlalchemy import Connection pytestmark = pytest.mark.public_api_schema @@ -145,17 +146,6 @@ def _count_runs_of(connection: Connection, simulation_id: UUID) -> int: ).scalar_one() -def _store_on(connection: Connection) -> EconomyRunStore: - """Return a store whose transactions are savepoints of ``connection``.""" - - @contextmanager - def savepoint() -> Iterator[Connection]: - with connection.begin_nested(): - yield connection - - return EconomyRunStore(savepoint) - - def test_a_stored_economy_reports_definition_is_read(public_api_database_url) -> None: with rolled_back_transaction(public_api_database_url) as connection: economy = insert_economy_catalog(connection) @@ -164,7 +154,7 @@ def test_a_stored_economy_reports_definition_is_read(public_api_database_url) -> stored = _stored_report(connection, economy, reform_choice=reform_choice) act_as_the_service(connection) - definition = read_report_definition(connection, stored.report_id) + definition = in_session(connection, read_report_definition, stored.report_id) assert definition == ReportDefinition( id=stored.report_id, @@ -202,8 +192,8 @@ def test_a_budget_window_or_household_report_is_read_as_its_kind( ) act_as_the_service(connection) - window = read_report_definition(connection, window_id) - household = read_report_definition(connection, household_report_id) + window = in_session(connection, read_report_definition, window_id) + household = in_session(connection, read_report_definition, household_report_id) assert window is not None assert household is not None @@ -216,8 +206,8 @@ def test_an_unknown_report_or_simulation_has_no_definition( with rolled_back_transaction(public_api_database_url) as connection: act_as_the_service(connection) - assert read_report_definition(connection, uuid4()) is None - assert read_simulation_definition(connection, uuid4()) is None + assert in_session(connection, read_report_definition, uuid4()) is None + assert in_session(connection, read_simulation_definition, uuid4()) is None def test_a_stored_economy_simulations_definition_is_read( @@ -232,7 +222,9 @@ def test_a_stored_economy_simulations_definition_is_read( ) act_as_the_service(connection) - definition = read_simulation_definition(connection, stored.simulation_id) + definition = in_session( + connection, read_simulation_definition, stored.simulation_id + ) assert definition == SimulationDefinition( id=stored.simulation_id, @@ -312,7 +304,7 @@ def test_a_report_rerun_joins_or_creates_runs_of_the_reports_own_simulations( rerun = _rerun_of(stored, later) act_as_the_service(connection) - insert_report_rerun(connection, rerun) + in_session(connection, insert_report_rerun, rerun) report_run = _row(connection, "report_runs", rerun.run_id) reform_run = _row(connection, "simulation_runs", rerun.reform.simulation_run_id) earlier_run = _row(connection, "report_runs", stored.report_run_id) @@ -396,7 +388,7 @@ def test_a_report_rerun_joins_the_reports_run_on_its_versions( rerun = _unforced_rerun_on_unchanged_versions(stored, economy) act_as_the_service(connection) - joined = insert_report_rerun(connection, rerun) + joined = in_session(connection, insert_report_rerun, rerun) created_report_runs = connection.execute( text("SELECT count(*) FROM public.report_runs WHERE id = :id"), {"id": rerun.run_id}, @@ -429,7 +421,7 @@ def test_a_report_rerun_beside_a_run_under_way_on_older_versions_creates_its_own rerun = _rerun_of(stored, later) act_as_the_service(connection) - created = insert_report_rerun(connection, rerun) + created = in_session(connection, insert_report_rerun, rerun) report_run = _row(connection, "report_runs", rerun.run_id) assert (created.id, created.inserted, created.status) == ( @@ -447,7 +439,7 @@ def test_a_report_whose_runs_have_ended_is_rerun_again(public_api_database_url) stored = _stored_report(connection, economy, run_status=ReportRunStatus.FAILED) first = _rerun_of(stored, later) act_as_the_service(connection) - insert_report_rerun(connection, first) + in_session(connection, insert_report_rerun, first) # The first re-run's run, and the simulation runs it created, fail # before a second re-run. connection.execute(text("RESET ROLE")) @@ -465,7 +457,7 @@ def test_a_report_whose_runs_have_ended_is_rerun_again(public_api_database_url) second = _rerun_of(stored, later) act_as_the_service(connection) - insert_report_rerun(connection, second) + in_session(connection, insert_report_rerun, second) assert _row(connection, "report_runs", second.run_id)["status"] == ( ReportRunStatus.PENDING @@ -488,13 +480,14 @@ def test_a_simulation_rerun_joins_its_run_on_the_current_versions_until_it_fails ) act_as_the_service(connection) - inserted = insert_simulation_rerun(connection, rerun) + inserted = in_session(connection, insert_simulation_rerun, rerun) created = _row(connection, "simulation_runs", rerun.run_id) - joined_under_way = insert_simulation_rerun( - connection, replace(rerun, run_id=uuid4()) + joined_under_way = in_session( + connection, insert_simulation_rerun, replace(rerun, run_id=uuid4()) ) - joined_succeeded = insert_simulation_rerun( + joined_succeeded = in_session( connection, + insert_simulation_rerun, NewSimulationRerun( simulation_id=stored.simulation_id, run_id=uuid4(), @@ -511,8 +504,8 @@ def test_a_simulation_rerun_joins_its_run_on_the_current_versions_until_it_fails {"id": rerun.run_id}, ) act_as_the_service(connection) - after_failure = insert_simulation_rerun( - connection, replace(rerun, run_id=uuid4()) + after_failure = in_session( + connection, insert_simulation_rerun, replace(rerun, run_id=uuid4()) ) runs_of_the_simulation = _count_runs_of(connection, stored.simulation_id) @@ -554,14 +547,14 @@ def test_a_rerun_holds_its_records_lock_until_its_transaction_ends( later = insert_later_catalog(holder, economy) stored = _stored_report(holder, economy, run_status=ReportRunStatus.FAILED) act_as_the_service(holder) - insert_report_rerun(holder, _rerun_of(stored, later)) + in_session(holder, insert_report_rerun, _rerun_of(stored, later)) with rolled_back_transaction(public_api_database_url) as waiter: act_as_the_service(waiter) waiter.execute(text("SET LOCAL lock_timeout = '200ms'")) - lock_reruns_of(waiter, uuid4()) + in_session(waiter, lock_reruns_of, uuid4()) with pytest.raises(OperationalError, match="lock timeout"): - lock_reruns_of(waiter, stored.report_id) + in_session(waiter, lock_reruns_of, stored.report_id) def test_a_report_reruns_inputs_are_read_on_the_current_catalog_version( @@ -577,7 +570,7 @@ def test_a_report_reruns_inputs_are_read_on_the_current_catalog_version( stored = _stored_report(connection, economy, reform_choice=reform_choice) act_as_the_service(connection) - inputs = _store_on(connection).read_report_rerun_inputs( + inputs = economy_store_on(connection).read_report_rerun_inputs( LATER_REQUEST, baseline=SideDefinition(stored.baseline.simulation_id, CURRENT_LAW), reform=SideDefinition(stored.reform.simulation_id, reform_choice), @@ -606,7 +599,7 @@ def test_a_rerun_names_the_side_whose_policy_the_current_catalog_cannot_set( act_as_the_service(connection) with pytest.raises(ReportSideError) as refused: - _store_on(connection).read_report_rerun_inputs( + economy_store_on(connection).read_report_rerun_inputs( LATER_REQUEST, baseline=SideDefinition(stored.baseline.simulation_id, CURRENT_LAW), reform=SideDefinition(stored.reform.simulation_id, reform_choice), @@ -644,7 +637,7 @@ def test_a_rerun_is_refused_when_the_current_catalog_lacks_its_region_or_dataset ) with pytest.raises(expected): - _store_on(connection).read_report_rerun_inputs( + economy_store_on(connection).read_report_rerun_inputs( request, baseline=SideDefinition(stored.baseline.simulation_id, CURRENT_LAW), reform=SideDefinition(stored.reform.simulation_id, CURRENT_LAW), @@ -664,7 +657,9 @@ def test_a_simulation_reruns_inputs_are_read_on_the_current_catalog_version( act_as_the_service(connection) # A simulation's re-run reads the current catalog version as a new one's. - inputs = _store_on(connection).read_simulation_inputs(LATER_REQUEST, choice) + inputs = economy_store_on(connection).read_simulation_inputs( + LATER_REQUEST, choice + ) assert inputs.catalog.catalog_version_id == later.catalog.version_id assert inputs.simulation.choice == choice @@ -688,7 +683,7 @@ def test_a_rerun_and_its_runs_are_created_through_the_store( versions=LATER_VERSIONS, ) act_as_the_service(connection) - store = _store_on(connection) + store = economy_store_on(connection) store.create_report_rerun(rerun) store.create_simulation_rerun(simulation_rerun) @@ -719,7 +714,7 @@ def test_a_household_or_filtered_simulation_and_another_model_are_read_as_such( household_simulation_id = uuid4() filtered_simulation_id = uuid4() connection.execute( - insert(simulations).values( + insert(Simulation).values( id=household_simulation_id, simulation_type=SimulationType.HOUSEHOLD, household_id=household_id, @@ -728,7 +723,7 @@ def test_a_household_or_filtered_simulation_and_another_model_are_read_as_such( ) ) connection.execute( - insert(simulations).values( + insert(Simulation).values( id=filtered_simulation_id, simulation_type=SimulationType.ECONOMY, dataset_id=economy.dataset_id, @@ -750,10 +745,14 @@ def test_a_household_or_filtered_simulation_and_another_model_are_read_as_such( other_simulation = insert_simulation_run(connection, other.population()) act_as_the_service(connection) - household = read_simulation_definition(connection, household_simulation_id) - filtered = read_simulation_definition(connection, filtered_simulation_id) - of_another_model = read_simulation_definition( - connection, other_simulation.simulation_id + household = in_session( + connection, read_simulation_definition, household_simulation_id + ) + filtered = in_session( + connection, read_simulation_definition, filtered_simulation_id + ) + of_another_model = in_session( + connection, read_simulation_definition, other_simulation.simulation_id ) assert household is not None @@ -791,7 +790,7 @@ def test_a_report_of_another_type_is_read_as_unsupported( ) act_as_the_service(connection) - definition = read_report_definition(connection, stored.report_id) + definition = in_session(connection, read_report_definition, stored.report_id) assert definition is not None assert definition.kind == "unsupported" @@ -808,12 +807,15 @@ def test_a_rerun_of_a_deleted_report_or_simulation_says_the_record_is_gone( act_as_the_service(connection) with pytest.raises(RerunRecordDeletedError), connection.begin_nested(): - insert_report_rerun( - connection, replace(_rerun_of(stored, later), report_id=uuid4()) + in_session( + connection, + insert_report_rerun, + replace(_rerun_of(stored, later), report_id=uuid4()), ) with pytest.raises(RerunRecordDeletedError), connection.begin_nested(): - insert_simulation_rerun( + in_session( connection, + insert_simulation_rerun, NewSimulationRerun( simulation_id=uuid4(), run_id=uuid4(), @@ -830,7 +832,7 @@ def test_a_rerun_of_a_deleted_report_or_simulation_says_the_record_is_gone( pytest.raises(EconomyRecordReferenceError) as refused, connection.begin_nested(), ): - insert_report_rerun(connection, gone_side) + in_session(connection, insert_report_rerun, gone_side) assert (refused.value.table_name, refused.value.column_name) == ( "simulation_runs", @@ -871,7 +873,7 @@ def test_a_forced_report_rerun_is_an_extra_run_beside_any_run_on_its_versions( rerun = _forced_rerun_of(stored, economy) act_as_the_service(connection) - insert_report_rerun(connection, rerun) + in_session(connection, insert_report_rerun, rerun) report_run = _row(connection, "report_runs", rerun.run_id) simulation_runs = [ _row(connection, "simulation_runs", run_id) @@ -931,7 +933,7 @@ def create_a_competing_run_first( act_as_the_service(connection) event.listen(connection, "before_cursor_execute", create_a_competing_run_first) try: - joined = insert_report_rerun(connection, rerun) + joined = in_session(connection, insert_report_rerun, rerun) finally: event.remove( connection, "before_cursor_execute", create_a_competing_run_first diff --git a/libs/policyengine-simulation-persistence/tests/test_economy_run_reads.py b/libs/policyengine-simulation-persistence/tests/test_economy_run_reads.py index 5e11ee4b1..90909c485 100644 --- a/libs/policyengine-simulation-persistence/tests/test_economy_run_reads.py +++ b/libs/policyengine-simulation-persistence/tests/test_economy_run_reads.py @@ -12,7 +12,7 @@ from uuid import UUID, uuid4 import pytest -from conftest import act_as_the_service, rolled_back_transaction +from conftest import act_as_the_service, in_session, rolled_back_transaction from economy_rows import ( COUNTRY_PACKAGE_VERSION, POLICYENGINE_VERSION, @@ -25,12 +25,12 @@ insert_simulation_run, ) from policyengine_simulation_persistence import ( + ReportRun, ReportRunStatus, + SimulationRun, SimulationStatus, read_economy_report, read_economy_simulation, - report_runs, - simulation_runs, ) from sqlalchemy import update @@ -61,7 +61,9 @@ def _report_with_runs(connection, economy: EconomyCatalog) -> tuple[UUID, list[U ) ] report_id = ( - connection.execute(report_runs.select().where(report_runs.c.id == run_ids[0])) + connection.execute( + ReportRun.__table__.select().where(ReportRun.id == run_ids[0]) + ) .one() .report_id ) @@ -73,7 +75,7 @@ def _report_with_runs(connection, economy: EconomyCatalog) -> tuple[UUID, list[U for side in (baseline, reform) ) connection.execute( - report_runs.insert().values( + ReportRun.__table__.insert().values( id=run_id, report_id=report_id, baseline_simulation_run_id=forced_baseline.simulation_run_id, @@ -96,20 +98,18 @@ def test_a_report_is_read_with_its_newest_run(public_api_database_url) -> None: # The first run is the oldest; the other two share a creation time, so # the larger ID is the newer. connection.execute( - update(report_runs) - .where(report_runs.c.id == first) - .values(created_at=EARLIER) + update(ReportRun).where(ReportRun.id == first).values(created_at=EARLIER) ) shared = EARLIER + timedelta(days=1) connection.execute( - update(report_runs) - .where(report_runs.c.id.in_([second, third])) + update(ReportRun) + .where(ReportRun.id.in_([second, third])) .values(created_at=shared) ) act_as_the_service(connection) - report = read_economy_report(connection, report_id) - missing = read_economy_report(connection, uuid4()) + report = in_session(connection, read_economy_report, report_id) + missing = in_session(connection, read_economy_report, uuid4()) assert missing is None assert report is not None @@ -142,13 +142,15 @@ def test_a_failed_newest_run_is_read_with_its_error(public_api_database_url) -> ), ) report_id = ( - connection.execute(report_runs.select().where(report_runs.c.id == run_id)) + connection.execute( + ReportRun.__table__.select().where(ReportRun.id == run_id) + ) .one() .report_id ) act_as_the_service(connection) - report = read_economy_report(connection, report_id) + report = in_session(connection, read_economy_report, report_id) assert report is not None assert (report.baseline_simulation_id, report.reform_simulation_id) == ( @@ -176,7 +178,7 @@ def test_a_simulation_is_read_with_its_newest_run(public_api_database_url) -> No stored = insert_simulation_run(connection, economy.population()) newer_run_id = uuid4() connection.execute( - simulation_runs.insert().values( + SimulationRun.__table__.insert().values( id=newer_run_id, simulation_id=stored.simulation_id, year=2026, @@ -190,14 +192,16 @@ def test_a_simulation_is_read_with_its_newest_run(public_api_database_url) -> No ) ) connection.execute( - update(simulation_runs) - .where(simulation_runs.c.id == stored.simulation_run_id) + update(SimulationRun) + .where(SimulationRun.id == stored.simulation_run_id) .values(created_at=EARLIER) ) act_as_the_service(connection) - simulation = read_economy_simulation(connection, stored.simulation_id) - missing = read_economy_simulation(connection, uuid4()) + simulation = in_session( + connection, read_economy_simulation, stored.simulation_id + ) + missing = in_session(connection, read_economy_simulation, uuid4()) assert missing is None assert simulation is not None diff --git a/libs/policyengine-simulation-persistence/tests/test_economy_run_records.py b/libs/policyengine-simulation-persistence/tests/test_economy_run_records.py index cb9776c3b..720fad2ff 100644 --- a/libs/policyengine-simulation-persistence/tests/test_economy_run_records.py +++ b/libs/policyengine-simulation-persistence/tests/test_economy_run_records.py @@ -13,7 +13,7 @@ from uuid import UUID, uuid4 import pytest -from conftest import act_as_the_service, rolled_back_transaction +from conftest import act_as_the_service, in_session, rolled_back_transaction from economy_rows import ( PendingReport, RunShape, @@ -23,7 +23,9 @@ insert_uk_area_regions, ) from policyengine_simulation_persistence import ( + ReportRun, ReportRunStatus, + SimulationRun, SimulationStatus, fail_unstarted_report_run, fail_unstarted_simulation_run, @@ -34,8 +36,6 @@ read_region, record_report_run_call, record_simulation_run_call, - report_runs, - simulation_runs, ) from public_api_rows import ( insert_catalog_version, @@ -63,8 +63,11 @@ def test_the_countrys_model_is_found_by_its_name(public_api_database_url) -> Non economy = insert_economy_catalog(connection) act_as_the_service(connection) - assert read_model_id(connection, country="us") == economy.catalog.model_id - assert read_model_id(connection, country="uk") is None + assert ( + in_session(connection, read_model_id, country="us") + == economy.catalog.model_id + ) + assert in_session(connection, read_model_id, country="uk") is None def test_a_region_is_found_by_code_in_its_catalog_version_only( @@ -82,14 +85,16 @@ def test_a_region_is_found_by_code_in_its_catalog_version_only( ) act_as_the_service(connection) - region = read_region( + region = in_session( connection, + read_region, catalog_version_id=economy.catalog.version_id, region_code="state/ca", ) assert ( - read_region( + in_session( connection, + read_region, catalog_version_id=other.catalog.version_id, region_code="state/ca", ) @@ -114,22 +119,34 @@ def test_a_dataset_is_found_by_name_or_id_in_its_catalog_version_only( act_as_the_service(connection) version_id = economy.catalog.version_id - by_name = read_dataset( - connection, catalog_version_id=version_id, name="acs_local_2024" + by_name = in_session( + connection, + read_dataset, + catalog_version_id=version_id, + name="acs_local_2024", ) - by_id = read_dataset_by_id( - connection, catalog_version_id=version_id, dataset_id=economy.dataset_id + by_id = in_session( + connection, + read_dataset_by_id, + catalog_version_id=version_id, + dataset_id=economy.dataset_id, ) other_version = other.catalog.version_id assert ( - read_dataset( - connection, catalog_version_id=other_version, name="acs_local_2024" + in_session( + connection, + read_dataset, + catalog_version_id=other_version, + name="acs_local_2024", ) is None ) assert ( - read_dataset_by_id( - connection, catalog_version_id=other_version, dataset_id=local_id + in_session( + connection, + read_dataset_by_id, + catalog_version_id=other_version, + dataset_id=local_id, ) is None ) @@ -170,8 +187,10 @@ def test_the_local_area_dataset_is_the_one_both_area_types_use( ) act_as_the_service(connection) - found = read_local_area_dataset( - connection, catalog_version_id=economy.catalog.version_id + found = in_session( + connection, + read_local_area_dataset, + catalog_version_id=economy.catalog.version_id, ) assert found is not None @@ -209,8 +228,10 @@ def test_a_catalog_version_without_one_dataset_for_both_area_types_has_none( ) act_as_the_service(connection) - found = read_local_area_dataset( - connection, catalog_version_id=economy.catalog.version_id + found = in_session( + connection, + read_local_area_dataset, + catalog_version_id=economy.catalog.version_id, ) assert found is None @@ -230,8 +251,10 @@ def test_the_local_area_dataset_may_be_the_national_default( ) act_as_the_service(connection) - found = read_local_area_dataset( - connection, catalog_version_id=economy.catalog.version_id + found = in_session( + connection, + read_local_area_dataset, + catalog_version_id=economy.catalog.version_id, ) assert found is not None @@ -251,14 +274,23 @@ def test_a_report_runs_call_is_recorded_once_while_it_is_pending_without_one( with rolled_back_transaction(public_api_database_url) as connection: report = _created_report(connection) - first = record_report_run_call( - connection, report_run_id=report.report_run_id, modal_call_id="fc-first" + first = in_session( + connection, + record_report_run_call, + report_run_id=report.report_run_id, + modal_call_id="fc-first", ) - second = record_report_run_call( - connection, report_run_id=report.report_run_id, modal_call_id="fc-second" + second = in_session( + connection, + record_report_run_call, + report_run_id=report.report_run_id, + modal_call_id="fc-second", ) - unknown = record_report_run_call( - connection, report_run_id=uuid4(), modal_call_id="fc-other" + unknown = in_session( + connection, + record_report_run_call, + report_run_id=uuid4(), + modal_call_id="fc-other", ) call_id = _row(connection, "report_runs", report.report_run_id)["modal_call_id"] @@ -275,17 +307,21 @@ def test_a_claimed_report_run_keeps_its_claim(public_api_database_url) -> None: report = _created_report(connection) connection.execute(text("RESET ROLE")) connection.execute( - update(report_runs) - .where(report_runs.c.id == report.report_run_id) + update(ReportRun) + .where(ReportRun.id == report.report_run_id) .values(status=ReportRunStatus.RUNNING, modal_call_id="fc-coordinator") ) act_as_the_service(connection) - status = record_report_run_call( - connection, report_run_id=report.report_run_id, modal_call_id="fc-late" + status = in_session( + connection, + record_report_run_call, + report_run_id=report.report_run_id, + modal_call_id="fc-late", ) - failed = fail_unstarted_report_run( + failed = in_session( connection, + fail_unstarted_report_run, report_run_id=report.report_run_id, simulation_run_ids=(report.baseline.simulation_run_id,), error_message="modal_call_refused: Modal refused the call.", @@ -308,8 +344,9 @@ def test_an_unstarted_report_run_and_its_new_simulation_runs_fail_at_once( with rolled_back_transaction(public_api_database_url) as connection: report = _created_report(connection) - failed = fail_unstarted_report_run( + failed = in_session( connection, + fail_unstarted_report_run, report_run_id=report.report_run_id, simulation_run_ids=( report.baseline.simulation_run_id, @@ -317,8 +354,9 @@ def test_an_unstarted_report_run_and_its_new_simulation_runs_fail_at_once( ), error_message=message, ) - again = fail_unstarted_report_run( + again = in_session( connection, + fail_unstarted_report_run, report_run_id=report.report_run_id, simulation_run_ids=(), error_message="a second failure", @@ -326,11 +364,11 @@ def test_an_unstarted_report_run_and_its_new_simulation_runs_fail_at_once( stored = _row(connection, "report_runs", report.report_run_id) stored_simulation_runs = connection.execute( select( - simulation_runs.c.status, - simulation_runs.c.error_message, - simulation_runs.c.completed_at, + SimulationRun.status, + SimulationRun.error_message, + SimulationRun.completed_at, ).where( - simulation_runs.c.id.in_( + SimulationRun.id.in_( [ report.baseline.simulation_run_id, report.reform.simulation_run_id, @@ -360,28 +398,35 @@ def test_a_simulation_runs_call_is_recorded_or_the_run_failed_while_unstarted( ) act_as_the_service(connection) - status = record_simulation_run_call( + status = in_session( connection, + record_simulation_run_call, simulation_run_id=recorded.simulation_run_id, modal_call_id="fc-worker", ) - not_failed = fail_unstarted_simulation_run( + not_failed = in_session( connection, + fail_unstarted_simulation_run, simulation_run_id=recorded.simulation_run_id, error_message="modal_call_refused: refused", ) - failed = fail_unstarted_simulation_run( + failed = in_session( connection, + fail_unstarted_simulation_run, simulation_run_id=refused.simulation_run_id, error_message="modal_call_refused: refused", ) - not_recorded = record_simulation_run_call( + not_recorded = in_session( connection, + record_simulation_run_call, simulation_run_id=refused.simulation_run_id, modal_call_id="fc-late", ) - unknown = record_simulation_run_call( - connection, simulation_run_id=uuid4(), modal_call_id="fc-other" + unknown = in_session( + connection, + record_simulation_run_call, + simulation_run_id=uuid4(), + modal_call_id="fc-other", ) recorded_run = _row(connection, "simulation_runs", recorded.simulation_run_id) refused_run = _row(connection, "simulation_runs", refused.simulation_run_id) diff --git a/libs/policyengine-simulation-persistence/tests/test_economy_run_store.py b/libs/policyengine-simulation-persistence/tests/test_economy_run_store.py index 6571833b8..4ecc8555d 100644 --- a/libs/policyengine-simulation-persistence/tests/test_economy_run_store.py +++ b/libs/policyengine-simulation-persistence/tests/test_economy_run_store.py @@ -1,9 +1,9 @@ """The economy store: each read and write in a transaction of its own. These run against a database the Public API's own v2 migration chain built -(``PUBLIC_API_SCHEMA_DATABASE_URL``). The store opens each transaction -through a function it is given; here each is a savepoint of one connection, -acting as the simulation service's role, which the test rolls back. +(``PUBLIC_API_SCHEMA_DATABASE_URL``). The store runs each operation in an ORM +session of its own; here each session's transaction is a savepoint of one +connection, acting as the simulation service's role, which the test rolls back. """ from __future__ import annotations @@ -15,7 +15,7 @@ from uuid import uuid4 import pytest -from conftest import act_as_the_service, rolled_back_transaction +from conftest import act_as_the_service, economy_store_on, rolled_back_transaction from economy_rows import ( CURRENT_LAW, POLICYENGINE_VERSION, @@ -56,10 +56,9 @@ from public_api_rows import StoredValue, insert_dataset, insert_dynamic, insert_policy from sqlalchemy import text from sqlalchemy.exc import OperationalError +from transaction_fakes import CommitFailingTransaction, RecordingSession if TYPE_CHECKING: - from collections.abc import Iterator - from sqlalchemy import Connection pytestmark = pytest.mark.public_api_schema @@ -76,17 +75,6 @@ ) -def _store_on(connection: Connection) -> EconomyRunStore: - """Return a store whose transactions are savepoints of ``connection``.""" - - @contextmanager - def savepoint() -> Iterator[Connection]: - with connection.begin_nested(): - yield connection - - return EconomyRunStore(savepoint) - - def test_a_reports_inputs_are_read_together(public_api_database_url) -> None: with rolled_back_transaction(public_api_database_url) as connection: economy = insert_economy_catalog( @@ -101,7 +89,7 @@ def test_a_reports_inputs_are_read_together(public_api_database_url) -> None: act_as_the_service(connection) reform_choice = SimulationChoice(policy_id=policy_id, dynamic_id=dynamic_id) - inputs = _store_on(connection).read_report_inputs( + inputs = economy_store_on(connection).read_report_inputs( REQUEST, baseline=CURRENT_LAW, reform=reform_choice ) @@ -148,7 +136,7 @@ def test_a_report_on_the_local_area_dataset_is_read_on_it( local_authority_dataset_id=local_area_id, ) act_as_the_service(connection) - store = _store_on(connection) + store = economy_store_on(connection) on_local_areas = store.read_report_inputs( UK_REQUEST, @@ -194,7 +182,7 @@ def test_a_declared_dataset_the_catalog_does_not_give_both_area_types_is_refused act_as_the_service(connection) with pytest.raises(LocalAreaDatasetMismatchError) as mismatch: - _store_on(connection).read_report_inputs( + economy_store_on(connection).read_report_inputs( UK_REQUEST, baseline=CURRENT_LAW, reform=CURRENT_LAW, @@ -212,7 +200,7 @@ def test_a_named_dataset_is_read_instead_of_the_regions_default( local_id = insert_dataset(connection, economy.catalog, "acs_local_2024") act_as_the_service(connection) - inputs = _store_on(connection).read_simulation_inputs( + inputs = economy_store_on(connection).read_simulation_inputs( EconomyRequest(**{**vars(REQUEST), "dataset_name": "acs_local_2024"}), CURRENT_LAW, ) @@ -249,7 +237,7 @@ def test_each_missing_row_a_request_names_is_refused( with rolled_back_transaction(public_api_database_url) as connection: insert_economy_catalog(connection) act_as_the_service(connection) - store = _store_on(connection) + store = economy_store_on(connection) request = EconomyRequest(**{**vars(REQUEST), **request_fields}) # A report names the side whose policy or dynamic is missing. @@ -285,7 +273,7 @@ def test_values_the_running_version_lacks_or_both_set_are_refused( connection, economy.catalog, [StoredValue(ELASTICITY, -0.05, START)] ) act_as_the_service(connection) - store = _store_on(connection) + store = economy_store_on(connection) with pytest.raises(MissingPolicyParametersError): store.read_simulation_inputs( @@ -318,7 +306,7 @@ def test_a_reports_call_is_recorded_or_its_run_failed( insert_pending_report(connection, economy) for _ in range(2) ) act_as_the_service(connection) - store = _store_on(connection) + store = economy_store_on(connection) status = store.record_report_call(recorded.report_run_id, "fc-coordinator") failed = store.fail_unstarted_report( @@ -349,7 +337,7 @@ def test_a_simulations_call_is_recorded_or_its_run_failed( for _ in range(2) ) act_as_the_service(connection) - store = _store_on(connection) + store = economy_store_on(connection) status = store.record_simulation_call(recorded.simulation_run_id, "fc-worker") failed = store.fail_unstarted_simulation( @@ -373,7 +361,7 @@ def test_a_report_and_a_simulation_are_read_as_they_stand( with rolled_back_transaction(public_api_database_url) as connection: report = insert_pending_report(connection, insert_economy_catalog(connection)) act_as_the_service(connection) - store = _store_on(connection) + store = economy_store_on(connection) store.record_report_call(report.report_run_id, "fc-coordinator") stored_report = store.read_report(report.report_id) @@ -430,7 +418,7 @@ def test_a_reports_refused_values_name_the_side_they_belong_to( choices = {"baseline": CURRENT_LAW, "reform": CURRENT_LAW, side: refused} with pytest.raises(ReportSideError) as raised: - _store_on(connection).read_report_inputs( + economy_store_on(connection).read_report_inputs( REQUEST, baseline=choices["baseline"], reform=choices["reform"], @@ -440,29 +428,6 @@ def test_a_reports_refused_values_name_the_side_they_belong_to( assert isinstance(raised.value.error, error_type) -class _CommitFails: - """A transaction whose block runs, and whose commit then fails.""" - - def __init__(self, connection) -> None: - self.connection = connection - - def __enter__(self): - return self.connection - - def __exit__(self, error_type, error, traceback) -> None: - if error is None: - statement = "COMMIT" - raise OperationalError(statement, {}, Exception("connection lost")) - - -class _RecordingConnection: - def __init__(self) -> None: - self.statements: list[object] = [] - - def execute(self, statement, *arguments): - self.statements.append(statement) - - def _forced_rerun() -> NewReportRerun: """Return a forced report re-run, whose create only inserts rows.""" return NewReportRerun( @@ -482,14 +447,14 @@ def _forced_rerun() -> NewReportRerun: def test_a_create_whose_commit_fails_may_have_been_written() -> None: - connection = _RecordingConnection() - store = EconomyRunStore(lambda: _CommitFails(connection)) + session = RecordingSession() + store = EconomyRunStore(lambda: CommitFailingTransaction(session)) with pytest.raises(EconomyRecordOutcomeUnknownError) as raised: store.create_report_rerun(_forced_rerun()) assert isinstance(raised.value.__cause__, OperationalError) - assert connection.statements + assert session.statements class _StatementFails: diff --git a/libs/policyengine-simulation-persistence/tests/test_economy_sessions.py b/libs/policyengine-simulation-persistence/tests/test_economy_sessions.py new file mode 100644 index 000000000..61a398054 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_economy_sessions.py @@ -0,0 +1,1337 @@ +"""Economy reports, simulations, re-runs and windows on ORM sessions. + +The economy code runs its statements through ORM sessions on the mapped +classes. Find-or-create inserts, conditional updates and re-run locks are single +statements whose meaning depends on the database, so these tests run them on +real sessions: at once from two transactions, through a lost connection, and +past the statement limit. + +They use a database API v1's own v2 migration chain built +(``PUBLIC_API_SCHEMA_DATABASE_URL``). A test of the store's transactions commits +a catalog of its own and deletes it after; the others work in one transaction +rolled back afterwards. +""" + +from __future__ import annotations + +import inspect +import threading +from dataclasses import dataclass, replace +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any +from uuid import UUID, uuid4 + +import pytest +from committed_rows import ( + LOCK_WAIT_TIMEOUT_SECONDS, + run_together, + wait_until_another_backend_waits_for, +) +from conftest import ( + act_as_the_service, + in_session, + rolled_back_transaction, + session_in, +) +from economy_rows import ( + CURRENT_LAW, + POLICYENGINE_VERSION, + VERSIONS, + ReportRunOutcome, + RunShape, + StoredRun, + insert_economy_catalog, + insert_pending_report, + insert_report_of, + insert_report_run_of, + insert_simulation_run, +) +from policyengine_simulation_persistence import ( + BudgetWindowRun, + EconomyRecordOutcomeUnknownError, + EconomyRecordReferenceError, + EconomyReportDefinition, + EconomyRequest, + EconomyRunStore, + EconomySimulationDefinition, + KeyedBudgetWindow, + KeyedEconomyReport, + KeyedEconomySimulation, + KeyedWindowYear, + NewReportRerun, + NewSimulationRerun, + NewWindowRerun, + ProposedSide, + Report, + ReportRun, + ReportRunStatus, + RerunRecordDeletedError, + RerunSide, + RunVersions, + SharedRowNotSettledError, + Simulation, + SimulationChoice, + SimulationRun, + SimulationStatus, + WindowRerunYear, + create_session_factory, + economy_inputs, + economy_rerun_records, + economy_run_reads, + economy_run_records, + fail_unstarted_report_run, + fail_unstarted_simulation_run, + find_or_create_budget_window, + find_or_create_economy_report, + find_or_create_economy_simulation, + insert_report_rerun, + insert_simulation_rerun, + insert_window_rerun, + keyed_economy_records, + keyed_window_records, + policy_reform, + read_economy_report, + read_economy_simulation, + read_report_definition, + read_simulation_definition, + record_report_run_call, + record_simulation_run_call, + rerun_definitions, + shared_rows, +) +from policyengine_simulation_persistence.economy_run_records import ( + references_checked, +) +from policyengine_simulation_persistence.keyed_window_records import ( + WindowYearToInsert, + insert_window_years, + window_run_row, +) +from policyengine_simulation_persistence.shared_rows import find_or_insert_window_run +from public_api_rows import StoredValue, insert_dynamic, insert_policy +from session_probes import ( + end_connection_at, + end_connection_before_statement, + statements_naming_no_mapped_class, + statements_sent, +) +from sqlalchemy import ForeignKeyConstraint, event, func, select, text +from sqlalchemy.exc import OperationalError + +if TYPE_CHECKING: + from collections.abc import Callable + + from economy_rows import EconomyCatalog, PendingReport + from sqlalchemy import Connection, Engine + from sqlalchemy.orm import Session, sessionmaker + +pytestmark = pytest.mark.public_api_schema + +STATEMENT_TIMEOUT_MILLISECONDS = 5_000 +SHORT_STATEMENT_TIMEOUT_MILLISECONDS = 200 +START = datetime(2026, 1, 1, tzinfo=UTC) +CTC_AMOUNT = "gov.irs.credits.ctc.amount.base[0].amount" +ELASTICITY = "gov.simulation.labor_supply_responses.elasticities.income" +WINDOW_YEARS = (2026, 2027) +# The .py version and so the catalog version of ``VERSIONS``, on a newer country +# package: a re-run on these versions joins no run on ``VERSIONS``. +NEWER_PACKAGE_VERSIONS = RunVersions( + country_package_version="1.770.0", policyengine_version=POLICYENGINE_VERSION +) +REQUEST = EconomyRequest( + country="us", + policyengine_version=POLICYENGINE_VERSION, + region_code="us", + dataset_name=None, + year=2026, +) +# What a store's session runs without naming a mapped class: the statement limit +# and the re-run lock, which read no table. +STATEMENTS_ON_NO_TABLE = ("set_config", "pg_advisory_xact_lock") + +# Every function that runs a statement, whose first parameter is its session. +SESSION_FUNCTIONS = ( + (shared_rows, "find_or_insert_simulation"), + (shared_rows, "find_or_insert_report"), + (shared_rows, "find_or_insert_simulation_run"), + (shared_rows, "find_or_insert_simulation_runs"), + (shared_rows, "find_or_insert_window_run"), + (shared_rows, "find_joinable_window_run"), + (shared_rows, "find_or_insert_report_run"), + (shared_rows, "find_joinable_report_run"), + (shared_rows, "_find_or_insert"), + (economy_run_records, "read_model_id"), + (economy_run_records, "read_region"), + (economy_run_records, "read_dataset"), + (economy_run_records, "read_dataset_by_id"), + (economy_run_records, "read_local_area_dataset"), + (economy_run_records, "_dataset"), + (economy_run_records, "insert_simulation_run"), + (economy_run_records, "record_report_run_call"), + (economy_run_records, "record_simulation_run_call"), + (economy_run_records, "_record_call"), + (economy_run_records, "fail_unstarted_report_run"), + (economy_run_records, "fail_unstarted_simulation_run"), + (economy_run_records, "_fail_unstarted"), + (economy_run_reads, "read_economy_report"), + (economy_run_reads, "read_economy_simulation"), + (economy_rerun_records, "lock_reruns_of"), + (economy_rerun_records, "insert_report_rerun"), + (economy_rerun_records, "_insert_forced_report_run"), + (economy_rerun_records, "insert_window_rerun"), + (economy_rerun_records, "insert_household_report_rerun"), + (economy_rerun_records, "insert_simulation_rerun"), + (economy_inputs, "read_catalog_rows"), + (economy_inputs, "read_simulation_values"), + (economy_inputs, "read_side_values"), + (economy_inputs, "read_report_sides"), + (economy_inputs, "read_window_sides"), + (keyed_economy_records, "find_or_create_economy_report"), + (keyed_economy_records, "find_or_create_economy_simulation"), + (keyed_economy_records, "_find_or_insert_simulations"), + (keyed_economy_records, "find_or_insert_report_run_on_sides"), + (keyed_economy_records, "find_or_insert_side_runs"), + (keyed_window_records, "find_or_create_budget_window"), + (keyed_window_records, "_find_or_insert_simulations"), + (keyed_window_records, "insert_window_years"), + (keyed_window_records, "stored_window_years"), + (rerun_definitions, "read_report_definition"), + (rerun_definitions, "_window_years"), + (rerun_definitions, "read_simulation_definition"), + (policy_reform, "read_policy_reform"), + (policy_reform, "read_dynamic_reform"), + (policy_reform, "_read_reform"), +) + + +def _keyed_report( + economy: EconomyCatalog, *, reform: SimulationChoice = CURRENT_LAW +) -> KeyedEconomyReport: + """Return a request's report on ``economy``, with fresh IDs to propose.""" + return KeyedEconomyReport( + definition=EconomyReportDefinition( + country=economy.catalog.country, + baseline=CURRENT_LAW, + reform=reform, + dataset_name="populace_us_2024", + region_code="us", + year=2026, + budget_window_size=None, + spm=None, + ), + report_id=uuid4(), + run_id=uuid4(), + baseline=ProposedSide(simulation_id=uuid4(), simulation_run_id=uuid4()), + reform=ProposedSide(simulation_id=uuid4(), simulation_run_id=uuid4()), + tax_benefit_model_id=economy.catalog.model_id, + population=economy.population(), + versions=VERSIONS, + inputs={}, + ) + + +def _keyed_simulation( + economy: EconomyCatalog, *, choice: SimulationChoice = CURRENT_LAW +) -> KeyedEconomySimulation: + return KeyedEconomySimulation( + definition=EconomySimulationDefinition( + country=economy.catalog.country, + choice=choice, + dataset_name="populace_us_2024", + region_code="us", + year=2026, + spm=None, + ), + simulation_id=uuid4(), + run_id=uuid4(), + population=economy.population(), + versions=VERSIONS, + ) + + +def _keyed_window( + economy: EconomyCatalog, *, reform: SimulationChoice = CURRENT_LAW +) -> KeyedBudgetWindow: + return KeyedBudgetWindow( + definition=EconomyReportDefinition( + country=economy.catalog.country, + baseline=CURRENT_LAW, + reform=reform, + dataset_name="populace_us_2024", + region_code="us", + year=WINDOW_YEARS[0], + budget_window_size=len(WINDOW_YEARS), + spm=None, + ), + report_id=uuid4(), + window_run_id=uuid4(), + tax_benefit_model_id=economy.catalog.model_id, + population=economy.population(), + versions=VERSIONS, + inputs={}, + years=tuple( + KeyedWindowYear( + year=year, + report_run_id=uuid4(), + baseline=ProposedSide(simulation_id=uuid4(), simulation_run_id=uuid4()), + reform=ProposedSide(simulation_id=uuid4(), simulation_run_id=uuid4()), + ) + for year in WINDOW_YEARS + ), + ) + + +@dataclass(frozen=True) +class _StoredRecords: + """A catalog, with a stored report and a stored budget-window report on it.""" + + economy: EconomyCatalog + report_id: UUID + window_report_id: UUID + baseline: StoredRun + reform: StoredRun + + +def _insert_stored_records( + connection: Connection, economy: EconomyCatalog +) -> _StoredRecords: + """Insert, as the superuser, the records the re-runs here calculate again.""" + baseline = insert_simulation_run(connection, economy.population()) + reform = insert_simulation_run(connection, economy.population()) + compared = (baseline, reform) + report_id = insert_report_of(connection, economy, compared) + insert_report_run_of( + connection, report_id, compared, ReportRunOutcome(status=ReportRunStatus.FAILED) + ) + window_report_id = insert_report_of( + connection, economy, compared, budget_window_size=len(WINDOW_YEARS) + ) + return _StoredRecords( + economy=economy, + report_id=report_id, + window_report_id=window_report_id, + baseline=baseline, + reform=reform, + ) + + +def _rerun_side(stored: StoredRun) -> RerunSide: + return RerunSide(simulation_id=stored.simulation_id, simulation_run_id=uuid4()) + + +def _report_rerun(records: _StoredRecords, *, forced: bool = False) -> NewReportRerun: + return NewReportRerun( + report_id=records.report_id, + run_id=uuid4(), + population=records.economy.population(), + versions=NEWER_PACKAGE_VERSIONS, + baseline=_rerun_side(records.baseline), + reform=_rerun_side(records.reform), + forced=forced, + ) + + +def _window_rerun(records: _StoredRecords, *, forced: bool = False) -> NewWindowRerun: + return NewWindowRerun( + report_id=records.window_report_id, + window_run_id=uuid4(), + population=records.economy.population(), + versions=NEWER_PACKAGE_VERSIONS, + recalculate=forced, + years=tuple( + WindowRerunYear( + year=year, + report_run_id=uuid4(), + baseline=_rerun_side(records.baseline), + reform=_rerun_side(records.reform), + ) + for year in WINDOW_YEARS + ), + ) + + +def _simulation_rerun(records: _StoredRecords) -> NewSimulationRerun: + return NewSimulationRerun( + simulation_id=records.baseline.simulation_id, + run_id=uuid4(), + population=records.economy.population(), + versions=NEWER_PACKAGE_VERSIONS, + ) + + +def _create_report(store: EconomyRunStore, records: _StoredRecords) -> object: + return store.find_or_create_report(_keyed_report(records.economy)) + + +def _create_simulation(store: EconomyRunStore, records: _StoredRecords) -> object: + return store.find_or_create_simulation(_keyed_simulation(records.economy)) + + +def _create_window(store: EconomyRunStore, records: _StoredRecords) -> object: + return store.find_or_create_budget_window(_keyed_window(records.economy)) + + +def _create_report_rerun(store: EconomyRunStore, records: _StoredRecords) -> object: + return store.create_report_rerun(_report_rerun(records)) + + +def _create_forced_report_rerun( + store: EconomyRunStore, records: _StoredRecords +) -> object: + return store.create_report_rerun(_report_rerun(records, forced=True)) + + +def _create_window_rerun(store: EconomyRunStore, records: _StoredRecords) -> object: + return store.create_window_rerun(_window_rerun(records)) + + +def _create_forced_window_rerun( + store: EconomyRunStore, records: _StoredRecords +) -> object: + return store.create_window_rerun(_window_rerun(records, forced=True)) + + +def _create_simulation_rerun(store: EconomyRunStore, records: _StoredRecords) -> object: + return store.create_simulation_rerun(_simulation_rerun(records)) + + +# Each way the store creates rows, and the first statement that writes the row +# the create exists to make. +CREATES: dict[str, tuple[Callable[[EconomyRunStore, _StoredRecords], object], str]] = { + "report": (_create_report, "INSERT INTO public.report_runs"), + "simulation": (_create_simulation, "INSERT INTO public.simulation_runs"), + "budget window": (_create_window, "INSERT INTO public.budget_window_runs"), + "report re-run": (_create_report_rerun, "INSERT INTO public.report_runs"), + "forced report re-run": ( + _create_forced_report_rerun, + "INSERT INTO public.report_runs", + ), + "window re-run": (_create_window_rerun, "INSERT INTO public.budget_window_runs"), + "forced window re-run": ( + _create_forced_window_rerun, + "INSERT INTO public.report_runs", + ), + "simulation re-run": ( + _create_simulation_rerun, + "INSERT INTO public.simulation_runs", + ), +} + + +@pytest.fixture +def session_factory(committed_engine) -> sessionmaker[Session]: + return create_session_factory(committed_engine) + + +@pytest.fixture +def stored_records(committed_engine, committed_catalog) -> _StoredRecords: + with committed_engine.begin() as connection: + return _insert_stored_records(connection, committed_catalog) + + +def _store( + session_factory: sessionmaker[Session], + *, + statement_timeout_milliseconds: int = STATEMENT_TIMEOUT_MILLISECONDS, +) -> EconomyRunStore: + return EconomyRunStore.from_session_factory( + session_factory, + statement_timeout_milliseconds=statement_timeout_milliseconds, + ) + + +def _rows_under(engine: Engine, economy: EconomyCatalog) -> tuple[int, int, int]: + """Count the catalog's simulation runs, report runs and window runs.""" + with engine.connect() as connection: + counts = connection.execute( + text( + "SELECT (SELECT count(*) FROM public.simulation_runs " + "WHERE tax_benefit_model_version_id = :version_id), " + "(SELECT count(*) FROM public.report_runs WHERE report_id IN " + "(SELECT id FROM public.reports WHERE tax_benefit_model_id = :model_id)" + "), (SELECT count(*) FROM public.budget_window_runs WHERE report_id IN " + "(SELECT id FROM public.reports WHERE tax_benefit_model_id = :model_id)" + ")" + ), + { + "version_id": economy.catalog.version_id, + "model_id": economy.catalog.model_id, + }, + ).one() + return (counts[0], counts[1], counts[2]) + + +def _only_statements_on_no_table(statements: list[str]) -> bool: + return all( + any(function in statement for function in STATEMENTS_ON_NO_TABLE) + for statement in statements + ) + + +@pytest.mark.parametrize( + ("module", "function_name"), + SESSION_FUNCTIONS, + ids=[f"{module.__name__}.{name}" for module, name in SESSION_FUNCTIONS], +) +def test_each_function_running_a_statement_takes_a_session( + module: Any, function_name: str +) -> None: + (first_parameter, *_) = inspect.signature( + getattr(module, function_name) + ).parameters.values() + + assert first_parameter.name == "session" + assert first_parameter.annotation == "Session" + + +def test_the_store_is_built_on_a_session_factory_alone() -> None: + assert "from_engine" not in vars(EconomyRunStore) + assert "from_session_factory" in vars(EconomyRunStore) + + +def test_each_lookup_names_the_mapped_class_it_inserts_into() -> None: + assert shared_rows._SIMULATION_DEFINITION.model is Simulation + assert shared_rows._REPORT_DEFINITION.model is Report + assert shared_rows._JOINABLE_SIMULATION_RUN.model is SimulationRun + assert shared_rows._JOINABLE_REPORT_RUN.model is ReportRun + assert shared_rows._JOINABLE_WINDOW_RUN.model is BudgetWindowRun + + +def test_each_refused_key_is_known_by_its_table_and_column() -> None: + """Every foreign key of the five classes is found through their arguments.""" + expected = { + str(constraint.name): (model.__tablename__, constraint.column_keys[0]) + for model in (Simulation, SimulationRun, Report, ReportRun, BudgetWindowRun) + for constraint in model.__table__.constraints + if isinstance(constraint, ForeignKeyConstraint) + } + + assert expected + assert expected == economy_run_records._KEY_COLUMNS + + +def test_every_economy_read_names_mapped_classes(public_api_database_url) -> None: + with rolled_back_transaction(public_api_database_url) as connection: + economy = insert_economy_catalog( + connection, parameter_names=(CTC_AMOUNT, ELASTICITY) + ) + policy_id = insert_policy( + connection, economy.catalog, [StoredValue(CTC_AMOUNT, 3_000, START)] + ) + dynamic_id = insert_dynamic( + connection, economy.catalog, [StoredValue(ELASTICITY, -0.05, START)] + ) + records = _insert_stored_records(connection, economy) + act_as_the_service(connection) + + with session_in(connection) as session: + core_statements = statements_naming_no_mapped_class(session) + catalog = economy_inputs.read_catalog_rows(session, REQUEST) + sides = economy_inputs.read_report_sides( + session, + REQUEST, + catalog, + baseline=CURRENT_LAW, + reform=SimulationChoice(policy_id=policy_id, dynamic_id=dynamic_id), + ) + local_area_dataset = economy_run_records.read_local_area_dataset( + session, catalog_version_id=economy.catalog.version_id + ) + named_dataset = economy_run_records.read_dataset( + session, + catalog_version_id=economy.catalog.version_id, + name="populace_us_2024", + ) + report_state = read_economy_report(session, records.report_id) + simulation_state = read_economy_simulation( + session, records.baseline.simulation_id + ) + report_definition = read_report_definition(session, records.report_id) + window_definition = read_report_definition( + session, records.window_report_id + ) + simulation_definition = read_simulation_definition( + session, records.baseline.simulation_id + ) + + assert core_statements == [] + assert catalog.dataset.name == named_dataset.name == "populace_us_2024" + assert sides.reform.reform == { + CTC_AMOUNT: {"2026-01-01.2200-12-31": 3_000}, + ELASTICITY: {"2026-01-01.2200-12-31": -0.05}, + } + assert local_area_dataset is None + assert report_state.run.status is ReportRunStatus.FAILED + assert simulation_state.run.status is SimulationStatus.SUCCEEDED + assert report_definition.kind == "economy" + assert report_definition.region_code == "us" + assert window_definition.kind == "budget_window" + assert simulation_definition.country == "us" + + +def test_every_economy_write_names_mapped_classes(public_api_database_url) -> None: + with rolled_back_transaction(public_api_database_url) as connection: + economy = insert_economy_catalog(connection) + policy_id = insert_policy(connection, economy.catalog) + records = _insert_stored_records(connection, economy) + pending = insert_pending_report(connection, economy) + act_as_the_service(connection) + reform = SimulationChoice(policy_id=policy_id, dynamic_id=None) + + with session_in(connection) as session: + core_statements = statements_naming_no_mapped_class(session) + report = find_or_create_economy_report( + session, _keyed_report(economy, reform=reform) + ) + simulation = find_or_create_economy_simulation( + session, _keyed_simulation(economy, choice=reform) + ) + window = find_or_create_budget_window( + session, _keyed_window(economy, reform=reform) + ) + insert_report_rerun(session, _report_rerun(records)) + insert_report_rerun(session, _report_rerun(records, forced=True)) + insert_window_rerun(session, _window_rerun(records)) + insert_window_rerun(session, _window_rerun(records, forced=True)) + insert_simulation_rerun(session, _simulation_rerun(records)) + record_report_run_call( + session, report_run_id=report.run.id, modal_call_id="fc-report" + ) + record_simulation_run_call( + session, simulation_run_id=simulation.run.id, modal_call_id="fc-sim" + ) + failed_report = fail_unstarted_report_run( + session, + report_run_id=pending.report_run_id, + simulation_run_ids=pending.simulation_run_ids, + error_message="Modal refused the call.", + ) + failed_simulation = fail_unstarted_simulation_run( + session, + simulation_run_id=pending.baseline.simulation_run_id, + error_message="already failed with its report", + ) + + assert _only_statements_on_no_table(core_statements) + assert report.run.inserted is True + assert window.window_run.inserted is True + assert failed_report is True + assert failed_simulation is False + + +def test_a_find_or_insert_sends_one_conflict_handling_insert_per_lookup( + public_api_database_url, +) -> None: + """Each insert names its unique index, partial or whole, and does nothing.""" + with rolled_back_transaction(public_api_database_url) as connection: + economy = insert_economy_catalog(connection) + act_as_the_service(connection) + statements = statements_sent(connection) + + in_session(connection, find_or_create_economy_report, _keyed_report(economy)) + in_session(connection, find_or_create_budget_window, _keyed_window(economy)) + + inserts = [ + " ".join(statement.split()) + for statement in statements + if statement.startswith("INSERT INTO") + ] + assert any( + statement.startswith("INSERT INTO public.simulations ") + and "ON CONFLICT (key_version, definition_key) DO NOTHING" in statement + for statement in inserts + ) + assert any( + statement.startswith("INSERT INTO public.reports ") + and "ON CONFLICT (key_version, definition_key) DO NOTHING" in statement + for statement in inserts + ) + assert any( + statement.startswith("INSERT INTO public.simulation_runs ") + and "WHERE status <> 'failed' AND NOT forced DO NOTHING" in statement + for statement in inserts + ) + assert any( + statement.startswith("INSERT INTO public.report_runs ") + and "budget_window_run_id IS NULL AND status <> 'failed' AND NOT forced " + "DO NOTHING" + in statement + for statement in inserts + ) + assert any( + statement.startswith("INSERT INTO public.budget_window_runs ") + and "ON CONFLICT" in statement + and "NOT recalculate DO NOTHING" in statement + for statement in inserts + ) + + +def test_a_report_goes_through_the_stores_sessions_and_stays_readable( + committed_engine, committed_catalog, session_factory +) -> None: + core_statements = statements_naming_no_mapped_class(session_factory) + store = _store(session_factory) + + found = store.find_or_create_report(_keyed_report(committed_catalog)) + joined = store.find_or_create_report(_keyed_report(committed_catalog)) + status = store.record_report_call(found.run.id, "fc-report") + state = store.read_report(found.report_id) + definition = store.read_report_definition(found.report_id) + + assert _only_statements_on_no_table(core_statements) + assert (found.run.inserted, joined.run.inserted) == (True, False) + assert joined.run.id == found.run.id + assert status is ReportRunStatus.PENDING + assert state.run.id == found.run.id + assert state.run.created_at is not None + assert definition.keyed is True + with committed_engine.connect() as connection: + assert ( + connection.execute( + select(ReportRun.modal_call_id).where(ReportRun.id == found.run.id) + ).scalar_one() + == "fc-report" + ) + + +def test_a_simulation_goes_through_the_stores_sessions_and_stays_readable( + committed_catalog, session_factory +) -> None: + core_statements = statements_naming_no_mapped_class(session_factory) + store = _store(session_factory) + + found = store.find_or_create_simulation(_keyed_simulation(committed_catalog)) + failed = store.fail_unstarted_simulation(found.run.id, "Modal refused the call.") + state = store.read_simulation(found.simulation_id) + definition = store.read_simulation_definition(found.simulation_id) + + assert _only_statements_on_no_table(core_statements) + assert failed is True + assert state.run.status is SimulationStatus.FAILED + assert state.run.error_message == "Modal refused the call." + assert state.run.completed_at is not None + assert definition.kind == "economy" + + +def test_a_budget_window_and_its_reruns_go_through_the_stores_sessions( + committed_catalog, stored_records, session_factory +) -> None: + core_statements = statements_naming_no_mapped_class(session_factory) + store = _store(session_factory) + rerun = _window_rerun(stored_records) + + created = store.find_or_create_budget_window(_keyed_window(committed_catalog)) + joined = store.find_or_create_budget_window(_keyed_window(committed_catalog)) + rerun_created = store.create_window_rerun(rerun) + rerun_joined = store.create_window_rerun(replace(rerun, window_run_id=uuid4())) + + assert _only_statements_on_no_table(core_statements) + assert (created.window_run.inserted, joined.window_run.inserted) == (True, False) + assert [year.year for year in joined.years] == list(WINDOW_YEARS) + assert rerun_created.window_run.inserted is True + assert rerun_joined.window_run.id == rerun_created.window_run.id + assert rerun_joined.window_run.inserted is False + assert [year.report_run_id for year in rerun_joined.years] == [ + year.report_run_id for year in rerun_created.years + ] + + +@pytest.mark.parametrize("create_name", CREATES) +def test_a_connection_lost_at_a_creates_commit_leaves_the_outcome_unknown( + committed_engine, stored_records, session_factory, create_name: str +) -> None: + """Every statement ran, and PostgreSQL may have applied the commit.""" + create, _ = CREATES[create_name] + unflushed_counts = end_connection_at( + session_factory, "before_commit", committed_engine + ) + + with pytest.raises(EconomyRecordOutcomeUnknownError) as raised: + create(_store(session_factory), stored_records) + + assert isinstance(raised.value.__cause__, OperationalError) + # Nothing was left for the commit's flush to send. + assert unflushed_counts == [0] + + +@pytest.mark.parametrize("create_name", CREATES) +def test_a_connection_lost_during_a_creates_statements_is_not_a_lost_commit( + committed_engine, stored_records, session_factory, create_name: str +) -> None: + create, statement_start = CREATES[create_name] + before = _rows_under(committed_engine, stored_records.economy) + ended_backends = end_connection_before_statement(committed_engine, statement_start) + + with pytest.raises(OperationalError): + create(_store(session_factory), stored_records) + + assert len(ended_backends) == 1 + assert _rows_under(committed_engine, stored_records.economy) == before + + +def test_a_statement_outlasting_the_limit_is_not_a_lost_commit( + committed_engine, stored_records, session_factory +) -> None: + """The limit covers each statement, and a cancelled INSERT wrote nothing.""" + store = _store( + session_factory, + statement_timeout_milliseconds=SHORT_STATEMENT_TIMEOUT_MILLISECONDS, + ) + before = _rows_under(committed_engine, stored_records.economy) + + with committed_engine.begin() as locking_connection: + locking_connection.execute( + text("LOCK TABLE public.simulation_runs IN ACCESS EXCLUSIVE MODE") + ) + with pytest.raises(OperationalError, match="statement timeout"): + store.create_simulation_rerun(_simulation_rerun(stored_records)) + + assert _rows_under(committed_engine, stored_records.economy) == before + + +def test_a_refused_reference_is_named_and_is_not_a_lost_commit( + committed_engine, stored_records, session_factory +) -> None: + economy = stored_records.economy + before = _rows_under(committed_engine, economy) + missing_policy = SimulationChoice(policy_id=uuid4(), dynamic_id=None) + + with pytest.raises(EconomyRecordReferenceError) as refused: + _store(session_factory).find_or_create_simulation( + _keyed_simulation(economy, choice=missing_policy) + ) + + assert (refused.value.table_name, refused.value.column_name) == ( + "simulations", + "policy_id", + ) + assert _rows_under(committed_engine, economy) == before + + +@pytest.mark.parametrize( + "create_with_a_deleted_record", + [ + lambda store, records: store.create_report_rerun( + replace(_report_rerun(records), report_id=uuid4()) + ), + lambda store, records: store.create_report_rerun( + replace(_report_rerun(records, forced=True), report_id=uuid4()) + ), + lambda store, records: store.create_window_rerun( + replace(_window_rerun(records), report_id=uuid4()) + ), + lambda store, records: store.create_window_rerun( + replace(_window_rerun(records, forced=True), report_id=uuid4()) + ), + lambda store, records: store.create_simulation_rerun( + replace(_simulation_rerun(records), simulation_id=uuid4()) + ), + ], + ids=[ + "report", + "forced report", + "window", + "forced window", + "simulation", + ], +) +def test_a_rerun_of_a_deleted_record_says_the_record_is_gone( + committed_engine, stored_records, session_factory, create_with_a_deleted_record +) -> None: + before = _rows_under(committed_engine, stored_records.economy) + + with pytest.raises(RerunRecordDeletedError): + create_with_a_deleted_record(_store(session_factory), stored_records) + + assert _rows_under(committed_engine, stored_records.economy) == before + + +def test_an_unforced_rerun_refused_inside_its_savepoint_keeps_nothing( + committed_engine, stored_records, session_factory +) -> None: + """The reform's simulation is gone: the baseline's new run is taken back.""" + rerun = replace( + _report_rerun(stored_records), + reform=RerunSide( + simulation_id=UUID("ffffffff-ffff-4fff-bfff-ffffffffffff"), + simulation_run_id=uuid4(), + ), + ) + before = _rows_under(committed_engine, stored_records.economy) + statements = statements_sent(committed_engine) + + with pytest.raises(EconomyRecordReferenceError) as refused: + _store(session_factory).create_report_rerun(rerun) + + assert (refused.value.table_name, refused.value.column_name) == ( + "simulation_runs", + "simulation_id", + ) + assert [ + statement.split(" sa_savepoint")[0] + for statement in statements + if "SAVEPOINT" in statement + ] == ["SAVEPOINT", "ROLLBACK TO SAVEPOINT"] + assert _rows_under(committed_engine, stored_records.economy) == before + + +def test_a_report_losing_its_run_on_a_session_joins_it_and_keeps_nothing( + committed_engine, committed_catalog, session_factory +) -> None: + """Another request inserts the report run between the lookup and the insert.""" + economy = committed_catalog + store = _store(session_factory) + first = store.find_or_create_report(_keyed_report(economy)) + with committed_engine.begin() as connection: + for run_table, run_id in ( + ("simulation_runs", first.baseline.simulation_run_id), + ("report_runs", first.run.id), + ): + connection.execute( + text( + f"UPDATE public.{run_table} SET status = 'failed', " # noqa: S608 # fixed table names + "completed_at = now() WHERE id = :id" + ), + {"id": run_id}, + ) + competing: list[UUID] = [] + request = _keyed_report(economy) + + def create_a_competing_run_first( + _connection, _cursor, statement, _parameters, _context, _executemany + ) -> None: + if competing or not statement.startswith("INSERT INTO public.report_runs"): + return + run_id = uuid4() + competing.append(run_id) + with committed_engine.begin() as other: + other.execute( + text( + "INSERT INTO public.report_runs (id, report_id, " + "baseline_simulation_run_id, reform_simulation_run_id, " + "country_package_version, policyengine_version, status) " + "SELECT :id, report_id, :side_run, :side_run, " + "country_package_version, policyengine_version, 'pending' " + "FROM public.report_runs WHERE id = :failed_run" + ), + { + "id": run_id, + "side_run": first.reform.simulation_run_id, + "failed_run": first.run.id, + }, + ) + + event.listen( + committed_engine, "before_cursor_execute", create_a_competing_run_first + ) + + joined = store.find_or_create_report(request) + + assert (joined.run.id, joined.run.inserted) == (competing[0], False) + assert joined.baseline.run_inserted is False + with committed_engine.connect() as connection: + kept = connection.execute( + select(func.count()) + .select_from(SimulationRun) + .where(SimulationRun.id == request.baseline.simulation_run_id) + ).scalar_one() + assert kept == 0 + + +def _at_once( + engine: Engine, + session_factory: sessionmaker[Session], + work: Callable[[Session, str], None], +) -> list[str]: + """Run ``work`` in two sessions, the first holding until the second waits. + + Return the statement the second was waiting in. + """ + first_has_written = threading.Event() + waited_on: list[str] = [] + + def first_request() -> None: + with session_factory.begin() as session: + act_as_the_service(session) + work(session, "first") + holder_pid = session.execute(select(func.pg_backend_pid())).scalar_one() + first_has_written.set() + waited_on.append(wait_until_another_backend_waits_for(engine, holder_pid)) + + def second_request() -> None: + assert first_has_written.wait(LOCK_WAIT_TIMEOUT_SECONDS) + with session_factory.begin() as session: + act_as_the_service(session) + work(session, "second") + + run_together((first_request, second_request), released=first_has_written) + return waited_on + + +def test_two_identical_report_requests_at_once_end_on_one_run( + committed_engine, committed_catalog, session_factory +) -> None: + found: dict[str, Any] = {} + + def create(session: Session, name: str) -> None: + found[name] = find_or_create_economy_report( + session, _keyed_report(committed_catalog) + ) + + _at_once(committed_engine, session_factory, create) + + first, second = found["first"], found["second"] + assert (second.report_id, second.run.id) == (first.report_id, first.run.id) + assert (first.run.inserted, second.run.inserted) == (True, False) + assert second.baseline == replace(first.baseline, run_inserted=False) + + +def test_two_identical_simulation_requests_at_once_end_on_one_run( + committed_engine, committed_catalog, session_factory +) -> None: + found: dict[str, Any] = {} + + def create(session: Session, name: str) -> None: + found[name] = find_or_create_economy_simulation( + session, _keyed_simulation(committed_catalog) + ) + + _at_once(committed_engine, session_factory, create) + + first, second = found["first"], found["second"] + assert second.simulation_id == first.simulation_id + assert second.run.id == first.run.id + assert (first.run.inserted, second.run.inserted) == (True, False) + + +def test_two_identical_window_requests_at_once_end_on_one_window_run( + committed_engine, committed_catalog, session_factory +) -> None: + found: dict[str, Any] = {} + + def create(session: Session, name: str) -> None: + found[name] = find_or_create_budget_window( + session, _keyed_window(committed_catalog) + ) + + _at_once(committed_engine, session_factory, create) + + first, second = found["first"], found["second"] + assert second.window_run.id == first.window_run.id + assert (first.window_run.inserted, second.window_run.inserted) == (True, False) + assert [year.report_run_id for year in second.years] == [ + year.report_run_id for year in first.years + ] + + +def test_a_second_unforced_rerun_waits_for_the_first_and_joins_its_run( + committed_engine, stored_records, session_factory +) -> None: + """The re-run lock holds it until the first transaction has committed.""" + found: dict[str, Any] = {} + + def rerun(session: Session, name: str) -> None: + found[name] = insert_report_rerun(session, _report_rerun(stored_records)) + + waited_on = _at_once(committed_engine, session_factory, rerun) + + assert "pg_advisory_xact_lock" in waited_on[0] + assert (found["first"].inserted, found["second"].inserted) == (True, False) + assert found["second"].id == found["first"].id + + +def test_a_second_unforced_window_rerun_waits_for_the_first_and_joins_it( + committed_engine, stored_records, session_factory +) -> None: + found: dict[str, Any] = {} + + def rerun(session: Session, name: str) -> None: + found[name] = insert_window_rerun(session, _window_rerun(stored_records)) + + waited_on = _at_once(committed_engine, session_factory, rerun) + + assert "pg_advisory_xact_lock" in waited_on[0] + first, second = found["first"], found["second"] + assert second.window_run.id == first.window_run.id + assert (first.window_run.inserted, second.window_run.inserted) == (True, False) + + +def _committed_pending_report(engine: Engine, economy: EconomyCatalog) -> PendingReport: + with engine.begin() as connection: + return insert_pending_report(connection, economy) + + +def test_a_call_recorded_at_once_by_two_sessions_keeps_the_first( + committed_engine, committed_catalog, session_factory +) -> None: + """The second waits on the row, then finds it no longer without a call.""" + pending = _committed_pending_report(committed_engine, committed_catalog) + statuses: dict[str, ReportRunStatus | None] = {} + + def record(session: Session, name: str) -> None: + statuses[name] = record_report_run_call( + session, report_run_id=pending.report_run_id, modal_call_id=f"fc-{name}" + ) + + waited_on = _at_once(committed_engine, session_factory, record) + + assert waited_on[0].lstrip().startswith("UPDATE public.report_runs") + assert statuses == { + "first": ReportRunStatus.PENDING, + "second": ReportRunStatus.PENDING, + } + with committed_engine.connect() as connection: + recorded = connection.execute( + select(ReportRun.modal_call_id).where(ReportRun.id == pending.report_run_id) + ).scalar_one() + assert recorded == "fc-first" + + +def test_a_failure_after_a_call_was_recorded_at_once_changes_nothing( + committed_engine, committed_catalog, session_factory +) -> None: + """A run whose call another session records first is left to that call.""" + pending = _committed_pending_report(committed_engine, committed_catalog) + failed: list[bool] = [] + + def record_or_fail(session: Session, name: str) -> None: + if name == "first": + record_report_run_call( + session, report_run_id=pending.report_run_id, modal_call_id="fc-first" + ) + return + failed.append( + fail_unstarted_report_run( + session, + report_run_id=pending.report_run_id, + simulation_run_ids=pending.simulation_run_ids, + error_message="Modal refused the call.", + ) + ) + + _at_once(committed_engine, session_factory, record_or_fail) + + assert failed == [False] + with committed_engine.connect() as connection: + statuses = connection.execute( + select(ReportRun.status, ReportRun.error_message).where( + ReportRun.id == pending.report_run_id + ) + ).one() + assert statuses == (ReportRunStatus.PENDING, None) + + +def test_each_conditional_update_is_one_statement_that_leaves_loaded_rows_alone( + public_api_database_url, +) -> None: + """A read after it returns the update, whatever the session has loaded.""" + with rolled_back_transaction(public_api_database_url) as connection: + economy = insert_economy_catalog(connection) + pending = insert_pending_report(connection, economy) + act_as_the_service(connection) + statements = statements_sent(connection) + + with session_in(connection) as session: + loaded_run = session.get(ReportRun, pending.report_run_id) + failed = fail_unstarted_report_run( + session, + report_run_id=pending.report_run_id, + simulation_run_ids=(), + error_message="Modal refused the call.", + ) + state = read_economy_report(session, pending.report_id) + loaded_status = loaded_run.status + + updates = [ + " ".join(statement.split()) + for statement in statements + if statement.startswith("UPDATE public.report_runs") + ] + assert failed is True + assert len(updates) == 1 + assert "status =" in updates[0] + assert "modal_call_id IS NULL" in updates[0] + assert "RETURNING" in updates[0] + assert state.run.status is ReportRunStatus.FAILED + assert state.run.completed_at is not None + # The loaded object is the session's own; the update didn't touch it. + assert loaded_status is ReportRunStatus.PENDING + + +def test_the_statements_of_a_forced_window_rerun_mark_every_new_run_forced( + public_api_database_url, +) -> None: + with rolled_back_transaction(public_api_database_url) as connection: + economy = insert_economy_catalog(connection) + records = _insert_stored_records(connection, economy) + act_as_the_service(connection) + rerun = _window_rerun(records, forced=True) + + with session_in(connection) as session: + found = insert_window_rerun(session, rerun) + year_runs = session.execute( + select( + ReportRun.year, + ReportRun.forced, + ReportRun.modal_call_id, + ReportRun.status, + ) + .where(ReportRun.budget_window_run_id == rerun.window_run_id) + .order_by(ReportRun.year) + ).all() + forced_side_runs = session.execute( + select(func.count()) + .select_from(SimulationRun) + .where( + SimulationRun.id.in_( + [year.baseline.simulation_run_id for year in rerun.years] + + [year.reform.simulation_run_id for year in rerun.years] + ), + SimulationRun.forced.is_(True), + ) + ).scalar_one() + + assert found.window_run.inserted is True + assert [tuple(row) for row in year_runs] == [ + (year, True, None, ReportRunStatus.PENDING) for year in WINDOW_YEARS + ] + assert forced_side_runs == 2 * len(WINDOW_YEARS) + + +def test_a_pending_simulation_run_the_session_inserts_is_a_stored_row( + public_api_database_url, +) -> None: + with rolled_back_transaction(public_api_database_url) as connection: + economy = insert_economy_catalog(connection) + stored = insert_simulation_run( + connection, economy.population(), RunShape(status=SimulationStatus.FAILED) + ) + act_as_the_service(connection) + run_id = uuid4() + + with session_in(connection) as session: + economy_run_records.insert_simulation_run( + session, + run_id=run_id, + simulation_id=stored.simulation_id, + population=economy.population(), + versions=VERSIONS, + forced=True, + ) + stored_run = session.execute( + select( + SimulationRun.status, + SimulationRun.forced, + SimulationRun.modal_call_id, + ).where(SimulationRun.id == run_id) + ).one() + + assert tuple(stored_run) == (SimulationStatus.PENDING, True, None) + + +def test_a_windows_year_runs_are_inserted_with_every_column_they_set( + public_api_database_url, +) -> None: + """A year's run sets ``modal_call_id`` to NULL, as a single insert would.""" + with rolled_back_transaction(public_api_database_url) as connection: + economy = insert_economy_catalog(connection) + records = _insert_stored_records(connection, economy) + act_as_the_service(connection) + statements = statements_sent(connection) + + in_session(connection, insert_window_rerun, _window_rerun(records, forced=True)) + + (year_runs_insert,) = { + statement + for statement in statements + if statement.startswith("INSERT INTO public.report_runs") + } + assert "modal_call_id" in year_runs_insert + + +def test_a_key_refused_among_a_windows_year_runs_is_named( + public_api_database_url, +) -> None: + """The multi-row insert's refusal names the key, as a single insert's does.""" + with rolled_back_transaction(public_api_database_url) as connection: + economy = insert_economy_catalog(connection) + records = _insert_stored_records(connection, economy) + act_as_the_service(connection) + rerun = _window_rerun(records, forced=True) + years = tuple( + WindowYearToInsert( + year=year.year, + report_run_id=year.report_run_id, + simulation_ids={ + "baseline": year.baseline.simulation_id, + "reform": year.reform.simulation_id, + }, + proposed_run_ids={ + "baseline": year.baseline.simulation_run_id, + "reform": year.reform.simulation_run_id, + }, + ) + for year in rerun.years + ) + + with session_in(connection) as session: + window_run = find_or_insert_window_run( + session, + window_run_row( + run_id=rerun.window_run_id, + report_id=records.window_report_id, + population=rerun.population, + versions=rerun.versions, + recalculate=False, + ), + ) + savepoint = session.begin_nested() + with ( + pytest.raises(EconomyRecordReferenceError) as refused, + references_checked(), + ): + insert_window_years( + session, + years, + report_id=uuid4(), + window_run_id=window_run.id, + population=rerun.population, + versions=rerun.versions, + forced=True, + ) + savepoint.rollback() + + assert (refused.value.table_name, refused.value.column_name) == ( + "report_runs", + "report_id", + ) + + +class _SessionFindingNothing: + """A session whose inserts conflict and whose reads then find no row.""" + + def __init__(self) -> None: + self.statements: list[object] = [] + + def execute(self, statement: object) -> _SessionFindingNothing: + self.statements.append(statement) + return self + + def one_or_none(self) -> None: + return None + + +def test_a_lookup_gives_up_on_a_run_that_keeps_failing_before_its_read() -> None: + session = _SessionFindingNothing() + run = { + "simulation_id": uuid4(), + "year": 2026, + "tax_benefit_model_version_id": uuid4(), + "country_package_version": VERSIONS.country_package_version, + "policyengine_version": VERSIONS.policyengine_version, + } + + with pytest.raises(SharedRowNotSettledError, match="simulation_runs"): + shared_rows.find_or_insert_simulation_run(session, run) + + assert len(session.statements) == 2 * shared_rows.INSERT_ATTEMPTS diff --git a/libs/policyengine-simulation-persistence/tests/test_economy_window_reruns.py b/libs/policyengine-simulation-persistence/tests/test_economy_window_reruns.py index 6519323f2..e3845ef85 100644 --- a/libs/policyengine-simulation-persistence/tests/test_economy_window_reruns.py +++ b/libs/policyengine-simulation-persistence/tests/test_economy_window_reruns.py @@ -11,13 +11,17 @@ from __future__ import annotations -from contextlib import contextmanager from dataclasses import dataclass, replace from typing import TYPE_CHECKING from uuid import UUID, uuid4 import pytest -from conftest import act_as_the_service, rolled_back_transaction +from conftest import ( + act_as_the_service, + economy_store_on, + in_session, + rolled_back_transaction, +) from economy_rows import ( CURRENT_LAW, LATER_POLICYENGINE_VERSION, @@ -32,16 +36,17 @@ insert_simulation_run, ) from policyengine_simulation_persistence import ( + BudgetWindowRun, EconomyPopulation, EconomyRecordReferenceError, EconomyReportDefinition, EconomyRequest, - EconomyRunStore, FoundBudgetWindow, KeyedBudgetWindow, KeyedWindowYear, NewWindowRerun, ProposedSide, + ReportRun, ReportRunStatus, ReportRunTrigger, RerunRecordDeletedError, @@ -49,17 +54,13 @@ SimulationStatus, WindowRerunYear, WindowYearDefinition, - budget_window_runs, find_or_create_budget_window, insert_window_rerun, read_report_definition, - report_runs, ) from sqlalchemy import insert, text if TYPE_CHECKING: - from collections.abc import Iterator - from sqlalchemy import Connection pytestmark = pytest.mark.public_api_schema @@ -125,7 +126,7 @@ def _insert_window_rows( ) window_run_id = uuid4() connection.execute( - insert(budget_window_runs).values( + insert(BudgetWindowRun).values( id=window_run_id, report_id=report_id, tax_benefit_model_version_id=economy.catalog.version_id, @@ -135,7 +136,7 @@ def _insert_window_rows( ) ) connection.execute( - insert(report_runs), + insert(ReportRun), [ { "id": window_year.report_run_id, @@ -206,17 +207,6 @@ def _stored_window( return report -def _store_on(connection: Connection) -> EconomyRunStore: - """Return a store whose transactions are savepoints of ``connection``.""" - - @contextmanager - def savepoint() -> Iterator[Connection]: - with connection.begin_nested(): - yield connection - - return EconomyRunStore(savepoint) - - def _row(connection: Connection, table_name: str, row_id: UUID) -> dict: row = connection.execute( text(f"SELECT * FROM public.{table_name} WHERE id = :id"), # noqa: S608 # fixed table names @@ -260,7 +250,7 @@ def test_a_budget_windows_definition_names_each_years_simulations( stored = _stored_window(connection, economy) act_as_the_service(connection) - definition = read_report_definition(connection, stored.report_id) + definition = in_session(connection, read_report_definition, stored.report_id) assert definition is not None assert (definition.kind, definition.year, definition.budget_window_size) == ( @@ -297,9 +287,9 @@ def test_a_definitions_years_come_from_the_newest_window_run( ), ) act_as_the_service(connection) - insert_window_rerun(connection, replace(rerun, years=tuple(years))) + in_session(connection, insert_window_rerun, replace(rerun, years=tuple(years))) - definition = read_report_definition(connection, stored.report_id) + definition = in_session(connection, read_report_definition, stored.report_id) assert definition is not None assert [year.baseline_simulation_id for year in definition.window_years] == [ @@ -320,7 +310,7 @@ def test_a_forced_window_rerun_creates_a_window_run_and_each_years_new_runs( rerun = _window_rerun_of(stored, later, recalculate=True) act_as_the_service(connection) - found = insert_window_rerun(connection, rerun) + found = in_session(connection, insert_window_rerun, rerun) window_run = _row(connection, "budget_window_runs", rerun.window_run_id) year_runs = [ _row(connection, "report_runs", year.report_run_id) for year in rerun.years @@ -401,8 +391,9 @@ def _keyed_window(connection: Connection, economy: EconomyCatalog) -> FoundBudge budget_window_size=len(WINDOW_YEARS), spm=None, ) - return find_or_create_budget_window( + return in_session( connection, + find_or_create_budget_window, KeyedBudgetWindow( definition=definition, report_id=uuid4(), @@ -465,7 +456,9 @@ def test_an_unforced_window_rerun_joins_a_keyed_windows_run_on_its_versions( act_as_the_service(connection) found = _keyed_window(connection, economy) - joined = insert_window_rerun(connection, _rerun_of_keyed(found, economy)) + joined = in_session( + connection, insert_window_rerun, _rerun_of_keyed(found, economy) + ) assert (joined.window_run.id, joined.window_run.inserted) == ( found.window_run.id, @@ -495,7 +488,7 @@ def test_an_unforced_window_rerun_after_a_failed_run_names_its_years_runs( act_as_the_service(connection) rerun = _rerun_of_keyed(found, economy) - created = insert_window_rerun(connection, rerun) + created = in_session(connection, insert_window_rerun, rerun) window_run = _row(connection, "budget_window_runs", rerun.window_run_id) assert (created.window_run.id, created.window_run.inserted) == ( @@ -517,7 +510,7 @@ def test_a_window_reruns_inputs_are_read_on_the_current_catalog_version( _stored_window(connection, economy) act_as_the_service(connection) - inputs = _store_on(connection).read_window_rerun_inputs( + inputs = economy_store_on(connection).read_window_rerun_inputs( LATER_REQUEST, baseline=CURRENT_LAW, reform=CURRENT_LAW ) @@ -536,7 +529,7 @@ def test_a_window_rerun_is_created_through_the_store(public_api_database_url) -> stored = _stored_window(connection, economy) rerun = _window_rerun_of(stored, later) act_as_the_service(connection) - store = _store_on(connection) + store = economy_store_on(connection) created = store.create_window_rerun(rerun) joined = store.create_window_rerun(_window_rerun_of(stored, later)) @@ -595,7 +588,7 @@ def test_a_year_naming_no_reform_is_left_out_of_a_windows_definition( ) act_as_the_service(connection) - definition = read_report_definition(connection, stored.report_id) + definition = in_session(connection, read_report_definition, stored.report_id) assert definition is not None assert [year.year for year in definition.window_years] == [2026, 2028] @@ -616,20 +609,25 @@ def test_a_window_rerun_of_a_deleted_report_or_year_simulation_is_refused( ) with pytest.raises(RerunRecordDeletedError), connection.begin_nested(): - insert_window_rerun(connection, replace(rerun, report_id=uuid4())) + in_session( + connection, insert_window_rerun, replace(rerun, report_id=uuid4()) + ) # A report deleted with a year's simulation: the window run is # inserted first, so the report's own key refuses it. with pytest.raises(RerunRecordDeletedError), connection.begin_nested(): - insert_window_rerun( + in_session( connection, + insert_window_rerun, replace(rerun, report_id=uuid4(), years=(*rerun.years[:2], gone_year)), ) with ( pytest.raises(EconomyRecordReferenceError) as refused, connection.begin_nested(), ): - insert_window_rerun( - connection, replace(rerun, years=(*rerun.years[:2], gone_year)) + in_session( + connection, + insert_window_rerun, + replace(rerun, years=(*rerun.years[:2], gone_year)), ) assert (refused.value.table_name, refused.value.column_name) == ( @@ -654,7 +652,7 @@ def test_a_forced_window_rerun_is_an_extra_run_beside_any_run_on_its_versions( ) act_as_the_service(connection) - insert_window_rerun(connection, rerun) + in_session(connection, insert_window_rerun, rerun) window_run = _row(connection, "budget_window_runs", rerun.window_run_id) year_runs = [ _row(connection, "report_runs", year.report_run_id) for year in rerun.years diff --git a/libs/policyengine-simulation-persistence/tests/test_engine.py b/libs/policyengine-simulation-persistence/tests/test_engine.py index 8d89249f9..fb0e74ac4 100644 --- a/libs/policyengine-simulation-persistence/tests/test_engine.py +++ b/libs/policyengine-simulation-persistence/tests/test_engine.py @@ -1,7 +1,4 @@ -"""Connection settings for the simulation service's database. - -The tests marked ``postgres`` need a disposable PostgreSQL server (``conftest``). -""" +"""Connection settings for the simulation service's database.""" from __future__ import annotations @@ -9,13 +6,15 @@ import pytest from policyengine_simulation_persistence import ( - bounded_transaction, - create_simulation_engine, - create_unpooled_engine, + create_simulation_session_factory, + create_single_connection_session_factory, + create_unpooled_session_factory, + dispose_session_factory, normalized_database_url, ) from policyengine_simulation_persistence import engine as engine_module from sqlalchemy import create_engine, text +from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import NullPool, QueuePool @@ -52,49 +51,87 @@ def test_malformed_database_urls_are_rejected_without_echoing_them( assert error.value.__cause__ is None or "secret" not in str(error.value.__cause__) -def test_runtime_engine_never_prepares_server_side_statements(monkeypatch) -> None: - captured_options = {} +def _capture_engine_options(monkeypatch) -> tuple[dict, object]: + captured_options: dict = {} + built_engine = object() def capture_engine(url, **options): captured_options.update(options, url=url) - return object() + return built_engine - monkeypatch.setattr( - "policyengine_simulation_persistence.engine.create_engine", - capture_engine, - ) + monkeypatch.setattr(engine_module, "create_engine", capture_engine) + return captured_options, built_engine - create_simulation_engine( + +def test_the_service_sessions_never_prepare_server_side_statements(monkeypatch) -> None: + captured_options, built_engine = _capture_engine_options(monkeypatch) + + session_factory = create_simulation_session_factory( "postgresql://simulation_service:secret@db.example/postgres" ) + assert session_factory.kw["bind"] is built_engine + assert session_factory.kw["expire_on_commit"] is False assert captured_options["url"].drivername == "postgresql+psycopg" assert captured_options["connect_args"]["prepare_threshold"] is None assert captured_options["connect_args"]["connect_timeout"] > 0 assert captured_options["pool_pre_ping"] is True - assert captured_options["pool_size"] >= 1 + assert captured_options["pool_size"] == engine_module.POOL_SIZE + +def test_the_unpooled_session_factory_keeps_no_connection(monkeypatch) -> None: + captured_options, built_engine = _capture_engine_options(monkeypatch) + + session_factory = create_unpooled_session_factory( + "postgresql://modal_worker:secret@db.example/postgres" + ) + + assert session_factory.kw["bind"] is built_engine + assert session_factory.kw["expire_on_commit"] is False + assert captured_options["poolclass"] is NullPool + assert captured_options["connect_args"] == {"connect_timeout": 10} -def test_the_unpooled_engine_holds_no_connection_and_selects_psycopg( + +def test_the_single_connection_session_factory_shares_one_connection( monkeypatch, ) -> None: - captured_options = {} + captured_options, built_engine = _capture_engine_options(monkeypatch) - def capture_engine(url, **options): - captured_options.update(options, url=url) - return object() + session_factory = create_single_connection_session_factory( + "postgresql://runtime:secret@db.example/postgres" + ) - monkeypatch.setattr(engine_module, "create_engine", capture_engine) + assert session_factory.kw["bind"] is built_engine + assert session_factory.kw["expire_on_commit"] is False + assert captured_options["poolclass"] is QueuePool + assert captured_options["pool_size"] == 1 + assert captured_options["max_overflow"] == 0 + assert ( + captured_options["pool_timeout"] == engine_module.SINGLE_CONNECTION_WAIT_SECONDS + ) + assert captured_options["pool_pre_ping"] is True + assert captured_options["connect_args"] == { + "connect_timeout": engine_module.SINGLE_CONNECTION_CONNECT_TIMEOUT_SECONDS, + "prepare_threshold": None, + } - create_unpooled_engine("postgresql://modal_worker:secret@db.example/postgres") - assert captured_options["url"].drivername == "postgresql+psycopg" - assert captured_options["poolclass"] is NullPool - assert captured_options["connect_args"] == {"connect_timeout": 10} +def test_disposing_a_session_factory_closes_its_pooled_connections() -> None: + engine = create_engine("sqlite://", poolclass=QueuePool) + session_factory = sessionmaker(engine) + with session_factory() as session: + session.execute(text("SELECT 1")) + pool_before = engine.pool + assert pool_before.checkedin() == 1 + + dispose_session_factory(session_factory) + + assert engine.pool is not pool_before + assert pool_before.checkedin() == 0 @pytest.mark.parametrize("fail_transaction", [False, True]) -def test_the_unpooled_engine_closes_connections_between_transactions( +def test_unpooled_sessions_close_connections_between_transactions( monkeypatch, fail_transaction ) -> None: """A long calculation holds no idle database session after its operation.""" @@ -107,7 +144,9 @@ def local_engine(_url, **options): return create_engine("sqlite://", **options) monkeypatch.setattr(engine_module, "create_engine", local_engine) - engine = create_unpooled_engine("postgresql://runtime:secret@db.example/postgres") + engine = create_unpooled_session_factory( + "postgresql://runtime:secret@db.example/postgres" + ).kw["bind"] try: try: with engine.begin() as connection: @@ -132,61 +171,3 @@ def _fail_the_calculation() -> None: """Raise the failure a calculation inside the transaction would.""" msg = "calculation failed" raise RuntimeError(msg) - - -@pytest.mark.postgres -def test_a_bounded_transaction_limits_its_own_statements_and_commits( - disposable_database, -) -> None: - """The limit is set for the transaction alone, so a pooled connection keeps none.""" - engine = create_simulation_engine(disposable_database.superuser_url) - try: - with engine.begin() as connection: - connection.execute(text("CREATE TABLE written (id integer)")) - with bounded_transaction( - engine, statement_timeout_milliseconds=1234 - ) as connection: - limit = connection.execute( - text("SELECT current_setting('statement_timeout')") - ).scalar_one() - connection.execute(text("INSERT INTO written VALUES (1)")) - with engine.connect() as connection: - later_limit = connection.execute( - text("SELECT current_setting('statement_timeout')") - ).scalar_one() - count = connection.execute( - text("SELECT count(*) FROM written") - ).scalar_one() - finally: - engine.dispose() - - assert limit == "1234ms" - assert later_limit == "0" - assert count == 1 - - -def _insert_then_fail(engine) -> None: - with bounded_transaction(engine, statement_timeout_milliseconds=1000) as connection: - connection.execute(text("INSERT INTO written VALUES (1)")) - msg = "block failed" - raise RuntimeError(msg) - - -@pytest.mark.postgres -def test_a_bounded_transaction_rolls_back_when_its_block_raises( - disposable_database, -) -> None: - engine = create_simulation_engine(disposable_database.superuser_url) - try: - with engine.begin() as connection: - connection.execute(text("CREATE TABLE written (id integer)")) - with pytest.raises(RuntimeError, match="block failed"): - _insert_then_fail(engine) - with engine.connect() as connection: - count = connection.execute( - text("SELECT count(*) FROM written") - ).scalar_one() - finally: - engine.dispose() - - assert count == 0 diff --git a/libs/policyengine-simulation-persistence/tests/test_household_job_records.py b/libs/policyengine-simulation-persistence/tests/test_household_job_records.py index c1642e2cf..20f652068 100644 --- a/libs/policyengine-simulation-persistence/tests/test_household_job_records.py +++ b/libs/policyengine-simulation-persistence/tests/test_household_job_records.py @@ -1,8 +1,8 @@ """The simulation service's reads and writes for a household calculation. -These run against a database the Public API's own v2 migration chain built +These run against a database API v1's own v2 migration chain built (``PUBLIC_API_SCHEMA_DATABASE_URL``), as the simulation service's role, each -in a transaction rolled back afterwards. The Public API's rows a job names are +in a transaction rolled back afterwards. API v1's rows a job names are inserted first, as the superuser. """ @@ -13,13 +13,13 @@ from uuid import uuid4 import pytest -from conftest import act_as_the_service, rolled_back_transaction +from conftest import act_as_the_service, in_session, rolled_back_transaction from policyengine_simulation_persistence import ( + HouseholdJob, HouseholdJobReferenceError, HouseholdJobStatus, NewHouseholdJob, fail_unstarted_household_job, - household_jobs, insert_household_job, read_catalog_version_id, read_household_job, @@ -102,14 +102,20 @@ def test_a_catalog_version_is_found_by_country_and_policyengine_version( act_as_the_service(connection) assert ( - read_catalog_version_id( - connection, country="us", policyengine_version=POLICYENGINE_VERSION + in_session( + connection, + read_catalog_version_id, + country="us", + policyengine_version=POLICYENGINE_VERSION, ) == catalog.version_id ) assert ( - read_catalog_version_id( - connection, country="us", policyengine_version="6.16.1" + in_session( + connection, + read_catalog_version_id, + country="us", + policyengine_version="6.16.1", ) is None ) @@ -122,15 +128,22 @@ def test_a_household_is_found_in_its_own_country_only(public_api_database_url) - ) act_as_the_service(connection) - stored = read_stored_household( - connection, country="us", household_id=household_id + stored = in_session( + connection, read_stored_household, country="us", household_id=household_id ) assert ( - read_stored_household(connection, country="uk", household_id=household_id) + in_session( + connection, + read_stored_household, + country="uk", + household_id=household_id, + ) is None ) assert ( - read_stored_household(connection, country="us", household_id=uuid4()) + in_session( + connection, read_stored_household, country="us", household_id=uuid4() + ) is None ) @@ -148,8 +161,8 @@ def test_a_job_is_created_pending_with_no_call(public_api_database_url) -> None: new_job = _stored_job(connection) act_as_the_service(connection) - insert_household_job(connection, new_job) - job = read_household_job(connection, new_job.id) + in_session(connection, insert_household_job, new_job) + job = in_session(connection, read_household_job, new_job.id) dynamic_id = connection.execute( text("SELECT dynamic_id FROM public.household_jobs WHERE id = :id"), {"id": new_job.id}, @@ -192,7 +205,7 @@ def test_an_unknown_job_reads_as_none(public_api_database_url) -> None: with rolled_back_transaction(public_api_database_url) as connection: act_as_the_service(connection) - assert read_household_job(connection, uuid4()) is None + assert in_session(connection, read_household_job, uuid4()) is None def test_a_call_is_recorded_once_while_the_job_is_pending_without_one( @@ -201,15 +214,21 @@ def test_a_call_is_recorded_once_while_the_job_is_pending_without_one( with rolled_back_transaction(public_api_database_url) as connection: new_job = _stored_job(connection) act_as_the_service(connection) - insert_household_job(connection, new_job) + in_session(connection, insert_household_job, new_job) - first = record_household_call( - connection, job_id=new_job.id, modal_call_id="fc-first" + first = in_session( + connection, + record_household_call, + job_id=new_job.id, + modal_call_id="fc-first", ) - second = record_household_call( - connection, job_id=new_job.id, modal_call_id="fc-second" + second = in_session( + connection, + record_household_call, + job_id=new_job.id, + modal_call_id="fc-second", ) - job = read_household_job(connection, new_job.id) + job = in_session(connection, read_household_job, new_job.id) assert (first, second) == (HouseholdJobStatus.PENDING, HouseholdJobStatus.PENDING) assert job is not None @@ -223,22 +242,28 @@ def test_a_job_the_function_already_claimed_keeps_its_claim( with rolled_back_transaction(public_api_database_url) as connection: new_job = _stored_job(connection) act_as_the_service(connection) - insert_household_job(connection, new_job) + in_session(connection, insert_household_job, new_job) connection.execute(text("RESET ROLE")) connection.execute( - update(household_jobs) - .where(household_jobs.c.id == new_job.id) + update(HouseholdJob) + .where(HouseholdJob.id == new_job.id) .values(status=HouseholdJobStatus.RUNNING, modal_call_id="fc-claimed") ) act_as_the_service(connection) - status = record_household_call( - connection, job_id=new_job.id, modal_call_id="fc-claimed" + status = in_session( + connection, + record_household_call, + job_id=new_job.id, + modal_call_id="fc-claimed", ) - failed = fail_unstarted_household_job( - connection, job_id=new_job.id, error_message="Modal refused the call." + failed = in_session( + connection, + fail_unstarted_household_job, + job_id=new_job.id, + error_message="Modal refused the call.", ) - job = read_household_job(connection, new_job.id) + job = in_session(connection, read_household_job, new_job.id) assert status is HouseholdJobStatus.RUNNING assert failed is False @@ -254,15 +279,21 @@ def test_a_job_whose_call_was_refused_fails_at_once(public_api_database_url) -> with rolled_back_transaction(public_api_database_url) as connection: new_job = _stored_job(connection) act_as_the_service(connection) - insert_household_job(connection, new_job) + in_session(connection, insert_household_job, new_job) - failed = fail_unstarted_household_job( - connection, job_id=new_job.id, error_message="Modal refused the call." + failed = in_session( + connection, + fail_unstarted_household_job, + job_id=new_job.id, + error_message="Modal refused the call.", ) - failed_again = fail_unstarted_household_job( - connection, job_id=new_job.id, error_message="A second refusal." + failed_again = in_session( + connection, + fail_unstarted_household_job, + job_id=new_job.id, + error_message="A second refusal.", ) - job = read_household_job(connection, new_job.id) + job = in_session(connection, read_household_job, new_job.id) assert (failed, failed_again) == (True, False) assert job is not None @@ -278,13 +309,18 @@ def test_a_job_whose_call_was_recorded_is_not_failed_as_refused( with rolled_back_transaction(public_api_database_url) as connection: new_job = _stored_job(connection) act_as_the_service(connection) - insert_household_job(connection, new_job) - record_household_call(connection, job_id=new_job.id, modal_call_id="fc-1") + in_session(connection, insert_household_job, new_job) + in_session( + connection, record_household_call, job_id=new_job.id, modal_call_id="fc-1" + ) - failed = fail_unstarted_household_job( - connection, job_id=new_job.id, error_message="Modal refused the call." + failed = in_session( + connection, + fail_unstarted_household_job, + job_id=new_job.id, + error_message="Modal refused the call.", ) - job = read_household_job(connection, new_job.id) + job = in_session(connection, read_household_job, new_job.id) assert failed is False assert job is not None @@ -304,10 +340,7 @@ def test_a_job_naming_a_row_that_does_not_exist_names_its_column( new_job = replace(stored, **{missing_column: uuid4()}) act_as_the_service(connection) - with ( - pytest.raises(HouseholdJobReferenceError) as raised, - connection.begin_nested(), - ): - insert_household_job(connection, new_job) + with pytest.raises(HouseholdJobReferenceError) as raised: + in_session(connection, insert_household_job, new_job) assert raised.value.column_name == missing_column diff --git a/libs/policyengine-simulation-persistence/tests/test_household_job_store.py b/libs/policyengine-simulation-persistence/tests/test_household_job_store.py index ba9ca2d78..cee581577 100644 --- a/libs/policyengine-simulation-persistence/tests/test_household_job_store.py +++ b/libs/policyengine-simulation-persistence/tests/test_household_job_store.py @@ -1,27 +1,35 @@ """The household calculation store: each read and write in a transaction of its own. -These run against a database the Public API's own v2 migration chain built -(``PUBLIC_API_SCHEMA_DATABASE_URL``). The store opens each transaction -through a function it is given; here each is a savepoint of one connection, -acting as the simulation service's role, which the test rolls back. +These run against a database API v1's own v2 migration chain built +(``PUBLIC_API_SCHEMA_DATABASE_URL``). The store opens each transaction as an +ORM session from the factory it is given; here each session's transaction is +a savepoint of one connection, acting as the simulation service's role, +which the test rolls back. """ from __future__ import annotations -from contextlib import contextmanager from datetime import UTC, datetime from typing import TYPE_CHECKING from uuid import uuid4 import pytest -from conftest import act_as_the_service, rolled_back_transaction +from conftest import ( + act_as_the_service, + rolled_back_transaction, + session_factory_on, + simulation_engine, +) from policyengine_simulation_persistence import ( + EconomyRecordOutcomeUnknownError, + HouseholdJobReferenceError, HouseholdJobStatus, HouseholdJobStore, NewHouseholdJob, UnknownCatalogVersionError, UnknownHouseholdError, UnknownPolicyError, + create_session_factory, ) from public_api_rows import ( StoredValue, @@ -30,14 +38,19 @@ insert_model, insert_policy, ) +from sqlalchemy.exc import OperationalError +from transaction_fakes import ( + CommitFailingTransaction, + FlushFailingTransaction, + RecordingSession, +) if TYPE_CHECKING: - from collections.abc import Iterator - from sqlalchemy import Connection pytestmark = pytest.mark.public_api_schema +STATEMENT_TIMEOUT_MILLISECONDS = 5_000 POLICYENGINE_VERSION = "6.17.1" CTC_AMOUNT = "gov.irs.credits.ctc.amount.base[0].amount" DOCUMENT = { @@ -54,13 +67,10 @@ def _store_on(connection: Connection) -> HouseholdJobStore: """Return a store whose transactions are savepoints of ``connection``.""" - - @contextmanager - def savepoint() -> Iterator[Connection]: - with connection.begin_nested(): - yield connection - - return HouseholdJobStore(savepoint) + return HouseholdJobStore.from_session_factory( + session_factory_on(connection), + statement_timeout_milliseconds=STATEMENT_TIMEOUT_MILLISECONDS, + ) def test_the_inputs_are_read_together(public_api_database_url) -> None: @@ -225,3 +235,59 @@ def test_a_refused_call_fails_its_job(public_api_database_url) -> None: HouseholdJobStatus.FAILED, "Modal refused the call.", ) + + +def _job_naming_no_rows() -> NewHouseholdJob: + """Return a job whose household and catalog version are random IDs.""" + return NewHouseholdJob( + id=uuid4(), + country="us", + household_id=uuid4(), + policy_id=None, + year=2026, + request_data={"country": "us"}, + tax_benefit_model_version_id=uuid4(), + country_package_version="2.2.1", + policyengine_version=POLICYENGINE_VERSION, + ) + + +def test_a_job_whose_commit_fails_may_have_been_created() -> None: + """PostgreSQL may have applied the commit before the connection failed.""" + session = RecordingSession() + store = HouseholdJobStore(lambda: CommitFailingTransaction(session)) + + with pytest.raises(EconomyRecordOutcomeUnknownError) as raised: + store.create_job(_job_naming_no_rows()) + + assert isinstance(raised.value.__cause__, OperationalError) + assert session.added + assert session.flushed + + +def test_a_job_whose_flush_fails_was_not_created() -> None: + store = HouseholdJobStore(FlushFailingTransaction) + + with pytest.raises(OperationalError): + store.create_job(_job_naming_no_rows()) + + +def test_a_job_naming_missing_rows_is_refused_before_its_commit( + public_api_database_url, +) -> None: + """The keys refuse the insert itself, in a transaction of its own. + + A key checked only at commit would make the refusal look like a lost + commit; this database's keys refuse the statement. + """ + engine = simulation_engine(public_api_database_url) + try: + store = HouseholdJobStore.from_session_factory( + create_session_factory(engine), + statement_timeout_milliseconds=STATEMENT_TIMEOUT_MILLISECONDS, + ) + + with pytest.raises(HouseholdJobReferenceError): + store.create_job(_job_naming_no_rows()) + finally: + engine.dispose() diff --git a/libs/policyengine-simulation-persistence/tests/test_household_report_reruns.py b/libs/policyengine-simulation-persistence/tests/test_household_report_reruns.py index 9a33414c0..702efa88c 100644 --- a/libs/policyengine-simulation-persistence/tests/test_household_report_reruns.py +++ b/libs/policyengine-simulation-persistence/tests/test_household_report_reruns.py @@ -2,7 +2,7 @@ A household report's re-run gives each side a new run of its own household simulation, never reusing one, under a new report run with trigger -``manual``. These run against a database the Public API's own v2 migration +``manual``. These run against a database API v1's own v2 migration chain built (``PUBLIC_API_SCHEMA_DATABASE_URL``). The stored report and its first runs are written as the superuser; the re-run's reads and writes run as the simulation service's role, in a transaction the test rolls back. @@ -10,13 +10,17 @@ from __future__ import annotations -from contextlib import contextmanager from dataclasses import replace from typing import TYPE_CHECKING from uuid import uuid4 import pytest -from conftest import act_as_the_service, rolled_back_transaction +from conftest import ( + act_as_the_service, + in_session, + rolled_back_transaction, + session_factory_on, +) from household_report_rows import insert_household_report_rows from policyengine_simulation_persistence import ( EconomyRecordOutcomeUnknownError, @@ -41,9 +45,9 @@ ) from sqlalchemy import text from sqlalchemy.exc import OperationalError +from transaction_fakes import CommitFailingTransaction, RecordingSession if TYPE_CHECKING: - from collections.abc import Iterator from uuid import UUID from household_report_rows import StoredHouseholdReport @@ -52,6 +56,7 @@ pytestmark = pytest.mark.public_api_schema +STATEMENT_TIMEOUT_MILLISECONDS = 5_000 VERSIONS = RunVersions(country_package_version="1.764.6", policyengine_version="6.17.1") # A later release's versions, with a catalog version of its own. LATER_VERSIONS = RunVersions( @@ -76,13 +81,10 @@ def _store_on(connection: Connection) -> HouseholdReportStore: """Return a store whose transactions are savepoints of ``connection``.""" - - @contextmanager - def savepoint() -> Iterator[Connection]: - with connection.begin_nested(): - yield connection - - return HouseholdReportStore(savepoint) + return HouseholdReportStore.from_session_factory( + session_factory_on(connection), + statement_timeout_milliseconds=STATEMENT_TIMEOUT_MILLISECONDS, + ) def _catalogs(connection: Connection) -> tuple[CatalogVersion, CatalogVersion]: @@ -125,8 +127,9 @@ def _stored_report( with_reform=with_reform, ) if first_run_ended: - fail_unstarted_report_run( + in_session( connection, + fail_unstarted_report_run, report_run_id=report.run_id, simulation_run_ids=report.simulation_run_ids, error_message="run_not_started: no call was recorded", @@ -204,8 +207,10 @@ def test_a_household_report_is_read_with_its_household_and_each_sides_policy( baseline_alone = _stored_report(connection, catalog, with_reform=False) act_as_the_service(connection) - definition = read_report_definition(connection, report.id) - alone_definition = read_report_definition(connection, baseline_alone.id) + definition = in_session(connection, read_report_definition, report.id) + alone_definition = in_session( + connection, read_report_definition, baseline_alone.id + ) assert definition is not None assert alone_definition is not None @@ -364,6 +369,11 @@ def test_a_forced_household_rerun_beside_a_run_under_way_is_an_extra_run( report.baseline.simulation_id, ) first_run = _row(connection, "report_runs", report.run_id) + extra_run = _row(connection, "report_runs", rerun.run_id) + extra_side_runs = [ + _row(connection, "simulation_runs", run_id) + for run_id in rerun.simulation_run_ids + ] assert (created.id, created.inserted, created.status) == ( rerun.run_id, @@ -376,6 +386,8 @@ def test_a_forced_household_rerun_beside_a_run_under_way_is_an_extra_run( ) == rerun.simulation_run_ids assert (report_runs, baseline_runs) == (2, 2) assert first_run["status"] == ReportRunStatus.PENDING + assert extra_run["forced"] is True + assert [run["forced"] for run in extra_side_runs] == [True, True] def test_a_rerun_naming_a_deleted_report_or_simulation_is_refused_by_its_key( @@ -407,40 +419,9 @@ def test_a_rerun_naming_a_deleted_report_or_simulation_is_refused_by_its_key( assert report_runs == 1 -class _CommitFails: - """A transaction whose block runs, and whose commit then fails.""" - - def __init__(self, connection) -> None: - self.connection = connection - - def __enter__(self): - return self.connection - - def __exit__(self, error_type, error, traceback) -> None: - if error is None: - statement = "COMMIT" - raise OperationalError(statement, {}, Exception("connection lost")) - - -class _RecordingConnection: - """Records each statement it is given, and answers each with no rows.""" - - def __init__(self) -> None: - self.statements: list[object] = [] - - def execute(self, statement, *arguments): - self.statements.append(statement) - return _NoRows() - - -class _NoRows: - def scalar_one_or_none(self) -> None: - return None - - def test_a_rerun_whose_commit_fails_may_have_been_written() -> None: - connection = _RecordingConnection() - store = HouseholdReportStore(lambda: _CommitFails(connection)) + session = RecordingSession() + store = HouseholdReportStore(lambda: CommitFailingTransaction(session)) with pytest.raises(EconomyRecordOutcomeUnknownError) as raised: store.create_report_rerun( @@ -459,4 +440,6 @@ def test_a_rerun_whose_commit_fails_may_have_been_written() -> None: ) assert isinstance(raised.value.__cause__, OperationalError) - assert connection.statements + assert session.added + assert session.flushed + assert session.statements diff --git a/libs/policyengine-simulation-persistence/tests/test_household_report_store.py b/libs/policyengine-simulation-persistence/tests/test_household_report_store.py index 1b34bda4c..e923065d8 100644 --- a/libs/policyengine-simulation-persistence/tests/test_household_report_store.py +++ b/libs/policyengine-simulation-persistence/tests/test_household_report_store.py @@ -1,21 +1,20 @@ """The household report store: each read and write in a transaction of its own. A household report calculates one stored household under a baseline policy -and, optionally, a reform. These run against a database the Public API's own +and, optionally, a reform. These run against a database API v1's own v2 migration chain built (``PUBLIC_API_SCHEMA_DATABASE_URL``); each store -transaction is a savepoint of one connection acting as the simulation -service's role, which the test rolls back. +transaction is an ORM session in a savepoint of one connection acting as the +simulation service's role, which the test rolls back. """ from __future__ import annotations -from contextlib import contextmanager from datetime import UTC, datetime from typing import TYPE_CHECKING from uuid import uuid4 import pytest -from conftest import act_as_the_service, rolled_back_transaction +from conftest import act_as_the_service, rolled_back_transaction, session_factory_on from household_report_rows import insert_household_report_rows from policyengine_simulation_persistence import ( HouseholdReportStore, @@ -37,7 +36,6 @@ from sqlalchemy import text if TYPE_CHECKING: - from collections.abc import Iterator from uuid import UUID from public_api_rows import CatalogVersion @@ -45,6 +43,7 @@ pytestmark = pytest.mark.public_api_schema +STATEMENT_TIMEOUT_MILLISECONDS = 5_000 POLICYENGINE_VERSION = "6.17.1" VERSIONS = RunVersions( country_package_version="1.764.6", policyengine_version=POLICYENGINE_VERSION @@ -65,13 +64,10 @@ def _store_on(connection: Connection) -> HouseholdReportStore: """Return a store whose transactions are savepoints of ``connection``.""" - - @contextmanager - def savepoint() -> Iterator[Connection]: - with connection.begin_nested(): - yield connection - - return HouseholdReportStore(savepoint) + return HouseholdReportStore.from_session_factory( + session_factory_on(connection), + statement_timeout_milliseconds=STATEMENT_TIMEOUT_MILLISECONDS, + ) def _catalog(connection: Connection) -> CatalogVersion: diff --git a/libs/policyengine-simulation-persistence/tests/test_household_sessions.py b/libs/policyengine-simulation-persistence/tests/test_household_sessions.py new file mode 100644 index 000000000..5fa970789 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_household_sessions.py @@ -0,0 +1,761 @@ +"""Household jobs and reports on ORM sessions, against a real database. + +The household code runs its statements through ORM sessions on the mapped +classes. A flush is where an added row's INSERT runs, so what a failure means +depends on when it happens: during the flush nothing was written, and at the +commit after a flush the rows may exist. These tests make each failure happen +for real, by ending the session's connection at that moment. + +They use a database API v1's own v2 migration chain built +(``PUBLIC_API_SCHEMA_DATABASE_URL``). A test of the stores' transactions needs +rows it can commit, so it commits a catalog of its own and deletes it after; +the others work in one transaction rolled back afterwards. +""" + +from __future__ import annotations + +import threading +from dataclasses import dataclass, replace +from typing import TYPE_CHECKING, Any +from uuid import UUID, uuid4 + +import pytest +from committed_rows import ( + LOCK_WAIT_TIMEOUT_SECONDS, + delete_catalog_rows, + run_together, + wait_until_another_backend_waits_for, +) +from conftest import ( + act_as_the_service, + in_session, + rolled_back_transaction, + session_in, +) +from economy_rows import POLICYENGINE_VERSION, VERSIONS, insert_economy_catalog +from household_report_rows import insert_household_report_rows +from policyengine_simulation_persistence import ( + EconomyRecordOutcomeUnknownError, + EconomyRecordReferenceError, + HouseholdJob, + HouseholdJobStatus, + HouseholdJobStore, + HouseholdReportDefinition, + HouseholdReportStore, + HouseholdRerunSide, + KeyedHouseholdReport, + NewHouseholdJob, + NewHouseholdReportRerun, + ProposedSide, + ReportRunStatus, + RunVersions, + SimulationChoice, + SimulationRun, + create_session_factory, + fail_unstarted_household_job, + find_or_create_household_report, + insert_household_job, + read_catalog_version_id, + read_household_job, + read_stored_household, + record_household_call, +) +from policyengine_simulation_persistence.household_report_records import ( + household_simulation_run_row, + insert_household_simulation_run, +) +from policyengine_simulation_persistence.keyed_economy_records import ( + find_or_insert_report_run_on_sides, +) +from public_api_rows import insert_household +from session_probes import ( + end_connection_at, + orm_statement_flags, + statements_sent, + terminate_backend, +) +from sqlalchemy import event, func, select, text +from sqlalchemy.exc import IntegrityError, OperationalError + +if TYPE_CHECKING: + from collections.abc import Iterator + + from economy_rows import EconomyCatalog + from public_api_rows import CatalogVersion + from sqlalchemy import Connection, Engine + from sqlalchemy.orm import Session, sessionmaker + +pytestmark = pytest.mark.public_api_schema + +STATEMENT_TIMEOUT_MILLISECONDS = 5_000 +SHORT_STATEMENT_TIMEOUT_MILLISECONDS = 200 +CURRENT_LAW = SimulationChoice(policy_id=None, dynamic_id=None) +# The .py version and so the catalog version of ``VERSIONS``, on a newer country +# package: a re-run on these versions joins no run on ``VERSIONS``. +NEWER_PACKAGE_VERSIONS = RunVersions( + country_package_version="1.770.0", policyengine_version=POLICYENGINE_VERSION +) +# No simulation has this ID, and it sorts after every generated one, so the +# lookups, which go in simulation ID order, reach it last. +LAST_MISSING_SIMULATION_ID = UUID("ffffffff-ffff-4fff-bfff-ffffffffffff") +DOCUMENT = { + "people": [ + { + "id": "you", + "values": {"age": 40}, + "memberships": {"household": "your household"}, + } + ], + "household": [{"id": "your household", "values": {}}], +} + + +@dataclass(frozen=True) +class _CommittedHousehold: + """A committed catalog version and a committed household.""" + + economy: EconomyCatalog + household_id: UUID + + @property + def catalog(self) -> CatalogVersion: + return self.economy.catalog + + +@pytest.fixture +def committed_household( + committed_engine, committed_catalog +) -> Iterator[_CommittedHousehold]: + """Commit a household; delete it after, with every row naming it.""" + with committed_engine.begin() as connection: + household_id = insert_household( + connection, country="us", default_year=2026, document=DOCUMENT + ) + try: + yield _CommittedHousehold(committed_catalog, household_id) + finally: + with committed_engine.begin() as connection: + connection.execute( + text("DELETE FROM public.household_jobs WHERE household_id = :id"), + {"id": household_id}, + ) + delete_catalog_rows(connection, committed_catalog) + connection.execute( + text("DELETE FROM public.households WHERE id = :id"), + {"id": household_id}, + ) + + +@pytest.fixture +def session_factory(committed_engine) -> sessionmaker[Session]: + return create_session_factory(committed_engine) + + +def _new_job(catalog: CatalogVersion, household_id: UUID) -> NewHouseholdJob: + return NewHouseholdJob( + id=uuid4(), + country="us", + household_id=household_id, + policy_id=None, + year=2026, + request_data={"country": "us"}, + tax_benefit_model_version_id=catalog.version_id, + country_package_version=VERSIONS.country_package_version, + policyengine_version=POLICYENGINE_VERSION, + ) + + +def _job_store( + session_factory: sessionmaker[Session], + *, + statement_timeout_milliseconds: int = STATEMENT_TIMEOUT_MILLISECONDS, +) -> HouseholdJobStore: + return HouseholdJobStore.from_session_factory( + session_factory, + statement_timeout_milliseconds=statement_timeout_milliseconds, + ) + + +def _report_store(session_factory: sessionmaker[Session]) -> HouseholdReportStore: + return HouseholdReportStore.from_session_factory( + session_factory, + statement_timeout_milliseconds=STATEMENT_TIMEOUT_MILLISECONDS, + ) + + +def _count(engine: Engine, table_name: str, row_id: UUID) -> int: + with engine.connect() as connection: + return connection.execute( + text(f"SELECT count(*) FROM public.{table_name} WHERE id = :id"), # noqa: S608 # fixed table names + {"id": row_id}, + ).scalar_one() + + +def _flushed_rows(session: Session) -> list[object]: + """Return a list that gains each new mapped object the session flushes.""" + flushed: list[object] = [] + event.listen( + session, + "after_flush", + lambda flushing_session, _flush_context: flushed.extend(flushing_session.new), + ) + return flushed + + +def _stored_job(connection: Connection) -> NewHouseholdJob: + """Insert, as the superuser, the rows a job names; return a job naming them.""" + economy = insert_economy_catalog(connection) + household_id = insert_household( + connection, country="us", default_year=2026, document=DOCUMENT + ) + return _new_job(economy.catalog, household_id) + + +def _keyed_report(catalog: CatalogVersion, household_id: UUID) -> KeyedHouseholdReport: + """Return a request's report of the baseline alone, with fresh IDs to propose.""" + return KeyedHouseholdReport( + definition=HouseholdReportDefinition( + country="us", + baseline=CURRENT_LAW, + reform=None, + household_id=household_id, + year=2026, + ), + report_id=uuid4(), + run_id=uuid4(), + baseline=ProposedSide(simulation_id=uuid4(), simulation_run_id=uuid4()), + reform=None, + tax_benefit_model_id=catalog.model_id, + tax_benefit_model_version_id=catalog.version_id, + versions=VERSIONS, + ) + + +def _forced_rerun( + committed_engine: Engine, household: _CommittedHousehold +) -> NewHouseholdReportRerun: + """Commit a stored report of the baseline alone; return its forced re-run.""" + with committed_engine.begin() as connection: + report = insert_household_report_rows( + connection, + household.catalog, + household_id=household.household_id, + year=2026, + versions=VERSIONS, + with_reform=False, + ) + return NewHouseholdReportRerun( + report_id=report.id, + run_id=uuid4(), + tax_benefit_model_version_id=household.catalog.version_id, + year=report.year, + versions=VERSIONS, + baseline=HouseholdRerunSide( + simulation_id=report.baseline.simulation_id, simulation_run_id=uuid4() + ), + reform=None, + forced=True, + ) + + +def _unforced_rerun( + committed_engine: Engine, household: _CommittedHousehold +) -> NewHouseholdReportRerun: + """Commit a stored report; return a re-run that joins none of its runs. + + On versions no run of the report has, it goes on to its savepoint. + """ + return replace( + _forced_rerun(committed_engine, household), + forced=False, + versions=NEWER_PACKAGE_VERSIONS, + ) + + +def _new_report_run_row(report_id: UUID, versions: RunVersions) -> dict[str, Any]: + """Return a report's new run as inserted, without its sides' runs.""" + return { + "id": uuid4(), + "report_id": report_id, + "budget_window_run_id": None, + "year": None, + "country_package_version": versions.country_package_version, + "policyengine_version": versions.policyengine_version, + "status": ReportRunStatus.PENDING, + "modal_call_id": None, + } + + +def test_the_job_functions_run_orm_statements_on_their_session( + public_api_database_url, +) -> None: + with rolled_back_transaction(public_api_database_url) as connection: + new_job = _stored_job(connection) + act_as_the_service(connection) + + with session_in(connection) as session: + orm_statements = orm_statement_flags(session) + read_catalog_version_id( + session, country="us", policyengine_version=POLICYENGINE_VERSION + ) + read_stored_household( + session, country="us", household_id=new_job.household_id + ) + insert_household_job(session, new_job) + record_household_call(session, job_id=new_job.id, modal_call_id="fc-1") + fail_unstarted_household_job( + session, job_id=new_job.id, error_message="Modal refused the call." + ) + read_household_job(session, new_job.id) + + expected_statements = 5 + assert orm_statements == [True] * expected_statements + + +def test_a_new_job_is_a_mapped_row_written_before_the_insert_returns( + public_api_database_url, +) -> None: + """The flush runs the INSERT at once, and its defaults outlive the session.""" + with rolled_back_transaction(public_api_database_url) as connection: + new_job = _stored_job(connection) + act_as_the_service(connection) + + with session_in(connection) as session: + flushed = _flushed_rows(session) + insert_household_job(session, new_job) + written = connection.execute( + text("SELECT count(*) FROM public.household_jobs WHERE id = :id"), + {"id": new_job.id}, + ).scalar_one() + + assert [type(row) for row in flushed] == [HouseholdJob] + job = flushed[0] + assert job.id == new_job.id + assert written == 1 + assert job.created_at is not None + assert job.updated_at is not None + + +def test_a_job_read_in_the_session_that_changed_it_is_current( + public_api_database_url, +) -> None: + """The conditional updates run in the database; the read must see their result.""" + with rolled_back_transaction(public_api_database_url) as connection: + new_job = _stored_job(connection) + act_as_the_service(connection) + + with session_in(connection) as session: + insert_household_job(session, new_job) + record_household_call(session, job_id=new_job.id, modal_call_id="fc-1") + recorded = read_household_job(session, new_job.id) + connection.execute( + text("RESET ROLE"), + ) + connection.execute( + text( + "UPDATE public.household_jobs SET status = 'running' WHERE id = :id" + ), + {"id": new_job.id}, + ) + act_as_the_service(connection) + claimed = read_household_job(session, new_job.id) + + assert recorded is not None + assert (recorded.status, recorded.modal_call_id) == ( + HouseholdJobStatus.PENDING, + "fc-1", + ) + assert claimed is not None + assert claimed.status is HouseholdJobStatus.RUNNING + + +def test_a_household_simulation_run_is_written_before_anything_names_it( + public_api_database_url, +) -> None: + """The classes declare no relationships, so the flush can't order the rows.""" + with rolled_back_transaction(public_api_database_url) as connection: + economy = insert_economy_catalog(connection) + household_id = insert_household( + connection, country="us", default_year=2026, document=DOCUMENT + ) + report = insert_household_report_rows( + connection, + economy.catalog, + household_id=household_id, + year=2026, + versions=VERSIONS, + with_reform=False, + ) + act_as_the_service(connection) + run_id = uuid4() + + with session_in(connection) as session: + flushed = _flushed_rows(session) + insert_household_simulation_run( + session, + run_id=run_id, + simulation_id=report.baseline.simulation_id, + year=2026, + tax_benefit_model_version_id=economy.catalog.version_id, + versions=VERSIONS, + forced=True, + ) + forced = connection.execute( + text("SELECT forced FROM public.simulation_runs WHERE id = :id"), + {"id": run_id}, + ).scalar_one() + + assert [type(row) for row in flushed] == [SimulationRun] + run = flushed[0] + assert run.id == run_id + assert forced is True + assert run.created_at is not None + + +def test_a_report_run_losing_a_race_on_a_session_keeps_nothing_and_goes_on( + public_api_database_url, +) -> None: + """The savepoint takes back this call's runs; the session's transaction lives.""" + with rolled_back_transaction(public_api_database_url) as connection: + economy = insert_economy_catalog(connection) + household_id = insert_household( + connection, country="us", default_year=2026, document=DOCUMENT + ) + request = _keyed_report(economy.catalog, household_id) + act_as_the_service(connection) + first = in_session(connection, find_or_create_household_report, request) + # With the first run failed, this call inserts a new simulation run. + connection.execute(text("RESET ROLE")) + connection.execute( + text( + "UPDATE public.simulation_runs SET status = 'failed', " + "completed_at = now() WHERE id = :id" + ), + {"id": first.baseline.simulation_run_id}, + ) + act_as_the_service(connection) + proposed_run_id = uuid4() + + with session_in(connection) as session: + run, side_runs = find_or_insert_report_run_on_sides( + session, + _new_report_run_row(first.report_id, VERSIONS), + { + "baseline": household_simulation_run_row( + run_id=proposed_run_id, + simulation_id=first.baseline.simulation_id, + year=2026, + tax_benefit_model_version_id=economy.catalog.version_id, + versions=VERSIONS, + ) + }, + ) + kept_runs = session.execute( + select(func.count()) + .select_from(SimulationRun) + .where(SimulationRun.id == proposed_run_id) + ).scalar_one() + + assert (run.id, run.inserted) == (first.run.id, False) + assert side_runs == {} + assert kept_runs == 0 + + +def test_a_job_goes_through_its_sessions_and_stays_readable( + committed_engine, committed_household, session_factory +) -> None: + store = _job_store(session_factory) + called = _new_job(committed_household.catalog, committed_household.household_id) + refused = _new_job(committed_household.catalog, committed_household.household_id) + + store.create_job(called) + store.create_job(refused) + status = store.record_call(called.id, "fc-1") + failed = store.fail_unstarted(refused.id, "Modal refused the call.") + called_job = store.read(called.id) + refused_job = store.read(refused.id) + + assert status is HouseholdJobStatus.PENDING + assert failed is True + assert called_job is not None + assert called_job.modal_call_id == "fc-1" + assert called_job.created_at is not None + assert refused_job is not None + assert refused_job.status is HouseholdJobStatus.FAILED + assert refused_job.completed_at is not None + assert _count(committed_engine, "household_jobs", called.id) == 1 + + +def test_a_connection_lost_at_a_jobs_commit_leaves_the_outcome_unknown( + committed_engine, committed_household, session_factory +) -> None: + """The flush succeeded; PostgreSQL may have applied the commit.""" + unflushed_counts = end_connection_at( + session_factory, "before_commit", committed_engine + ) + job = _new_job(committed_household.catalog, committed_household.household_id) + + with pytest.raises(EconomyRecordOutcomeUnknownError) as raised: + _job_store(session_factory).create_job(job) + + assert isinstance(raised.value.__cause__, OperationalError) + assert unflushed_counts == [0] + + +def test_a_connection_lost_during_a_jobs_flush_is_not_a_lost_commit( + committed_engine, committed_household, session_factory +) -> None: + unflushed_counts = end_connection_at( + session_factory, "before_flush", committed_engine + ) + job = _new_job(committed_household.catalog, committed_household.household_id) + + with pytest.raises(OperationalError): + _job_store(session_factory).create_job(job) + + assert unflushed_counts == [1] + assert _count(committed_engine, "household_jobs", job.id) == 0 + + +def test_a_flush_outlasting_the_statement_limit_is_not_a_lost_commit( + committed_engine, committed_household, session_factory +) -> None: + """The limit covers the flush, and a cancelled INSERT wrote nothing.""" + store = _job_store( + session_factory, + statement_timeout_milliseconds=SHORT_STATEMENT_TIMEOUT_MILLISECONDS, + ) + job = _new_job(committed_household.catalog, committed_household.household_id) + + with committed_engine.begin() as locking_connection: + locking_connection.execute( + text("LOCK TABLE public.household_jobs IN ACCESS EXCLUSIVE MODE") + ) + with pytest.raises(OperationalError, match="statement timeout"): + store.create_job(job) + + assert _count(committed_engine, "household_jobs", job.id) == 0 + + +def test_a_forced_rerun_commits_its_runs_through_a_session( + committed_engine, committed_household, session_factory +) -> None: + rerun = _forced_rerun(committed_engine, committed_household) + + created = _report_store(session_factory).create_report_rerun(rerun) + + assert (created.id, created.inserted) == (rerun.run_id, True) + assert _count(committed_engine, "report_runs", rerun.run_id) == 1 + assert ( + _count(committed_engine, "simulation_runs", rerun.baseline.simulation_run_id) + == 1 + ) + + +def test_a_connection_lost_at_a_reruns_commit_leaves_the_outcome_unknown( + committed_engine, committed_household, session_factory +) -> None: + rerun = _forced_rerun(committed_engine, committed_household) + statements = statements_sent(committed_engine) + unflushed_counts = end_connection_at( + session_factory, "before_commit", committed_engine + ) + + with pytest.raises(EconomyRecordOutcomeUnknownError) as raised: + _report_store(session_factory).create_report_rerun(rerun) + + assert isinstance(raised.value.__cause__, OperationalError) + assert unflushed_counts == [0] + # A forced re-run inserts its report run directly, joining nothing. + assert any( + statement.startswith("INSERT INTO public.report_runs") + and "ON CONFLICT" not in statement + for statement in statements + ) + + +def test_a_connection_lost_during_a_reruns_flush_is_not_a_lost_commit( + committed_engine, committed_household, session_factory +) -> None: + rerun = _forced_rerun(committed_engine, committed_household) + end_connection_at(session_factory, "before_flush", committed_engine) + + with pytest.raises(OperationalError): + _report_store(session_factory).create_report_rerun(rerun) + + assert _count(committed_engine, "report_runs", rerun.run_id) == 0 + + +def test_a_forced_rerun_naming_a_missing_simulation_writes_nothing( + committed_engine, committed_household, session_factory +) -> None: + """The flush itself is refused, which is no lost commit.""" + rerun = _forced_rerun(committed_engine, committed_household) + missing_simulation = replace( + rerun, + baseline=HouseholdRerunSide(simulation_id=uuid4(), simulation_run_id=uuid4()), + ) + + with pytest.raises(EconomyRecordReferenceError) as refused: + _report_store(session_factory).create_report_rerun(missing_simulation) + + assert (refused.value.table_name, refused.value.column_name) == ( + "simulation_runs", + "simulation_id", + ) + assert _count(committed_engine, "report_runs", rerun.run_id) == 0 + + +def test_a_refusal_inside_the_savepoint_takes_back_the_other_sides_run( + public_api_database_url, +) -> None: + """The session goes on afterwards, without the run this call had inserted.""" + with rolled_back_transaction(public_api_database_url) as connection: + economy = insert_economy_catalog(connection) + household_id = insert_household( + connection, country="us", default_year=2026, document=DOCUMENT + ) + report = insert_household_report_rows( + connection, + economy.catalog, + household_id=household_id, + year=2026, + versions=VERSIONS, + with_reform=False, + ) + act_as_the_service(connection) + inserted_run_id = uuid4() + statements = statements_sent(connection) + + with session_in(connection) as session: + with pytest.raises(IntegrityError): + find_or_insert_report_run_on_sides( + session, + _new_report_run_row(report.id, NEWER_PACKAGE_VERSIONS), + { + "baseline": household_simulation_run_row( + run_id=inserted_run_id, + simulation_id=report.baseline.simulation_id, + year=2026, + tax_benefit_model_version_id=economy.catalog.version_id, + versions=NEWER_PACKAGE_VERSIONS, + ), + "reform": household_simulation_run_row( + run_id=uuid4(), + simulation_id=LAST_MISSING_SIMULATION_ID, + year=2026, + tax_benefit_model_version_id=economy.catalog.version_id, + versions=NEWER_PACKAGE_VERSIONS, + ), + }, + ) + kept_runs = session.execute( + select(func.count()) + .select_from(SimulationRun) + .where(SimulationRun.id == inserted_run_id) + ).scalar_one() + + side_run_inserts = [ + statement + for statement in statements + if statement.startswith("INSERT INTO public.simulation_runs") + ] + assert len(side_run_inserts) == len(("baseline", "reform")) + assert kept_runs == 0 + + +def test_an_unforced_rerun_refused_inside_its_savepoint_names_the_refusal( + committed_engine, committed_household, session_factory +) -> None: + rerun = _unforced_rerun(committed_engine, committed_household) + missing_simulation = replace( + rerun, + baseline=HouseholdRerunSide(simulation_id=uuid4(), simulation_run_id=uuid4()), + ) + statements = statements_sent(committed_engine) + + with pytest.raises(EconomyRecordReferenceError) as refused: + _report_store(session_factory).create_report_rerun(missing_simulation) + + assert (refused.value.table_name, refused.value.column_name) == ( + "simulation_runs", + "simulation_id", + ) + assert [ + statement.split(" sa_savepoint")[0] + for statement in statements + if "SAVEPOINT" in statement + ] == ["SAVEPOINT", "ROLLBACK TO SAVEPOINT"] + assert _count(committed_engine, "report_runs", rerun.run_id) == 0 + + +def test_a_connection_lost_inside_an_unforced_reruns_savepoint_is_not_a_lost_commit( + committed_engine, committed_household, session_factory +) -> None: + """The find-or-insert of the report run fails after the side's run was inserted.""" + rerun = _unforced_rerun(committed_engine, committed_household) + statements = statements_sent(committed_engine) + ended_backends: list[int] = [] + + def end_the_connection_at_the_report_run( + _connection, cursor, statement, _parameters, _context, _executemany + ) -> None: + if ( + ended_backends + or not statement.startswith("INSERT INTO public.report_runs") + or "ON CONFLICT" not in statement + ): + return + ended_backends.append(cursor.connection.info.backend_pid) + terminate_backend(committed_engine, ended_backends[0]) + + event.listen( + committed_engine, "before_cursor_execute", end_the_connection_at_the_report_run + ) + + with pytest.raises(OperationalError): + _report_store(session_factory).create_report_rerun(rerun) + + assert len(ended_backends) == 1 + assert any(statement.startswith("SAVEPOINT") for statement in statements) + assert _count(committed_engine, "report_runs", rerun.run_id) == 0 + assert ( + _count(committed_engine, "simulation_runs", rerun.baseline.simulation_run_id) + == 0 + ) + + +def test_two_identical_report_requests_at_once_end_on_one_run( + committed_engine, committed_household, session_factory +) -> None: + """The first holds its session open until the second waits on its rows.""" + first_has_inserted = threading.Event() + found: dict[str, Any] = {} + + def create(session: Session, name: str) -> None: + act_as_the_service(session) + found[name] = find_or_create_household_report( + session, + _keyed_report( + committed_household.catalog, committed_household.household_id + ), + ) + + def first_request() -> None: + with session_factory.begin() as session: + create(session, "first") + holder_pid = session.execute(select(func.pg_backend_pid())).scalar_one() + first_has_inserted.set() + wait_until_another_backend_waits_for(committed_engine, holder_pid) + + def second_request() -> None: + assert first_has_inserted.wait(LOCK_WAIT_TIMEOUT_SECONDS) + with session_factory.begin() as session: + create(session, "second") + + run_together((first_request, second_request), released=first_has_inserted) + + first, second = found["first"], found["second"] + assert (second.report_id, second.run.id) == (first.report_id, first.run.id) + assert (first.run.inserted, second.run.inserted) == (True, False) + assert second.baseline == replace(first.baseline, run_inserted=False) diff --git a/libs/policyengine-simulation-persistence/tests/test_keyed_economy_records.py b/libs/policyengine-simulation-persistence/tests/test_keyed_economy_records.py index 592b06423..7d577ba38 100644 --- a/libs/policyengine-simulation-persistence/tests/test_keyed_economy_records.py +++ b/libs/policyengine-simulation-persistence/tests/test_keyed_economy_records.py @@ -12,7 +12,6 @@ from __future__ import annotations import threading -from contextlib import contextmanager from dataclasses import replace from typing import TYPE_CHECKING, Any from uuid import UUID, uuid4 @@ -23,7 +22,12 @@ run_together, wait_until_another_backend_waits_for, ) -from conftest import act_as_the_service, rolled_back_transaction +from conftest import ( + act_as_the_service, + economy_store_on, + in_session, + rolled_back_transaction, +) from economy_rows import ( CURRENT_LAW, VERSIONS, @@ -34,29 +38,26 @@ KEY_VERSION, EconomyRecordReferenceError, EconomyReportDefinition, - EconomyRunStore, EconomySimulationDefinition, KeyedEconomyReport, KeyedEconomySimulation, NewReportRerun, ProposedSide, + ReportRun, ReportRunStatus, RerunSide, SimulationChoice, + SimulationRun, SimulationStatus, find_or_create_economy_report, find_or_create_economy_simulation, insert_report_rerun, read_report_definition, - report_runs, - simulation_runs, ) from public_api_rows import insert_policy from sqlalchemy import event, func, insert, select, text, update if TYPE_CHECKING: - from collections.abc import Iterator - from sqlalchemy import Connection pytestmark = pytest.mark.public_api_schema @@ -131,7 +132,9 @@ def _set_status( connection: Connection, table_name: str, row_id: UUID, status: str ) -> None: """Set a run's status as the superuser, as a worker or the sweep would.""" - table = report_runs if table_name == "report_runs" else simulation_runs + table = ( + ReportRun.__table__ if table_name == "report_runs" else SimulationRun.__table__ + ) ended = status in {"succeeded", "failed"} connection.execute(text("RESET ROLE")) connection.execute( @@ -155,7 +158,7 @@ def test_a_keyed_report_is_created_with_its_simulations_and_first_run( ) act_as_the_service(connection) - found = find_or_create_economy_report(connection, request) + found = in_session(connection, find_or_create_economy_report, request) report = _row(connection, "reports", found.report_id) run = _row(connection, "report_runs", found.run.id) baseline = _row(connection, "simulations", found.baseline.simulation_id) @@ -233,16 +236,20 @@ def test_an_identical_request_ends_on_the_same_rows_and_inserts_nothing( with rolled_back_transaction(public_api_database_url) as connection: economy = insert_economy_catalog(connection) act_as_the_service(connection) - first = find_or_create_economy_report( - connection, _keyed_report(economy, spm=NATIONAL_SPM) + first = in_session( + connection, + find_or_create_economy_report, + _keyed_report(economy, spm=NATIONAL_SPM), ) counts = [ _count(connection, name) for name in ("simulations", "reports", "simulation_runs", "report_runs") ] - again = find_or_create_economy_report( - connection, _keyed_report(economy, spm=NATIONAL_SPM) + again = in_session( + connection, + find_or_create_economy_report, + _keyed_report(economy, spm=NATIONAL_SPM), ) counts_after = [ _count(connection, name) @@ -269,7 +276,7 @@ def test_the_same_policy_on_both_sides_names_one_simulation_run_twice( request = _keyed_report(economy) act_as_the_service(connection) - found = find_or_create_economy_report(connection, request) + found = in_session(connection, find_or_create_economy_report, request) run = _row(connection, "report_runs", found.run.id) simulation_count = _count(connection, "simulations") @@ -289,8 +296,10 @@ def test_a_report_whose_run_failed_gets_a_new_run_joining_its_simulation_runs( policy_id = insert_policy(connection, economy.catalog) reform = SimulationChoice(policy_id=policy_id, dynamic_id=None) act_as_the_service(connection) - first = find_or_create_economy_report( - connection, _keyed_report(economy, reform=reform) + first = in_session( + connection, + find_or_create_economy_report, + _keyed_report(economy, reform=reform), ) _set_status( connection, "simulation_runs", first.baseline.simulation_run_id, "succeeded" @@ -300,8 +309,10 @@ def test_a_report_whose_run_failed_gets_a_new_run_joining_its_simulation_runs( ) _set_status(connection, "report_runs", first.run.id, "failed") - second = find_or_create_economy_report( - connection, _keyed_report(economy, reform=reform) + second = in_session( + connection, + find_or_create_economy_report, + _keyed_report(economy, reform=reform), ) assert second.report_id == first.report_id @@ -318,14 +329,16 @@ def test_a_forced_run_is_never_joined_by_a_create(public_api_database_url) -> No with rolled_back_transaction(public_api_database_url) as connection: economy = insert_economy_catalog(connection) act_as_the_service(connection) - first = find_or_create_economy_report(connection, _keyed_report(economy)) + first = in_session( + connection, find_or_create_economy_report, _keyed_report(economy) + ) _set_status( connection, "simulation_runs", first.baseline.simulation_run_id, "failed" ) _set_status(connection, "report_runs", first.run.id, "failed") forced_simulation_run_id = uuid4() connection.execute( - insert(simulation_runs).values( + insert(SimulationRun).values( id=forced_simulation_run_id, simulation_id=first.baseline.simulation_id, year=2026, @@ -339,7 +352,7 @@ def test_a_forced_run_is_never_joined_by_a_create(public_api_database_url) -> No ) ) connection.execute( - insert(report_runs).values( + insert(ReportRun).values( id=uuid4(), report_id=first.report_id, baseline_simulation_run_id=forced_simulation_run_id, @@ -352,7 +365,9 @@ def test_a_forced_run_is_never_joined_by_a_create(public_api_database_url) -> No ) ) - second = find_or_create_economy_report(connection, _keyed_report(economy)) + second = in_session( + connection, find_or_create_economy_report, _keyed_report(economy) + ) assert second.run.inserted is True assert second.baseline.run_inserted is True @@ -370,9 +385,13 @@ def test_another_spm_selection_is_another_report_with_other_simulations( economy = insert_economy_catalog(connection) act_as_the_service(connection) - without = find_or_create_economy_report(connection, _keyed_report(economy)) - national = find_or_create_economy_report( - connection, _keyed_report(economy, spm=NATIONAL_SPM) + without = in_session( + connection, find_or_create_economy_report, _keyed_report(economy) + ) + national = in_session( + connection, + find_or_create_economy_report, + _keyed_report(economy, spm=NATIONAL_SPM), ) assert national.report_id != without.report_id @@ -389,16 +408,18 @@ def test_a_report_sharing_a_side_with_another_names_that_sides_run_under_way( first_policy = insert_policy(connection, economy.catalog) second_policy = insert_policy(connection, economy.catalog) act_as_the_service(connection) - first = find_or_create_economy_report( + first = in_session( connection, + find_or_create_economy_report, _keyed_report(economy, reform=SimulationChoice(first_policy, None)), ) _set_status( connection, "simulation_runs", first.baseline.simulation_run_id, "running" ) - second = find_or_create_economy_report( + second = in_session( connection, + find_or_create_economy_report, _keyed_report(economy, reform=SimulationChoice(second_policy, None)), ) @@ -416,17 +437,19 @@ def test_a_keyed_simulation_joins_its_run_or_creates_it( with rolled_back_transaction(public_api_database_url) as connection: economy = insert_economy_catalog(connection) act_as_the_service(connection) - report = find_or_create_economy_report(connection, _keyed_report(economy)) + report = in_session( + connection, find_or_create_economy_report, _keyed_report(economy) + ) # The report's current-law simulation is this request's simulation. - joined = find_or_create_economy_simulation( - connection, _keyed_simulation(economy) + joined = in_session( + connection, find_or_create_economy_simulation, _keyed_simulation(economy) ) _set_status( connection, "simulation_runs", report.baseline.simulation_run_id, "failed" ) request = _keyed_simulation(economy) - created = find_or_create_economy_simulation(connection, request) + created = in_session(connection, find_or_create_economy_simulation, request) simulation = _row(connection, "simulations", created.simulation_id) assert (joined.simulation_id, joined.run.id) == ( @@ -455,7 +478,9 @@ def test_two_identical_requests_at_once_end_on_the_same_rows_and_start_one_call( def create_and_start(connection: Connection, name: str) -> None: act_as_the_service(connection) - result = find_or_create_economy_report(connection, _keyed_report(economy)) + result = in_session( + connection, find_or_create_economy_report, _keyed_report(economy) + ) found[name] = result if result.run.inserted: started_calls.append(result.run.id) @@ -487,16 +512,18 @@ def test_a_request_losing_the_report_run_to_another_joins_it_and_keeps_nothing( economy = committed_catalog with committed_engine.begin() as connection: act_as_the_service(connection) - first = find_or_create_economy_report(connection, _keyed_report(economy)) + first = in_session( + connection, find_or_create_economy_report, _keyed_report(economy) + ) with committed_engine.begin() as connection: connection.execute( - update(simulation_runs) - .where(simulation_runs.c.id == first.baseline.simulation_run_id) + update(SimulationRun) + .where(SimulationRun.id == first.baseline.simulation_run_id) .values(status=SimulationStatus.FAILED, completed_at=func.now()) ) connection.execute( - update(report_runs) - .where(report_runs.c.id == first.run.id) + update(ReportRun) + .where(ReportRun.id == first.run.id) .values(status=ReportRunStatus.FAILED, completed_at=func.now()) ) competing: list[UUID] = [] @@ -514,7 +541,7 @@ def create_a_competing_run_first( run_id = uuid4() with committed_engine.begin() as other: other.execute( - insert(report_runs).values( + insert(ReportRun).values( id=run_id, report_id=first.report_id, baseline_simulation_run_id=first.baseline.simulation_run_id, @@ -531,15 +558,15 @@ def create_a_competing_run_first( act_as_the_service(connection) event.listen(connection, "before_cursor_execute", create_a_competing_run_first) try: - joined = find_or_create_economy_report(connection, request) + joined = in_session(connection, find_or_create_economy_report, request) finally: event.remove( connection, "before_cursor_execute", create_a_competing_run_first ) kept = connection.execute( select(func.count()) - .select_from(simulation_runs) - .where(simulation_runs.c.id == request.baseline.simulation_run_id) + .select_from(SimulationRun) + .where(SimulationRun.id == request.baseline.simulation_run_id) ).scalar_one() assert (joined.run.id, joined.run.inserted) == (competing[0], False) @@ -547,24 +574,13 @@ def create_a_competing_run_first( assert kept == 0 -def _store_on(connection: Connection) -> EconomyRunStore: - """Return a store whose transactions are savepoints of ``connection``.""" - - @contextmanager - def savepoint() -> Iterator[Connection]: - with connection.begin_nested(): - yield connection - - return EconomyRunStore(savepoint) - - def test_the_store_finds_or_creates_reports_and_simulations_in_a_transaction( public_api_database_url, ) -> None: with rolled_back_transaction(public_api_database_url) as connection: economy = insert_economy_catalog(connection) act_as_the_service(connection) - store = _store_on(connection) + store = economy_store_on(connection) created = store.find_or_create_report(_keyed_report(economy)) joined = store.find_or_create_report(_keyed_report(economy)) @@ -586,8 +602,9 @@ def test_a_keyed_reports_rerun_joins_each_sides_run_on_its_versions( economy = insert_economy_catalog(connection) policy_id = insert_policy(connection, economy.catalog) act_as_the_service(connection) - created = find_or_create_economy_report( + created = in_session( connection, + find_or_create_economy_report, _keyed_report(economy, reform=SimulationChoice(policy_id, None)), ) _set_status( @@ -597,7 +614,7 @@ def test_a_keyed_reports_rerun_joins_each_sides_run_on_its_versions( "succeeded", ) _set_status(connection, "report_runs", created.run.id, "failed") - definition = read_report_definition(connection, created.report_id) + definition = in_session(connection, read_report_definition, created.report_id) rerun = NewReportRerun( report_id=created.report_id, run_id=uuid4(), @@ -607,7 +624,7 @@ def test_a_keyed_reports_rerun_joins_each_sides_run_on_its_versions( reform=RerunSide(created.reform.simulation_id, uuid4()), ) - found = insert_report_rerun(connection, rerun) + found = in_session(connection, insert_report_rerun, rerun) assert definition is not None assert definition.keyed is True @@ -625,7 +642,9 @@ def test_a_keyed_rerun_with_one_simulation_on_both_sides_inserts_one_run( with rolled_back_transaction(public_api_database_url) as connection: economy = insert_economy_catalog(connection) act_as_the_service(connection) - created = find_or_create_economy_report(connection, _keyed_report(economy)) + created = in_session( + connection, find_or_create_economy_report, _keyed_report(economy) + ) _set_status( connection, "simulation_runs", created.baseline.simulation_run_id, "failed" ) @@ -640,11 +659,11 @@ def test_a_keyed_rerun_with_one_simulation_on_both_sides_inserts_one_run( reform=RerunSide(simulation_id, uuid4()), ) - found = insert_report_rerun(connection, rerun) + found = in_session(connection, insert_report_rerun, rerun) runs = connection.execute( select(func.count()) - .select_from(simulation_runs) - .where(simulation_runs.c.simulation_id == simulation_id) + .select_from(SimulationRun) + .where(SimulationRun.simulation_id == simulation_id) ).scalar_one() assert found.inserted is True @@ -686,7 +705,7 @@ def test_a_report_naming_a_missing_row_names_the_column_that_refused_it( act_as_the_service(connection) with pytest.raises(EconomyRecordReferenceError) as refused: - find_or_create_economy_report(connection, report) + in_session(connection, find_or_create_economy_report, report) assert (refused.value.table_name, refused.value.column_name) == ( "simulations", diff --git a/libs/policyengine-simulation-persistence/tests/test_keyed_household_records.py b/libs/policyengine-simulation-persistence/tests/test_keyed_household_records.py index 120d20a70..bda88b6f1 100644 --- a/libs/policyengine-simulation-persistence/tests/test_keyed_household_records.py +++ b/libs/policyengine-simulation-persistence/tests/test_keyed_household_records.py @@ -2,20 +2,24 @@ Identical household report requests, together or apart, end on the same simulation, report and run rows, as economy ones do. These run against a -database the Public API's own v2 migration chain built +database API v1's own v2 migration chain built (``PUBLIC_API_SCHEMA_DATABASE_URL``), as the simulation service's role, in a transaction rolled back afterwards. """ from __future__ import annotations -from contextlib import contextmanager from dataclasses import dataclass, replace from typing import TYPE_CHECKING from uuid import UUID, uuid4 import pytest -from conftest import act_as_the_service, rolled_back_transaction +from conftest import ( + act_as_the_service, + in_session, + rolled_back_transaction, + session_factory_on, +) from policyengine_simulation_persistence import ( KEY_VERSION, EconomyRecordReferenceError, @@ -44,12 +48,11 @@ from sqlalchemy import text if TYPE_CHECKING: - from collections.abc import Iterator - from sqlalchemy import Connection pytestmark = pytest.mark.public_api_schema +STATEMENT_TIMEOUT_MILLISECONDS = 5_000 VERSIONS = RunVersions(country_package_version="1.764.6", policyengine_version="6.17.1") CURRENT_LAW = SimulationChoice(policy_id=None, dynamic_id=None) DOCUMENT = { @@ -155,7 +158,7 @@ def test_a_keyed_household_report_is_created_with_its_simulations_and_first_run( request = _keyed_report(stored, reform=stored.reform) act_as_the_service(connection) - found = find_or_create_household_report(connection, request) + found = in_session(connection, find_or_create_household_report, request) report = _row(connection, "reports", found.report_id) report_run = _row(connection, "report_runs", found.run.id) sides = { @@ -260,16 +263,20 @@ def test_an_identical_household_request_ends_on_the_same_rows_and_inserts_nothin with rolled_back_transaction(public_api_database_url) as connection: stored = _stored_inputs(connection) act_as_the_service(connection) - first = find_or_create_household_report( - connection, _keyed_report(stored, reform=stored.reform) + first = in_session( + connection, + find_or_create_household_report, + _keyed_report(stored, reform=stored.reform), ) counts = [ _count(connection, name) for name in ("simulations", "reports", "simulation_runs", "report_runs") ] - again = find_or_create_household_report( - connection, _keyed_report(stored, reform=stored.reform) + again = in_session( + connection, + find_or_create_household_report, + _keyed_report(stored, reform=stored.reform), ) counts_again = [ _count(connection, name) @@ -292,7 +299,7 @@ def test_a_household_policy_compared_with_itself_names_one_simulation_run_twice( request = _keyed_report(stored, reform=CURRENT_LAW) act_as_the_service(connection) - found = find_or_create_household_report(connection, request) + found = in_session(connection, find_or_create_household_report, request) report_run = _row(connection, "report_runs", found.run.id) assert found.reform is not None @@ -313,9 +320,11 @@ def test_a_household_report_of_the_baseline_alone_names_no_reform( request = _keyed_report(stored, reform=None) act_as_the_service(connection) - found = find_or_create_household_report(connection, request) - again = find_or_create_household_report( - connection, _keyed_report(stored, reform=None) + found = in_session(connection, find_or_create_household_report, request) + again = in_session( + connection, + find_or_create_household_report, + _keyed_report(stored, reform=None), ) report = _row(connection, "reports", found.report_id) report_run = _row(connection, "report_runs", found.run.id) @@ -338,12 +347,16 @@ def test_a_household_report_sharing_a_side_names_that_sides_run_under_way( with rolled_back_transaction(public_api_database_url) as connection: stored = _stored_inputs(connection) act_as_the_service(connection) - with_reform = find_or_create_household_report( - connection, _keyed_report(stored, reform=stored.reform) + with_reform = in_session( + connection, + find_or_create_household_report, + _keyed_report(stored, reform=stored.reform), ) - alone = find_or_create_household_report( - connection, _keyed_report(stored, reform=None) + alone = in_session( + connection, + find_or_create_household_report, + _keyed_report(stored, reform=None), ) assert alone.report_id != with_reform.report_id @@ -358,13 +371,17 @@ def test_a_household_report_whose_run_failed_gets_a_new_run_on_its_simulations( with rolled_back_transaction(public_api_database_url) as connection: stored = _stored_inputs(connection) act_as_the_service(connection) - first = find_or_create_household_report( - connection, _keyed_report(stored, reform=stored.reform) + first = in_session( + connection, + find_or_create_household_report, + _keyed_report(stored, reform=stored.reform), ) _fail_report_run(connection, first.run.id) - second = find_or_create_household_report( - connection, _keyed_report(stored, reform=stored.reform) + second = in_session( + connection, + find_or_create_household_report, + _keyed_report(stored, reform=stored.reform), ) assert second.report_id == first.report_id @@ -380,12 +397,10 @@ def test_the_household_store_finds_or_creates_a_report_in_a_transaction( stored = _stored_inputs(connection) act_as_the_service(connection) - @contextmanager - def savepoint() -> Iterator[Connection]: - with connection.begin_nested(): - yield connection - - store = HouseholdReportStore(savepoint) + store = HouseholdReportStore.from_session_factory( + session_factory_on(connection), + statement_timeout_milliseconds=STATEMENT_TIMEOUT_MILLISECONDS, + ) found = store.find_or_create_report(_keyed_report(stored, reform=None)) again = store.find_or_create_report(_keyed_report(stored, reform=None)) @@ -446,14 +461,18 @@ def test_a_keyed_household_reports_rerun_joins_its_run_on_the_versions( with rolled_back_transaction(public_api_database_url) as connection: stored = _stored_inputs(connection) act_as_the_service(connection) - found = find_or_create_household_report( - connection, _keyed_report(stored, reform=stored.reform) + found = in_session( + connection, + find_or_create_household_report, + _keyed_report(stored, reform=stored.reform), ) counts = [ _count(connection, name) for name in ("simulation_runs", "report_runs") ] - joined = insert_household_report_rerun(connection, _rerun_of(found, stored)) + joined = in_session( + connection, insert_household_report_rerun, _rerun_of(found, stored) + ) counts_after = [ _count(connection, name) for name in ("simulation_runs", "report_runs") ] @@ -469,13 +488,15 @@ def test_a_keyed_household_rerun_after_a_failed_run_names_its_simulations_runs( with rolled_back_transaction(public_api_database_url) as connection: stored = _stored_inputs(connection) act_as_the_service(connection) - found = find_or_create_household_report( - connection, _keyed_report(stored, reform=stored.reform) + found = in_session( + connection, + find_or_create_household_report, + _keyed_report(stored, reform=stored.reform), ) _fail_report_run(connection, found.run.id) rerun = _rerun_of(found, stored) - created = insert_household_report_rerun(connection, rerun) + created = in_session(connection, insert_household_report_rerun, rerun) report_run = _row(connection, "report_runs", created.id) assert found.reform is not None @@ -494,12 +515,14 @@ def test_a_forced_keyed_household_rerun_is_an_extra_run_beside_the_one_under_way with rolled_back_transaction(public_api_database_url) as connection: stored = _stored_inputs(connection) act_as_the_service(connection) - found = find_or_create_household_report( - connection, _keyed_report(stored, reform=stored.reform) + found = in_session( + connection, + find_or_create_household_report, + _keyed_report(stored, reform=stored.reform), ) rerun = _rerun_of(found, stored, forced=True) - created = insert_household_report_rerun(connection, rerun) + created = in_session(connection, insert_household_report_rerun, rerun) report_run = _row(connection, "report_runs", created.id) side_runs = [ _row(connection, "simulation_runs", run_id) @@ -523,6 +546,6 @@ def test_a_household_report_naming_a_missing_household_is_refused_by_its_key( act_as_the_service(connection) with pytest.raises(EconomyRecordReferenceError) as refused: - find_or_create_household_report(connection, missing_household) + in_session(connection, find_or_create_household_report, missing_household) assert refused.value.column_name == "household_id" diff --git a/libs/policyengine-simulation-persistence/tests/test_keyed_window_records.py b/libs/policyengine-simulation-persistence/tests/test_keyed_window_records.py index 746f4355b..33bc7e5fe 100644 --- a/libs/policyengine-simulation-persistence/tests/test_keyed_window_records.py +++ b/libs/policyengine-simulation-persistence/tests/test_keyed_window_records.py @@ -11,7 +11,6 @@ from __future__ import annotations import threading -from contextlib import contextmanager from dataclasses import replace from typing import TYPE_CHECKING from uuid import UUID, uuid4 @@ -22,7 +21,12 @@ run_together, wait_until_another_backend_waits_for, ) -from conftest import act_as_the_service, rolled_back_transaction +from conftest import ( + act_as_the_service, + economy_store_on, + in_session, + rolled_back_transaction, +) from economy_rows import ( CURRENT_LAW, VERSIONS, @@ -34,7 +38,6 @@ EconomyPopulation, EconomyRecordReferenceError, EconomyReportDefinition, - EconomyRunStore, FoundBudgetWindow, KeyedBudgetWindow, KeyedEconomyReport, @@ -49,8 +52,6 @@ from sqlalchemy import func, select, text if TYPE_CHECKING: - from collections.abc import Iterator - from sqlalchemy import Connection pytestmark = pytest.mark.public_api_schema @@ -156,7 +157,7 @@ def test_a_keyed_window_is_created_with_each_years_simulations_and_runs( request = _keyed_window(economy, _reform(connection, economy)) act_as_the_service(connection) - found = find_or_create_budget_window(connection, request) + found = in_session(connection, find_or_create_budget_window, request) report = _row(connection, "reports", found.report_id) window_run = _row(connection, "budget_window_runs", found.window_run.id) year_runs = [ @@ -234,10 +235,14 @@ def test_an_identical_window_request_joins_its_window_run_and_inserts_nothing( economy = insert_economy_catalog(connection) reform = _reform(connection, economy) act_as_the_service(connection) - first = find_or_create_budget_window(connection, _keyed_window(economy, reform)) + first = in_session( + connection, find_or_create_budget_window, _keyed_window(economy, reform) + ) counts = _counts(connection) - again = find_or_create_budget_window(connection, _keyed_window(economy, reform)) + again = in_session( + connection, find_or_create_budget_window, _keyed_window(economy, reform) + ) counts_again = _counts(connection) assert (again.report_id, again.report_inserted) == (first.report_id, False) @@ -264,8 +269,9 @@ def test_a_window_year_names_a_single_year_reports_simulation_run( economy = insert_economy_catalog(connection) reform = _reform(connection, economy) act_as_the_service(connection) - single_year = find_or_create_economy_report( + single_year = in_session( connection, + find_or_create_economy_report, KeyedEconomyReport( definition=_definition( reform, country=economy.catalog.country, year=2027, size=None @@ -281,8 +287,8 @@ def test_a_window_year_names_a_single_year_reports_simulation_run( ), ) - window = find_or_create_budget_window( - connection, _keyed_window(economy, reform) + window = in_session( + connection, find_or_create_budget_window, _keyed_window(economy, reform) ) year_2027 = window.years[1] @@ -303,11 +309,13 @@ def test_a_window_whose_run_failed_gets_a_new_run_on_its_years_simulation_runs( economy = insert_economy_catalog(connection) reform = _reform(connection, economy) act_as_the_service(connection) - first = find_or_create_budget_window(connection, _keyed_window(economy, reform)) + first = in_session( + connection, find_or_create_budget_window, _keyed_window(economy, reform) + ) _fail_window_run(connection, first.window_run.id) - second = find_or_create_budget_window( - connection, _keyed_window(economy, reform) + second = in_session( + connection, find_or_create_budget_window, _keyed_window(economy, reform) ) assert second.report_id == first.report_id @@ -329,7 +337,7 @@ def test_a_window_comparing_a_policy_with_itself_names_one_run_a_year_twice( request = _keyed_window(economy, CURRENT_LAW) act_as_the_service(connection) - found = find_or_create_budget_window(connection, request) + found = in_session(connection, find_or_create_budget_window, request) for proposed, year in zip(request.years, found.years, strict=True): assert year.baseline.simulation_id == year.reform.simulation_id @@ -345,12 +353,7 @@ def test_the_store_finds_or_creates_a_window_in_a_transaction( reform = _reform(connection, economy) act_as_the_service(connection) - @contextmanager - def savepoint() -> Iterator[Connection]: - with connection.begin_nested(): - yield connection - - store = EconomyRunStore(savepoint) + store = economy_store_on(connection) found = store.find_or_create_budget_window(_keyed_window(economy, reform)) again = store.find_or_create_budget_window(_keyed_window(economy, reform)) @@ -403,8 +406,10 @@ def test_two_identical_window_requests_at_once_end_on_one_window_run_started_onc economy = committed_catalog with committed_engine.begin() as connection: act_as_the_service(connection) - earlier = find_or_create_budget_window( - connection, _keyed_window(economy, CURRENT_LAW) + earlier = in_session( + connection, + find_or_create_budget_window, + _keyed_window(economy, CURRENT_LAW), ) with committed_engine.begin() as connection: _fail_window_run(connection, earlier.window_run.id) @@ -415,8 +420,10 @@ def test_two_identical_window_requests_at_once_end_on_one_window_run_started_onc def create_and_start(connection: Connection, name: str) -> None: act_as_the_service(connection) - result = find_or_create_budget_window( - connection, _keyed_window(economy, CURRENT_LAW) + result = in_session( + connection, + find_or_create_budget_window, + _keyed_window(economy, CURRENT_LAW), ) found[name] = result if result.window_run.inserted: @@ -467,8 +474,10 @@ def test_a_window_naming_a_missing_policy_is_refused_and_writes_nothing( pytest.raises(EconomyRecordReferenceError) as refused, connection.begin_nested(), ): - find_or_create_budget_window( - connection, _keyed_window(economy, missing_policy) + in_session( + connection, + find_or_create_budget_window, + _keyed_window(economy, missing_policy), ) after = _counts(connection) diff --git a/libs/policyengine-simulation-persistence/tests/test_mapped_classes.py b/libs/policyengine-simulation-persistence/tests/test_mapped_classes.py new file mode 100644 index 000000000..cfaf93b03 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_mapped_classes.py @@ -0,0 +1,366 @@ +"""The ORM mapped classes of API v1's tables. + +Each table has one typed mapped class, and the stores' statements name those +classes. The modules no longer export a Core table under the table's name. +""" + +from __future__ import annotations + +import inspect as python_inspect +from types import NoneType, UnionType +from typing import Any, Union, get_args, get_origin +from uuid import UUID, uuid4 + +import pytest +from conftest import SERVICE_TEST_ROLE, simulation_engine +from policyengine_simulation_persistence import ( + BudgetSummary, + BudgetWindowRun, + CongressionalDistrictImpact, + ConstituencyImpact, + Dataset, + DecileImpact, + Dynamic, + Household, + HouseholdJob, + HouseholdJobStatus, + Inequality, + InputBase, + IntraDecileImpact, + LaborSupplyDecileResponse, + LaborSupplyResponse, + LocalAuthorityImpact, + Parameter, + ParameterValue, + Policy, + Poverty, + ProgramStatistics, + ReferencedBase, + Region, + Report, + ReportRun, + ResultBase, + ServiceBase, + Simulation, + SimulationRun, + SimulationStatus, + SimulationType, + TaxBenefitModel, + TaxBenefitModelVersion, + WithheldAreaCode, + create_session_factory, + input_metadata, + input_tables, + referenced_metadata, + referenced_tables, + result_metadata, + result_tables, + service_metadata, + tables, +) +from public_api_keys import KEYS_TO_PUBLIC_API_TABLES +from schema_comparison import describe_tables +from sqlalchemy import create_engine, inspect, select, text, update +from sqlalchemy.dialects import postgresql +from sqlalchemy.engine import make_url +from sqlalchemy.orm import Mapped, configure_mappers + +SERVICE_CLASSES = { + "simulations": Simulation, + "reports": Report, + "simulation_runs": SimulationRun, + "budget_window_runs": BudgetWindowRun, + "report_runs": ReportRun, + "household_jobs": HouseholdJob, +} +REFERENCED_CLASSES = { + "datasets": referenced_tables.ReferencedDataset, + "dynamics": referenced_tables.ReferencedDynamic, + "households": referenced_tables.ReferencedHousehold, + "policies": referenced_tables.ReferencedPolicy, + "regions": referenced_tables.ReferencedRegion, + "tax_benefit_model_versions": referenced_tables.ReferencedTaxBenefitModelVersion, + "tax_benefit_models": referenced_tables.ReferencedTaxBenefitModel, + "users": referenced_tables.ReferencedUser, +} +INPUT_CLASSES = { + "tax_benefit_models": TaxBenefitModel, + "tax_benefit_model_versions": TaxBenefitModelVersion, + "regions": Region, + "datasets": Dataset, + "households": Household, + "policies": Policy, + "parameters": Parameter, + "parameter_values": ParameterValue, + "dynamics": Dynamic, +} +RESULT_CLASSES = { + "labor_supply_responses": LaborSupplyResponse, + "labor_supply_decile_responses": LaborSupplyDecileResponse, + "budget_summary": BudgetSummary, + "decile_impacts": DecileImpact, + "intra_decile_impacts": IntraDecileImpact, + "program_statistics": ProgramStatistics, + "congressional_district_impacts": CongressionalDistrictImpact, + "constituency_impacts": ConstituencyImpact, + "local_authority_impacts": LocalAuthorityImpact, + "withheld_area_codes": WithheldAreaCode, + "poverty": Poverty, + "inequality": Inequality, +} +GROUPS = [ + pytest.param(ServiceBase, service_metadata, SERVICE_CLASSES, id="service"), + pytest.param(ReferencedBase, referenced_metadata, REFERENCED_CLASSES, id="ref"), + pytest.param(InputBase, input_metadata, INPUT_CLASSES, id="input"), + pytest.param(ResultBase, result_metadata, RESULT_CLASSES, id="result"), +] +# The modules whose Core table names the stores import. +TABLE_NAME_MODULES = [ + pytest.param(tables, SERVICE_CLASSES, id="service"), + pytest.param(input_tables, INPUT_CLASSES, id="input"), + pytest.param(result_tables, RESULT_CLASSES, id="result"), +] +ALL_CLASSES = [ + *SERVICE_CLASSES.values(), + *REFERENCED_CLASSES.values(), + *INPUT_CLASSES.values(), + *RESULT_CLASSES.values(), +] + + +def _allows_none(annotation: object) -> bool: + return get_origin(annotation) in (Union, UnionType) and NoneType in get_args( + annotation + ) + + +@pytest.mark.parametrize(("base", "metadata", "classes"), GROUPS) +def test_each_base_maps_exactly_the_tables_of_its_metadata( + base, metadata, classes +) -> None: + mapped_classes = {mapper.class_ for mapper in base.registry.mappers} + + assert base.metadata is metadata + assert metadata.schema == "public" + assert mapped_classes == set(classes.values()) + assert {table.name for table in metadata.tables.values()} == set(classes) + for table_name, mapped_class in classes.items(): + assert mapped_class.__tablename__ == table_name + assert mapped_class.__table__.metadata is metadata + + +def test_the_groups_keep_the_tables_they_share_a_name_with_apart() -> None: + """A read model and a key's target of one table are separate classes.""" + assert INPUT_CLASSES["households"].__table__ is not ( + REFERENCED_CLASSES["households"].__table__ + ) + assert [column.name for column in REFERENCED_CLASSES["households"].__table__.c] == [ + "id" + ] + + +@pytest.mark.parametrize(("module", "classes"), TABLE_NAME_MODULES) +def test_the_table_modules_export_no_core_table_names(module, classes) -> None: + exported_table_names = [ + table_name for table_name in classes if hasattr(module, table_name) + ] + + assert exported_table_names == [] + + +@pytest.mark.parametrize("mapped_class", ALL_CLASSES, ids=lambda item: item.__name__) +def test_each_column_is_the_attribute_of_its_own_name(mapped_class) -> None: + mapper = inspect(mapped_class) + + assert [attribute.key for attribute in mapper.column_attrs] == [ + column.name for column in mapped_class.__table__.columns + ] + + +@pytest.mark.parametrize("mapped_class", ALL_CLASSES, ids=lambda item: item.__name__) +def test_each_annotation_states_whether_its_column_allows_null(mapped_class) -> None: + annotations = python_inspect.get_annotations(mapped_class, eval_str=True) + + for column in mapped_class.__table__.columns: + annotation = annotations[column.name] + assert get_origin(annotation) is Mapped, column.name + (value_type,) = get_args(annotation) + # A JSON value may be anything, so its annotation can't say. + if value_type is not Any: + assert _allows_none(value_type) is column.nullable, column.name + + +@pytest.mark.parametrize("mapped_class", ALL_CLASSES, ids=lambda item: item.__name__) +def test_each_class_is_identified_by_the_id_column(mapped_class) -> None: + primary_key = inspect(mapped_class).primary_key + + assert [column.name for column in primary_key] == ["id"] + + +def test_the_keys_to_api_v1s_tables_name_the_referenced_classes() -> None: + referenced = {model.__table__ for model in REFERENCED_CLASSES.values()} + named_tables = { + constraint.referred_table + for model in SERVICE_CLASSES.values() + for constraint in model.__table__.foreign_key_constraints + if constraint.referred_table in referenced + } + + assert named_tables == referenced + assert len(KEYS_TO_PUBLIC_API_TABLES) == 20 + + +def test_every_mapper_configures() -> None: + configure_mappers() + + +def _postgres_sql(statement) -> str: + return str(statement.compile(dialect=postgresql.dialect())) + + +def test_statements_on_a_mapped_class_name_its_table_in_the_public_schema() -> None: + read = select(HouseholdJob).where(HouseholdJob.status == HouseholdJobStatus.PENDING) + claim = ( + update(HouseholdJob) + .where(HouseholdJob.modal_call_id.is_(None)) + .values(modal_call_id="call") + .returning(HouseholdJob.id) + ) + find_or_create = ( + postgresql.insert(Simulation) + .values(id=uuid4()) + .on_conflict_do_nothing( + index_elements=[Simulation.key_version, Simulation.definition_key] + ) + ) + + assert "FROM public.household_jobs" in _postgres_sql(read) + assert "UPDATE public.household_jobs" in _postgres_sql(claim) + assert "ON CONFLICT (key_version, definition_key)" in _postgres_sql(find_or_create) + + +@pytest.mark.postgres +def test_a_session_writes_and_reads_a_household_job_through_its_class( + disposable_database, +) -> None: + """Python and server defaults, the enum and JSON ``NULL`` all round-trip.""" + engine = simulation_engine(disposable_database.superuser_url) + session_factory = create_session_factory(engine) + try: + # Tests may build the models' tables; runtime code never does. + referenced_metadata.create_all(engine) + service_metadata.create_all(engine) + household = referenced_tables.ReferencedHousehold(id=uuid4()) + version = referenced_tables.ReferencedTaxBenefitModelVersion(id=uuid4()) + with session_factory.begin() as session: + session.add_all([household, version]) + # No relationship orders these inserts, so the keys' targets go first. + session.flush() + job = HouseholdJob( + country="us", + request_data={"people": {}}, + status=HouseholdJobStatus.PENDING, + household_id=household.id, + year=2026, + tax_benefit_model_version_id=version.id, + country_package_version="1.0.0", + policyengine_version="2.0.0", + result=None, + ) + session.add(job) + with session_factory() as session: + stored_job = session.get(HouseholdJob, job.id) + result_is_sql_null = session.scalar( + select(HouseholdJob.result.is_(None)).where(HouseholdJob.id == job.id) + ) + finally: + engine.dispose() + + assert isinstance(job.id, UUID) + # Read back at the insert, and still loaded after the session closed. + assert job.created_at is not None + assert stored_job is not None + assert stored_job.status is HouseholdJobStatus.PENDING + assert stored_job.request_data == {"people": {}} + assert stored_job.result is None + assert result_is_sql_null is True + + +def _insert_simulation_run(session_factory) -> UUID: + """Insert a simulation run and the rows its keys name; return its ``id``.""" + household = referenced_tables.ReferencedHousehold(id=uuid4()) + version = referenced_tables.ReferencedTaxBenefitModelVersion(id=uuid4()) + simulation = Simulation( + simulation_type=SimulationType.HOUSEHOLD, + household_id=household.id, + tax_benefit_model_version_id=version.id, + ) + with session_factory.begin() as session: + session.add_all([household, version]) + session.flush() + session.add(simulation) + session.flush() + run = SimulationRun( + simulation_id=simulation.id, + year=2026, + tax_benefit_model_version_id=version.id, + country_package_version="1.0.0", + policyengine_version="2.0.0", + status=SimulationStatus.PENDING, + ) + session.add(run) + return run.id + + +@pytest.mark.postgres +def test_a_written_result_stays_readable_after_its_session_closes( + disposable_database, +) -> None: + """A role with API v1's grants on a result table reads its defaults back.""" + owner_engine = simulation_engine(disposable_database.superuser_url) + writer_engine = simulation_engine(disposable_database.service_url) + try: + # Tests may build the models' tables; runtime code never does. + referenced_metadata.create_all(owner_engine) + service_metadata.create_all(owner_engine) + result_metadata.create_all(owner_engine) + run_id = _insert_simulation_run(create_session_factory(owner_engine)) + with owner_engine.begin() as connection: + connection.execute( + text(f"GRANT SELECT, INSERT ON public.poverty TO {SERVICE_TEST_ROLE}") + ) + result = Poverty(simulation_run_id=run_id, poverty_type="spm", entity="person") + + with create_session_factory(writer_engine).begin() as session: + session.add(result) + finally: + owner_engine.dispose() + writer_engine.dispose() + + assert isinstance(result.id, UUID) + assert result.created_at is not None + + +@pytest.mark.public_api_schema +def test_the_input_classes_are_identified_as_api_v1s_tables_are( + public_api_database_url, +) -> None: + """A mapped class needs a primary key, which must be the table's real one. + + The comparison of the read-only models looks only at their columns, so it + wouldn't notice an identity that isn't unique in API v1's table, + and a session would merge the rows that shared it. + """ + engine = create_engine( + make_url(public_api_database_url).set(drivername="postgresql+psycopg") + ) + try: + with engine.connect() as connection: + real = describe_tables(connection, list(INPUT_CLASSES)) + finally: + engine.dispose() + + for table_name in INPUT_CLASSES: + assert "PRIMARY KEY (id)" in real.tables[table_name].constraints.values(), ( + table_name + ) diff --git a/libs/policyengine-simulation-persistence/tests/test_modal_worker_writes.py b/libs/policyengine-simulation-persistence/tests/test_modal_worker_writes.py index 44de60c0e..6646a54b7 100644 --- a/libs/policyengine-simulation-persistence/tests/test_modal_worker_writes.py +++ b/libs/policyengine-simulation-persistence/tests/test_modal_worker_writes.py @@ -18,20 +18,23 @@ from conftest import ( act_as_the_modal_worker, act_as_the_service, + in_session, rolled_back_transaction, ) from policyengine_simulation_persistence import ( MODAL_WORKER_TABLE_PRIVILEGES, MODAL_WORKER_UPDATABLE_COLUMNS, - budget_window_runs, - household_jobs, + BudgetWindowRun, + HouseholdJob, + Poverty, + ReportRun, + SimulationRun, missing_table_privileges, - report_runs, result_metadata, - simulation_runs, ) from service_rows import ServiceRows, insert_public_api_rows, insert_service_rows from sqlalchemy import Enum, Float, Integer, String, func, insert, select, text, update +from sqlalchemy.orm import Session if TYPE_CHECKING: from sqlalchemy import Connection, Table @@ -49,8 +52,9 @@ def test_the_modal_worker_role_holds_every_privilege_the_modal_calls_need( act_as_the_modal_worker(connection) assert ( - missing_table_privileges( + in_session( connection, + missing_table_privileges, table_privileges=MODAL_WORKER_TABLE_PRIVILEGES, updatable_columns=MODAL_WORKER_UPDATABLE_COLUMNS, ) @@ -71,7 +75,7 @@ def _run_state(service_rows: ServiceRows) -> dict[Table, tuple[Any, dict[str, An "updated_at": NOW, } return { - report_runs: ( + ReportRun.__table__: ( service_rows.report_run_id, { **finished, @@ -85,7 +89,7 @@ def _run_state(service_rows: ServiceRows) -> dict[Table, tuple[Any, dict[str, An "data_version": "test-data-version", }, ), - simulation_runs: ( + SimulationRun.__table__: ( service_rows.baseline_run_id, { **finished, @@ -96,7 +100,7 @@ def _run_state(service_rows: ServiceRows) -> dict[Table, tuple[Any, dict[str, An "household_result": None, }, ), - household_jobs: ( + HouseholdJob.__table__: ( service_rows.household_job_id, { **finished, @@ -105,7 +109,7 @@ def _run_state(service_rows: ServiceRows) -> dict[Table, tuple[Any, dict[str, An "result": None, }, ), - budget_window_runs: ( + BudgetWindowRun.__table__: ( service_rows.budget_window_run_id, { "status": "running", @@ -161,6 +165,29 @@ def _stored_as_sql_null(connection: Connection, table: Table, column_name: str) ).scalar_one() +def test_the_modal_worker_role_inserts_a_result_through_its_class( + public_api_database_url, +) -> None: + """A session's insert reads the row's server defaults back with ``RETURNING``.""" + with rolled_back_transaction(public_api_database_url) as connection: + public_api_rows = insert_public_api_rows(connection) + act_as_the_service(connection) + service_rows = insert_service_rows(connection, public_api_rows) + act_as_the_modal_worker(connection) + result = Poverty( + simulation_run_id=service_rows.baseline_run_id, + poverty_type="spm", + entity="person", + ) + + with Session(connection, join_transaction_mode="create_savepoint") as session: + session.add(result) + session.flush() + created_at = result.created_at + + assert created_at is not None + + def test_the_modal_worker_role_records_its_runs_and_inserts_every_result( public_api_database_url, ) -> None: @@ -185,6 +212,11 @@ def test_the_modal_worker_role_records_its_runs_and_inserts_every_result( select(func.count()).select_from(table) ).scalar_one() assert stored >= 1, table.name - assert _stored_as_sql_null(connection, report_runs, "spm_provenance") == 1 - assert _stored_as_sql_null(connection, simulation_runs, "household_result") == 1 - assert _stored_as_sql_null(connection, household_jobs, "result") == 1 + assert ( + _stored_as_sql_null(connection, ReportRun.__table__, "spm_provenance") == 1 + ) + assert ( + _stored_as_sql_null(connection, SimulationRun.__table__, "household_result") + == 1 + ) + assert _stored_as_sql_null(connection, HouseholdJob.__table__, "result") == 1 diff --git a/libs/policyengine-simulation-persistence/tests/test_policy_reform.py b/libs/policyengine-simulation-persistence/tests/test_policy_reform.py index 153171b51..fb93b3681 100644 --- a/libs/policyengine-simulation-persistence/tests/test_policy_reform.py +++ b/libs/policyengine-simulation-persistence/tests/test_policy_reform.py @@ -11,7 +11,7 @@ from uuid import uuid4 import pytest -from conftest import act_as_the_service, rolled_back_transaction +from conftest import act_as_the_service, in_session, rolled_back_transaction from policyengine_simulation_persistence import ( MissingPolicyParametersError, UnknownPolicyError, @@ -42,8 +42,9 @@ def _reform_of(public_api_database_url, values: list[StoredValue]) -> dict: ) policy_id = insert_policy(connection, catalog, values) act_as_the_service(connection) - return read_policy_reform( + return in_session( connection, + read_policy_reform, country="us", policy_id=policy_id, catalog_version_id=catalog.version_id, @@ -169,8 +170,9 @@ def test_a_policy_is_read_as_its_values_by_parameter_name_and_period( ) act_as_the_service(connection) - reform = read_policy_reform( + reform = in_session( connection, + read_policy_reform, country="us", policy_id=policy_id, catalog_version_id=run_on.version_id, @@ -255,8 +257,9 @@ def test_a_policy_that_changes_nothing_is_read_as_no_values( policy_id = insert_policy(connection, catalog) act_as_the_service(connection) - reform = read_policy_reform( + reform = in_session( connection, + read_policy_reform, country="uk", policy_id=policy_id, catalog_version_id=catalog.version_id, @@ -281,8 +284,9 @@ def test_a_policy_of_another_country_or_none_at_all_is_unknown( for country, unknown_policy_id in (("uk", policy_id), ("us", uuid4())): with pytest.raises(UnknownPolicyError) as raised: - read_policy_reform( + in_session( connection, + read_policy_reform, country=country, policy_id=unknown_policy_id, catalog_version_id=catalog.version_id, @@ -322,8 +326,9 @@ def test_a_policy_changing_parameters_the_run_lacks_names_each_of_them( act_as_the_service(connection) with pytest.raises(MissingPolicyParametersError) as raised: - read_policy_reform( + in_session( connection, + read_policy_reform, country="us", policy_id=policy_id, catalog_version_id=run_on.version_id, diff --git a/libs/policyengine-simulation-persistence/tests/test_result_tables.py b/libs/policyengine-simulation-persistence/tests/test_result_tables.py index 7b6e008f5..8df437bff 100644 --- a/libs/policyengine-simulation-persistence/tests/test_result_tables.py +++ b/libs/policyengine-simulation-persistence/tests/test_result_tables.py @@ -3,10 +3,10 @@ from __future__ import annotations from policyengine_simulation_persistence import ( - report_runs, + ReportRun, + SimulationRun, result_metadata, service_metadata, - simulation_runs, ) from sqlalchemy import Enum @@ -45,14 +45,20 @@ def test_each_result_names_the_service_models_run_it_belongs_to() -> None: for constraint in table.foreign_key_constraints } if table.name in SIMULATION_RUN_RESULT_TABLES: - assert referred_tables == {"simulation_run_id": simulation_runs}, table + assert referred_tables == {"simulation_run_id": SimulationRun.__table__}, ( + table + ) continue - assert referred_tables["report_run_id"] is report_runs, table + assert referred_tables["report_run_id"] is ReportRun.__table__, table if table.name.startswith("labor_supply_"): assert set(referred_tables) == {"report_run_id"}, table else: - assert referred_tables["baseline_simulation_run_id"] is simulation_runs - assert referred_tables["reform_simulation_run_id"] is simulation_runs + assert ( + referred_tables["baseline_simulation_run_id"] is SimulationRun.__table__ + ) + assert ( + referred_tables["reform_simulation_run_id"] is SimulationRun.__table__ + ) def test_every_result_table_and_enum_type_names_the_public_schema() -> None: diff --git a/libs/policyengine-simulation-persistence/tests/test_service_writes.py b/libs/policyengine-simulation-persistence/tests/test_service_writes.py index ec72684a4..8bf76b69e 100644 --- a/libs/policyengine-simulation-persistence/tests/test_service_writes.py +++ b/libs/policyengine-simulation-persistence/tests/test_service_writes.py @@ -13,16 +13,17 @@ from conftest import ( act_as_the_service, connection_as_the_service, + in_session, rolled_back_transaction, ) from policyengine_simulation_persistence import ( - budget_window_runs, - household_jobs, + BudgetWindowRun, + HouseholdJob, + Report, + ReportRun, + Simulation, + SimulationRun, missing_table_privileges, - report_runs, - reports, - simulation_runs, - simulations, ) from service_rows import insert_public_api_rows, insert_service_rows from sqlalchemy import func, select @@ -34,7 +35,7 @@ def test_the_service_role_holds_every_privilege_the_service_needs( public_api_database_url, ) -> None: with connection_as_the_service(public_api_database_url) as connection: - assert missing_table_privileges(connection) == [] + assert in_session(connection, missing_table_privileges) == [] def test_the_service_role_creates_and_reads_a_row_in_every_table_it_writes( @@ -47,10 +48,10 @@ def test_the_service_role_creates_and_reads_a_row_in_every_table_it_writes( compared_runs = connection.execute( select( - report_runs.c.baseline_simulation_run_id, - report_runs.c.reform_simulation_run_id, - report_runs.c.budget_window_run_id, - ).where(report_runs.c.id == service_rows.report_run_id) + ReportRun.baseline_simulation_run_id, + ReportRun.reform_simulation_run_id, + ReportRun.budget_window_run_id, + ).where(ReportRun.id == service_rows.report_run_id) ).one() assert tuple(compared_runs) == ( service_rows.baseline_run_id, @@ -58,12 +59,12 @@ def test_the_service_role_creates_and_reads_a_row_in_every_table_it_writes( service_rows.budget_window_run_id, ) for table in ( - simulations, - reports, - simulation_runs, - budget_window_runs, - report_runs, - household_jobs, + Simulation.__table__, + Report.__table__, + SimulationRun.__table__, + BudgetWindowRun.__table__, + ReportRun.__table__, + HouseholdJob.__table__, ): assert ( connection.execute(select(func.count()).select_from(table)).scalar_one() diff --git a/libs/policyengine-simulation-persistence/tests/test_sessions.py b/libs/policyengine-simulation-persistence/tests/test_sessions.py new file mode 100644 index 000000000..9e9a0c88a --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_sessions.py @@ -0,0 +1,199 @@ +"""The ORM sessions the library's mapped classes are read and written through. + +The tests marked ``postgres`` need a disposable PostgreSQL server (``conftest``). +""" + +from __future__ import annotations + +import pytest +from conftest import simulation_engine +from policyengine_simulation_persistence import ( + bounded_session, + create_session_factory, +) +from sqlalchemy import Integer, String, create_engine, func, text +from sqlalchemy.exc import OperationalError +from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column +from sqlalchemy.orm.exc import DetachedInstanceError + + +class _Base(DeclarativeBase): + """The base of this module's own table.""" + + +class _Written(_Base): + """A row these tests write, in a table of their own.""" + + __tablename__ = "written" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + label: Mapped[str | None] = mapped_column(String) + + +def _statement_timeout(session: Session) -> str: + return session.execute( + text("SELECT current_setting('statement_timeout')") + ).scalar_one() + + +def test_the_session_factory_opens_sessions_on_the_engine_it_is_given() -> None: + engine = create_engine("sqlite://") + try: + session_factory = create_session_factory(engine) + + with session_factory() as session: + assert isinstance(session, Session) + assert session.get_bind() is engine + finally: + engine.dispose() + + +def test_a_committed_object_stays_readable_after_its_session_closes() -> None: + """Nothing is expired on commit, so a closed session's rows can be returned.""" + engine = create_engine("sqlite://") + try: + _Base.metadata.create_all(engine) + session_factory = create_session_factory(engine) + + with session_factory.begin() as session: + row = _Written(id=1) + session.add(row) + + assert row.id == 1 + finally: + engine.dispose() + + +def test_an_attribute_set_to_a_sql_expression_needs_a_refresh_to_stay_readable() -> ( + None +): + """Only the database knows the value, so it must be read before the close.""" + engine = create_engine("sqlite://") + try: + _Base.metadata.create_all(engine) + session_factory = create_session_factory(engine) + + with session_factory.begin() as session: + refreshed_row = _Written(id=1) + unrefreshed_row = _Written(id=2) + session.add_all([refreshed_row, unrefreshed_row]) + session.flush() + refreshed_row.label = func.lower("LABEL") + unrefreshed_row.label = func.lower("LABEL") + session.flush() + session.refresh(refreshed_row) + + assert refreshed_row.label == "label" + with pytest.raises(DetachedInstanceError): + _ = unrefreshed_row.label + finally: + engine.dispose() + + +@pytest.mark.postgres +def test_a_bounded_session_limits_its_own_statements_and_commits( + disposable_database, +) -> None: + """The limit is set for the transaction alone, so a pooled connection keeps none.""" + engine = simulation_engine(disposable_database.superuser_url) + session_factory = create_session_factory(engine) + try: + _Base.metadata.create_all(engine) + with bounded_session( + session_factory, statement_timeout_milliseconds=1234 + ) as session: + limit = _statement_timeout(session) + session.add(_Written(id=1)) + with session_factory() as session: + later_limit = _statement_timeout(session) + stored_row = session.get(_Written, 1) + finally: + engine.dispose() + + assert limit == "1234ms" + assert later_limit == "0" + assert stored_row is not None + + +def _add_then_fail(session_factory) -> None: + with bounded_session( + session_factory, statement_timeout_milliseconds=1000 + ) as session: + session.add(_Written(id=1)) + session.flush() + msg = "block failed" + raise RuntimeError(msg) + + +@pytest.mark.postgres +def test_a_bounded_session_rolls_back_when_its_block_raises( + disposable_database, +) -> None: + engine = simulation_engine(disposable_database.superuser_url) + session_factory = create_session_factory(engine) + try: + _Base.metadata.create_all(engine) + with pytest.raises(RuntimeError, match="block failed"): + _add_then_fail(session_factory) + with session_factory() as session: + stored_row = session.get(_Written, 1) + finally: + engine.dispose() + + assert stored_row is None + + +def _add_without_flushing(session_factory) -> None: + with bounded_session(session_factory, statement_timeout_milliseconds=50) as session: + session.add(_Written(id=1)) + + +@pytest.mark.postgres +def test_a_bounded_session_limits_the_flush_at_its_commit( + disposable_database, +) -> None: + """What the block leaves pending is written under the same limit.""" + engine = simulation_engine(disposable_database.superuser_url) + session_factory = create_session_factory(engine) + try: + _Base.metadata.create_all(engine) + with engine.begin() as connection: + connection.execute( + text( + "CREATE FUNCTION slow_insert() RETURNS trigger " + "LANGUAGE plpgsql AS " + "$$ BEGIN PERFORM pg_sleep(5); RETURN NEW; END $$" + ) + ) + connection.execute( + text( + "CREATE TRIGGER slow_insert BEFORE INSERT ON written " + "FOR EACH ROW EXECUTE FUNCTION slow_insert()" + ) + ) + with pytest.raises(OperationalError, match="statement timeout"): + _add_without_flushing(session_factory) + with session_factory() as session: + stored_row = session.get(_Written, 1) + finally: + engine.dispose() + + assert stored_row is None + + +def _sleep_past_the_limit(session_factory) -> None: + with bounded_session(session_factory, statement_timeout_milliseconds=50) as session: + session.execute(text("SELECT pg_sleep(5)")) + + +@pytest.mark.postgres +def test_a_bounded_session_cancels_a_statement_past_its_limit( + disposable_database, +) -> None: + engine = simulation_engine(disposable_database.superuser_url) + session_factory = create_session_factory(engine) + try: + with pytest.raises(OperationalError, match="statement timeout"): + _sleep_past_the_limit(session_factory) + finally: + engine.dispose() diff --git a/libs/policyengine-simulation-persistence/tests/test_shared_rows.py b/libs/policyengine-simulation-persistence/tests/test_shared_rows.py index 47f225be5..d83038f1c 100644 --- a/libs/policyengine-simulation-persistence/tests/test_shared_rows.py +++ b/libs/policyengine-simulation-persistence/tests/test_shared_rows.py @@ -19,7 +19,7 @@ run_together, wait_until_another_backend_waits_for, ) -from conftest import act_as_the_service, rolled_back_transaction +from conftest import act_as_the_service, in_session, rolled_back_transaction from economy_rows import ( CURRENT_LAW, LATER_VERSIONS, @@ -32,6 +32,7 @@ ReportRunStatus, ReportRunTrigger, RunVersions, + SimulationRun, SimulationStatus, SimulationType, definition_key, @@ -39,7 +40,6 @@ find_or_insert_report_run, find_or_insert_simulation, find_or_insert_simulation_run, - simulation_runs, ) from sqlalchemy import event, insert, text, update @@ -139,12 +139,16 @@ def test_a_simulation_is_inserted_once_and_then_found_by_its_key( act_as_the_service(connection) first_values = _simulation_values(economy, _key("simulation")) - first = find_or_insert_simulation(connection, first_values) - again = find_or_insert_simulation( - connection, _simulation_values(economy, _key("simulation")) + first = in_session(connection, find_or_insert_simulation, first_values) + again = in_session( + connection, + find_or_insert_simulation, + _simulation_values(economy, _key("simulation")), ) - other = find_or_insert_simulation( - connection, _simulation_values(economy, _key("another simulation")) + other = in_session( + connection, + find_or_insert_simulation, + _simulation_values(economy, _key("another simulation")), ) assert (first.id, first.inserted) == (first_values["id"], True) @@ -159,14 +163,18 @@ def test_a_report_is_inserted_once_and_then_found_by_its_key( with rolled_back_transaction(public_api_database_url) as connection: economy = insert_economy_catalog(connection) act_as_the_service(connection) - simulation = find_or_insert_simulation( - connection, _simulation_values(economy, _key("simulation")) + simulation = in_session( + connection, + find_or_insert_simulation, + _simulation_values(economy, _key("simulation")), ) first_values = _report_values(economy, _key("report"), simulation.id) - first = find_or_insert_report(connection, first_values) - again = find_or_insert_report( - connection, _report_values(economy, _key("report"), simulation.id) + first = in_session(connection, find_or_insert_report, first_values) + again = in_session( + connection, + find_or_insert_report, + _report_values(economy, _key("report"), simulation.id), ) assert (first.id, first.inserted) == (first_values["id"], True) @@ -179,24 +187,34 @@ def test_a_simulation_run_is_joined_on_its_version_set_until_it_fails( with rolled_back_transaction(public_api_database_url) as connection: economy = insert_economy_catalog(connection) act_as_the_service(connection) - simulation = find_or_insert_simulation( - connection, _simulation_values(economy, _key("simulation")) + simulation = in_session( + connection, + find_or_insert_simulation, + _simulation_values(economy, _key("simulation")), ) - first = find_or_insert_simulation_run( - connection, _simulation_run_values(economy, simulation.id) + first = in_session( + connection, + find_or_insert_simulation_run, + _simulation_run_values(economy, simulation.id), ) - joined = find_or_insert_simulation_run( - connection, _simulation_run_values(economy, simulation.id) + joined = in_session( + connection, + find_or_insert_simulation_run, + _simulation_run_values(economy, simulation.id), ) - later_versions = find_or_insert_simulation_run( - connection, _simulation_run_values(economy, simulation.id, LATER_VERSIONS) + later_versions = in_session( + connection, + find_or_insert_simulation_run, + _simulation_run_values(economy, simulation.id, LATER_VERSIONS), ) connection.execute(text("RESET ROLE")) _fail(connection, "simulation_runs", first.id) act_as_the_service(connection) - after_failure = find_or_insert_simulation_run( - connection, _simulation_run_values(economy, simulation.id) + after_failure = in_session( + connection, + find_or_insert_simulation_run, + _simulation_run_values(economy, simulation.id), ) assert first.inserted is True @@ -212,18 +230,22 @@ def test_a_forced_run_is_never_joined_and_never_takes_a_runs_place( with rolled_back_transaction(public_api_database_url) as connection: economy = insert_economy_catalog(connection) act_as_the_service(connection) - simulation = find_or_insert_simulation( - connection, _simulation_values(economy, _key("simulation")) + simulation = in_session( + connection, + find_or_insert_simulation, + _simulation_values(economy, _key("simulation")), ) forced_values = _simulation_run_values(economy, simulation.id) - connection.execute(insert(simulation_runs).values(**forced_values, forced=True)) + connection.execute(insert(SimulationRun).values(**forced_values, forced=True)) - first = find_or_insert_simulation_run( - connection, _simulation_run_values(economy, simulation.id) + first = in_session( + connection, + find_or_insert_simulation_run, + _simulation_run_values(economy, simulation.id), ) # Another forced run beside an unforced one conflicts with neither. connection.execute( - insert(simulation_runs).values( + insert(SimulationRun).values( **_simulation_run_values(economy, simulation.id), forced=True ) ) @@ -231,8 +253,9 @@ def test_a_forced_run_is_never_joined_and_never_takes_a_runs_place( assert first.inserted is True assert first.id != forced_values["id"] with pytest.raises(ValueError, match="forced"): - find_or_insert_simulation_run( + in_session( connection, + find_or_insert_simulation_run, {**_simulation_run_values(economy, simulation.id), "forced": True}, ) @@ -243,21 +266,31 @@ def test_a_report_run_is_joined_on_its_versions_outside_budget_windows( with rolled_back_transaction(public_api_database_url) as connection: economy = insert_economy_catalog(connection) act_as_the_service(connection) - simulation = find_or_insert_simulation( - connection, _simulation_values(economy, _key("simulation")) + simulation = in_session( + connection, + find_or_insert_simulation, + _simulation_values(economy, _key("simulation")), ) - simulation_run = find_or_insert_simulation_run( - connection, _simulation_run_values(economy, simulation.id) + simulation_run = in_session( + connection, + find_or_insert_simulation_run, + _simulation_run_values(economy, simulation.id), ) - report = find_or_insert_report( - connection, _report_values(economy, _key("report"), simulation.id) + report = in_session( + connection, + find_or_insert_report, + _report_values(economy, _key("report"), simulation.id), ) - first = find_or_insert_report_run( - connection, _report_run_values(report.id, simulation_run.id) + first = in_session( + connection, + find_or_insert_report_run, + _report_run_values(report.id, simulation_run.id), ) - joined = find_or_insert_report_run( - connection, _report_run_values(report.id, simulation_run.id) + joined = in_session( + connection, + find_or_insert_report_run, + _report_run_values(report.id, simulation_run.id), ) assert first.inserted is True @@ -266,8 +299,9 @@ def test_a_report_run_is_joined_on_its_versions_outside_budget_windows( assert joined.baseline_simulation_run_id == simulation_run.id assert joined.reform_simulation_run_id == simulation_run.id with pytest.raises(ValueError, match="budget window"): - find_or_insert_report_run( + in_session( connection, + find_or_insert_report_run, { **_report_run_values(report.id, simulation_run.id), "budget_window_run_id": uuid4(), @@ -282,8 +316,10 @@ def test_a_transaction_inserting_a_run_another_has_inserted_waits_and_joins_it( economy = committed_catalog with committed_engine.begin() as connection: act_as_the_service(connection) - simulation = find_or_insert_simulation( - connection, _simulation_values(economy, _key("shared simulation")) + simulation = in_session( + connection, + find_or_insert_simulation, + _simulation_values(economy, _key("shared simulation")), ) first_has_inserted = threading.Event() found: dict[str, Any] = {} @@ -291,8 +327,10 @@ def test_a_transaction_inserting_a_run_another_has_inserted_waits_and_joins_it( def insert_then_commit_once_the_other_waits() -> None: with committed_engine.begin() as connection: act_as_the_service(connection) - found["first"] = find_or_insert_simulation_run( - connection, _simulation_run_values(economy, simulation.id) + found["first"] = in_session( + connection, + find_or_insert_simulation_run, + _simulation_run_values(economy, simulation.id), ) holder_pid = connection.execute( text("SELECT pg_backend_pid()") @@ -304,8 +342,10 @@ def insert_once_the_first_has() -> None: assert first_has_inserted.wait(LOCK_WAIT_TIMEOUT_SECONDS) with committed_engine.begin() as connection: act_as_the_service(connection) - found["second"] = find_or_insert_simulation_run( - connection, _simulation_run_values(economy, simulation.id) + found["second"] = in_session( + connection, + find_or_insert_simulation_run, + _simulation_run_values(economy, simulation.id), ) run_together( @@ -323,11 +363,15 @@ def test_a_run_that_fails_between_the_insert_and_the_read_is_inserted_again( economy = committed_catalog with committed_engine.begin() as connection: act_as_the_service(connection) - simulation = find_or_insert_simulation( - connection, _simulation_values(economy, _key("shared simulation")) + simulation = in_session( + connection, + find_or_insert_simulation, + _simulation_values(economy, _key("shared simulation")), ) - earlier = find_or_insert_simulation_run( - connection, _simulation_run_values(economy, simulation.id) + earlier = in_session( + connection, + find_or_insert_simulation_run, + _simulation_run_values(economy, simulation.id), ) def fail_the_earlier_run_before_the_read( @@ -336,10 +380,10 @@ def fail_the_earlier_run_before_the_read( if statement.lstrip().startswith("SELECT") and "simulation_runs" in statement: with committed_engine.begin() as other: other.execute( - update(simulation_runs) + update(SimulationRun) .where( - simulation_runs.c.id == earlier.id, - simulation_runs.c.status != SimulationStatus.FAILED, + SimulationRun.id == earlier.id, + SimulationRun.status != SimulationStatus.FAILED, ) .values(status=SimulationStatus.FAILED, completed_at=text("now()")) ) @@ -350,8 +394,10 @@ def fail_the_earlier_run_before_the_read( connection, "before_cursor_execute", fail_the_earlier_run_before_the_read ) try: - again = find_or_insert_simulation_run( - connection, _simulation_run_values(economy, simulation.id) + again = in_session( + connection, + find_or_insert_simulation_run, + _simulation_run_values(economy, simulation.id), ) finally: event.remove( diff --git a/libs/policyengine-simulation-persistence/tests/test_store_transactions.py b/libs/policyengine-simulation-persistence/tests/test_store_transactions.py new file mode 100644 index 000000000..5a9d84639 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_store_transactions.py @@ -0,0 +1,81 @@ +"""How a store runs a create, telling a lost commit from a refused write.""" + +from __future__ import annotations + +import pytest +from policyengine_simulation_persistence import EconomyRecordOutcomeUnknownError +from policyengine_simulation_persistence.store_transactions import ( + create_in_transaction, +) +from sqlalchemy.exc import OperationalError +from transaction_fakes import CommitFailingTransaction, RecordingSession + +COMMIT_FAILURE = "the commit failed after the rows were inserted" + + +class _OpenTransaction: + """A transaction whose commit succeeds.""" + + def __init__(self, session: RecordingSession) -> None: + self.session = session + + def __enter__(self) -> RecordingSession: + return self.session + + def __exit__(self, error_type, error, traceback) -> None: + return None + + +def test_a_create_returns_what_its_insert_returns() -> None: + session = RecordingSession() + + created = create_in_transaction( + lambda: _OpenTransaction(session), + lambda given_session: given_session is session, + commit_failure=COMMIT_FAILURE, + ) + + assert created is True + + +def test_a_failed_commit_after_the_insert_leaves_the_outcome_unknown() -> None: + session = RecordingSession() + + with pytest.raises( + EconomyRecordOutcomeUnknownError, match=COMMIT_FAILURE + ) as raised: + create_in_transaction( + lambda: CommitFailingTransaction(session), + lambda _session: "created", + commit_failure=COMMIT_FAILURE, + ) + + assert isinstance(raised.value.__cause__, OperationalError) + + +def test_a_database_error_from_the_insert_is_raised_as_it_is() -> None: + refused = OperationalError("INSERT", {}, Exception("connection lost")) + + def refuse(_session: RecordingSession) -> None: + raise refused + + with pytest.raises(OperationalError) as raised: + create_in_transaction( + lambda: _OpenTransaction(RecordingSession()), + refuse, + commit_failure=COMMIT_FAILURE, + ) + + assert raised.value is refused + + +def test_any_other_error_is_never_taken_for_a_lost_commit() -> None: + def refuse(_session: RecordingSession) -> None: + raise LookupError + + with pytest.raises(LookupError): + create_in_transaction( + lambda: _OpenTransaction(RecordingSession()), + refuse, + commit_failure=COMMIT_FAILURE, + ) diff --git a/libs/policyengine-simulation-persistence/tests/test_tables.py b/libs/policyengine-simulation-persistence/tests/test_tables.py index 6c3d7685b..ecac00d78 100644 --- a/libs/policyengine-simulation-persistence/tests/test_tables.py +++ b/libs/policyengine-simulation-persistence/tests/test_tables.py @@ -4,18 +4,18 @@ from policyengine_simulation_persistence import ( SERVICE_TABLE_PRIVILEGES, + BudgetWindowRun, + HouseholdJob, + Report, + ReportRun, + Simulation, + SimulationRun, SimulationStatus, - budget_window_runs, - household_jobs, input_metadata, referenced_metadata, - report_runs, - reports, service_metadata, - simulation_runs, - simulations, ) -from policyengine_simulation_persistence import tables as tables_module +from policyengine_simulation_persistence.orm_base import enum_type from public_api_keys import KEYS_TO_PUBLIC_API_TABLES from sqlalchemy import Enum, ForeignKeyConstraint, Index, Table, UniqueConstraint @@ -103,12 +103,12 @@ def test_the_public_apis_tables_are_modelled_by_their_id_alone() -> None: def test_the_columns_the_public_apis_deletes_search_are_indexed() -> None: """Deleting a user or model version searches these columns for its rows.""" - assert _indexed_columns(reports, "ix_reports_user_id") == ("user_id",) + assert _indexed_columns(Report.__table__, "ix_reports_user_id") == ("user_id",) assert _indexed_columns( - simulation_runs, "ix_simulation_runs_tax_benefit_model_version_id" + SimulationRun.__table__, "ix_simulation_runs_tax_benefit_model_version_id" ) == ("tax_benefit_model_version_id",) assert _indexed_columns( - household_jobs, "ix_household_jobs_tax_benefit_model_version_id" + HouseholdJob.__table__, "ix_household_jobs_tax_benefit_model_version_id" ) == ("tax_benefit_model_version_id",) @@ -136,16 +136,16 @@ def test_every_enum_type_names_the_public_schema() -> None: for column in enum_columns: assert column.type.schema == "public", f"{column.table.name}.{column.name}" # The helper every enum column uses, before any table adopts its type. - helper_type = tables_module._enum_type(SimulationStatus, "v2_example") + helper_type = enum_type(SimulationStatus, "v2_example") assert helper_type.schema == "public" def test_a_report_run_names_its_baseline_and_reform_simulation_runs() -> None: """A report run compares one pair; no table lists a report run's members.""" - assert report_runs.c.baseline_simulation_run_id.nullable is False + assert ReportRun.baseline_simulation_run_id.nullable is False # A household report with only a baseline has no reform run. - assert report_runs.c.reform_simulation_run_id.nullable is True - keys = _foreign_keys(report_runs) + assert ReportRun.reform_simulation_run_id.nullable is True + keys = _foreign_keys(ReportRun.__table__) assert keys["fk_report_runs_baseline_simulation_run_id_simulation_runs"] == ( ("baseline_simulation_run_id",), @@ -164,21 +164,23 @@ def test_a_report_run_names_its_baseline_and_reform_simulation_runs() -> None: def test_a_budget_window_has_one_report_run_per_year_of_one_report() -> None: """A window compares the report's two simulations in each of its years.""" - assert reports.c.budget_window_size.nullable is True - assert simulation_runs.c.year.nullable is False - assert _indexed_columns(simulation_runs, "ix_simulation_runs_current_output")[ - :2 - ] == ("simulation_id", "year") + assert Report.budget_window_size.nullable is True + assert SimulationRun.year.nullable is False + assert _indexed_columns( + SimulationRun.__table__, "ix_simulation_runs_current_output" + )[:2] == ("simulation_id", "year") # The window run must belong to the year run's own report, and owns it. - assert _foreign_keys(report_runs)[ + assert _foreign_keys(ReportRun.__table__)[ "fk_report_runs_budget_window_run_id_budget_window_runs" ] == ( ("budget_window_run_id", "report_id"), "budget_window_runs(id, report_id)", "CASCADE", ) - assert _unique_keys(report_runs)["uq_report_runs_budget_window_run_id_year"] == ( + assert _unique_keys(ReportRun.__table__)[ + "uq_report_runs_budget_window_run_id_year" + ] == ( "budget_window_run_id", "year", ) @@ -192,9 +194,9 @@ def test_a_python_none_in_a_runs_json_result_is_stored_as_sql_null() -> None: write ``None`` for both, and a query for ``IS NULL`` must find them. """ for result_column in ( - simulation_runs.c.household_result, - household_jobs.c.result, - report_runs.c.spm_provenance, + SimulationRun.household_result, + HouseholdJob.result, + ReportRun.spm_provenance, ): assert result_column.type.none_as_null is True, result_column @@ -237,7 +239,7 @@ def _index(table: Table, index_name: str) -> Index: def test_simulations_and_reports_are_unique_on_their_definition_key() -> None: """Find-or-create names this pair as its conflict target (C1 · 25).""" - for table in (simulations, reports): + for table in (Simulation.__table__, Report.__table__): assert table.c.key_version.nullable is True, table.name assert table.c.definition_key.nullable is True, table.name assert table.c.definition_key.type.length == 64, table.name @@ -245,9 +247,9 @@ def test_simulations_and_reports_are_unique_on_their_definition_key() -> None: "key_version", "definition_key", ) - assert reports.c.dataset_independent_key.nullable is True + assert Report.dataset_independent_key.nullable is True # The app finds a report on another dataset by this key alone. - assert _indexed_columns(reports, "ix_reports_dataset_independent_key") == ( + assert _indexed_columns(Report.__table__, "ix_reports_dataset_independent_key") == ( "dataset_independent_key", ) @@ -255,9 +257,11 @@ def test_simulations_and_reports_are_unique_on_their_definition_key() -> None: def test_a_definition_and_version_set_has_one_unfailed_unforced_run() -> None: """A failed or forced run is left out, so it never takes a joinable run's place.""" simulation_run_index = _index( - simulation_runs, "uq_simulation_runs_unfailed_unforced_versions" + SimulationRun.__table__, "uq_simulation_runs_unfailed_unforced_versions" + ) + report_run_index = _index( + ReportRun.__table__, "uq_report_runs_unfailed_unforced_versions" ) - report_run_index = _index(report_runs, "uq_report_runs_unfailed_unforced_versions") assert simulation_run_index.unique is True assert tuple(column.name for column in simulation_run_index.columns) == ( @@ -282,26 +286,26 @@ def test_a_definition_and_version_set_has_one_unfailed_unforced_run() -> None: def test_a_run_is_unforced_unless_its_insert_marks_it() -> None: - for table in (simulation_runs, report_runs): + for table in (SimulationRun.__table__, ReportRun.__table__): assert table.c.forced.nullable is False, table.name assert str(table.c.forced.server_default.arg) == "false", table.name def test_the_per_user_columns_are_nullable() -> None: """Answer 6: the service writes no request's label, description or trigger.""" - assert reports.c.label.nullable is True - assert reports.c.description.nullable is True - assert reports.c.user_id.nullable is True - assert report_runs.c.trigger.nullable is True - assert budget_window_runs.c.trigger.nullable is True + assert Report.label.nullable is True + assert Report.description.nullable is True + assert Report.user_id.nullable is True + assert ReportRun.trigger.nullable is True + assert BudgetWindowRun.trigger.nullable is True def test_a_window_run_is_one_per_report_and_version_set_unless_recalculated() -> None: """Answer 8: a window run on the versions is joined; a forced one is extra.""" - assert budget_window_runs.c.policyengine_version.nullable is True - assert budget_window_runs.c.policyengine_version.type.length == 128 + assert BudgetWindowRun.policyengine_version.nullable is True + assert BudgetWindowRun.policyengine_version.type.length == 128 window_run_index = _index( - budget_window_runs, "uq_budget_window_runs_unfailed_unforced_versions" + BudgetWindowRun.__table__, "uq_budget_window_runs_unfailed_unforced_versions" ) assert window_run_index.unique is True diff --git a/libs/policyengine-simulation-persistence/tests/transaction_fakes.py b/libs/policyengine-simulation-persistence/tests/transaction_fakes.py new file mode 100644 index 000000000..4b36a4c83 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/transaction_fakes.py @@ -0,0 +1,63 @@ +"""Transactions that fail at a chosen point, for the stores' create tests. + +A store's create tells a commit that failed after its rows were flushed, so +the rows may exist, from a flush that failed, so nothing was written. These +fakes stand in for the store's transaction factory to make each failure +happen without a database. +""" + +from __future__ import annotations + +from typing import Self + +from sqlalchemy.exc import OperationalError + + +class RecordingSession: + """A session that keeps what it is given and runs nothing.""" + + def __init__(self) -> None: + self.added: list[object] = [] + self.statements: list[object] = [] + self.flushed = False + + def add(self, instance: object) -> None: + self.added.append(instance) + + def flush(self) -> None: + self.flushed = True + + def execute(self, statement: object) -> None: + self.statements.append(statement) + + +class CommitFailingTransaction: + """A transaction whose block runs, and whose commit then fails.""" + + def __init__(self, session: RecordingSession) -> None: + self.session = session + + def __enter__(self) -> RecordingSession: + return self.session + + def __exit__(self, error_type, error, traceback) -> None: + if error is None: + statement = "COMMIT" + raise OperationalError(statement, {}, Exception("connection lost")) + + +class FlushFailingTransaction: + """A transaction whose flush fails, so nothing is committed.""" + + def __enter__(self) -> Self: + return self + + def __exit__(self, error_type, error, traceback) -> None: + return None + + def add(self, instance: object) -> None: + return None + + def flush(self) -> None: + failed_statement = "INSERT" + raise OperationalError(failed_statement, {}, Exception("connection lost")) diff --git a/libs/policyengine-stage12-persistence/README.md b/libs/policyengine-stage12-persistence/README.md index 8229d40d3..04543a512 100644 --- a/libs/policyengine-stage12-persistence/README.md +++ b/libs/policyengine-stage12-persistence/README.md @@ -7,9 +7,25 @@ package never changes a schema. The models of the Public API's run and result tables are the simulation service library's (`libs/policyengine-simulation-persistence`: `tables.py` and `result_tables.py`), which CI compares with the tables the Public API's -chain builds; this package's stores import them, and its engines come from -that library's `create_unpooled_engine`. Its only models of its own are the -comparison path's tables, in `tables.py`. +chain builds; this package's stores import them, and its sessions come from +that library's `create_unpooled_session_factory`. Its only models of its own are the +comparison path's tables, the mapped classes `ComparisonReport` and +`ComparisonSimulation` in `tables.py`, on `ComparisonBase` in `orm_base.py`. + +Every store runs each operation on one ORM `Session` from the factory it is +given, or that `create_stage12_session_factory` builds from its URL (`engine.py`). +Opening the session sends `SELECT true`, which makes it check out its +connection. A `SessionOpener` wraps only that step, and the v2 Modal runtime's +opener repeats it after an `OperationalError`, so a retry can't repeat anything +the operation wrote: its statements and COMMIT run after the opener returns, +and their failures are raised without a retry. + +The deployment check (`validate_runtime_database`) holds one connection at a +time and repeats a step after a connection failure, up to five attempts, since +each step only reads or rolls back what it wrote. The deployment command gives +it `create_stage12_check_session_factory`, whose one-connection pool lets the +role check, every schema read and the rolled-back writes share one connection, +and ends the process with a failure if the check runs for 15 minutes. - **The Stage 12 comparison tables** (`Stage12PersistenceStore`): temporary state of the comparison path, removed with it. diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/__init__.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/__init__.py index bf45c8159..6e8311eb3 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/__init__.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/__init__.py @@ -8,6 +8,11 @@ YearStarter, YearToStart, ) +from policyengine_stage12_persistence.engine import ( + SessionOpener, + create_stage12_check_session_factory, + create_stage12_session_factory, +) from policyengine_stage12_persistence.household_report_run_store import ( HouseholdReportRunStore, ) @@ -24,11 +29,7 @@ RunWrite, SimulationRunState, ) -from policyengine_stage12_persistence.store import ( - ConnectionOpener, - Stage12PersistenceStore, - create_stage12_engine, -) +from policyengine_stage12_persistence.store import Stage12PersistenceStore from policyengine_stage12_persistence.validation import ( Stage12RuntimeAccessError, validate_runtime_database, @@ -38,7 +39,6 @@ "BUDGET_WINDOW_MAX_RUNNING_YEARS", "ActiveRun", "BudgetWindowStore", - "ConnectionOpener", "DistrictCodes", "HouseholdReportRunStore", "ReportRunResult", @@ -46,6 +46,7 @@ "RunKind", "RunStatus", "RunWrite", + "SessionOpener", "SimulationRunState", "Stage12PersistenceStore", "Stage12RuntimeAccessError", @@ -53,7 +54,8 @@ "YearStartDeferred", "YearStarter", "YearToStart", - "create_stage12_engine", + "create_stage12_check_session_factory", + "create_stage12_session_factory", "report_run_result", "validate_runtime_database", ] diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/budget_window_store.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/budget_window_store.py index 3ff656684..7eddcec67 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/budget_window_store.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/budget_window_store.py @@ -23,36 +23,35 @@ from __future__ import annotations import logging -from contextlib import contextmanager from dataclasses import dataclass from typing import TYPE_CHECKING, Protocol from policyengine_simulation_contract.stage12_execution import ReportRunIds from policyengine_simulation_persistence.tables import ( - budget_window_runs, - report_runs, - simulation_runs, + BudgetWindowRun, + ReportRun, + SimulationRun, ) from sqlalchemy import and_, exists, func, or_, select, update +from policyengine_stage12_persistence.engine import ( + WITHOUT_OBJECT_SYNC, + SessionOpener, + StoreSessions, + open_session_once, +) from policyengine_stage12_persistence.report_run_store import ( ACTIVE_STATUSES, RunStatus, no_active_report_run_names, ) -from policyengine_stage12_persistence.store import ( - ConnectionOpener, - create_stage12_engine, - open_connection_once, -) if TYPE_CHECKING: - from collections.abc import Iterator from datetime import datetime from uuid import UUID - from sqlalchemy import ColumnElement, Engine - from sqlalchemy.engine import Connection + from sqlalchemy import ColumnElement + from sqlalchemy.orm import Session, sessionmaker logger = logging.getLogger(__name__) @@ -173,42 +172,30 @@ def __init__( self, database_url: str | None = None, *, - engine: Engine | None = None, - open_connection: ConnectionOpener = open_connection_once, + session_factory: sessionmaker[Session] | None = None, + open_session: SessionOpener = open_session_once, ) -> None: - """Use ``engine`` if given, else build one from ``database_url``. + """Use ``session_factory`` if given, else build one from ``database_url``. Raises ValueError when neither is given or the URL is invalid. """ - if engine is None: - if database_url is None: - msg = "budget window database URL is required" - raise ValueError(msg) - engine = create_stage12_engine(database_url) - self.engine = engine - self._open_connection = open_connection - - @contextmanager - def _transaction(self, operation_name: str) -> Iterator[Connection]: - # The opener only opens the connection; every statement and the - # COMMIT run after it returns. - connection = self._open_connection( - self.engine.connect, - operation_name=operation_name, + self.sessions = StoreSessions.for_store( + database_url, + session_factory=session_factory, + open_session=open_session, + missing_url_message="budget window database URL is required", ) - with connection, connection.begin(): - yield connection def window_of_year(self, report_run_id: UUID) -> UUID | None: """Return the budget-window run a report run is a year of. None when the report run is no window's year, or does not exist. """ - statement = select(report_runs.c.budget_window_run_id).where( - report_runs.c.id == report_run_id + statement = select(ReportRun.budget_window_run_id).where( + ReportRun.id == report_run_id ) - with self._transaction("window_of_year") as connection: - return connection.execute(statement).scalar_one_or_none() + with self.sessions.transaction("window_of_year") as session: + return session.execute(statement).scalar_one_or_none() def start_window(self, report_run_id: UUID, *, started_at: datetime) -> bool: """Mark the window a claimed year belongs to running; return whether it changed. @@ -218,25 +205,28 @@ def start_window(self, report_run_id: UUID, *, started_at: datetime) -> bool: nothing. """ window_of_run = ( - select(report_runs.c.budget_window_run_id) - .where(report_runs.c.id == report_run_id) + select(ReportRun.budget_window_run_id) + .where(ReportRun.id == report_run_id) .scalar_subquery() ) statement = ( - update(budget_window_runs) + update(BudgetWindowRun) .where( - budget_window_runs.c.id == window_of_run, - budget_window_runs.c.status == RunStatus.PENDING.value, + BudgetWindowRun.id == window_of_run, + BudgetWindowRun.status == RunStatus.PENDING.value, ) .values( status=RunStatus.RUNNING.value, - started_at=func.coalesce(budget_window_runs.c.started_at, started_at), + started_at=func.coalesce(BudgetWindowRun.started_at, started_at), updated_at=started_at, ) - .returning(budget_window_runs.c.id) + .returning(BudgetWindowRun.id) ) - with self._transaction("start_window") as connection: - return connection.execute(statement).scalar_one_or_none() is not None + with self.sessions.transaction("start_window") as session: + changed = session.execute( + statement, execution_options=WITHOUT_OBJECT_SYNC + ).scalar_one_or_none() + return changed is not None def advance_window( self, @@ -259,18 +249,18 @@ def advance_window( deferred with ``YearStartDeferred`` leaves that year and the later ones waiting. Raises LookupError when the window run does not exist. """ - with self._transaction("advance_window") as connection: - window = _lock_window(connection, window_run_id) + with self.sessions.transaction("advance_window") as session: + window = _lock_window(session, window_run_id) if window.finished: return WindowAdvance(status=window.status, changed=False) - years = _window_years(connection, window_run_id) - finished = _finish_ended_window(connection, window_run_id, years, now=now) + years = _window_years(session, window_run_id) + finished = _finish_ended_window(session, window_run_id, years, now=now) if finished is not None: return finished if start_year is None: return WindowAdvance(status=window.status, changed=False) return _start_waiting_years( - connection, + session, window_run_id, years, status=window.status, @@ -287,27 +277,27 @@ def list_idle_windows(self, *, idle_since_before: datetime) -> tuple[UUID, ...]: last year ended, or the simulation service created it, without anything recording what follows. """ - year_of_window = report_runs.c.budget_window_run_id == budget_window_runs.c.id + year_of_window = ReportRun.budget_window_run_id == BudgetWindowRun.id last_year_change = ( - select(func.max(report_runs.c.updated_at)) + select(func.max(ReportRun.updated_at)) .where(year_of_window) .scalar_subquery() ) statement = ( - select(budget_window_runs.c.id) + select(BudgetWindowRun.id) .where( - budget_window_runs.c.status.in_(ACTIVE_STATUSES), + BudgetWindowRun.status.in_(ACTIVE_STATUSES), ~exists().where(year_of_window, _under_way()), func.greatest( - budget_window_runs.c.updated_at, - func.coalesce(last_year_change, budget_window_runs.c.updated_at), + BudgetWindowRun.updated_at, + func.coalesce(last_year_change, BudgetWindowRun.updated_at), ) < idle_since_before, ) - .order_by(budget_window_runs.c.created_at, budget_window_runs.c.id) + .order_by(BudgetWindowRun.created_at, BudgetWindowRun.id) ) - with self._transaction("list_idle_windows") as connection: - return tuple(connection.execute(statement).scalars()) + with self.sessions.transaction("list_idle_windows") as session: + return tuple(session.execute(statement).scalars()) def settle_idle_window( self, @@ -324,21 +314,21 @@ def settle_idle_window( ``idle_since_before`` is left as it is. Raises LookupError when the window run does not exist. """ - with self._transaction("settle_idle_window") as connection: - window = _lock_window(connection, window_run_id) + with self.sessions.transaction("settle_idle_window") as session: + window = _lock_window(session, window_run_id) if window.finished: return WindowAdvance(status=window.status, changed=False) - years = _window_years(connection, window_run_id) + years = _window_years(session, window_run_id) last_change = max([window.updated_at, *(year.updated_at for year in years)]) if any(year.under_way for year in years) or ( last_change >= idle_since_before ): return WindowAdvance(status=window.status, changed=False) - finished = _finish_ended_window(connection, window_run_id, years, now=now) + finished = _finish_ended_window(session, window_run_id, years, now=now) if finished is not None: return finished _fail_window( - connection, + session, window_run_id, WINDOW_STALLED_MESSAGE, now=now, @@ -347,11 +337,11 @@ def settle_idle_window( return WindowAdvance(status=RunStatus.FAILED, changed=True) -def _lock_window(connection: Connection, window_run_id: UUID) -> _LockedWindow: +def _lock_window(session: Session, window_run_id: UUID) -> _LockedWindow: """Lock the window run's row until the transaction ends, and read it.""" - locked = connection.execute( - select(budget_window_runs.c.status, budget_window_runs.c.updated_at) - .where(budget_window_runs.c.id == window_run_id) + locked = session.execute( + select(BudgetWindowRun.status, BudgetWindowRun.updated_at) + .where(BudgetWindowRun.id == window_run_id) .with_for_update() ).one_or_none() if locked is None: @@ -360,20 +350,20 @@ def _lock_window(connection: Connection, window_run_id: UUID) -> _LockedWindow: return _LockedWindow(status=RunStatus(locked.status), updated_at=locked.updated_at) -def _window_years(connection: Connection, window_run_id: UUID) -> list[_WindowYear]: +def _window_years(session: Session, window_run_id: UUID) -> list[_WindowYear]: """Return the window's years, earliest first.""" statement = ( select( - report_runs.c.id, - report_runs.c.baseline_simulation_run_id, - report_runs.c.reform_simulation_run_id, - report_runs.c.year, - report_runs.c.status, - report_runs.c.modal_call_id, - report_runs.c.updated_at, + ReportRun.id, + ReportRun.baseline_simulation_run_id, + ReportRun.reform_simulation_run_id, + ReportRun.year, + ReportRun.status, + ReportRun.modal_call_id, + ReportRun.updated_at, ) - .where(report_runs.c.budget_window_run_id == window_run_id) - .order_by(report_runs.c.year) + .where(ReportRun.budget_window_run_id == window_run_id) + .order_by(ReportRun.year) ) return [ _WindowYear( @@ -387,12 +377,12 @@ def _window_years(connection: Connection, window_run_id: UUID) -> list[_WindowYe modal_call_id=row.modal_call_id, updated_at=row.updated_at, ) - for row in connection.execute(statement) + for row in session.execute(statement) ] def _finish_ended_window( - connection: Connection, + session: Session, window_run_id: UUID, years: list[_WindowYear], *, @@ -407,32 +397,33 @@ def _finish_ended_window( ) if failed_year is not None: _fail_window( - connection, + session, window_run_id, YEAR_FAILED_MESSAGE.format(year=failed_year), now=now, ) return WindowAdvance(status=RunStatus.FAILED, changed=True) if years and all(year.status is RunStatus.SUCCEEDED for year in years): - connection.execute( - update(budget_window_runs) + session.execute( + update(BudgetWindowRun) .where( - budget_window_runs.c.id == window_run_id, - budget_window_runs.c.status.in_(ACTIVE_STATUSES), + BudgetWindowRun.id == window_run_id, + BudgetWindowRun.status.in_(ACTIVE_STATUSES), ) .values( status=RunStatus.SUCCEEDED.value, error_message=None, completed_at=now, updated_at=now, - ) + ), + execution_options=WITHOUT_OBJECT_SYNC, ) return WindowAdvance(status=RunStatus.SUCCEEDED, changed=True) return None def _start_waiting_years( # noqa: PLR0913 # the window's locked state, the starter and the clock - connection: Connection, + session: Session, window_run_id: UUID, years: list[_WindowYear], *, @@ -466,7 +457,7 @@ def _start_waiting_years( # noqa: PLR0913 # the window's locked state, the sta type(error).__name__, ) refused_years = _fail_waiting_years( - connection, + session, window_run_id, CALL_REFUSED_MESSAGE.format(error_type=type(error).__name__), now=now, @@ -476,7 +467,7 @@ def _start_waiting_years( # noqa: PLR0913 # the window's locked state, the sta started.append(year.year) continue _fail_window( - connection, + session, window_run_id, YEAR_FAILED_MESSAGE.format(year=year.year), now=now, @@ -484,17 +475,18 @@ def _start_waiting_years( # noqa: PLR0913 # the window's locked state, the sta return WindowAdvance( status=RunStatus.FAILED, changed=True, started_years=tuple(started) ) - connection.execute( - update(report_runs) - .where(report_runs.c.id == year.report_run_id, _waiting()) - .values(modal_call_id=modal_call_id, updated_at=now) + session.execute( + update(ReportRun) + .where(ReportRun.id == year.report_run_id, _waiting()) + .values(modal_call_id=modal_call_id, updated_at=now), + execution_options=WITHOUT_OBJECT_SYNC, ) started.append(year.year) return WindowAdvance(status=status, changed=False, started_years=tuple(started)) def _fail_window( - connection: Connection, + session: Session, window_run_id: UUID, error_message: str, *, @@ -506,24 +498,25 @@ def _fail_window( The window records ``error_message``, and its waiting years ``waiting_years_message``: by default, that another year failed. """ - connection.execute( - update(budget_window_runs) + session.execute( + update(BudgetWindowRun) .where( - budget_window_runs.c.id == window_run_id, - budget_window_runs.c.status.in_(ACTIVE_STATUSES), + BudgetWindowRun.id == window_run_id, + BudgetWindowRun.status.in_(ACTIVE_STATUSES), ) .values( status=RunStatus.FAILED.value, error_message=error_message, completed_at=now, updated_at=now, - ) + ), + execution_options=WITHOUT_OBJECT_SYNC, ) - _fail_waiting_years(connection, window_run_id, waiting_years_message, now=now) + _fail_waiting_years(session, window_run_id, waiting_years_message, now=now) def _fail_waiting_years( - connection: Connection, + session: Session, window_run_id: UUID, error_message: str, *, @@ -539,11 +532,11 @@ def _fail_waiting_years( report run names it, so a run another report shares is left to that report. """ - conditions = [report_runs.c.budget_window_run_id == window_run_id, _waiting()] + conditions = [ReportRun.budget_window_run_id == window_run_id, _waiting()] if only is not None: - conditions.append(report_runs.c.id == only) - failed_years = connection.execute( - update(report_runs) + conditions.append(ReportRun.id == only) + failed_years = session.execute( + update(ReportRun) .where(*conditions) .values( status=RunStatus.FAILED.value, @@ -552,9 +545,10 @@ def _fail_waiting_years( updated_at=now, ) .returning( - report_runs.c.baseline_simulation_run_id, - report_runs.c.reform_simulation_run_id, - ) + ReportRun.baseline_simulation_run_id, + ReportRun.reform_simulation_run_id, + ), + execution_options=WITHOUT_OBJECT_SYNC, ).all() simulation_run_ids = { run_id @@ -564,20 +558,21 @@ def _fail_waiting_years( } if not simulation_run_ids: return len(failed_years) - connection.execute( - update(simulation_runs) + session.execute( + update(SimulationRun) .where( - simulation_runs.c.id.in_(simulation_run_ids), - simulation_runs.c.status == RunStatus.PENDING.value, - simulation_runs.c.modal_call_id.is_(None), - no_active_report_run_names(simulation_runs), + SimulationRun.id.in_(simulation_run_ids), + SimulationRun.status == RunStatus.PENDING.value, + SimulationRun.modal_call_id.is_(None), + no_active_report_run_names(), ) .values( status=RunStatus.FAILED.value, error_message=error_message, completed_at=now, updated_at=now, - ) + ), + execution_options=WITHOUT_OBJECT_SYNC, ) return len(failed_years) @@ -585,17 +580,17 @@ def _fail_waiting_years( def _waiting() -> ColumnElement[bool]: """Return the condition that a report run is pending with no call recorded.""" return and_( - report_runs.c.status == RunStatus.PENDING.value, - report_runs.c.modal_call_id.is_(None), + ReportRun.status == RunStatus.PENDING.value, + ReportRun.modal_call_id.is_(None), ) def _under_way() -> ColumnElement[bool]: """Return the condition that a report run is running, or started and pending.""" return or_( - report_runs.c.status == RunStatus.RUNNING.value, + ReportRun.status == RunStatus.RUNNING.value, and_( - report_runs.c.status == RunStatus.PENDING.value, - report_runs.c.modal_call_id.is_not(None), + ReportRun.status == RunStatus.PENDING.value, + ReportRun.modal_call_id.is_not(None), ), ) diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/engine.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/engine.py new file mode 100644 index 000000000..b10f898a3 --- /dev/null +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/engine.py @@ -0,0 +1,158 @@ +"""The ORM sessions the Stage 12 stores run their operations on. + +Each operation of a store runs in one ``Session`` of its own. Opening that +session sends one harmless statement, ``CONNECTION_CHECK``, so the session +checks out its connection and any failure to connect happens there, before the +operation has sent anything. A ``SessionOpener`` wraps only that step, so it +may repeat it without any risk of repeating a write. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from typing import TYPE_CHECKING, Any, Final, Protocol + +# Every Stage 12 store connects through the simulation service library's +# unpooled sessions, which check the URL with that library's one URL check. +from policyengine_simulation_persistence.engine import ( + create_single_connection_session_factory, + create_unpooled_session_factory, +) +from sqlalchemy import inspect, select, true + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator, Mapping + + from sqlalchemy.engine.reflection import Inspector + from sqlalchemy.orm import Session, sessionmaker + +CONNECTION_CHECK: Final = select(true()) +# The stores read rows as plain values and keep no mapped object in a session, +# so an UPDATE or DELETE has no object to bring up to date. Without this the +# session would add the primary key to the statement's RETURNING to find some. +WITHOUT_OBJECT_SYNC: Final[Mapping[str, Any]] = {"synchronize_session": False} + + +class SessionOpener(Protocol): + """Open the session that one store operation runs on. + + ``open_session`` only connects: it sends ``CONNECTION_CHECK`` and no + statement of the operation. The operation's statements, and the COMMIT or + ROLLBACK that ends it, run after the opener returns. An opener may therefore + call ``open_session`` again after a failure without risking a duplicate + write, and never sees a later failure. + """ + + def __call__( + self, + open_session: Callable[[], Session], + *, + operation_name: str, + ) -> Session: + """Return the session ``open_session`` opens for ``operation_name``.""" + ... + + +def open_session_once( + open_session: Callable[[], Session], + *, + operation_name: str, # noqa: ARG001 # the SessionOpener protocol passes it by keyword +) -> Session: + """Open a session once, so a refused connection fails the operation.""" + return open_session() + + +def create_stage12_session_factory(database_url: str) -> sessionmaker[Session]: + """Return a factory of sessions that each open their own connection. + + Raises ValueError when the URL is invalid. Nothing connects until a session + runs its first statement. + """ + return create_unpooled_session_factory(database_url) + + +def create_stage12_check_session_factory( + database_url: str, +) -> sessionmaker[Session]: + """Return the session factory of the deployment check. + + Its sessions and schema reads take turns on one connection, so the check + needs one free client slot of the database's pooler. Raises ValueError when + the URL is invalid. Close the connection with ``dispose_session_factory``. + """ + return create_single_connection_session_factory(database_url) + + +def inspect_live_schema(session_factory: sessionmaker[Session]) -> Inspector: + """Return an inspector of the database ``session_factory`` connects to. + + It takes a connection from the factory's engine at once, and again for each + thing it reads. Without a pool, each of those is a new connection. + """ + return inspect(session_factory.kw["bind"]) + + +class StoreSessions: + """Opens the session of each operation of one store.""" + + def __init__( + self, + session_factory: sessionmaker[Session], + open_session: SessionOpener = open_session_once, + ) -> None: + """Open sessions of ``session_factory`` through ``open_session``.""" + self._session_factory = session_factory + self._open_session = open_session + + @classmethod + def for_store( + cls, + database_url: str | None, + *, + session_factory: sessionmaker[Session] | None, + open_session: SessionOpener, + missing_url_message: str, + ) -> StoreSessions: + """Use ``session_factory`` if given, else build one from ``database_url``. + + Raises ValueError with ``missing_url_message`` when neither is given, + and ValueError when the URL is invalid. + """ + if session_factory is None: + if database_url is None: + raise ValueError(missing_url_message) + session_factory = create_stage12_session_factory(database_url) + return cls(session_factory, open_session) + + def _connected_session(self) -> Session: + session = self._session_factory() + try: + session.execute(CONNECTION_CHECK).scalar_one() + except BaseException: + session.close() + raise + return session + + @contextmanager + def reading(self, operation_name: str) -> Iterator[Session]: + """Yield a connected session; closing it ends what it did, uncommitted.""" + # Opening is the only step the opener sees. Every statement of the + # operation, its COMMIT and its ROLLBACK run after the opener returns, + # so an opener that retries can never repeat what the database received. + session = self._open_session( + self._connected_session, + operation_name=operation_name, + ) + with session: + yield session + + @contextmanager + def transaction(self, operation_name: str) -> Iterator[Session]: + """Yield a connected session and commit when the block ends. + + An exception from the block rolls everything back, and so does a + COMMIT that fails; neither is retried. + """ + with self.reading(operation_name) as session: + yield session + session.commit() diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/household_report_run_store.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/household_report_run_store.py index b3515e167..f6d6cd512 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/household_report_run_store.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/household_report_run_store.py @@ -20,10 +20,7 @@ from typing import TYPE_CHECKING, Any -from policyengine_simulation_persistence.tables import ( - report_runs, - simulation_runs, -) +from policyengine_simulation_persistence.tables import ReportRun, SimulationRun from sqlalchemy import select, update from policyengine_stage12_persistence.report_run_store import ( @@ -52,11 +49,11 @@ def household_report_simulation_runs( LookupError when the report run does not exist. """ statement = select( - report_runs.c.baseline_simulation_run_id, - report_runs.c.reform_simulation_run_id, - ).where(report_runs.c.id == report_run_id) - with self._transaction("household_report_simulation_runs") as connection: - named = connection.execute(statement).one_or_none() + ReportRun.baseline_simulation_run_id, + ReportRun.reform_simulation_run_id, + ).where(ReportRun.id == report_run_id) + with self.sessions.transaction("household_report_simulation_runs") as session: + named = session.execute(statement).one_or_none() if named is None: msg = f"report run {report_run_id} does not exist" raise LookupError(msg) @@ -76,10 +73,10 @@ def complete_household_simulation_run( as JSON. """ statement = ( - update(simulation_runs) + update(SimulationRun) .where( - simulation_runs.c.id == simulation_run_id, - running_under_call(simulation_runs, modal_call_id), + SimulationRun.id == simulation_run_id, + running_under_call(SimulationRun, modal_call_id), ) .values( status=RunStatus.SUCCEEDED.value, @@ -90,7 +87,7 @@ def complete_household_simulation_run( ) ) return self._write( - simulation_runs, + SimulationRun, simulation_run_id, statement, "complete_household_simulation_run", @@ -112,10 +109,10 @@ def fail_household_simulation_run( for a failure outside the calculation. """ statement = ( - update(simulation_runs) + update(SimulationRun) .where( - simulation_runs.c.id == simulation_run_id, - running_under_call(simulation_runs, modal_call_id), + SimulationRun.id == simulation_run_id, + running_under_call(SimulationRun, modal_call_id), ) .values( status=RunStatus.FAILED.value, @@ -128,7 +125,7 @@ def fail_household_simulation_run( ) ) return self._write( - simulation_runs, + SimulationRun, simulation_run_id, statement, "fail_household_simulation_run", @@ -147,10 +144,10 @@ def complete_household_report_run( run holds its calculated household. """ statement = ( - update(report_runs) + update(ReportRun) .where( - report_runs.c.id == report_run_id, - running_under_call(report_runs, modal_call_id), + ReportRun.id == report_run_id, + running_under_call(ReportRun, modal_call_id), ) .values( status=RunStatus.SUCCEEDED.value, @@ -160,5 +157,5 @@ def complete_household_report_run( ) ) return self._write( - report_runs, report_run_id, statement, "complete_household_report_run" + ReportRun, report_run_id, statement, "complete_household_report_run" ) diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/orm_base.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/orm_base.py new file mode 100644 index 000000000..632012824 --- /dev/null +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/orm_base.py @@ -0,0 +1,32 @@ +"""The declarative base of the comparison path's mapped classes.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, ClassVar + +from sqlalchemy import MetaData +from sqlalchemy.orm import DeclarativeBase + +if TYPE_CHECKING: + from sqlalchemy import Table + + +class _TableModel: + """Types ``__table__`` as what every class here maps: one ``Table``.""" + + __table__: ClassVar[Table] + + +class ComparisonBase(_TableModel, DeclarativeBase): + """Base of the comparison tables, in the ``public`` schema.""" + + metadata = MetaData( + schema="public", + naming_convention={ + "ix": "ix_%(table_name)s_%(column_0_N_name)s", + "uq": "uq_%(table_name)s_%(column_0_N_name)s", + "ck": "%(constraint_name)s", + "fk": "fk_%(table_name)s_%(column_0_N_name)s_%(referred_table_name)s", + "pk": "pk_%(table_name)s", + }, + ) diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_results.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_results.py index 6b2592362..299b02efe 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_results.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_results.py @@ -20,18 +20,18 @@ from policyengine_simulation_contract.macro_output import SingleYearMacroOutput from policyengine_simulation_persistence.result_tables import ( - budget_summary, - congressional_district_impacts, - constituency_impacts, - decile_impacts, - inequality, - intra_decile_impacts, - labor_supply_decile_responses, - labor_supply_responses, - local_authority_impacts, - poverty, - program_statistics, - withheld_area_codes, + BudgetSummary, + CongressionalDistrictImpact, + ConstituencyImpact, + DecileImpact, + Inequality, + IntraDecileImpact, + LaborSupplyDecileResponse, + LaborSupplyResponse, + LocalAuthorityImpact, + Poverty, + ProgramStatistics, + WithheldAreaCode, ) if TYPE_CHECKING: @@ -50,7 +50,7 @@ RacePovertyOutput, ) from policyengine_simulation_contract.stage12_execution import ReportRunIds - from sqlalchemy import Table + from policyengine_simulation_persistence.orm_base import ResultBase # The household variable each headline budget figure totals, and the column # its figure goes in. ``budgetary_impact`` is tax minus benefits, as the @@ -119,13 +119,13 @@ class DistrictCodes: @dataclass(frozen=True) class ResultRows: - """The rows one result table receives. + """The rows one result table receives, and that table's mapped class. Rows ``shared_by_reports`` belong to a simulation run, which other report runs may share; a row such a run already holds is kept as stored. """ - table: Table + model: type[ResultBase] rows: tuple[dict[str, Any], ...] shared_by_reports: bool @@ -170,64 +170,64 @@ def report_run_result( report_run_columns=_report_run_columns(economy), rows=( ResultRows( - labor_supply_responses, + LaborSupplyResponse, _labor_supply_rows(economy, report_run_key), shared_by_reports=False, ), ResultRows( - labor_supply_decile_responses, + LaborSupplyDecileResponse, _labor_supply_decile_rows(economy, report_run_key), shared_by_reports=False, ), ResultRows( - budget_summary, + BudgetSummary, _budget_rows(economy, report_keys, country), shared_by_reports=False, ), ResultRows( - decile_impacts, + DecileImpact, _decile_rows(economy, report_keys), shared_by_reports=False, ), ResultRows( - intra_decile_impacts, + IntraDecileImpact, _intra_decile_rows(economy, report_keys), shared_by_reports=False, ), ResultRows( - poverty, + Poverty, _poverty_rows(economy, run_ids, country), shared_by_reports=True, ), ResultRows( - inequality, + Inequality, _inequality_rows(economy, run_ids, country), shared_by_reports=True, ), ResultRows( - program_statistics, + ProgramStatistics, _program_rows(economy, report_keys), shared_by_reports=False, ), ResultRows( - congressional_district_impacts, + CongressionalDistrictImpact, _district_rows(economy, report_keys, district_codes), shared_by_reports=False, ), ResultRows( - constituency_impacts, + ConstituencyImpact, _area_rows(economy.constituency_impact, report_keys, "constituency"), shared_by_reports=False, ), ResultRows( - local_authority_impacts, + LocalAuthorityImpact, _area_rows( economy.local_authority_impact, report_keys, "local_authority" ), shared_by_reports=False, ), ResultRows( - withheld_area_codes, + WithheldAreaCode, _withheld_rows(economy, report_keys), shared_by_reports=False, ), diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py index 8dc6548d5..1a4fc2b80 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py @@ -11,34 +11,34 @@ from __future__ import annotations -from contextlib import contextmanager from dataclasses import dataclass from enum import StrEnum -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Final from policyengine_simulation_persistence.tables import ( - budget_window_runs, - household_jobs, - report_runs, - simulation_runs, + BudgetWindowRun, + HouseholdJob, + ReportRun, + SimulationRun, ) from sqlalchemy import and_, case, exists, func, insert, or_, select, true, update from sqlalchemy.dialects.postgresql import insert as postgresql_insert -from policyengine_stage12_persistence.store import ( - ConnectionOpener, - create_stage12_engine, - open_connection_once, +from policyengine_stage12_persistence.engine import ( + WITHOUT_OBJECT_SYNC, + SessionOpener, + StoreSessions, + open_session_once, ) if TYPE_CHECKING: - from collections.abc import Callable, Iterator, Mapping + from collections.abc import Callable, Mapping from datetime import datetime from uuid import UUID from policyengine_simulation_contract.stage12_execution import ReportRunIds - from sqlalchemy import ColumnElement, Engine, Table, Update - from sqlalchemy.engine import Connection + from sqlalchemy import ColumnElement, SQLColumnExpression, Update + from sqlalchemy.orm import Session, sessionmaker from policyengine_stage12_persistence.report_run_results import ReportRunResult @@ -63,17 +63,22 @@ class RunKind(StrEnum): HOUSEHOLD_JOB = "household_job" -RUN_TABLES = { - RunKind.REPORT_RUN: report_runs, - RunKind.SIMULATION_RUN: simulation_runs, - RunKind.HOUSEHOLD_JOB: household_jobs, +type RunModel = type[ReportRun | SimulationRun | HouseholdJob] + +RUN_MODELS: Final[dict[RunKind, RunModel]] = { + RunKind.REPORT_RUN: ReportRun, + RunKind.SIMULATION_RUN: SimulationRun, + RunKind.HOUSEHOLD_JOB: HouseholdJob, } -# What a missing row is called, by table name. -_ROW_NOUNS = { - report_runs.name: "report run", - simulation_runs.name: "simulation run", - household_jobs.name: "household job", +# What a missing row is called. +_ROW_NOUNS: Final[dict[RunModel, str]] = { + ReportRun: "report run", + SimulationRun: "simulation run", + HouseholdJob: "household job", } +# A bulk insert otherwise leaves out each column a row sets to None. Every row +# names the same columns, so the insert stays one statement that sets them NULL. +_EVERY_NAMED_COLUMN: Final = {"render_nulls": True} @dataclass(frozen=True) @@ -134,31 +139,19 @@ def __init__( self, database_url: str | None = None, *, - engine: Engine | None = None, - open_connection: ConnectionOpener = open_connection_once, + session_factory: sessionmaker[Session] | None = None, + open_session: SessionOpener = open_session_once, ) -> None: - """Use ``engine`` if given, else build one from ``database_url``. + """Use ``session_factory`` if given, else build one from ``database_url``. Raises ValueError when neither is given or the URL is invalid. """ - if engine is None: - if database_url is None: - msg = "report run database URL is required" - raise ValueError(msg) - engine = create_stage12_engine(database_url) - self.engine = engine - self._open_connection = open_connection - - @contextmanager - def _transaction(self, operation_name: str) -> Iterator[Connection]: - # As in the Stage 12 store, the opener only opens the connection; every - # statement and the COMMIT run after it returns. - connection = self._open_connection( - self.engine.connect, - operation_name=operation_name, + self.sessions = StoreSessions.for_store( + database_url, + session_factory=session_factory, + open_session=open_session, + missing_url_message="report run database URL is required", ) - with connection, connection.begin(): - yield connection def require_simulation_runs(self, run_ids: ReportRunIds) -> None: """Check the report run names the call's baseline and reform simulation runs. @@ -167,11 +160,11 @@ def require_simulation_runs(self, run_ids: ReportRunIds) -> None: when it names other simulation runs. """ statement = select( - report_runs.c.baseline_simulation_run_id, - report_runs.c.reform_simulation_run_id, - ).where(report_runs.c.id == run_ids.report_run_id) - with self._transaction("require_simulation_runs") as connection: - named = connection.execute(statement).one_or_none() + ReportRun.baseline_simulation_run_id, + ReportRun.reform_simulation_run_id, + ).where(ReportRun.id == run_ids.report_run_id) + with self.sessions.transaction("require_simulation_runs") as session: + named = session.execute(statement).one_or_none() if named is None: msg = f"report run {run_ids.report_run_id} does not exist" raise LookupError(msg) @@ -192,19 +185,19 @@ def get_simulation_run(self, simulation_run_id: UUID) -> SimulationRunState: It only reads the run. Raises LookupError when the run does not exist. """ statement = select( - simulation_runs.c.status, - simulation_runs.c.year, - simulation_runs.c.country_package_version, - simulation_runs.c.policyengine_version, - simulation_runs.c.dataset_id, - simulation_runs.c.region_id, - simulation_runs.c.output_uri, - simulation_runs.c.output_sha256, - simulation_runs.c.modal_call_id, - simulation_runs.c.error_message, - ).where(simulation_runs.c.id == simulation_run_id) - with self._transaction("get_simulation_run") as connection: - stored = connection.execute(statement).one_or_none() + SimulationRun.status, + SimulationRun.year, + SimulationRun.country_package_version, + SimulationRun.policyengine_version, + SimulationRun.dataset_id, + SimulationRun.region_id, + SimulationRun.output_uri, + SimulationRun.output_sha256, + SimulationRun.modal_call_id, + SimulationRun.error_message, + ).where(SimulationRun.id == simulation_run_id) + with self.sessions.transaction("get_simulation_run") as session: + stored = session.execute(statement).one_or_none() if stored is None: msg = f"simulation run {simulation_run_id} does not exist" raise LookupError(msg) @@ -236,23 +229,23 @@ def claim_report_run( call ID. A run recording another call is that call's. """ statement = ( - update(report_runs) + update(ReportRun) .where( - report_runs.c.id == report_run_id, - report_runs.c.status.in_(ACTIVE_STATUSES), + ReportRun.id == report_run_id, + ReportRun.status.in_(ACTIVE_STATUSES), or_( - report_runs.c.modal_call_id.is_(None), - report_runs.c.modal_call_id == modal_call_id, + ReportRun.modal_call_id.is_(None), + ReportRun.modal_call_id == modal_call_id, ), ) .values( status=RunStatus.RUNNING.value, modal_call_id=modal_call_id, - started_at=func.coalesce(report_runs.c.started_at, claimed_at), + started_at=func.coalesce(ReportRun.started_at, claimed_at), updated_at=claimed_at, ) ) - return self._write(report_runs, report_run_id, statement, "claim_report_run") + return self._write(ReportRun, report_run_id, statement, "claim_report_run") def complete_report_run( self, @@ -272,10 +265,10 @@ def complete_report_run( that simulation run, keeps the stored row. """ statement = ( - update(report_runs) + update(ReportRun) .where( - report_runs.c.id == run_ids.report_run_id, - running_under_call(report_runs, modal_call_id), + ReportRun.id == run_ids.report_run_id, + running_under_call(ReportRun, modal_call_id), ) .values( status=RunStatus.SUCCEEDED.value, @@ -286,7 +279,7 @@ def complete_report_run( ) ) return self._write( - report_runs, + ReportRun, run_ids.report_run_id, statement, "complete_report_run", @@ -303,10 +296,10 @@ def fail_report_run( ) -> RunWrite: """Mark the report run failed while ``modal_call_id`` holds it running.""" statement = ( - update(report_runs) + update(ReportRun) .where( - report_runs.c.id == report_run_id, - running_under_call(report_runs, modal_call_id), + ReportRun.id == report_run_id, + running_under_call(ReportRun, modal_call_id), ) .values( status=RunStatus.FAILED.value, @@ -315,7 +308,7 @@ def fail_report_run( updated_at=failed_at, ) ) - return self._write(report_runs, report_run_id, statement, "fail_report_run") + return self._write(ReportRun, report_run_id, statement, "fail_report_run") def claim_simulation_run( self, @@ -333,24 +326,24 @@ def claim_simulation_run( is that call's, which every report run naming the run waits for. """ statement = ( - update(simulation_runs) + update(SimulationRun) .where( - simulation_runs.c.id == simulation_run_id, - simulation_runs.c.status.in_(ACTIVE_STATUSES), + SimulationRun.id == simulation_run_id, + SimulationRun.status.in_(ACTIVE_STATUSES), or_( - simulation_runs.c.modal_call_id.is_(None), - simulation_runs.c.modal_call_id == modal_call_id, + SimulationRun.modal_call_id.is_(None), + SimulationRun.modal_call_id == modal_call_id, ), ) .values( status=RunStatus.RUNNING.value, modal_call_id=modal_call_id, - started_at=func.coalesce(simulation_runs.c.started_at, claimed_at), + started_at=func.coalesce(SimulationRun.started_at, claimed_at), updated_at=claimed_at, ) ) return self._write( - simulation_runs, simulation_run_id, statement, "claim_simulation_run" + SimulationRun, simulation_run_id, statement, "claim_simulation_run" ) def complete_simulation_run( @@ -368,10 +361,10 @@ def complete_simulation_run( run was failed meanwhile records nothing. """ statement = ( - update(simulation_runs) + update(SimulationRun) .where( - simulation_runs.c.id == simulation_run_id, - running_under_call(simulation_runs, modal_call_id), + SimulationRun.id == simulation_run_id, + running_under_call(SimulationRun, modal_call_id), ) .values( status=RunStatus.SUCCEEDED.value, @@ -383,7 +376,7 @@ def complete_simulation_run( ) ) return self._write( - simulation_runs, simulation_run_id, statement, "complete_simulation_run" + SimulationRun, simulation_run_id, statement, "complete_simulation_run" ) def fail_simulation_run( @@ -402,13 +395,13 @@ def fail_simulation_run( fails a run that way, since other report runs may share it. """ held = ( - simulation_runs.c.status.in_(ACTIVE_STATUSES) + SimulationRun.status.in_(ACTIVE_STATUSES) if modal_call_id is None - else running_under_call(simulation_runs, modal_call_id) + else running_under_call(SimulationRun, modal_call_id) ) statement = ( - update(simulation_runs) - .where(simulation_runs.c.id == simulation_run_id, held) + update(SimulationRun) + .where(SimulationRun.id == simulation_run_id, held) .values( status=RunStatus.FAILED.value, error_message=error_message, @@ -417,7 +410,7 @@ def fail_simulation_run( ) ) return self._write( - simulation_runs, simulation_run_id, statement, "fail_simulation_run" + SimulationRun, simulation_run_id, statement, "fail_simulation_run" ) def fail_simulation_run_held_by( @@ -438,11 +431,11 @@ def fail_simulation_run_held_by( waiting for one run fail it once. """ statement = ( - update(simulation_runs) + update(SimulationRun) .where( - simulation_runs.c.id == simulation_run_id, - simulation_runs.c.status.in_(ACTIVE_STATUSES), - simulation_runs.c.modal_call_id == modal_call_id, + SimulationRun.id == simulation_run_id, + SimulationRun.status.in_(ACTIVE_STATUSES), + SimulationRun.modal_call_id == modal_call_id, ) .values( status=RunStatus.FAILED.value, @@ -452,7 +445,7 @@ def fail_simulation_run_held_by( ) ) return self._write( - simulation_runs, + SimulationRun, simulation_run_id, statement, "fail_simulation_run_held_by", @@ -473,24 +466,24 @@ def claim_household_job( call ID. A job recording another call is that call's. """ statement = ( - update(household_jobs) + update(HouseholdJob) .where( - household_jobs.c.id == household_job_id, - household_jobs.c.status.in_(ACTIVE_STATUSES), + HouseholdJob.id == household_job_id, + HouseholdJob.status.in_(ACTIVE_STATUSES), or_( - household_jobs.c.modal_call_id.is_(None), - household_jobs.c.modal_call_id == modal_call_id, + HouseholdJob.modal_call_id.is_(None), + HouseholdJob.modal_call_id == modal_call_id, ), ) .values( status=RunStatus.RUNNING.value, modal_call_id=modal_call_id, - started_at=func.coalesce(household_jobs.c.started_at, claimed_at), + started_at=func.coalesce(HouseholdJob.started_at, claimed_at), updated_at=claimed_at, ) ) return self._write( - household_jobs, household_job_id, statement, "claim_household_job" + HouseholdJob, household_job_id, statement, "claim_household_job" ) def complete_household_job( @@ -507,10 +500,10 @@ def complete_household_job( The job must be running under ``modal_call_id``. """ statement = ( - update(household_jobs) + update(HouseholdJob) .where( - household_jobs.c.id == household_job_id, - running_under_call(household_jobs, modal_call_id), + HouseholdJob.id == household_job_id, + running_under_call(HouseholdJob, modal_call_id), ) .values( status=RunStatus.SUCCEEDED.value, @@ -521,7 +514,7 @@ def complete_household_job( ) ) return self._write( - household_jobs, household_job_id, statement, "complete_household_job" + HouseholdJob, household_job_id, statement, "complete_household_job" ) def fail_household_job( @@ -540,10 +533,10 @@ def fail_household_job( calculation. ``error_message`` is the failure's one-line summary. """ statement = ( - update(household_jobs) + update(HouseholdJob) .where( - household_jobs.c.id == household_job_id, - running_under_call(household_jobs, modal_call_id), + HouseholdJob.id == household_job_id, + running_under_call(HouseholdJob, modal_call_id), ) .values( status=RunStatus.FAILED.value, @@ -554,7 +547,7 @@ def fail_household_job( ) ) return self._write( - household_jobs, household_job_id, statement, "fail_household_job" + HouseholdJob, household_job_id, statement, "fail_household_job" ) def list_active_runs( @@ -585,21 +578,19 @@ def list_active_runs( its call long after the service created it, and nothing else changes a year that is pending with a call. """ - table = RUN_TABLES[kind] + model = RUN_MODELS[kind] active_since = _active_since(kind) conditions = [ - table.c.status.in_(ACTIVE_STATUSES), + model.status.in_(ACTIVE_STATUSES), active_since < active_since_before, _sweep_judges(kind, regardless_of_report_runs=regardless_of_report_runs), ] - columns = [table.c.id, table.c.status, table.c.modal_call_id] + columns = [model.id, model.status, model.modal_call_id] if kind is RunKind.REPORT_RUN: conditions.append(~_waiting_window_year()) - columns.append(table.c.budget_window_run_id) - statement = ( - select(*columns).where(*conditions).order_by(active_since, table.c.id) - ) - with self._transaction("list_active_runs") as connection: + columns.append(ReportRun.budget_window_run_id) + statement = select(*columns).where(*conditions).order_by(active_since, model.id) + with self.sessions.transaction("list_active_runs") as session: return tuple( ActiveRun( kind=kind, @@ -610,7 +601,7 @@ def list_active_runs( row.budget_window_run_id if kind is RunKind.REPORT_RUN else None ), ) - for row in connection.execute(statement) + for row in session.execute(statement) ) def fail_stuck_run( @@ -636,17 +627,17 @@ def fail_stuck_run( simulation run that records a call is failed whatever report runs name it. """ - table = RUN_TABLES[run.kind] + model = RUN_MODELS[run.kind] same_call = ( - table.c.modal_call_id.is_(None) + model.modal_call_id.is_(None) if run.modal_call_id is None - else table.c.modal_call_id == run.modal_call_id + else model.modal_call_id == run.modal_call_id ) statement = ( - update(table) + update(model) .where( - table.c.id == run.run_id, - table.c.status == run.status.value, + model.id == run.run_id, + model.status == run.status.value, same_call, _sweep_judges( run.kind, regardless_of_report_runs=regardless_of_report_runs @@ -659,52 +650,57 @@ def fail_stuck_run( updated_at=failed_at, ) ) - return self._write(table, run.run_id, statement, "fail_stuck_run") + return self._write(model, run.run_id, statement, "fail_stuck_run") def _write( self, - table: Table, + model: RunModel, run_id: UUID, statement: Update, operation_name: str, *, - after_change: Callable[[Connection], None] | None = None, + after_change: Callable[[Session], None] | None = None, ) -> RunWrite: """Run a conditional update; return whether it changed the run, and its status. ``after_change`` runs in the same transaction, only after the update changed the run. Raises LookupError when the run does not exist. """ - with self._transaction(operation_name) as connection: - changed = connection.execute( - statement.returning(table.c.status) + with self.sessions.transaction(operation_name) as session: + changed = session.execute( + statement.returning(model.status), + execution_options=WITHOUT_OBJECT_SYNC, ).scalar_one_or_none() if changed is not None: if after_change is not None: - after_change(connection) + after_change(session) return RunWrite(changed=True, status=RunStatus(changed)) - current = connection.execute( - select(table.c.status).where(table.c.id == run_id) + current = session.execute( + select(model.status).where(model.id == run_id) ).scalar_one_or_none() if current is None: - msg = f"{_ROW_NOUNS[table.name]} {run_id} does not exist" + msg = f"{_ROW_NOUNS[model]} {run_id} does not exist" raise LookupError(msg) return RunWrite(changed=False, status=RunStatus(current)) -def _result_inserts(result: ReportRunResult) -> Callable[[Connection], None]: - """Return the step that inserts ``result``'s rows on an open transaction.""" +def _result_inserts(result: ReportRunResult) -> Callable[[Session], None]: + """Return the step that inserts ``result``'s rows in an open transaction.""" - def insert_rows(connection: Connection) -> None: + def insert_rows(session: Session) -> None: for result_rows in result.rows: if not result_rows.rows: continue statement = ( - postgresql_insert(result_rows.table).on_conflict_do_nothing() + postgresql_insert(result_rows.model).on_conflict_do_nothing() if result_rows.shared_by_reports - else insert(result_rows.table) + else insert(result_rows.model) + ) + session.execute( + statement, + _uniform_rows(result_rows.rows), + execution_options=_EVERY_NAMED_COLUMN, ) - connection.execute(statement, _uniform_rows(result_rows.rows)) return insert_rows @@ -715,37 +711,34 @@ def _uniform_rows(rows: tuple[dict[str, Any], ...]) -> list[dict[str, Any]]: return [dict.fromkeys(columns) | row for row in rows] -def running_under_call(table: Table, modal_call_id: str) -> ColumnElement[bool]: +def running_under_call(model: RunModel, modal_call_id: str) -> ColumnElement[bool]: """Return the condition that a row is running under ``modal_call_id``. A run or household job is completed or failed by a call only while that call holds it running. """ return and_( - table.c.status == RunStatus.RUNNING.value, - table.c.modal_call_id == modal_call_id, + model.status == RunStatus.RUNNING.value, + model.modal_call_id == modal_call_id, ) -def _active_since(kind: RunKind) -> ColumnElement[datetime]: +def _active_since(kind: RunKind) -> SQLColumnExpression[datetime]: """Return when a run of ``kind`` became active, for the sweep's ages. A household job counts from its creation. A run counts from its first claim or, before it, from its creation, except a budget window's year, which counts from its last change: when its call was recorded. """ - table = RUN_TABLES[kind] if kind is RunKind.HOUSEHOLD_JOB: - return table.c.created_at - unclaimed_since = ( - case( - (table.c.budget_window_run_id.is_not(None), table.c.updated_at), - else_=table.c.created_at, - ) - if kind is RunKind.REPORT_RUN - else table.c.created_at + return HouseholdJob.created_at + if kind is RunKind.SIMULATION_RUN: + return func.coalesce(SimulationRun.started_at, SimulationRun.created_at) + unclaimed_since = case( + (ReportRun.budget_window_run_id.is_not(None), ReportRun.updated_at), + else_=ReportRun.created_at, ) - return func.coalesce(table.c.started_at, unclaimed_since) + return func.coalesce(ReportRun.started_at, unclaimed_since) def _waiting_window_year() -> ColumnElement[bool]: @@ -755,11 +748,11 @@ def _waiting_window_year() -> ColumnElement[bool]: run, so an earlier year's coordinator will start it. """ return and_( - report_runs.c.status == RunStatus.PENDING.value, - report_runs.c.modal_call_id.is_(None), + ReportRun.status == RunStatus.PENDING.value, + ReportRun.modal_call_id.is_(None), exists().where( - budget_window_runs.c.id == report_runs.c.budget_window_run_id, - budget_window_runs.c.status.in_(ACTIVE_STATUSES), + BudgetWindowRun.id == ReportRun.budget_window_run_id, + BudgetWindowRun.status.in_(ACTIVE_STATUSES), ), ) @@ -773,23 +766,22 @@ def _sweep_judges( for the age limit, one that records a call needs none. One no call has claimed has no call to outlast: it waits for the report runs naming it. """ - table = RUN_TABLES[kind] if kind is not RunKind.SIMULATION_RUN: return true() if not regardless_of_report_runs: - return no_active_report_run_names(table) - return or_(table.c.modal_call_id.is_not(None), no_active_report_run_names(table)) + return no_active_report_run_names() + return or_(SimulationRun.modal_call_id.is_not(None), no_active_report_run_names()) -def no_active_report_run_names(table: Table) -> ColumnElement[bool]: +def no_active_report_run_names() -> ColumnElement[bool]: """Return the condition that no pending or running report run names the run. A report run names its simulation runs as its baseline and its reform. """ return ~exists().where( or_( - report_runs.c.baseline_simulation_run_id == table.c.id, - report_runs.c.reform_simulation_run_id == table.c.id, + ReportRun.baseline_simulation_run_id == SimulationRun.id, + ReportRun.reform_simulation_run_id == SimulationRun.id, ), - report_runs.c.status.in_(ACTIVE_STATUSES), + ReportRun.status.in_(ACTIVE_STATUSES), ) diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/store.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/store.py index 91caaa169..864d5dd60 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/store.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/store.py @@ -2,9 +2,8 @@ from __future__ import annotations -from contextlib import contextmanager from enum import StrEnum -from typing import TYPE_CHECKING, Any, Protocol +from typing import TYPE_CHECKING, Any from policyengine_simulation_contract.stage12_execution import ( ComparisonReportPersistenceResult, @@ -14,20 +13,22 @@ ComparisonSimulationRecord, ResultComparisonStatus, ) - -# Every Stage 12 store builds its engine with the simulation service library's -# unpooled engine, which checks the URL with that library's one URL check. -from policyengine_simulation_persistence.engine import ( - create_unpooled_engine as create_stage12_engine, -) -from sqlalchemy import Engine, select, update +from sqlalchemy import select, update from sqlalchemy.dialects.postgresql import insert +from policyengine_stage12_persistence.engine import ( + WITHOUT_OBJECT_SYNC, + SessionOpener, + StoreSessions, + open_session_once, +) from policyengine_stage12_persistence.tables import ( + REPORT_ATTRIBUTES, REPORT_IDENTITY_CONSTRAINT, + SIMULATION_ATTRIBUTES, SIMULATION_IDENTITY_CONSTRAINT, - comparison_reports, - comparison_simulations, + ComparisonReport, + ComparisonSimulation, ) from policyengine_stage12_persistence.validators import ( require_aggregation_transition, @@ -45,12 +46,11 @@ ) if TYPE_CHECKING: - from collections.abc import Callable, Iterator from datetime import datetime from uuid import UUID - from sqlalchemy import ColumnElement, RowMapping, Table - from sqlalchemy.engine import Connection + from sqlalchemy import ColumnElement, RowMapping + from sqlalchemy.orm import Session, sessionmaker REPORT_IDENTITY_COLUMNS = ( "environment", @@ -106,34 +106,6 @@ ) -class ConnectionOpener(Protocol): - """Open the database connection that one store operation uses. - - ``connect`` only opens a connection; it sends no application statement. - The operation's statements, and the COMMIT or ROLLBACK that ends it, run - after the opener returns. An opener may therefore retry ``connect`` - without risking a duplicate write, and never sees a later failure. - """ - - def __call__( - self, - connect: Callable[[], Connection], - *, - operation_name: str, - ) -> Connection: - """Return the connection ``connect`` opens for ``operation_name``.""" - ... - - -def open_connection_once( - connect: Callable[[], Connection], - *, - operation_name: str, # noqa: ARG001 # the ConnectionOpener protocol passes it by keyword -) -> Connection: - """Open a connection once, so a refused connection fails the operation.""" - return connect() - - def _database_value(value: object) -> object: if isinstance(value, StrEnum): return value.value @@ -156,9 +128,11 @@ def _simulation(row: RowMapping) -> ComparisonSimulationRecord: def _identity_predicates( - table: Table, values: dict[str, Any], columns: tuple[str, ...] + model: type[ComparisonReport | ComparisonSimulation], + values: dict[str, Any], + columns: tuple[str, ...], ) -> tuple[ColumnElement[bool], ...]: - return tuple(table.c[name] == values[name] for name in columns) + return tuple(getattr(model, name) == values[name] for name in columns) class Stage12PersistenceStore: @@ -168,37 +142,19 @@ def __init__( self, database_url: str | None = None, *, - engine: Engine | None = None, - open_connection: ConnectionOpener = open_connection_once, + session_factory: sessionmaker[Session] | None = None, + open_session: SessionOpener = open_session_once, ) -> None: - """Use ``engine`` if given, else build one from ``database_url``. + """Use ``session_factory`` if given, else build one from ``database_url``. Raises ValueError when neither is given or the URL is invalid. """ - if engine is None: - if database_url is None: - msg = "Stage 12 database URL is required" - raise ValueError(msg) - engine = create_stage12_engine(database_url) - self.engine = engine - self._open_connection = open_connection - - @contextmanager - def _connection(self, operation_name: str) -> Iterator[Connection]: - # Opening the connection is the only step the opener sees. Every - # statement, COMMIT and ROLLBACK runs after the opener returns, so - # an opener that retries can never repeat work the database received. - connection = self._open_connection( - self.engine.connect, - operation_name=operation_name, + self.sessions = StoreSessions.for_store( + database_url, + session_factory=session_factory, + open_session=open_session, + missing_url_message="Stage 12 database URL is required", ) - with connection: - yield connection - - @contextmanager - def _transaction(self, operation_name: str) -> Iterator[Connection]: - with self._connection(operation_name) as connection, connection.begin(): - yield connection def create_or_resolve_report( self, @@ -211,23 +167,23 @@ def create_or_resolve_report( """ values = _record_values(record) statement = ( - insert(comparison_reports) + insert(ComparisonReport) .values(**values) .on_conflict_do_nothing(constraint=REPORT_IDENTITY_CONSTRAINT) - .returning(*comparison_reports.c) + .returning(*REPORT_ATTRIBUTES) ) - with self._transaction("create_or_resolve_report") as connection: - created = connection.execute(statement).mappings().one_or_none() + with self.sessions.transaction("create_or_resolve_report") as session: + created = session.execute(statement).mappings().one_or_none() if created is not None: return ComparisonReportPersistenceResult( record=_report(created), created=True, ) existing = ( - connection.execute( - select(comparison_reports).where( + session.execute( + select(*REPORT_ATTRIBUTES).where( *_identity_predicates( - comparison_reports, + ComparisonReport, values, REPORT_IDENTITY_COLUMNS, ) @@ -247,11 +203,11 @@ def create_or_resolve_report( def get_report(self, evaluation_id: UUID) -> ComparisonReportRecord: """Return the stored report; raise LookupError if there is none.""" - with self._connection("get_report") as connection: + with self.sessions.reading("get_report") as session: row = ( - connection.execute( - select(comparison_reports).where( - comparison_reports.c.evaluation_id == evaluation_id + session.execute( + select(*REPORT_ATTRIBUTES).where( + ComparisonReport.evaluation_id == evaluation_id ) ) .mappings() @@ -268,15 +224,15 @@ def list_simulations( ) -> tuple[ComparisonSimulationRecord, ...]: """Return the report's simulations ordered by role, then execution ID.""" statement = ( - select(comparison_simulations) - .where(comparison_simulations.c.evaluation_id == evaluation_id) + select(*SIMULATION_ATTRIBUTES) + .where(ComparisonSimulation.evaluation_id == evaluation_id) .order_by( - comparison_simulations.c.role, - comparison_simulations.c.simulation_execution_id, + ComparisonSimulation.role, + ComparisonSimulation.simulation_execution_id, ) ) - with self._connection("list_simulations") as connection: - rows = connection.execute(statement).mappings().all() + with self.sessions.reading("list_simulations") as session: + rows = session.execute(statement).mappings().all() return tuple(_simulation(row) for row in rows) def replace_report( @@ -291,11 +247,11 @@ def replace_report( it exactly. """ values = _record_values(record) - with self._transaction("replace_report") as connection: + with self.sessions.transaction("replace_report") as session: row = ( - connection.execute( - select(comparison_reports) - .where(comparison_reports.c.evaluation_id == record.evaluation_id) + session.execute( + select(*REPORT_ATTRIBUTES) + .where(ComparisonReport.evaluation_id == record.evaluation_id) .with_for_update() ) .mappings() @@ -316,11 +272,12 @@ def replace_report( require_successful_report_replay(existing, record) return existing updated = ( - connection.execute( - update(comparison_reports) - .where(comparison_reports.c.evaluation_id == record.evaluation_id) + session.execute( + update(ComparisonReport) + .where(ComparisonReport.evaluation_id == record.evaluation_id) .values(**{name: values[name] for name in REPORT_MUTABLE_COLUMNS}) - .returning(*comparison_reports.c) + .returning(*REPORT_ATTRIBUTES), + execution_options=WITHOUT_OBJECT_SYNC, ) .mappings() .one() @@ -339,11 +296,11 @@ def replace_report_result_comparison( report is missing and ValueError for any other change or disallowed move. """ values = _record_values(record) - with self._transaction("replace_report_result_comparison") as connection: + with self.sessions.transaction("replace_report_result_comparison") as session: row = ( - connection.execute( - select(comparison_reports) - .where(comparison_reports.c.evaluation_id == record.evaluation_id) + session.execute( + select(*REPORT_ATTRIBUTES) + .where(ComparisonReport.evaluation_id == record.evaluation_id) .with_for_update() ) .mappings() @@ -365,13 +322,14 @@ def replace_report_result_comparison( require_result_comparison_unchanged(existing, record) return existing updated = ( - connection.execute( - update(comparison_reports) - .where(comparison_reports.c.evaluation_id == record.evaluation_id) + session.execute( + update(ComparisonReport) + .where(ComparisonReport.evaluation_id == record.evaluation_id) .values( **{name: values[name] for name in REPORT_COMPARISON_COLUMNS} ) - .returning(*comparison_reports.c) + .returning(*REPORT_ATTRIBUTES), + execution_options=WITHOUT_OBJECT_SYNC, ) .mappings() .one() @@ -389,16 +347,16 @@ def create_or_resolve_simulation( """ values = _record_values(record) statement = ( - insert(comparison_simulations) + insert(ComparisonSimulation) .values(**values) .on_conflict_do_nothing(constraint=SIMULATION_IDENTITY_CONSTRAINT) - .returning(*comparison_simulations.c) + .returning(*SIMULATION_ATTRIBUTES) ) - with self._transaction("create_or_resolve_simulation") as connection: + with self.sessions.transaction("create_or_resolve_simulation") as session: parent_row = ( - connection.execute( - select(comparison_reports).where( - comparison_reports.c.evaluation_id == record.evaluation_id + session.execute( + select(*REPORT_ATTRIBUTES).where( + ComparisonReport.evaluation_id == record.evaluation_id ) ) .mappings() @@ -408,17 +366,17 @@ def create_or_resolve_simulation( msg = f"comparison report {record.evaluation_id} does not exist" raise LookupError(msg) require_simulation_parent_matches(_report(parent_row), record) - created = connection.execute(statement).mappings().one_or_none() + created = session.execute(statement).mappings().one_or_none() if created is not None: return ComparisonSimulationPersistenceResult( record=_simulation(created), created=True, ) existing = ( - connection.execute( - select(comparison_simulations).where( + session.execute( + select(*SIMULATION_ATTRIBUTES).where( *_identity_predicates( - comparison_simulations, + ComparisonSimulation, values, SIMULATION_IDENTITY_COLUMNS, ) @@ -445,11 +403,11 @@ def get_simulation( simulation_execution_id: UUID, ) -> ComparisonSimulationRecord: """Return the stored simulation; raise LookupError if there is none.""" - with self._connection("get_simulation") as connection: + with self.sessions.reading("get_simulation") as session: row = ( - connection.execute( - select(comparison_simulations).where( - comparison_simulations.c.simulation_execution_id + session.execute( + select(*SIMULATION_ATTRIBUTES).where( + ComparisonSimulation.simulation_execution_id == simulation_execution_id ) ) @@ -472,12 +430,12 @@ def replace_simulation( succeeded simulation is returned as stored when ``record`` replays it exactly. """ values = _record_values(record) - with self._transaction("replace_simulation") as connection: + with self.sessions.transaction("replace_simulation") as session: row = ( - connection.execute( - select(comparison_simulations) + session.execute( + select(*SIMULATION_ATTRIBUTES) .where( - comparison_simulations.c.simulation_execution_id + ComparisonSimulation.simulation_execution_id == record.simulation_execution_id ) .with_for_update() @@ -498,16 +456,17 @@ def replace_simulation( require_successful_simulation_replay(existing, record) return existing updated = ( - connection.execute( - update(comparison_simulations) + session.execute( + update(ComparisonSimulation) .where( - comparison_simulations.c.simulation_execution_id + ComparisonSimulation.simulation_execution_id == record.simulation_execution_id ) .values( **{name: values[name] for name in SIMULATION_MUTABLE_COLUMNS} ) - .returning(*comparison_simulations.c) + .returning(*SIMULATION_ATTRIBUTES), + execution_options=WITHOUT_OBJECT_SYNC, ) .mappings() .one() @@ -522,12 +481,12 @@ def replace_simulation_if_status( ) -> tuple[ComparisonSimulationRecord, bool]: """Replace a simulation only while it remains in ``expected_status``.""" values = _record_values(record) - with self._transaction("replace_simulation_if_status") as connection: + with self.sessions.transaction("replace_simulation_if_status") as session: row = ( - connection.execute( - select(comparison_simulations) + session.execute( + select(*SIMULATION_ATTRIBUTES) .where( - comparison_simulations.c.simulation_execution_id + ComparisonSimulation.simulation_execution_id == record.simulation_execution_id ) .with_for_update() @@ -547,16 +506,17 @@ def replace_simulation_if_status( return existing, False require_lifecycle_transition(existing.status, record.status) updated = ( - connection.execute( - update(comparison_simulations) + session.execute( + update(ComparisonSimulation) .where( - comparison_simulations.c.simulation_execution_id + ComparisonSimulation.simulation_execution_id == record.simulation_execution_id ) .values( **{name: values[name] for name in SIMULATION_MUTABLE_COLUMNS} ) - .returning(*comparison_simulations.c) + .returning(*SIMULATION_ATTRIBUTES), + execution_options=WITHOUT_OBJECT_SYNC, ) .mappings() .one() @@ -579,25 +539,28 @@ def attach_simulation_invocation( invocation ID is already attached. """ statement = ( - update(comparison_simulations) + update(ComparisonSimulation) .where( - comparison_simulations.c.simulation_execution_id - == simulation_execution_id, - comparison_simulations.c.modal_invocation_id == expected_placeholder, + ComparisonSimulation.simulation_execution_id == simulation_execution_id, + ComparisonSimulation.modal_invocation_id == expected_placeholder, ) .values( modal_invocation_id=modal_invocation_id, updated_at=updated_at, ) - .returning(*comparison_simulations.c) + .returning(*SIMULATION_ATTRIBUTES) ) - with self._transaction("attach_simulation_invocation") as connection: - row = connection.execute(statement).mappings().one_or_none() + with self.sessions.transaction("attach_simulation_invocation") as session: + row = ( + session.execute(statement, execution_options=WITHOUT_OBJECT_SYNC) + .mappings() + .one_or_none() + ) if row is None: row = ( - connection.execute( - select(comparison_simulations).where( - comparison_simulations.c.simulation_execution_id + session.execute( + select(*SIMULATION_ATTRIBUTES).where( + ComparisonSimulation.simulation_execution_id == simulation_execution_id ) ) diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/tables.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/tables.py index d3169fa02..34eacf9c3 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/tables.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/tables.py @@ -1,25 +1,31 @@ -"""Synchronized SQLAlchemy mappings for the temporary Stage 12 tables. +"""Synchronized SQLAlchemy mapped classes for the temporary Stage 12 tables. -``policyengine-api`` owns these tables and every schema migration. These Core -mappings exist only so the temporary simulation runtime can issue typed DML; +``policyengine-api`` owns these tables and every schema migration. These +classes exist only so the temporary simulation runtime can issue typed DML; runtime code must never call ``metadata.create_all`` or otherwise emit DDL. """ from __future__ import annotations +import uuid # noqa: TC003 # SQLAlchemy reads Mapped annotations at run time +from datetime import datetime # noqa: TC003 # SQLAlchemy reads Mapped annotations +from typing import ClassVar + from sqlalchemy import ( JSON, - Column, DateTime, + FetchedValue, ForeignKey, Index, Integer, - MetaData, String, - Table, UniqueConstraint, + inspect, ) from sqlalchemy.dialects.postgresql import ENUM, UUID +from sqlalchemy.orm import Mapped, mapped_column + +from policyengine_stage12_persistence.orm_base import ComparisonBase REPORT_TABLE_NAME = "stage12_evaluation_reports" SIMULATION_TABLE_NAME = "stage12_evaluation_simulations" @@ -71,142 +77,163 @@ create_type=False, ) -metadata = MetaData( - schema="public", - naming_convention={ - "ix": "ix_%(table_name)s_%(column_0_N_name)s", - "uq": "uq_%(table_name)s_%(column_0_N_name)s", - "ck": "%(constraint_name)s", - "fk": "fk_%(table_name)s_%(column_0_N_name)s_%(referred_table_name)s", - "pk": "pk_%(table_name)s", - }, -) +metadata = ComparisonBase.metadata -comparison_reports = Table( - REPORT_TABLE_NAME, - metadata, - Column("evaluation_id", UUID(as_uuid=True), primary_key=True), - Column("contract_version", Integer, nullable=False), - Column("status", run_status_type, nullable=False), - Column("aggregation_status", aggregation_status_type, nullable=False), - Column("environment", String(255), nullable=False), - Column("calculation_flow", String(255), nullable=False), - Column("originating_request_id", String(255), nullable=False), - Column("observability_id", String(36)), - Column("production_identity", String(255), nullable=False), - Column("incumbent_execution_id", String(255)), - Column("worker_version", String(255), nullable=False), - Column("modal_application", String(255), nullable=False), - Column("report_coordinator_callable", String(255), nullable=False), - Column("version_manifest_sha256", String(64), nullable=False), - Column("policyengine_version", String(255), nullable=False), - Column("country_package_name", String(255), nullable=False), - Column("country_package_version", String(255), nullable=False), - Column("country", String(2), nullable=False), - Column("dataset_identity", String(255), nullable=False), - Column("dataset_uri", String(2048), nullable=False), - Column("data_package_name", String(255), nullable=False), - Column("data_package_version", String(255), nullable=False), - Column("data_artifact_revision", String(255), nullable=False), - Column("coordinator_invocation_id", String(255)), - Column("error_code", String(64)), - Column("error_summary", String(512)), - Column("aggregate_output_uri", String(2048)), - Column("aggregate_output_sha256", String(64)), - Column("aggregate_schema_version", Integer), - Column( - "comparison_status", - result_comparison_status_type, - nullable=False, - ), - Column("comparison_output_uri", String(2048)), - Column("comparison_output_sha256", String(64)), - Column("comparison_schema_version", Integer), - Column("comparison_completed_at", DateTime(timezone=True)), - Column("comparison_error_code", String(64)), - Column("comparison_error_summary", String(512)), - Column("created_at", DateTime(timezone=True), nullable=False), - Column("updated_at", DateTime(timezone=True), nullable=False), - Column("started_at", DateTime(timezone=True)), - Column("completed_at", DateTime(timezone=True)), - Column("retention_expires_at", DateTime(timezone=True), nullable=False), - UniqueConstraint( - "environment", - "calculation_flow", - "production_identity", - "worker_version", - "version_manifest_sha256", - name=REPORT_IDENTITY_CONSTRAINT, - ), -) -Index( - "ix_stage12_eval_reports_status_retention", - comparison_reports.c.status, - comparison_reports.c.retention_expires_at, -) -Index( - "ix_stage12_eval_reports_production_identity", - comparison_reports.c.calculation_flow, - comparison_reports.c.production_identity, -) -comparison_simulations = Table( - SIMULATION_TABLE_NAME, - metadata, - Column("simulation_execution_id", UUID(as_uuid=True), primary_key=True), - Column( - "evaluation_id", +class ComparisonReport(ComparisonBase): + """One report's run on the comparison path.""" + + __tablename__ = REPORT_TABLE_NAME + __table_args__: ClassVar = ( + UniqueConstraint( + "environment", + "calculation_flow", + "production_identity", + "worker_version", + "version_manifest_sha256", + name=REPORT_IDENTITY_CONSTRAINT, + ), + Index( + "ix_stage12_eval_reports_status_retention", + "status", + "retention_expires_at", + ), + Index( + "ix_stage12_eval_reports_production_identity", + "calculation_flow", + "production_identity", + ), + ) + + evaluation_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True + ) + contract_version: Mapped[int] = mapped_column(Integer) + status: Mapped[str] = mapped_column(run_status_type) + aggregation_status: Mapped[str] = mapped_column(aggregation_status_type) + environment: Mapped[str] = mapped_column(String(255)) + calculation_flow: Mapped[str] = mapped_column(String(255)) + originating_request_id: Mapped[str] = mapped_column(String(255)) + observability_id: Mapped[str | None] = mapped_column(String(36)) + production_identity: Mapped[str] = mapped_column(String(255)) + incumbent_execution_id: Mapped[str | None] = mapped_column(String(255)) + worker_version: Mapped[str] = mapped_column(String(255)) + modal_application: Mapped[str] = mapped_column(String(255)) + report_coordinator_callable: Mapped[str] = mapped_column(String(255)) + version_manifest_sha256: Mapped[str] = mapped_column(String(64)) + policyengine_version: Mapped[str] = mapped_column(String(255)) + country_package_name: Mapped[str] = mapped_column(String(255)) + country_package_version: Mapped[str] = mapped_column(String(255)) + country: Mapped[str] = mapped_column(String(2)) + dataset_identity: Mapped[str] = mapped_column(String(255)) + dataset_uri: Mapped[str] = mapped_column(String(2048)) + data_package_name: Mapped[str] = mapped_column(String(255)) + data_package_version: Mapped[str] = mapped_column(String(255)) + data_artifact_revision: Mapped[str] = mapped_column(String(255)) + coordinator_invocation_id: Mapped[str | None] = mapped_column(String(255)) + error_code: Mapped[str | None] = mapped_column(String(64)) + error_summary: Mapped[str | None] = mapped_column(String(512)) + aggregate_output_uri: Mapped[str | None] = mapped_column(String(2048)) + aggregate_output_sha256: Mapped[str | None] = mapped_column(String(64)) + aggregate_schema_version: Mapped[int | None] = mapped_column(Integer) + # This and the two timestamps below have defaults in API v1's table, which + # a session leaves to the database when they aren't set. + comparison_status: Mapped[str] = mapped_column( + result_comparison_status_type, server_default=FetchedValue() + ) + comparison_output_uri: Mapped[str | None] = mapped_column(String(2048)) + comparison_output_sha256: Mapped[str | None] = mapped_column(String(64)) + comparison_schema_version: Mapped[int | None] = mapped_column(Integer) + comparison_completed_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True) + ) + comparison_error_code: Mapped[str | None] = mapped_column(String(64)) + comparison_error_summary: Mapped[str | None] = mapped_column(String(512)) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=FetchedValue() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=FetchedValue() + ) + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + retention_expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + + +class ComparisonSimulation(ComparisonBase): + """One simulation's execution within a comparison report.""" + + __tablename__ = SIMULATION_TABLE_NAME + __table_args__: ClassVar = ( + UniqueConstraint( + "evaluation_id", + "role", + "input_sha256", + "worker_version", + "version_manifest_sha256", + name=SIMULATION_IDENTITY_CONSTRAINT, + ), + Index( + "ix_stage12_eval_simulations_parent_status", + "evaluation_id", + "status", + ), + Index( + "ix_stage12_eval_simulations_status_retention", + "status", + "retention_expires_at", + ), + ) + + simulation_execution_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True + ) + evaluation_id: Mapped[uuid.UUID] = mapped_column( UUID(as_uuid=True), ForeignKey( "public.stage12_evaluation_reports.evaluation_id", ondelete="CASCADE", ), - nullable=False, - ), - Column("contract_version", Integer, nullable=False), - Column("role", simulation_role_type, nullable=False), - Column("input_sha256", String(64), nullable=False), - Column("worker_version", String(255), nullable=False), - Column("modal_application", String(255), nullable=False), - Column("simulation_callable", String(255), nullable=False), - Column("version_manifest_sha256", String(64), nullable=False), - Column("modal_invocation_id", String(255)), - Column("status", run_status_type, nullable=False), - Column("error_code", String(64)), - Column("error_summary", String(512)), - Column("output_uri", String(2048)), - Column("output_sha256", String(64)), - Column("output_schema_version", Integer), - Column("row_identity_columns", JSON(none_as_null=True)), - Column("row_count", Integer), - Column("row_identity_sha256", String(64)), - Column("created_at", DateTime(timezone=True), nullable=False), - Column("updated_at", DateTime(timezone=True), nullable=False), - Column("started_at", DateTime(timezone=True)), - Column("completed_at", DateTime(timezone=True)), - Column("retention_expires_at", DateTime(timezone=True), nullable=False), - UniqueConstraint( - "evaluation_id", - "role", - "input_sha256", - "worker_version", - "version_manifest_sha256", - name=SIMULATION_IDENTITY_CONSTRAINT, - ), -) -Index( - "ix_stage12_eval_simulations_parent_status", - comparison_simulations.c.evaluation_id, - comparison_simulations.c.status, -) -Index( - "ix_stage12_eval_simulations_status_retention", - comparison_simulations.c.status, - comparison_simulations.c.retention_expires_at, -) + ) + contract_version: Mapped[int] = mapped_column(Integer) + role: Mapped[str] = mapped_column(simulation_role_type) + input_sha256: Mapped[str] = mapped_column(String(64)) + worker_version: Mapped[str] = mapped_column(String(255)) + modal_application: Mapped[str] = mapped_column(String(255)) + simulation_callable: Mapped[str] = mapped_column(String(255)) + version_manifest_sha256: Mapped[str] = mapped_column(String(64)) + modal_invocation_id: Mapped[str | None] = mapped_column(String(255)) + status: Mapped[str] = mapped_column(run_status_type) + error_code: Mapped[str | None] = mapped_column(String(64)) + error_summary: Mapped[str | None] = mapped_column(String(512)) + output_uri: Mapped[str | None] = mapped_column(String(2048)) + output_sha256: Mapped[str | None] = mapped_column(String(64)) + output_schema_version: Mapped[int | None] = mapped_column(Integer) + row_identity_columns: Mapped[list[str] | None] = mapped_column( + JSON(none_as_null=True) + ) + row_count: Mapped[int | None] = mapped_column(Integer) + row_identity_sha256: Mapped[str | None] = mapped_column(String(64)) + # Both have defaults in API v1's table. + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=FetchedValue() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=FetchedValue() + ) + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + retention_expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) -REPORT_COLUMNS = tuple(comparison_reports.c.keys()) -SIMULATION_COLUMNS = tuple(comparison_simulations.c.keys()) + +REPORT_COLUMNS = tuple(inspect(ComparisonReport).columns.keys()) +SIMULATION_COLUMNS = tuple(inspect(ComparisonSimulation).columns.keys()) +# Each class's mapped attributes, which the store selects and returns: its rows +# stay plain values, so no statement depends on what a session already holds. +REPORT_ATTRIBUTES = tuple(getattr(ComparisonReport, name) for name in REPORT_COLUMNS) +SIMULATION_ATTRIBUTES = tuple( + getattr(ComparisonSimulation, name) for name in SIMULATION_COLUMNS +) REPORT_CHECK_CONSTRAINTS = frozenset( { diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/validation.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/validation.py index 8c96658a1..0c7be63d6 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/validation.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/validation.py @@ -2,15 +2,19 @@ from __future__ import annotations +import time from datetime import UTC, datetime, timedelta +from functools import partial from typing import TYPE_CHECKING from uuid import uuid4 from sqlalchemy import ( JSON, DateTime, + Index, Integer, String, + UniqueConstraint, delete, func, inspect, @@ -18,8 +22,12 @@ update, ) from sqlalchemy.dialects.postgresql import ENUM, UUID, insert -from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.exc import DBAPIError, OperationalError, SQLAlchemyError +from policyengine_stage12_persistence.engine import ( + WITHOUT_OBJECT_SYNC, + inspect_live_schema, +) from policyengine_stage12_persistence.tables import ( REPORT_CHECK_CONSTRAINTS, REPORT_IDENTITY_CONSTRAINT, @@ -27,15 +35,22 @@ SIMULATION_CHECK_CONSTRAINTS, SIMULATION_IDENTITY_CONSTRAINT, SIMULATION_TABLE_NAME, - comparison_reports, - comparison_simulations, + ComparisonReport, + ComparisonSimulation, ) if TYPE_CHECKING: - from sqlalchemy.engine import Connection, Engine + from collections.abc import Callable + + from sqlalchemy.engine.reflection import Inspector + from sqlalchemy.orm import Session, sessionmaker REQUIRED_TABLE_PRIVILEGES = ("SELECT", "INSERT", "UPDATE", "DELETE") FORBIDDEN_TABLE_PRIVILEGES = ("TRUNCATE", "REFERENCES", "TRIGGER") +# The check connects through a session pooler that refuses clients while it is +# full, and may lose a connection, so each step gets several attempts. +STEP_ATTEMPTS = 5 +FIRST_RETRY_DELAY_SECONDS = 1.0 class Stage12RuntimeAccessError(RuntimeError): @@ -80,19 +95,19 @@ def _column_type_matches(expected: object, actual: object) -> bool: def _validate_table_schema( - connection: Connection, + live_schema: Inspector, *, - table: object, + model: type[ComparisonReport | ComparisonSimulation], table_name: str, identity_constraint: str, check_constraints: frozenset[str], ) -> None: - inspector = inspect(connection) + mapper = inspect(model) live_columns = { item["name"]: item - for item in inspector.get_columns(table_name, schema="public") + for item in live_schema.get_columns(table_name, schema="public") } - declared_columns = list(table.c) # type: ignore[attr-defined] + declared_columns = list(mapper.columns) if set(live_columns) != {item.name for item in declared_columns}: msg = f"{table_name} columns do not match the synchronized mapping" raise Stage12RuntimeAccessError(msg) @@ -107,20 +122,21 @@ def _validate_table_schema( column=f"{table_name}.{declared.name}", ) - primary_key = inspector.get_pk_constraint(table_name, schema="public") - declared_primary_key = [column.name for column in table.primary_key] # type: ignore[attr-defined] + primary_key = live_schema.get_pk_constraint(table_name, schema="public") + declared_primary_key = [column.name for column in mapper.primary_key] if primary_key.get("constrained_columns") != declared_primary_key: msg = f"{table_name} has an unexpected primary key" raise Stage12RuntimeAccessError(msg) unique_constraints = { item.get("name"): tuple(item.get("column_names") or ()) - for item in inspector.get_unique_constraints(table_name, schema="public") + for item in live_schema.get_unique_constraints(table_name, schema="public") } declared_unique = next( constraint - for constraint in table.constraints # type: ignore[attr-defined] - if constraint.name == identity_constraint + for constraint in model.__table_args__ + if isinstance(constraint, UniqueConstraint) + and constraint.name == identity_constraint ) if unique_constraints.get(identity_constraint) != tuple( column.name for column in declared_unique.columns @@ -130,7 +146,7 @@ def _validate_table_schema( live_checks = { item.get("name") - for item in inspector.get_check_constraints(table_name, schema="public") + for item in live_schema.get_check_constraints(table_name, schema="public") } if not check_constraints.issubset(live_checks): msg = f"{table_name} is missing required check constraints" @@ -138,11 +154,12 @@ def _validate_table_schema( live_indexes = { item.get("name"): tuple(item.get("column_names") or ()) - for item in inspector.get_indexes(table_name, schema="public") + for item in live_schema.get_indexes(table_name, schema="public") } declared_indexes = { index.name: tuple(column.name for column in index.columns) - for index in table.indexes # type: ignore[attr-defined] + for index in model.__table_args__ + if isinstance(index, Index) } for name, columns in declared_indexes.items(): if live_indexes.get(name) != columns: @@ -150,8 +167,8 @@ def _validate_table_schema( raise Stage12RuntimeAccessError(msg) -def _validate_foreign_key(connection: Connection) -> None: - foreign_keys = inspect(connection).get_foreign_keys( +def _validate_foreign_key(live_schema: Inspector) -> None: + foreign_keys = live_schema.get_foreign_keys( SIMULATION_TABLE_NAME, schema="public", ) @@ -176,21 +193,21 @@ def _validate_foreign_key(connection: Connection) -> None: raise Stage12RuntimeAccessError(msg) -def _has_schema_privilege(connection: Connection, privilege: str) -> bool: +def _has_schema_privilege(session: Session, privilege: str) -> bool: return bool( - connection.execute( + session.execute( select(func.has_schema_privilege("public", privilege)) ).scalar_one() ) def _has_table_privilege( - connection: Connection, + session: Session, table_name: str, privilege: str, ) -> bool: return bool( - connection.execute( + session.execute( select( func.has_table_privilege( f"public.{table_name}", @@ -201,29 +218,29 @@ def _has_table_privilege( ) -def _validate_permissions(connection: Connection, *, expected_role: str) -> None: - current_role = connection.execute(select(func.current_user())).scalar_one() +def _validate_permissions(session: Session, *, expected_role: str) -> None: + current_role = session.execute(select(func.current_user())).scalar_one() if current_role != expected_role: msg = "Stage 12 credential authenticated as an unexpected role" raise Stage12RuntimeAccessError(msg) - if not _has_schema_privilege(connection, "USAGE"): + if not _has_schema_privilege(session, "USAGE"): msg = "runtime role lacks public-schema USAGE" raise Stage12RuntimeAccessError(msg) - if _has_schema_privilege(connection, "CREATE"): + if _has_schema_privilege(session, "CREATE"): msg = "runtime role has public-schema CREATE" raise Stage12RuntimeAccessError(msg) for table_name in (REPORT_TABLE_NAME, SIMULATION_TABLE_NAME): for privilege in REQUIRED_TABLE_PRIVILEGES: - if not _has_table_privilege(connection, table_name, privilege): + if not _has_table_privilege(session, table_name, privilege): msg = f"runtime role lacks {privilege} on {table_name}" raise Stage12RuntimeAccessError(msg) for privilege in FORBIDDEN_TABLE_PRIVILEGES: - if _has_table_privilege(connection, table_name, privilege): + if _has_table_privilege(session, table_name, privilege): msg = f"runtime role unexpectedly has {privilege} on {table_name}" raise Stage12RuntimeAccessError(msg) -def _exercise_runtime_dml(connection: Connection, *, environment: str) -> None: +def _exercise_runtime_dml(session: Session, *, environment: str) -> None: evaluation_id = uuid4() simulation_execution_id = uuid4() now = datetime.now(UTC) @@ -272,27 +289,28 @@ def _exercise_runtime_dml(connection: Connection, *, environment: str) -> None: "completed_at": None, "retention_expires_at": retention_expires_at, } - inserted_report = connection.execute( - insert(comparison_reports) + inserted_report = session.execute( + insert(ComparisonReport) .values(**report_values) - .returning(comparison_reports.c.evaluation_id) + .returning(ComparisonReport.evaluation_id) ).scalar_one() if inserted_report != evaluation_id: msg = "report DML canary insert failed" raise Stage12RuntimeAccessError(msg) - selected_report = connection.execute( - select(comparison_reports.c.evaluation_id).where( - comparison_reports.c.evaluation_id == evaluation_id + selected_report = session.execute( + select(ComparisonReport.evaluation_id).where( + ComparisonReport.evaluation_id == evaluation_id ) ).scalar_one() if selected_report != evaluation_id: msg = "report DML canary select failed" raise Stage12RuntimeAccessError(msg) - updated_report = connection.execute( - update(comparison_reports) - .where(comparison_reports.c.evaluation_id == evaluation_id) + updated_report = session.execute( + update(ComparisonReport) + .where(ComparisonReport.evaluation_id == evaluation_id) .values(updated_at=datetime.now(UTC)) - .returning(comparison_reports.c.evaluation_id) + .returning(ComparisonReport.evaluation_id), + execution_options=WITHOUT_OBJECT_SYNC, ).scalar_one() if updated_report != evaluation_id: msg = "report DML canary update failed" @@ -324,87 +342,141 @@ def _exercise_runtime_dml(connection: Connection, *, environment: str) -> None: "completed_at": None, "retention_expires_at": retention_expires_at, } - inserted_simulation = connection.execute( - insert(comparison_simulations) + inserted_simulation = session.execute( + insert(ComparisonSimulation) .values(**simulation_values) - .returning(comparison_simulations.c.simulation_execution_id) + .returning(ComparisonSimulation.simulation_execution_id) ).scalar_one() if inserted_simulation != simulation_execution_id: msg = "simulation DML canary insert failed" raise Stage12RuntimeAccessError(msg) - selected_simulation = connection.execute( - select(comparison_simulations.c.simulation_execution_id).where( - comparison_simulations.c.simulation_execution_id == simulation_execution_id + selected_simulation = session.execute( + select(ComparisonSimulation.simulation_execution_id).where( + ComparisonSimulation.simulation_execution_id == simulation_execution_id ) ).scalar_one() if selected_simulation != simulation_execution_id: msg = "simulation DML canary select failed" raise Stage12RuntimeAccessError(msg) - updated_simulation = connection.execute( - update(comparison_simulations) - .where( - comparison_simulations.c.simulation_execution_id == simulation_execution_id - ) + updated_simulation = session.execute( + update(ComparisonSimulation) + .where(ComparisonSimulation.simulation_execution_id == simulation_execution_id) .values(updated_at=datetime.now(UTC)) - .returning(comparison_simulations.c.simulation_execution_id) + .returning(ComparisonSimulation.simulation_execution_id), + execution_options=WITHOUT_OBJECT_SYNC, ).scalar_one() if updated_simulation != simulation_execution_id: msg = "simulation DML canary update failed" raise Stage12RuntimeAccessError(msg) - deleted_simulation = connection.execute( - delete(comparison_simulations) - .where( - comparison_simulations.c.simulation_execution_id == simulation_execution_id - ) - .returning(comparison_simulations.c.simulation_execution_id) + deleted_simulation = session.execute( + delete(ComparisonSimulation) + .where(ComparisonSimulation.simulation_execution_id == simulation_execution_id) + .returning(ComparisonSimulation.simulation_execution_id), + execution_options=WITHOUT_OBJECT_SYNC, ).scalar_one() if deleted_simulation != simulation_execution_id: msg = "simulation DML canary delete failed" raise Stage12RuntimeAccessError(msg) - deleted_report = connection.execute( - delete(comparison_reports) - .where(comparison_reports.c.evaluation_id == evaluation_id) - .returning(comparison_reports.c.evaluation_id) + deleted_report = session.execute( + delete(ComparisonReport) + .where(ComparisonReport.evaluation_id == evaluation_id) + .returning(ComparisonReport.evaluation_id), + execution_options=WITHOUT_OBJECT_SYNC, ).scalar_one() if deleted_report != evaluation_id: msg = "report DML canary delete failed" raise Stage12RuntimeAccessError(msg) +def _connection_failed(error: DBAPIError) -> bool: + """Return whether ``error`` is about the connection rather than the check. + + A pooler ending a connection can report it with any error class, such as + ``InternalError``, so SQLAlchemy's finding that the connection is gone counts + as well as ``OperationalError``. + """ + return isinstance(error, OperationalError) or error.connection_invalidated + + +def _retrying_connection_failures[StepResult]( + step: Callable[[], StepResult], +) -> StepResult: + """Run a step of the check, again after each connection failure. + + Every step can be repeated: it only reads, or its writes are rolled back. + The last attempt's error is raised. + """ + delay = FIRST_RETRY_DELAY_SECONDS + for _ in range(STEP_ATTEMPTS - 1): + try: + return step() + except DBAPIError as error: + if not _connection_failed(error): + raise + time.sleep(delay) + delay *= 2 + return step() + + +def _check_role(session_factory: sessionmaker[Session], expected_role: str) -> None: + with session_factory() as session: + _validate_permissions(session, expected_role=expected_role) + + +def _check_schema(live_schema: Inspector) -> None: + """Compare the live tables with the models.""" + _validate_table_schema( + live_schema, + model=ComparisonReport, + table_name=REPORT_TABLE_NAME, + identity_constraint=REPORT_IDENTITY_CONSTRAINT, + check_constraints=REPORT_CHECK_CONSTRAINTS, + ) + _validate_table_schema( + live_schema, + model=ComparisonSimulation, + table_name=SIMULATION_TABLE_NAME, + identity_constraint=SIMULATION_IDENTITY_CONSTRAINT, + check_constraints=SIMULATION_CHECK_CONSTRAINTS, + ) + _validate_foreign_key(live_schema) + + +def _check_writes(session_factory: sessionmaker[Session], environment: str) -> None: + with session_factory() as session: + try: + _exercise_runtime_dml(session, environment=environment) + finally: + session.rollback() + + def validate_runtime_database( - engine: Engine, + session_factory: sessionmaker[Session], *, expected_role: str, environment: str, ) -> None: - """Inspect the live schema and roll back a complete typed DML canary.""" + """Inspect the live schema and roll back a complete typed DML canary. + + It holds one connection at a time, since a session pooler may have one + free, and retries each step a bounded number of times when the database + refuses or loses a connection. With a factory from + ``create_stage12_check_session_factory``, every step shares one connection. + """ if environment not in {"staging", "production"}: msg = "environment must be staging or production" raise Stage12RuntimeAccessError(msg) try: - with engine.connect() as connection: - _validate_permissions(connection, expected_role=expected_role) - _validate_table_schema( - connection, - table=comparison_reports, - table_name=REPORT_TABLE_NAME, - identity_constraint=REPORT_IDENTITY_CONSTRAINT, - check_constraints=REPORT_CHECK_CONSTRAINTS, - ) - _validate_table_schema( - connection, - table=comparison_simulations, - table_name=SIMULATION_TABLE_NAME, - identity_constraint=SIMULATION_IDENTITY_CONSTRAINT, - check_constraints=SIMULATION_CHECK_CONSTRAINTS, - ) - _validate_foreign_key(connection) - connection.rollback() - transaction = connection.begin() - try: - _exercise_runtime_dml(connection, environment=environment) - finally: - transaction.rollback() + _retrying_connection_failures( + partial(_check_role, session_factory, expected_role) + ) + live_schema = _retrying_connection_failures( + partial(inspect_live_schema, session_factory) + ) + _retrying_connection_failures(partial(_check_schema, live_schema)) + _retrying_connection_failures( + partial(_check_writes, session_factory, environment) + ) except Stage12RuntimeAccessError: raise except SQLAlchemyError as error: diff --git a/libs/policyengine-stage12-persistence/tests/conftest.py b/libs/policyengine-stage12-persistence/tests/conftest.py index 75298c0c3..9b77e010c 100644 --- a/libs/policyengine-stage12-persistence/tests/conftest.py +++ b/libs/policyengine-stage12-persistence/tests/conftest.py @@ -6,6 +6,11 @@ from typing import TYPE_CHECKING import pytest +from policyengine_simulation_persistence.engine import ( + create_session_factory, + create_unpooled_session_factory, + dispose_session_factory, +) from report_run_database import ( SEED_URL_ENV, ReportRunSeeder, @@ -13,11 +18,15 @@ SeededReportRun, worker_database_url, ) +from session_database import RUNTIME_ROLE, StatementLog, act_as_role from sqlalchemy import create_engine +from sqlalchemy.pool import NullPool if TYPE_CHECKING: from collections.abc import Iterator + from sqlalchemy.orm import Session, sessionmaker + @pytest.fixture def seeder() -> Iterator[ReportRunSeeder]: @@ -48,3 +57,33 @@ def seeded_household_job(seeder: ReportRunSeeder) -> Iterator[SeededHouseholdJob yield seeded finally: seeder.remove_household_job(seeded) + + +@pytest.fixture +def worker_sessions() -> Iterator[sessionmaker[Session]]: + """Yield a session factory connecting as the Modal worker role.""" + session_factory = create_unpooled_session_factory(worker_database_url()) + try: + yield session_factory + finally: + dispose_session_factory(session_factory) + + +@pytest.fixture +def statements(worker_sessions: sessionmaker[Session]) -> Iterator[StatementLog]: + """Record the worker sessions; fail if a statement bypassed them.""" + log = StatementLog(worker_sessions) + yield log + log.assert_every_statement_ran_on_a_session() + + +@pytest.fixture +def runtime_sessions() -> Iterator[sessionmaker[Session]]: + """Yield a session factory acting as the comparison path's runtime role.""" + worker_database_url() + engine = create_engine(os.environ[SEED_URL_ENV], poolclass=NullPool) + act_as_role(engine, RUNTIME_ROLE) + try: + yield create_session_factory(engine) + finally: + engine.dispose() diff --git a/libs/policyengine-stage12-persistence/tests/session_database.py b/libs/policyengine-stage12-persistence/tests/session_database.py new file mode 100644 index 000000000..d6fec9940 --- /dev/null +++ b/libs/policyengine-stage12-persistence/tests/session_database.py @@ -0,0 +1,159 @@ +"""ORM sessions on the disposable Postgres, and a record of what they send. + +The stores run every statement on a ``Session``. ``StatementLog`` counts what +each session executes and what reaches the database, so a store still using a +Core connection, whose statements no session sees, fails the test using it. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from sqlalchemy import event +from sqlalchemy.exc import OperationalError + +if TYPE_CHECKING: + from collections.abc import Callable, Collection + + from sqlalchemy.orm import ORMExecuteState, Session, sessionmaker + +CONNECTION_REFUSED_MESSAGE = "(EMAXCONNSESSION) max clients reached in session mode" +# The role the comparison path's tables grant to, which a test connection +# takes on with SET ROLE: it has no login in the disposable database. +RUNTIME_ROLE = "policyengine_v2_runtime" + + +class StatementLog: + """What a session factory's sessions executed and sent to the database.""" + + def __init__(self, session_factory: sessionmaker[Session]) -> None: + """Start recording ``session_factory``'s sessions and their engine.""" + self.session_statements = 0 + self.sent: list[str] = [] + self.commits = 0 + engine = session_factory.kw["bind"] + event.listen(session_factory, "do_orm_execute", self._count_session_statement) + event.listen(engine, "before_cursor_execute", self._record_sent) + event.listen(engine, "commit", self._count_commit) + + def _count_session_statement(self, _state: ORMExecuteState) -> None: + self.session_statements += 1 + + def _record_sent( + self, _connection, _cursor, statement, _parameters, _context, _many + ) -> None: + self.sent.append(" ".join(statement.split())) + + def _count_commit(self, _connection) -> None: + self.commits += 1 + + def starting_with(self, prefix: str) -> list[str]: + """Return the statements sent that begin with ``prefix``.""" + return [statement for statement in self.sent if statement.startswith(prefix)] + + def assert_every_statement_ran_on_a_session(self) -> None: + """Fail if anything reached the database without passing through a session.""" + assert self.sent, "the store sent no statement" + # A session statement that could not connect reached no database, so + # sessions may have run more statements than arrived, never fewer. + assert len(self.sent) <= self.session_statements, self.sent + + +class RetryingSessionOpener: + """Open a session, and open another after one refused connection. + + It records each operation's name, each refusal and, given a log, what + reached the database while the session opened. + """ + + def __init__(self, statements: StatementLog | None = None) -> None: + """Report what ``statements``, if given, records during each opening.""" + self.statements = statements + self.operation_names: list[str] = [] + self.refusals: list[OperationalError] = [] + self.sent_while_opening: list[list[str]] = [] + + def __call__( + self, open_session: Callable[[], Session], *, operation_name: str + ) -> Session: + """Return ``open_session``'s session, retrying one refused connection.""" + self.operation_names.append(operation_name) + already_sent = len(self.statements.sent) if self.statements else 0 + try: + session = open_session() + except OperationalError as refusal: + self.refusals.append(refusal) + session = open_session() + if self.statements: + self.sent_while_opening.append(self.statements.sent[already_sent:]) + return session + + +def refuse_connection_numbers( + session_factory: sessionmaker[Session], numbers: Collection[int] +) -> list[Any]: + """Refuse the factory's attempts to connect that have these 1-based numbers. + + Returns a list that gains one entry per attempt to connect. + """ + attempts: list[Any] = [] + + def refuse(dialect, _record, _arguments, _keywords) -> None: + attempts.append(object()) + if len(attempts) in numbers: + raise dialect.loaded_dbapi.OperationalError(CONNECTION_REFUSED_MESSAGE) + + event.listen(session_factory.kw["bind"], "do_connect", refuse) + return attempts + + +def refuse_connections(session_factory: sessionmaker[Session], count: int) -> list[Any]: + """Refuse the factory's first ``count`` connections, as a full pooler does. + + Returns a list that gains one entry per attempt to connect. + """ + return refuse_connection_numbers(session_factory, range(1, count + 1)) + + +def limit_open_connections(session_factory: sessionmaker[Session], limit: int) -> None: + """Refuse a connection while ``limit`` are open, as a full session pooler does.""" + engine = session_factory.kw["bind"] + open_connections: list[Any] = [] + + def refuse_over_the_limit(dialect, _record, _arguments, _keywords) -> None: + if len(open_connections) >= limit: + raise dialect.loaded_dbapi.OperationalError(CONNECTION_REFUSED_MESSAGE) + + def record_opened(dbapi_connection, _connection_record) -> None: + open_connections.append(dbapi_connection) + + def record_closed(dbapi_connection, _connection_record) -> None: + open_connections.remove(dbapi_connection) + + event.listen(engine, "do_connect", refuse_over_the_limit) + event.listen(engine, "connect", record_opened) + event.listen(engine, "close", record_closed) + + +def act_as_role(engine: Any, role: str) -> None: + """Make every connection of ``engine`` act as ``role``.""" + + def set_role(dbapi_connection, _connection_record) -> None: + with dbapi_connection.cursor() as cursor: + cursor.execute(f"SET ROLE {role}") + # A pooled connection outlives its first transaction, and rolling that + # back would undo the role. + dbapi_connection.commit() + + event.listen(engine, "connect", set_role) + + +def record_opened_connections(session_factory: sessionmaker[Session]) -> list[Any]: + """Return a list that gains each real connection the factory's engine opens.""" + opened: list[Any] = [] + + def record_opened(dbapi_connection, _connection_record) -> None: + opened.append(dbapi_connection) + + event.listen(session_factory.kw["bind"], "connect", record_opened) + return opened diff --git a/libs/policyengine-stage12-persistence/tests/test_budget_window_sessions.py b/libs/policyengine-stage12-persistence/tests/test_budget_window_sessions.py new file mode 100644 index 000000000..6196f7f0f --- /dev/null +++ b/libs/policyengine-stage12-persistence/tests/test_budget_window_sessions.py @@ -0,0 +1,237 @@ +"""The budget-window store on ORM sessions, as the Modal worker role.""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from datetime import timedelta +from threading import Barrier, Lock +from typing import TYPE_CHECKING +from uuid import uuid4 + +import pytest +from budget_window_database import BudgetWindowSeeder +from policyengine_stage12_persistence import ( + BudgetWindowStore, + RunStatus, + WindowAdvance, + YearStartDeferred, +) +from report_run_database import NOW + +if TYPE_CHECKING: + from collections.abc import Iterator + + from budget_window_database import SeededWindow + from policyengine_stage12_persistence import YearToStart + from report_run_database import ReportRunSeeder + from session_database import StatementLog + from sqlalchemy.orm import Session, sessionmaker + +LATER = NOW + timedelta(minutes=5) +IDLE_CUTOFF = NOW + timedelta(minutes=1) + + +class CountingStarter: + """Start each year by returning a new call ID, recording which years started.""" + + def __init__(self) -> None: + self.started: list[int] = [] + self._lock = Lock() + + def __call__(self, year_to_start: YearToStart) -> str: + with self._lock: + self.started.append(year_to_start.year) + return f"fc-started-{year_to_start.year}-{len(self.started)}" + + +@pytest.fixture +def windows(seeder: ReportRunSeeder) -> BudgetWindowSeeder: + return BudgetWindowSeeder(seeder) + + +@pytest.fixture +def window(windows: BudgetWindowSeeder) -> Iterator[SeededWindow]: + seeded = windows.seed_window(first_year=2026, size=7) + try: + yield seeded + finally: + windows.remove(seeded) + + +@pytest.fixture +def store( + worker_sessions: sessionmaker[Session], statements: StatementLog +) -> BudgetWindowStore: + assert statements.sent == [] + return BudgetWindowStore(session_factory=worker_sessions) + + +def test_a_year_names_its_window(store, window) -> None: + year = window.year(2026) + + assert store.window_of_year(year.report_run_id) == window.window_run_id + assert store.window_of_year(uuid4()) is None + + +def test_the_first_claimed_year_starts_its_window_once(store, windows, window) -> None: + report_run_id = window.year(2026).report_run_id + + first = store.start_window(report_run_id, started_at=NOW) + second = store.start_window(report_run_id, started_at=LATER) + + assert (first, second) == (True, False) + stored = windows.window_row(window) + assert stored["status"] == "running" + assert stored["started_at"] == NOW + + +def test_an_advance_starts_the_earliest_waiting_years( + store, statements, windows, window +) -> None: + starter = CountingStarter() + + advance = store.advance_window(window.window_run_id, now=LATER, start_year=starter) + + assert advance == WindowAdvance( + status=RunStatus.PENDING, + changed=False, + started_years=(2026, 2027, 2028, 2029, 2030), + ) + assert starter.started == [2026, 2027, 2028, 2029, 2030] + years = windows.year_rows(window) + assert years[2026]["modal_call_id"].startswith("fc-started-2026") + assert years[2031]["modal_call_id"] is None + assert statements.commits == 1 + [lock] = [sent for sent in statements.sent if sent.endswith("FOR UPDATE")] + assert "public.budget_window_runs" in lock + + +def test_concurrent_advances_start_each_year_once( + worker_sessions, statements, windows, window +) -> None: + assert statements.sent == [] + advancers = 4 + start_together = Barrier(advancers) + starter = CountingStarter() + + def advance(_advancer: int) -> WindowAdvance: + store = BudgetWindowStore(session_factory=worker_sessions) + start_together.wait(timeout=10) + return store.advance_window(window.window_run_id, now=LATER, start_year=starter) + + with ThreadPoolExecutor(max_workers=advancers) as executor: + list(executor.map(advance, range(advancers))) + + assert sorted(starter.started) == [2026, 2027, 2028, 2029, 2030] + started = [ + year + for year, row in windows.year_rows(window).items() + if row["modal_call_id"] is not None + ] + assert sorted(started) == [2026, 2027, 2028, 2029, 2030] + + +def test_a_refused_start_fails_the_year_and_the_window(store, windows, window) -> None: + def refuse(_year_to_start: YearToStart) -> str: + message = "Modal refused the call" + raise RuntimeError(message) + + advance = store.advance_window(window.window_run_id, now=LATER, start_year=refuse) + + assert advance == WindowAdvance(status=RunStatus.FAILED, changed=True) + assert windows.window_row(window)["status"] == "failed" + years = windows.year_rows(window) + assert years[2026]["error_message"].startswith("modal_call_refused: ") + assert years[2027]["error_message"].startswith("budget_window_failed: ") + assert windows.simulation_run_statuses(window.year(2027)) == ("failed", "failed") + + +def test_a_deferred_start_leaves_the_years_waiting(store, windows, window) -> None: + def defer(_year_to_start: YearToStart) -> str: + raise YearStartDeferred + + advance = store.advance_window(window.window_run_id, now=LATER, start_year=defer) + + assert advance == WindowAdvance(status=RunStatus.PENDING, changed=False) + assert windows.year_rows(window)[2026]["status"] == "pending" + + +def test_every_year_succeeded_marks_the_window_succeeded( + store, windows, window +) -> None: + windows.set_years( + window, + range(2026, 2033), + status="succeeded", + modal_call_id="fc-finished", + started_at=NOW, + completed_at=NOW, + ) + + first = store.advance_window(window.window_run_id, now=LATER) + second = store.advance_window(window.window_run_id, now=LATER) + + assert first == WindowAdvance(status=RunStatus.SUCCEEDED, changed=True) + assert second == WindowAdvance(status=RunStatus.SUCCEEDED, changed=False) + assert windows.window_row(window)["completed_at"] == LATER + + +def test_a_failed_year_fails_the_window_with_its_waiting_years( + store, windows, window +) -> None: + windows.set_years( + window, + [2027], + status="failed", + modal_call_id="fc-failed", + error_message="it broke", + started_at=NOW, + completed_at=NOW, + ) + + advance = store.advance_window(window.window_run_id, now=LATER) + + assert advance == WindowAdvance(status=RunStatus.FAILED, changed=True) + stored = windows.window_row(window) + assert stored["error_message"].startswith("budget_window_year_failed: ") + assert "2027" in stored["error_message"] + assert windows.year_rows(window)[2026]["status"] == "failed" + + +def test_a_missing_window_cannot_be_advanced(store) -> None: + with pytest.raises(LookupError, match="does not exist"): + store.advance_window(uuid4(), now=LATER) + + +def test_an_idle_window_is_listed_and_settled_as_stalled( + store, windows, window +) -> None: + listed = store.list_idle_windows(idle_since_before=IDLE_CUTOFF) + + settled = store.settle_idle_window( + window.window_run_id, idle_since_before=IDLE_CUTOFF, now=LATER + ) + repeated = store.settle_idle_window( + window.window_run_id, idle_since_before=IDLE_CUTOFF, now=LATER + ) + + assert window.window_run_id in listed + assert settled == WindowAdvance(status=RunStatus.FAILED, changed=True) + assert repeated == WindowAdvance(status=RunStatus.FAILED, changed=False) + assert windows.window_row(window)["error_message"].startswith( + "budget_window_stalled: " + ) + + +def test_a_window_with_a_year_under_way_is_not_idle(store, windows, window) -> None: + windows.set_years( + window, [2026], status="running", modal_call_id="fc-running", started_at=NOW + ) + + listed = store.list_idle_windows(idle_since_before=IDLE_CUTOFF) + settled = store.settle_idle_window( + window.window_run_id, idle_since_before=IDLE_CUTOFF, now=LATER + ) + + assert window.window_run_id not in listed + assert settled == WindowAdvance(status=RunStatus.PENDING, changed=False) diff --git a/libs/policyengine-stage12-persistence/tests/test_budget_window_store.py b/libs/policyengine-stage12-persistence/tests/test_budget_window_store.py index 4c3f2cad4..d1137fd1e 100644 --- a/libs/policyengine-stage12-persistence/tests/test_budget_window_store.py +++ b/libs/policyengine-stage12-persistence/tests/test_budget_window_store.py @@ -23,6 +23,7 @@ import pytest from budget_window_database import BudgetWindowSeeder, SeededWindow from policyengine_simulation_contract.stage12_execution import ReportRunIds +from policyengine_simulation_persistence.engine import create_session_factory from policyengine_stage12_persistence import ( BUDGET_WINDOW_MAX_RUNNING_YEARS, ActiveRun, @@ -81,12 +82,8 @@ def __call__(self, year_to_start: YearToStart) -> str: @pytest.fixture -def store() -> Iterator[BudgetWindowStore]: - window_store = BudgetWindowStore(worker_database_url()) - try: - yield window_store - finally: - window_store.engine.dispose() +def store() -> BudgetWindowStore: + return BudgetWindowStore(worker_database_url()) @pytest.fixture @@ -327,7 +324,8 @@ def test_a_refused_start_names_its_cause_on_its_own_simulations( def test_the_service_role_starts_and_fails_a_window(windows, window) -> None: # The simulation service starts a window's first years with the same # store, as its own role, and fails the window when Modal refuses one. - service_store = BudgetWindowStore(engine=_engine_as_the_service(windows)) + engine = _engine_as_the_service(windows) + service_store = BudgetWindowStore(session_factory=create_session_factory(engine)) try: starter = RecordingStarter(refused=_run_ids(window, [2028])) @@ -335,7 +333,7 @@ def test_the_service_role_starts_and_fails_a_window(windows, window) -> None: window.window_run_id, now=LATER, start_year=starter ) finally: - service_store.engine.dispose() + engine.dispose() assert advance.started_years == (2026, 2027) assert windows.window_row(window)["status"] == "failed" @@ -599,12 +597,8 @@ def test_a_window_whose_year_changed_recently_is_not_idle( @pytest.fixture -def run_store() -> Iterator[ReportRunStore]: - report_run_store = ReportRunStore(worker_database_url()) - try: - yield report_run_store - finally: - report_run_store.engine.dispose() +def run_store() -> ReportRunStore: + return ReportRunStore(worker_database_url()) def _listed_report_runs(run_store: ReportRunStore) -> dict[UUID, ActiveRun]: diff --git a/libs/policyengine-stage12-persistence/tests/test_comparison_store_sessions.py b/libs/policyengine-stage12-persistence/tests/test_comparison_store_sessions.py new file mode 100644 index 000000000..9a81c4a60 --- /dev/null +++ b/libs/policyengine-stage12-persistence/tests/test_comparison_store_sessions.py @@ -0,0 +1,766 @@ +"""The comparison store and the deployment check on ORM sessions. + +They act as the comparison path's runtime role, which API v1's +manifest grants on the two temporary tables. +""" + +from __future__ import annotations + +import os +import time +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING, Any, Self +from uuid import UUID, uuid4 + +import pytest +from policyengine_simulation_contract.stage12_execution import ( + ComparisonReportRecord, + ComparisonRunAggregationStatus, + ComparisonRunLifecycleStatus, + ComparisonSimulationRecord, + ResultComparisonStatus, + SimulationRole, +) +from policyengine_simulation_persistence import engine as persistence_engine +from policyengine_simulation_persistence.engine import ( + create_session_factory, + dispose_session_factory, +) +from policyengine_stage12_persistence import ( + Stage12PersistenceStore, + Stage12RuntimeAccessError, + create_stage12_check_session_factory, + validate_runtime_database, + validation, +) +from policyengine_stage12_persistence.tables import ( + REPORT_CHECK_CONSTRAINTS, + REPORT_IDENTITY_CONSTRAINT, + REPORT_TABLE_NAME, + ComparisonReport, +) +from policyengine_stage12_persistence.validation import ( + _validate_foreign_key, + _validate_table_schema, +) +from report_run_database import SEED_URL_ENV, worker_database_url +from session_database import ( + RUNTIME_ROLE, + StatementLog, + act_as_role, + limit_open_connections, + record_opened_connections, + refuse_connection_numbers, +) +from sqlalchemy import delete, event, func, inspect, make_url, select, true +from sqlalchemy.exc import InternalError, OperationalError, ProgrammingError +from sqlalchemy.orm import Session + +if TYPE_CHECKING: + from collections.abc import Iterator + + from report_run_database import ReportRunSeeder + from sqlalchemy.orm import sessionmaker + +NOW = datetime(2026, 9, 22, tzinfo=UTC) +LATER = NOW + timedelta(minutes=5) +PLACEHOLDER_CALL = "dispatch-pending-1" +DIGEST = "a" * 64 +# The connections a passing deployment check opens without a pool, one at a +# time: its role check, the schema inspector, eleven schema reads and its +# rolled-back writes. +CHECK_CONNECTIONS = 14 + + +def _report(evaluation_id: UUID) -> ComparisonReportRecord: + return ComparisonReportRecord( + evaluation_id=evaluation_id, + status=ComparisonRunLifecycleStatus.PENDING, + aggregation_status=ComparisonRunAggregationStatus.NOT_STARTED, + environment="staging", + calculation_flow="economy", + originating_request_id="request-1", + observability_id="00000000-0000-4000-8000-000000000012", + production_identity=f"job-{evaluation_id}", + incumbent_execution_id="job-1", + worker_version="5.2.0", + modal_application="policyengine-simulation-v2-py5-2-0", + report_coordinator_callable="coordinate_report", + version_manifest_sha256=DIGEST, + policyengine_version="5.2.0", + country_package_name="policyengine-us", + country_package_version="1.764.6", + country="us", + dataset_identity="populace_us_2024", + dataset_uri="hf://policyengine/data/populace_us_2024.h5@revision", + data_package_name="populace-data", + data_package_version="0.1.0", + data_artifact_revision="revision", + created_at=NOW, + updated_at=NOW, + retention_expires_at=NOW + timedelta(days=30), + ) + + +def _simulation(evaluation_id: UUID) -> ComparisonSimulationRecord: + return ComparisonSimulationRecord( + simulation_execution_id=uuid4(), + evaluation_id=evaluation_id, + role=SimulationRole.BASELINE, + input_sha256="b" * 64, + worker_version="5.2.0", + modal_application="policyengine-simulation-v2-py5-2-0", + simulation_callable="run_single_simulation_us", + version_manifest_sha256=DIGEST, + modal_invocation_id=PLACEHOLDER_CALL, + status=ComparisonRunLifecycleStatus.PENDING, + created_at=NOW, + updated_at=NOW, + retention_expires_at=NOW + timedelta(days=30), + ) + + +@pytest.fixture +def statements(runtime_sessions: sessionmaker[Session]) -> Iterator[StatementLog]: + log = StatementLog(runtime_sessions) + yield log + log.assert_every_statement_ran_on_a_session() + + +@pytest.fixture +def store( + runtime_sessions: sessionmaker[Session], statements: StatementLog +) -> Stage12PersistenceStore: + assert statements.sent == [] + return Stage12PersistenceStore(session_factory=runtime_sessions) + + +def _deployment_command_sessions() -> sessionmaker[Session]: + """Return the deployment command's session factory, as the runtime role.""" + worker_database_url() + database_url = make_url(os.environ[SEED_URL_ENV]).set(drivername="postgresql") + session_factory = create_stage12_check_session_factory( + database_url.render_as_string(hide_password=False) + ) + act_as_role(session_factory.kw["bind"], RUNTIME_ROLE) + return session_factory + + +@pytest.fixture +def check_sessions() -> Iterator[sessionmaker[Session]]: + """Yield the deployment command's session factory, as the runtime role.""" + session_factory = _deployment_command_sessions() + try: + yield session_factory + finally: + dispose_session_factory(session_factory) + + +@pytest.fixture +def retry_waits(monkeypatch: pytest.MonkeyPatch) -> list[float]: + """Record the deployment check's waits between attempts instead of waiting.""" + waits: list[float] = [] + monkeypatch.setattr(time, "sleep", waits.append) + return waits + + +@pytest.fixture +def evaluation_id(seeder: ReportRunSeeder) -> Iterator[UUID]: + """Yield a report's ID; its row and simulations are removed after the test.""" + new_id = uuid4() + yield new_id + with Session(seeder.engine) as session: + session.execute( + delete(ComparisonReport).where(ComparisonReport.evaluation_id == new_id) + ) + session.commit() + + +def test_a_report_is_created_once_and_read_back(store, evaluation_id) -> None: + report = _report(evaluation_id) + + created = store.create_or_resolve_report(report) + resolved = store.create_or_resolve_report(report) + + assert created.created is True + assert created.record == report + assert resolved.created is False + assert resolved.record == report + assert store.get_report(evaluation_id) == report + with pytest.raises(LookupError, match="does not exist"): + store.get_report(uuid4()) + + +def test_a_report_with_the_same_identity_and_other_provenance_is_refused( + store, evaluation_id +) -> None: + report = _report(evaluation_id) + store.create_or_resolve_report(report) + conflicting = report.model_copy( + update={"evaluation_id": uuid4(), "country_package_version": "9.9.9"} + ) + + with pytest.raises(ValueError, match="country_package_version"): + store.create_or_resolve_report(conflicting) + + +def test_a_reports_lifecycle_is_replaced_under_its_row_lock( + store, statements, evaluation_id +) -> None: + report = _report(evaluation_id) + store.create_or_resolve_report(report) + running = report.model_copy( + update={ + "status": ComparisonRunLifecycleStatus.RUNNING, + "started_at": LATER, + "updated_at": LATER, + } + ) + + replaced = store.replace_report(running) + + assert replaced == running + assert store.get_report(evaluation_id) == running + assert [sent for sent in statements.sent if sent.endswith("FOR UPDATE")] + + +def test_a_reports_result_comparison_is_replaced_once_final( + store, evaluation_id +) -> None: + report = _report(evaluation_id).model_copy( + update={"comparison_status": ResultComparisonStatus.PENDING} + ) + store.create_or_resolve_report(report) + matched = report.model_copy( + update={ + "comparison_status": ResultComparisonStatus.MATCHED, + "comparison_output_uri": "gs://bucket/comparison", + "comparison_output_sha256": DIGEST, + "comparison_schema_version": 1, + "comparison_completed_at": LATER, + "updated_at": LATER, + } + ) + + first = store.replace_report_result_comparison(matched) + replayed = store.replace_report_result_comparison(matched) + + assert first == matched + assert replayed == matched + + +def test_a_simulation_needs_its_report_and_is_created_once( + store, evaluation_id +) -> None: + simulation = _simulation(evaluation_id) + with pytest.raises(LookupError, match="does not exist"): + store.create_or_resolve_simulation(simulation) + store.create_or_resolve_report(_report(evaluation_id)) + + created = store.create_or_resolve_simulation(simulation) + resolved = store.create_or_resolve_simulation(simulation) + + assert created.created is True + assert created.record == simulation + assert resolved.created is False + assert store.get_simulation(simulation.simulation_execution_id) == simulation + assert store.list_simulations(evaluation_id) == (simulation,) + + +def test_a_simulation_is_replaced_only_in_its_expected_status( + store, evaluation_id +) -> None: + store.create_or_resolve_report(_report(evaluation_id)) + simulation = _simulation(evaluation_id) + store.create_or_resolve_simulation(simulation) + running = simulation.model_copy( + update={ + "status": ComparisonRunLifecycleStatus.RUNNING, + "started_at": LATER, + "updated_at": LATER, + } + ) + assert store.replace_simulation(running) == running + failed = running.model_copy( + update={ + "status": ComparisonRunLifecycleStatus.FAILED, + "error_code": "simulation_execution_failed", + "completed_at": LATER, + } + ) + + stale, stale_replaced = store.replace_simulation_if_status( + failed, expected_status=ComparisonRunLifecycleStatus.PENDING + ) + current, replaced = store.replace_simulation_if_status( + failed, expected_status=ComparisonRunLifecycleStatus.RUNNING + ) + + assert (stale, stale_replaced) == (running, False) + assert (current, replaced) == (failed, True) + + +def test_a_simulations_invocation_replaces_only_the_placeholder( + store, evaluation_id +) -> None: + store.create_or_resolve_report(_report(evaluation_id)) + simulation = _simulation(evaluation_id) + store.create_or_resolve_simulation(simulation) + simulation_id = simulation.simulation_execution_id + + def attach(call: str) -> ComparisonSimulationRecord: + return store.attach_simulation_invocation( + simulation_id, + expected_placeholder=PLACEHOLDER_CALL, + modal_invocation_id=call, + updated_at=LATER, + ) + + attached = attach("fc-123") + repeated = attach("fc-123") + + assert attached.modal_invocation_id == "fc-123" + assert repeated == attached + with pytest.raises(ValueError, match="invocation identity changed"): + attach("fc-other") + with pytest.raises(LookupError, match="does not exist"): + store.attach_simulation_invocation( + uuid4(), + expected_placeholder=PLACEHOLDER_CALL, + modal_invocation_id="fc-123", + updated_at=LATER, + ) + + +def test_the_deployment_check_passes_and_leaves_no_row( + runtime_sessions, seeder +) -> None: + # The schema inspection reads through the engine, outside any session. + statements = StatementLog(runtime_sessions) + + def comparison_rows() -> int: + with Session(seeder.engine) as session: + return session.execute( + select(func.count()).select_from(ComparisonReport) + ).scalar_one() + + before = comparison_rows() + + validate_runtime_database( + runtime_sessions, expected_role=RUNTIME_ROLE, environment="staging" + ) + + assert statements.starting_with("INSERT INTO public.stage12_evaluation_reports") + assert statements.starting_with("DELETE FROM public.stage12_evaluation_reports") + assert statements.commits == 0 + assert comparison_rows() == before + + +def _comparison_report_count(seeder: ReportRunSeeder) -> int: + with Session(seeder.engine) as session: + return session.execute( + select(func.count()).select_from(ComparisonReport) + ).scalar_one() + + +def test_the_deployment_command_runs_its_check_on_one_real_connection( + check_sessions, retry_waits +) -> None: + opened = record_opened_connections(check_sessions) + + validate_runtime_database( + check_sessions, expected_role=RUNTIME_ROLE, environment="staging" + ) + + assert len(opened) == 1 + assert retry_waits == [] + + +def _terminate_the_connection_at_the_first_insert( + session_factory: sessionmaker[Session], seeder: ReportRunSeeder +) -> list[int]: + """End the server process during the check's first insert; return its ID.""" + terminated_backends: list[int] = [] + + def terminate_during_the_first_insert( + connection, _cursor, statement, _parameters, _context, _many + ) -> None: + if terminated_backends or not statement.startswith( + f"INSERT INTO public.{REPORT_TABLE_NAME}" + ): + return + backend = connection.connection.dbapi_connection.info.backend_pid + terminated_backends.append(backend) + with seeder.engine.begin() as administrator: + administrator.execute(select(func.pg_terminate_backend(backend))) + + event.listen( + session_factory.kw["bind"], + "before_cursor_execute", + terminate_during_the_first_insert, + ) + return terminated_backends + + +def test_the_deployment_command_replaces_a_connection_it_loses( + check_sessions, seeder, retry_waits +) -> None: + opened = record_opened_connections(check_sessions) + reports_before = _comparison_report_count(seeder) + terminated_backends = _terminate_the_connection_at_the_first_insert( + check_sessions, seeder + ) + + validate_runtime_database( + check_sessions, expected_role=RUNTIME_ROLE, environment="staging" + ) + + assert len(terminated_backends) == 1 + assert len(opened) == 2 + assert retry_waits == [1.0] + assert _comparison_report_count(seeder) == reports_before + + +def test_the_deployment_command_retries_a_refused_replacement_connection( + check_sessions, seeder, retry_waits +) -> None: + attempts = refuse_connection_numbers(check_sessions, {2}) + terminated_backends = _terminate_the_connection_at_the_first_insert( + check_sessions, seeder + ) + + validate_runtime_database( + check_sessions, expected_role=RUNTIME_ROLE, environment="staging" + ) + + assert len(terminated_backends) == 1 + assert len(attempts) == 3 + assert retry_waits == [1.0, 2.0] + + +def test_the_deployment_command_replaces_a_connection_lost_while_idle( + check_sessions, seeder, retry_waits +) -> None: + # The connection is checked before reuse, so no step fails or waits. + opened = record_opened_connections(check_sessions) + terminated_backends: list[int] = [] + + def terminate_the_first_returned_connection( + dbapi_connection, _connection_record + ) -> None: + if terminated_backends: + return + backend = dbapi_connection.info.backend_pid + terminated_backends.append(backend) + with seeder.engine.begin() as administrator: + administrator.execute(select(func.pg_terminate_backend(backend))) + + event.listen( + check_sessions.kw["bind"], "checkin", terminate_the_first_returned_connection + ) + + validate_runtime_database( + check_sessions, expected_role=RUNTIME_ROLE, environment="staging" + ) + + assert len(terminated_backends) == 1 + assert len(opened) == 2 + assert retry_waits == [] + + +def test_the_deployment_check_reports_a_connection_another_step_kept( + retry_waits, monkeypatch +) -> None: + monkeypatch.setattr(persistence_engine, "SINGLE_CONNECTION_WAIT_SECONDS", 0.05) + session_factory = _deployment_command_sessions() + try: + with session_factory() as holder: + holder.execute(select(true())).scalar_one() + with pytest.raises(Stage12RuntimeAccessError, match="TimeoutError"): + validate_runtime_database( + session_factory, + expected_role=RUNTIME_ROLE, + environment="staging", + ) + finally: + dispose_session_factory(session_factory) + + assert retry_waits == [] + + +def test_the_deployment_check_needs_one_free_pooler_slot(runtime_sessions) -> None: + # The deployment URL names a session pooler with a fixed client limit, so + # the check never holds one connection while it opens another. + limit_open_connections(runtime_sessions, limit=1) + + validate_runtime_database( + runtime_sessions, expected_role=RUNTIME_ROLE, environment="staging" + ) + + +@pytest.mark.parametrize("refused_connection", range(1, CHECK_CONNECTIONS + 1)) +def test_the_deployment_check_survives_any_one_refused_connection( + runtime_sessions, retry_waits, refused_connection +) -> None: + # A session pooler refuses clients while it is full, which says nothing + # about the schema or the role. Every step of the check may be repeated. + attempts = refuse_connection_numbers(runtime_sessions, {refused_connection}) + + validate_runtime_database( + runtime_sessions, expected_role=RUNTIME_ROLE, environment="staging" + ) + + assert len(attempts) > CHECK_CONNECTIONS + assert retry_waits == [1.0] + + +def test_the_deployment_check_stops_when_connections_stay_refused( + runtime_sessions, retry_waits +) -> None: + attempts = refuse_connection_numbers(runtime_sessions, range(1, 1000)) + + with pytest.raises(Stage12RuntimeAccessError, match="OperationalError"): + validate_runtime_database( + runtime_sessions, expected_role=RUNTIME_ROLE, environment="staging" + ) + + assert len(attempts) == 5 + assert retry_waits == [1.0, 2.0, 4.0, 8.0] + + +def test_the_deployment_check_refuses_another_role( + runtime_sessions, retry_waits +) -> None: + attempts = refuse_connection_numbers(runtime_sessions, ()) + + with pytest.raises(Stage12RuntimeAccessError, match="unexpected role"): + validate_runtime_database( + runtime_sessions, expected_role="another_role", environment="staging" + ) + + assert len(attempts) == 1 + assert retry_waits == [] + + +def test_the_deployment_check_refuses_an_unknown_environment( + runtime_sessions, +) -> None: + with pytest.raises(Stage12RuntimeAccessError, match="staging or production"): + validate_runtime_database( + runtime_sessions, expected_role=RUNTIME_ROLE, environment="local" + ) + + +class FailingSessions: + """A session factory whose sessions fail every statement with one error.""" + + def __init__(self, error: Exception) -> None: + self.error = error + + def __call__(self) -> Self: + return self + + def __enter__(self) -> Self: + return self + + def __exit__(self, *_exception_details) -> None: + return None + + def rollback(self) -> None: + return None + + def execute(self, _statement: object) -> None: + raise self.error + + +@pytest.mark.parametrize( + ("error_type", "connection_invalidated", "retries"), + [ + pytest.param(OperationalError, False, 4, id="connection-error-is-retried"), + # A pooler ending a connection can report it as an internal error. + pytest.param(InternalError, True, 4, id="lost-connection-is-retried"), + pytest.param(InternalError, False, 0, id="internal-error-is-not"), + pytest.param(ProgrammingError, False, 0, id="other-database-error-is-not"), + ], +) +def test_the_deployment_check_reports_a_database_error_without_its_text( + retry_waits, error_type, connection_invalidated, retries +) -> None: + sessions = FailingSessions( + error_type( + None, + None, + Exception("password=hidden"), + connection_invalidated=connection_invalidated, + ) + ) + + with pytest.raises(Stage12RuntimeAccessError) as failure: + validate_runtime_database( + sessions, # type: ignore[arg-type] + expected_role=RUNTIME_ROLE, + environment="staging", + ) + + assert error_type.__name__ in str(failure.value) + assert "password" not in str(failure.value) + assert len(retry_waits) == retries + + +def test_the_deployment_check_refuses_a_role_missing_a_table_privilege( + worker_sessions, retry_waits +) -> None: + with pytest.raises( + Stage12RuntimeAccessError, + match=f"lacks SELECT on {REPORT_TABLE_NAME}", + ): + validate_runtime_database( + worker_sessions, + expected_role="policyengine_v2_modal_worker", + environment="staging", + ) + + assert retry_waits == [] + + +def test_the_deployment_check_refuses_a_role_that_may_change_the_schema( + seeder, retry_waits +) -> None: + with pytest.raises(Stage12RuntimeAccessError, match="public-schema CREATE"): + validate_runtime_database( + create_session_factory(seeder.engine), + expected_role="postgres", + environment="staging", + ) + + assert retry_waits == [] + + +def test_the_deployment_check_refuses_a_forbidden_table_privilege( + runtime_sessions, retry_waits, monkeypatch +) -> None: + has_privilege = validation._has_table_privilege + + def has_truncate_too(session: Session, table_name: str, privilege: str) -> bool: + return privilege == "TRUNCATE" or has_privilege(session, table_name, privilege) + + monkeypatch.setattr(validation, "_has_table_privilege", has_truncate_too) + + with pytest.raises(Stage12RuntimeAccessError, match="unexpectedly has TRUNCATE"): + validate_runtime_database( + runtime_sessions, expected_role=RUNTIME_ROLE, environment="staging" + ) + + assert retry_waits == [] + + +class AlteredSchema: + """A live schema with one inspector answer replaced.""" + + def __init__(self, inspector: Any, method_name: str, answer: Any) -> None: + self._inspector = inspector + self._method_name = method_name + self._answer = answer + + def __getattr__(self, name: str) -> Any: + if name == self._method_name: + return lambda *_args, **_keywords: self._answer( + getattr(self._inspector, name)(*_args, **_keywords) + ) + return getattr(self._inspector, name) + + +def _without_first_column(columns: list[dict]) -> list[dict]: + return columns[1:] + + +def _with_changed_nullability(columns: list[dict]) -> list[dict]: + first = {**columns[0], "nullable": not columns[0]["nullable"]} + return [first, *columns[1:]] + + +def _no_columns_constrained(constraint: dict) -> dict: + return {**constraint, "constrained_columns": []} + + +def _nothing(_answer: object) -> list: + return [] + + +def _keys_to_another_table(foreign_keys: list[dict]) -> list[dict]: + return [{**key, "referred_table": "another_table"} for key in foreign_keys] + + +@pytest.mark.parametrize( + ("method_name", "answer", "message"), + [ + pytest.param( + "get_columns", _without_first_column, "columns do not match", id="columns" + ), + pytest.param( + "get_columns", + _with_changed_nullability, + "unexpected nullability", + id="nullability", + ), + pytest.param( + "get_pk_constraint", + _no_columns_constrained, + "unexpected primary key", + id="primary-key", + ), + pytest.param( + "get_unique_constraints", + _nothing, + "unexpected identity constraint", + id="identity", + ), + pytest.param( + "get_check_constraints", + _nothing, + "missing required check constraints", + id="checks", + ), + pytest.param("get_indexes", _nothing, "is missing the expected", id="indexes"), + ], +) +def test_the_schema_check_compares_the_live_table_with_its_model( + seeder, method_name, answer, message +) -> None: + live_schema = inspect(seeder.engine) + _validate_table_schema( + live_schema, + model=ComparisonReport, + table_name=REPORT_TABLE_NAME, + identity_constraint=REPORT_IDENTITY_CONSTRAINT, + check_constraints=REPORT_CHECK_CONSTRAINTS, + ) + + with pytest.raises(Stage12RuntimeAccessError, match=message): + _validate_table_schema( + AlteredSchema(inspect(seeder.engine), method_name, answer), + model=ComparisonReport, + table_name=REPORT_TABLE_NAME, + identity_constraint=REPORT_IDENTITY_CONSTRAINT, + check_constraints=REPORT_CHECK_CONSTRAINTS, + ) + + +@pytest.mark.parametrize( + ("answer", "message"), + [ + pytest.param(_nothing, "foreign key is missing", id="missing"), + pytest.param( + _keys_to_another_table, "unexpected definition", id="another-table" + ), + ], +) +def test_the_schema_check_compares_the_simulations_foreign_key( + seeder, answer, message +) -> None: + _validate_foreign_key(inspect(seeder.engine)) + + with pytest.raises(Stage12RuntimeAccessError, match=message): + _validate_foreign_key( + AlteredSchema(inspect(seeder.engine), "get_foreign_keys", answer) + ) diff --git a/libs/policyengine-stage12-persistence/tests/test_household_job_store.py b/libs/policyengine-stage12-persistence/tests/test_household_job_store.py index 1950ab3ea..4979b0969 100644 --- a/libs/policyengine-stage12-persistence/tests/test_household_job_store.py +++ b/libs/policyengine-stage12-persistence/tests/test_household_job_store.py @@ -87,11 +87,7 @@ @pytest.fixture def store(): - run_store = ReportRunStore(worker_database_url()) - try: - yield run_store - finally: - run_store.engine.dispose() + return ReportRunStore(worker_database_url()) def _set_job(seeder, job_id, **values): diff --git a/libs/policyengine-stage12-persistence/tests/test_household_report_run_store.py b/libs/policyengine-stage12-persistence/tests/test_household_report_run_store.py index 62dd39d1d..cac44fc4c 100644 --- a/libs/policyengine-stage12-persistence/tests/test_household_report_run_store.py +++ b/libs/policyengine-stage12-persistence/tests/test_household_report_run_store.py @@ -92,12 +92,8 @@ @pytest.fixture -def store() -> Iterator[HouseholdReportRunStore]: - run_store = HouseholdReportRunStore(worker_database_url()) - try: - yield run_store - finally: - run_store.engine.dispose() +def store() -> HouseholdReportRunStore: + return HouseholdReportRunStore(worker_database_url()) @pytest.fixture diff --git a/libs/policyengine-stage12-persistence/tests/test_mapped_classes.py b/libs/policyengine-stage12-persistence/tests/test_mapped_classes.py new file mode 100644 index 000000000..ae7fa02d7 --- /dev/null +++ b/libs/policyengine-stage12-persistence/tests/test_mapped_classes.py @@ -0,0 +1,163 @@ +"""The ORM mapped classes of the comparison path's two tables.""" + +from __future__ import annotations + +import inspect as python_inspect +import os +from datetime import UTC, datetime, timedelta +from types import NoneType, UnionType +from typing import Union, get_args, get_origin +from uuid import uuid4 + +import pytest +from policyengine_stage12_persistence.orm_base import ComparisonBase +from policyengine_stage12_persistence.tables import ( + REPORT_TABLE_NAME, + SIMULATION_TABLE_NAME, + ComparisonReport, + ComparisonSimulation, + metadata, +) +from report_run_database import SEED_URL_ENV, worker_database_url +from sqlalchemy import FetchedValue, create_engine, inspect, select +from sqlalchemy.dialects import postgresql +from sqlalchemy.orm import Mapped, Session, configure_mappers +from test_store import _report, _simulation + +MAPPED_CLASSES = [ + pytest.param( + ComparisonReport, + ComparisonReport.__table__, + REPORT_TABLE_NAME, + "evaluation_id", + id="report", + ), + pytest.param( + ComparisonSimulation, + ComparisonSimulation.__table__, + SIMULATION_TABLE_NAME, + "simulation_execution_id", + id="simulation", + ), +] +# The columns API v1's tables give a server default. +REPORT_DATABASE_DEFAULTS = {"comparison_status", "created_at", "updated_at"} +SIMULATION_DATABASE_DEFAULTS = {"created_at", "updated_at"} + + +def _allows_none(annotation: object) -> bool: + return get_origin(annotation) in (Union, UnionType) and NoneType in get_args( + annotation + ) + + +def test_the_base_maps_exactly_the_comparison_tables() -> None: + assert ComparisonBase.metadata is metadata + assert {mapper.class_ for mapper in ComparisonBase.registry.mappers} == { + ComparisonReport, + ComparisonSimulation, + } + configure_mappers() + + +@pytest.mark.parametrize( + ("mapped_class", "table", "table_name", "primary_key"), MAPPED_CLASSES +) +def test_each_table_name_the_store_imports_is_its_classs_table( + mapped_class, table, table_name, primary_key +) -> None: + assert mapped_class.__table__ is table + assert mapped_class.__tablename__ == table_name + assert [column.name for column in inspect(mapped_class).primary_key] == [ + primary_key + ] + + +@pytest.mark.parametrize( + ("mapped_class", "table", "table_name", "primary_key"), MAPPED_CLASSES +) +def test_each_column_is_a_typed_attribute_of_its_own_name( + mapped_class, table, table_name, primary_key +) -> None: + annotations = python_inspect.get_annotations(mapped_class, eval_str=True) + + assert [attribute.key for attribute in inspect(mapped_class).column_attrs] == [ + column.name for column in table.columns + ] + for column in table.columns: + annotation = annotations[column.name] + assert get_origin(annotation) is Mapped, column.name + (value_type,) = get_args(annotation) + assert _allows_none(value_type) is column.nullable, column.name + + +def test_a_statement_on_a_class_names_its_table_in_the_public_schema() -> None: + statement = select(ComparisonSimulation).where( + ComparisonSimulation.status == "pending" + ) + + assert f"FROM public.{SIMULATION_TABLE_NAME}" in str( + statement.compile(dialect=postgresql.dialect()) + ) + + +@pytest.mark.parametrize( + ("table", "column_names"), + [ + pytest.param(ComparisonReport.__table__, REPORT_DATABASE_DEFAULTS, id="report"), + pytest.param( + ComparisonSimulation.__table__, + SIMULATION_DATABASE_DEFAULTS, + id="simulation", + ), + ], +) +def test_the_columns_the_database_fills_are_marked_as_such(table, column_names) -> None: + marked = { + column.name + for column in table.columns + if isinstance(column.server_default, FetchedValue) + } + + assert marked == column_names + + +def test_a_session_insert_leaves_the_tables_defaults_to_the_database() -> None: + worker_database_url() + report_id = uuid4() + # The tables require retention to end after the database's created_at. + retention_expires_at = datetime.now(UTC) + timedelta(days=30) + report_values = { + **_report().model_dump(exclude=REPORT_DATABASE_DEFAULTS), + "evaluation_id": report_id, + "retention_expires_at": retention_expires_at, + } + simulation_values = { + **_simulation().model_dump(exclude=SIMULATION_DATABASE_DEFAULTS), + "simulation_execution_id": uuid4(), + "evaluation_id": report_id, + "retention_expires_at": retention_expires_at, + } + report = ComparisonReport(**report_values) + simulation = ComparisonSimulation(**simulation_values) + engine = create_engine(os.environ[SEED_URL_ENV]) + try: + with engine.connect() as connection: + transaction = connection.begin() + with Session( + connection, join_transaction_mode="create_savepoint" + ) as session: + session.add(report) + session.flush() + session.add(simulation) + session.flush() + stored_status = report.comparison_status + report_created_at = report.created_at + simulation_created_at = simulation.created_at + transaction.rollback() + finally: + engine.dispose() + + assert stored_status == "not_requested" + assert report_created_at is not None + assert simulation_created_at is not None diff --git a/libs/policyengine-stage12-persistence/tests/test_one_simulation_run_results.py b/libs/policyengine-stage12-persistence/tests/test_one_simulation_run_results.py index b8973f85b..10768dd43 100644 --- a/libs/policyengine-stage12-persistence/tests/test_one_simulation_run_results.py +++ b/libs/policyengine-stage12-persistence/tests/test_one_simulation_run_results.py @@ -11,11 +11,8 @@ from economy_output import US_ECONOMY_OUTPUT, uk_economy_output from policyengine_simulation_contract.stage12_execution import ReportRunIds -from policyengine_stage12_persistence.report_run_results import ( - inequality, - poverty, - report_run_result, -) +from policyengine_simulation_persistence.result_tables import Inequality, Poverty +from policyengine_stage12_persistence.report_run_results import report_run_result SIMULATION_RUN_ID = UUID("00000000-0000-0000-0000-00000000000b") ONE_RUN_ON_BOTH_SIDES = ReportRunIds( @@ -34,12 +31,12 @@ def _run_rows(output, run_ids: ReportRunIds, country: str): """Return the poverty and inequality rows of ``output``, without their IDs.""" result = report_run_result(output, run_ids=run_ids, country=country) rows_by_table = { - rows.table.name: [ + rows.model.__tablename__: [ {name: value for name, value in row.items() if name != "id"} for row in rows.rows ] for rows in result.rows - if rows.table in {poverty, inequality} + if rows.model in {Poverty, Inequality} } return rows_by_table["poverty"], rows_by_table["inequality"] diff --git a/libs/policyengine-stage12-persistence/tests/test_report_run_results.py b/libs/policyengine-stage12-persistence/tests/test_report_run_results.py index 1cd058005..e2cfeabdc 100644 --- a/libs/policyengine-stage12-persistence/tests/test_report_run_results.py +++ b/libs/policyengine-stage12-persistence/tests/test_report_run_results.py @@ -17,6 +17,7 @@ uk_output_with_withheld_codes, us_output_with_every_part, ) +from policyengine_simulation_persistence.result_tables import BudgetSummary from policyengine_stage12_persistence import ( DistrictCodes, ReportRunStore, @@ -59,11 +60,7 @@ @pytest.fixture def store(): - run_store = ReportRunStore(worker_database_url()) - try: - yield run_store - finally: - run_store.engine.dispose() + return ReportRunStore(worker_database_url()) DISTRICT_CODES = DistrictCodes( @@ -434,10 +431,10 @@ def test_a_result_that_cannot_be_stored_leaves_the_report_run_running( run_ids.report_run_id, modal_call_id="fc-coordinator", claimed_at=NOW ) result = report_run_result(US_ECONOMY_OUTPUT, run_ids=run_ids, country="us") - budget = next(rows for rows in result.rows if rows.table.name == "budget_summary") + budget = next(rows for rows in result.rows if rows.model is BudgetSummary) # The same row twice breaks the table's unique key in the last insert. duplicated = ResultRows( - table=budget.table, + model=budget.model, rows=(*budget.rows, {**budget.rows[0], "id": uuid4()}), shared_by_reports=False, ) diff --git a/libs/policyengine-stage12-persistence/tests/test_report_run_reuse_store.py b/libs/policyengine-stage12-persistence/tests/test_report_run_reuse_store.py index 94ffcc71b..b04599758 100644 --- a/libs/policyengine-stage12-persistence/tests/test_report_run_reuse_store.py +++ b/libs/policyengine-stage12-persistence/tests/test_report_run_reuse_store.py @@ -28,11 +28,7 @@ @pytest.fixture def store(): - run_store = ReportRunStore(worker_database_url()) - try: - yield run_store - finally: - run_store.engine.dispose() + return ReportRunStore(worker_database_url()) def _regions(seeder) -> Table: diff --git a/libs/policyengine-stage12-persistence/tests/test_report_run_store.py b/libs/policyengine-stage12-persistence/tests/test_report_run_store.py index 036880e45..a7a76b344 100644 --- a/libs/policyengine-stage12-persistence/tests/test_report_run_store.py +++ b/libs/policyengine-stage12-persistence/tests/test_report_run_store.py @@ -29,29 +29,25 @@ @pytest.fixture def store(): - run_store = ReportRunStore(worker_database_url()) - try: - yield run_store - finally: - run_store.engine.dispose() + return ReportRunStore(worker_database_url()) def test_the_store_connects_as_the_modal_worker_role(store) -> None: - with store.engine.connect() as connection: - role = connection.execute(select(func.current_user())).scalar_one() + with store.sessions.reading("read_role") as session: + role = session.execute(select(func.current_user())).scalar_one() assert role == "policyengine_v2_modal_worker" def test_the_modal_worker_role_cannot_create_a_report_run(store, seeded_run) -> None: - from policyengine_simulation_persistence.tables import report_runs + from policyengine_simulation_persistence.tables import ReportRun with ( pytest.raises(ProgrammingError, match="permission denied"), - store.engine.begin() as connection, + store.sessions.transaction("create_report_run") as session, ): - connection.execute( - insert(report_runs).values(id=uuid4(), status="pending", updated_at=NOW) + session.execute( + insert(ReportRun).values(id=uuid4(), status="pending", updated_at=NOW) ) @@ -106,17 +102,17 @@ def test_the_modal_worker_role_may_update_only_a_runs_state_and_results( ) # The same update of a column the role may update succeeds, so the # refusal below is of the column itself, not of the row it selects. - with store.engine.begin() as connection: - allowed = connection.execute( + with store.sessions.transaction("update_allowed_column") as session: + allowed = session.execute( update(run_table).where(run_table.c.id == row_id).values(updated_at=LATER) ) assert allowed.rowcount == 1 with ( pytest.raises(ProgrammingError, match="permission denied"), - store.engine.begin() as connection, + store.sessions.transaction("update_forbidden_column") as session, ): - connection.execute( + session.execute( update(run_table) .where(run_table.c.id == row_id) .values({column_name: value}) diff --git a/libs/policyengine-stage12-persistence/tests/test_report_run_sweep_store.py b/libs/policyengine-stage12-persistence/tests/test_report_run_sweep_store.py index e67ef4e61..60dec9b00 100644 --- a/libs/policyengine-stage12-persistence/tests/test_report_run_sweep_store.py +++ b/libs/policyengine-stage12-persistence/tests/test_report_run_sweep_store.py @@ -29,11 +29,7 @@ @pytest.fixture def store(): - run_store = ReportRunStore(worker_database_url()) - try: - yield run_store - finally: - run_store.engine.dispose() + return ReportRunStore(worker_database_url()) def _set_run(seeder, table_name, run_id, **values): @@ -301,12 +297,8 @@ def sweep(run_store): for run in runs ] - try: - with ThreadPoolExecutor(max_workers=len(stores)) as pool: - changes = list(pool.map(sweep, stores)) - finally: - for run_store in stores: - run_store.engine.dispose() + with ThreadPoolExecutor(max_workers=len(stores)) as pool: + changes = list(pool.map(sweep, stores)) assert [sum(column) for column in zip(*changes, strict=True)] == [1, 1, 1] assert seeder.row("report_runs", seeded_run.run_ids.report_run_id)["status"] == ( diff --git a/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py b/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py index f0aa0d6c1..d696f2865 100644 --- a/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py +++ b/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py @@ -39,8 +39,9 @@ report_run_store, store, ) +from policyengine_stage12_persistence import engine as stage12_engine from policyengine_stage12_persistence.report_run_store import ( - RUN_TABLES, + RUN_MODELS, RunKind, RunStatus, ) @@ -57,21 +58,21 @@ def _is_one_of(table: Table, tables: Iterable[Table]) -> bool: def test_the_run_stores_write_the_shared_run_table_models() -> None: - assert RUN_TABLES[RunKind.REPORT_RUN] is shared_tables.report_runs - assert RUN_TABLES[RunKind.SIMULATION_RUN] is shared_tables.simulation_runs - assert RUN_TABLES[RunKind.HOUSEHOLD_JOB] is shared_tables.household_jobs - assert report_run_store.budget_window_runs is shared_tables.budget_window_runs + assert RUN_MODELS[RunKind.REPORT_RUN] is shared_tables.ReportRun + assert RUN_MODELS[RunKind.SIMULATION_RUN] is shared_tables.SimulationRun + assert RUN_MODELS[RunKind.HOUSEHOLD_JOB] is shared_tables.HouseholdJob + assert report_run_store.BudgetWindowRun is shared_tables.BudgetWindowRun def test_the_budget_window_store_writes_the_shared_run_table_models() -> None: - assert budget_window_store.budget_window_runs is shared_tables.budget_window_runs - assert budget_window_store.report_runs is shared_tables.report_runs - assert budget_window_store.simulation_runs is shared_tables.simulation_runs + assert budget_window_store.BudgetWindowRun is shared_tables.BudgetWindowRun + assert budget_window_store.ReportRun is shared_tables.ReportRun + assert budget_window_store.SimulationRun is shared_tables.SimulationRun def test_the_household_report_store_writes_the_shared_run_table_models() -> None: - assert household_report_run_store.report_runs is shared_tables.report_runs - assert household_report_run_store.simulation_runs is shared_tables.simulation_runs + assert household_report_run_store.ReportRun is shared_tables.ReportRun + assert household_report_run_store.SimulationRun is shared_tables.SimulationRun def test_every_table_the_library_names_is_shared_or_a_comparison_table() -> None: @@ -89,11 +90,12 @@ def test_every_table_the_library_names_is_shared_or_a_comparison_table() -> None f"{policyengine_stage12_persistence.__name__}.{module_info.name}" ) for name, value in vars(module).items(): - if not isinstance(value, Table): + table = getattr(value, "__table__", value) + if not isinstance(table, Table): continue - assert _is_one_of(value, shared_models) or _is_one_of( - value, comparison_tables - ), f"{module_info.name}.{name} models {value.name} itself" + assert _is_one_of(table, shared_models) or _is_one_of( + table, comparison_tables + ), f"{module_info.name}.{name} models {table.name} itself" def test_every_result_row_goes_in_a_shared_result_table_model() -> None: @@ -119,15 +121,19 @@ def test_every_result_row_goes_in_a_shared_result_table_model() -> None: for result in results: for result_rows in result.rows: - assert _is_one_of(result_rows.table, result_models), result_rows.table.name + table = result_rows.model.__table__ + assert _is_one_of(table, result_models), table.name def test_the_stores_connect_with_the_service_librarys_one_engine_helper() -> None: """One URL check and one unpooled engine serve every Modal call's store.""" - assert store.create_stage12_engine is engine.create_unpooled_engine + assert ( + stage12_engine.create_unpooled_session_factory + is engine.create_unpooled_session_factory + ) assert not hasattr(store, "normalized_database_url") - assert report_run_store.create_stage12_engine is engine.create_unpooled_engine - assert budget_window_store.create_stage12_engine is engine.create_unpooled_engine + for store_module in (store, report_run_store, budget_window_store): + assert not hasattr(store_module, "create_unpooled_session_factory") @pytest.mark.parametrize( diff --git a/libs/policyengine-stage12-persistence/tests/test_simulation_run_age_store.py b/libs/policyengine-stage12-persistence/tests/test_simulation_run_age_store.py index ab51b1f29..106dd9ace 100644 --- a/libs/policyengine-stage12-persistence/tests/test_simulation_run_age_store.py +++ b/libs/policyengine-stage12-persistence/tests/test_simulation_run_age_store.py @@ -34,11 +34,7 @@ @pytest.fixture def store(): - run_store = ReportRunStore(worker_database_url()) - try: - yield run_store - finally: - run_store.engine.dispose() + return ReportRunStore(worker_database_url()) def _listed(store, run_id, **options): diff --git a/libs/policyengine-stage12-persistence/tests/test_simulation_run_held_by_store.py b/libs/policyengine-stage12-persistence/tests/test_simulation_run_held_by_store.py index 0d1087574..82f6fdbbb 100644 --- a/libs/policyengine-stage12-persistence/tests/test_simulation_run_held_by_store.py +++ b/libs/policyengine-stage12-persistence/tests/test_simulation_run_held_by_store.py @@ -26,11 +26,7 @@ @pytest.fixture def store(): - run_store = ReportRunStore(worker_database_url()) - try: - yield run_store - finally: - run_store.engine.dispose() + return ReportRunStore(worker_database_url()) def _recorded_call(seeder, run_id, modal_call_id: str) -> None: diff --git a/libs/policyengine-stage12-persistence/tests/test_store.py b/libs/policyengine-stage12-persistence/tests/test_store.py index cc9119402..dac76f924 100644 --- a/libs/policyengine-stage12-persistence/tests/test_store.py +++ b/libs/policyengine-stage12-persistence/tests/test_store.py @@ -1,7 +1,6 @@ from __future__ import annotations import sqlite3 -from contextlib import contextmanager from datetime import UTC, datetime, timedelta from uuid import UUID @@ -14,6 +13,7 @@ SimulationRole, ) from policyengine_stage12_persistence import Stage12PersistenceStore +from session_database import RetryingSessionOpener from sqlalchemy import create_engine from sqlalchemy.exc import OperationalError from sqlalchemy.sql import ClauseElement @@ -91,6 +91,9 @@ def one(self): assert self.row is not None return self.row + def scalar_one(self): + return self.one() + def all(self): return self.rows or [] @@ -105,57 +108,61 @@ def _refused_connection_error() -> OperationalError: ) -class FakeConnection: - def __init__(self, results, *, commit_error=None): - self.results = list(results) - self.statements = [] - self.commit_error = commit_error +class FakeSession: + """Answer statements with scripted results, connecting on the first one. + + As on a real session, the first statement checks out the connection, and + only what follows it is the operation's. + """ + + def __init__(self, database): + self.database = database + self.connected = False def __enter__(self): return self def __exit__(self, *_exception_details): - return None + self.close() - @contextmanager - def begin(self): - yield - if self.commit_error is not None: - raise self.commit_error + def close(self): + self.database.closed_sessions.add(self) - def execute(self, statement): + def commit(self): + if self.database.commit_error is not None: + raise self.database.commit_error + + def execute(self, statement, *_parameters, **_options): assert isinstance(statement, ClauseElement) - self.statements.append(statement) - return self.results.pop(0) + if not self.connected: + self.database.connect() + self.connected = True + return FakeResult(row=True) + self.database.statements.append(statement) + return self.database.results.pop(0) + +class FakeSessions: + """A session factory over scripted results, refusing the first connections.""" -class FakeEngine: def __init__(self, results, *, refused_connections=0, commit_error=None): - self.connection = FakeConnection(results, commit_error=commit_error) + self.results = list(results) + self.statements = [] + self.commit_error = commit_error self.refused_connections = refused_connections self.connection_attempts = 0 + self.opened_sessions = set() + self.closed_sessions = set() + + def __call__(self): + session = FakeSession(self) + self.opened_sessions.add(session) + return session def connect(self): self.connection_attempts += 1 if self.connection_attempts <= self.refused_connections: raise _refused_connection_error() - return self.connection - - -class RetryingConnectionOpener: - """Record each operation's connection and retry one refused connection.""" - - def __init__(self): - self.operation_names = [] - self.refusals = [] - - def __call__(self, connect, *, operation_name): - self.operation_names.append(operation_name) - try: - return connect() - except OperationalError as refusal: - self.refusals.append(refusal) - return connect() def _sqlalchemy_commit_failure(monkeypatch) -> OperationalError: @@ -179,10 +186,10 @@ def commit_then_lose_the_reply(dbapi_connection): return commit_failure.value -def test_operations_open_connections_through_the_opener_before_any_statement() -> None: +def test_operations_open_sessions_through_the_opener_before_any_statement() -> None: report = _report() simulation = _simulation() - engine = FakeEngine( + sessions = FakeSessions( [ FakeResult(row=_row(report)), FakeResult(row=_row(simulation)), @@ -190,10 +197,10 @@ def test_operations_open_connections_through_the_opener_before_any_statement() - ], refused_connections=1, ) - opener = RetryingConnectionOpener() + opener = RetryingSessionOpener() store = Stage12PersistenceStore( - engine=engine, # type: ignore[arg-type] - open_connection=opener, + session_factory=sessions, # type: ignore[arg-type] + open_session=opener, ) created = store.create_or_resolve_simulation(simulation) @@ -204,8 +211,8 @@ def test_operations_open_connections_through_the_opener_before_any_statement() - assert opener.operation_names == ["create_or_resolve_simulation", "get_simulation"] # The refusal reached the opener before the store sent any statement. assert len(opener.refusals) == 1 - assert engine.connection_attempts == 3 - assert len(engine.connection.statements) == 3 + assert sessions.connection_attempts == 3 + assert len(sessions.statements) == 3 def test_commit_failure_is_raised_after_the_opener_returns(monkeypatch) -> None: @@ -214,14 +221,14 @@ def test_commit_failure_is_raised_after_the_opener_returns(monkeypatch) -> None: lost_commit_reply = _sqlalchemy_commit_failure(monkeypatch) assert lost_commit_reply.statement is None report = _report() - engine = FakeEngine( + sessions = FakeSessions( [FakeResult(row=_row(report))], commit_error=lost_commit_reply, ) - opener = RetryingConnectionOpener() + opener = RetryingSessionOpener() store = Stage12PersistenceStore( - engine=engine, # type: ignore[arg-type] - open_connection=opener, + session_factory=sessions, # type: ignore[arg-type] + open_session=opener, ) with pytest.raises(OperationalError) as failure: @@ -230,32 +237,59 @@ def test_commit_failure_is_raised_after_the_opener_returns(monkeypatch) -> None: assert failure.value is lost_commit_reply assert opener.operation_names == ["create_or_resolve_report"] assert opener.refusals == [] - assert engine.connection_attempts == 1 - assert len(engine.connection.statements) == 1 + assert sessions.connection_attempts == 1 + assert len(sessions.statements) == 1 + + +def test_every_operation_closes_its_session() -> None: + report = _report() + sessions = FakeSessions( + [ + FakeResult(row=_row(report)), + FakeResult(row=_row(report)), + FakeResult(row=None), + FakeResult(row=None), + ], + refused_connections=1, + ) + store = Stage12PersistenceStore( + session_factory=sessions, # type: ignore[arg-type] + open_session=RetryingSessionOpener(), + ) + + store.create_or_resolve_report(report) + store.get_report(REPORT_ID) + with pytest.raises(LookupError): + store.get_report(REPORT_ID) + with pytest.raises(LookupError): + store.replace_report(report) + + assert len(sessions.opened_sessions) == 5 + assert sessions.closed_sessions == sessions.opened_sessions def test_default_store_does_not_retry_a_refused_connection() -> None: - engine = FakeEngine([], refused_connections=1) - store = Stage12PersistenceStore(engine=engine) # type: ignore[arg-type] + sessions = FakeSessions([], refused_connections=1) + store = Stage12PersistenceStore(session_factory=sessions) # type: ignore[arg-type] with pytest.raises(OperationalError, match="EMAXCONNSESSION"): store.get_report(REPORT_ID) - assert engine.connection_attempts == 1 + assert sessions.connection_attempts == 1 def test_create_and_read_report_use_sqlalchemy_statements_only() -> None: report = _report() - engine = FakeEngine([FakeResult(row=_row(report))]) - store = Stage12PersistenceStore(engine=engine) # type: ignore[arg-type] + sessions = FakeSessions([FakeResult(row=_row(report))]) + store = Stage12PersistenceStore(session_factory=sessions) # type: ignore[arg-type] result = store.create_or_resolve_report(report) assert result.created is True assert result.record == report assert result.record.observability_id == OBSERVABILITY_ID - assert len(engine.connection.statements) == 1 - assert "observability_id" in str(engine.connection.statements[0]) + assert len(sessions.statements) == 1 + assert "observability_id" in str(sessions.statements[0]) def test_report_observability_identifier_is_limited_to_36_characters() -> None: @@ -268,13 +302,13 @@ def test_report_observability_identifier_is_limited_to_36_characters() -> None: def test_report_identity_conflict_resolves_existing_row() -> None: report = _report() - engine = FakeEngine( + sessions = FakeSessions( [ FakeResult(row=None), FakeResult(row=_row(report)), ] ) - store = Stage12PersistenceStore(engine=engine) # type: ignore[arg-type] + store = Stage12PersistenceStore(session_factory=sessions) # type: ignore[arg-type] result = store.create_or_resolve_report(report) @@ -285,19 +319,19 @@ def test_report_identity_conflict_resolves_existing_row() -> None: def test_simulation_creation_checks_the_parent_in_the_same_transaction() -> None: report = _report() simulation = _simulation() - engine = FakeEngine( + sessions = FakeSessions( [ FakeResult(row=_row(report)), FakeResult(row=_row(simulation)), ] ) - store = Stage12PersistenceStore(engine=engine) # type: ignore[arg-type] + store = Stage12PersistenceStore(session_factory=sessions) # type: ignore[arg-type] result = store.create_or_resolve_simulation(simulation) assert result.created is True assert result.record == simulation - assert len(engine.connection.statements) == 2 + assert len(sessions.statements) == 2 def test_invalid_lifecycle_transition_is_rejected_before_update() -> None: @@ -305,21 +339,21 @@ def test_invalid_lifecycle_transition_is_rejected_before_update() -> None: candidate = existing.model_copy( update={"status": ComparisonRunLifecycleStatus.SUCCEEDED} ) - engine = FakeEngine([FakeResult(row=_row(existing))]) - store = Stage12PersistenceStore(engine=engine) # type: ignore[arg-type] + sessions = FakeSessions([FakeResult(row=_row(existing))]) + store = Stage12PersistenceStore(session_factory=sessions) # type: ignore[arg-type] with pytest.raises(ValueError, match="lifecycle cannot move"): store.replace_report(candidate) - assert len(engine.connection.statements) == 1 + assert len(sessions.statements) == 1 def test_invocation_attachment_is_compare_and_set() -> None: attached = _simulation().model_copy( update={"modal_invocation_id": "fc-123", "updated_at": NOW} ) - engine = FakeEngine([FakeResult(row=_row(attached))]) - store = Stage12PersistenceStore(engine=engine) # type: ignore[arg-type] + sessions = FakeSessions([FakeResult(row=_row(attached))]) + store = Stage12PersistenceStore(session_factory=sessions) # type: ignore[arg-type] result = store.attach_simulation_invocation( SIMULATION_ID, @@ -342,16 +376,16 @@ def test_simulation_status_replacement_is_compare_and_set() -> None: "completed_at": NOW, } ) - engine = FakeEngine( + sessions = FakeSessions( [ FakeResult(row=_row(running)), FakeResult(row=_row(failed)), ] ) - opener = RetryingConnectionOpener() + opener = RetryingSessionOpener() store = Stage12PersistenceStore( - engine=engine, # type: ignore[arg-type] - open_connection=opener, + session_factory=sessions, # type: ignore[arg-type] + open_session=opener, ) result, replaced = store.replace_simulation_if_status( @@ -362,7 +396,7 @@ def test_simulation_status_replacement_is_compare_and_set() -> None: assert replaced is True assert result == failed assert opener.operation_names == ["replace_simulation_if_status"] - assert len(engine.connection.statements) == 2 + assert len(sessions.statements) == 2 def test_simulation_status_replacement_preserves_a_concurrent_cancellation() -> None: @@ -386,8 +420,8 @@ def test_simulation_status_replacement_preserves_a_concurrent_cancellation() -> "completed_at": NOW, } ) - engine = FakeEngine([FakeResult(row=_row(cancelled))]) - store = Stage12PersistenceStore(engine=engine) # type: ignore[arg-type] + sessions = FakeSessions([FakeResult(row=_row(cancelled))]) + store = Stage12PersistenceStore(session_factory=sessions) # type: ignore[arg-type] result, replaced = store.replace_simulation_if_status( succeeded, @@ -396,4 +430,4 @@ def test_simulation_status_replacement_preserves_a_concurrent_cancellation() -> assert replaced is False assert result == cancelled - assert len(engine.connection.statements) == 1 + assert len(sessions.statements) == 1 diff --git a/libs/policyengine-stage12-persistence/tests/test_store_sessions.py b/libs/policyengine-stage12-persistence/tests/test_store_sessions.py new file mode 100644 index 000000000..6ce489ece --- /dev/null +++ b/libs/policyengine-stage12-persistence/tests/test_store_sessions.py @@ -0,0 +1,628 @@ +"""The run stores on ORM sessions, as the Modal worker role. + +Each test runs a store on a real ``Session`` against a disposable Postgres +that API v1's v2 migrations created (see ``report_run_database``), and +the ``statements`` fixture fails it if a statement bypassed the session. +""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from datetime import timedelta +from threading import Barrier +from typing import TYPE_CHECKING +from uuid import uuid4 + +import pytest +from economy_output import US_ECONOMY_OUTPUT +from household_report_database import ( + remove_household_report_run, + seed_household_report_run, +) +from policyengine_simulation_contract.stage12_execution import ReportRunIds +from policyengine_simulation_persistence.result_tables import BudgetSummary, Poverty +from policyengine_stage12_persistence import ( + ActiveRun, + HouseholdReportRunStore, + ReportRunStore, + RunKind, + RunStatus, + RunWrite, + report_run_result, +) +from policyengine_stage12_persistence.report_run_results import ResultRows +from report_run_database import NOW +from session_database import ( + CONNECTION_REFUSED_MESSAGE, + RetryingSessionOpener, + limit_open_connections, + refuse_connections, +) +from sqlalchemy import insert, select +from sqlalchemy.exc import IntegrityError, OperationalError + +if TYPE_CHECKING: + from collections.abc import Iterator + + from household_report_database import SeededHouseholdReportRun + from report_run_database import ReportRunSeeder, SeededHouseholdJob, SeededReportRun + from session_database import StatementLog + from sqlalchemy.orm import Session, sessionmaker + +LATER = NOW + timedelta(minutes=5) +CUTOFF = NOW + timedelta(minutes=30) +CALL = "fc-session-call" +OTHER_CALL = "fc-other-call" +CONNECTION_CHECK_SENT = "SELECT true AS anon_1" +HOUSEHOLD_DOCUMENT = {"status": "ok", "values": [1, 2]} + + +@pytest.fixture +def store( + worker_sessions: sessionmaker[Session], statements: StatementLog +) -> ReportRunStore: + assert statements.sent == [] + return ReportRunStore(session_factory=worker_sessions) + + +@pytest.fixture +def household_store( + worker_sessions: sessionmaker[Session], statements: StatementLog +) -> HouseholdReportRunStore: + assert statements.sent == [] + return HouseholdReportRunStore(session_factory=worker_sessions) + + +@pytest.fixture +def household_report( + seeder: ReportRunSeeder, +) -> Iterator[SeededHouseholdReportRun]: + seeded = seed_household_report_run(seeder) + try: + yield seeded + finally: + remove_household_report_run(seeder, seeded) + + +def _result(run_ids: ReportRunIds): + return report_run_result(US_ECONOMY_OUTPUT, run_ids=run_ids, country="us") + + +def _claimed(store: ReportRunStore, seeded_run: SeededReportRun) -> ReportRunIds: + run_ids = seeded_run.run_ids + store.claim_report_run(run_ids.report_run_id, modal_call_id=CALL, claimed_at=NOW) + return run_ids + + +def test_a_refused_connection_is_opened_again_and_the_claim_runs_once( + worker_sessions, statements, seeder, seeded_run +) -> None: + attempts = refuse_connections(worker_sessions, count=1) + opener = RetryingSessionOpener(statements) + store = ReportRunStore(session_factory=worker_sessions, open_session=opener) + report_run_id = seeded_run.run_ids.report_run_id + + write = store.claim_report_run(report_run_id, modal_call_id=CALL, claimed_at=NOW) + + assert write == RunWrite(changed=True, status=RunStatus.RUNNING) + assert len(attempts) == 2 + assert CONNECTION_REFUSED_MESSAGE in str(opener.refusals[0]) + assert opener.operation_names == ["claim_report_run"] + assert len(statements.starting_with("UPDATE")) == 1 + assert statements.commits == 1 + assert seeder.row("report_runs", report_run_id)["modal_call_id"] == CALL + + +def test_a_failed_connection_check_closes_its_session_before_the_retry( + worker_sessions, statements, seeder, seeded_run, monkeypatch +) -> None: + # With one connection allowed, the retry connects only if the first + # session gave its connection back. + limit_open_connections(worker_sessions, limit=1) + dialect = worker_sessions.kw["bind"].dialect + run_statement = dialect.do_execute + lost_checks = [CONNECTION_CHECK_SENT] + + def lose_the_first_check(cursor, statement, parameters, context=None) -> None: + if lost_checks and statement == lost_checks[0]: + lost_checks.clear() + message = "server closed the connection unexpectedly" + raise dialect.loaded_dbapi.OperationalError(message) + run_statement(cursor, statement, parameters, context) + + monkeypatch.setattr(dialect, "do_execute", lose_the_first_check) + opener = RetryingSessionOpener(statements) + store = ReportRunStore(session_factory=worker_sessions, open_session=opener) + report_run_id = seeded_run.run_ids.report_run_id + + write = store.claim_report_run(report_run_id, modal_call_id=CALL, claimed_at=NOW) + + assert write == RunWrite(changed=True, status=RunStatus.RUNNING) + assert len(opener.refusals) == 1 + assert len(statements.starting_with("UPDATE")) == 1 + assert seeder.row("report_runs", report_run_id)["modal_call_id"] == CALL + + +def test_only_the_connection_check_runs_while_a_session_opens( + worker_sessions, statements, seeded_run +) -> None: + opener = RetryingSessionOpener(statements) + store = ReportRunStore(session_factory=worker_sessions, open_session=opener) + + store.claim_report_run( + seeded_run.run_ids.report_run_id, modal_call_id=CALL, claimed_at=NOW + ) + + assert opener.sent_while_opening == [[CONNECTION_CHECK_SENT]] + assert statements.sent[0] == CONNECTION_CHECK_SENT + + +def test_a_lost_commit_reply_is_raised_and_the_write_is_not_repeated( + worker_sessions, statements, seeder, seeded_run, monkeypatch +) -> None: + dialect = worker_sessions.kw["bind"].dialect + + def commit_then_lose_the_reply(dbapi_connection) -> None: + dbapi_connection.commit() + message = "server closed the connection unexpectedly" + raise dialect.loaded_dbapi.OperationalError(message) + + monkeypatch.setattr(dialect, "do_commit", commit_then_lose_the_reply) + opener = RetryingSessionOpener(statements) + store = ReportRunStore(session_factory=worker_sessions, open_session=opener) + report_run_id = seeded_run.run_ids.report_run_id + + with pytest.raises(OperationalError, match="server closed the connection"): + store.claim_report_run(report_run_id, modal_call_id=CALL, claimed_at=NOW) + + assert opener.operation_names == ["claim_report_run"] + assert opener.refusals == [] + assert len(statements.starting_with("UPDATE")) == 1 + # The commit reached the database: only its reply was lost. + assert seeder.row("report_runs", report_run_id)["status"] == "running" + + +def test_a_statement_failure_after_opening_reaches_no_opener( + worker_sessions, statements, seeded_run +) -> None: + opener = RetryingSessionOpener(statements) + store = ReportRunStore(session_factory=worker_sessions, open_session=opener) + run_ids = _claimed(store, seeded_run) + result = _result(run_ids) + budget = next(rows for rows in result.rows if rows.model is BudgetSummary) + duplicated = ResultRows( + model=BudgetSummary, + rows=(*budget.rows, {**budget.rows[0], "id": uuid4()}), + shared_by_reports=False, + ) + broken = replace(result, rows=(duplicated,)) + + with pytest.raises(IntegrityError): + store.complete_report_run( + run_ids, modal_call_id=CALL, completed_at=LATER, result=broken + ) + + assert opener.operation_names == ["claim_report_run", "complete_report_run"] + assert opener.refusals == [] + + +def test_each_operation_commits_its_own_session(store, statements, seeded_run) -> None: + _claimed(store, seeded_run) + assert statements.commits == 1 + store.get_simulation_run(seeded_run.run_ids.baseline_simulation_run_id) + assert statements.commits == 2 + + +def test_a_conditional_update_returns_only_the_status_it_reads( + store, statements, seeded_run +) -> None: + _claimed(store, seeded_run) + + [claim] = statements.starting_with("UPDATE") + assert claim.startswith("UPDATE public.report_runs SET ") + assert claim.endswith("RETURNING public.report_runs.status") + assert statements.starting_with("SELECT public.") == [] + + +def test_concurrent_claims_leave_the_run_with_one_call( + worker_sessions, statements, seeder, seeded_run +) -> None: + assert statements.sent == [] + report_run_id = seeded_run.run_ids.report_run_id + claimants = 6 + start_together = Barrier(claimants) + + def claim(claimant: int) -> tuple[str, RunWrite]: + store = ReportRunStore(session_factory=worker_sessions) + call = f"fc-claimant-{claimant}" + start_together.wait(timeout=10) + return call, store.claim_report_run( + report_run_id, modal_call_id=call, claimed_at=NOW + ) + + with ThreadPoolExecutor(max_workers=claimants) as executor: + outcomes = list(executor.map(claim, range(claimants))) + + winners = [call for call, write in outcomes if write.changed] + assert len(winners) == 1 + assert seeder.row("report_runs", report_run_id)["modal_call_id"] == winners[0] + assert all(write.status is RunStatus.RUNNING for _call, write in outcomes) + + +def test_a_report_run_names_its_simulation_runs(store, seeded_run) -> None: + run_ids = seeded_run.run_ids + store.require_simulation_runs(run_ids) + + other = ReportRunIds( + report_run_id=run_ids.report_run_id, + baseline_simulation_run_id=run_ids.reform_simulation_run_id, + reform_simulation_run_id=run_ids.baseline_simulation_run_id, + ) + with pytest.raises(ValueError, match="names other simulation runs"): + store.require_simulation_runs(other) + missing = other.model_copy(update={"report_run_id": uuid4()}) + with pytest.raises(LookupError, match="does not exist"): + store.require_simulation_runs(missing) + + +def test_a_simulation_run_is_read_as_plain_values(store, seeded_run) -> None: + simulation_run_id = seeded_run.run_ids.baseline_simulation_run_id + + state = store.get_simulation_run(simulation_run_id) + + assert state.status is RunStatus.PENDING + assert state.year == 2026 + assert state.modal_call_id is None + with pytest.raises(LookupError, match="does not exist"): + store.get_simulation_run(uuid4()) + + +def test_a_completion_stores_results_once_with_null_columns_null( + store, seeder, seeded_run +) -> None: + run_ids = _claimed(store, seeded_run) + result = _result(run_ids) + + first = store.complete_report_run( + run_ids, modal_call_id=CALL, completed_at=LATER, result=result + ) + repeated = store.complete_report_run( + run_ids, modal_call_id=CALL, completed_at=LATER, result=_result(run_ids) + ) + + assert first == RunWrite(changed=True, status=RunStatus.SUCCEEDED) + assert repeated == RunWrite(changed=False, status=RunStatus.SUCCEEDED) + baseline = run_ids.baseline_simulation_run_id + stored = seeder.rows("poverty", simulation_run_id=baseline) + expected = next(rows for rows in result.rows if rows.model is Poverty) + baseline_rows = [ + row for row in expected.rows if row["simulation_run_id"] == baseline + ] + assert len(stored) == len(baseline_rows) + assert [row for row in stored if row["filter_variable"] is None] + budget = seeder.rows("budget_summary", report_run_id=run_ids.report_run_id) + assert budget + assert seeder.row("report_runs", run_ids.report_run_id)["status"] == "succeeded" + + +def test_a_failed_result_insert_leaves_the_run_running_without_results( + store, seeder, seeded_run +) -> None: + run_ids = _claimed(store, seeded_run) + result = _result(run_ids) + budget = next(rows for rows in result.rows if rows.model is BudgetSummary) + duplicated = ResultRows( + model=BudgetSummary, + rows=(*budget.rows, {**budget.rows[0], "id": uuid4()}), + shared_by_reports=False, + ) + other_rows = tuple(rows for rows in result.rows if rows is not budget) + + with pytest.raises(IntegrityError): + store.complete_report_run( + run_ids, + modal_call_id=CALL, + completed_at=LATER, + result=replace(result, rows=(*other_rows, duplicated)), + ) + + assert seeder.row("report_runs", run_ids.report_run_id)["status"] == "running" + assert seeder.rows("decile_impacts", report_run_id=run_ids.report_run_id) == [] + baseline = run_ids.baseline_simulation_run_id + assert seeder.rows("poverty", simulation_run_id=baseline) == [] + + +def test_a_shared_simulation_run_keeps_its_stored_result_row( + store, seeder, seeded_run +) -> None: + run_ids = _claimed(store, seeded_run) + baseline = run_ids.baseline_simulation_run_id + with seeder.engine.begin() as connection: + connection.execute( + insert(seeder.table("poverty")).values( + id=uuid4(), + simulation_run_id=baseline, + poverty_type="spm", + entity="person", + filter_variable=None, + filter_group=None, + rate=0.5, + ) + ) + + store.complete_report_run( + run_ids, modal_call_id=CALL, completed_at=LATER, result=_result(run_ids) + ) + + stored = seeder.rows( + "poverty", + simulation_run_id=baseline, + poverty_type="spm", + filter_variable=None, + ) + assert [row["rate"] for row in stored] == [0.5] + + +def test_a_report_run_fails_only_under_its_call(store, seeder, seeded_run) -> None: + run_ids = _claimed(store, seeded_run) + report_run_id = run_ids.report_run_id + + refused = store.fail_report_run( + report_run_id, modal_call_id=OTHER_CALL, error_message="no", failed_at=LATER + ) + failed = store.fail_report_run( + report_run_id, modal_call_id=CALL, error_message="it broke", failed_at=LATER + ) + + assert refused == RunWrite(changed=False, status=RunStatus.RUNNING) + assert failed == RunWrite(changed=True, status=RunStatus.FAILED) + assert seeder.row("report_runs", report_run_id)["error_message"] == "it broke" + with pytest.raises(LookupError, match=r"report run .* does not exist"): + store.fail_report_run( + uuid4(), modal_call_id=CALL, error_message="x", failed_at=LATER + ) + + +def test_a_simulation_run_is_claimed_and_completed_under_its_call( + store, seeder, seeded_run +) -> None: + simulation_run_id = seeded_run.run_ids.baseline_simulation_run_id + + claim = store.claim_simulation_run( + simulation_run_id, modal_call_id=CALL, claimed_at=NOW + ) + taken = store.claim_simulation_run( + simulation_run_id, modal_call_id=OTHER_CALL, claimed_at=LATER + ) + completed = store.complete_simulation_run( + simulation_run_id, + modal_call_id=CALL, + output_uri="gs://bucket/output", + output_sha256="a" * 64, + completed_at=LATER, + ) + + assert claim.changed + assert taken == RunWrite(changed=False, status=RunStatus.RUNNING) + assert completed == RunWrite(changed=True, status=RunStatus.SUCCEEDED) + stored = seeder.row("simulation_runs", simulation_run_id) + assert stored["output_uri"] == "gs://bucket/output" + assert stored["started_at"] == NOW + + +@pytest.mark.parametrize("names_the_call", [True, False]) +def test_a_simulation_run_is_failed_with_or_without_its_call( + store, seeder, seeded_run, names_the_call +) -> None: + simulation_run_id = seeded_run.run_ids.reform_simulation_run_id + store.claim_simulation_run(simulation_run_id, modal_call_id=CALL, claimed_at=NOW) + + failed = store.fail_simulation_run( + simulation_run_id, + error_message="it broke", + failed_at=LATER, + modal_call_id=CALL if names_the_call else None, + ) + + assert failed == RunWrite(changed=True, status=RunStatus.FAILED) + stored = seeder.row("simulation_runs", simulation_run_id) + assert stored["error_message"] == "it broke" + + +def test_a_run_held_by_an_ended_call_is_failed_once(store, seeded_run) -> None: + simulation_run_id = seeded_run.run_ids.baseline_simulation_run_id + store.claim_simulation_run(simulation_run_id, modal_call_id=CALL, claimed_at=NOW) + + first, second = ( + store.fail_simulation_run_held_by( + simulation_run_id, + modal_call_id=CALL, + error_message="its call ended", + failed_at=LATER, + ) + for _ in range(2) + ) + + assert first == RunWrite(changed=True, status=RunStatus.FAILED) + assert second == RunWrite(changed=False, status=RunStatus.FAILED) + + +def test_a_household_job_is_claimed_and_completed_with_its_document( + store, seeder, seeded_household_job: SeededHouseholdJob +) -> None: + job_id = seeded_household_job.job_id + + claim = store.claim_household_job(job_id, modal_call_id=CALL, claimed_at=NOW) + completed = store.complete_household_job( + job_id, + modal_call_id=CALL, + result=HOUSEHOLD_DOCUMENT, + completed_at=LATER, + ) + + assert claim.changed + assert completed == RunWrite(changed=True, status=RunStatus.SUCCEEDED) + assert seeder.row("household_jobs", job_id)["result"] == HOUSEHOLD_DOCUMENT + + +def test_a_household_job_failed_without_a_document_stores_sql_null( + store, seeder, seeded_household_job: SeededHouseholdJob +) -> None: + job_id = seeded_household_job.job_id + store.claim_household_job(job_id, modal_call_id=CALL, claimed_at=NOW) + + failed = store.fail_household_job( + job_id, modal_call_id=CALL, error_message="it broke", failed_at=LATER + ) + + assert failed == RunWrite(changed=True, status=RunStatus.FAILED) + jobs = seeder.table("household_jobs") + with seeder.engine.connect() as connection: + result_is_null = connection.execute( + select(jobs.c.result.is_(None)).where(jobs.c.id == job_id) + ).scalar_one() + assert result_is_null + + +@pytest.mark.parametrize("regardless_of_report_runs", [False, True]) +def test_the_sweep_lists_and_fails_a_stuck_report_run_once( + store, seeder, seeded_run, regardless_of_report_runs +) -> None: + report_run_id = _claimed(store, seeded_run).report_run_id + + [listed] = [ + run + for run in store.list_active_runs( + RunKind.REPORT_RUN, + active_since_before=CUTOFF, + regardless_of_report_runs=regardless_of_report_runs, + ) + if run.run_id == report_run_id + ] + first, second = ( + store.fail_stuck_run( + listed, + error_message="stuck", + failed_at=LATER, + regardless_of_report_runs=regardless_of_report_runs, + ) + for _ in range(2) + ) + + assert listed == ActiveRun( + kind=RunKind.REPORT_RUN, + run_id=report_run_id, + status=RunStatus.RUNNING, + modal_call_id=CALL, + ) + assert first == RunWrite(changed=True, status=RunStatus.FAILED) + assert second == RunWrite(changed=False, status=RunStatus.FAILED) + assert seeder.row("report_runs", report_run_id)["error_message"] == "stuck" + + +def test_concurrent_sweeps_fail_a_stuck_household_job_once( + store, worker_sessions, seeded_household_job: SeededHouseholdJob +) -> None: + job_id = seeded_household_job.job_id + store.claim_household_job(job_id, modal_call_id=CALL, claimed_at=NOW) + [listed] = [ + run + for run in store.list_active_runs( + RunKind.HOUSEHOLD_JOB, active_since_before=NOW + timedelta(days=36500) + ) + if run.run_id == job_id + ] + sweeps = 4 + start_together = Barrier(sweeps) + + def sweep(_sweep_number: int) -> RunWrite: + sweeping_store = ReportRunStore(session_factory=worker_sessions) + start_together.wait(timeout=10) + return sweeping_store.fail_stuck_run( + listed, error_message="stuck", failed_at=LATER + ) + + with ThreadPoolExecutor(max_workers=sweeps) as executor: + writes = list(executor.map(sweep, range(sweeps))) + + assert sum(write.changed for write in writes) == 1 + + +def test_the_sweep_waits_for_report_runs_naming_a_simulation_run( + store, statements, seeder, seeded_run +) -> None: + run_ids = seeded_run.run_ids + baseline = run_ids.baseline_simulation_run_id + store.claim_simulation_run(baseline, modal_call_id=CALL, claimed_at=NOW) + + def listed_ids(*, regardless_of_report_runs: bool) -> set: + return { + run.run_id + for run in store.list_active_runs( + RunKind.SIMULATION_RUN, + active_since_before=CUTOFF, + regardless_of_report_runs=regardless_of_report_runs, + ) + } + + assert baseline not in listed_ids(regardless_of_report_runs=False) + assert baseline in listed_ids(regardless_of_report_runs=True) + named = ActiveRun( + kind=RunKind.SIMULATION_RUN, + run_id=baseline, + status=RunStatus.RUNNING, + modal_call_id=CALL, + ) + left_alone = store.fail_stuck_run(named, error_message="stuck", failed_at=LATER) + failed = store.fail_stuck_run( + named, + error_message="stuck", + failed_at=LATER, + regardless_of_report_runs=True, + ) + + assert left_alone == RunWrite(changed=False, status=RunStatus.RUNNING) + assert failed == RunWrite(changed=True, status=RunStatus.FAILED) + assert seeder.row("simulation_runs", baseline)["error_message"] == "stuck" + # The session adds no primary key to find objects these updates changed. + sweep_updates = statements.starting_with("UPDATE public.simulation_runs")[1:] + assert len(sweep_updates) == 2 + for update_sent in sweep_updates: + assert update_sent.endswith("RETURNING public.simulation_runs.status") + + +def test_a_household_report_run_records_each_side_under_its_call( + household_store, seeder, household_report +) -> None: + report_run_id = household_report.report_run_id + baseline = household_report.baseline_simulation_run_id + reform = household_report.reform_simulation_run_id + + named = household_store.household_report_simulation_runs(report_run_id) + household_store.claim_report_run(report_run_id, modal_call_id=CALL, claimed_at=NOW) + for simulation_run_id in (baseline, reform): + household_store.claim_simulation_run( + simulation_run_id, modal_call_id=CALL, claimed_at=NOW + ) + completed = household_store.complete_household_simulation_run( + baseline, + modal_call_id=CALL, + household_result=HOUSEHOLD_DOCUMENT, + completed_at=LATER, + ) + failed = household_store.fail_household_simulation_run( + reform, modal_call_id=CALL, error_message="it broke", failed_at=LATER + ) + report = household_store.complete_household_report_run( + report_run_id, modal_call_id=CALL, completed_at=LATER + ) + + assert named == (baseline, reform) + assert completed == RunWrite(changed=True, status=RunStatus.SUCCEEDED) + assert failed == RunWrite(changed=True, status=RunStatus.FAILED) + assert report == RunWrite(changed=True, status=RunStatus.SUCCEEDED) + stored = seeder.row("simulation_runs", baseline) + assert stored["household_result"] == HOUSEHOLD_DOCUMENT + with pytest.raises(LookupError, match="does not exist"): + household_store.household_report_simulation_runs(uuid4()) diff --git a/libs/policyengine-stage12-persistence/tests/test_tables.py b/libs/policyengine-stage12-persistence/tests/test_tables.py index b95d1aefb..142f4c6ff 100644 --- a/libs/policyengine-stage12-persistence/tests/test_tables.py +++ b/libs/policyengine-stage12-persistence/tests/test_tables.py @@ -11,12 +11,15 @@ SIMULATION_CHECK_CONSTRAINTS, SIMULATION_COLUMNS, SIMULATION_IDENTITY_CONSTRAINT, - comparison_reports, - comparison_simulations, + ComparisonReport, + ComparisonSimulation, metadata, ) from sqlalchemy import ForeignKeyConstraint, String, UniqueConstraint +comparison_reports = ComparisonReport.__table__ +comparison_simulations = ComparisonSimulation.__table__ + def test_mappings_cover_every_cross_service_record_field() -> None: assert set(REPORT_COLUMNS) == set(ComparisonReportRecord.model_fields) diff --git a/libs/policyengine-stage12-persistence/tests/test_withheld_area_codes.py b/libs/policyengine-stage12-persistence/tests/test_withheld_area_codes.py index 01eae40ba..8c58de534 100644 --- a/libs/policyengine-stage12-persistence/tests/test_withheld_area_codes.py +++ b/libs/policyengine-stage12-persistence/tests/test_withheld_area_codes.py @@ -12,10 +12,8 @@ ) from policyengine_simulation_contract.macro_output import SingleYearMacroOutput from policyengine_simulation_contract.stage12_execution import ReportRunIds -from policyengine_stage12_persistence.report_run_results import ( - report_run_result, - withheld_area_codes, -) +from policyengine_simulation_persistence.result_tables import WithheldAreaCode +from policyengine_stage12_persistence.report_run_results import report_run_result RUN_IDS = ReportRunIds( report_run_id=UUID("00000000-0000-0000-0000-00000000000a"), @@ -31,7 +29,7 @@ def _withheld_rows(output: dict[str, Any]) -> list[dict[str, Any]]: result = report_run_result(output, run_ids=RUN_IDS, country="uk") - (rows,) = [rows for rows in result.rows if rows.table is withheld_area_codes] + (rows,) = [rows for rows in result.rows if rows.model is WithheldAreaCode] assert rows.shared_by_reports is False return [ {name: value for name, value in row.items() if name != "id"} @@ -55,7 +53,7 @@ def test_withheld_rows_have_their_own_ids() -> None: } result = report_run_result(output, run_ids=RUN_IDS, country="uk") - (rows,) = [rows for rows in result.rows if rows.table is withheld_area_codes] + (rows,) = [rows for rows in result.rows if rows.model is WithheldAreaCode] assert len({row["id"] for row in rows.rows}) == 2 diff --git a/projects/policyengine-simulation-entry/README.md b/projects/policyengine-simulation-entry/README.md index d4e20b444..db161ed07 100644 --- a/projects/policyengine-simulation-entry/README.md +++ b/projects/policyengine-simulation-entry/README.md @@ -98,14 +98,19 @@ default year, the policy sets parameters that catalog version lacks, or the household contract refuses the stored data. A job naming a row deleted since the reads is refused by the database's foreign keys, also with 400. The routes answer 503 with `Retry-After` when the manifest or the database -can't be read. When Modal refuses the call, or doesn't confirm it within ten -seconds, the job is failed at once and the 503 (`modal_call_refused`) names -it; a job the household function claimed meanwhile is answered with 202 as -it stands, and a job whose state couldn't then be recorded or read with 202 -`pending`, to be read rather than submitted again (the stuck-run sweep fails -it if no call ever starts). Every 500 of either route, from an unexpected -failure or a request the service built and the household contract refuses, -is the JSON error with code `internal_error`. `DEPLOYMENT.md` describes the deployment settings and the +can't be read and no job was written. When Modal refuses the call, or doesn't +confirm it within ten seconds, the job is failed at once and the 503 +(`modal_call_refused`) names it; a job the household function claimed +meanwhile is answered with 202 as it stands, and a job whose state couldn't +then be recorded or read with 202 `pending`, to be read rather than +submitted again (the stuck-run sweep fails it if no call ever starts). Every +500 of either route is the JSON error with code `internal_error`: an +unexpected failure, a request the service built and the household contract +refuses, or a job whose commit failed after its row was inserted. That job +may exist, so the 500 never says nothing was created; no call is started for +it, the stuck-run sweep fails it if it was written, and the failure is +logged as `household_job_outcome_unknown` with the commit error's type. +`DEPLOYMENT.md` describes the deployment settings and the staging check. ## Economy reports and simulations diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py index ba75fa1d8..b106e6c38 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py @@ -532,8 +532,8 @@ def household_calculations_for( ): return None return HouseholdCalculationService( - store=HouseholdJobStore.from_engine( - database.engine, + store=HouseholdJobStore.from_session_factory( + database.session_factory, statement_timeout_milliseconds=HOUSEHOLD_STATEMENT_TIMEOUT_MILLISECONDS, ), workers=ManifestHouseholdWorkers.from_modal( @@ -586,9 +586,9 @@ def economy_submissions_for( return EconomyRerunService( household_reports=household_reports, # The service starts a budget window's first years as its own role. - windows=BudgetWindowStore(engine=database.engine), - store=EconomyRunStore.from_engine( - database.engine, + windows=BudgetWindowStore(session_factory=database.session_factory), + store=EconomyRunStore.from_session_factory( + database.session_factory, statement_timeout_milliseconds=ECONOMY_STATEMENT_TIMEOUT_MILLISECONDS, ), workers=ManifestEconomyWorkers.from_modal( diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_api_models.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_api_models.py index 36e4bf49b..5a8888510 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_api_models.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_api_models.py @@ -33,17 +33,21 @@ "missing_reference", ] # Why a route cannot answer now (503); retry after ``Retry-After`` seconds. +# A submission answered with one of these created no job that will run. HouseholdUnavailableCode = Literal[ # The service has no database or v2 Modal environment configured. "not_configured", "manifest_unavailable", + # The database could not be read, or the job's insert failed: nothing + # was written. "database_unavailable", # Modal refused the household function's call, or did not accept it in # time; the job was failed at once. "modal_call_refused", ] -# The service failed unexpectedly, or built a household function request its -# contract refuses. +# The service failed unexpectedly, built a household function request its +# contract refuses, or could not confirm the commit of a job it may have +# created. HouseholdInternalCode = Literal["internal_error"] # Every code a household route's error answer carries. HouseholdErrorCode = Literal[ diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_calculations.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_calculations.py index bc085df3e..6835a9a4b 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_calculations.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_calculations.py @@ -9,7 +9,10 @@ worker's .py version, the stored household and the policy's reform values, and builds the household contract's request for the household's ``default_year``, with the new job's ID as its calculation ID; -3. creates the job ``pending`` with no Modal call ID; +3. creates the job ``pending`` with no Modal call ID. When the commit fails + after the job's row was inserted, PostgreSQL may have applied it: the job + may exist, so no call is started and the submission is answered as one + that may have created it; 4. starts the household function's call with ``spawn``, passing the job's ID, then records the call's ID while the job is still pending with none (the function may have claimed it already). When Modal refuses the call, @@ -35,6 +38,7 @@ HouseholdExecutionRequest, ) from policyengine_simulation_persistence import ( + EconomyRecordOutcomeUnknownError, HouseholdJobReferenceError, HouseholdJobStatus, MissingPolicyParametersError, @@ -130,6 +134,21 @@ class HouseholdCalculationInternalError(Exception): """ +class HouseholdCalculationOutcomeUnknownError(Exception): + """The job may have been created, but that can't be confirmed. + + The commit failed after the job's row was inserted, and no call was + started. The route answers 500, which the Public API reads as a job that + may exist, never a 503, which it reads as nothing created. + ``error_type`` is the commit error's type. + """ + + def __init__(self, *, error_type: str) -> None: + """Keep the commit error's type.""" + super().__init__("the household job may have been created") + self.error_type = error_type + + class HouseholdCalculationUnavailableError(Exception): """A dependency could not be read or refused the call; ask again later. @@ -351,8 +370,10 @@ async def submit( Raise HouseholdCalculationRefusedError (400) or HouseholdCalculationUnprocessableError (422) before writing anything, HouseholdCalculationInternalError (500) when the service builds a - request the contract refuses, and HouseholdCalculationUnavailableError - (503) when the manifest or database cannot be read, or Modal doesn't + request the contract refuses, HouseholdCalculationOutcomeUnknownError + (500) when the job's commit failed after its row was inserted, and + HouseholdCalculationUnavailableError (503) when the manifest or + database cannot be read and no job was written, or Modal doesn't confirm the call and the job was failed. """ worker = await self._default_worker(request.country) @@ -472,6 +493,11 @@ async def _create(self, job: NewHouseholdJob) -> None: raise HouseholdCalculationRefusedError( code="missing_reference", detail=detail ) from None + except EconomyRecordOutcomeUnknownError as error: + # The route logs it, with the commit error's type. + raise HouseholdCalculationOutcomeUnknownError( + error_type=type(error.__cause__).__name__ + ) from None except SQLAlchemyError as error: logger.warning( "household_job_not_created", diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_report_reruns.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_report_reruns.py index 127c8ab68..2a16e73dc 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_report_reruns.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_report_reruns.py @@ -310,8 +310,8 @@ def household_reports_for( ): return None return HouseholdReportRerunService( - store=HouseholdReportStore.from_engine( - database.engine, + store=HouseholdReportStore.from_session_factory( + database.session_factory, statement_timeout_milliseconds=STATEMENT_TIMEOUT_MILLISECONDS, ), workers=ManifestHouseholdWorkers.from_modal( diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_routes.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_routes.py index 660d4d13b..c58b172e1 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_routes.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_routes.py @@ -14,9 +14,11 @@ be calculated), naming why. Either route answers 503 with ``Retry-After`` when household calculations are not configured or a dependency cannot be read, and a submission does too when Modal refuses the call and its job was -failed. Every 500 of either route, an unexpected failure or a request the -service built and the household contract refuses, is the JSON error with -code ``internal_error``. +failed; a submission's 503 always means no job will run. Every 500 of either +route is the JSON error with code ``internal_error``: an unexpected failure, +a request the service built and the household contract refuses, or a +submission whose job's commit failed after its row was inserted, so the job +may exist. A submission gets its observability ID from the caller's header, or a new one, and passes it to the household function. ``household_jobs`` keeps no @@ -41,6 +43,7 @@ ) from policyengine_simulation_entry.household_calculations import ( HouseholdCalculationInternalError, + HouseholdCalculationOutcomeUnknownError, HouseholdCalculationRefusedError, HouseholdCalculationUnavailableError, HouseholdCalculationUnprocessableError, @@ -80,19 +83,20 @@ } _SUBMISSION_UNAVAILABLE = { "description": ( - "Household calculations are not configured; the v2 version manifest " - "or the database cannot be read; or Modal refused the household " - "function's call, or didn't accept it in time, and the job named by " - "household_job_id was failed (modal_call_refused). Retry after " - "Retry-After seconds" + "No job will run: household calculations are not configured; the v2 " + "version manifest or the database cannot be read, and no job was " + "written; or Modal refused the household function's call, or didn't " + "accept it in time, and the job named by household_job_id was failed " + "(modal_call_refused). Retry after Retry-After seconds" ), "headers": _RETRY_AFTER, **_ERROR, } _SUBMISSION_INTERNAL_ERROR = { "description": ( - "The service failed unexpectedly, or built a household function " - "request the household contract refuses (internal_error)" + "The service failed unexpectedly; built a household function request " + "the household contract refuses; or may have created the job but " + "could not confirm its commit, so the job may exist (internal_error)" ), **_ERROR, } @@ -171,10 +175,40 @@ def _unavailable(error: HouseholdCalculationUnavailableError) -> JSONResponse: ) +def _outcome_unknown_response( + error: HouseholdCalculationOutcomeUnknownError, +) -> JSONResponse: + """Answer a create whose commit can't be confirmed with the JSON 500. + + The Public API reads a 500 as a job that may exist; a 503 would tell it + nothing was created. The log is the fixed event + ``household_job_outcome_unknown`` with the commit error's type, never + its text, as ``economy_record_outcome_unknown`` logs. It names no job: + the observability runtime drops every attribute whose key contains + ``household``, and ``job_id`` names a Modal invocation. + """ + fields = {"error_type": error.error_type} + logger.error( + "household_job_outcome_unknown", + extra={**fields, "policyengine_attributes": fields}, + ) + return _error_response( + status.HTTP_500_INTERNAL_SERVER_ERROR, + HouseholdCalculationError( + code="internal_error", + detail=( + "The service may have created the household job, but could not " + "confirm it was written." + ), + ), + ) + + _SUBMISSION_FAILURES = ( HouseholdCalculationRefusedError, HouseholdCalculationUnprocessableError, HouseholdCalculationInternalError, + HouseholdCalculationOutcomeUnknownError, HouseholdCalculationUnavailableError, ) @@ -198,6 +232,8 @@ def _submission_failure_response(error: Exception) -> JSONResponse: ) if isinstance(error, HouseholdCalculationUnavailableError): return _unavailable(error) + if isinstance(error, HouseholdCalculationOutcomeUnknownError): + return _outcome_unknown_response(error) # The message names the refused fields the service built, never a value. logger.error("household_request_built_wrong: %s", error) return _error_response( diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/simulation_database.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/simulation_database.py index 7ef3be61c..86b57aa72 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/simulation_database.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/simulation_database.py @@ -11,7 +11,8 @@ from typing import TYPE_CHECKING from policyengine_simulation_persistence import ( - create_simulation_engine, + create_simulation_session_factory, + dispose_session_factory, read_missing_table_privileges, ) @@ -20,7 +21,7 @@ if TYPE_CHECKING: from collections.abc import Callable, Sequence - from sqlalchemy import Engine + from sqlalchemy.orm import Session, sessionmaker from policyengine_simulation_entry.entry_runtime import ( SimulationDatabaseConnection, @@ -90,9 +91,9 @@ def missing_privilege_log_messages( class SimulationDatabase: """Run short, blocking database operations off the event loop.""" - def __init__(self, engine: Engine) -> None: - """Use ``engine``, which connects as the simulation service's role.""" - self.engine = engine + def __init__(self, session_factory: sessionmaker[Session]) -> None: + """Use ``session_factory``, which connects as the service's role.""" + self.session_factory = session_factory # One thread of its own: waiting for a pooled connection, connecting # and a dead socket aren't bounded by the database's time limit, so a # stalled database may hold this thread, but never more than this one, @@ -103,11 +104,11 @@ def __init__(self, engine: Engine) -> None: @classmethod def from_url(cls, database_url: str) -> SimulationDatabase: - """Build the runtime engine; no connection opens until first use. + """Build the runtime sessions; no connection opens until first use. Raise ValueError, without echoing the URL, when it is malformed. """ - return cls(create_simulation_engine(database_url)) + return cls(create_simulation_session_factory(database_url)) async def missing_privileges(self) -> list[str]: """Return each table and column privilege the service needs and lacks.""" @@ -115,7 +116,7 @@ async def missing_privileges(self) -> list[str]: self.executor, partial( read_missing_table_privileges, - self.engine, + self.session_factory, statement_timeout_milliseconds=READINESS_STATEMENT_TIMEOUT_MILLISECONDS, ), ) @@ -129,7 +130,7 @@ def dispose(self) -> None: once, at shutdown. """ self.executor.shutdown(wait=False, cancel_futures=True) - self.engine.dispose() + dispose_session_factory(self.session_factory) class SimulationDatabaseReadinessCheck: diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/stage12_infrastructure.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/stage12_infrastructure.py index ccae557c9..97cf5cea0 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/stage12_infrastructure.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/stage12_infrastructure.py @@ -3,14 +3,49 @@ from __future__ import annotations import argparse +import os +import sys +import threading +from contextlib import suppress from pathlib import Path +from policyengine_simulation_persistence import dispose_session_factory from policyengine_stage12_persistence import ( Stage12RuntimeAccessError, - create_stage12_engine, + create_stage12_check_session_factory, validate_runtime_database, ) +# Room for every attempt and wait of the check's steps, each attempt connecting +# once, when the database's host name has up to four addresses with a connect +# timeout each. A connection that goes silent raises no error, so nothing else +# would end a read blocked on it. +CHECK_TIME_LIMIT_SECONDS = 900 +# How long the limit waits to report itself: standard error may be blocked. +TIME_LIMIT_MESSAGE_WAIT_SECONDS = 5 + + +def _report_the_time_limit() -> None: + with suppress(OSError, ValueError): + print( + "Stage 12 runtime database verification did not finish within " + f"{CHECK_TIME_LIMIT_SECONDS} seconds", + file=sys.stderr, + flush=True, + ) + + +def _end_the_stalled_check() -> None: + """Fail the deployment at once: cleaning up could block on the same read. + + The message is written on its own thread, so a blocked standard error can't + keep the process alive. + """ + report = threading.Thread(target=_report_the_time_limit, daemon=True) + report.start() + report.join(TIME_LIMIT_MESSAGE_WAIT_SECONDS) + os._exit(1) + def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser() @@ -28,14 +63,23 @@ def main() -> None: """ args = _parser().parse_args() database_url = args.database_url_file.read_text(encoding="utf-8").strip() + time_limit = threading.Timer(CHECK_TIME_LIMIT_SECONDS, _end_the_stalled_check) + time_limit.daemon = True + time_limit.start() try: - validate_runtime_database( - create_stage12_engine(database_url), - expected_role=args.expected_role, - environment=args.environment, - ) + session_factory = create_stage12_check_session_factory(database_url) + try: + validate_runtime_database( + session_factory, + expected_role=args.expected_role, + environment=args.environment, + ) + finally: + dispose_session_factory(session_factory) except (Stage12RuntimeAccessError, ValueError) as error: raise SystemExit(str(error)) from None + finally: + time_limit.cancel() print( "Shared API v2 runtime database access verified for Stage 12: " f"environment={args.environment}, role={args.expected_role}" diff --git a/projects/policyengine-simulation-entry/tests/orm_access_support.py b/projects/policyengine-simulation-entry/tests/orm_access_support.py new file mode 100644 index 000000000..d7ce28285 --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/orm_access_support.py @@ -0,0 +1,701 @@ +"""Checks behind sim.persistence.orm-only in rules/registry.toml. + +Database code declares tables only as SQLAlchemy ORM mapped classes and runs +every statement through an ORM Session against them. +``orm_access_findings`` reads source, so the rule test runs it on the +repository and the proof test runs it on the fixtures under +``rules/fixtures/sim.persistence.orm-only/``. ``unmapped_table_findings`` +reports every table that given modules hold and no mapped class maps, and +``unmapped_tables_in_packages`` imports whole packages for it, which finds +tables built in ways the source checks can't see. + +The source checks look for Core rather than for the ORM, so any statement on +mapped classes passes. Each file is checked on its own, through its own imports: +``from sqlalchemy import text`` and ``import sqlalchemy as sa`` both count, and +a ``text``, ``Table`` or ``update`` from another package doesn't. A Core table is +reported where it starts: where it is declared, where it is read from a mapped +class, mapper or MetaData or as the bare name ``__table__``, and where an +annotation names ``Table`` or ``TableClause``, apart from a class's bare +``__table__`` annotation in a declarative base module. Its later uses aren't +reported, because the rule has no baseline and the first finding already fails +it. + +A public function of an engine module may not return an Engine or Connection, +and only an engine module may read ``factory.kw["bind"]``. The rules that go by a +method or attribute name alone apply only in modules that import sqlalchemy: any +``.table`` read, ``kw["bind"]``, ``.execute()`` on anything not named as a +session, ``.scalar()``, ``.scalars()``, ``.stream()`` and ``.stream_scalars()`` on +a name such as ``connection``, and the ``.connect()``, ``.connection()``, +``.raw_connection()``, ``.get_bind()``, ``.begin()``, ``.cursor()`` and +``.exec_driver_sql()`` calls. + +The checks guard against ordinary Core use, not against deliberate evasion. They +don't see imports made through ``importlib`` or ``__import__``, engines or +connections passed around without an ``Engine`` or ``Connection`` annotation, so +a connection held under a name such as ``session`` isn't reported, a statement +method passed without being called, a function that returns an engine with no +return annotation, the engine an ``Inspector`` holds in ``bind`` or ``engine``, a +quoted name inside an annotation such as ``Optional["Table"]``, ``Table`` under +another local name such as ``Jobs = Table``, names from ``from sqlalchemy import +*``, a ``.table`` read in a module that doesn't import sqlalchemy, or a Session's +``bind`` attribute. ``.tables`` of +a MetaData isn't reported when the name holding it doesn't end in ``metadata`` and +wasn't assigned ``MetaData(...)``. +""" + +from __future__ import annotations + +import ast +import importlib +import pkgutil +import re +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator, Mapping + from types import ModuleType + +SQLALCHEMY_MODULE = "sqlalchemy" +SQLMODEL_MODULE = "sqlmodel" +# Core's table declarations, the lightweight table() and column() included. +# MetaData may be built only in a declarative base module, as the metadata of +# one of the library's DeclarativeBase classes. +TABLE_CONSTRUCTORS = frozenset( + {"Column", "ColumnClause", "MetaData", "Table", "TableClause", "column", "table"} +) +TABLE_TYPES = frozenset({"Table", "TableClause"}) +# A declarative base's classes may annotate what each of them maps. +MAPPED_TABLE_ATTRIBUTE = "__table__" +METADATA_CONSTRUCTOR = "MetaData" +DECLARATIVE_BASE_MODULE_NAME = "orm_base.py" +# The one module per library that may create and hold an engine, to hand out +# a sessionmaker. It may not open a Connection either. +SESSION_FACTORY_PATTERN = "libs/*/src/*/engine.py" +ENGINE_NAMES = frozenset( + { + "AsyncEngine", + "Engine", + "create_async_engine", + "create_engine", + "engine_from_config", + } +) +CONNECTION_NAMES = frozenset({"AsyncConnection", "Connection"}) +ENGINE_FACTORIES = frozenset({"create_async_engine", "create_engine"}) +ENGINE_TYPES = frozenset({"AsyncEngine", "Engine"}) +# A name such as engine or simulation_engine, but not policyengine. +ENGINE_NAME_PATTERN = re.compile(r"(?:^|_)engine$", re.IGNORECASE) +# Calls that open a Connection: on anything, and begin() on an engine. +CONNECTION_METHODS = frozenset({"connect", "connection", "get_bind", "raw_connection"}) +ENGINE_TRANSACTION_METHOD = "begin" +# execute runs only on a name such as session or store_session, which finds a +# connection passed around without a Connection annotation. A Result has scalar +# and scalars too, so the others are reported only on a name such as connection. +EXECUTE_METHOD = "execute" +SESSION_NAME_PATTERN = re.compile(r"(?:^|_)session$", re.IGNORECASE) +CONNECTION_STATEMENT_METHODS = frozenset( + {"scalar", "scalars", "stream", "stream_scalars"} +) +CONNECTION_NAME_PATTERN = re.compile(r"(?:^|_)(?:connection|conn)$", re.IGNORECASE) +# What an engine module keeps to itself: no public function returns one of +# these, and only it reads the engine out of a sessionmaker's keywords. +HELD_DATABASE_TYPES = ENGINE_TYPES | CONNECTION_NAMES +SESSION_FACTORY_KEYWORDS = "kw" +SESSION_FACTORY_BIND_KEY = "bind" +# Attributes that reach a Core table from a mapped class, its mapper or a +# MetaData, and the ones that do so only on a mapper, inspect() or a MetaData. +TABLE_ATTRIBUTES = frozenset( + {"__table__", "local_table", "persist_selectable", "sorted_tables"} +) +MAPPER_TABLE_ATTRIBUTES = frozenset({"selectable", "tables"}) +MAPPER_FUNCTIONS = frozenset({"class_mapper", "inspect", "object_mapper"}) +MAPPER_NAMES = frozenset({"__mapper__", "mapper"}) +# A column's or mapped attribute's table, as in HouseholdJob.id.table: read in a +# module that imports sqlalchemy, any .table is reported. +COLUMN_TABLE_ATTRIBUTE = "table" +METADATA_SUFFIX = "metadata" +RAW_SQL_FUNCTIONS = frozenset({"TextClause", "literal_column", "text"}) +RAW_SQL_METHODS = frozenset({"cursor", "exec_driver_sql"}) +# A model's schema details given as SQL text: a server default and a partial +# index's condition, by keyword, and a check constraint's condition. +SCHEMA_DETAIL_KEYWORDS = frozenset( + {"postgresql_where", "server_default", "server_onupdate", "sqlite_where"} +) +CHECK_CONSTRAINT = "CheckConstraint" + +CORE_TABLE_DECLARATION = "Core table declaration" +CORE_TABLE_ACCESS = "Core table access" +CORE_CONNECTION = "Core engine or connection" +RAW_SQL = "raw SQL" +SQLMODEL_IMPORT = "sqlmodel import" +UNMAPPED_TABLE = "table without a mapped class" + + +@dataclass(frozen=True) +class Finding: + """One place where code reaches the database without the ORM.""" + + file: str + line: int + check: str + source: str + + def __str__(self) -> str: + """Describe the finding as ``file:line: check: source``.""" + return f"{self.file}:{self.line}: {self.check}: {self.source}" + + +def _matches_module(module_name: str, parent_module: str) -> bool: + return module_name == parent_module or module_name.startswith(f"{parent_module}.") + + +def _sqlalchemy_imports(tree: ast.Module) -> dict[str, str]: + """Map each name the module imports from sqlalchemy to its dotted name.""" + imported: dict[str, str] = {} + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + if not _matches_module(alias.name, SQLALCHEMY_MODULE): + continue + if alias.asname: + imported[alias.asname] = alias.name + else: + imported[alias.name.split(".")[0]] = SQLALCHEMY_MODULE + elif ( + isinstance(node, ast.ImportFrom) + and not node.level + and node.module is not None + and _matches_module(node.module, SQLALCHEMY_MODULE) + ): + for alias in node.names: + imported[alias.asname or alias.name] = f"{node.module}.{alias.name}" + return imported + + +def _qualified_name(node: ast.expr, imported: Mapping[str, str]) -> str | None: + if isinstance(node, ast.Name): + return imported.get(node.id) + if isinstance(node, ast.Attribute): + parent = _qualified_name(node.value, imported) + return None if parent is None else f"{parent}.{node.attr}" + return None + + +def _sqlalchemy_name(node: ast.expr, imported: Mapping[str, str]) -> str: + """Return the last part of the sqlalchemy name ``node`` refers to, or ``""``.""" + qualified = _qualified_name(node, imported) + return qualified.rsplit(".", 1)[-1] if qualified else "" + + +def _annotation_names( + annotation: ast.expr | None, imported: Mapping[str, str] +) -> set[str]: + """Return the sqlalchemy names an annotation refers to, unions included. + + A wholly quoted annotation is parsed. Strings inside one aren't, because + they may be anything, such as ``Literal`` values or ``Annotated`` metadata. + """ + if annotation is None: + return set() + if isinstance(annotation, ast.Constant) and isinstance(annotation.value, str): + try: + annotation = ast.parse(annotation.value, mode="eval").body + except SyntaxError: + return set() + return { + name + for node in ast.walk(annotation) + if isinstance(node, (ast.Name, ast.Attribute)) + and (name := _sqlalchemy_name(node, imported)) + } + + +def _bound_targets(target: ast.expr) -> list[str]: + """Return the names and attributes an assignment target binds.""" + if isinstance(target, ast.Name): + return [target.id] + if isinstance(target, ast.Attribute): + return [target.attr] + if isinstance(target, (ast.Tuple, ast.List)): + return [name for element in target.elts for name in _bound_targets(element)] + return [] + + +def _last_name(node: ast.expr) -> str: + if isinstance(node, ast.Call): + return _last_name(node.func) + if isinstance(node, ast.Attribute): + return node.attr + return node.id if isinstance(node, ast.Name) else "" + + +@dataclass +class _Module: + """One scanned file and the names it imports from sqlalchemy.""" + + file: str + tree: ast.Module + imported: dict[str, str] + + @property + def uses_sqlalchemy(self) -> bool: + """Whether the module imports anything from sqlalchemy.""" + return bool(self.imported) + + @property + def is_session_factory(self) -> bool: + return PurePosixPath(self.file).match(SESSION_FACTORY_PATTERN) + + @property + def is_declarative_base(self) -> bool: + return PurePosixPath(self.file).name == DECLARATIVE_BASE_MODULE_NAME + + +def _annotated_as(module: _Module, types: frozenset[str]) -> set[str]: + """Return the names and attributes annotated with one of ``types``.""" + names: set[str] = set() + for node in ast.walk(module.tree): + if isinstance(node, ast.arg): + if types & _annotation_names(node.annotation, module.imported): + names.add(node.arg) + elif isinstance(node, ast.AnnAssign) and types & _annotation_names( + node.annotation, module.imported + ): + names.update(_bound_targets(node.target)) + return names + + +def _bound_to_calls(module: _Module, functions: frozenset[str]) -> set[str]: + """Return the names and attributes bound to a call of one of ``functions``.""" + return { + name + for target, value in _assignment_nodes(module.tree) + if isinstance(value, ast.Call) + and _sqlalchemy_name(value.func, module.imported) in functions + for name in _bound_targets(target) + } + + +def _paired(target: ast.expr, value: ast.expr) -> Iterator[tuple[ast.expr, ast.expr]]: + """Pair each part of a tuple target with its value, nested tuples included.""" + if ( + isinstance(target, (ast.Tuple, ast.List)) + and isinstance(value, (ast.Tuple, ast.List)) + and len(target.elts) == len(value.elts) + ): + for element, element_value in zip(target.elts, value.elts, strict=True): + yield from _paired(element, element_value) + else: + yield target, value + + +def _assignment_nodes(tree: ast.AST) -> Iterator[tuple[ast.expr, ast.expr]]: + """Yield each assignment's target and value, pairing tuples. + + ``a, b = x, y`` yields ``a`` with ``x`` and ``b`` with ``y``. + """ + for node in ast.walk(tree): + if isinstance(node, ast.Assign): + for target in node.targets: + yield from _paired(target, node.value) + elif (isinstance(node, ast.AnnAssign) and node.value is not None) or isinstance( + node, ast.NamedExpr + ): + yield node.target, node.value + + +def _bound_to_mappers(module: _Module) -> set[str]: + """Return the names and attributes assigned a mapper, as in ``m = X.__mapper__``.""" + return { + name + for target, value in _assignment_nodes(module.tree) + if isinstance(value, ast.Attribute) and value.attr in MAPPER_NAMES + for name in _bound_targets(target) + } + + +def _call_summary(call: ast.Call) -> str: + parts = [ast.unparse(call.args[0])] if call.args else [] + if len(call.args) > 1 or call.keywords: + parts.append("...") + return f"{ast.unparse(call.func)}({', '.join(parts)})" + + +def _schema_detail_calls(tree: ast.Module, imported: Mapping[str, str]) -> set[int]: + """Return the calls given as a default or as an index's or check's condition.""" + details = { + id(keyword.value) + for node in ast.walk(tree) + if isinstance(node, ast.Call) + for keyword in node.keywords + if keyword.arg in SCHEMA_DETAIL_KEYWORDS + } + details.update( + id(node.args[0]) + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and node.args + and _sqlalchemy_name(node.func, imported) == CHECK_CONSTRAINT + ) + return details + + +@dataclass(frozen=True) +class _ModuleContext: + module: _Module + metadata_names: set[str] + mapper_names: set[str] + engine_names: set[str] + schema_details: set[int] + called: set[int] + mapped_table_annotations: set[int] + + +def _forbidden_connection_name(name: str, context: _ModuleContext) -> bool: + if name in CONNECTION_NAMES: + return True + return name in ENGINE_NAMES and not context.module.is_session_factory + + +def _import_findings( + node: ast.Import | ast.ImportFrom, context: _ModuleContext +) -> Iterator[tuple[str, str]]: + if isinstance(node, ast.Import): + for alias in node.names: + if _matches_module(alias.name, SQLMODEL_MODULE): + yield SQLMODEL_IMPORT, alias.name + return + if node.level or node.module is None: + return + if _matches_module(node.module, SQLMODEL_MODULE): + yield SQLMODEL_IMPORT, node.module + elif _matches_module(node.module, SQLALCHEMY_MODULE): + for alias in node.names: + if _forbidden_connection_name(alias.name, context): + yield CORE_CONNECTION, f"{node.module}.{alias.name}" + + +def _reaches_mapped_table(node: ast.Attribute, context: _ModuleContext) -> bool: + if node.attr in TABLE_ATTRIBUTES: + return True + receiver = node.value + if node.attr == COLUMN_TABLE_ATTRIBUTE: + # A column's table is read, never called: sa.table(...) is a function. + return ( + context.module.uses_sqlalchemy + and isinstance(node.ctx, ast.Load) + and id(node) not in context.called + ) + if node.attr not in MAPPER_TABLE_ATTRIBUTES: + return False + if isinstance(receiver, ast.Call): + function = _sqlalchemy_name(receiver.func, context.module.imported) + return function in MAPPER_FUNCTIONS + receiver_name = _last_name(receiver) + return ( + receiver_name in MAPPER_NAMES + or receiver_name in context.mapper_names + or receiver_name in context.metadata_names + or receiver_name.lower().endswith(METADATA_SUFFIX) + ) + + +def _attribute_findings( + node: ast.Attribute, context: _ModuleContext +) -> Iterator[tuple[str, str]]: + if _reaches_mapped_table(node, context): + yield CORE_TABLE_ACCESS, ast.unparse(node) + elif _forbidden_connection_name( + _sqlalchemy_name(node, context.module.imported), context + ): + yield CORE_CONNECTION, str(_qualified_name(node, context.module.imported)) + + +def _runs_raw_sql(node: ast.Call, function: str, context: _ModuleContext) -> bool: + if function in RAW_SQL_FUNCTIONS: + return id(node) not in context.schema_details + return ( + context.module.uses_sqlalchemy + and isinstance(node.func, ast.Attribute) + and node.func.attr in RAW_SQL_METHODS + ) + + +def _opens_connection(node: ast.Call, context: _ModuleContext) -> bool: + if not context.module.uses_sqlalchemy or not isinstance(node.func, ast.Attribute): + return False + if node.func.attr in CONNECTION_METHODS: + return True + if node.func.attr != ENGINE_TRANSACTION_METHOD: + return False + receiver = _last_name(node.func.value) + return receiver in context.engine_names or bool( + ENGINE_NAME_PATTERN.search(receiver) + ) + + +def _runs_outside_a_session(node: ast.Call, context: _ModuleContext) -> bool: + """Whether a statement runs on something other than a session.""" + if not context.module.uses_sqlalchemy or not isinstance(node.func, ast.Attribute): + return False + receiver = _last_name(node.func.value) + if node.func.attr == EXECUTE_METHOD: + return not SESSION_NAME_PATTERN.search(receiver) + return node.func.attr in CONNECTION_STATEMENT_METHODS and bool( + CONNECTION_NAME_PATTERN.search(receiver) + ) + + +def _call_findings( + node: ast.Call, context: _ModuleContext +) -> Iterator[tuple[str, str]]: + function = _sqlalchemy_name(node.func, context.module.imported) + if function in TABLE_CONSTRUCTORS and not ( + function == METADATA_CONSTRUCTOR and context.module.is_declarative_base + ): + yield CORE_TABLE_DECLARATION, _call_summary(node) + elif _runs_raw_sql(node, function, context): + yield RAW_SQL, _call_summary(node) + elif _opens_connection(node, context): + yield CORE_CONNECTION, _call_summary(node) + elif _runs_outside_a_session(node, context): + yield CORE_CONNECTION, f"{ast.unparse(node.func)}(...)" + + +def _annotation_text(annotation: ast.expr) -> str: + if isinstance(annotation, ast.Constant) and isinstance(annotation.value, str): + return annotation.value + return ast.unparse(annotation) + + +def _table_annotation_findings( + label: str, annotation: ast.expr | None, context: _ModuleContext +) -> list[tuple[str, str]]: + """Report an annotation naming a Core table type, as in ``jobs: Table``.""" + names = _annotation_names(annotation, context.module.imported) + if annotation is None or not names & TABLE_TYPES: + return [] + return [(CORE_TABLE_ACCESS, f"{label}{_annotation_text(annotation)}")] + + +def _class_level_statements(node: ast.ClassDef) -> Iterator[ast.stmt]: + """Yield a class's own statements, including those under an ``if``.""" + pending: list[ast.stmt] = list(node.body) + while pending: + statement = pending.pop() + yield statement + if isinstance(statement, ast.If): + pending.extend([*statement.body, *statement.orelse]) + + +def _mapped_table_annotations(module: _Module) -> set[int]: + """Return each class's bare ``__table__`` annotation in a declarative base.""" + if not module.is_declarative_base: + return set() + return { + id(statement) + for node in ast.walk(module.tree) + if isinstance(node, ast.ClassDef) + for statement in _class_level_statements(node) + if isinstance(statement, ast.AnnAssign) + and statement.value is None + and isinstance(statement.target, ast.Name) + and statement.target.id == MAPPED_TABLE_ATTRIBUTE + } + + +def _annotated_assignment_findings( + node: ast.AnnAssign, context: _ModuleContext +) -> list[tuple[str, str]]: + if id(node) in context.mapped_table_annotations: + return [] + return _table_annotation_findings( + f"{ast.unparse(node.target)}: ", node.annotation, context + ) + + +def _function_findings( + node: ast.FunctionDef | ast.AsyncFunctionDef, context: _ModuleContext +) -> list[tuple[str, str]]: + """Report a function returning a Core table, or an engine module's engine.""" + findings = _table_annotation_findings(f"{node.name}() -> ", node.returns, context) + if not context.module.is_session_factory or node.name.startswith("_"): + return findings + returned = _annotation_names(node.returns, context.module.imported) + return findings + [ + (CORE_CONNECTION, f"{node.name}() -> {type_name}") + for type_name in sorted(returned & HELD_DATABASE_TYPES) + ] + + +def _reads_session_factory_engine(node: ast.Subscript, context: _ModuleContext) -> bool: + """Whether ``node`` is ``factory.kw["bind"]`` outside an engine module.""" + return ( + context.module.uses_sqlalchemy + and not context.module.is_session_factory + and isinstance(node.value, ast.Attribute) + and node.value.attr == SESSION_FACTORY_KEYWORDS + and isinstance(node.slice, ast.Constant) + and node.slice.value == SESSION_FACTORY_BIND_KEY + ) + + +def _subscript_findings( + node: ast.Subscript, context: _ModuleContext +) -> list[tuple[str, str]]: + if _reads_session_factory_engine(node, context): + return [(CORE_CONNECTION, ast.unparse(node))] + return [] + + +def _name_findings(node: ast.Name) -> list[tuple[str, str]]: + """Report a read of the bare name ``__table__``, as in a class body.""" + if node.id == MAPPED_TABLE_ATTRIBUTE and isinstance(node.ctx, ast.Load): + return [(CORE_TABLE_ACCESS, node.id)] + return [] + + +def _node_findings(node: ast.AST, context: _ModuleContext) -> list[tuple[str, str]]: + findings: Iterable[tuple[str, str]] = () + match node: + case ast.Import() | ast.ImportFrom(): + findings = _import_findings(node, context) + case ast.Name(): + findings = _name_findings(node) + case ast.Attribute(): + findings = _attribute_findings(node, context) + case ast.Call(): + findings = _call_findings(node, context) + case ast.FunctionDef() | ast.AsyncFunctionDef(): + findings = _function_findings(node, context) + case ast.arg(): + findings = _table_annotation_findings( + f"{node.arg}: ", node.annotation, context + ) + case ast.AnnAssign(): + findings = _annotated_assignment_findings(node, context) + case ast.Subscript(): + findings = _subscript_findings(node, context) + return list(findings) + + +def _module_context(module: _Module) -> _ModuleContext: + return _ModuleContext( + module=module, + metadata_names=_bound_to_calls(module, frozenset({METADATA_CONSTRUCTOR})), + mapper_names=_bound_to_calls(module, MAPPER_FUNCTIONS) + | _bound_to_mappers(module), + engine_names=_bound_to_calls(module, ENGINE_FACTORIES) + | _annotated_as(module, ENGINE_TYPES), + schema_details=_schema_detail_calls(module.tree, module.imported), + called={ + id(node.func) + for node in ast.walk(module.tree) + if isinstance(node, ast.Call) + }, + mapped_table_annotations=_mapped_table_annotations(module), + ) + + +def orm_access_findings(sources: Mapping[str, str]) -> list[Finding]: + """Return every place the sources reach the database without the ORM. + + ``sources`` maps a repository-relative file name to its source. + """ + findings: list[Finding] = [] + for file, source in sorted(sources.items()): + tree = ast.parse(source, filename=file) + module = _Module(file=file, tree=tree, imported=_sqlalchemy_imports(tree)) + context = _module_context(module) + findings.extend( + Finding(file, getattr(node, "lineno", 0), check, finding_source) + for node in ast.walk(tree) + for check, finding_source in _node_findings(node, context) + ) + return sorted(findings, key=lambda finding: (finding.file, finding.line)) + + +def _relative_file(path: str | None, repository_root: Path) -> str: + if path is None: + return "" + return Path(path).resolve().relative_to(repository_root).as_posix() + + +def unmapped_table_findings( + modules: Iterable[ModuleType], repository_root: Path, *, location: str | None = None +) -> list[Finding]: + """Return each table the modules hold that no mapped class maps. + + A module holds a table through a global bound to the table, to a MetaData, + or to a declarative base or mapped class. Each table is reported once, at + ``location`` when given, otherwise at the first module that holds it. + """ + return _unmapped_findings( + (module, location or _relative_file(module.__file__, repository_root)) + for module in modules + ) + + +def _unmapped_findings( + placed_modules: Iterable[tuple[ModuleType, str]], +) -> list[Finding]: + """Report each held table no mapped class of any of the modules maps.""" + from sqlalchemy import MetaData, Table + from sqlalchemy.orm import DeclarativeBase, DeclarativeBaseNoMeta + + held: dict[int, tuple[Table, str]] = {} + mapped: set[int] = set() + for module, place in placed_modules: + for value in vars(module).values(): + if isinstance(value, Table): + tables = [value] + elif isinstance(value, MetaData): + tables = list(value.tables.values()) + elif ( + isinstance(value, type) + and issubclass(value, (DeclarativeBase, DeclarativeBaseNoMeta)) + and not value.__module__.startswith(SQLALCHEMY_MODULE) + ): + mapped.update( + id(mapper.local_table) for mapper in value.registry.mappers + ) + tables = list(value.registry.metadata.tables.values()) + else: + tables = [] + for table in tables: + held.setdefault(id(table), (table, place)) + findings = [ + Finding(place, 0, UNMAPPED_TABLE, table.fullname) + for key, (table, place) in held.items() + if key not in mapped + ] + return sorted(findings, key=lambda finding: (finding.file, finding.source)) + + +def _package_modules(package_name: str) -> list[ModuleType]: + package = importlib.import_module(package_name) + names = sorted( + module.name + for module in pkgutil.walk_packages(package.__path__, f"{package_name}.") + ) + return [package, *(importlib.import_module(name) for name in names)] + + +def unmapped_tables_in_packages( + package_names: Iterable[str], repository_root: Path +) -> list[Finding]: + """Import the packages and return each table they hold that no class maps. + + A table is reported once, at the source directory of the first package + that holds it, in the order given. + """ + placed_modules: list[tuple[ModuleType, str]] = [] + for package_name in package_names: + package_modules = _package_modules(package_name) + directory = PurePosixPath( + _relative_file(package_modules[0].__file__, repository_root) + ).parent + placed_modules.extend((module, str(directory)) for module in package_modules) + return _unmapped_findings(placed_modules) diff --git a/projects/policyengine-simulation-entry/tests/test_economy_routes.py b/projects/policyengine-simulation-entry/tests/test_economy_routes.py index 65886fbfe..e584830a1 100644 --- a/projects/policyengine-simulation-entry/tests/test_economy_routes.py +++ b/projects/policyengine-simulation-entry/tests/test_economy_routes.py @@ -559,6 +559,40 @@ def test_a_configured_service_builds_economy_submissions_without_connecting(): assert isinstance(submissions, EconomySubmissionService) +def test_the_economy_store_runs_in_sessions_on_the_service_database(monkeypatch): + """Each operation is a session on the database's engine, with the same limit.""" + stores_built: list[tuple[object, int]] = [] + + def build_store(session_factory, *, statement_timeout_milliseconds): + stores_built.append((session_factory, statement_timeout_milliseconds)) + return object() + + monkeypatch.setattr( + app_module.EconomyRunStore, + "from_session_factory", + build_store, + raising=False, + ) + database = SimulationDatabase.from_url(DATABASE_URL) + try: + app_module.economy_submissions_for( + make_settings( + simulation_database_url=DATABASE_URL, + v2_modal_environment="staging", + ), + database, + ) + finally: + database.dispose() + + assert stores_built == [ + ( + database.session_factory, + app_module.ECONOMY_STATEMENT_TIMEOUT_MILLISECONDS, + ) + ] + + @pytest.mark.parametrize( "settings_changes", [ diff --git a/projects/policyengine-simulation-entry/tests/test_household_calculations.py b/projects/policyengine-simulation-entry/tests/test_household_calculations.py index dd7aa455c..a9b0d0597 100644 --- a/projects/policyengine-simulation-entry/tests/test_household_calculations.py +++ b/projects/policyengine-simulation-entry/tests/test_household_calculations.py @@ -23,6 +23,7 @@ ) from policyengine_simulation_entry.household_calculations import ( HouseholdCalculationInternalError, + HouseholdCalculationOutcomeUnknownError, HouseholdCalculationRefusedError, HouseholdCalculationService, HouseholdCalculationUnavailableError, @@ -36,6 +37,7 @@ default_household_worker, ) from policyengine_simulation_persistence import ( + EconomyRecordOutcomeUnknownError, HouseholdCalculationInputs, HouseholdJobRecord, HouseholdJobReferenceError, @@ -623,6 +625,39 @@ async def test_a_row_deleted_before_the_job_is_created_is_named(column_name, det assert caller.calls == [] +@pytest.mark.asyncio +async def test_a_job_whose_commit_is_unconfirmed_may_exist_and_starts_no_call( + caplog, +): + """The commit may have been applied, so no 503 may say nothing was created.""" + lost_commit = EconomyRecordOutcomeUnknownError("the commit failed") + lost_commit.__cause__ = DATABASE_ERROR + service, _, _, caller = _service(FakeStore(create_failure=lost_commit)) + + with pytest.raises(HouseholdCalculationOutcomeUnknownError) as raised: + await service.submit(_request(), observability_context=None) + + assert raised.value.error_type == "OperationalError" + assert caller.calls == [] + # The route logs the outcome, once; the service logs no failed create. + assert "household_job_not_created" not in caplog.messages + + +@pytest.mark.asyncio +async def test_a_job_whose_insert_fails_is_logged_by_the_service(caplog): + """The service logs a create that wrote nothing, unlike an unconfirmed one. + + The test above relies on this event to show that the service logs no + unconfirmed commit. + """ + service, _, _, _ = _service(FakeStore(create_failure=DATABASE_ERROR)) + + with pytest.raises(HouseholdCalculationUnavailableError): + await service.submit(_request(), observability_context=None) + + assert caplog.messages.count("household_job_not_created") == 1 + + @pytest.mark.asyncio async def test_a_call_modal_refuses_fails_its_job_at_once(): service, store, _, _ = _service( diff --git a/projects/policyengine-simulation-entry/tests/test_household_report_reruns.py b/projects/policyengine-simulation-entry/tests/test_household_report_reruns.py index 9bae4879d..33893629b 100644 --- a/projects/policyengine-simulation-entry/tests/test_household_report_reruns.py +++ b/projects/policyengine-simulation-entry/tests/test_household_report_reruns.py @@ -51,6 +51,7 @@ HouseholdReportCreated, ) from policyengine_simulation_entry.household_report_reruns import ( + STATEMENT_TIMEOUT_MILLISECONDS, HouseholdReportRerunService, ) from policyengine_simulation_entry.household_workers import ( @@ -61,6 +62,7 @@ EconomyRecordOutcomeUnknownError, EconomyRecordReferenceError, FoundReportRun, + HouseholdReportStore, HouseholdRerunSide, MissingPolicyParametersError, NewHouseholdReportRerun, @@ -566,6 +568,31 @@ def test_the_app_reruns_household_reports_through_its_household_report_service( assert builders_given == [{"household_reports": household_reports}] +def test_the_household_report_store_runs_in_sessions_on_the_service_database( + monkeypatch, +) -> None: + """Each operation is a session on the database's engine, with the same limit.""" + stores_built: list[tuple[object, int]] = [] + + def build_store(session_factory, *, statement_timeout_milliseconds): + stores_built.append((session_factory, statement_timeout_milliseconds)) + return object() + + monkeypatch.setattr(HouseholdReportStore, "from_session_factory", build_store) + database = SimulationDatabase.from_url(DATABASE_URL) + try: + app_module.household_reports_for( + make_settings( + simulation_database_url=DATABASE_URL, v2_modal_environment="staging" + ), + database, + ) + finally: + database.dispose() + + assert stores_built == [(database.session_factory, STATEMENT_TIMEOUT_MILLISECONDS)] + + def test_a_configured_service_builds_a_household_report_service_that_reruns() -> None: """Nothing reaches the database or Modal until the first submission.""" database = SimulationDatabase.from_url(DATABASE_URL) diff --git a/projects/policyengine-simulation-entry/tests/test_household_routes.py b/projects/policyengine-simulation-entry/tests/test_household_routes.py index 3c1863336..ab431ca1d 100644 --- a/projects/policyengine-simulation-entry/tests/test_household_routes.py +++ b/projects/policyengine-simulation-entry/tests/test_household_routes.py @@ -21,6 +21,7 @@ ) from policyengine_simulation_entry.household_calculations import ( HouseholdCalculationInternalError, + HouseholdCalculationOutcomeUnknownError, HouseholdCalculationRefusedError, HouseholdCalculationService, HouseholdCalculationUnavailableError, @@ -400,6 +401,39 @@ def test_a_job_id_that_is_not_a_uuid_is_refused(client_for): DATABASE_URL = "postgresql://simulation_service:secret@db.example:5432/postgres" +def test_the_household_store_runs_in_sessions_on_the_service_database(monkeypatch): + """Each operation is a session on the database's engine, with the same limit.""" + stores_built: list[tuple[object, int]] = [] + + def build_store(session_factory, *, statement_timeout_milliseconds): + stores_built.append((session_factory, statement_timeout_milliseconds)) + return object() + + monkeypatch.setattr( + app_module.HouseholdJobStore, "from_session_factory", build_store + ) + database = SimulationDatabase.from_url(DATABASE_URL) + try: + app_module.household_calculations_for( + make_settings( + simulation_database_url=DATABASE_URL, v2_modal_environment="staging" + ), + database, + ) + with database.session_factory() as session: + session_engine = session.get_bind() + finally: + database.dispose() + + assert stores_built == [ + ( + database.session_factory, + app_module.HOUSEHOLD_STATEMENT_TIMEOUT_MILLISECONDS, + ) + ] + assert session_engine is database.session_factory.kw["bind"] + + def test_a_configured_service_builds_household_calculations_without_connecting(): """Nothing reaches the database or Modal until the first submission.""" database = SimulationDatabase.from_url(DATABASE_URL) @@ -503,6 +537,44 @@ def test_an_unexpected_failure_answers_a_json_500_without_its_values( assert "a value that must not be logged" not in printed +def test_a_job_whose_commit_is_unconfirmed_answers_a_500_and_is_logged( + client_for, capsys +): + """A 500 tells the Public API the job may exist; a 503 would say it doesn't.""" + calculations = FakeHouseholdCalculations( + failure=HouseholdCalculationOutcomeUnknownError(error_type="OperationalError") + ) + + with client_for(calculations) as client: + result = client.post( + "/household-calculations", + json=SUBMISSION, + headers={OBSERVABILITY_ID_HEADER: OBSERVABILITY_ID}, + ) + + assert result.status_code == 500 + assert "Retry-After" not in result.headers + assert result.json() == { + "code": "internal_error", + "detail": ( + "The service may have created the household job, but could not " + "confirm it was written." + ), + } + printed = capsys.readouterr().out + (record,) = _emitted_records(printed, "household_job_outcome_unknown") + # Every attribute was kept, none with a key the runtime drops, and the + # submission's observability ID ties the event to its request. + assert record["attributes"] == { + "error_type": "OperationalError", + "observability_id": OBSERVABILITY_ID, + } + assert "attributes.omitted_count" not in record + assert record["severity"] == "ERROR" + # One event for one outcome. + assert _emitted_records(printed, "household_route_failed") == [] + + # A string limit that holds a few frames only, and enough calls to pass it. SMALL_STRING_LIMIT = 600 NESTED_CALLS = 20 diff --git a/projects/policyengine-simulation-entry/tests/test_orm_access_support.py b/projects/policyengine-simulation-entry/tests/test_orm_access_support.py new file mode 100644 index 000000000..31952d86d --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_orm_access_support.py @@ -0,0 +1,757 @@ +"""Unit tests for the checks behind sim.persistence.orm-only.""" + +from __future__ import annotations + +import sys +import types +from pathlib import Path +from textwrap import dedent + +import pytest +from orm_access_support import ( + CORE_CONNECTION, + CORE_TABLE_ACCESS, + CORE_TABLE_DECLARATION, + RAW_SQL, + SQLMODEL_IMPORT, + UNMAPPED_TABLE, + orm_access_findings, + unmapped_table_findings, +) + +STORE = "libs/example/src/example/store.py" +ENGINE = "libs/example/src/example/engine.py" +ORM_BASE = "libs/example/src/example/orm_base.py" +GATEWAY = "projects/gateway/src/gateway/regions.py" +MODELS = "libs/example/src/example/models.py" + + +def _checks(sources: dict[str, str], file: str) -> list[tuple[str, str]]: + """Return the (check, source) findings in ``file``.""" + findings = orm_access_findings( + {name: dedent(source) for name, source in sources.items()} + ) + return [ + (finding.check, finding.source) for finding in findings if finding.file == file + ] + + +@pytest.mark.parametrize( + "statement", + [ + "select(func.count().label('jobs'))", + "select(cast(HouseholdJob.year, String).label('year'))", + "select(exists().where(HouseholdJob.id == job_id))", + "select(*columns)", + "select(pending.c.id).where(pending.c.id == job_id)", + "select(func.count()).select_from(HouseholdJob).join(Policy)", + "insert(HouseholdJob).values(id=job_id).on_conflict_do_nothing()", + "select(reports.c.id)", + ], +) +def test_orm_statements_are_accepted(statement): + store = f""" + from sqlalchemy import String, cast, exists, func, insert, select + + pending = select(HouseholdJob.id).subquery() + reports = select(Report.id).where(Report.year == 2026).subquery() + statement = {statement} + """ + + assert _checks({STORE: store}, STORE) == [] + + +@pytest.mark.parametrize( + ("declaration", "summary"), + [ + ("def read(jobs: Table): ...", "jobs: Table"), + ("def read(jobs: TableClause): ...", "jobs: TableClause"), + ("def read(jobs: 'sa.Table | None'): ...", "jobs: sa.Table | None"), + ("def read(*tables: Table): ...", "tables: Table"), + ("def read(**tables: Table): ...", "tables: Table"), + ("def read() -> Table: ...", "read() -> Table"), + ("async def read() -> list[Table]: ...", "read() -> list[Table]"), + ("selected: Table = choose()", "selected: Table"), + ("self.jobs: Table = source", "self.jobs: Table"), + ("class Rows:\n target: Table", "target: Table"), + ( + "def read(jobs: Annotated[Any, Marker(kind=Table)]): ...", + "jobs: Annotated[Any, Marker(kind=Table)]", + ), + ("def read(jobs: Literal[Table]): ...", "jobs: Literal[Table]"), + ], +) +def test_an_annotation_naming_a_core_table_type_is_reported(declaration, summary): + store = f""" +import sqlalchemy as sa +from sqlalchemy import Table, TableClause + +{declaration} +""" + + assert _checks({STORE: store}, STORE) == [(CORE_TABLE_ACCESS, summary)] + + +@pytest.mark.parametrize( + "annotation", + [ + 'Literal["Table"]', + 'Choice["Table", "View"]', + 'Annotated[str, "Table"]', + 'Annotated[str, "' + "x." * 1000 + 'y"]', + 'Optional["Table"]', + "\"list['Table']\"", + '"not a type ("', + ], +) +def test_a_quoted_name_inside_an_annotation_is_not_read(annotation): + """Only a wholly quoted annotation is parsed; other strings may be anything.""" + store = f""" +from sqlalchemy import Table + +def read(value: {annotation}): ... +""" + + assert _checks({STORE: store}, STORE) == [] + + +def test_an_engine_type_in_annotation_metadata_is_followed(): + source = """ + from sqlalchemy import Engine + + def hand_out() -> Annotated[Any, Marker(kind=Engine)]: ... + + def run(handle: Annotated[Any, Marker(kind=Engine)]): + return handle.begin() + """ + + assert _checks({ENGINE: source}, ENGINE) == [ + (CORE_CONNECTION, "hand_out() -> Engine"), + (CORE_CONNECTION, "handle.begin()"), + ] + + +def test_a_declarative_base_module_may_annotate_a_mapped_classs_table(): + source = """ + from typing import ClassVar + + from sqlalchemy import Table + + class TableModel: + __table__: ClassVar[Table] + """ + + assert _checks({ORM_BASE: source}, ORM_BASE) == [] + assert _checks({STORE: source}, STORE) == [ + (CORE_TABLE_ACCESS, "__table__: ClassVar[Table]") + ] + + +@pytest.mark.parametrize( + ("declaration", "summary"), + [ + ("def read(jobs: Table): ...", "jobs: Table"), + ("selected: Table = choose()", "selected: Table"), + ("class Rows:\n target: Table", "target: Table"), + ("def read(__table__: Table): ...", "__table__: Table"), + ("__table__: Table = choose()", "__table__: Table"), + ( + "class Rows:\n def read(self):\n __table__: Table = choose()", + "__table__: Table", + ), + ], +) +def test_a_declarative_base_module_annotates_only_table_of_its_classes( + declaration, summary +): + source = f"from sqlalchemy import Table\n\n{declaration}\n" + + assert _checks({ORM_BASE: source}, ORM_BASE) == [(CORE_TABLE_ACCESS, summary)] + + +@pytest.mark.parametrize( + "guarded_class_body", + [ + "if TYPE_CHECKING:\n __table__: ClassVar[Table]", + "if RUNTIME:\n pass\n else:\n __table__: ClassVar[Table]", + ( + "if RUNTIME:\n pass\n elif TYPE_CHECKING:\n" + " __table__: ClassVar[Table]" + ), + ], +) +def test_a_guarded_table_annotation_in_a_class_is_accepted(guarded_class_body): + source = f""" +from typing import TYPE_CHECKING, ClassVar + +from sqlalchemy import Table + +class TableModel: + {guarded_class_body} +""" + + assert _checks({ORM_BASE: source}, ORM_BASE) == [] + + +@pytest.mark.parametrize("file", [ORM_BASE, STORE]) +def test_reading_a_bare_mapped_table_name_is_core_table_access(file): + source = """ + from typing import ClassVar + + from sqlalchemy import select + + class JobRecord: + __table__ = build_jobs_table() + pending = select(__table__) + """ + + assert _checks({file: source}, file) == [(CORE_TABLE_ACCESS, "__table__")] + + +def test_a_declarative_base_module_may_not_declare_a_core_table(): + source = """ + from sqlalchemy import Table + + jobs = Table("household_jobs", metadata) + """ + + assert _checks({ORM_BASE: source}, ORM_BASE) == [ + (CORE_TABLE_DECLARATION, "Table('household_jobs', ...)") + ] + + +@pytest.mark.parametrize( + "table_expression", + [ + "HouseholdJob.__table__", + "inspect(HouseholdJob).local_table", + "inspect(HouseholdJob).persist_selectable", + "Base.metadata.sorted_tables", + ], +) +def test_a_table_is_reported_where_it_is_reached_and_not_where_it_is_used( + table_expression, +): + models = f""" + from example.orm_base import Base, HouseholdJob + from sqlalchemy import inspect, select + + jobs = {table_expression} + statement = select(jobs) + """ + store = """ + from example.models import jobs + from sqlalchemy import select + + statement = select(jobs) + """ + sources = {MODELS: models, STORE: store} + + assert _checks(sources, MODELS) == [(CORE_TABLE_ACCESS, table_expression)] + assert _checks(sources, STORE) == [] + + +def test_a_table_declared_in_another_module_is_reported_where_it_is_declared(): + tables = """ + from sqlalchemy import MetaData, Table + + metadata = MetaData() + household_jobs = Table("household_jobs", metadata) + """ + store = """ + from example.tables import household_jobs + from sqlalchemy import select + + statement = select(household_jobs) + """ + sources = {"libs/example/src/example/tables.py": tables, STORE: store} + + assert _checks(sources, "libs/example/src/example/tables.py") == [ + (CORE_TABLE_DECLARATION, "MetaData()"), + (CORE_TABLE_DECLARATION, "Table('household_jobs', ...)"), + ] + assert _checks(sources, STORE) == [] + + +def test_two_files_may_share_a_module_name(): + sources = { + "projects/first/src/modal/app.py": "", + "projects/second/src/modal/app.py": "", + } + + assert orm_access_findings(sources) == [] + + +def test_a_bare_mapped_table_read_is_reported_without_a_sqlalchemy_import(): + source = """ + class JobRecord: + __table__ = build_jobs_table() + pending = __table__.select() + """ + + assert _checks({STORE: source}, STORE) == [(CORE_TABLE_ACCESS, "__table__")] + + +def test_names_shared_with_core_tables_elsewhere_are_not_core_tables(): + gateway = """ + import pyarrow as pa + + def describe(regions: list[str], households, frame: pa.Table) -> str: + return ", ".join(regions) + str(households.columns) + str(frame.columns) + """ + + assert _checks({GATEWAY: gateway}, GATEWAY) == [] + + +def test_sql_text_is_accepted_as_a_server_default_or_index_condition(): + store = """ + from sqlalchemy import CheckConstraint, Index, text + from sqlalchemy.orm import mapped_column + + created_at = mapped_column(server_default=text("now()")) + positive = CheckConstraint(text("key_version > 0"), name="ck_key_version") + index = Index("ix_pending", "status", postgresql_where=text("status = 'pending'")) + """ + + assert _checks({STORE: store}, STORE) == [] + + +def test_names_from_other_packages_are_not_sqlalchemy(): + store = """ + from textwrap import dedent as text + + settings.update(region="uk") + text("SELECT 1") + """ + + assert _checks({STORE: store}, STORE) == [] + + +def test_metadata_is_accepted_only_in_a_declarative_base_module(): + source = """ + from sqlalchemy import MetaData + + metadata = MetaData(schema="public") + """ + + assert _checks({ORM_BASE: source}, ORM_BASE) == [] + assert _checks({STORE: source}, STORE) == [ + (CORE_TABLE_DECLARATION, "MetaData(...)") + ] + + +def test_lightweight_tables_and_columns_are_core_declarations(): + store = """ + import sqlalchemy as sa + + pg_class = sa.table("pg_class", sa.column("relname")) + """ + + assert _checks({STORE: store}, STORE) == [ + (CORE_TABLE_DECLARATION, "sa.table('pg_class', ...)"), + (CORE_TABLE_DECLARATION, "sa.column('relname')"), + ] + + +@pytest.mark.parametrize( + "expression", + [ + "Base.metadata.tables['household_jobs']", + "schema.tables['household_jobs']", + "schema.sorted_tables", + "HouseholdJob.__mapper__.local_table", + "HouseholdJob.__mapper__.tables", + "inspect(HouseholdJob).persist_selectable", + "inspect(HouseholdJob).selectable", + "HouseholdJob.__table__", + "class_mapper(HouseholdJob).tables", + "inspect(HouseholdJob).mapper.tables", + "job_mapper.tables", + "held_mapper.tables", + "inspected_mapper.tables", + "HouseholdJob.id.table", + ], +) +def test_reaching_a_mapped_class_or_metadata_table_is_core_table_access(expression): + store = f""" + from sqlalchemy import MetaData, inspect + from sqlalchemy.orm import class_mapper + + schema = MetaData() + job_mapper = inspect(HouseholdJob) + held_mapper = HouseholdJob.__mapper__ + inspected_mapper = inspect(job).mapper + table = {expression} + """ + + assert [ + check + for check, _ in _checks({STORE: store}, STORE) + if check != CORE_TABLE_DECLARATION + ] == [CORE_TABLE_ACCESS] + + +def test_engines_are_held_only_by_a_library_engine_module(): + source = """ + from sqlalchemy import Engine, create_engine + from sqlalchemy.orm import sessionmaker + """ + + assert _checks({ENGINE: source}, ENGINE) == [] + assert _checks({STORE: source}, STORE) == [ + (CORE_CONNECTION, "sqlalchemy.Engine"), + (CORE_CONNECTION, "sqlalchemy.create_engine"), + ] + + +def test_an_engine_module_opens_sessions_not_connections(): + engine_module = """ + from sqlalchemy import Engine + from sqlalchemy.engine import Connection + + def bounded_transaction(engine: Engine): + with engine.begin() as connection: + yield connection + + def store_transaction(store): + return store._engine.begin() + + def session_transaction(session_factory): + return session_factory.begin() + """ + + assert _checks({ENGINE: engine_module}, ENGINE) == [ + (CORE_CONNECTION, "sqlalchemy.engine.Connection"), + (CORE_CONNECTION, "engine.begin()"), + (CORE_CONNECTION, "store._engine.begin()"), + ] + + +@pytest.mark.parametrize( + "call", + [ + "engine.connect()", + "session.connection()", + "session.get_bind()", + "engine.raw_connection()", + ], +) +def test_opening_a_connection_is_reported_in_every_module_using_sqlalchemy(call): + source = f"import sqlalchemy\n\nvalue = {call}" + + assert _checks({ENGINE: source}, ENGINE) == [(CORE_CONNECTION, call)] + + +@pytest.mark.parametrize( + ("call", "summary"), + [ + ("connection.execute(statement)", "connection.execute(...)"), + ("conn.scalar(statement)", "conn.scalar(...)"), + ("store_connection.scalars(statement)", "store_connection.scalars(...)"), + ("self._connection.execute(statement)", "self._connection.execute(...)"), + ("connection.stream(statement)", "connection.stream(...)"), + ("connection.stream_scalars(statement)", "connection.stream_scalars(...)"), + ("bind.execute(statement)", "bind.execute(...)"), + ("database.execute(statement)", "database.execute(...)"), + ("self.store.execute(statement)", "self.store.execute(...)"), + ("connections.execute(statement)", "connections.execute(...)"), + ], +) +def test_a_statement_on_an_unannotated_connection_is_reported(call, summary): + """``execute`` must run on a session, whatever else its receiver is called.""" + source = f"import sqlalchemy\n\nvalue = {call}" + + assert _checks({STORE: source}, STORE) == [(CORE_CONNECTION, summary)] + + +@pytest.mark.parametrize( + "call", + [ + "session.execute(statement)", + "session.scalar(statement)", + "session.scalars(statement)", + "self._session.execute(statement)", + "store_session.execute(statement)", + "Session(bind).execute(statement)", + "session.execute(statement).scalars()", + "result.scalars()", + "result.scalar()", + ], +) +def test_a_statement_on_a_session_is_not_reported(call): + source = f"import sqlalchemy\n\nvalue = {call}" + + assert _checks({STORE: source}, STORE) == [] + + +def test_a_connection_outside_sqlalchemy_is_not_reported(): + assert _checks({GATEWAY: "value = connection.execute(statement)"}, GATEWAY) == [] + + +def test_an_engine_module_hands_out_no_engine_or_connection(): + source = """ + from sqlalchemy import Engine, create_engine + from sqlalchemy.orm import Session, sessionmaker + + def _private_engine(url) -> Engine: + return create_engine(url) + + def public_engine(url) -> Engine: + return create_engine(url) + + def optional_engine(url) -> "Engine | None": + return None + + def session_factory(engine: Engine) -> sessionmaker[Session]: + return sessionmaker(engine) + + class Sessions: + def engine(self) -> Engine: + return self._engine + """ + + assert _checks({ENGINE: source}, ENGINE) == [ + (CORE_CONNECTION, "public_engine() -> Engine"), + (CORE_CONNECTION, "optional_engine() -> Engine"), + (CORE_CONNECTION, "engine() -> Engine"), + ] + + +def test_only_an_engine_module_reads_a_session_factorys_engine(): + source = """ + import sqlalchemy + + engine = session_factory.kw["bind"] + autoflush = session_factory.kw["autoflush"] + """ + + assert _checks({ENGINE: source}, ENGINE) == [] + assert _checks({STORE: source}, STORE) == [ + (CORE_CONNECTION, "session_factory.kw['bind']"), + ] + + +@pytest.mark.parametrize( + ("source", "summary"), + [ + ("sa.text('SELECT 1')", "sa.text('SELECT 1')"), + ("sa.literal_column('count(*)')", "sa.literal_column('count(*)')"), + ("session.exec_driver_sql('SELECT 1')", "session.exec_driver_sql('SELECT 1')"), + ("driver_connection.cursor()", "driver_connection.cursor()"), + ], +) +def test_raw_sql_is_reported(source, summary): + store = f""" + import sqlalchemy as sa + + statement = {source} + """ + + assert _checks({STORE: store}, STORE) == [(RAW_SQL, summary)] + + +@pytest.mark.parametrize( + "statement", ["from sqlmodel import Session", "import sqlmodel"] +) +def test_sqlmodel_imports_are_reported(statement): + assert _checks({STORE: statement}, STORE) == [(SQLMODEL_IMPORT, "sqlmodel")] + + +@pytest.mark.parametrize( + ("binding", "receiver"), + [ + ("first, label = create_engine(url), 'primary'", "first"), + ("(first, label), other = (create_engine(url), 'a'), 1", "first"), + ("[first, label] = [create_engine(url), 'primary']", "first"), + ("self.first = create_engine(url)", "self.first"), + ("first: object = create_engine(url)", "first"), + ("value = (first := create_engine(url))", "first"), + ], +) +def test_every_way_of_binding_an_engine_is_followed_to_begin(binding, receiver): + source = f""" + from sqlalchemy import create_engine + + {binding} + transaction = {receiver}.begin() + """ + + assert _checks({ENGINE: source}, ENGINE) == [ + (CORE_CONNECTION, f"{receiver}.begin()") + ] + + +def test_a_name_annotated_as_an_engine_is_followed_to_begin(): + source = """ + from sqlalchemy import Engine + + def open_transaction(primary: "Engine | None", held: Engine): + return primary.begin(), held.begin() + """ + + assert _checks({ENGINE: source}, ENGINE) == [ + (CORE_CONNECTION, "primary.begin()"), + (CORE_CONNECTION, "held.begin()"), + ] + + +@pytest.mark.parametrize( + ("binding", "read"), + [ + ("schema, label = MetaData(), 'public'", "schema.tables"), + ("job_mapper, label = inspect(HouseholdJob), 'jobs'", "job_mapper.tables"), + ("job_mapper, label = HouseholdJob.__mapper__, 'jobs'", "job_mapper.tables"), + ], +) +def test_tuple_assignments_bind_metadata_and_mappers(binding, read): + source = f""" + from sqlalchemy import MetaData, inspect + + {binding} + value = {read} + """ + + assert _checks({ORM_BASE: source}, ORM_BASE) == [(CORE_TABLE_ACCESS, read)] + + +def test_begin_counts_as_opening_a_connection_only_on_an_engine(): + engine_module = """ + from sqlalchemy import create_engine + + primary = create_engine(url) + first = primary.begin() + second = policyengine_settings.begin() + """ + + assert _checks({ENGINE: engine_module}, ENGINE) == [ + (CORE_CONNECTION, "primary.begin()") + ] + + +def test_async_engines_and_connections_are_reported(): + store = "from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine" + + assert _checks({STORE: store}, STORE) == [ + (CORE_CONNECTION, "sqlalchemy.ext.asyncio.AsyncConnection"), + (CORE_CONNECTION, "sqlalchemy.ext.asyncio.AsyncEngine"), + ] + + +@pytest.mark.parametrize( + ("source", "check"), + [ + ("sa.TableClause('household_jobs')", CORE_TABLE_DECLARATION), + ("sa.ColumnClause('id')", CORE_TABLE_DECLARATION), + ("sa.TextClause('SELECT 1')", RAW_SQL), + ], +) +def test_core_clause_classes_are_reported(source, check): + store = f""" + import sqlalchemy as sa + + clause = {source} + """ + + assert [found for found, _ in _checks({STORE: store}, STORE)] == [check] + + +def _load_module(name: str, source: str, monkeypatch) -> types.ModuleType: + module = types.ModuleType(name) + # SQLAlchemy resolves a mapped class's annotations in its module. + monkeypatch.setitem(sys.modules, name, module) + exec(compile(dedent(source), name, "exec"), module.__dict__) + return module + + +def test_the_runtime_check_accepts_tables_mapped_by_either_declarative_base( + monkeypatch, +): + mapped = _load_module( + "orm_only_mapped_models", + """ + from sqlalchemy import MetaData, Table + from sqlalchemy.orm import ( + DeclarativeBase, + DeclarativeBaseNoMeta, + Mapped, + mapped_column, + ) + + class Base(DeclarativeBase): + pass + + class NoMetaBase(DeclarativeBaseNoMeta): + metadata = MetaData() + + class HouseholdJob(Base): + __tablename__ = "household_jobs" + id: Mapped[int] = mapped_column(primary_key=True) + + class Report(NoMetaBase): + __tablename__ = "reports" + id: Mapped[int] = mapped_column(primary_key=True) + + stray = Table("stray", Base.metadata) + """, + monkeypatch, + ) + + findings = unmapped_table_findings([mapped], Path(), location="example") + + assert [(finding.check, finding.source) for finding in findings] == [ + (UNMAPPED_TABLE, "stray") + ] + + +@pytest.mark.parametrize( + "expression", ["key.table", "job_id.table", "self.model.id.table", "rows.table"] +) +def test_any_table_read_in_a_module_using_sqlalchemy_is_core_table_access( + expression, +): + store = f""" + from sqlalchemy import insert + + def write(self, key, job_id, rows): + return insert({expression}) + """ + + assert _checks({STORE: store}, STORE) == [(CORE_TABLE_ACCESS, expression)] + + +def test_a_called_or_assigned_table_attribute_is_not_a_table_read(): + store = """ + import sqlalchemy as sa + + class Report: + def __init__(self, rows, model): + self.table = model + self.rows = rows.table(columns=["id"]) + """ + + assert _checks({STORE: store}, STORE) == [] + + +def test_name_rules_apply_only_in_modules_that_use_sqlalchemy(): + gateway = """ + def publish(settings, socket, cache_engine, driver): + settings.bigquery.table + socket.connect() + cache_engine.begin() + driver.cursor() + """ + + assert _checks({GATEWAY: gateway}, GATEWAY) == [] + + +def test_a_name_assigned_a_mapped_class_or_its_table_name_is_not_a_table(): + store = """ + from example.orm_base import HouseholdJob + from sqlalchemy import select + + job_model = HouseholdJob + table_name = HouseholdJob.__tablename__ + statement = select(job_model).where(job_model.id == table_name) + """ + + assert _checks({STORE: store}, STORE) == [] diff --git a/projects/policyengine-simulation-entry/tests/test_orm_only.py b/projects/policyengine-simulation-entry/tests/test_orm_only.py new file mode 100644 index 000000000..ac01572fe --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_orm_only.py @@ -0,0 +1,136 @@ +"""Guard sim.persistence.orm-only over every source directory. + +Database code declares tables only as SQLAlchemy ORM mapped classes and runs +every statement through an ORM Session against them; +docs/engineering/skills/database-access.md#orm explains the rule. The checks +live in orm_access_support.py, and any finding fails. The source directories +are the ones sim.boundaries.sqlalchemy-and-alembic-only scans, whose +test_every_source_directory_is_scanned checks that each one contributes files. +""" + +from __future__ import annotations + +import importlib.util +import sys +from typing import TYPE_CHECKING + +import tomllib +from orm_access_support import ( + CORE_CONNECTION, + CORE_TABLE_ACCESS, + CORE_TABLE_DECLARATION, + RAW_SQL, + UNMAPPED_TABLE, + orm_access_findings, + unmapped_table_findings, + unmapped_tables_in_packages, +) +from test_database_access_boundaries import REPOSITORY_ROOT, SOURCE_ROOT_PATTERNS + +if TYPE_CHECKING: + from types import ModuleType + + import pytest + +FIXTURES_ROOT = REPOSITORY_ROOT / "rules" / "fixtures" / "sim.persistence.orm-only" +PROJECT_PATTERNS = ("projects/*/pyproject.toml", "libs/*/pyproject.toml") +# The libraries that declare tables; every table they hold needs a mapped class. +TABLE_PACKAGES = ( + "policyengine_simulation_persistence", + "policyengine_stage12_persistence", +) + + +def _sources() -> dict[str, str]: + return { + source_file.relative_to(REPOSITORY_ROOT).as_posix(): source_file.read_text( + encoding="utf-8" + ) + for pattern in SOURCE_ROOT_PATTERNS + for source_root in REPOSITORY_ROOT.glob(pattern) + for source_file in sorted(source_root.rglob("*.py")) + } + + +def _load_fixture(fixture_name: str, monkeypatch: pytest.MonkeyPatch) -> ModuleType: + path = FIXTURES_ROOT / fixture_name + spec = importlib.util.spec_from_file_location(f"orm_only_{path.stem}", path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + # SQLAlchemy resolves a mapped class's annotations in its module, which it + # finds in sys.modules. + monkeypatch.setitem(sys.modules, spec.name, module) + spec.loader.exec_module(module) + return module + + +def _fixture_findings(fixture_name: str) -> list[tuple[int, str, str]]: + fixture = FIXTURES_ROOT / fixture_name + findings = orm_access_findings({fixture_name: fixture.read_text(encoding="utf-8")}) + return [(finding.line, finding.check, finding.source) for finding in findings] + + +def test_database_code_uses_only_the_orm() -> None: + findings = orm_access_findings(_sources()) + + assert [str(finding) for finding in findings] == [], ( + "Code reaches the database without the ORM; use a mapped class and a " + "Session (docs/engineering/skills/database-access.md#orm). For a .table " + "finding on converted code, name the attribute after what it holds, such " + "as model" + ) + + +def test_every_table_is_mapped_by_a_class() -> None: + findings = unmapped_tables_in_packages(TABLE_PACKAGES, REPOSITORY_ROOT) + + assert [str(finding) for finding in findings] == [] + + +def test_no_project_installs_sqlmodel() -> None: + """The rule's ORM is SQLAlchemy's own, so no installable project locks SQLModel.""" + projects_with_sqlmodel = [] + for pattern in PROJECT_PATTERNS: + for pyproject in sorted(REPOSITORY_ROOT.glob(pattern)): + lock = tomllib.loads( + (pyproject.parent / "uv.lock").read_text(encoding="utf-8") + ) + if any(package["name"] == "sqlmodel" for package in lock["package"]): + projects_with_sqlmodel.append( + pyproject.parent.relative_to(REPOSITORY_ROOT).as_posix() + ) + + assert projects_with_sqlmodel == [] + + +def test_rule_proof_orm_only(monkeypatch: pytest.MonkeyPatch) -> None: + """Proof for rule sim.persistence.orm-only.""" + assert _fixture_findings("bad_household_job_records.py") == [ + (15, CORE_CONNECTION, "sqlalchemy.create_engine"), + (20, CORE_CONNECTION, "sqlalchemy.engine.Connection"), + (20, CORE_CONNECTION, "sqlalchemy.engine.Engine"), + (22, CORE_TABLE_DECLARATION, "MetaData(...)"), + (23, CORE_TABLE_DECLARATION, "Table('household_jobs', ...)"), + (26, CORE_TABLE_DECLARATION, "Column('id', ...)"), + (27, CORE_TABLE_DECLARATION, "Column('status', ...)"), + (28, CORE_TABLE_DECLARATION, "Column('modal_call_id', ...)"), + (39, CORE_CONNECTION, "connection.execute(...)"), + (48, CORE_CONNECTION, "connection.execute(...)"), + (61, CORE_CONNECTION, "engine.connect()"), + (62, CORE_CONNECTION, "connection.execute(...)"), + ( + 63, + RAW_SQL, + "text(\"SELECT count(*) FROM household_jobs WHERE status = 'pending'\")", + ), + (67, CORE_TABLE_ACCESS, "jobs: Table"), + ] + assert _fixture_findings("good_household_job_records.py") == [] + bad_module = _load_fixture("bad_household_job_records.py", monkeypatch) + good_module = _load_fixture("good_household_job_records.py", monkeypatch) + assert [ + (finding.check, finding.source) + for finding in unmapped_table_findings([bad_module], REPOSITORY_ROOT) + ] == [(UNMAPPED_TABLE, "public.household_jobs")] + assert unmapped_table_findings([good_module], REPOSITORY_ROOT) == [] diff --git a/projects/policyengine-simulation-entry/tests/test_simulation_database.py b/projects/policyengine-simulation-entry/tests/test_simulation_database.py index ac0ba00e5..148e673c6 100644 --- a/projects/policyengine-simulation-entry/tests/test_simulation_database.py +++ b/projects/policyengine-simulation-entry/tests/test_simulation_database.py @@ -20,6 +20,7 @@ missing_privilege_log_messages, ) from sqlalchemy.exc import OperationalError +from sqlalchemy.orm import sessionmaker DATABASE_URL = "postgresql://simulation_service:secret@db.example:5432/postgres" @@ -318,17 +319,22 @@ def test_the_check_runs_on_its_own_thread_within_the_database_time_limit( ): calls = [] - def read_missing_table_privileges(engine, *, statement_timeout_milliseconds): + def read_missing_table_privileges( + session_factory, *, statement_timeout_milliseconds + ): calls.append( - (engine, statement_timeout_milliseconds, threading.current_thread().name) + ( + session_factory, + statement_timeout_milliseconds, + threading.current_thread().name, + ) ) return ["INSERT on reports"] monkeypatch.setattr( database_module, "read_missing_table_privileges", read_missing_table_privileges ) - engine = object() - database = SimulationDatabase(engine) + database = SimulationDatabase(sessionmaker(object())) try: missing = asyncio.run(database.missing_privileges()) @@ -336,8 +342,8 @@ def read_missing_table_privileges(engine, *, statement_timeout_milliseconds): database.executor.shutdown(wait=True) assert missing == ["INSERT on reports"] - [(called_engine, timeout_milliseconds, thread_name)] = calls - assert called_engine is engine + [(session_factory, timeout_milliseconds, thread_name)] = calls + assert session_factory is database.session_factory expected_timeout = database_module.READINESS_STATEMENT_TIMEOUT_MILLISECONDS assert timeout_milliseconds == expected_timeout assert timeout_milliseconds < ( @@ -355,7 +361,7 @@ def test_a_stalled_database_holds_one_thread_and_no_thread_of_the_default_pool( most_running_at_once = [] lock = threading.Lock() - def stall(engine, *, statement_timeout_milliseconds): + def stall(session_factory, *, statement_timeout_milliseconds): with lock: running.append(threading.current_thread().name) most_running_at_once.append(len(running)) @@ -365,7 +371,7 @@ def stall(engine, *, statement_timeout_milliseconds): return [] monkeypatch.setattr(database_module, "read_missing_table_privileges", stall) - database = SimulationDatabase(object()) + database = SimulationDatabase(sessionmaker(object())) async def three_checks_that_time_out(): results = await asyncio.gather( @@ -395,6 +401,39 @@ async def three_checks_that_time_out(): assert default_pool_threads == [] +def test_the_database_holds_the_session_factory_it_is_given(): + session_factory = sessionmaker(object()) + + database = SimulationDatabase(session_factory) + + try: + assert database.session_factory is session_factory + assert not hasattr(database, "engine") + finally: + database.executor.shutdown(wait=True) + + +def test_a_database_from_a_url_builds_the_services_session_factory(monkeypatch): + built_from = [] + session_factory = sessionmaker(object()) + + def build_session_factory(database_url): + built_from.append(database_url) + return session_factory + + monkeypatch.setattr( + database_module, "create_simulation_session_factory", build_session_factory + ) + + database = SimulationDatabase.from_url(DATABASE_URL) + + try: + assert built_from == [DATABASE_URL] + assert database.session_factory is session_factory + finally: + database.executor.shutdown(wait=True) + + def test_disposing_the_database_also_stops_its_thread(monkeypatch): class FakeEngine: disposed = False @@ -405,10 +444,10 @@ def dispose(self): monkeypatch.setattr( database_module, "read_missing_table_privileges", - lambda engine, *, statement_timeout_milliseconds: [], + lambda session_factory, *, statement_timeout_milliseconds: [], ) engine = FakeEngine() - database = SimulationDatabase(engine) + database = SimulationDatabase(sessionmaker(engine)) asyncio.run(database.missing_privileges()) [worker] = [ thread @@ -431,9 +470,9 @@ def dispose(self): monkeypatch.setattr( database_module, "read_missing_table_privileges", - lambda engine, *, statement_timeout_milliseconds: [], + lambda session_factory, *, statement_timeout_milliseconds: [], ) - database = SimulationDatabase(FakeEngine()) + database = SimulationDatabase(sessionmaker(FakeEngine())) database.dispose() with _client(FakeBackend(), database) as client: diff --git a/projects/policyengine-simulation-entry/tests/test_stage12_infrastructure.py b/projects/policyengine-simulation-entry/tests/test_stage12_infrastructure.py index 7548e2243..3f60414d1 100644 --- a/projects/policyengine-simulation-entry/tests/test_stage12_infrastructure.py +++ b/projects/policyengine-simulation-entry/tests/test_stage12_infrastructure.py @@ -5,11 +5,23 @@ import json import os import subprocess +import sys import textwrap from pathlib import Path +from typing import TYPE_CHECKING, ClassVar import pytest import yaml +from policyengine_simulation_entry import stage12_infrastructure +from policyengine_simulation_persistence.engine import ( + SINGLE_CONNECTION_CONNECT_TIMEOUT_SECONDS, +) +from policyengine_stage12_persistence import validation +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import QueuePool + +if TYPE_CHECKING: + from collections.abc import Callable REPO_ROOT = Path(__file__).resolve().parents[3] SCRIPT = REPO_ROOT / ".github/scripts/stage12-validate-infrastructure.sh" @@ -411,3 +423,240 @@ def test_both_stage12_deploy_steps_receive_the_simulation_output_bucket() -> Non assert step["env"]["V2_SIMULATION_OUTPUT_BUCKET"] == ( "${{ vars.V2_SIMULATION_OUTPUT_BUCKET }}" ) + + +def _prepare_the_check_command(tmp_path, monkeypatch) -> list[object]: + """Prepare the command's arguments; return the factories it disposes of.""" + url_file = tmp_path / "database-url" + url_file.write_text("postgresql://role@database.example/postgres\n") + monkeypatch.setattr( + "sys.argv", + [ + "stage12_infrastructure", + "--database-url-file", + str(url_file), + "--expected-role", + "policyengine_v2_runtime", + "--environment", + "staging", + ], + ) + disposed: list[object] = [] + monkeypatch.setattr( + stage12_infrastructure, "dispose_session_factory", disposed.append + ) + return disposed + + +def test_the_deployment_check_shares_one_connection_of_the_given_url( + tmp_path, monkeypatch, capsys +) -> None: + disposed = _prepare_the_check_command(tmp_path, monkeypatch) + checked: list[tuple[object, dict[str, str]]] = [] + monkeypatch.setattr( + stage12_infrastructure, + "validate_runtime_database", + lambda session_factory, **options: checked.append((session_factory, options)), + ) + + stage12_infrastructure.main() + + [(session_factory, options)] = checked + assert isinstance(session_factory, sessionmaker) + engine = session_factory.kw["bind"] + assert engine.url.host == "database.example" + assert engine.url.username == "role" + assert isinstance(engine.pool, QueuePool) + assert engine.pool.size() == 1 + assert options == { + "expected_role": "policyengine_v2_runtime", + "environment": "staging", + } + assert disposed == [session_factory] + assert "role=policyengine_v2_runtime" in capsys.readouterr().out + + +def test_a_failed_deployment_check_closes_its_connection(tmp_path, monkeypatch) -> None: + disposed = _prepare_the_check_command(tmp_path, monkeypatch) + + def refuse(_session_factory, **_options) -> None: + message = "runtime role lacks SELECT" + raise stage12_infrastructure.Stage12RuntimeAccessError(message) + + monkeypatch.setattr(stage12_infrastructure, "validate_runtime_database", refuse) + + with pytest.raises(SystemExit, match="runtime role lacks SELECT"): + stage12_infrastructure.main() + + assert len(disposed) == 1 + + +class RecordedTimer: + """Stands in for ``threading.Timer``, recording how the command uses it.""" + + created: ClassVar[list[RecordedTimer]] = [] + + def __init__(self, interval: float, function: Callable[[], None]) -> None: + self.interval = interval + self.function = function + self.daemon = False + self.started = False + self.cancelled = False + self.created.append(self) + + def start(self) -> None: + self.started = True + + def cancel(self) -> None: + self.cancelled = True + + +@pytest.mark.parametrize("check_passes", [True, False]) +def test_the_check_time_limit_ends_with_the_command( + tmp_path, monkeypatch, check_passes +) -> None: + _prepare_the_check_command(tmp_path, monkeypatch) + monkeypatch.setattr(RecordedTimer, "created", []) + monkeypatch.setattr(stage12_infrastructure.threading, "Timer", RecordedTimer) + + def dispose_while_the_limit_runs(_session_factory) -> None: + [timer] = RecordedTimer.created + assert not timer.cancelled + + monkeypatch.setattr( + stage12_infrastructure, "dispose_session_factory", dispose_while_the_limit_runs + ) + + def finish_the_check(_session_factory, **_options) -> None: + [timer] = RecordedTimer.created + assert timer.started + assert not timer.cancelled + if not check_passes: + message = "runtime role lacks SELECT" + raise stage12_infrastructure.Stage12RuntimeAccessError(message) + + monkeypatch.setattr( + stage12_infrastructure, "validate_runtime_database", finish_the_check + ) + + if check_passes: + stage12_infrastructure.main() + else: + with pytest.raises(SystemExit): + stage12_infrastructure.main() + + [timer] = RecordedTimer.created + assert timer.interval == stage12_infrastructure.CHECK_TIME_LIMIT_SECONDS + assert timer.daemon is True + assert timer.cancelled + + +def _run_a_stalled_check_command( + tmp_path, *, before_the_command: str = "" +) -> subprocess.CompletedProcess[str]: + """Run the real command in a process whose check blocks forever.""" + url_file = tmp_path / "database-url" + url_file.write_text("postgresql://role@database.example/postgres\n") + program = "\n".join( + [ + "import os", + "import sys", + "import threading", + "from policyengine_simulation_entry import (", + " stage12_infrastructure as command,", + ")", + before_the_command, + "command.CHECK_TIME_LIMIT_SECONDS = 0.2", + "command.TIME_LIMIT_MESSAGE_WAIT_SECONDS = 0.2", + "command.validate_runtime_database = (", + " lambda _session_factory, **_options: threading.Event().wait()", + ")", + "sys.argv = [", + ' "stage12_infrastructure",', + ' "--database-url-file",', + f" {str(url_file)!r},", + ' "--expected-role",', + ' "policyengine_v2_runtime",', + ' "--environment",', + ' "staging",', + "]", + "command.main()", + ] + ) + return subprocess.run( + [sys.executable, "-c", program], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + + +def test_a_stalled_deployment_check_ends_the_real_process(tmp_path) -> None: + # A connection that goes silent blocks its reader with no error to handle, + # so the limit must end the process while the main thread is still blocked. + result = _run_a_stalled_check_command(tmp_path) + + assert result.returncode == 1 + assert "did not finish within 0.2 seconds" in result.stderr + + +def test_the_time_limit_ends_the_process_when_standard_error_is_full( + tmp_path, +) -> None: + fill_standard_error = textwrap.dedent( + """ + reader, writer = os.pipe() + os.set_blocking(writer, False) + try: + while True: + os.write(writer, b"x" * 4096) + except BlockingIOError: + pass + os.set_blocking(writer, True) + os.dup2(writer, 2) + """ + ) + + result = _run_a_stalled_check_command( + tmp_path, before_the_command=fill_standard_error + ) + + assert result.returncode == 1 + + +def test_the_time_limit_ends_the_process_when_its_message_cannot_be_written( + monkeypatch, +) -> None: + class ClosedStream: + def write(self, _text: str) -> int: + raise BrokenPipeError + + def flush(self) -> None: + raise BrokenPipeError + + exit_statuses: list[int] = [] + monkeypatch.setattr(stage12_infrastructure.sys, "stderr", ClosedStream()) + monkeypatch.setattr(stage12_infrastructure.os, "_exit", exit_statuses.append) + + stage12_infrastructure._end_the_stalled_check() + + assert exit_statuses == [1] + + +def test_the_time_limit_leaves_room_for_every_attempt_of_every_step() -> None: + # The driver gives each address of the host name the full connect timeout, + # and the database pooler's host names resolve to as many as three. This + # counts one connection per attempt, what a lost or refused connection costs. + addresses = 4 + steps = 4 + waits = sum( + validation.FIRST_RETRY_DELAY_SECONDS * 2**attempt + for attempt in range(validation.STEP_ATTEMPTS - 1) + ) + connecting = ( + validation.STEP_ATTEMPTS * addresses * SINGLE_CONNECTION_CONNECT_TIMEOUT_SECONDS + ) + time_needed = steps * (connecting + waits) + + assert time_needed < stage12_infrastructure.CHECK_TIME_LIMIT_SECONDS diff --git a/projects/policyengine-simulation-executor/pyproject.toml b/projects/policyengine-simulation-executor/pyproject.toml index c76ebc2bc..ed61d30d4 100644 --- a/projects/policyengine-simulation-executor/pyproject.toml +++ b/projects/policyengine-simulation-executor/pyproject.toml @@ -66,7 +66,7 @@ modal-simulation-image = [ # this directly; it is also a transitive dependency of policyengine. "requests>=2.32,<3", "pyarrow>=20,<24", - # The Stage 12 stores use typed SQLAlchemy Core DML through the shared + # The Stage 12 stores use SQLAlchemy ORM sessions and the shared # table-model library (policyengine-simulation-persistence), whose ranges # these are. They are explicit because only this group is installed in # Modal. diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/database_retry.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/database_retry.py index 4f6c10097..88b159ae5 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/database_retry.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/database_retry.py @@ -1,4 +1,4 @@ -"""Bounded retry for Stage 12 database connections that could not be opened.""" +"""Bounded retry for Stage 12 database sessions that could not connect.""" from __future__ import annotations @@ -15,7 +15,7 @@ from collections.abc import Callable from policyengine_observability import ObservabilityRuntime - from sqlalchemy.engine import Connection + from sqlalchemy.orm import Session logger = logging.getLogger(__name__) @@ -40,22 +40,23 @@ class RetryTiming: choose_delay: Callable[[float, float], float] = random.uniform -def open_connection_with_retry( - connect: Callable[[], Connection], +def open_session_with_retry( + open_session: Callable[[], Session], *, operation_name: str, runtime: ObservabilityRuntime | None = None, timing: RetryTiming | None = None, -) -> Connection: - """Open a Stage 12 database connection, retrying while it is refused. - - The persistence store passes only its connect step here. Failing that step - means no connection opened and no statement reached the database, so - opening it again cannot duplicate a write. The store runs the operation's - statements and its COMMIT or ROLLBACK after this returns; their failures, - including a COMMIT whose reply was lost, propagate without a retry. - Errors other than ``OperationalError`` propagate immediately, and retries - stop once the retry budget is spent. +) -> Session: + """Open a Stage 12 database session, retrying while it cannot connect. + + The persistence store passes only its opening step here: a new session + connecting with one statement that changes nothing. Failing that step means + none of the operation's statements reached the database, so opening another + session cannot duplicate a write. The store runs the operation's statements + and its COMMIT or ROLLBACK after this returns; their failures, including a + COMMIT whose reply was lost, propagate without a retry. Errors other than + ``OperationalError`` propagate immediately, and retries stop once the retry + budget is spent. """ timing = timing or RetryTiming() started = timing.monotonic() @@ -63,7 +64,7 @@ def open_connection_with_retry( delay_ceiling = INITIAL_RETRY_DELAY_SECONDS while True: try: - return connect() + return open_session() except OperationalError as error: remaining_budget = CONNECTION_RETRY_BUDGET_SECONDS - ( timing.monotonic() - started diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/dependencies.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/dependencies.py index 854ca2255..dc0ee0773 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/dependencies.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/dependencies.py @@ -15,7 +15,7 @@ Stage12ArtifactStore, ) -from .database_retry import open_connection_with_retry +from .database_retry import open_session_with_retry if TYPE_CHECKING: from datetime import datetime @@ -176,7 +176,7 @@ def runtime_store( # refused connection rather than failing the report during a burst. return Stage12PersistenceStore( os.environ.get("STAGE12_DATABASE_URL", ""), - open_connection=partial(open_connection_with_retry, runtime=runtime), + open_session=partial(open_session_with_retry, runtime=runtime), ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/household_report_recording.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/household_report_recording.py index 2a64a359e..eaebc73d7 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/household_report_recording.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/household_report_recording.py @@ -53,7 +53,7 @@ HOUSEHOLD_SIDE_WAIT_SECONDS, ) -from .database_retry import open_connection_with_retry +from .database_retry import open_session_with_retry from .dependencies import ModalChildInvoker from .failure_recording import persist_failure_record from .household_job_recording import ( @@ -201,7 +201,7 @@ def household_report_run_store( """ return HouseholdReportRunStore( os.environ.get(MODAL_WORKER_DATABASE_URL_ENV, ""), - open_connection=partial(open_connection_with_retry, runtime=runtime), + open_session=partial(open_session_with_retry, runtime=runtime), ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/report_run_access.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/report_run_access.py index e85cdfffe..4465ccd57 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/report_run_access.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/report_run_access.py @@ -19,7 +19,7 @@ from policyengine_stage12_persistence import BudgetWindowStore, ReportRunStore -from .database_retry import open_connection_with_retry +from .database_retry import open_session_with_retry if TYPE_CHECKING: from collections.abc import Mapping @@ -178,7 +178,7 @@ def report_run_store(runtime: ObservabilityRuntime | None = None) -> ReportRunSt """ return ReportRunStore( os.environ.get(MODAL_WORKER_DATABASE_URL_ENV, ""), - open_connection=partial(open_connection_with_retry, runtime=runtime), + open_session=partial(open_session_with_retry, runtime=runtime), ) @@ -192,5 +192,5 @@ def budget_window_store( """ return BudgetWindowStore( os.environ.get(MODAL_WORKER_DATABASE_URL_ENV, ""), - open_connection=partial(open_connection_with_retry, runtime=runtime), + open_session=partial(open_session_with_retry, runtime=runtime), ) diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_database_retry.py b/projects/policyengine-simulation-executor/tests/test_stage12_database_retry.py index 29cfbf255..da4e889d5 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_database_retry.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_database_retry.py @@ -1,4 +1,4 @@ -"""Tests for retrying Stage 12 database connections that could not be opened.""" +"""Tests for retrying Stage 12 database sessions that could not connect.""" from __future__ import annotations @@ -8,7 +8,7 @@ from policyengine_simulation_executor.stage12_runtime import database_retry from policyengine_simulation_executor.stage12_runtime.database_retry import ( RetryTiming, - open_connection_with_retry, + open_session_with_retry, ) from sqlalchemy.exc import OperationalError @@ -50,8 +50,8 @@ def log(self, message, *, severity="INFO", attributes=None, error=None): ) -class OpenedConnection: - """Stand in for the connection a successful connect step returns.""" +class OpenedSession: + """Stand in for the session a successful opening step returns.""" def _upper_bound_delay(_lower_bound: float, upper_bound: float) -> float: @@ -61,16 +61,16 @@ def _upper_bound_delay(_lower_bound: float, upper_bound: float) -> float: def test_refused_connections_are_retried_with_growing_bounded_delays() -> None: clock = ManualClock() runtime = RetryLogRuntime() - opened_connection = OpenedConnection() + opened_session = OpenedSession() attempts: list[int] = [] - def connect_after_two_refusals() -> OpenedConnection: + def connect_after_two_refusals() -> OpenedSession: attempts.append(len(attempts) + 1) if len(attempts) <= 2: raise _connection_limit_error() - return opened_connection + return opened_session - result = open_connection_with_retry( + result = open_session_with_retry( connect_after_two_refusals, operation_name="create_or_resolve_simulation", runtime=runtime, @@ -81,7 +81,7 @@ def connect_after_two_refusals() -> OpenedConnection: ), ) - assert result is opened_connection + assert result is opened_session assert attempts == [1, 2, 3] assert clock.sleeps == [1.0, 2.0] assert [record["attributes"] for record in runtime.records] == [ @@ -112,13 +112,13 @@ def lowest_delay(lower_bound: float, upper_bound: float) -> float: attempts: list[int] = [] - def connect_on_third_attempt() -> OpenedConnection: + def connect_on_third_attempt() -> OpenedSession: attempts.append(1) if len(attempts) < 3: raise _connection_limit_error() - return OpenedConnection() + return OpenedSession() - open_connection_with_retry( + open_session_with_retry( connect_on_third_attempt, operation_name="get_simulation", timing=RetryTiming( @@ -142,7 +142,7 @@ def fail_with_invalid_connection_options() -> None: raise ValueError(msg) with pytest.raises(ValueError, match="invalid connection option"): - open_connection_with_retry( + open_session_with_retry( fail_with_invalid_connection_options, operation_name="get_report", timing=RetryTiming( @@ -164,7 +164,7 @@ def never_connect() -> None: raise _connection_limit_error() with pytest.raises(OperationalError, match="EMAXCONNSESSION"): - open_connection_with_retry( + open_session_with_retry( never_connect, operation_name="create_or_resolve_report", timing=RetryTiming( @@ -187,17 +187,17 @@ def log(self, *_args, **_kwargs): raise RuntimeError(msg) clock = ManualClock() - opened_connection = OpenedConnection() + opened_session = OpenedSession() attempts: list[int] = [] - def connect_on_second_attempt() -> OpenedConnection: + def connect_on_second_attempt() -> OpenedSession: attempts.append(1) if len(attempts) == 1: raise _connection_limit_error() - return opened_connection + return opened_session assert ( - open_connection_with_retry( + open_session_with_retry( connect_on_second_attempt, operation_name="create_or_resolve_report", runtime=FailingRuntime(), @@ -206,7 +206,7 @@ def connect_on_second_attempt() -> OpenedConnection: monotonic=clock.monotonic, ), ) - is opened_connection + is opened_session ) @@ -216,14 +216,14 @@ def test_retry_is_logged_when_no_observability_runtime_is_available( clock = ManualClock() attempts: list[int] = [] - def connect_on_second_attempt() -> OpenedConnection: + def connect_on_second_attempt() -> OpenedSession: attempts.append(1) if len(attempts) == 1: raise _connection_limit_error() - return OpenedConnection() + return OpenedSession() with caplog.at_level(logging.WARNING, logger=database_retry.__name__): - open_connection_with_retry( + open_session_with_retry( connect_on_second_attempt, operation_name="replace_simulation", timing=RetryTiming( @@ -249,7 +249,7 @@ def never_connect() -> None: raise _connection_limit_error() with pytest.raises(OperationalError): - open_connection_with_retry( + open_session_with_retry( never_connect, operation_name="get_report", timing=RetryTiming( @@ -271,7 +271,7 @@ def slowly_refuse() -> None: raise _connection_limit_error() with pytest.raises(OperationalError): - open_connection_with_retry( + open_session_with_retry( slowly_refuse, operation_name="get_report", timing=RetryTiming( diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_local_area_aggregation.py b/projects/policyengine-simulation-executor/tests/test_stage12_local_area_aggregation.py index a0331f394..5922cb838 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_local_area_aggregation.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_local_area_aggregation.py @@ -28,10 +28,8 @@ build_aggregate_report, local_area_datasets, ) -from policyengine_stage12_persistence.report_run_results import ( - report_run_result, - withheld_area_codes, -) +from policyengine_simulation_persistence import WithheldAreaCode +from policyengine_stage12_persistence.report_run_results import report_run_result from stage12_fake_report_runs import REPORT_RUN_IDS from stage12_fake_stores import FakeArtifacts from stage12_runtime_support import bundle_provenance, report_execution_input @@ -277,7 +275,7 @@ def test_a_local_area_aggregate_is_stored_with_its_withheld_codes(monkeypatch) - # The coordinator validates the aggregate, then stores its result. result = AggregateReportArtifactPayload.model_validate(aggregate).result stored = report_run_result(result, run_ids=REPORT_RUN_IDS, country="uk") - (withheld,) = [rows for rows in stored.rows if rows.table is withheld_area_codes] + (withheld,) = [rows for rows in stored.rows if rows.model is WithheldAreaCode] assert [(row["area_type"], row["code"]) for row in withheld.rows] == [ ("constituency", "W07000081"), ("local_authority", "E06000064"), diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_report_run_coordinator.py b/projects/policyengine-simulation-executor/tests/test_stage12_report_run_coordinator.py index d61bc1d91..3676fd582 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_report_run_coordinator.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_report_run_coordinator.py @@ -139,7 +139,7 @@ def test_coordinator_completes_its_report_run_with_the_economy_output_as_rows() _coordinate(report_runs, invoker, artifacts) result = report_runs.report_runs[REPORT_RUN_ID].result - rows = {rows.table.name: rows.rows for rows in result.rows} + rows = {rows.model.__tablename__: rows.rows for rows in result.rows} budget = {row["variable_name"]: row for row in rows["budget_summary"]} assert budget["household_tax"]["change"] == 100.0 assert budget["household_tax"]["report_run_id"] == REPORT_RUN_ID diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_runtime_store_retries.py b/projects/policyengine-simulation-executor/tests/test_stage12_runtime_store_retries.py index 4ae09c3e5..8fd7fd8c4 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_runtime_store_retries.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_runtime_store_retries.py @@ -2,13 +2,23 @@ from __future__ import annotations -from contextlib import contextmanager +from datetime import UTC, datetime +from typing import Any +from uuid import UUID import pytest from policyengine_simulation_contract.stage12_execution import SimulationRole -from policyengine_simulation_executor.stage12_runtime import database_retry +from policyengine_simulation_executor.stage12_runtime import ( + database_retry, + household_report_recording, +) from policyengine_simulation_executor.stage12_runtime.dependencies import runtime_store -from policyengine_stage12_persistence import store as persistence_store_module +from policyengine_simulation_executor.stage12_runtime.report_run_access import ( + MODAL_WORKER_DATABASE_URL_ENV, + budget_window_store, + report_run_store, +) +from policyengine_stage12_persistence import engine as persistence_engine_module from sqlalchemy.exc import OperationalError from stage12_runtime_support import ( TelemetryRecordingRuntime, @@ -18,6 +28,9 @@ planned_simulation, ) +REPORT_RUN_ID = UUID("00000000-0000-0000-0000-00000000000a") +CLAIMED_AT = datetime(2026, 11, 4, tzinfo=UTC) + class ScriptedResult: def __init__(self, row): @@ -32,14 +45,23 @@ def one_or_none(self): def one(self): return self.row + def scalar_one(self): + return self.row -class ScriptedConnection: - """Answer each statement with the next scripted row.""" + def scalar_one_or_none(self): + return self.row - def __init__(self, rows, *, commit_error=None): - self.rows = list(rows) - self.statement_count = 0 - self.commit_error = commit_error + +class ScriptedSession: + """Answer each statement after the first with the next scripted row. + + As on a real session, the first statement checks out the connection, which + the pooler may refuse. + """ + + def __init__(self, pooler): + self.pooler = pooler + self.connected = False def __enter__(self): return self @@ -47,39 +69,52 @@ def __enter__(self): def __exit__(self, *_exception_details): return None - @contextmanager - def begin(self): - yield - if self.commit_error is not None: - raise self.commit_error + def close(self): + return None - def execute(self, _statement): - self.statement_count += 1 - return ScriptedResult(self.rows.pop(0)) + def commit(self): + if self.pooler.commit_error is not None: + raise self.pooler.commit_error + def execute(self, _statement, *_parameters, **_options): + if not self.connected: + self.pooler.connect() + self.connected = True + return ScriptedResult(row=True) + self.pooler.statement_count += 1 + return ScriptedResult(self.pooler.rows.pop(0)) -class BusyPoolerEngine: - """Refuse the first connections, as a session pooler at its limit does.""" + +class BusyPooler: + """A session factory refusing the first connections, as a full pooler does.""" def __init__(self, rows, *, refused_connections=0, commit_error=None): - self.connection = ScriptedConnection(rows, commit_error=commit_error) + self.rows = list(rows) + self.statement_count = 0 + self.commit_error = commit_error self.refused_connections = refused_connections self.connection_attempts = 0 + def __call__(self): + return ScriptedSession(self) + def connect(self): self.connection_attempts += 1 if self.connection_attempts <= self.refused_connections: raise connection_limit_error() - return self.connection -def _runtime_store_over(engine, monkeypatch, runtime): +def _use_pooler(pooler, monkeypatch) -> None: monkeypatch.setattr(database_retry, "INITIAL_RETRY_DELAY_SECONDS", 0.0) monkeypatch.setattr( - persistence_store_module, - "create_stage12_engine", - lambda _database_url: engine, + persistence_engine_module, + "create_stage12_session_factory", + lambda _database_url: pooler, ) + + +def _runtime_store_over(pooler, monkeypatch, runtime): + _use_pooler(pooler, monkeypatch) return runtime_store(runtime) @@ -88,20 +123,20 @@ def test_runtime_store_retries_a_refused_connection_for_child_state_creation( ) -> None: ordered_events: list[tuple[str, object]] = [] child = child_record(planned_simulation(SimulationRole.BASELINE)) - engine = BusyPoolerEngine( + pooler = BusyPooler( [parent_record().model_dump(mode="python"), child.model_dump(mode="python")], refused_connections=1, ) store = _runtime_store_over( - engine, monkeypatch, TelemetryRecordingRuntime(ordered_events) + pooler, monkeypatch, TelemetryRecordingRuntime(ordered_events) ) result = store.create_or_resolve_simulation(child) assert result.created is True assert result.record == child - assert engine.connection_attempts == 2 - assert engine.connection.statement_count == 2 + assert pooler.connection_attempts == 2 + assert pooler.statement_count == 2 [(event_kind, (message, attributes, error))] = ordered_events assert event_kind == "telemetry" assert message == "Stage 12 database connection unavailable; retrying" @@ -118,18 +153,56 @@ def test_runtime_store_does_not_retry_a_failed_commit(monkeypatch) -> None: Exception("server closed the connection unexpectedly"), ) report = parent_record() - engine = BusyPoolerEngine( + pooler = BusyPooler( [report.model_dump(mode="python")], commit_error=lost_commit_reply, ) store = _runtime_store_over( - engine, monkeypatch, TelemetryRecordingRuntime(ordered_events) + pooler, monkeypatch, TelemetryRecordingRuntime(ordered_events) ) with pytest.raises(OperationalError) as failure: store.create_or_resolve_report(report) assert failure.value is lost_commit_reply - assert engine.connection_attempts == 1 - assert engine.connection.statement_count == 1 + assert pooler.connection_attempts == 1 + assert pooler.statement_count == 1 assert ordered_events == [] + + +def _claim_a_report_run(store) -> Any: + return store.claim_report_run( + REPORT_RUN_ID, modal_call_id="fc-call", claimed_at=CLAIMED_AT + ) + + +@pytest.mark.parametrize( + ("build_store", "write"), + [ + pytest.param(report_run_store, _claim_a_report_run, id="report-runs"), + pytest.param( + household_report_recording.household_report_run_store, + _claim_a_report_run, + id="household-reports", + ), + pytest.param( + budget_window_store, + lambda store: store.window_of_year(REPORT_RUN_ID), + id="budget-windows", + ), + ], +) +def test_the_modal_worker_stores_retry_a_refused_connection( + build_store, write, monkeypatch +) -> None: + ordered_events: list[tuple[str, object]] = [] + monkeypatch.setenv(MODAL_WORKER_DATABASE_URL_ENV, "postgresql://user@host/database") + pooler = BusyPooler(["running"], refused_connections=1) + _use_pooler(pooler, monkeypatch) + + write(build_store(TelemetryRecordingRuntime(ordered_events))) + + assert pooler.connection_attempts == 2 + assert pooler.statement_count == 1 + [(event_kind, _event)] = ordered_events + assert event_kind == "telemetry" diff --git a/projects/policyengine-simulation-executor/uv.lock b/projects/policyengine-simulation-executor/uv.lock index a129f4d4c..f99b7392f 100644 --- a/projects/policyengine-simulation-executor/uv.lock +++ b/projects/policyengine-simulation-executor/uv.lock @@ -1813,7 +1813,6 @@ dependencies = [ { name = "opentelemetry-sdk" }, { name = "pyjwt" }, { name = "python-json-logger" }, - { name = "sqlmodel" }, { name = "uvicorn" }, ] @@ -1833,7 +1832,6 @@ requires-dist = [ { name = "pytest-asyncio", marker = "extra == 'test'", specifier = ">=0.25.3" }, { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=6.1.1" }, { name = "python-json-logger", specifier = ">=3.2.1,<4.0.0" }, - { name = "sqlmodel", specifier = ">=0.0.22,<0.0.23" }, { name = "uvicorn", specifier = ">=0.35.0" }, ] provides-extras = ["test", "build"] @@ -2784,19 +2782,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/30/8519fdde58a7bdf155b714359791ad1dc018b47d60269d5d160d311fdc36/sqlalchemy-2.0.49-py3-none-any.whl", hash = "sha256:ec44cfa7ef1a728e88ad41674de50f6db8cfdb3e2af84af86e0041aaf02d43d0", size = 1942158, upload-time = "2026-04-03T16:53:44.135Z" }, ] -[[package]] -name = "sqlmodel" -version = "0.0.22" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "sqlalchemy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b5/39/8641040ab0d5e1d8a1c2325ae89a01ae659fc96c61a43d158fb71c9a0bf0/sqlmodel-0.0.22.tar.gz", hash = "sha256:7d37c882a30c43464d143e35e9ecaf945d88035e20117bf5ec2834a23cbe505e", size = 116392, upload-time = "2024-08-31T09:43:24.088Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dd/b1/3af5104b716c420e40a6ea1b09886cae3a1b9f4538343875f637755cae5b/sqlmodel-0.0.22-py3-none-any.whl", hash = "sha256:a1ed13e28a1f4057cbf4ff6cdb4fc09e85702621d3259ba17b3c230bfb2f941b", size = 28276, upload-time = "2024-08-31T09:43:22.358Z" }, -] - [[package]] name = "stack-data" version = "0.6.3" diff --git a/projects/policyengine-simulation-gateway/uv.lock b/projects/policyengine-simulation-gateway/uv.lock index be91bef48..f7f3ae3e6 100644 --- a/projects/policyengine-simulation-gateway/uv.lock +++ b/projects/policyengine-simulation-gateway/uv.lock @@ -595,22 +595,6 @@ grpc = [ { name = "grpcio" }, ] -[[package]] -name = "greenlet" -version = "3.5.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e2/f1/fbbfef6af0bad0548f09bc28948ea3c275b4edb19e17fc5ca9900a6a634d/greenlet-3.5.3.tar.gz", hash = "sha256:a61efc018fd3eb317eeca31aba90ee9e7f26f22884a79b6c6ec715bf71bb62f1", size = 200270, upload-time = "2026-06-26T19:28:24.832Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9b/ff/a620267401db30a50cc8450ee90730e2d4a85658c055c0e760d4ed47fb13/greenlet-3.5.3-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:c8d87c2134d871df96ecdea9cec7cbaab286dadab0f56476e57aaf9e8ac11550", size = 287609, upload-time = "2026-06-26T18:21:14.724Z" }, - { url = "https://files.pythonhosted.org/packages/d6/fa/5401ac78021c826a25b6dde0c705e0a8f29b617509f9185a31dac15fbe1b/greenlet-3.5.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2d185dd1621757e70c3861cceffd5317ab4e7ed7eb09c82994828468527ade5", size = 607435, upload-time = "2026-06-26T19:07:11.412Z" }, - { url = "https://files.pythonhosted.org/packages/e9/76/1dc144a2e56e65d36405078ed774224375ea520a1870a6e46e08bb4ac7bf/greenlet-3.5.3-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1c514a468149bf8fbbab874188a3535cd8a48a3e353eb53a3d424296f8dbacd3", size = 619787, upload-time = "2026-06-26T19:10:08.396Z" }, - { url = "https://files.pythonhosted.org/packages/bf/87/c298cee62df1de4ad7fec32abda73526cff347fd143a6ed4ac369246668a/greenlet-3.5.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:915f887cf2682b66419b879423a2e072634aa7b7dce6f3ada4957cfced3f1e9a", size = 616786, upload-time = "2026-06-26T18:32:19.128Z" }, - { url = "https://files.pythonhosted.org/packages/9e/2e/e6f009885ed0705ccf33fe0583c117cfd03cde77e31a596dd5785a30762b/greenlet-3.5.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:766cfd421c13e450feb340cd472a3ed9957d438727b7b4593ad7c76c5d2b0deb", size = 1574316, upload-time = "2026-06-26T19:09:04.273Z" }, - { url = "https://files.pythonhosted.org/packages/ef/fe/43fd110b01e40da0adb7c90ac7ea744bef2d43dca00de5095fd2351c2a68/greenlet-3.5.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2ecda9ec22edf38fa389369eaed8c3d37c05f3c54e69f69438dbb2cc1de1458b", size = 1638614, upload-time = "2026-06-26T18:31:46.297Z" }, - { url = "https://files.pythonhosted.org/packages/0f/7c/062447147a61f8b4337b156fe70d32a165fcf2f89d7ca6255e572806705c/greenlet-3.5.3-cp313-cp313-win_amd64.whl", hash = "sha256:c82304750f057167ff60d188df1d0cc1764ce9567eadf03e6a7443bcedd0b30b", size = 239850, upload-time = "2026-06-26T18:21:54.613Z" }, - { url = "https://files.pythonhosted.org/packages/c7/7e/220a7f5824a64a60443fc03b39dfac4ea63a7fb6d481efa27eafa928e7f4/greenlet-3.5.3-cp313-cp313-win_arm64.whl", hash = "sha256:dc133a1569ee667b2a6ef56ce551084aeefd87a5acbc4736d336d1e2edc6cfc4", size = 238141, upload-time = "2026-06-26T18:22:48.507Z" }, -] - [[package]] name = "grpc-google-iam-v1" version = "0.14.5" @@ -1234,7 +1218,6 @@ dependencies = [ { name = "opentelemetry-sdk" }, { name = "pyjwt" }, { name = "python-json-logger" }, - { name = "sqlmodel" }, { name = "uvicorn" }, ] @@ -1254,7 +1237,6 @@ requires-dist = [ { name = "pytest-asyncio", marker = "extra == 'test'", specifier = ">=0.25.3" }, { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=6.1.1" }, { name = "python-json-logger", specifier = ">=3.2.1,<4.0.0" }, - { name = "sqlmodel", specifier = ">=0.0.22,<0.0.23" }, { name = "uvicorn", specifier = ">=0.35.0" }, ] provides-extras = ["test", "build"] @@ -1795,39 +1777,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, ] -[[package]] -name = "sqlalchemy" -version = "2.0.51" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/02/f1/a7a892f18d4d224e6b26f706531eafccc41e37594d37d304786969ee13cb/sqlalchemy-2.0.51.tar.gz", hash = "sha256:804dccd8a4a6242c4e30ad961e540e18a588f6527202f2d6791b01845d59fdc9", size = 9912201, upload-time = "2026-06-15T15:41:20.012Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/54/fe/a210d52fd1a90ecfae8a78e9d8b27e18d733d60818a8bf250ff690b75120/sqlalchemy-2.0.51-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7c2056838b6685b72fdb36c99996cf862753461a62f2e84f4196371d3b2d6a07", size = 2157184, upload-time = "2026-06-15T16:08:50.374Z" }, - { url = "https://files.pythonhosted.org/packages/17/6b/2dce8369b199cb855110e056032f94a9f66dacc2237d3d39c115a86eac56/sqlalchemy-2.0.51-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:483b11bd46bf35fc14c52faf338b04300c9e6ce554bce9b11be85bfec3bc3195", size = 3284735, upload-time = "2026-06-15T16:19:46.934Z" }, - { url = "https://files.pythonhosted.org/packages/53/ff/dbc495b8a14da840faffb353857a72d4190113cac33727906fb997047f0f/sqlalchemy-2.0.51-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1bed1ee8b01da6088210aa9412023326fb98a599ba502e6118308601dcbef77f", size = 3302756, upload-time = "2026-06-15T16:26:41.336Z" }, - { url = "https://files.pythonhosted.org/packages/cf/d5/fde8f4dddcf518ee15ab35a7c6a28acc32c8ba548d1d2aa451f96e6dbb0b/sqlalchemy-2.0.51-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:72ca54c952107ba5cd58854b67a5a6268631289d21651a1235396f3b98b47400", size = 3232055, upload-time = "2026-06-15T16:19:49.286Z" }, - { url = "https://files.pythonhosted.org/packages/67/d1/43d3a0ac955a58601c24fa23038b1c55ee3a1ec02c0f96ebb1eae2bcf614/sqlalchemy-2.0.51-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b3e693d15533a45cd5906f0589f9c35090bef6ef45bf1e8195c424aa0ae06a8d", size = 3269850, upload-time = "2026-06-15T16:26:43.017Z" }, - { url = "https://files.pythonhosted.org/packages/94/df/de669c7054cd47c4439ac34b1b2ee8b804a794791fbb10720e997a2c87c7/sqlalchemy-2.0.51-cp313-cp313-win32.whl", hash = "sha256:b93ab07b5292dbe7e6b8da89475275e7042744283921344b56105f3eeb0f828b", size = 2117721, upload-time = "2026-06-15T16:23:12.36Z" }, - { url = "https://files.pythonhosted.org/packages/d0/8a/403c51d064196bae20a0bc2476577f83a3f8dd299719a97417086b7f2ec5/sqlalchemy-2.0.51-cp313-cp313-win_amd64.whl", hash = "sha256:0f053118c30e53161857a953e4de667d90e274980dccbe5dd3829bbbeece72a5", size = 2143615, upload-time = "2026-06-15T16:23:13.906Z" }, - { url = "https://files.pythonhosted.org/packages/e2/22/dbf013a12ec759e54a34a119e9e217435b3f71b2dd5c61a7ade0a25dae87/sqlalchemy-2.0.51-py3-none-any.whl", hash = "sha256:bb024d8b621d0be75f4f44ecc7c950450026e76d66dc8f791bb5331d7fed59d5", size = 1944334, upload-time = "2026-06-15T16:09:22.418Z" }, -] - -[[package]] -name = "sqlmodel" -version = "0.0.22" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "sqlalchemy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b5/39/8641040ab0d5e1d8a1c2325ae89a01ae659fc96c61a43d158fb71c9a0bf0/sqlmodel-0.0.22.tar.gz", hash = "sha256:7d37c882a30c43464d143e35e9ecaf945d88035e20117bf5ec2834a23cbe505e", size = 116392, upload-time = "2024-08-31T09:43:24.088Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dd/b1/3af5104b716c420e40a6ea1b09886cae3a1b9f4538343875f637755cae5b/sqlmodel-0.0.22-py3-none-any.whl", hash = "sha256:a1ed13e28a1f4057cbf4ff6cdb4fc09e85702621d3259ba17b3c230bfb2f941b", size = 28276, upload-time = "2024-08-31T09:43:22.358Z" }, -] - [[package]] name = "starlette" version = "0.46.2" diff --git a/rules/fixtures/sim.persistence.orm-only/bad_household_job_records.py b/rules/fixtures/sim.persistence.orm-only/bad_household_job_records.py new file mode 100644 index 000000000..d993c6fb1 --- /dev/null +++ b/rules/fixtures/sim.persistence.orm-only/bad_household_job_records.py @@ -0,0 +1,69 @@ +"""Proof fixture: household job records written on SQLAlchemy Core. + +The rule must reject it. The table, the insert and the call recording are the +Core code policyengine-simulation-persistence has until its conversion, +shortened: household_jobs is a Core Table, written through a Connection. The +engine factory imports create_engine outside a library's engine.py, and the +count runs SQL text; both are illustrative, not taken from the library. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sqlalchemy import Column, MetaData, String, Table, Uuid, insert, text, update +from sqlalchemy import create_engine as build_engine + +if TYPE_CHECKING: + from uuid import UUID + + from sqlalchemy.engine import Connection, Engine + +service_metadata = MetaData(schema="public") +household_jobs = Table( + "household_jobs", + service_metadata, + Column("id", Uuid, primary_key=True), + Column("status", String, nullable=False), + Column("modal_call_id", String, nullable=True), +) + + +def create_household_engine(database_url: str) -> Engine: + """Build an engine outside the library's session factory.""" + return build_engine(database_url) + + +def insert_household_job(connection: Connection, job_id: UUID) -> None: + """Create the job ``pending``, with no Modal call ID.""" + connection.execute( + insert(household_jobs).values(id=job_id, status="pending", modal_call_id=None) + ) + + +def record_household_call( + connection: Connection, *, job_id: UUID, modal_call_id: str +) -> str | None: + """Record the job's Modal call ID while it is pending with none.""" + return connection.execute( + update(household_jobs) + .where( + household_jobs.c.id == job_id, + household_jobs.c.modal_call_id.is_(None), + ) + .values(modal_call_id=modal_call_id) + .returning(household_jobs.c.status) + ).scalar_one_or_none() + + +def count_pending_jobs(engine: Engine) -> int: + """Count pending jobs with SQL text on a connection of its own.""" + with engine.connect() as connection: + return connection.execute( + text("SELECT count(*) FROM household_jobs WHERE status = 'pending'") + ).scalar_one() + + +def household_job_columns(jobs: Table) -> list[str]: + """Read column names from a Core table passed in.""" + return list(jobs.c.keys()) diff --git a/rules/fixtures/sim.persistence.orm-only/good_household_job_records.py b/rules/fixtures/sim.persistence.orm-only/good_household_job_records.py new file mode 100644 index 000000000..49af3887e --- /dev/null +++ b/rules/fixtures/sim.persistence.orm-only/good_household_job_records.py @@ -0,0 +1,66 @@ +"""Proof fixture: household job records written on the SQLAlchemy ORM. + +The rule must accept it. It is the bad fixture's records rewritten: the table +is a typed mapped class, every statement names the class or its attributes, +and each function runs on the ORM Session it is given. A partial index's +condition is the one place SQL text may appear, as a model's schema detail. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, ClassVar +from uuid import UUID # noqa: TC003 # SQLAlchemy reads Mapped annotations at run time + +from sqlalchemy import Index, String, func, select, text, update +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column + +if TYPE_CHECKING: + from sqlalchemy.orm import Session + + +class Base(DeclarativeBase): + """The declarative base every mapped class derives from.""" + + +class HouseholdJob(Base): + """One household calculation, as a row of household_jobs.""" + + __tablename__ = "household_jobs" + __table_args__: ClassVar = ( + Index( + "ix_household_jobs_pending", + "status", + postgresql_where=text("status = 'pending'"), + ), + ) + + id: Mapped[UUID] = mapped_column(primary_key=True) + status: Mapped[str] = mapped_column(String) + modal_call_id: Mapped[str | None] = mapped_column(String) + + +def insert_household_job(session: Session, job_id: UUID) -> None: + """Create the job ``pending``, with no Modal call ID.""" + session.add(HouseholdJob(id=job_id, status="pending", modal_call_id=None)) + session.flush() + + +def record_household_call( + session: Session, *, job_id: UUID, modal_call_id: str +) -> str | None: + """Record the job's Modal call ID while it is pending with none.""" + return session.execute( + update(HouseholdJob) + .where(HouseholdJob.id == job_id, HouseholdJob.modal_call_id.is_(None)) + .values(modal_call_id=modal_call_id) + .returning(HouseholdJob.status) + ).scalar_one_or_none() + + +def count_pending_jobs(session: Session) -> int: + """Count pending jobs with a typed select on the mapped class.""" + return session.scalars( + select(func.count()) + .select_from(HouseholdJob) + .where(HouseholdJob.status == "pending") + ).one() diff --git a/rules/registry.toml b/rules/registry.toml index 7314d8237..a9ecfbc63 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -136,6 +136,34 @@ good_fixture = "rules/fixtures/sim.boundaries.sqlalchemy-and-alembic-only/good_d project = "projects/policyengine-simulation-entry" test = "tests/test_database_access_boundaries.py::test_rule_proof_sqlalchemy_and_alembic_only" +[[rule]] +id = "sim.persistence.orm-only" +category = "boundaries" +statement = "Code that reads or writes database rows declares tables only as SQLAlchemy ORM mapped classes and runs every statement through an ORM Session against those classes, never through Core Table objects or Connections." +rationale = "docs/engineering/skills/database-access.md#orm" +scope = ["projects/*/src/**/*.py", "libs/*/src/**/*.py", ".github/scripts/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = [ + "tests/test_orm_only.py::test_database_code_uses_only_the_orm", + "tests/test_orm_only.py::test_every_table_is_mapped_by_a_class", + "tests/test_orm_only.py::test_no_project_installs_sqlmodel", + "tests/test_database_access_boundaries.py::test_every_source_directory_is_scanned", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run --frozen pytest tests/" +[rule.proof] +bad_fixture = "rules/fixtures/sim.persistence.orm-only/bad_household_job_records.py" +good_fixture = "rules/fixtures/sim.persistence.orm-only/good_household_job_records.py" +project = "projects/policyengine-simulation-entry" +test = "tests/test_orm_only.py::test_rule_proof_orm_only" + # --- Simulation service database ------------------------------------------ [[rule]] @@ -177,6 +205,8 @@ category = "persistence" statement = "Sim's one set of models of the Public API's tables, those the simulation service writes and the result tables the v2 Modal calls write, match the tables the Public API's v2 migration chain builds at the pinned Public API commit, which is on the Public API's default branch: every column's type, nullability and default, every constraint and index by name and definition, and every enum's values; every column its read-only models of the Public API's input tables name exists there with the same type and nullability; and, with the privileges the Public API's manifest declares, the simulation service's role writes every table the service writes, and the Modal worker role records its runs and inserts a row in every result table." rationale = "libs/policyengine-simulation-persistence/README.md#checking-the-models-against-the-public-api" scope = [ + "libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/orm_base.py", + "libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/referenced_tables.py", "libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/tables.py", "libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/result_tables.py", "libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/input_tables.py", @@ -278,7 +308,7 @@ invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.stage12.persistence-typed-dml-only" category = "stage12" -statement = "The Stage 12 persistence library and runtime use typed SQLAlchemy Core DML only: no raw SQL text, driver-level SQL or schema operations." +statement = "The Stage 12 persistence library and runtime use typed SQLAlchemy statements only: no raw SQL text, driver-level SQL or schema operations." rationale = "libs/policyengine-stage12-persistence/README.md" scope = ["libs/policyengine-stage12-persistence/src/**/*.py"] tier = "mechanical"