From 0d635a20b40da28e18608aabacc0b58d74f32f83 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:36:45 +0400 Subject: [PATCH 01/10] Test a cliff result per simulation run and its grants (failing) Each simulation run can have one cliff calculation that hasn't failed, holding its state and, once it succeeds, the run's cliff gap and cliff share. The tests cover the model, its checks and partial unique index, the revision that adds the table, and the grants: the simulation service starts calculations, the Modal workers finish them, the Public API reads them, and the catalog writer and the Stage 12 role can do neither. Co-Authored-By: Claude Opus 5.5 --- .../test_alembic_v2_cliff_impacts.py | 369 ++++++++++++++++++ .../test_v2_simulation_role_privileges.py | 159 +++++++- tests/unit/test_alembic_workflows.py | 1 + tests/unit/v2/test_alembic_v2_revisions.py | 62 ++- tests/unit/v2/test_cliff_impact_models.py | 224 +++++++++++ tests/unit/v2/test_privilege_manifest.py | 33 +- tests/unit/v2/test_simulation_run_models.py | 3 +- 7 files changed, 844 insertions(+), 7 deletions(-) create mode 100644 tests/integration/test_alembic_v2_cliff_impacts.py create mode 100644 tests/unit/v2/test_cliff_impact_models.py diff --git a/tests/integration/test_alembic_v2_cliff_impacts.py b/tests/integration/test_alembic_v2_cliff_impacts.py new file mode 100644 index 000000000..56f9b3013 --- /dev/null +++ b/tests/integration/test_alembic_v2_cliff_impacts.py @@ -0,0 +1,369 @@ +"""Check the cliff impacts revision against disposable Postgres. + +Revision bc334f9877b7, on the user invitations revision f519a77ba122, adds +``cliff_impacts``: one row per cliff calculation of a simulation run, with its +state and, once it succeeds, the run's cliff gap and cliff share. Its status +reuses the ``v2_simulation_status`` type, and it changes no other table. The +downgrade drops the table with its rows and leaves the type, which +``simulation_runs`` still uses. +""" + +from collections.abc import Iterator +from typing import Any +from uuid import UUID, uuid4 + +import pytest +from alembic import command +from sqlalchemy import Connection, Engine, create_engine, inspect, text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.pool import NullPool + +from tests.fixtures.integration.alembic_v2 import ( + alembic_v2_config, + disposable_v2_url, +) +from tests.integration.simulation_rows import ( + RUN_VERSIONS, + insert_public_api_rows, + insert_row, + refused, +) + +USER_INVITATIONS_REVISION = "f519a77ba122" +CLIFF_IMPACTS_REVISION = "bc334f9877b7" +# What a request starting a run's cliff calculation inserts: the conflict +# target names the partial unique index's whole condition. +START_CLIFF_CALCULATION = ( + "INSERT INTO cliff_impacts (id, simulation_run_id, status) " + "VALUES (:id, :simulation_run_id, 'pending') " + "ON CONFLICT (simulation_run_id) WHERE status <> 'failed' DO NOTHING " + "RETURNING id" +) + + +def _schema_shape(engine: Engine) -> dict[str, dict[str, Any]]: + """Reflect every table's columns, keys, indexes and checks.""" + inspector = inspect(engine) + return { + table_name: { + "columns": [ + ( + column["name"], + str(column["type"]), + column["nullable"], + column.get("default"), + ) + for column in inspector.get_columns(table_name) + ], + "primary_key": inspector.get_pk_constraint(table_name), + "foreign_keys": sorted( + ( + key["name"], + key["constrained_columns"], + key["referred_table"], + key["options"].get("ondelete"), + ) + for key in inspector.get_foreign_keys(table_name) + ), + "unique_constraints": sorted( + (unique["name"], unique["column_names"]) + for unique in inspector.get_unique_constraints(table_name) + ), + "indexes": sorted( + ( + index["name"], + index["column_names"], + index["unique"], + index.get("dialect_options", {}).get("postgresql_where"), + ) + for index in inspector.get_indexes(table_name) + ), + "checks": { + check["name"]: check["sqltext"] + for check in inspector.get_check_constraints(table_name) + }, + } + for table_name in inspector.get_table_names(schema="public") + if table_name != "alembic_version" + } + + +def _enum_labels(engine: Engine) -> dict[str, list[str]]: + """Return each enum type in the public schema with its labels in order.""" + with engine.connect() as connection: + rows = connection.execute( + text( + "SELECT type.typname, enum.enumlabel FROM pg_type AS type " + "JOIN pg_enum AS enum ON enum.enumtypid = type.oid " + "JOIN pg_namespace AS namespace ON namespace.oid = type.typnamespace " + "WHERE namespace.nspname = 'public' " + "ORDER BY type.typname, enum.enumsortorder" + ) + ).all() + labels: dict[str, list[str]] = {} + for type_name, label in rows: + labels.setdefault(type_name, []).append(label) + return labels + + +@pytest.fixture +def engine() -> Iterator[Engine]: + """Yield an engine on the disposable database, left at the v2 head.""" + database_engine = create_engine(disposable_v2_url(), poolclass=NullPool) + try: + yield database_engine + finally: + command.upgrade(alembic_v2_config(), "head") + database_engine.dispose() + + +def test_cliff_impacts_revision_adds_only_its_table(engine: Engine) -> None: + """The revision adds one table, with no new enum type, and changes nothing else.""" + config = alembic_v2_config() + + command.upgrade(config, "head") + command.downgrade(config, USER_INVITATIONS_REVISION) + parent_shape = _schema_shape(engine) + parent_enums = _enum_labels(engine) + command.upgrade(config, CLIFF_IMPACTS_REVISION) + revision_shape = _schema_shape(engine) + + assert set(revision_shape) - set(parent_shape) == {"cliff_impacts"} + assert { + table_name: shape + for table_name, shape in revision_shape.items() + if table_name != "cliff_impacts" + } == parent_shape + assert _enum_labels(engine) == parent_enums + cliff_impacts = revision_shape["cliff_impacts"] + assert cliff_impacts["primary_key"]["constrained_columns"] == ["id"] + assert [ + (columns, referred_table, ondelete) + for _name, columns, referred_table, ondelete in cliff_impacts["foreign_keys"] + ] == [(["simulation_run_id"], "simulation_runs", "CASCADE")] + assert cliff_impacts["unique_constraints"] == [] + assert cliff_impacts["indexes"] == [ + ("ix_cliff_impacts_simulation_run_id", ["simulation_run_id"], False, None), + ("ix_cliff_impacts_status_started_at", ["status", "started_at"], False, None), + ( + "uq_cliff_impacts_unfailed", + ["simulation_run_id"], + True, + "(status <> 'failed'::v2_simulation_status)", + ), + ] + assert set(cliff_impacts["checks"]) == { + "ck_cliff_impacts_finished_completion", + "ck_cliff_impacts_result_when_succeeded", + } + with engine.connect() as connection: + column_types = dict( + connection.execute( + text( + "SELECT column_name, udt_name || ':' || is_nullable " + "FROM information_schema.columns " + "WHERE table_schema = 'public' AND table_name = 'cliff_impacts'" + ) + ).all() + ) + assert column_types == { + "id": "uuid:NO", + "created_at": "timestamptz:NO", + "updated_at": "timestamptz:NO", + "simulation_run_id": "uuid:NO", + "status": "v2_simulation_status:NO", + "modal_call_id": "varchar:YES", + "error_message": "varchar:YES", + "started_at": "timestamptz:YES", + "completed_at": "timestamptz:YES", + "cliff_gap": "float8:YES", + "cliff_share": "float8:YES", + } + + +def test_cliff_impacts_downgrade_drops_the_table_and_keeps_the_status_type( + engine: Engine, +) -> None: + """Downgrading drops only the table, so upgrading again works.""" + config = alembic_v2_config() + + command.upgrade(config, "head") + command.downgrade(config, USER_INVITATIONS_REVISION) + + assert "cliff_impacts" not in inspect(engine).get_table_names() + # ``simulation_runs.status`` still uses the type. + assert _enum_labels(engine)["v2_simulation_status"] == [ + "pending", + "running", + "succeeded", + "failed", + ] + + command.upgrade(config, CLIFF_IMPACTS_REVISION) + + assert "cliff_impacts" in inspect(engine).get_table_names() + + +def _insert_simulation_run(connection: Connection) -> UUID: + """Store an economy simulation and one pending run of it.""" + public_api_rows = insert_public_api_rows(connection) + simulation_id = uuid4() + run_id = uuid4() + insert_row( + connection, + "simulations", + id=simulation_id, + simulation_type="economy", + dataset_id=public_api_rows.dataset_id, + tax_benefit_model_version_id=public_api_rows.version_id, + year=2026, + ) + insert_row( + connection, + "simulation_runs", + id=run_id, + simulation_id=simulation_id, + tax_benefit_model_version_id=public_api_rows.version_id, + year=2026, + status="pending", + **RUN_VERSIONS, + ) + return run_id + + +def _start_cliff_calculation(connection: Connection, run_id: UUID) -> UUID | None: + """Insert a pending calculation unless one that hasn't failed exists.""" + return connection.execute( + text(START_CLIFF_CALCULATION), {"id": uuid4(), "simulation_run_id": run_id} + ).scalar_one_or_none() + + +def _fail_cliff_calculation(connection: Connection, calculation_id: UUID) -> None: + connection.execute( + text( + "UPDATE cliff_impacts SET status = 'failed', " + "error_message = 'modal_call_ended: the call ended', " + "completed_at = now() WHERE id = :id" + ), + {"id": calculation_id}, + ) + + +def _cliff_statuses(connection: Connection, run_id: UUID) -> list[str]: + return list( + connection.execute( + text( + "SELECT status::text FROM cliff_impacts " + "WHERE simulation_run_id = :run_id ORDER BY created_at, status" + ), + {"run_id": run_id}, + ).scalars() + ) + + +def test_a_run_starts_one_cliff_calculation_until_it_fails(engine: Engine) -> None: + """Two requests at once start one calculation; a failed one never counts.""" + command.upgrade(alembic_v2_config(), "head") + + with engine.connect() as connection: + transaction = connection.begin() + try: + run_id = _insert_simulation_run(connection) + first_calculation = _start_cliff_calculation(connection, run_id) + joined_calculation = _start_cliff_calculation(connection, run_id) + + assert first_calculation is not None + # The second request joins the pending calculation. + assert joined_calculation is None + with refused(connection, IntegrityError, match="uq_cliff_impacts_unfailed"): + insert_row( + connection, + "cliff_impacts", + id=uuid4(), + simulation_run_id=run_id, + status="running", + ) + + _fail_cliff_calculation(connection, first_calculation) + retried_calculation = _start_cliff_calculation(connection, run_id) + + assert retried_calculation is not None + connection.execute( + text( + "UPDATE cliff_impacts SET status = 'succeeded', " + "cliff_gap = 1.5e9, cliff_share = 0.04, completed_at = now() " + "WHERE id = :id" + ), + {"id": retried_calculation}, + ) + assert _start_cliff_calculation(connection, run_id) is None + assert sorted(_cliff_statuses(connection, run_id)) == [ + "failed", + "succeeded", + ] + finally: + transaction.rollback() + + +@pytest.mark.parametrize( + ("assignments", "check_name"), + [ + ("status = 'succeeded', completed_at = now()", "result_when_succeeded"), + ( + "status = 'succeeded', cliff_gap = 1.0, completed_at = now()", + "result_when_succeeded", + ), + ("cliff_gap = 1.0, cliff_share = 0.1", "result_when_succeeded"), + ( + "status = 'failed', cliff_gap = 1.0, cliff_share = 0.1, " + "completed_at = now()", + "result_when_succeeded", + ), + ("status = 'failed'", "finished_completion"), + ( + "status = 'succeeded', cliff_gap = 1.0, cliff_share = 0.1", + "finished_completion", + ), + ], +) +def test_only_a_finished_calculation_ends_and_only_a_succeeded_one_has_numbers( + engine: Engine, assignments: str, check_name: str +) -> None: + command.upgrade(alembic_v2_config(), "head") + + with engine.connect() as connection: + transaction = connection.begin() + try: + run_id = _insert_simulation_run(connection) + calculation_id = _start_cliff_calculation(connection, run_id) + + with refused( + connection, IntegrityError, match=f"ck_cliff_impacts_{check_name}" + ): + connection.execute( + text(f"UPDATE cliff_impacts SET {assignments} WHERE id = :id"), + {"id": calculation_id}, + ) + finally: + transaction.rollback() + + +def test_a_runs_cliff_calculations_are_deleted_with_it(engine: Engine) -> None: + command.upgrade(alembic_v2_config(), "head") + + with engine.connect() as connection: + transaction = connection.begin() + try: + run_id = _insert_simulation_run(connection) + failed_calculation = _start_cliff_calculation(connection, run_id) + assert failed_calculation is not None + _fail_cliff_calculation(connection, failed_calculation) + _start_cliff_calculation(connection, run_id) + + connection.execute( + text("DELETE FROM simulation_runs WHERE id = :id"), {"id": run_id} + ) + + assert _cliff_statuses(connection, run_id) == [] + finally: + transaction.rollback() diff --git a/tests/integration/test_v2_simulation_role_privileges.py b/tests/integration/test_v2_simulation_role_privileges.py index 2bc48db88..944aeeb9e 100644 --- a/tests/integration/test_v2_simulation_role_privileges.py +++ b/tests/integration/test_v2_simulation_role_privileges.py @@ -1,8 +1,9 @@ """Run the simulation tables' writes as the roles that do them in production. The simulation service creates simulations, reports, their runs, budget-window -runs and household jobs; its Modal workers write their runs' status and -results and their budget-window runs' status; the Public API reads them. Each +runs, household jobs and simulation runs' cliff calculations; its Modal +workers write their runs' and cliff calculations' status and results and +their budget-window runs' status; the Public API reads them. Each role holds only its declared privileges (``policyengine_api/data/v2/privileges.py``), so a statement outside them fails here as it would in staging and production. @@ -561,6 +562,7 @@ def test_each_simulation_role_updates_exactly_its_declared_columns( "report_runs", "household_jobs", "budget_window_runs", + "cliff_impacts", ): updatable = { column @@ -577,3 +579,156 @@ def test_each_simulation_role_updates_exactly_its_declared_columns( connection.execute(text(statement)) else: _refused(connection, statement) + + +# What the simulation service inserts to start a run's cliff calculation, +# unless one that hasn't failed exists. Naming the partial unique index's +# conflict target needs SELECT as well as INSERT. +START_CLIFF_CALCULATION = ( + "INSERT INTO cliff_impacts (id, simulation_run_id, status) " + "VALUES (:id, :simulation_run_id, 'pending') " + "ON CONFLICT (simulation_run_id) WHERE status <> 'failed' DO NOTHING " + "RETURNING id" +) + + +def _start_cliff_calculation(connection: Connection, run_id: object) -> object: + return connection.execute( + text(START_CLIFF_CALCULATION), {"id": uuid4(), "simulation_run_id": run_id} + ).scalar_one_or_none() + + +def test_the_simulation_service_starts_cliff_calculations_and_records_calls( + connection: Connection, managed_v2_roles, public_api_rows: PublicApiRows +) -> None: + """The service inserts a run's calculation once, and records or fails its call. + + It never claims a calculation, writes its numbers, changes the run it + calculates or deletes it. + """ + _act_as(connection, managed_v2_roles.simulation_service) + ids = _create_simulation_rows(connection, public_api_rows) + + calculation_id = _start_cliff_calculation(connection, ids["run"]) + joined_calculation = _start_cliff_calculation(connection, ids["run"]) + recorded = connection.execute( + text( + "UPDATE cliff_impacts SET modal_call_id = 'fc-cliffs', " + "updated_at = now() WHERE id = :id AND status = 'pending' " + "AND modal_call_id IS NULL RETURNING id" + ), + {"id": calculation_id}, + ).scalar_one() + failed = connection.execute( + text( + "UPDATE cliff_impacts SET status = 'failed', " + "error_message = 'modal_refused: refused', completed_at = now(), " + "updated_at = now() WHERE id = :id AND status = 'pending' RETURNING id" + ), + {"id": calculation_id}, + ).scalar_one() + + assert calculation_id is not None + assert joined_calculation is None + assert recorded == failed == calculation_id + for statement in ( + "UPDATE cliff_impacts SET started_at = now() WHERE id = :id", + "UPDATE cliff_impacts SET cliff_gap = 1.0 WHERE id = :id", + "UPDATE cliff_impacts SET simulation_run_id = simulation_run_id WHERE id = :id", + "DELETE FROM cliff_impacts WHERE id = :id", + ): + _refused(connection, statement, id=calculation_id) + + +def test_modal_workers_claim_cliff_calculations_and_write_their_results( + connection: Connection, managed_v2_roles, public_api_rows: PublicApiRows +) -> None: + """A worker claims a pending calculation under its call and finishes it. + + It never inserts a calculation, changes the run it calculates or deletes + one. + """ + _act_as(connection, managed_v2_roles.simulation_service) + ids = _create_simulation_rows(connection, public_api_rows) + calculation_id = _start_cliff_calculation(connection, ids["run"]) + _act_as(connection, managed_v2_roles.modal_worker) + + claimed = connection.execute( + text( + "UPDATE cliff_impacts SET status = 'running', started_at = now(), " + "modal_call_id = 'fc-cliffs', updated_at = now() " + "WHERE id = :id AND status = 'pending' RETURNING id" + ), + {"id": calculation_id}, + ).scalar_one() + succeeded = connection.execute( + text( + "UPDATE cliff_impacts SET status = 'succeeded', cliff_gap = 1.5e9, " + "cliff_share = 0.04, completed_at = now(), updated_at = now() " + "WHERE id = :id AND status = 'running' RETURNING id" + ), + {"id": calculation_id}, + ).scalar_one() + + assert claimed == succeeded == calculation_id + _refused( + connection, + "INSERT INTO cliff_impacts (id, simulation_run_id, status) " + "VALUES (:id, :run_id, 'pending')", + id=uuid4(), + run_id=ids["run"], + ) + for statement in ( + "UPDATE cliff_impacts SET simulation_run_id = simulation_run_id WHERE id = :id", + "DELETE FROM cliff_impacts WHERE id = :id", + ): + _refused(connection, statement, id=calculation_id) + + +def test_the_public_api_reads_cliff_results_and_other_roles_cannot( + connection: Connection, managed_v2_roles, public_api_rows: PublicApiRows +) -> None: + """The Public API reads a report run's two cliff results and writes none. + + The catalog writer and the Stage 12 role can't read them. + """ + _act_as(connection, managed_v2_roles.simulation_service) + ids = _create_simulation_rows(connection, public_api_rows) + calculation_id = _start_cliff_calculation(connection, ids["run"]) + _act_as(connection, managed_v2_roles.modal_worker) + connection.execute( + text( + "UPDATE cliff_impacts SET status = 'succeeded', cliff_gap = 1.5e9, " + "cliff_share = 0.04, completed_at = now() WHERE id = :id" + ), + {"id": calculation_id}, + ) + _act_as(connection, managed_v2_roles.runtime) + + results = connection.execute( + text( + "SELECT cliff_impacts.cliff_gap, cliff_impacts.cliff_share " + "FROM report_runs JOIN cliff_impacts ON cliff_impacts.simulation_run_id " + "IN (report_runs.baseline_simulation_run_id, " + "report_runs.reform_simulation_run_id) " + "WHERE report_runs.id = :id AND cliff_impacts.status = 'succeeded'" + ), + {"id": ids["report_run"]}, + ).all() + + assert [tuple(result) for result in results] == [(1.5e9, 0.04)] + _refused( + connection, + "INSERT INTO cliff_impacts (id, simulation_run_id, status) " + "VALUES (:id, :run_id, 'pending')", + id=uuid4(), + run_id=ids["run"], + ) + _refused( + connection, + "UPDATE cliff_impacts SET cliff_gap = 0 WHERE id = :id", + id=calculation_id, + ) + for role in (managed_v2_roles.stage12, CATALOG_WRITER_ROLE): + _act_as(connection, role) + _refused(connection, "SELECT count(*) FROM cliff_impacts") diff --git a/tests/unit/test_alembic_workflows.py b/tests/unit/test_alembic_workflows.py index 73b063d32..7676d8a93 100644 --- a/tests/unit/test_alembic_workflows.py +++ b/tests/unit/test_alembic_workflows.py @@ -214,6 +214,7 @@ def test_reusable_v2_alembic_check_uses_only_disposable_postgres(): assert "test_alembic_v2_user_types.py" in lifecycle_script assert "test_alembic_v2_withheld_area_codes.py" in lifecycle_script assert "test_alembic_v2_shared_report_keys.py" in lifecycle_script + assert "test_alembic_v2_cliff_impacts.py" in lifecycle_script assert "redis:7.2-alpine" not in workflow assert "RUNTIME_CACHE_TEST_URL" not in workflow assert "test_v2_catalog_installed.py" not in workflow diff --git a/tests/unit/v2/test_alembic_v2_revisions.py b/tests/unit/v2/test_alembic_v2_revisions.py index 2c484b86d..108846b6c 100644 --- a/tests/unit/v2/test_alembic_v2_revisions.py +++ b/tests/unit/v2/test_alembic_v2_revisions.py @@ -21,6 +21,7 @@ POST_BASELINE_TABLE_NAMES = frozenset( { "budget_window_runs", + "cliff_impacts", "countries", "legacy_household_mappings", "legacy_policy_mappings", @@ -66,16 +67,21 @@ # The shared reports revision, on the withheld area codes revision, and parent # of the user invitations revision. SHARED_REPORTS_REVISION = "94dc43079186" -# The user invitations revision, on the shared reports revision. +# The user invitations revision, on the shared reports revision, and parent of +# the cliff impacts revision. USER_INVITATIONS_REVISION = "f519a77ba122" +# The cliff impacts revision, on the user invitations revision. +CLIFF_IMPACTS_REVISION = "bc334f9877b7" def test_v2_revision_chain_has_generated_policy_and_user_identity_changes() -> None: config = Config(str(REPO / "alembic-v2.ini")) script = ScriptDirectory.from_config(config) assert script.get_heads() == [V2_ALEMBIC_HEAD_REVISION] + assert V2_ALEMBIC_HEAD_REVISION == CLIFF_IMPACTS_REVISION assert [revision.revision for revision in script.walk_revisions()] == [ - V2_ALEMBIC_HEAD_REVISION, + CLIFF_IMPACTS_REVISION, + USER_INVITATIONS_REVISION, SHARED_REPORTS_REVISION, WITHHELD_AREA_CODES_REVISION, SPM_AVAILABILITY_REVISION, @@ -1132,3 +1138,55 @@ def test_user_invitations_revision_adds_its_table_and_the_dismissal_column() -> ) assert "ck_user_type_changes_source_allowed" in upgrade assert "source IN ('admin_route', 'script', 'invitation')" in upgrade + + +def test_cliff_impacts_revision_creates_only_its_table_with_one_correction() -> None: + """The revision after user invitations adds ``cliff_impacts``. + + It is generated, creates the table with its two indexes and its partial + unique index, and writes no data. Its one post-generation correction makes + ``status`` reuse the ``v2_simulation_status`` type the baseline created, + which the generated ``create_table`` would create again and fail on; so + the downgrade, which drops only the table, leaves the type to + ``simulation_runs``. + """ + (path,) = (REPO / "migrations/v2/versions").glob(f"{CLIFF_IMPACTS_REVISION}_*.py") + revision = path.read_text(encoding="utf-8") + upgrade = revision[ + revision.index("def upgrade()") : revision.index("def downgrade()") + ] + downgrade = revision[revision.index("def downgrade()") :] + + assert ( + "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" + in revision + ) + assert f'down_revision: Union[str, None] = "{USER_INVITATIONS_REVISION}"' in ( + revision + ) + assert revision.count("Dialect correction:") == 1 + assert re.search( + r"""name=["']v2_simulation_status["'],\s*create_type=False""", upgrade + ) + assert re.findall(r"""op\.create_table\(\s*["'](\w+)["']""", upgrade) == [ + "cliff_impacts" + ] + assert sorted( + re.findall(r"""op\.create_index\(\s*(?:op\.f\()?["'](\w+)["']""", upgrade) + ) == [ + "ix_cliff_impacts_simulation_run_id", + "ix_cliff_impacts_status_started_at", + "uq_cliff_impacts_unfailed", + ] + assert re.search( + r"""postgresql_where=sa\.text\(["']status <> 'failed'["']\)""", upgrade + ) + assert re.findall(r"""op\.drop_table\(\s*["'](\w+)["']""", downgrade) == [ + "cliff_impacts" + ] + for body in (upgrade, downgrade): + assert "op.execute(" not in body + assert "op.bulk_insert(" not in body + assert ".drop(op.get_bind())" not in body + assert "op.alter_column(" not in upgrade + assert "op.add_column(" not in upgrade diff --git a/tests/unit/v2/test_cliff_impact_models.py b/tests/unit/v2/test_cliff_impact_models.py new file mode 100644 index 000000000..c2009d4bd --- /dev/null +++ b/tests/unit/v2/test_cliff_impact_models.py @@ -0,0 +1,224 @@ +"""Each simulation run can have one cliff result (C1 ยท 24's answer 4b). + +A cliff calculation of a simulation run records its state and, once it +succeeds, the run's cliff gap and cliff share. A run has at most one +calculation that hasn't failed, so two requests asking for its cliffs at once +start one calculation; a failed calculation never counts, and the next request +starts another. The result inherits its run's versions, and is linked to the +run, not to any report. +""" + +from collections.abc import Iterator +from datetime import UTC, datetime +from uuid import UUID, uuid4 + +import pytest +import sqlalchemy as sa +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session + +from policyengine_api.data.v2.models import ( + V2_METADATA, + CliffImpact, + SimulationRun, + SimulationStatus, +) + +CLIFF_IMPACT_COLUMNS = frozenset( + { + "id", + "created_at", + "updated_at", + "simulation_run_id", + "status", + "modal_call_id", + "error_message", + "started_at", + "completed_at", + "cliff_gap", + "cliff_share", + } +) +UNFAILED_CALCULATIONS = "status <> 'failed'" +FINISHED_STATUSES = frozenset({SimulationStatus.SUCCEEDED, SimulationStatus.FAILED}) + + +def _cliff_impacts() -> sa.Table: + return V2_METADATA.tables["cliff_impacts"] + + +def _checks() -> dict[str, str]: + return { + str(constraint.name): str(constraint.sqltext) + for constraint in _cliff_impacts().constraints + if isinstance(constraint, sa.CheckConstraint) + } + + +def _indexes() -> dict[str, sa.Index]: + return {str(index.name): index for index in _cliff_impacts().indexes} + + +def test_cliff_impacts_hold_one_calculation_of_a_simulation_run() -> None: + """The run it calculates, the calculation's state and the two numbers.""" + cliff_impacts = _cliff_impacts() + + assert set(cliff_impacts.c.keys()) == CLIFF_IMPACT_COLUMNS + assert not cliff_impacts.c.simulation_run_id.nullable + (run_key,) = cliff_impacts.c.simulation_run_id.foreign_keys + assert run_key.target_fullname == "simulation_runs.id" + # The result is part of its run, as poverty and inequality rows are. + assert run_key.ondelete == "CASCADE" + status = cliff_impacts.c.status + assert not status.nullable + assert isinstance(status.type, sa.Enum) + # A cliff calculation has a simulation run's states, in the same type. + assert status.type.name == "v2_simulation_status" + assert status.type.enums == ["pending", "running", "succeeded", "failed"] + assert cliff_impacts.c.modal_call_id.type.length == 255 + for timestamp in ("started_at", "completed_at"): + assert cliff_impacts.c[timestamp].nullable + assert cliff_impacts.c[timestamp].type.timezone + for number in ("cliff_gap", "cliff_share"): + assert cliff_impacts.c[number].nullable + assert isinstance(cliff_impacts.c[number].type, sa.Float) + # No version of its own: it inherits its run's. + assert { + "tax_benefit_model_version_id", + "country_package_version", + "policyengine_version", + "report_run_id", + }.isdisjoint(cliff_impacts.c.keys()) + + +def test_only_a_succeeded_cliff_calculation_has_a_result() -> None: + assert _checks() == { + "ck_cliff_impacts_finished_completion": ( + "status NOT IN ('succeeded', 'failed') OR completed_at IS NOT NULL" + ), + "ck_cliff_impacts_result_when_succeeded": ( + "(cliff_gap IS NULL) = (cliff_share IS NULL) " + "AND (status = 'succeeded') = (cliff_gap IS NOT NULL)" + ), + } + + +def test_a_simulation_run_has_one_cliff_calculation_that_has_not_failed() -> None: + """The partial unique index the simulation service names on conflict.""" + indexes = _indexes() + unfailed = indexes["uq_cliff_impacts_unfailed"] + + assert unfailed.unique + assert tuple(column.name for column in unfailed.columns) == ("simulation_run_id",) + assert str(unfailed.dialect_options["postgresql"]["where"]) == ( + UNFAILED_CALCULATIONS + ) + assert str(unfailed.dialect_options["sqlite"]["where"]) == UNFAILED_CALCULATIONS + # The run's key, for its cascade and for reads that include failed rows, + # and the stuck-run sweep's search by state and age. + assert { + name: tuple(column.name for column in index.columns) + for name, index in indexes.items() + if not index.unique + } == { + "ix_cliff_impacts_simulation_run_id": ("simulation_run_id",), + "ix_cliff_impacts_status_started_at": ("status", "started_at"), + } + + +def test_a_simulation_runs_cliff_calculations_go_with_it() -> None: + relationship = sa.inspect(SimulationRun).relationships["cliff_impacts"] + + assert relationship.mapper.local_table is _cliff_impacts() + assert "delete" in relationship.cascade + assert sa.inspect(CliffImpact).relationships["simulation_run"].mapper.class_ is ( + SimulationRun + ) + + +@pytest.fixture +def sqlite_session() -> Iterator[Session]: + engine = sa.create_engine("sqlite://") + V2_METADATA.create_all(engine) + with Session(engine) as session: + yield session + engine.dispose() + + +def _cliff_calculation( + simulation_run_id: UUID, + *, + status: SimulationStatus = SimulationStatus.PENDING, + cliff_gap: float | None = None, + cliff_share: float | None = None, +) -> CliffImpact: + return CliffImpact( + simulation_run_id=simulation_run_id, + status=status, + completed_at=datetime.now(UTC) if status in FINISHED_STATUSES else None, + cliff_gap=cliff_gap, + cliff_share=cliff_share, + ) + + +def test_sqlite_refuses_a_second_unfailed_cliff_calculation_like_postgres( + sqlite_session: Session, +) -> None: + """Failed calculations stay beside the one that hasn't failed.""" + simulation_run_id = uuid4() + sqlite_session.add_all( + [ + _cliff_calculation(simulation_run_id, status=SimulationStatus.FAILED), + _cliff_calculation(simulation_run_id, status=SimulationStatus.FAILED), + _cliff_calculation( + simulation_run_id, + status=SimulationStatus.SUCCEEDED, + cliff_gap=1.5e9, + cliff_share=0.04, + ), + _cliff_calculation(uuid4(), status=SimulationStatus.RUNNING), + ] + ) + sqlite_session.commit() + + sqlite_session.add(_cliff_calculation(simulation_run_id)) + with pytest.raises(IntegrityError, match="UNIQUE constraint failed: cliff_impacts"): + sqlite_session.commit() + + +@pytest.mark.parametrize( + ("status", "cliff_gap", "cliff_share", "check_name"), + [ + (SimulationStatus.SUCCEEDED, None, None, "ck_cliff_impacts_result_when"), + (SimulationStatus.SUCCEEDED, 1.0, None, "ck_cliff_impacts_result_when"), + (SimulationStatus.PENDING, 1.0, 0.1, "ck_cliff_impacts_result_when"), + (SimulationStatus.FAILED, 1.0, 0.1, "ck_cliff_impacts_result_when"), + (SimulationStatus.RUNNING, None, 0.1, "ck_cliff_impacts_result_when"), + ], +) +def test_sqlite_refuses_numbers_on_any_row_but_a_succeeded_one( + sqlite_session: Session, + status: SimulationStatus, + cliff_gap: float | None, + cliff_share: float | None, + check_name: str, +) -> None: + sqlite_session.add( + _cliff_calculation( + uuid4(), status=status, cliff_gap=cliff_gap, cliff_share=cliff_share + ) + ) + + with pytest.raises(IntegrityError, match=check_name): + sqlite_session.commit() + + +def test_sqlite_refuses_a_finished_cliff_calculation_without_its_end( + sqlite_session: Session, +) -> None: + sqlite_session.add( + CliffImpact(simulation_run_id=uuid4(), status=SimulationStatus.FAILED) + ) + + with pytest.raises(IntegrityError, match="ck_cliff_impacts_finished_completion"): + sqlite_session.commit() diff --git a/tests/unit/v2/test_privilege_manifest.py b/tests/unit/v2/test_privilege_manifest.py index 0a20c74a3..857872b5b 100644 --- a/tests/unit/v2/test_privilege_manifest.py +++ b/tests/unit/v2/test_privilege_manifest.py @@ -45,7 +45,8 @@ PROOF_FIXTURES = PROJECT_ROOT / "rules/fixtures/api.v2.runtime-privileges-declared" STAGE12_PACKAGES = frozenset({"comparison_runs"}) TEST_ENVIRONMENT = "staging" -# The rows the simulation service creates, and the run tables among them. +# The rows the simulation service creates, and the run tables among them. A +# simulation run's cliff calculation is started and finished like a run. SIMULATION_SERVICE_CREATES = frozenset( { "simulations", @@ -54,9 +55,12 @@ "report_runs", "budget_window_runs", "household_jobs", + "cliff_impacts", } ) -RUN_STATUS_TABLES = frozenset({"simulation_runs", "report_runs", "household_jobs"}) +RUN_STATUS_TABLES = frozenset( + {"simulation_runs", "report_runs", "household_jobs", "cliff_impacts"} +) # The run columns the Modal workers update: a run's state and its results. WORKER_RUN_STATE_COLUMNS = frozenset( { @@ -81,6 +85,7 @@ } ), "household_jobs": frozenset({"result"}), + "cliff_impacts": frozenset({"cliff_gap", "cliff_share"}), } # The run columns the simulation service updates: it records a Modal call's ID # when Modal accepts the call, and fails the run when Modal refuses it. @@ -150,6 +155,8 @@ "recalculate", } ), + # A cliff calculation always calculates the run it was inserted for. + "cliff_impacts": frozenset({"simulation_run_id"}), } # The policy, dynamic, household and catalog rows a run needs. RUN_INPUT_TABLES = frozenset( @@ -480,3 +487,25 @@ def test_the_public_api_records_each_users_report_run_requests() -> None: for role, tables in declared.items(): if role != RUNTIME_ROLE: assert tables["user_report_run_associations"] == NO_ACCESS, role + + +def test_cliff_calculations_are_started_by_the_service_and_finished_by_workers() -> ( + None +): + """The simulation service inserts a run's cliff calculation and records its call. + + The Modal worker claims it, writes the cliff gap and share or the failure, + and never inserts one; the Public API reads the results. The catalog + writer and the Stage 12 role can do neither. + """ + declared = declared_privileges(TEST_ENVIRONMENT) + + assert declared[SIMULATION_SERVICE_ROLE]["cliff_impacts"] == READ_INSERT | { + update_columns(*SERVICE_RUN_STATE_COLUMNS) + } + assert declared[MODAL_WORKER_ROLE]["cliff_impacts"] == READ | { + update_columns(*WORKER_RUN_STATE_COLUMNS, "cliff_gap", "cliff_share") + } + assert declared[RUNTIME_ROLE]["cliff_impacts"] == READ + for role in (CATALOG_WRITER_ROLE, stage12_role(TEST_ENVIRONMENT)): + assert declared[role]["cliff_impacts"] == NO_ACCESS, role diff --git a/tests/unit/v2/test_simulation_run_models.py b/tests/unit/v2/test_simulation_run_models.py index 0abdaca73..d3282dcb6 100644 --- a/tests/unit/v2/test_simulation_run_models.py +++ b/tests/unit/v2/test_simulation_run_models.py @@ -46,7 +46,8 @@ "relative_income_effect", "relative_substitution_effect", ) -PER_SIMULATION_RESULT_TABLES = ("inequality", "poverty") +# A simulation run's cliff calculation is one of its results too. +PER_SIMULATION_RESULT_TABLES = ("inequality", "poverty", "cliff_impacts") COMPARISON_RESULT_TABLES = ( "budget_summary", "congressional_district_impacts", From a2b29deac551c48fde6d77b445b2323e8548a338 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:42:11 +0400 Subject: [PATCH 02/10] Record each simulation run's cliff calculations, with grants Add cliff_impacts (revision bc334f9877b7, autogenerated on f519a77ba122): one row per cliff calculation of a simulation run, with its state and, once it succeeds, the run's cliff gap and cliff share. A partial unique index allows one calculation per run that hasn't failed, so identical requests join it and a failed one never counts. The row is deleted with its run, and inherits the run's versions. Its status reuses v2_simulation_status, the revision's one post-generation correction. The simulation service may insert calculations and record or fail their Modal calls, the Modal workers may write their state and results, the Public API may read them, and no other role may touch them. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/test_alembic_v2_lifecycle.sh | 3 +- .../simulation-run-cliff-impacts.added.md | 1 + docs/engineering/skills/alembic-migrations.md | 75 +++++++++++++++- ...7b7_record_each_simulation_run_s_cliff_.py | 86 ++++++++++++++++++ policyengine_api/data/v2/migration_target.py | 2 +- policyengine_api/data/v2/models/__init__.py | 2 + .../data/v2/models/run_results.py | 88 ++++++++++++++++++- .../data/v2/models/simulations.py | 14 ++- policyengine_api/data/v2/privileges.py | 23 +++-- tests/unit/v2/test_alembic_v2_revisions.py | 5 +- 10 files changed, 279 insertions(+), 20 deletions(-) create mode 100644 changelog.d/simulation-run-cliff-impacts.added.md create mode 100644 migrations/v2/versions/bc334f9877b7_record_each_simulation_run_s_cliff_.py diff --git a/.github/scripts/test_alembic_v2_lifecycle.sh b/.github/scripts/test_alembic_v2_lifecycle.sh index b122419df..7b6293df5 100755 --- a/.github/scripts/test_alembic_v2_lifecycle.sh +++ b/.github/scripts/test_alembic_v2_lifecycle.sh @@ -11,4 +11,5 @@ uv run --frozen --extra dev pytest -q \ tests/integration/test_alembic_v2_saved_item_keys.py \ tests/integration/test_alembic_v2_user_types.py \ tests/integration/test_alembic_v2_withheld_area_codes.py \ - tests/integration/test_alembic_v2_shared_report_keys.py + tests/integration/test_alembic_v2_shared_report_keys.py \ + tests/integration/test_alembic_v2_cliff_impacts.py diff --git a/changelog.d/simulation-run-cliff-impacts.added.md b/changelog.d/simulation-run-cliff-impacts.added.md new file mode 100644 index 000000000..3fa2c8e95 --- /dev/null +++ b/changelog.d/simulation-run-cliff-impacts.added.md @@ -0,0 +1 @@ +The v2 database can store one cliff result per simulation run, its cliff gap and cliff share, with the state of its calculation. The simulation service starts each calculation, its Modal workers write the result, and the Public API may read it. diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index 157f8aecf..46577ed82 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -255,8 +255,8 @@ and a revision runs once per database, so it can't repair drift. | `policyengine_v2_runtime` | The Public API on Cloud Run | | `policyengine_v2_catalog_writer` | `scripts/publish_v2_metadata_catalog.py`, which also runs `ANALYZE` on seven catalog tables after every publication, with `MAINTAIN` (PostgreSQL 17) | | `policyengine_stage12_` | The simulation API and its Modal worker, on the temporary comparison-run tables | -| `policyengine_v2_simulation_service` | The simulation API on Cloud Run: creates simulations, reports, their runs, budget-window runs and household jobs, and reads the policy, dynamic, household and catalog rows a run needs. It updates only the status, Modal call ID, error and completion columns of the runs and household jobs whose Modal calls it starts, and the status, error and completion columns of a budget-window run, which it fails at once when Modal refuses the calls of the window's first years | -| `policyengine_v2_modal_worker` | The simulation API's Modal workers: write their runs' and household jobs' status and results, and budget-window runs' status, and read the rows a run needs. It updates only a run's status, Modal call ID, error, time and result columns, and a budget-window run's status, error and time columns: each year's coordinator starts the window's next unstarted year when its own year finishes, the last year to finish marks the window succeeded, and a failed year fails it. The workers' code, not the database, keeps a worker to its own run | +| `policyengine_v2_simulation_service` | The simulation API on Cloud Run: creates simulations, reports, their runs, budget-window runs, household jobs and simulation runs' cliff calculations, and reads the policy, dynamic, household and catalog rows a run needs. It updates only the status, Modal call ID, error and completion columns of the runs, household jobs and cliff calculations whose Modal calls it starts, and the status, error and completion columns of a budget-window run, which it fails at once when Modal refuses the calls of the window's first years | +| `policyengine_v2_modal_worker` | The simulation API's Modal workers: write their runs', household jobs' and cliff calculations' status and results, and budget-window runs' status, and read the rows a run needs. It updates only a run's or cliff calculation's status, Modal call ID, error, time and result columns, and a budget-window run's status, error and time columns: each year's coordinator starts the window's next unstarted year when its own year finishes, the last year to finish marks the window succeeded, and a failed year fails it. The workers' code, not the database, keeps a worker to its own run | Every v2 table has an entry for every role, so "no access" is a decision, not an omission. A privilege is declared on the whole table, or with @@ -456,6 +456,8 @@ it publishes that version, with nothing to restore it. Downgrading past the revision is therefore safe only while no version is pruned (no `tax_benefit_model_versions` row has `catalog_values_pruned_at` set). From the head, such a downgrade first runs the later revisions' downgrades: +`bc334f9877b7`'s drops `cliff_impacts` with every simulation run's cliff +results (see "API v2 simulation-run cliff impacts"), `f519a77ba122`'s drops `user_invitations` and needs no recorded change with source `invitation` (see "API v2 user invitations"), `94dc43079186`'s drops every user's report-run record, the simulation and @@ -1012,3 +1014,72 @@ change's source is `invitation`. `test_user_invitations_revision_adds_invitations_and_dismissals_only` and `test_user_invitations_revision_records_invitation_changes_until_downgraded` in `tests/integration/test_alembic_v2_user_types.py` check both. + +### API v2 simulation-run cliff impacts + +Revision `bc334f9877b7` (parent `f519a77ba122`) was autogenerated, with its +ID given by `--rev-id`, after the `CliffImpact` model was added. Each +simulation run can have one cliff result, linked to the run and not to any +report, which every report naming the run shares: cliffs need marginal tax +rates, which recalculate each household with extra earnings, so a shared run +never stores the cliff columns (US national runs keep their region-group +split), and a request that asks for cliffs starts a separate calculation of +the run on its dataset, policy and dynamic. + +It creates `cliff_impacts`, one row per cliff calculation of a simulation run: + +- `simulation_run_id`, a foreign key to `simulation_runs` with `CASCADE`, as + `poverty` and `inequality` have: the result is part of its run, and nothing + references it. A run can be deleted only once no report run names it. +- The calculation's state: `status` (`pending`, `running`, `succeeded` or + `failed`, in the `v2_simulation_status` type simulation runs use), + `modal_call_id`, `error_message` (`code: message`, as runs record theirs), + `started_at` and `completed_at`, with the run tables' check that a + succeeded or failed row has `completed_at` + (`ck_cliff_impacts_finished_completion`). +- The result: `cliff_gap`, the weighted total of the net income people would + lose from a small rise in their earnings, and `cliff_share`, the weighted + number of people on a cliff divided by the weighted number of adults, both + double precision. A succeeded row has both, and no other row has either + (`ck_cliff_impacts_result_when_succeeded`). Neither is range-checked: .py + doesn't keep the share to at most 1, and a filtered simulation with no + adults gives NaN. +- `uq_cliff_impacts_unfailed`, a partial unique index on `simulation_run_id` + where `status <> 'failed'` (also declared for SQLite): a run has at most one + calculation that hasn't failed, and so at most one result. The simulation + service starts one with `INSERT ... ON CONFLICT (simulation_run_id) WHERE + status <> 'failed' DO NOTHING`, so two requests at once start one + calculation and the second joins it. A failed calculation never counts: it + stays as a record, and the next request inserts another. +- An index on `(status, started_at)`, as `simulation_runs` has, for finding + unfinished calculations by age, as the simulation API's stuck-run sweep + does for runs: a calculation whose worker ended without recording its + outcome stays unfinished, and keeps later requests from starting another, + until something fails it. + +A result has no versions of its own: it inherits its run's catalog and +package versions, so it is calculated only for a run on the versions the +simulation API serves. The simulation service may read and insert rows and update their +status, Modal call ID, error and completion columns; the Modal worker may read +them and update their state and result columns, never insert them; the +runtime role may read them; neither simulation role may change the run a row +names or delete a row. The reconcile step after each schema upgrade grants +them. `report_runs`' four cliff columns stay until a later revision drops +them, once the Public API reads cliffs from here. + +It has one post-generation correction, commented in the revision: `status` +reuses the baseline's `v2_simulation_status` type +(`postgresql.ENUM(..., create_type=False)`), as `1356f658347d` does for +`simulation_runs.status`; the generated `create_table` would create the type +again and fail. So the downgrade, which drops only the table, leaves the type +to `simulation_runs`. + +The revision writes no data and changes no other table, and the previous +Public API revision, which doesn't map the table, keeps working at it. The +downgrade drops the table with every calculation and result; a later +calculation of the same runs on the same versions gives the same numbers. +`tests/integration/test_alembic_v2_cliff_impacts.py` checks that nothing +else changes, that the downgrade leaves the type and an upgrade works again, +and the index, checks and delete rule; +`tests/integration/test_v2_simulation_role_privileges.py` runs the writes +and reads as the roles that do them. diff --git a/migrations/v2/versions/bc334f9877b7_record_each_simulation_run_s_cliff_.py b/migrations/v2/versions/bc334f9877b7_record_each_simulation_run_s_cliff_.py new file mode 100644 index 000000000..b9bd3badd --- /dev/null +++ b/migrations/v2/versions/bc334f9877b7_record_each_simulation_run_s_cliff_.py @@ -0,0 +1,86 @@ +"""record each simulation run's cliff calculations + +Revision ID: bc334f9877b7 +Revises: f519a77ba122 +Create Date: 2026-11-17 17:38:09.806260 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate --rev-id bc334f9877b7 + +One post-generation correction, commented where it is made: +cliff_impacts.status reuses the existing v2_simulation_status type instead of +creating it again. + +Each simulation run can have one cliff result: its cliff gap and cliff share, +linked to the run and not to any report. It adds cliff_impacts, one row per +cliff calculation of a simulation run, holding the calculation's state +(status, Modal call, error and times) and, once it succeeds, the two numbers. +A partial unique index allows one calculation per run that hasn't failed, so +identical requests join it; a failed calculation never counts. Deleting a run +deletes its calculations. A result has no versions of its own: it inherits its +run's. + +It writes no data and changes no other table, and the previous Public API +revision, which doesn't map the table, keeps working at it. The downgrade +drops the table with every calculation and result, and leaves the type, +which simulation_runs still uses. +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel +from sqlalchemy.dialects import postgresql + + +revision: str = 'bc334f9877b7' +down_revision: Union[str, None] = 'f519a77ba122' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('cliff_impacts', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('simulation_run_id', sa.Uuid(), nullable=False), + sa.Column( + 'status', + # Dialect correction: a cliff calculation has a simulation run's + # states, so the column reuses the baseline's v2_simulation_status + # type instead of creating it again, which would fail. + postgresql.ENUM( + 'pending', + 'running', + 'succeeded', + 'failed', + name='v2_simulation_status', + create_type=False, + ), + nullable=False, + ), + sa.Column('modal_call_id', sqlmodel.sql.sqltypes.AutoString(length=255), nullable=True), + sa.Column('error_message', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('started_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('completed_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('cliff_gap', sa.Float(), nullable=True), + sa.Column('cliff_share', sa.Float(), nullable=True), + sa.CheckConstraint("(cliff_gap IS NULL) = (cliff_share IS NULL) AND (status = 'succeeded') = (cliff_gap IS NOT NULL)", name=op.f('ck_cliff_impacts_result_when_succeeded')), + sa.CheckConstraint("status NOT IN ('succeeded', 'failed') OR completed_at IS NOT NULL", name=op.f('ck_cliff_impacts_finished_completion')), + sa.ForeignKeyConstraint(['simulation_run_id'], ['simulation_runs.id'], name=op.f('fk_cliff_impacts_simulation_run_id_simulation_runs'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_cliff_impacts')) + ) + op.create_index(op.f('ix_cliff_impacts_simulation_run_id'), 'cliff_impacts', ['simulation_run_id'], unique=False) + op.create_index('ix_cliff_impacts_status_started_at', 'cliff_impacts', ['status', 'started_at'], unique=False) + op.create_index('uq_cliff_impacts_unfailed', 'cliff_impacts', ['simulation_run_id'], unique=True, postgresql_where=sa.text("status <> 'failed'"), sqlite_where=sa.text("status <> 'failed'")) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index('uq_cliff_impacts_unfailed', table_name='cliff_impacts', postgresql_where=sa.text("status <> 'failed'"), sqlite_where=sa.text("status <> 'failed'")) + op.drop_index('ix_cliff_impacts_status_started_at', table_name='cliff_impacts') + op.drop_index(op.f('ix_cliff_impacts_simulation_run_id'), table_name='cliff_impacts') + op.drop_table('cliff_impacts') + # ### end Alembic commands ### diff --git a/policyengine_api/data/v2/migration_target.py b/policyengine_api/data/v2/migration_target.py index b51fd461c..21c2a2a5c 100644 --- a/policyengine_api/data/v2/migration_target.py +++ b/policyengine_api/data/v2/migration_target.py @@ -24,7 +24,7 @@ from collections.abc import Mapping V2_ALEMBIC_DISPOSABLE_TEST = "V2_ALEMBIC_DISPOSABLE_TEST" -V2_ALEMBIC_HEAD_REVISION = "f519a77ba122" +V2_ALEMBIC_HEAD_REVISION = "bc334f9877b7" DISPOSABLE_DATABASE_NAME = "policyengine_v2_alembic_test" DISPOSABLE_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "postgres"}) MIGRATION_ROLE = "policyengine_v2_migrator" diff --git a/policyengine_api/data/v2/models/__init__.py b/policyengine_api/data/v2/models/__init__.py index 973eb9c11..dfa9c0bdf 100644 --- a/policyengine_api/data/v2/models/__init__.py +++ b/policyengine_api/data/v2/models/__init__.py @@ -78,6 +78,7 @@ AggregateType, BudgetSummary, ChangeAggregate, + CliffImpact, CongressionalDistrictImpact, ConstituencyImpact, DecileImpact, @@ -124,6 +125,7 @@ "CatalogEntity", "CatalogEnumValue", "ChangeAggregate", + "CliffImpact", "CongressionalDistrictImpact", "ConstituencyImpact", "Country", diff --git a/policyengine_api/data/v2/models/run_results.py b/policyengine_api/data/v2/models/run_results.py index 2e691bd2f..3451515bf 100644 --- a/policyengine_api/data/v2/models/run_results.py +++ b/policyengine_api/data/v2/models/run_results.py @@ -2,10 +2,11 @@ Results that compare a report run's baseline and reform belong to the report run and name its two simulation runs; each such row is unique within its report -run. Poverty and inequality, which describe one simulation, belong to a -simulation run. +run. Poverty, inequality and cliff impacts, which describe one simulation, +belong to a simulation run. """ +from datetime import datetime from enum import StrEnum from typing import Any from uuid import UUID @@ -13,9 +14,16 @@ import sqlalchemy as sa from sqlmodel import Field, Relationship -from policyengine_api.data.v2.models.base import IdentifiedModel, enum_type +from policyengine_api.data.v2.models.base import ( + IdentifiedModel, + TimestampedModel, + enum_type, +) from policyengine_api.data.v2.models.reports import ReportRun -from policyengine_api.data.v2.models.simulations import SimulationRun +from policyengine_api.data.v2.models.simulations import ( + SimulationRun, + SimulationStatus, +) class OutputStatus(StrEnum): @@ -401,6 +409,78 @@ class Poverty(IdentifiedModel, table=True): simulation_run: SimulationRun = Relationship(back_populates="poverty_results") +# The cliff calculations a new request may join, of which a simulation run has +# at most one: a failed calculation never counts. +UNFAILED_CLIFF_CALCULATIONS = "status <> 'failed'" + + +class CliffImpact(TimestampedModel, table=True): + """One cliff calculation of a simulation run, and its result once it succeeds. + + ``cliff_gap`` is the weighted total of the net income people would lose + from a small rise in their earnings, and ``cliff_share`` the weighted + number of people on a cliff divided by the weighted number of adults. + They need marginal tax rates, which recalculate each household with extra + earnings, so a run's cliffs are calculated apart from the run, on its + dataset, policy and dynamic, only when a request asks for them, and only + for a run on the current versions. The result has no versions of its own: + it inherits its run's, and every report naming the run shares it. + + The simulation service inserts a pending calculation and records its + Modal call, or fails it when Modal refuses the call; the Modal worker + claims it under its call, then writes the two numbers with ``succeeded``, + or ``failed`` with its ``code: message`` error. A run has at most one + calculation that hasn't failed, which identical requests join; a failed + calculation never counts, and the next request starts another. Deleting + the run deletes its calculations. + """ + + __tablename__ = "cliff_impacts" + __table_args__ = ( + sa.CheckConstraint( + "status NOT IN ('succeeded', 'failed') OR completed_at IS NOT NULL", + name="ck_cliff_impacts_finished_completion", + ), + sa.CheckConstraint( + "(cliff_gap IS NULL) = (cliff_share IS NULL) " + "AND (status = 'succeeded') = (cliff_gap IS NOT NULL)", + name="ck_cliff_impacts_result_when_succeeded", + ), + sa.Index("ix_cliff_impacts_status_started_at", "status", "started_at"), + sa.Index( + "uq_cliff_impacts_unfailed", + "simulation_run_id", + unique=True, + postgresql_where=sa.text(UNFAILED_CLIFF_CALCULATIONS), + sqlite_where=sa.text(UNFAILED_CLIFF_CALCULATIONS), + ), + ) + + simulation_run_id: UUID = Field( + foreign_key="simulation_runs.id", + ondelete="CASCADE", + index=True, + ) + status: SimulationStatus = Field( + default=SimulationStatus.PENDING, + sa_type=enum_type(SimulationStatus, "v2_simulation_status"), + ) + modal_call_id: str | None = Field(default=None, max_length=255) + error_message: str | None = None + started_at: datetime | None = Field( + default=None, + sa_type=sa.DateTime(timezone=True), + ) + completed_at: datetime | None = Field( + default=None, + sa_type=sa.DateTime(timezone=True), + ) + cliff_gap: float | None = None + cliff_share: float | None = None + + simulation_run: SimulationRun = Relationship(back_populates="cliff_impacts") + + class ProgramStatistics(IdentifiedModel, table=True): """Baseline and reform totals, counts, winners and losers for one program. diff --git a/policyengine_api/data/v2/models/simulations.py b/policyengine_api/data/v2/models/simulations.py index ca213d3fb..28ba6b052 100644 --- a/policyengine_api/data/v2/models/simulations.py +++ b/policyengine_api/data/v2/models/simulations.py @@ -24,7 +24,11 @@ if TYPE_CHECKING: from policyengine_api.data.v2.models.reports import Report - from policyengine_api.data.v2.models.run_results import Inequality, Poverty + from policyengine_api.data.v2.models.run_results import ( + CliffImpact, + Inequality, + Poverty, + ) class SimulationStatus(StrEnum): @@ -168,7 +172,9 @@ class SimulationRun(TimestampedModel, table=True): dataset can't be deleted while a run names them, and deleting the region clears the reference. A household simulation's run stores its household in ``household_result``; an economy run's output file is ``output_uri``, with - its SHA-256. Each poverty and inequality result belongs to one run. + its SHA-256. Each poverty and inequality result belongs to one run, and so + does each calculation of its cliff impact (``CliffImpact``), which a + request asking for cliffs starts apart from the run. A simulation has at most one run that hasn't failed for each year, catalog version and pair of package versions, which identical requests join; a @@ -268,3 +274,7 @@ class SimulationRun(TimestampedModel, table=True): back_populates="simulation_run", cascade_delete=True, ) + cliff_impacts: list["CliffImpact"] = Relationship( + back_populates="simulation_run", + cascade_delete=True, + ) diff --git a/policyengine_api/data/v2/privileges.py b/policyengine_api/data/v2/privileges.py index ba116de22..5a27d1d6f 100644 --- a/policyengine_api/data/v2/privileges.py +++ b/policyengine_api/data/v2/privileges.py @@ -29,20 +29,22 @@ analyzes afterwards; its code isn't organized as connector modules, so the deploy check, not the connector analysis, covers it. - ``policyengine_v2_simulation_service``: the simulation API on Cloud Run. It - creates simulations, reports, their runs, budget-window runs and household - jobs, and reads the policy, dynamic, household and catalog rows a run needs. - It starts the Modal calls of runs and household jobs, so it may update their + creates simulations, reports, their runs, budget-window runs, household + jobs and simulation runs' cliff calculations, and reads the policy, + dynamic, household and catalog rows a run needs. It starts the Modal calls + of runs, household jobs and cliff calculations, so it may update their status, Modal call ID, error and completion columns: it records a call's ID once Modal accepts the call, and fails the run when Modal refuses it. It starts a budget-window run's first years at submit, so it may update the window run's status, error and completion columns, to fail it at once when Modal refuses those years' calls. - ``policyengine_v2_modal_worker``: the simulation API's Modal workers. They - write the status of the runs and household jobs they were given, and their - results, and read the rows a run needs. They also write a budget-window - run's status, error and times: each year's coordinator starts the window's - next unstarted year when its own year finishes, the last year to finish - marks the window succeeded, and a failed year fails it. + write the status of the runs, household jobs and cliff calculations they + were given, and their results, and read the rows a run needs. They also + write a budget-window run's status, error and times: each year's + coordinator starts the window's next unstarted year when its own year + finishes, the last year to finish marks the window succeeded, and a failed + year fails it. Both simulation API roles update only the columns ``update_columns`` names, so the database refuses any change to what a run is (its simulation or report, @@ -267,6 +269,11 @@ def _run_table( "reports": replace(_SIMULATION_DEFINITION, catalog_writer=READ), "simulation_runs": _run_table("output_uri", "output_sha256", "household_result"), "simulations": replace(_SIMULATION_DEFINITION, catalog_writer=READ), + # A simulation run's cliff calculations, started and finished as a run is: + # the simulation service inserts one and records or fails its Modal call, + # and the Modal worker claims it and writes its cliff gap and share. Neither + # may change the run a calculation names. + "cliff_impacts": _run_table("cliff_gap", "cliff_share"), # Results of simulation runs and report runs. "labor_supply_decile_responses": _RESULT, "labor_supply_responses": _RESULT, diff --git a/tests/unit/v2/test_alembic_v2_revisions.py b/tests/unit/v2/test_alembic_v2_revisions.py index 108846b6c..326a811f7 100644 --- a/tests/unit/v2/test_alembic_v2_revisions.py +++ b/tests/unit/v2/test_alembic_v2_revisions.py @@ -1161,8 +1161,9 @@ def test_cliff_impacts_revision_creates_only_its_table_with_one_correction() -> "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" in revision ) - assert f'down_revision: Union[str, None] = "{USER_INVITATIONS_REVISION}"' in ( - revision + assert re.search( + rf"""down_revision: Union\[str, None\] = ["']{USER_INVITATIONS_REVISION}["']""", + revision, ) assert revision.count("Dialect correction:") == 1 assert re.search( From becca3b366955769bde10cb5d71782de4bffe52c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:59:32 +0400 Subject: [PATCH 03/10] Name the cliff impacts revision in the downgrade notes and table lists The Stage 9 and Stage 12A runbooks' downgrade chains now start with bc334f9877b7, and the head foreign-key test counts cliff_impacts among the simulation side's tables. The changelog fragment and the skill section say what each role may do, since the simulation API writes the table only once its own change lands. Co-Authored-By: Claude Opus 5.5 --- changelog.d/simulation-run-cliff-impacts.added.md | 2 +- docs/engineering/skills/alembic-migrations.md | 14 ++++++++------ docs/migration/stage-12a-user-login.md | 4 +++- docs/migration/stage-9-v2-metadata.md | 6 ++++-- .../integration/test_alembic_v2_simulation_runs.py | 5 +++-- 5 files changed, 19 insertions(+), 12 deletions(-) diff --git a/changelog.d/simulation-run-cliff-impacts.added.md b/changelog.d/simulation-run-cliff-impacts.added.md index 3fa2c8e95..f858abb2b 100644 --- a/changelog.d/simulation-run-cliff-impacts.added.md +++ b/changelog.d/simulation-run-cliff-impacts.added.md @@ -1 +1 @@ -The v2 database can store one cliff result per simulation run, its cliff gap and cliff share, with the state of its calculation. The simulation service starts each calculation, its Modal workers write the result, and the Public API may read it. +The v2 database can store one cliff result per simulation run, its cliff gap and cliff share, with the state of its calculation. The simulation service may start each calculation, its Modal workers may write the result, and the Public API may read it. diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index 46577ed82..9e0f86351 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -1047,10 +1047,10 @@ It creates `cliff_impacts`, one row per cliff calculation of a simulation run: - `uq_cliff_impacts_unfailed`, a partial unique index on `simulation_run_id` where `status <> 'failed'` (also declared for SQLite): a run has at most one calculation that hasn't failed, and so at most one result. The simulation - service starts one with `INSERT ... ON CONFLICT (simulation_run_id) WHERE - status <> 'failed' DO NOTHING`, so two requests at once start one - calculation and the second joins it. A failed calculation never counts: it - stays as a record, and the next request inserts another. + service is to start one with `INSERT ... ON CONFLICT (simulation_run_id) + WHERE status <> 'failed' DO NOTHING`, so that two requests at once start + one calculation and the second joins it. A failed calculation never counts: + it stays as a record, and the next request inserts another. - An index on `(status, started_at)`, as `simulation_runs` has, for finding unfinished calculations by age, as the simulation API's stuck-run sweep does for runs: a calculation whose worker ended without recording its @@ -1064,8 +1064,10 @@ status, Modal call ID, error and completion columns; the Modal worker may read them and update their state and result columns, never insert them; the runtime role may read them; neither simulation role may change the run a row names or delete a row. The reconcile step after each schema upgrade grants -them. `report_runs`' four cliff columns stay until a later revision drops -them, once the Public API reads cliffs from here. +them. Until the simulation API writes the table, no run has a cliff +calculation, and its cliffs stay refused. `report_runs`' four cliff columns +stay until a later revision drops them, once the Public API reads cliffs from +here. It has one post-generation correction, commented in the revision: `status` reuses the baseline's `v2_simulation_status` type diff --git a/docs/migration/stage-12a-user-login.md b/docs/migration/stage-12a-user-login.md index d93eec83a..13f8475d8 100644 --- a/docs/migration/stage-12a-user-login.md +++ b/docs/migration/stage-12a-user-login.md @@ -770,7 +770,9 @@ admin; `scripts/set_v2_user_type.py` makes another admin afterwards. `users.primary_country`, so it fails while legacy-only rows, any simulation or report associations, or any unauthenticated user exist. Removing them first deletes users' saved work. From the head, the later revisions' - downgrades run first: the user invitations revision (`f519a77ba122`) drops + downgrades run first: the cliff impacts revision (`bc334f9877b7`) drops + every simulation run's cliff results, the user invitations revision + (`f519a77ba122`) drops every invitation and dismissal and needs no change with source `invitation`, the shared reports revision (`94dc43079186`) drops every user's report-run record, every simulation and report key, the runs' diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 785d4a49a..431a5a148 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -835,8 +835,10 @@ pruning"). A schema downgrade past revision `334859ef4fec` drops `catalog_last_published_at` and `catalog_values_pruned_at`. From the head, -it first runs the later revisions' downgrades: `f519a77ba122`'s drops -`user_invitations` and needs no recorded change with source `invitation`, +it first runs the later revisions' downgrades: `bc334f9877b7`'s drops +`cliff_impacts` with every simulation run's cliff results, `f519a77ba122`'s +drops `user_invitations` and needs no recorded change with source +`invitation`, `94dc43079186`'s drops `user_report_run_associations` with every user's report-run record, the simulation and report keys, the runs' `forced` markers, the window runs' .py diff --git a/tests/integration/test_alembic_v2_simulation_runs.py b/tests/integration/test_alembic_v2_simulation_runs.py index 7d2ddf406..f70d2fe22 100644 --- a/tests/integration/test_alembic_v2_simulation_runs.py +++ b/tests/integration/test_alembic_v2_simulation_runs.py @@ -72,11 +72,12 @@ } ) # The tables the simulation service and its Modal workers write; the Public API -# writes every other table. withheld_area_codes came in a later revision. +# writes every other table. withheld_area_codes and cliff_impacts came in later +# revisions. SIMULATION_SIDE_TABLES = ( CHANGED_TABLES | NEW_TABLES - | {"aggregates", "change_aggregates", "withheld_area_codes"} + | {"aggregates", "change_aggregates", "withheld_area_codes", "cliff_impacts"} ) # The foreign keys from the simulation service's tables to the Public API's # rows, with their delete rules. From a4be631104c435d6788395ea2a95dc2485d5c4ae Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:56:08 +0400 Subject: [PATCH 04/10] Count cliff_impacts in the import test's v2 table total The v2 metadata now holds 51 tables. Co-Authored-By: Claude Opus 5.5 --- tests/unit/v2/test_import_side_effects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/v2/test_import_side_effects.py b/tests/unit/v2/test_import_side_effects.py index 9bd2d7389..637e6d406 100644 --- a/tests/unit/v2/test_import_side_effects.py +++ b/tests/unit/v2/test_import_side_effects.py @@ -69,7 +69,7 @@ def reject_ddl(*args, **kwargs): import sys after = set(pathlib.Path.cwd().iterdir()) assert before == after -assert len(V2_METADATA.tables) == 50 +assert len(V2_METADATA.tables) == 51 assert "policyengine_api.data.v2.catalog.initialization" not in sys.modules assert "policyengine_api.data.v2.policy_migration_qualification" not in sys.modules assert "policyengine_api.data.v2.household_migration_qualification" not in sys.modules From ddc422cf6a9be9bb17a0f9a33e7d57f0bdebcdea Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 19 Nov 2026 20:41:10 +0400 Subject: [PATCH 05/10] Call the service API v1 in the cliff impacts text The changelog fragment, the migrations skill, the revision's docstring and two test docstrings now name the service API v1, and the test of its reads is renamed to match. The revision's schema and grants are unchanged. Co-Authored-By: Claude Opus 5.5 --- changelog.d/simulation-run-cliff-impacts.added.md | 2 +- docs/engineering/skills/alembic-migrations.md | 6 +++--- .../bc334f9877b7_record_each_simulation_run_s_cliff_.py | 4 ++-- tests/integration/test_v2_simulation_role_privileges.py | 6 +++--- tests/unit/v2/test_privilege_manifest.py | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/changelog.d/simulation-run-cliff-impacts.added.md b/changelog.d/simulation-run-cliff-impacts.added.md index f858abb2b..eed61136b 100644 --- a/changelog.d/simulation-run-cliff-impacts.added.md +++ b/changelog.d/simulation-run-cliff-impacts.added.md @@ -1 +1 @@ -The v2 database can store one cliff result per simulation run, its cliff gap and cliff share, with the state of its calculation. The simulation service may start each calculation, its Modal workers may write the result, and the Public API may read it. +The v2 database can store one cliff result per simulation run, its cliff gap and cliff share, with the state of its calculation. The simulation service may start each calculation, its Modal workers may write the result, and API v1 may read it. diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index 9e0f86351..5a10cf95d 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -1066,7 +1066,7 @@ runtime role may read them; neither simulation role may change the run a row names or delete a row. The reconcile step after each schema upgrade grants them. Until the simulation API writes the table, no run has a cliff calculation, and its cliffs stay refused. `report_runs`' four cliff columns -stay until a later revision drops them, once the Public API reads cliffs from +stay until a later revision drops them, once API v1 reads cliffs from here. It has one post-generation correction, commented in the revision: `status` @@ -1076,8 +1076,8 @@ reuses the baseline's `v2_simulation_status` type again and fail. So the downgrade, which drops only the table, leaves the type to `simulation_runs`. -The revision writes no data and changes no other table, and the previous -Public API revision, which doesn't map the table, keeps working at it. The +The revision writes no data and changes no other table, and API v1's +previous release, which doesn't map the table, keeps working at it. The downgrade drops the table with every calculation and result; a later calculation of the same runs on the same versions gives the same numbers. `tests/integration/test_alembic_v2_cliff_impacts.py` checks that nothing diff --git a/migrations/v2/versions/bc334f9877b7_record_each_simulation_run_s_cliff_.py b/migrations/v2/versions/bc334f9877b7_record_each_simulation_run_s_cliff_.py index b9bd3badd..ad8b683c8 100644 --- a/migrations/v2/versions/bc334f9877b7_record_each_simulation_run_s_cliff_.py +++ b/migrations/v2/versions/bc334f9877b7_record_each_simulation_run_s_cliff_.py @@ -18,8 +18,8 @@ deletes its calculations. A result has no versions of its own: it inherits its run's. -It writes no data and changes no other table, and the previous Public API -revision, which doesn't map the table, keeps working at it. The downgrade +It writes no data and changes no other table, and API v1's previous +release, which doesn't map the table, keeps working at it. The downgrade drops the table with every calculation and result, and leaves the type, which simulation_runs still uses. """ diff --git a/tests/integration/test_v2_simulation_role_privileges.py b/tests/integration/test_v2_simulation_role_privileges.py index 944aeeb9e..e64be932a 100644 --- a/tests/integration/test_v2_simulation_role_privileges.py +++ b/tests/integration/test_v2_simulation_role_privileges.py @@ -3,7 +3,7 @@ The simulation service creates simulations, reports, their runs, budget-window runs, household jobs and simulation runs' cliff calculations; its Modal workers write their runs' and cliff calculations' status and results and -their budget-window runs' status; the Public API reads them. Each +their budget-window runs' status; API v1 reads them. Each role holds only its declared privileges (``policyengine_api/data/v2/privileges.py``), so a statement outside them fails here as it would in staging and production. @@ -685,10 +685,10 @@ def test_modal_workers_claim_cliff_calculations_and_write_their_results( _refused(connection, statement, id=calculation_id) -def test_the_public_api_reads_cliff_results_and_other_roles_cannot( +def test_api_v1_reads_cliff_results_and_other_roles_cannot( connection: Connection, managed_v2_roles, public_api_rows: PublicApiRows ) -> None: - """The Public API reads a report run's two cliff results and writes none. + """API v1 reads a report run's two cliff results and writes none. The catalog writer and the Stage 12 role can't read them. """ diff --git a/tests/unit/v2/test_privilege_manifest.py b/tests/unit/v2/test_privilege_manifest.py index 857872b5b..7eca8c53d 100644 --- a/tests/unit/v2/test_privilege_manifest.py +++ b/tests/unit/v2/test_privilege_manifest.py @@ -495,7 +495,7 @@ def test_cliff_calculations_are_started_by_the_service_and_finished_by_workers() """The simulation service inserts a run's cliff calculation and records its call. The Modal worker claims it, writes the cliff gap and share or the failure, - and never inserts one; the Public API reads the results. The catalog + and never inserts one; API v1 reads the results. The catalog writer and the Stage 12 role can do neither. """ declared = declared_privileges(TEST_ENVIRONMENT) From c0b879a139e90344e4eb90ce2db26e6bdae079f9 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 20 Nov 2026 01:13:03 +0400 Subject: [PATCH 06/10] Reach the dynamics revision by a downgrade in its downgrade test The cliff impacts revision now follows the dynamics revision, so the test's upgrade to the dynamics revision from the head did nothing, and its failing downgrade then rolled back the cliff impacts step too, leaving the database at bc334f9877b7. A downgrade to the dynamics revision restores what the test checks: the dynamics downgrade fails and changes nothing while a dynamic is stored. Co-Authored-By: Claude Opus 5.5 --- tests/integration/test_alembic_v2_dynamics.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_alembic_v2_dynamics.py b/tests/integration/test_alembic_v2_dynamics.py index 3da55d7cf..79932106a 100644 --- a/tests/integration/test_alembic_v2_dynamics.py +++ b/tests/integration/test_alembic_v2_dynamics.py @@ -323,7 +323,10 @@ def test_the_downgrade_fails_changing_nothing_while_a_dynamic_is_stored( engine: Engine, ) -> None: config = alembic_v2_config() - command.upgrade(config, DYNAMICS_REVISION) + # The fixture leaves the database at the head, which a later revision may + # be, so a downgrade reaches the dynamics revision; an upgrade to an older + # revision does nothing. + command.downgrade(config, DYNAMICS_REVISION) with engine.begin() as connection: rows = insert_public_api_rows(connection) dynamic_id = _insert_dynamic(connection, rows) From ac9f05ca69d0c1538c80426fdf4360c5654d3e80 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 20 Nov 2026 01:16:18 +0400 Subject: [PATCH 07/10] Check the dynamics downgrade's rollback directly, and reword its comment The failed downgrade leaves legacy_policy_mappings.dynamic_id in place, which the dynamics downgrade drops before the step that fails. Co-Authored-By: Claude Opus 5.5 --- tests/integration/test_alembic_v2_dynamics.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_alembic_v2_dynamics.py b/tests/integration/test_alembic_v2_dynamics.py index 79932106a..658e244ca 100644 --- a/tests/integration/test_alembic_v2_dynamics.py +++ b/tests/integration/test_alembic_v2_dynamics.py @@ -323,9 +323,9 @@ def test_the_downgrade_fails_changing_nothing_while_a_dynamic_is_stored( engine: Engine, ) -> None: config = alembic_v2_config() - # The fixture leaves the database at the head, which a later revision may - # be, so a downgrade reaches the dynamics revision; an upgrade to an older - # revision does nothing. + # The fixture upgrades to the head before the test, and the head can be + # later than the dynamics revision, so a downgrade reaches it; an upgrade + # to an older revision does nothing. command.downgrade(config, DYNAMICS_REVISION) with engine.begin() as connection: rows = insert_public_api_rows(connection) @@ -334,6 +334,8 @@ def test_the_downgrade_fails_changing_nothing_while_a_dynamic_is_stored( with pytest.raises(IntegrityError, match="name"): command.downgrade(config, PREVIOUS_REVISION) assert _current_revision(engine) == DYNAMICS_REVISION + mapping_columns = inspect(engine).get_columns("legacy_policy_mappings") + assert "dynamic_id" in {column["name"] for column in mapping_columns} finally: with engine.begin() as connection: connection.execute( From d18a530f48f9fe61bc291b8afe6072166f0d338f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 20 Nov 2026 01:29:16 +0400 Subject: [PATCH 08/10] Require CI to run the dynamics migration and persistence tests (failing) The V2 Postgres migration lifecycle job is to run test_alembic_v2_dynamics.py, and the cross-database integration job test_v2_dynamic_persistence.py, after the managed roles get their privileges, since it runs as the runtime role. Both came with #4325, and no CI job runs either yet. Co-Authored-By: Claude Opus 5.5 --- tests/unit/test_alembic_workflows.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/unit/test_alembic_workflows.py b/tests/unit/test_alembic_workflows.py index 7676d8a93..334f41380 100644 --- a/tests/unit/test_alembic_workflows.py +++ b/tests/unit/test_alembic_workflows.py @@ -215,6 +215,7 @@ def test_reusable_v2_alembic_check_uses_only_disposable_postgres(): assert "test_alembic_v2_withheld_area_codes.py" in lifecycle_script assert "test_alembic_v2_shared_report_keys.py" in lifecycle_script assert "test_alembic_v2_cliff_impacts.py" in lifecycle_script + assert "test_alembic_v2_dynamics.py" in lifecycle_script assert "redis:7.2-alpine" not in workflow assert "RUNTIME_CACHE_TEST_URL" not in workflow assert "test_v2_catalog_installed.py" not in workflow @@ -251,6 +252,14 @@ def test_reusable_v2_integration_check_runs_the_users_report_run_tests(): assert privileges_applied < workflow.index(module), module +def test_reusable_v2_integration_check_runs_the_dynamic_persistence_tests(): + """Storing dynamics and the v1-to-v2 copy run as the runtime role.""" + workflow = _workflow("v2-integration-check.yml") + + privileges_applied = workflow.index("reconcile_v2_privileges.py apply") + assert privileges_applied < workflow.index("test_v2_dynamic_persistence.py") + + def test_reusable_v2_integration_check_uses_disposable_databases_and_coverage(): workflow = _workflow("v2-integration-check.yml") From dc706b590808705f988342905cee7d96f7e5412d Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 20 Nov 2026 01:31:22 +0400 Subject: [PATCH 09/10] Run the dynamics migration and persistence tests in CI The V2 Postgres migration lifecycle script now runs test_alembic_v2_dynamics.py after the cliff impacts module, against its own disposable database, and the cross-database integration job runs test_v2_dynamic_persistence.py in its policy persistence step, after the managed roles get their privileges. Both read only V2_MIGRATION_DATABASE_URL, V2_ALEMBIC_DISPOSABLE_TEST and, for the managed roles, DEPLOYMENT_ENVIRONMENT, which each job's env block already sets. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/test_alembic_v2_lifecycle.sh | 3 ++- .github/workflows/v2-integration-check.yml | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/scripts/test_alembic_v2_lifecycle.sh b/.github/scripts/test_alembic_v2_lifecycle.sh index 7b6293df5..aa86cfa4e 100755 --- a/.github/scripts/test_alembic_v2_lifecycle.sh +++ b/.github/scripts/test_alembic_v2_lifecycle.sh @@ -12,4 +12,5 @@ uv run --frozen --extra dev pytest -q \ tests/integration/test_alembic_v2_user_types.py \ tests/integration/test_alembic_v2_withheld_area_codes.py \ tests/integration/test_alembic_v2_shared_report_keys.py \ - tests/integration/test_alembic_v2_cliff_impacts.py + tests/integration/test_alembic_v2_cliff_impacts.py \ + tests/integration/test_alembic_v2_dynamics.py diff --git a/.github/workflows/v2-integration-check.yml b/.github/workflows/v2-integration-check.yml index 61d18738a..3c45278f7 100644 --- a/.github/workflows/v2-integration-check.yml +++ b/.github/workflows/v2-integration-check.yml @@ -75,8 +75,8 @@ jobs: run: uv run --frozen --extra dev python scripts/reconcile_v2_privileges.py verify - name: Test v2 metadata publication, resource routes, the countries list, reports, simulations and users' report runs run: uv run --frozen --extra dev coverage run --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_catalog_publication_time_periods.py tests/integration/test_v2_catalog_publication_pruning.py tests/integration/test_v2_catalog_spm_availability.py tests/integration/test_v2_catalog_statistics.py tests/integration/test_v2_metadata_routes.py tests/integration/test_v2_country_routes.py tests/integration/test_v2_report_reads.py tests/integration/test_v2_report_creation.py tests/integration/test_v2_budget_window_report_reads.py tests/integration/test_v2_budget_window_report_creation.py tests/integration/test_v2_report_runs.py tests/integration/test_v2_report_reads_by_run.py tests/integration/test_v2_user_report_runs.py tests/integration/test_v2_user_report_run_requesters.py tests/integration/test_v2_user_budget_window_runs.py tests/integration/test_v2_forced_report_reruns.py tests/integration/test_v2_user_report_views.py - - name: Test v2 policy persistence and immediate v1 mirroring - run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_policy_persistence.py tests/integration/test_v1_policy_dual_write.py tests/integration/test_mysql_policy_dual_write.py tests/integration/test_v2_user_policy_mirroring.py tests/integration/test_v1_user_policy_dual_write.py + - name: Test v2 policy and dynamic persistence and immediate v1 mirroring + run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_policy_persistence.py tests/integration/test_v2_dynamic_persistence.py tests/integration/test_v1_policy_dual_write.py tests/integration/test_mysql_policy_dual_write.py tests/integration/test_v2_user_policy_mirroring.py tests/integration/test_v1_user_policy_dual_write.py - name: Test signed-in identity provisioning, user associations, saved detailed-save records and user types run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_user_identity_persistence.py tests/integration/test_v2_user_type_persistence.py tests/integration/test_v2_saved_detailed_save_records.py tests/integration/test_v2_saved_budget_window_reports.py - name: Test comparison-run persistence, the simulation API's roles, shared reports and runs, and privilege reconciliation as the managed roles From 22616f102a5d1e37538b341ba25483b437f1cb4c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 20 Nov 2026 01:41:53 +0400 Subject: [PATCH 10/10] Check the dynamics tests' target first, and keep them out of the integration job The dynamics migration tests' engine fixture now checks that the target is the disposable database before upgrading it, as the cliff impacts fixture does, so without one the tests are skipped instead of migrating whatever is configured. The integration workflow test now also requires that those migration tests stay out of that job, and says why in its docstring. Co-Authored-By: Claude Opus 5.5 --- tests/integration/test_alembic_v2_dynamics.py | 4 +++- tests/unit/test_alembic_workflows.py | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_alembic_v2_dynamics.py b/tests/integration/test_alembic_v2_dynamics.py index 658e244ca..819344839 100644 --- a/tests/integration/test_alembic_v2_dynamics.py +++ b/tests/integration/test_alembic_v2_dynamics.py @@ -40,8 +40,10 @@ @pytest.fixture def engine() -> Iterator[Engine]: """Yield an engine on a database at head, and leave it at head.""" + # Check that the target is the disposable database before migrating it. + database_url = disposable_v2_url() command.upgrade(alembic_v2_config(), "head") - database_engine = create_engine(disposable_v2_url()) + database_engine = create_engine(database_url) try: yield database_engine finally: diff --git a/tests/unit/test_alembic_workflows.py b/tests/unit/test_alembic_workflows.py index 334f41380..b1f49c857 100644 --- a/tests/unit/test_alembic_workflows.py +++ b/tests/unit/test_alembic_workflows.py @@ -253,11 +253,17 @@ def test_reusable_v2_integration_check_runs_the_users_report_run_tests(): def test_reusable_v2_integration_check_runs_the_dynamic_persistence_tests(): - """Storing dynamics and the v1-to-v2 copy run as the runtime role.""" + """The dynamic persistence tests run after the managed roles get their privileges. + + The dynamics revision's migration tests downgrade and upgrade the schema, + which would drop tables the job granted privileges on, so they run in the + lifecycle job instead. + """ workflow = _workflow("v2-integration-check.yml") privileges_applied = workflow.index("reconcile_v2_privileges.py apply") assert privileges_applied < workflow.index("test_v2_dynamic_persistence.py") + assert "test_alembic_v2_dynamics.py" not in workflow def test_reusable_v2_integration_check_uses_disposable_databases_and_coverage():