diff --git a/.github/scripts/test_alembic_v2_lifecycle.sh b/.github/scripts/test_alembic_v2_lifecycle.sh index 7c64360a9..ab5575c07 100755 --- a/.github/scripts/test_alembic_v2_lifecycle.sh +++ b/.github/scripts/test_alembic_v2_lifecycle.sh @@ -8,4 +8,5 @@ uv run --frozen --extra dev pytest -q \ tests/integration/test_alembic_v2_catalog_tables.py \ tests/integration/test_alembic_v2_simulation_runs.py \ tests/integration/test_alembic_v2_saved_item_keys.py \ - tests/integration/test_alembic_v2_user_types.py + tests/integration/test_alembic_v2_user_types.py \ + tests/integration/test_alembic_v2_withheld_area_codes.py diff --git a/changelog.d/v2-withheld-area-codes.added.md b/changelog.d/v2-withheld-area-codes.added.md new file mode 100644 index 000000000..f2ed5d70b --- /dev/null +++ b/changelog.d/v2-withheld-area-codes.added.md @@ -0,0 +1 @@ +The v2 database gains a `withheld_area_codes` table: one row per UK constituency or local authority that disclosure control withheld from a report run's local-area results, keyed by the report run and its two simulation runs. The simulation service's Modal workers may insert its rows and the Public API may read them; no route reads it yet. diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index 406ba6d40..ec29a5489 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -828,3 +828,32 @@ The revision writes no data, and the previous Public API revision, which doesn't map the column, keeps working at it. The downgrade drops the column with its values, which the next publication run of each PolicyEngine.py version records again. + +### API v2 withheld area codes + +Revision `1e1e3f079f2c` (parent `0c136a278f29`) was autogenerated, with +`--rev-id 1e1e3f079f2c`, after the `WithheldAreaCode` model was added. It +creates `withheld_area_codes`, one row per UK area that disclosure control +withheld from a report run's local-area results: the report run (`CASCADE`), +its baseline and reform simulation runs (`RESTRICT`, as the other comparison +result tables have), the `area_type` (the new native enum `v2_uk_area_type`, +`constituency` or `local_authority`) and the area's `code`, unique per report +run and area type. The run's writer leaves a withheld area out of +`constituency_impacts` and `local_authority_impacts`; no constraint across the +tables checks that. Nothing records whether a run's list was written, or which +writer completed the run, so no rows means no area was withheld only for a +succeeded run whose writer records the list, and the database can't tell such +a run from one whose writer doesn't. Until the Modal workers write this table, +every run has no rows. The Modal worker role may read and insert its rows, +never update or delete them, and the runtime role may read them; the reconcile +step after each schema upgrade grants them. + +It has one post-generation correction, for reversibility: Alembic's +generated downgrade drops the table but leaves the enum type, so the +downgrade also drops `v2_uk_area_type` after the table, as `439303be14fe` +does for its enums. Without it, upgrading again after a downgrade fails on +the existing type. The revision writes no data, and the previous Public API +revision, which doesn't map the table, keeps working at it. The downgrade +drops the table with its rows. `tests/integration/test_alembic_v2_withheld_area_codes.py` +checks that nothing else changes, that the downgrade removes the type and an +upgrade works again, and the table's keys and delete rules. diff --git a/migrations/v2/versions/1e1e3f079f2c_record_withheld_uk_area_codes.py b/migrations/v2/versions/1e1e3f079f2c_record_withheld_uk_area_codes.py new file mode 100644 index 000000000..daab1b7e1 --- /dev/null +++ b/migrations/v2/versions/1e1e3f079f2c_record_withheld_uk_area_codes.py @@ -0,0 +1,55 @@ +"""record withheld uk area codes + +Revision ID: 1e1e3f079f2c +Revises: 0c136a278f29 +Create Date: 2026-11-14 01:14:50.900369 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate + +Generated on 0c136a278f29, with --rev-id 1e1e3f079f2c. It creates the +withheld_area_codes table and its v2_uk_area_type enum type and changes no +other table, so it writes no data, and the previous Public API revision, which +doesn't map the table, keeps working. One correction after generation, for +reversibility: Alembic's generated downgrade drops the table but leaves the +enum type, so the downgrade also drops the type, after the table. The +downgrade drops the table with its rows. +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +revision: str = '1e1e3f079f2c' +down_revision: Union[str, None] = '0c136a278f29' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('withheld_area_codes', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('report_run_id', sa.Uuid(), nullable=False), + sa.Column('baseline_simulation_run_id', sa.Uuid(), nullable=False), + sa.Column('reform_simulation_run_id', sa.Uuid(), nullable=False), + sa.Column('area_type', sa.Enum('constituency', 'local_authority', name='v2_uk_area_type'), nullable=False), + sa.Column('code', sqlmodel.sql.sqltypes.AutoString(length=64), nullable=False), + sa.ForeignKeyConstraint(['baseline_simulation_run_id'], ['simulation_runs.id'], name=op.f('fk_withheld_area_codes_baseline_simulation_run_id_simulation_runs'), ondelete='RESTRICT'), + sa.ForeignKeyConstraint(['reform_simulation_run_id'], ['simulation_runs.id'], name=op.f('fk_withheld_area_codes_reform_simulation_run_id_simulation_runs'), ondelete='RESTRICT'), + sa.ForeignKeyConstraint(['report_run_id'], ['report_runs.id'], name=op.f('fk_withheld_area_codes_report_run_id_report_runs'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_withheld_area_codes')), + sa.UniqueConstraint('report_run_id', 'area_type', 'code', name='uq_withheld_area_codes_run_type_code') + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('withheld_area_codes') + # Post-generation reversibility correction: Alembic removes the table that + # uses this native PostgreSQL enum but leaves the schema-level type. + sa.Enum(name="v2_uk_area_type").drop(op.get_bind()) + # ### end Alembic commands ### diff --git a/policyengine_api/data/v2/migration_target.py b/policyengine_api/data/v2/migration_target.py index f52dc5987..02ddddab5 100644 --- a/policyengine_api/data/v2/migration_target.py +++ b/policyengine_api/data/v2/migration_target.py @@ -24,7 +24,7 @@ from collections.abc import Mapping V2_ALEMBIC_DISPOSABLE_TEST = "V2_ALEMBIC_DISPOSABLE_TEST" -V2_ALEMBIC_HEAD_REVISION = "0c136a278f29" +V2_ALEMBIC_HEAD_REVISION = "1e1e3f079f2c" DISPOSABLE_DATABASE_NAME = "policyengine_v2_alembic_test" DISPOSABLE_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "postgres"}) MIGRATION_ROLE = "policyengine_v2_migrator" diff --git a/policyengine_api/data/v2/models/__init__.py b/policyengine_api/data/v2/models/__init__.py index 0eeafa5a5..8169bf866 100644 --- a/policyengine_api/data/v2/models/__init__.py +++ b/policyengine_api/data/v2/models/__init__.py @@ -89,6 +89,8 @@ OutputStatus, Poverty, ProgramStatistics, + UKAreaType, + WithheldAreaCode, ) from policyengine_api.data.v2.models.simulations import ( # noqa: E402 Simulation, @@ -165,6 +167,7 @@ "Stage12SimulationRole", "TaxBenefitModel", "TaxBenefitModelVersion", + "UKAreaType", "User", "UserHouseholdAssociation", "UserIdentity", @@ -175,4 +178,5 @@ "UserTypeChange", "UserTypeChangeSource", "Variable", + "WithheldAreaCode", ] diff --git a/policyengine_api/data/v2/models/reports.py b/policyengine_api/data/v2/models/reports.py index 9e3770d2c..8a1ae4b7e 100644 --- a/policyengine_api/data/v2/models/reports.py +++ b/policyengine_api/data/v2/models/reports.py @@ -41,6 +41,7 @@ LaborSupplyResponse, LocalAuthorityImpact, ProgramStatistics, + WithheldAreaCode, ) # A budget window compares at most this many years, none after LAST_YEAR. @@ -376,3 +377,7 @@ class ReportRun(TimestampedModel, table=True): back_populates="report_run", cascade_delete=True, ) + withheld_area_codes: list["WithheldAreaCode"] = Relationship( + back_populates="report_run", + cascade_delete=True, + ) diff --git a/policyengine_api/data/v2/models/run_results.py b/policyengine_api/data/v2/models/run_results.py index 1f6943783..2e691bd2f 100644 --- a/policyengine_api/data/v2/models/run_results.py +++ b/policyengine_api/data/v2/models/run_results.py @@ -42,6 +42,13 @@ class DecileType(StrEnum): WEALTH = "wealth" +class UKAreaType(StrEnum): + """The level of UK area a local-area breakdown reports.""" + + CONSTITUENCY = "constituency" + LOCAL_AUTHORITY = "local_authority" + + class LaborSupplyResponse(IdentifiedModel, table=True): """The labour supply response to the reform in one report run. @@ -552,3 +559,44 @@ class LocalAuthorityImpact(UKAreaImpactBase, table=True): local_authority_name: str = Field(max_length=255) report_run: ReportRun = Relationship(back_populates="local_authority_impacts") + + +class WithheldAreaCode(IdentifiedModel, table=True): + """A UK area whose results disclosure control withheld from a report run. + + One row per report run, area type and area code. The run's writer leaves a + withheld area out of ``constituency_impacts`` and + ``local_authority_impacts``; no constraint across the tables checks that. + Nothing records whether a run's list was written, or which writer completed + the run, so no rows means no area was withheld only for a succeeded run + whose writer records the list, and the database can't tell such a run from + one whose writer doesn't. Until the Modal workers write this table, every + run has no rows. + """ + + __tablename__ = "withheld_area_codes" + __table_args__ = ( + sa.UniqueConstraint( + "report_run_id", + "area_type", + "code", + name="uq_withheld_area_codes_run_type_code", + ), + ) + + report_run_id: UUID = Field( + foreign_key="report_runs.id", + ondelete="CASCADE", + ) + baseline_simulation_run_id: UUID = Field( + foreign_key="simulation_runs.id", + ondelete="RESTRICT", + ) + reform_simulation_run_id: UUID = Field( + foreign_key="simulation_runs.id", + ondelete="RESTRICT", + ) + area_type: UKAreaType = Field(sa_type=enum_type(UKAreaType, "v2_uk_area_type")) + code: str = Field(max_length=64) + + report_run: ReportRun = Relationship(back_populates="withheld_area_codes") diff --git a/policyengine_api/data/v2/privileges.py b/policyengine_api/data/v2/privileges.py index 4079c8a7b..1d2862745 100644 --- a/policyengine_api/data/v2/privileges.py +++ b/policyengine_api/data/v2/privileges.py @@ -264,6 +264,8 @@ def _run_table( "local_authority_impacts": _RESULT, "poverty": _RESULT, "program_statistics": _RESULT, + # The UK areas disclosure control withheld from a local-area run's results. + "withheld_area_codes": _RESULT, # Metadata catalog: the catalog writer publishes it, the Public API reads it. "dataset_versions": _access(catalog_writer=READ), "datasets": _access( diff --git a/tests/integration/test_alembic_v2_simulation_runs.py b/tests/integration/test_alembic_v2_simulation_runs.py index 69544e145..202d03bf7 100644 --- a/tests/integration/test_alembic_v2_simulation_runs.py +++ b/tests/integration/test_alembic_v2_simulation_runs.py @@ -72,9 +72,11 @@ } ) # The tables the simulation service and its Modal workers write; the Public API -# writes every other table. +# writes every other table. withheld_area_codes came in a later revision. SIMULATION_SIDE_TABLES = ( - CHANGED_TABLES | NEW_TABLES | {"aggregates", "change_aggregates"} + CHANGED_TABLES + | NEW_TABLES + | {"aggregates", "change_aggregates", "withheld_area_codes"} ) # The foreign keys from the simulation service's tables to the Public API's # rows, with their delete rules. diff --git a/tests/integration/test_alembic_v2_withheld_area_codes.py b/tests/integration/test_alembic_v2_withheld_area_codes.py new file mode 100644 index 000000000..150400663 --- /dev/null +++ b/tests/integration/test_alembic_v2_withheld_area_codes.py @@ -0,0 +1,311 @@ +"""Check the withheld area codes revision against disposable Postgres. + +Revision 1e1e3f079f2c, on the SPM availability revision 0c136a278f29, adds +``withheld_area_codes`` and its ``v2_uk_area_type`` enum and changes no other +table. Each row names a UK area that disclosure control withheld from a +report run's local-area results. The downgrade drops the table, with its rows, +then the type. +""" + +from collections.abc import Iterator +from typing import Any +from uuid import UUID, uuid4 + +import pytest +from alembic import command +from sqlalchemy import Connection, Engine, create_engine, inspect, text +from sqlalchemy.exc import DataError, IntegrityError +from sqlalchemy.pool import NullPool + +from tests.fixtures.integration.alembic_v2 import ( + alembic_v2_config, + disposable_v2_url, +) +from tests.integration.simulation_rows import ( + RUN_VERSIONS, + insert_public_api_rows, + insert_row, + refused, +) + +SPM_AVAILABILITY_REVISION = "0c136a278f29" +WITHHELD_AREA_CODES_REVISION = "1e1e3f079f2c" + + +def _schema_shape(engine: Engine) -> dict[str, dict[str, Any]]: + """Reflect every table's columns, keys, indexes and checks.""" + inspector = inspect(engine) + return { + table_name: { + "columns": [ + ( + column["name"], + str(column["type"]), + column["nullable"], + column.get("default"), + ) + for column in inspector.get_columns(table_name) + ], + "primary_key": inspector.get_pk_constraint(table_name), + "foreign_keys": sorted( + ( + key["name"], + key["constrained_columns"], + key["referred_table"], + key["options"].get("ondelete"), + ) + for key in inspector.get_foreign_keys(table_name) + ), + "unique_constraints": sorted( + (unique["name"], unique["column_names"]) + for unique in inspector.get_unique_constraints(table_name) + ), + "indexes": sorted( + (index["name"], index["column_names"]) + for index in inspector.get_indexes(table_name) + ), + "checks": { + check["name"]: check["sqltext"] + for check in inspector.get_check_constraints(table_name) + }, + } + for table_name in inspector.get_table_names(schema="public") + if table_name != "alembic_version" + } + + +def _enum_labels(engine: Engine) -> dict[str, list[str]]: + """Return each enum type in the public schema with its labels in order.""" + with engine.connect() as connection: + rows = connection.execute( + text( + "SELECT type.typname, enum.enumlabel FROM pg_type AS type " + "JOIN pg_enum AS enum ON enum.enumtypid = type.oid " + "JOIN pg_namespace AS namespace ON namespace.oid = type.typnamespace " + "WHERE namespace.nspname = 'public' " + "ORDER BY type.typname, enum.enumsortorder" + ) + ).all() + labels: dict[str, list[str]] = {} + for type_name, label in rows: + labels.setdefault(type_name, []).append(label) + return labels + + +@pytest.fixture +def engine() -> Iterator[Engine]: + """Yield an engine on the disposable database, left at the v2 head.""" + database_url = disposable_v2_url() + database_engine = create_engine(database_url, poolclass=NullPool) + try: + yield database_engine + finally: + command.upgrade(alembic_v2_config(), "head") + database_engine.dispose() + + +def test_withheld_area_codes_revision_adds_only_the_table_and_its_type( + engine: Engine, +) -> None: + """The revision adds one table and one enum type and changes nothing else.""" + config = alembic_v2_config() + + command.upgrade(config, "head") + command.downgrade(config, SPM_AVAILABILITY_REVISION) + parent_shape = _schema_shape(engine) + parent_enums = _enum_labels(engine) + command.upgrade(config, WITHHELD_AREA_CODES_REVISION) + revision_shape = _schema_shape(engine) + revision_enums = _enum_labels(engine) + + assert set(revision_shape) - set(parent_shape) == {"withheld_area_codes"} + assert { + table_name: shape + for table_name, shape in revision_shape.items() + if table_name != "withheld_area_codes" + } == parent_shape + assert revision_enums == { + **parent_enums, + "v2_uk_area_type": ["constituency", "local_authority"], + } + withheld = revision_shape["withheld_area_codes"] + assert withheld["primary_key"]["constrained_columns"] == ["id"] + assert withheld["unique_constraints"] == [ + ( + "uq_withheld_area_codes_run_type_code", + ["report_run_id", "area_type", "code"], + ) + ] + assert sorted( + (columns, referred_table, ondelete) + for _name, columns, referred_table, ondelete in withheld["foreign_keys"] + ) == [ + (["baseline_simulation_run_id"], "simulation_runs", "RESTRICT"), + (["reform_simulation_run_id"], "simulation_runs", "RESTRICT"), + (["report_run_id"], "report_runs", "CASCADE"), + ] + + +def test_withheld_area_codes_downgrade_removes_the_table_and_its_type( + engine: Engine, +) -> None: + """Downgrading drops the table and the type, so upgrading again works.""" + config = alembic_v2_config() + + command.upgrade(config, "head") + command.downgrade(config, WITHHELD_AREA_CODES_REVISION) + command.downgrade(config, SPM_AVAILABILITY_REVISION) + + assert "withheld_area_codes" not in inspect(engine).get_table_names() + assert "v2_uk_area_type" not in _enum_labels(engine) + + command.upgrade(config, WITHHELD_AREA_CODES_REVISION) + + assert "withheld_area_codes" in inspect(engine).get_table_names() + assert _enum_labels(engine)["v2_uk_area_type"] == [ + "constituency", + "local_authority", + ] + + +def _insert_report_run(connection: Connection) -> dict[str, UUID]: + """Store a UK report run comparing two simulation runs, and a spare run.""" + public_api_rows = insert_public_api_rows(connection) + ids = { + name: uuid4() + for name in ( + "simulation", + "baseline_run", + "reform_run", + "unrelated_run", + "report", + "report_run", + ) + } + insert_row( + connection, + "simulations", + id=ids["simulation"], + simulation_type="economy", + dataset_id=public_api_rows.dataset_id, + tax_benefit_model_version_id=public_api_rows.version_id, + year=2026, + ) + for run_name in ("baseline_run", "reform_run", "unrelated_run"): + insert_row( + connection, + "simulation_runs", + id=ids[run_name], + simulation_id=ids["simulation"], + tax_benefit_model_version_id=public_api_rows.version_id, + year=2026, + status="pending", + **RUN_VERSIONS, + ) + insert_row( + connection, + "reports", + id=ids["report"], + label="Local areas", + country="uk", + tax_benefit_model_id=public_api_rows.model_id, + year=2026, + inputs="{}", + ) + insert_row( + connection, + "report_runs", + id=ids["report_run"], + report_id=ids["report"], + baseline_simulation_run_id=ids["baseline_run"], + reform_simulation_run_id=ids["reform_run"], + status="pending", + trigger="initial", + **RUN_VERSIONS, + ) + return ids + + +def _insert_withheld_code( + connection: Connection, + ids: dict[str, UUID], + *, + area_type: str, + code: str, + reform_run: str = "reform_run", +) -> None: + insert_row( + connection, + "withheld_area_codes", + id=uuid4(), + report_run_id=ids["report_run"], + baseline_simulation_run_id=ids["baseline_run"], + reform_simulation_run_id=ids[reform_run], + area_type=area_type, + code=code, + ) + + +def _withheld_codes(connection: Connection, report_run_id: UUID) -> list[tuple]: + return connection.execute( + text( + "SELECT area_type::text, code FROM withheld_area_codes " + "WHERE report_run_id = :report_run_id ORDER BY area_type, code" + ), + {"report_run_id": report_run_id}, + ).all() + + +def test_withheld_area_codes_hold_each_area_once_and_go_with_their_run( + engine: Engine, +) -> None: + """Each run lists an area once, of a known type; deleting the run drops it.""" + command.upgrade(alembic_v2_config(), "head") + + with engine.connect() as connection: + transaction = connection.begin() + try: + ids = _insert_report_run(connection) + _insert_withheld_code( + connection, ids, area_type="constituency", code="E14001063" + ) + # The same code under the other area type is a different area. + _insert_withheld_code( + connection, ids, area_type="local_authority", code="E14001063" + ) + _insert_withheld_code( + connection, + ids, + area_type="local_authority", + code="E06000001", + reform_run="unrelated_run", + ) + + with refused(connection, IntegrityError, match="uq_withheld_area_codes"): + _insert_withheld_code( + connection, ids, area_type="constituency", code="E14001063" + ) + with refused(connection, DataError, match="v2_uk_area_type"): + _insert_withheld_code( + connection, ids, area_type="ward", code="E05000001" + ) + # A simulation run that withheld codes name stays while they do. + with refused(connection, IntegrityError, match="withheld_area_codes"): + connection.execute( + text("DELETE FROM simulation_runs WHERE id = :id"), + {"id": ids["unrelated_run"]}, + ) + assert _withheld_codes(connection, ids["report_run"]) == [ + ("constituency", "E14001063"), + ("local_authority", "E06000001"), + ("local_authority", "E14001063"), + ] + + connection.execute( + text("DELETE FROM report_runs WHERE id = :id"), + {"id": ids["report_run"]}, + ) + + assert _withheld_codes(connection, ids["report_run"]) == [] + finally: + transaction.rollback() diff --git a/tests/integration/test_v2_simulation_role_privileges.py b/tests/integration/test_v2_simulation_role_privileges.py index 8ebe80efa..2bc48db88 100644 --- a/tests/integration/test_v2_simulation_role_privileges.py +++ b/tests/integration/test_v2_simulation_role_privileges.py @@ -23,6 +23,7 @@ from policyengine_api.constants import REPO from policyengine_api.data.v2.models import V2_METADATA from policyengine_api.data.v2.privileges import ( + CATALOG_WRITER_ROLE, ColumnPrivilege, TablePrivilege, declared_privileges, @@ -322,6 +323,74 @@ def test_modal_workers_write_their_run_status_and_results_only( _refused(connection, statement, id=ids[run_key]) +def test_modal_workers_insert_withheld_area_codes_that_the_public_api_reads( + connection: Connection, managed_v2_roles, public_api_rows: PublicApiRows +) -> None: + """The workers insert and read the rows, the Public API only reads them. + + The simulation service, the Stage 12 role and the catalog writer can't read + them. + """ + _act_as(connection, managed_v2_roles.simulation_service) + ids = _create_simulation_rows(connection, public_api_rows) + _act_as(connection, managed_v2_roles.modal_worker) + withheld_code_id = uuid4() + + insert_row( + connection, + "withheld_area_codes", + id=withheld_code_id, + report_run_id=ids["report_run"], + baseline_simulation_run_id=ids["run"], + reform_simulation_run_id=ids["run"], + area_type="constituency", + code="E14001063", + ) + assert ( + connection.execute( + text("SELECT count(*) FROM withheld_area_codes WHERE report_run_id = :id"), + {"id": ids["report_run"]}, + ).scalar_one() + == 1 + ) + + # Once written, a withheld code is never changed or removed by the workers. + _refused( + connection, + "UPDATE withheld_area_codes SET code = 'E14001064' WHERE id = :id", + id=withheld_code_id, + ) + _refused( + connection, + "DELETE FROM withheld_area_codes WHERE id = :id", + id=withheld_code_id, + ) + _act_as(connection, managed_v2_roles.runtime) + assert ( + connection.execute( + text("SELECT code FROM withheld_area_codes WHERE id = :id"), + {"id": withheld_code_id}, + ).scalar_one() + == "E14001063" + ) + _refused( + connection, + "INSERT INTO withheld_area_codes (id, report_run_id, " + "baseline_simulation_run_id, reform_simulation_run_id, area_type, code) " + "VALUES (:id, :report_run_id, :run_id, :run_id, 'constituency', 'E14001064')", + id=uuid4(), + report_run_id=ids["report_run"], + run_id=ids["run"], + ) + for role in ( + managed_v2_roles.simulation_service, + managed_v2_roles.stage12, + CATALOG_WRITER_ROLE, + ): + _act_as(connection, role) + _refused(connection, "SELECT count(*) FROM withheld_area_codes") + + def test_the_simulation_service_fails_a_window_whose_first_calls_modal_refuses( connection: Connection, managed_v2_roles, public_api_rows: PublicApiRows ) -> None: diff --git a/tests/unit/test_alembic_workflows.py b/tests/unit/test_alembic_workflows.py index 7f3058b9a..405e42080 100644 --- a/tests/unit/test_alembic_workflows.py +++ b/tests/unit/test_alembic_workflows.py @@ -211,6 +211,7 @@ def test_reusable_v2_alembic_check_uses_only_disposable_postgres(): assert "test_alembic_v2_simulation_runs.py" in lifecycle_script assert "test_alembic_v2_saved_item_keys.py" in lifecycle_script assert "test_alembic_v2_user_types.py" in lifecycle_script + assert "test_alembic_v2_withheld_area_codes.py" in lifecycle_script assert "redis:7.2-alpine" not in workflow assert "RUNTIME_CACHE_TEST_URL" not in workflow assert "test_v2_catalog_installed.py" not in workflow diff --git a/tests/unit/v2/test_alembic_v2.py b/tests/unit/v2/test_alembic_v2.py index 1c27b6f0d..88d7441b7 100644 --- a/tests/unit/v2/test_alembic_v2.py +++ b/tests/unit/v2/test_alembic_v2.py @@ -55,6 +55,7 @@ "stage12_evaluation_simulations", "user_identities", "user_type_changes", + "withheld_area_codes", } ) POOLER_URL = ( @@ -79,6 +80,11 @@ # The user type changes revision, on the user types revision, and parent of # the SPM availability revision. USER_TYPE_CHANGES_REVISION = "970be5d29283" +# The SPM availability revision, on the user type changes revision, and parent +# of the withheld area codes revision. +SPM_AVAILABILITY_REVISION = "0c136a278f29" +# The withheld area codes revision, on the SPM availability revision. +WITHHELD_AREA_CODES_REVISION = "1e1e3f079f2c" DISPOSABLE_URL = ( "postgresql+psycopg://postgres:test-password@127.0.0.1:5432/" f"{DISPOSABLE_DATABASE_NAME}" @@ -260,6 +266,7 @@ def test_v2_revision_chain_has_generated_policy_and_user_identity_changes() -> N assert script.get_heads() == [V2_ALEMBIC_HEAD_REVISION] assert [revision.revision for revision in script.walk_revisions()] == [ V2_ALEMBIC_HEAD_REVISION, + SPM_AVAILABILITY_REVISION, USER_TYPE_CHANGES_REVISION, USER_TYPES_REVISION, SAVED_ITEM_KEYS_REVISION, @@ -1298,7 +1305,7 @@ def test_spm_availability_revision_adds_only_one_nullable_column() -> None: if revision.down_revision == USER_TYPE_CHANGES_REVISION ] assert len(children) == 1 - assert children[0].revision == V2_ALEMBIC_HEAD_REVISION + assert children[0].revision == SPM_AVAILABILITY_REVISION revision = Path(children[0].path).read_text(encoding="utf-8") upgrade = revision[ revision.index("def upgrade()") : revision.index("def downgrade()") @@ -1328,3 +1335,56 @@ def test_spm_availability_revision_adds_only_one_nullable_column() -> None: assert "op.bulk_insert(" not in body assert "op.create_table(" not in body assert "op.drop_table(" not in body + + +def test_withheld_area_codes_revision_creates_only_the_table_and_its_type() -> None: + """The revision after SPM availability adds ``withheld_area_codes``. + + It is generated, creates the table with its ``v2_uk_area_type`` enum, and + writes no data. Its one post-generation correction drops the enum type on + downgrade, after the table, which Alembic's generated downgrade leaves. + """ + script = ScriptDirectory.from_config(Config(str(REPO / "alembic-v2.ini"))) + children = [ + revision + for revision in script.walk_revisions() + if revision.down_revision == SPM_AVAILABILITY_REVISION + ] + assert [child.revision for child in children] == [WITHHELD_AREA_CODES_REVISION] + revision = Path(children[0].path).read_text(encoding="utf-8") + upgrade = revision[ + revision.index("def upgrade()") : revision.index("def downgrade()") + ] + downgrade = revision[revision.index("def downgrade()") :] + + assert ( + "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" + in revision + ) + assert sorted(set(re.findall(r"op\.\w+", upgrade))) == [ + "op.create_table", + "op.f", + ] + assert upgrade.count("op.create_table(") == 1 + assert re.search(r"""op\.create_table\(\s*["']withheld_area_codes["']""", upgrade) + assert re.search( + r"""sa\.Enum\(\s*["']constituency["'],\s*["']local_authority["'],\s*""" + r"""name=["']v2_uk_area_type["']""", + upgrade, + ) + assert sorted(set(re.findall(r"op\.\w+", downgrade))) == [ + "op.drop_table", + "op.get_bind", + ] + assert re.search(r"""op\.drop_table\(\s*["']withheld_area_codes["']""", downgrade) + assert revision.count("Post-generation reversibility correction:") == 1 + corrected_enum_names = re.findall( + r"sa\.Enum\(name=[\"']([^\"']+)[\"']\)\.drop\(op\.get_bind\(\)\)", + downgrade, + ) + assert corrected_enum_names == ["v2_uk_area_type"] + # The type is dropped after the table that uses it. + assert downgrade.index("drop_table(") < downgrade.index(".drop(op.get_bind())") + for body in (upgrade, downgrade): + assert "op.execute(" not in body + assert "op.bulk_insert(" not in body diff --git a/tests/unit/v2/test_import_side_effects.py b/tests/unit/v2/test_import_side_effects.py index cbeae024e..88cb604b6 100644 --- a/tests/unit/v2/test_import_side_effects.py +++ b/tests/unit/v2/test_import_side_effects.py @@ -69,7 +69,7 @@ def reject_ddl(*args, **kwargs): import sys after = set(pathlib.Path.cwd().iterdir()) assert before == after -assert len(V2_METADATA.tables) == 47 +assert len(V2_METADATA.tables) == 48 assert "policyengine_api.data.v2.catalog.initialization" not in sys.modules assert "policyengine_api.data.v2.policy_migration_qualification" not in sys.modules assert "policyengine_api.data.v2.household_migration_qualification" not in sys.modules diff --git a/tests/unit/v2/test_models.py b/tests/unit/v2/test_models.py index ebedf109f..6cc1f5446 100644 --- a/tests/unit/v2/test_models.py +++ b/tests/unit/v2/test_models.py @@ -61,6 +61,7 @@ "intra_decile_impacts", "local_authority_impacts", "program_statistics", + "withheld_area_codes", } ) diff --git a/tests/unit/v2/test_privilege_manifest.py b/tests/unit/v2/test_privilege_manifest.py index b5e18e507..e25eb6098 100644 --- a/tests/unit/v2/test_privilege_manifest.py +++ b/tests/unit/v2/test_privilege_manifest.py @@ -165,6 +165,7 @@ "local_authority_impacts", "poverty", "program_statistics", + "withheld_area_codes", } ) WRITE_PRIVILEGES = frozenset( diff --git a/tests/unit/v2/test_simulation_run_models.py b/tests/unit/v2/test_simulation_run_models.py index 24298ab60..2fdd3acc4 100644 --- a/tests/unit/v2/test_simulation_run_models.py +++ b/tests/unit/v2/test_simulation_run_models.py @@ -55,6 +55,7 @@ "intra_decile_impacts", "local_authority_impacts", "program_statistics", + "withheld_area_codes", ) # The tables the simulation service and its Modal workers write. The Public API # writes every other table. @@ -341,6 +342,42 @@ def test_uk_area_impacts_record_disclosure_control() -> None: assert f"ck_{table_name}_disclosure_tier" in _check_names(table) +def test_withheld_area_codes_list_each_area_a_report_run_held_back() -> None: + """One row per UK area that disclosure control withheld from a report run.""" + withheld = V2_METADATA.tables["withheld_area_codes"] + + assert set(withheld.c.keys()) == { + "id", + "created_at", + "report_run_id", + "baseline_simulation_run_id", + "reform_simulation_run_id", + "area_type", + "code", + } + area_type = withheld.c.area_type + assert not area_type.nullable + assert isinstance(area_type.type, sa.Enum) + assert area_type.type.name == "v2_uk_area_type" + assert area_type.type.enums == ["constituency", "local_authority"] + assert not withheld.c.code.nullable + assert withheld.c.code.type.length == 64 + assert { + constraint.name: tuple(column.name for column in constraint.columns) + for constraint in withheld.constraints + if isinstance(constraint, sa.UniqueConstraint) + } == { + "uq_withheld_area_codes_run_type_code": ( + "report_run_id", + "area_type", + "code", + ) + } + relationship = sa.inspect(ReportRun).relationships["withheld_area_codes"] + assert relationship.mapper.local_table is withheld + assert "delete" in relationship.cascade + + def test_congressional_district_impacts_record_winner_and_loser_shares() -> None: districts = V2_METADATA.tables["congressional_district_impacts"]