diff --git a/changelog.d/stop-mapping-report-per-user-columns.removed.md b/changelog.d/stop-mapping-report-per-user-columns.removed.md new file mode 100644 index 000000000..579927b93 --- /dev/null +++ b/changelog.d/stop-mapping-report-per-user-columns.removed.md @@ -0,0 +1 @@ +`/v2/reports` run items and budget-window run items no longer have a `trigger` field: a run is shared between requests, and each user's trigger is on their record of it. API v1 no longer selects or writes the per-user columns of reports, report runs and budget-window runs, which a later database revision drops. diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index 98ac08c83..770993991 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -931,10 +931,9 @@ from there gives the same operations. - The per-user columns of shared rows become nullable, with no default: `reports.label`, `report_runs.trigger` and `budget_window_runs.trigger` (`reports.description` and `user_id` already were). A shared row describes - no one caller, so the simulation service stops writing them, and a later - revision drops them; until then the report read and run lists answer a - missing label or trigger as null. A user's trigger is on their - `user_report_run_associations` row. + no one caller, so the simulation service stops writing them, API v1's models + leave them out, and a later revision drops them. A user's trigger is on + their `user_report_run_associations` row. - `user_report_run_associations` records each user's request for a report run: the user (`CASCADE`), the run (`RESTRICT`, so a run a user asked for can't be deleted), `requested_at` (required, with no default: the writer diff --git a/policyengine_api/data/v2/models/reports.py b/policyengine_api/data/v2/models/reports.py index aae5ef99e..586ad6874 100644 --- a/policyengine_api/data/v2/models/reports.py +++ b/policyengine_api/data/v2/models/reports.py @@ -28,7 +28,6 @@ ) from policyengine_api.data.v2.models.policies import Policy from policyengine_api.data.v2.models.simulations import Simulation -from policyengine_api.data.v2.models.users import User if TYPE_CHECKING: from policyengine_api.data.v2.models.run_results import ( @@ -61,7 +60,10 @@ class ReportRunStatus(StrEnum): class ReportRunTrigger(StrEnum): - """Why a report run started: the report's first run, a manual or a system rerun.""" + """Why a user asked for a report run: the report's creation or a re-run. + + ``system`` remains in the database type; no user's record may hold it. + """ INITIAL = "initial" MANUAL = "manual" @@ -71,13 +73,18 @@ class ReportRunTrigger(StrEnum): class Report(TimestampedModel, table=True): """Stable report definition; execution state belongs to ReportRun. - The simulation service writes reports, and the Public API the user, model, - policy, household, dataset and region a report names; each service writes - only its own tables, and the foreign keys between them stay. Deleting the - user, policy, dataset or region clears the reference, which for the user - is found through ``user_id``'s index; the model and household can't be + The simulation service writes reports, and API v1 the model, policy, + household, dataset and region a report names; each service writes only its + own tables, and the foreign keys between them stay. Deleting the policy, + dataset or region clears the reference; the model and household can't be deleted while a report names them. + A report is shared by everyone who asks for it, so it describes no one + caller: a report's name and description are on each user's saved report. + The table's ``label``, ``description`` and ``user_id`` are left out of this + model, so API v1 neither loads nor writes them and a release keeps working + while a later revision drops them. + The simulation service finds a report by ``definition_key``, the SHA-256 of what it compares (its country, type, each side's policy and dynamic, the dataset name, region code, year and SPM selection both sides share, @@ -89,7 +96,19 @@ class Report(TimestampedModel, table=True): """ __tablename__ = "reports" + __mapper_args__: ClassVar[dict[str, Any]] = { + "exclude_properties": ("label", "description", "user_id") + } __table_args__ = ( + # Kept in the table, unmapped, until a revision drops them. + sa.Column("label", sa.String(255)), + sa.Column("description", sa.String), + sa.Column( + "user_id", + sa.Uuid, + sa.ForeignKey("users.id", ondelete="SET NULL"), + index=True, + ), sa.CheckConstraint( "year IS NULL OR year BETWEEN 1900 AND 2200", name="ck_reports_year", @@ -117,18 +136,8 @@ class Report(TimestampedModel, table=True): sa.Index("ix_reports_dataset_independent_key", "dataset_independent_key"), ) - # Per-user: nullable until a later revision drops it, since a shared report - # describes no one caller. - label: str | None = Field(default=None, max_length=255) - description: str | None = None country: str = Field(max_length=16) type: str | None = Field(default=None, max_length=128) - user_id: UUID | None = Field( - default=None, - foreign_key="users.id", - ondelete="SET NULL", - index=True, - ) tax_benefit_model_id: UUID = Field( foreign_key="tax_benefit_models.id", ondelete="RESTRICT", @@ -171,7 +180,6 @@ class Report(TimestampedModel, table=True): definition_key: str | None = Field(default=None, max_length=64) dataset_independent_key: str | None = Field(default=None, max_length=64) - user: User | None = Relationship(back_populates="reports") tax_benefit_model: TaxBenefitModel = Relationship(back_populates="reports") policy: Policy | None = Relationship(back_populates="reports") baseline_simulation: Simulation | None = Relationship( @@ -217,7 +225,12 @@ class BudgetWindowRun(TimestampedModel, table=True): """ __tablename__ = "budget_window_runs" + __mapper_args__: ClassVar[dict[str, Any]] = {"exclude_properties": ("trigger",)} __table_args__ = ( + # Kept in the table, unmapped, until a revision drops it: a shared run + # answers more than one request, and each user's trigger is on their + # record of it. + sa.Column("trigger", enum_type(ReportRunTrigger, "v2_report_run_trigger")), sa.UniqueConstraint("id", "report_id", name="uq_budget_window_runs_id_report"), sa.CheckConstraint( "status NOT IN ('succeeded', 'failed') OR completed_at IS NOT NULL", @@ -247,12 +260,6 @@ class BudgetWindowRun(TimestampedModel, table=True): index=True, ) policyengine_version: str | None = Field(default=None, max_length=128) - # Per-user: nullable until a later revision drops it, since a shared run - # answers more than one request; each user's trigger is on their record. - trigger: ReportRunTrigger | None = Field( - default=None, - sa_type=enum_type(ReportRunTrigger, "v2_report_run_trigger"), - ) recalculate: bool = False status: ReportRunStatus = Field( default=ReportRunStatus.PENDING, @@ -280,11 +287,11 @@ class ReportRun(TimestampedModel, table=True): and ``year``, at most one report run per year of a window run. ``spm_provenance`` is the SPM receipt and ``data_version`` the data release it used. Its labour supply response has its own two tables. Its cliff impact - is its two simulation runs' (``CliffImpact``). The table's four older cliff - columns are declared in ``__table_args__`` and left out of the mapper, so API - v1 neither loads nor writes them. This covers only those four: ``trigger`` is - still mapped, so it must stop being mapped in an earlier release before a - revision drops it. + is its two simulation runs' (``CliffImpact``), and each user's trigger is on + their record of the run. The table's four older cliff columns and + ``trigger`` are declared in ``__table_args__`` and left out of the mapper, so + API v1 neither loads nor writes them and a release keeps working while a + later revision drops them. Outside budget windows, a report has at most one run that hasn't failed for each pair of package versions, which identical requests join; a @@ -295,6 +302,7 @@ class ReportRun(TimestampedModel, table=True): __tablename__ = "report_runs" __mapper_args__: ClassVar[dict[str, Any]] = { "exclude_properties": ( + "trigger", "baseline_cliff_gap", "baseline_cliff_share", "reform_cliff_gap", @@ -303,6 +311,7 @@ class ReportRun(TimestampedModel, table=True): } __table_args__ = ( # Kept in the table, unmapped, until a revision drops them. + sa.Column("trigger", enum_type(ReportRunTrigger, "v2_report_run_trigger")), sa.Column("baseline_cliff_gap", sa.Float), sa.Column("baseline_cliff_share", sa.Float), sa.Column("reform_cliff_gap", sa.Float), @@ -391,12 +400,6 @@ class ReportRun(TimestampedModel, table=True): default=ReportRunStatus.PENDING, sa_type=enum_type(ReportRunStatus, "v2_report_run_status"), ) - # Per-user: nullable until a later revision drops it, since a shared run - # answers more than one request; each user's trigger is on their record. - trigger: ReportRunTrigger | None = Field( - default=None, - sa_type=enum_type(ReportRunTrigger, "v2_report_run_trigger"), - ) idempotency_key: UUID | None = Field(default=None) modal_call_id: str | None = Field(default=None, max_length=255) started_at: datetime | None = Field( diff --git a/policyengine_api/data/v2/models/users.py b/policyengine_api/data/v2/models/users.py index 0a4d55018..ee923cb08 100644 --- a/policyengine_api/data/v2/models/users.py +++ b/policyengine_api/data/v2/models/users.py @@ -27,7 +27,6 @@ UserReportRunAssociation, UserSimulationAssociation, ) - from policyengine_api.data.v2.models.reports import Report class UserType(StrEnum): @@ -79,8 +78,8 @@ class User(IdentifiedModel, table=True): A signed-in user, of any type but ``unauthenticated``, always has a primary country. Deleting a user deletes their sign-in identities, saved - associations and records of the report runs they asked for; reports they - own are kept without an owner. + associations and records of the report runs they asked for; the reports + and runs themselves are shared and stay. """ __tablename__ = "users" @@ -115,7 +114,6 @@ class User(IdentifiedModel, table=True): sa_column_kwargs={"server_default": UserType.STANDARD.value}, ) - reports: list["Report"] = Relationship(back_populates="user") identities: list["UserIdentity"] = Relationship( back_populates="user", cascade_delete=True, diff --git a/policyengine_api/data/v2/report_runs.py b/policyengine_api/data/v2/report_runs.py index f1980aadd..61c14fc07 100644 --- a/policyengine_api/data/v2/report_runs.py +++ b/policyengine_api/data/v2/report_runs.py @@ -20,7 +20,6 @@ Report, ReportRun, ReportRunStatus, - ReportRunTrigger, ) from policyengine_api.data.v2.models.base import utc_now @@ -113,7 +112,6 @@ def create_report_run( *, report_id: UUID, inputs: ReportRunInputs, - trigger: ReportRunTrigger, idempotency_key: UUID | None = None, ) -> ReportRun: """Create one run, or return the run for a retried idempotent request. @@ -122,9 +120,6 @@ def create_report_run( simulation runs that attempt named. """ _locked_report(session, report_id) - if trigger is ReportRunTrigger.MANUAL and idempotency_key is None: - msg = "manual report reruns require an idempotency key" - raise ValueError(msg) versions = inputs.versions if not versions.country_package_version or not versions.policyengine_version: msg = "report run package versions must be non-empty" @@ -136,7 +131,6 @@ def create_report_run( reform_simulation_run_id=inputs.simulation_runs.reform_simulation_run_id, country_package_version=versions.country_package_version, policyengine_version=versions.policyengine_version, - trigger=trigger, idempotency_key=idempotency_key, ) if idempotency_key is None: diff --git a/policyengine_api/fastapi_routes/v2/reports/response_models.py b/policyengine_api/fastapi_routes/v2/reports/response_models.py index 600213ccf..4b1ea4958 100644 --- a/policyengine_api/fastapi_routes/v2/reports/response_models.py +++ b/policyengine_api/fastapi_routes/v2/reports/response_models.py @@ -56,15 +56,11 @@ class ReportRunItem(StrictReportAPIModel): """One calculation of a report, on one country package and .py version. ``status`` is ``pending``, ``running``, ``succeeded`` or ``failed``, and - ``error_message`` says why a failed run failed. ``trigger`` is - ``initial`` for the run the report's creation started, and null for a run - the simulation service shares between requests, which records no one - request's trigger. + ``error_message`` says why a failed run failed. """ id: UUID status: Literal["pending", "running", "succeeded", "failed"] - trigger: Literal["initial", "manual", "system"] | None country_package_version: str policyengine_version: str baseline_simulation_run_id: UUID @@ -299,9 +295,7 @@ class BudgetWindowRunItem(StrictReportAPIModel): ``status`` is ``pending`` until a year starts running, ``running`` while its years run, ``failed`` once a year has failed, with ``error_message`` - saying which, and ``succeeded`` once every year has. ``trigger`` is - ``initial`` for the run the report's creation started (null for a shared - run, which records no one request's trigger), and ``recalculate`` says + saying which, and ``succeeded`` once every year has. ``recalculate`` says whether its years calculate their simulations again rather than reusing successful runs. ``country_package_version`` and ``policyengine_version`` are the versions @@ -310,7 +304,6 @@ class BudgetWindowRunItem(StrictReportAPIModel): id: UUID status: Literal["pending", "running", "succeeded", "failed"] - trigger: Literal["initial", "manual", "system"] | None recalculate: bool country_package_version: str | None policyengine_version: str | None diff --git a/policyengine_api/services/v2/stored_reports/transformations.py b/policyengine_api/services/v2/stored_reports/transformations.py index 36c9f0033..e67e76911 100644 --- a/policyengine_api/services/v2/stored_reports/transformations.py +++ b/policyengine_api/services/v2/stored_reports/transformations.py @@ -68,11 +68,10 @@ def report_kind(report: Report) -> ReportKind: def report_run_read(run: ReportRun) -> ReportRunRead: - """Return the run's status, trigger, versions and times.""" + """Return the run's status, versions and times.""" return ReportRunRead( id=run.id, status=run.status.value, - trigger=None if run.trigger is None else run.trigger.value, country_package_version=run.country_package_version, policyengine_version=run.policyengine_version, baseline_simulation_run_id=run.baseline_simulation_run_id, @@ -236,7 +235,7 @@ def household_report_read( def budget_window_run_read( window_run: BudgetWindowRun, years: Sequence[ReportRun] ) -> BudgetWindowRunRead: - """Return the window run's status, trigger and times, with its years. + """Return the window run's status and times, with its years. Its versions are its first year's, since the simulation service creates every year on one version; a run with no year has none. @@ -250,7 +249,6 @@ def budget_window_run_read( return BudgetWindowRunRead( id=window_run.id, status=window_run.status.value, - trigger=None if window_run.trigger is None else window_run.trigger.value, recalculate=window_run.recalculate, country_package_version=( None if first_year is None else first_year.country_package_version diff --git a/policyengine_api/services/v2/stored_reports/types.py b/policyengine_api/services/v2/stored_reports/types.py index 96f8f6cea..30765f965 100644 --- a/policyengine_api/services/v2/stored_reports/types.py +++ b/policyengine_api/services/v2/stored_reports/types.py @@ -52,7 +52,6 @@ # The kinds whose runs are their report runs; a budget window's are its # budget-window runs. ReportRunsKind = Literal["economy", "household"] -ReportRunTriggerName = Literal["initial", "manual", "system"] # How far a side's cliff calculation has got: not_calculated while its # simulation run has none, else its calculation's status (failed once its # newest calculation failed and none is pending, running or succeeded). @@ -65,15 +64,12 @@ class ReportRunRead: """One calculation of a report, and the package versions it ran on. - ``trigger`` is ``initial`` for the run the report's creation started, and - null for a run the simulation service shares between requests. A user's - trigger is on their record of the run. ``error_message`` says why a failed - run failed. + ``error_message`` says why a failed run failed. A run is shared between + requests, so each user's trigger is on their record of the run. """ id: UUID status: Literal["pending", "running", "succeeded", "failed"] - trigger: ReportRunTriggerName | None country_package_version: str policyengine_version: str baseline_simulation_run_id: UUID @@ -237,7 +233,6 @@ class BudgetWindowRunRead: id: UUID status: Literal["pending", "running", "succeeded", "failed"] - trigger: ReportRunTriggerName | None recalculate: bool country_package_version: str | None policyengine_version: str | None diff --git a/tests/integration/stored_report_rows.py b/tests/integration/stored_report_rows.py index 59756f1c2..e952d612a 100644 --- a/tests/integration/stored_report_rows.py +++ b/tests/integration/stored_report_rows.py @@ -299,12 +299,12 @@ def store_report( # noqa: PLR0913 # the connection and catalog, then each colu ) connection.execute( text( - "INSERT INTO reports (id, label, description, country, type, " + "INSERT INTO reports (id, country, type, " "tax_benefit_model_id, policy_id, baseline_simulation_id, " "reform_simulation_id, household_id, dataset_id, region_id, year, " "budget_window_size, inputs, key_version, definition_key, " - "dataset_independent_key, created_at) VALUES (:id, :label, " - ":description, :country, :type, :model_id, :policy_id, :baseline, " + "dataset_independent_key, created_at) VALUES (:id, " + ":country, :type, :model_id, :policy_id, :baseline, " ":reform, :household_id, :dataset_id, :region_id, :year, " ":budget_window_size, CAST(:inputs AS JSON), :key_version, " ":definition_key, :dataset_independent_key, " @@ -312,8 +312,6 @@ def store_report( # noqa: PLR0913 # the connection and catalog, then each colu ), { "id": report.report_id, - "label": f"{catalog.country} reform", - "description": "A stored report", "country": catalog.country, "type": "economy", "model_id": catalog.model_id, @@ -387,13 +385,13 @@ def store_report_run( "INSERT INTO report_runs (id, created_at, report_id, " "baseline_simulation_run_id, reform_simulation_run_id, " "budget_window_run_id, year, " - "country_package_version, policyengine_version, status, trigger, " + "country_package_version, policyengine_version, status, " "started_at, completed_at, error_message, spm_provenance, forced" + "".join(f", {column}" for column in result_columns) + ") VALUES (:id, :created_at, :report_id, :baseline_run, " ":reform_run, :budget_window_run_id, :year, " ":country_package_version, :policyengine_version, " - ":status, 'initial', :started_at, :completed_at, :error_message, " + ":status, :started_at, :completed_at, :error_message, " ":spm_provenance, :forced" + "".join(f", :{column}" for column in result_columns) + ")" @@ -485,7 +483,6 @@ def store_budget_window_run( created_at=created_at, report_id=report.report_id, tax_benefit_model_version_id=catalog.version_id, - trigger="initial", recalculate=False, status=settings.status, error_message=settings.error_message, diff --git a/tests/integration/test_v2_budget_window_report_creation.py b/tests/integration/test_v2_budget_window_report_creation.py index 25f8d1909..73873b46b 100644 --- a/tests/integration/test_v2_budget_window_report_creation.py +++ b/tests/integration/test_v2_budget_window_report_creation.py @@ -215,7 +215,7 @@ def test_a_created_budget_window_is_answered_as_the_tables_hold_it( assert (item["id"], item["kind"]) == (str(catalog.report_ids[0]), "budget_window") assert (item["year"], item["budget_window_size"]) == (2026, 3) assert item["status"] == "pending" - assert item["run"]["trigger"] == "initial" + assert "trigger" not in item["run"] assert item["run"]["policyengine_version"] == "4.2.0" assert [(year["year"], year["status"]) for year in item["run"]["years"]] == [ (2026, "pending"), diff --git a/tests/integration/test_v2_budget_window_report_reads.py b/tests/integration/test_v2_budget_window_report_reads.py index 71548db1e..5cf0b2d87 100644 --- a/tests/integration/test_v2_budget_window_report_reads.py +++ b/tests/integration/test_v2_budget_window_report_reads.py @@ -220,11 +220,8 @@ def test_a_budget_window_reads_its_newest_run_years_and_result( assert item["status"] == "succeeded" run = item["run"] assert run["id"] == str(window_run_id) - assert (run["status"], run["trigger"], run["recalculate"]) == ( - "succeeded", - "initial", - False, - ) + assert (run["status"], run["recalculate"]) == ("succeeded", False) + assert "trigger" not in run assert (run["country_package_version"], run["policyengine_version"]) == ( "1.4.0", "4.1.0", diff --git a/tests/integration/test_v2_catalog_publication.py b/tests/integration/test_v2_catalog_publication.py index c46fa0f6e..e6c0e1814 100644 --- a/tests/integration/test_v2_catalog_publication.py +++ b/tests/integration/test_v2_catalog_publication.py @@ -505,7 +505,6 @@ def test_republish_preserves_existing_run_and_dataset_version_records( region_id=region.id, ) report = Report( - label="Existing report", country="us", tax_benefit_model_id=model.id, dataset_id=input_dataset.id, diff --git a/tests/integration/test_v2_household_reports.py b/tests/integration/test_v2_household_reports.py index a64d971e8..d5bd68ab6 100644 --- a/tests/integration/test_v2_household_reports.py +++ b/tests/integration/test_v2_household_reports.py @@ -259,7 +259,6 @@ def store_household_report( connection, "reports", id=report_id, - label="My household under the reform", country=catalog.country, tax_benefit_model_id=catalog.model_id, policy_id=catalog.policy_id if with_reform else None, @@ -279,7 +278,6 @@ def store_household_report( baseline_simulation_run_id=baseline_run_id, reform_simulation_run_id=reform_run_id, status=status, - trigger="initial", started_at=STARTED_AT, completed_at=( STARTED_AT + timedelta(minutes=1) diff --git a/tests/integration/test_v2_report_creation.py b/tests/integration/test_v2_report_creation.py index 79eefbcac..a37c3838d 100644 --- a/tests/integration/test_v2_report_creation.py +++ b/tests/integration/test_v2_report_creation.py @@ -253,7 +253,7 @@ def test_a_created_report_is_answered_as_the_tables_hold_it( item = response.json()["result"]["item"] assert item["id"] == str(catalog.report_ids[0]) assert item["status"] == "pending" - assert item["run"]["trigger"] == "initial" + assert "trigger" not in item["run"] assert item["run"]["policyengine_version"] == "4.2.0" assert item["result"] is None assert item["current_versions"] == CURRENT_VERSIONS diff --git a/tests/integration/test_v2_report_runs.py b/tests/integration/test_v2_report_runs.py index 3696cdd46..9b263ac77 100644 --- a/tests/integration/test_v2_report_runs.py +++ b/tests/integration/test_v2_report_runs.py @@ -103,23 +103,12 @@ def current_versions(country_id: str) -> PackageVersions: return PackageVersions(**CURRENT_VERSIONS) -def mark_manual(connection: Connection, run_id: UUID) -> None: - """Record the report run as a re-run, as the simulation service creates one.""" - connection.execute( - text("UPDATE report_runs SET trigger = 'manual' WHERE id = :id"), - {"id": run_id}, - ) - - -def mark_window_manual( +def set_window_recalculate( connection: Connection, window_run_id: UUID, *, recalculate: bool ) -> None: - """Record the window run as a re-run, as the simulation service creates one.""" + """Record whether the window run is forced, as the simulation service does.""" connection.execute( - text( - "UPDATE budget_window_runs SET trigger = 'manual', recalculate = " - ":recalculate WHERE id = :id" - ), + text("UPDATE budget_window_runs SET recalculate = :recalculate WHERE id = :id"), {"id": window_run_id, "recalculate": recalculate}, ) @@ -201,7 +190,9 @@ def rerun_report(self, rerun: ReportRerunInput) -> StartedReportRun: window_run_id = store_window_run( connection, self.catalog, report, settings, self.window_years ) - mark_window_manual(connection, window_run_id, recalculate=rerun.force) + set_window_recalculate( + connection, window_run_id, recalculate=rerun.force + ) self.new_run_ids.append(window_run_id) started = StartedReportRun( report_run_id=connection.execute( @@ -215,7 +206,6 @@ def rerun_report(self, rerun: ReportRerunInput) -> StartedReportRun: ) else: run_ids = store_report_run(connection, self.catalog, report, settings) - mark_manual(connection, run_ids.report_run_id) if rerun.include_cliffs: for simulation_run_id in ( run_ids.baseline_simulation_run_id, @@ -357,7 +347,6 @@ def test_a_reports_runs_are_listed_newest_first_with_their_versions( error_message="The reform simulation failed", ), ) - mark_manual(connection, failed.report_run_id) newest = store_report_run( connection, catalog, @@ -368,7 +357,6 @@ def test_a_reports_runs_are_listed_newest_first_with_their_versions( created_at=STARTED_AT + timedelta(days=2), ), ) - mark_manual(connection, newest.report_run_id) other_report = store_report(connection, catalog) store_report_run( connection, @@ -393,11 +381,8 @@ def test_a_reports_runs_are_listed_newest_first_with_their_versions( str(failed.report_run_id), str(first.report_run_id), ] - assert [(item["status"], item["trigger"]) for item in items] == [ - ("running", "manual"), - ("failed", "manual"), - ("succeeded", "initial"), - ] + assert [item["status"] for item in items] == ["running", "failed", "succeeded"] + assert all("trigger" not in item for item in items) assert [item["policyengine_version"] for item in items] == [ "4.2.0", "4.2.0", @@ -492,7 +477,6 @@ def test_a_household_reports_runs_are_listed_newest_first_with_their_versions( created_at=STARTED_AT + timedelta(days=1), ), ) - mark_manual(connection, newest.report_run_id) client = client_for( runtime_sessions, RerunningSimulationService(owner_engine, catalog) ) @@ -509,10 +493,8 @@ def test_a_household_reports_runs_are_listed_newest_first_with_their_versions( str(newest.report_run_id), str(first.report_run_id), ] - assert [(item["status"], item["trigger"]) for item in items] == [ - ("pending", "manual"), - ("succeeded", "initial"), - ] + assert [item["status"] for item in items] == ["pending", "succeeded"] + assert all("trigger" not in item for item in items) assert [item["policyengine_version"] for item in items] == ["4.2.0", "4.1.0"] @@ -551,7 +533,7 @@ def test_a_household_report_runs_again_where_households_are_calculated( assert item["kind"] == "household" assert item["status"] == "pending" assert item["run"]["id"] == str(new_run_id) - assert item["run"]["trigger"] == "manual" + assert "trigger" not in item["run"] assert item["run"]["policyengine_version"] == "4.2.0" assert item["result"] is None runs = client.get(f"/v2/reports/{report.report_id}/runs?country_id=uk").json() @@ -599,7 +581,7 @@ def test_running_a_report_again_answers_it_with_its_new_run( assert item["id"] == str(report.report_id) assert item["status"] == "pending" assert item["run"]["id"] == str(new_run_id) - assert item["run"]["trigger"] == "manual" + assert "trigger" not in item["run"] assert item["run"]["policyengine_version"] == "4.2.0" assert item["current_versions"] == CURRENT_VERSIONS assert item["result"] is None @@ -799,7 +781,7 @@ def test_a_budget_windows_runs_are_listed_newest_first_with_their_years( ), (2026, 2027), ) - mark_window_manual(connection, newest_run_id, recalculate=True) + set_window_recalculate(connection, newest_run_id, recalculate=True) client = client_for( runtime_sessions, RerunningSimulationService(owner_engine, catalog) ) @@ -813,9 +795,11 @@ def test_a_budget_windows_runs_are_listed_newest_first_with_their_years( assert result["current_versions"] == CURRENT_VERSIONS items = result["items"] assert [item["id"] for item in items] == [str(newest_run_id), str(first_run_id)] - assert [ - (item["status"], item["trigger"], item["recalculate"]) for item in items - ] == [("running", "manual", True), ("succeeded", "initial", False)] + assert [(item["status"], item["recalculate"]) for item in items] == [ + ("running", True), + ("succeeded", False), + ] + assert all("trigger" not in item for item in items) assert [item["policyengine_version"] for item in items] == ["4.2.0", "4.1.0"] assert [[year["year"] for year in item["years"]] for item in items] == [ [2026, 2027], @@ -867,7 +851,7 @@ def test_running_a_budget_window_again_answers_it_with_its_new_run( assert item["kind"] == "budget_window" assert item["status"] == "pending" assert item["run"]["id"] == str(new_run_id) - assert item["run"]["trigger"] == "manual" + assert "trigger" not in item["run"] assert item["run"]["recalculate"] is force assert item["run"]["policyengine_version"] == "4.2.0" assert item["result"] is None diff --git a/tests/unit/v2/budget_window_report_support.py b/tests/unit/v2/budget_window_report_support.py index b364168c4..4817290b1 100644 --- a/tests/unit/v2/budget_window_report_support.py +++ b/tests/unit/v2/budget_window_report_support.py @@ -62,7 +62,6 @@ def budget_window_year_read(year: int, **changes: Any) -> BudgetWindowYearRead: "year": year, "id": YEAR_RUN_IDS[year], "status": "succeeded", - "trigger": "initial", "country_package_version": "1.4.0", "policyengine_version": "4.1.0", "baseline_simulation_run_id": UUID(int=year * 2), @@ -79,7 +78,6 @@ def budget_window_run_read(**changes: Any) -> BudgetWindowRunRead: values: dict[str, Any] = { "id": BUDGET_WINDOW_RUN_ID, "status": "succeeded", - "trigger": "initial", "recalculate": False, "country_package_version": "1.4.0", "policyengine_version": "4.1.0", diff --git a/tests/unit/v2/report_run_support.py b/tests/unit/v2/report_run_support.py index 83c4317e0..efe1beee3 100644 --- a/tests/unit/v2/report_run_support.py +++ b/tests/unit/v2/report_run_support.py @@ -44,7 +44,6 @@ def report_runs_read() -> ReportRunsRead: report_run_read( id=NEW_RUN_ID, status="pending", - trigger="manual", country_package_version=CURRENT_VERSIONS.country_package_version, policyengine_version=CURRENT_VERSIONS.policyengine_version, created_at=CREATED_AT + timedelta(days=3), @@ -75,7 +74,6 @@ def budget_window_runs_read() -> BudgetWindowRunsRead: budget_window_run_read( id=NEW_WINDOW_RUN_ID, status="pending", - trigger="manual", recalculate=True, created_at=rerun_at, started_at=None, @@ -87,7 +85,6 @@ def budget_window_runs_read() -> BudgetWindowRunsRead: baseline_simulation_run_id=UUID(int=year * 2 + 10_000), reform_simulation_run_id=UUID(int=year * 2 + 10_001), status="pending", - trigger="manual", created_at=rerun_at, started_at=None, completed_at=None, diff --git a/tests/unit/v2/sqlite_report_rows.py b/tests/unit/v2/sqlite_report_rows.py index 6a3efa5e8..c96aac0d3 100644 --- a/tests/unit/v2/sqlite_report_rows.py +++ b/tests/unit/v2/sqlite_report_rows.py @@ -111,7 +111,6 @@ def store_economy_report( ) stored = Report( id=uuid4() if report.report_id is None else report.report_id, - label="A stored report", country=place.country, tax_benefit_model_id=place.model_id, baseline_simulation_id=simulations[0].id, diff --git a/tests/unit/v2/stored_report_support.py b/tests/unit/v2/stored_report_support.py index bb3c89757..8f191bfdd 100644 --- a/tests/unit/v2/stored_report_support.py +++ b/tests/unit/v2/stored_report_support.py @@ -65,7 +65,6 @@ def report_run_read(**changes: Any) -> ReportRunRead: values: dict[str, Any] = { "id": REPORT_RUN_ID, "status": "succeeded", - "trigger": "initial", "country_package_version": "1.4.0", "policyengine_version": "4.1.0", "baseline_simulation_run_id": BASELINE_RUN_ID, diff --git a/tests/unit/v2/test_budget_window_report_read_routes.py b/tests/unit/v2/test_budget_window_report_read_routes.py index e27cc49ba..6d237ffa3 100644 --- a/tests/unit/v2/test_budget_window_report_read_routes.py +++ b/tests/unit/v2/test_budget_window_report_read_routes.py @@ -15,7 +15,6 @@ budget_window_report_read, budget_window_run_read, budget_window_summary_read, - budget_window_year_read, ) from tests.unit.v2.stored_report_support import ( REPORT_ID, @@ -42,11 +41,8 @@ def test_a_budget_window_answers_its_window_newest_run_years_and_result() -> Non assert (item["year"], item["budget_window_size"]) == (2026, 2) run = item["run"] assert run["id"] == str(BUDGET_WINDOW_RUN_ID) - assert (run["status"], run["trigger"], run["recalculate"]) == ( - "succeeded", - "initial", - False, - ) + assert (run["status"], run["recalculate"]) == ("succeeded", False) + assert "trigger" not in run assert (run["country_package_version"], run["policyengine_version"]) == ( "1.4.0", "4.1.0", @@ -58,7 +54,6 @@ def test_a_budget_window_answers_its_window_newest_run_years_and_result() -> Non "year", "id", "status", - "trigger", "country_package_version", "policyengine_version", "baseline_simulation_run_id", @@ -81,20 +76,14 @@ def test_a_budget_window_answers_its_window_newest_run_years_and_result() -> Non ] -def test_a_window_run_and_its_years_without_a_trigger_answer_it_as_null() -> None: - run = budget_window_run_read( - trigger=None, - years=tuple( - budget_window_year_read(year, trigger=None) for year in YEAR_RUN_IDS - ), - ) - service = FakeReportService(report=budget_window_report_read(run=run, label=None)) +def test_a_window_run_and_its_years_answer_no_trigger() -> None: + service = FakeReportService(report=budget_window_report_read(label=None)) item = read_item(service) assert item["label"] is None - assert item["run"]["trigger"] is None - assert {year["trigger"] for year in item["run"]["years"]} == {None} + assert "trigger" not in item["run"] + assert all("trigger" not in year for year in item["run"]["years"]) def test_a_budget_window_without_a_run_answers_no_run_and_no_result() -> None: diff --git a/tests/unit/v2/test_household_report_reads.py b/tests/unit/v2/test_household_report_reads.py index 14ec4b7ef..b9903c377 100644 --- a/tests/unit/v2/test_household_report_reads.py +++ b/tests/unit/v2/test_household_report_reads.py @@ -25,7 +25,6 @@ Report, ReportRun, ReportRunStatus, - ReportRunTrigger, Simulation, SimulationRun, SimulationStatus, @@ -225,8 +224,6 @@ def simulation(policy_id: UUID | None) -> Simulation: session.add_all([side for side in (baseline, reform) if side is not None]) session.flush() report = Report( - label="My household under the reform", - description="Doubles the credit", country=country, tax_benefit_model_id=model.id, baseline_simulation_id=baseline.id, @@ -275,7 +272,6 @@ def simulation_run( baseline_simulation_run_id=baseline_run.id, reform_simulation_run_id=None if reform_run is None else reform_run.id, status=run.status, - trigger=ReportRunTrigger.INITIAL, created_at=STORED_AT, started_at=STORED_AT, completed_at=completed_at, @@ -535,7 +531,6 @@ def store_economy_report(session: Session) -> UUID: session.add(model) session.flush() report = Report( - label="Economy", country="us", tax_benefit_model_id=model.id, year=2026, diff --git a/tests/unit/v2/test_report_column_mapping.py b/tests/unit/v2/test_report_column_mapping.py new file mode 100644 index 000000000..e622276e1 --- /dev/null +++ b/tests/unit/v2/test_report_column_mapping.py @@ -0,0 +1,68 @@ +"""API v1's models leave out the report columns a later revision drops. + +The deploy upgrades the database before new code takes traffic, so the release +serving during that upgrade must not name a column it removes. The tables keep +the columns until then: only the models stop selecting and writing them. +""" + +import pytest +import sqlalchemy as sa +from sqlmodel import select + +from policyengine_api.data.v2.models import ( + V2_METADATA, + BudgetWindowRun, + Report, + ReportRun, + User, +) + +UNMAPPED_COLUMNS = [ + (Report, "label"), + (Report, "description"), + (Report, "user_id"), + (ReportRun, "trigger"), + (BudgetWindowRun, "trigger"), +] + + +@pytest.mark.parametrize(("model", "column_name"), UNMAPPED_COLUMNS) +def test_a_whole_row_read_names_no_unmapped_column( + model: type, column_name: str +) -> None: + selected_names = {column.name for column in select(model).selected_columns} + + assert column_name not in selected_names + + +@pytest.mark.parametrize(("model", "column_name"), UNMAPPED_COLUMNS) +def test_a_model_has_no_attribute_for_an_unmapped_column( + model: type, column_name: str +) -> None: + assert column_name not in sa.inspect(model).column_attrs + assert column_name not in model.model_fields + assert not hasattr(model, column_name) + + +@pytest.mark.parametrize(("model", "column_name"), UNMAPPED_COLUMNS) +def test_the_table_keeps_an_unmapped_column_until_a_revision_drops_it( + model: type, column_name: str +) -> None: + column = V2_METADATA.tables[model.__tablename__].c[column_name] + + assert column.nullable + assert column.server_default is None + + +def test_the_reports_user_key_and_index_stay_with_the_column() -> None: + reports = V2_METADATA.tables["reports"] + + (user_key,) = reports.c.user_id.foreign_keys + assert user_key.target_fullname == "users.id" + assert user_key.ondelete == "SET NULL" + assert "ix_reports_user_id" in {index.name for index in reports.indexes} + + +def test_reports_and_users_have_no_relationship() -> None: + assert "user" not in Report.__sqlmodel_relationships__ + assert "reports" not in User.__sqlmodel_relationships__ diff --git a/tests/unit/v2/test_report_dataset_independent_keys.py b/tests/unit/v2/test_report_dataset_independent_keys.py index 9cac9c32b..5f49158b7 100644 --- a/tests/unit/v2/test_report_dataset_independent_keys.py +++ b/tests/unit/v2/test_report_dataset_independent_keys.py @@ -121,7 +121,6 @@ def store_household_report( session.add_all(simulations) session.flush() report = Report( - label="My household", country=catalog.country, tax_benefit_model_id=catalog.model_id, baseline_simulation_id=simulations[0].id, diff --git a/tests/unit/v2/test_report_read_routes.py b/tests/unit/v2/test_report_read_routes.py index 7513986be..7270c4acc 100644 --- a/tests/unit/v2/test_report_read_routes.py +++ b/tests/unit/v2/test_report_read_routes.py @@ -73,12 +73,10 @@ def test_a_reports_spm_receipts_are_answered_with_package_names_as_keys() -> Non } -def test_a_report_without_a_label_or_run_trigger_answers_them_as_null() -> None: +def test_a_report_without_a_label_answers_null_and_its_run_no_trigger() -> None: """A shared report holds no caller's label, and its runs no request's trigger.""" service = FakeReportService() - service.report = replace( - service.report, label=None, run=report_run_read(trigger=None) - ) + service.report = replace(service.report, label=None, run=report_run_read()) client = report_route_client(reports=service) response = client.get(f"/v2/reports/{REPORT_ID}?country_id=us") @@ -86,7 +84,7 @@ def test_a_report_without_a_label_or_run_trigger_answers_them_as_null() -> None: assert response.status_code == 200, response.text item = response.json()["result"]["item"] assert item["label"] is None - assert item["run"]["trigger"] is None + assert "trigger" not in item["run"] def test_a_report_without_a_run_answers_with_no_run_and_no_result() -> None: diff --git a/tests/unit/v2/test_report_rerun_service.py b/tests/unit/v2/test_report_rerun_service.py index bd948a040..34a58fd5d 100644 --- a/tests/unit/v2/test_report_rerun_service.py +++ b/tests/unit/v2/test_report_rerun_service.py @@ -14,7 +14,6 @@ BudgetWindowRun, Report, ReportRunStatus, - ReportRunTrigger, ) from policyengine_api.services.v2.simulation_service.types import ( PackageVersions, @@ -74,7 +73,6 @@ def rerun_report(self, rerun: ReportRerunInput) -> StartedReportRun: def stored_rows(**report_columns: object) -> StoredReportRows: report = Report( id=REPORT_ID, - label="Reform", country="us", tax_benefit_model_id=uuid4(), year=2026, @@ -147,7 +145,6 @@ def test_a_window_run_with_no_year_lists_no_years_and_no_versions() -> None: updated_at=STORED_AT, report_id=REPORT_ID, tax_benefit_model_version_id=uuid4(), - trigger=ReportRunTrigger.MANUAL, recalculate=True, status=ReportRunStatus.PENDING, ) @@ -162,7 +159,7 @@ def test_a_window_run_with_no_year_lists_no_years_and_no_versions() -> None: assert runs.kind == "budget_window" (run,) = runs.runs assert run.id == window_run.id - assert (run.trigger, run.recalculate, run.status) == ("manual", True, "pending") + assert (run.recalculate, run.status) == (True, "pending") assert run.years == () assert run.country_package_version is None assert run.policyengine_version is None diff --git a/tests/unit/v2/test_report_run_creation.py b/tests/unit/v2/test_report_run_creation.py index ac0959f9b..21a03297d 100644 --- a/tests/unit/v2/test_report_run_creation.py +++ b/tests/unit/v2/test_report_run_creation.py @@ -15,7 +15,6 @@ Report, ReportRun, ReportRunStatus, - ReportRunTrigger, TaxBenefitModel, ) from policyengine_api.data.v2.report_runs import ( @@ -50,7 +49,6 @@ def _create_report(session: Session) -> Report: session.add(model) session.flush() report = Report( - label="Distributional report", country="us", tax_benefit_model_id=model.id, inputs={}, @@ -75,31 +73,9 @@ def test_unknown_report_is_rejected_before_any_validation(engine) -> None: ), simulation_runs=SIMULATION_RUNS, ), - trigger=ReportRunTrigger.MANUAL, ) -def test_manual_run_without_a_key_is_rejected_before_version_checks(engine) -> None: - with Session(engine) as session: - report = _create_report(session) - - with pytest.raises( - ValueError, match=r"^manual report reruns require an idempotency key$" - ): - create_report_run( - session, - report_id=report.id, - inputs=ReportRunInputs( - versions=ReportRunPackageVersions( - country_package_version="", - policyengine_version="", - ), - simulation_runs=SIMULATION_RUNS, - ), - trigger=ReportRunTrigger.MANUAL, - ) - - @pytest.mark.parametrize( ("country_package_version", "policyengine_version"), [("", "4.5.6"), ("1.2.3", "")], @@ -125,7 +101,6 @@ def test_empty_package_versions_are_rejected( ), simulation_runs=SIMULATION_RUNS, ), - trigger=ReportRunTrigger.SYSTEM, ) @@ -143,7 +118,6 @@ def test_run_without_a_key_is_inserted_without_a_savepoint(engine) -> None: ), simulation_runs=SIMULATION_RUNS, ), - trigger=ReportRunTrigger.INITIAL, ) stored = session.exec(select(ReportRun)).one() @@ -151,7 +125,6 @@ def test_run_without_a_key_is_inserted_without_a_savepoint(engine) -> None: assert run.report_id == report.id assert run.country_package_version == "1.2.3" assert run.policyengine_version == "4.5.6" - assert run.trigger is ReportRunTrigger.INITIAL assert run.idempotency_key is None assert run.status is ReportRunStatus.PENDING assert run.baseline_simulation_run_id == UUID(int=101) @@ -184,7 +157,6 @@ def failing_savepoint(): ), simulation_runs=SIMULATION_RUNS, ), - trigger=ReportRunTrigger.MANUAL, idempotency_key=uuid4(), ) diff --git a/tests/unit/v2/test_report_run_routes.py b/tests/unit/v2/test_report_run_routes.py index 9ce831810..0760fa244 100644 --- a/tests/unit/v2/test_report_run_routes.py +++ b/tests/unit/v2/test_report_run_routes.py @@ -73,7 +73,7 @@ def test_a_reports_runs_are_answered_newest_first_with_versions_and_current_ones str(NEW_RUN_ID), str(REPORT_RUN_ID), ] - assert [run["trigger"] for run in result["items"]] == ["manual", "initial"] + assert all("trigger" not in run for run in result["items"]) assert [ (run["country_package_version"], run["policyengine_version"]) for run in result["items"] @@ -134,7 +134,8 @@ def test_a_budget_windows_runs_are_answered_with_their_years() -> None: assert result["kind"] == "budget_window" newest, first = result["items"] assert newest["id"] == str(NEW_WINDOW_RUN_ID) - assert (newest["trigger"], newest["recalculate"]) == ("manual", True) + assert newest["recalculate"] is True + assert "trigger" not in newest assert newest["policyengine_version"] == "4.2.0" assert [year["year"] for year in first["years"]] == [2026, 2027] assert first["years"][0]["country_package_version"] == "1.4.0" @@ -225,7 +226,7 @@ def test_running_a_report_again_answers_the_report_with_its_new_run( service.report = replace( service.report, status="pending", - run=report_run_read(id=NEW_RUN_ID, status="pending", trigger="manual"), + run=report_run_read(id=NEW_RUN_ID, status="pending"), result=None, ) @@ -236,7 +237,7 @@ def test_running_a_report_again_answers_the_report_with_its_new_run( assert item["id"] == str(REPORT_ID) assert item["status"] == "pending" assert item["run"]["id"] == str(NEW_RUN_ID) - assert item["run"]["trigger"] == "manual" + assert "trigger" not in item["run"] assert item["result"] is None assert item["current_versions"]["policyengine_version"] == "4.2.0" assert service.calls == [ diff --git a/tests/unit/v2/test_report_runs.py b/tests/unit/v2/test_report_runs.py index 44558041b..c942dd28a 100644 --- a/tests/unit/v2/test_report_runs.py +++ b/tests/unit/v2/test_report_runs.py @@ -20,7 +20,6 @@ Report, ReportRun, ReportRunStatus, - ReportRunTrigger, Simulation, SimulationType, TaxBenefitModel, @@ -67,7 +66,6 @@ def _create_report(session: Session, *, report_type: str | None = None) -> Repor session.add(model) session.flush() report = Report( - label="Distributional report", country="us", type=report_type, tax_benefit_model_id=model.id, @@ -96,7 +94,6 @@ def _create_run( ), simulation_runs=SIMULATION_RUNS, ), - trigger=ReportRunTrigger.MANUAL, idempotency_key=key if key is not None else uuid4(), ) @@ -113,7 +110,6 @@ def _create_forced_run(session: Session, report: Report) -> ReportRun: reform_simulation_run_id=SIMULATION_RUNS.reform_simulation_run_id, country_package_version="1.2.3", policyengine_version=DEPLOYED_POLICYENGINE_VERSION, - trigger=ReportRunTrigger.MANUAL, idempotency_key=uuid4(), forced=True, ) @@ -159,7 +155,7 @@ def test_report_type_can_change_before_first_run_but_not_after(engine) -> None: ) -def test_new_manual_keys_create_a_same_version_run_once_the_last_one_failed( +def test_new_keys_create_a_same_version_run_once_the_last_one_failed( engine, ) -> None: with Session(engine) as session: @@ -179,25 +175,6 @@ def test_new_manual_keys_create_a_same_version_run_once_the_last_one_failed( assert len(session.exec(select(ReportRun)).all()) == 2 -def test_manual_rerun_requires_an_idempotency_key(engine) -> None: - with Session(engine) as session: - report = _create_report(session) - - with pytest.raises(ValueError, match="require an idempotency key"): - create_report_run( - session, - report_id=report.id, - inputs=ReportRunInputs( - versions=ReportRunPackageVersions( - country_package_version="1.2.3", - policyengine_version=DEPLOYED_POLICYENGINE_VERSION, - ), - simulation_runs=SIMULATION_RUNS, - ), - trigger=ReportRunTrigger.MANUAL, - ) - - def test_idempotency_key_round_trips_as_a_python_uuid(engine) -> None: with Session(engine) as session: report = _create_report(session) @@ -259,7 +236,6 @@ def request_rerun() -> UUID: ), simulation_runs=SIMULATION_RUNS, ), - trigger=ReportRunTrigger.MANUAL, idempotency_key=request_key, ) run_id = run.id diff --git a/tests/unit/v2/test_shared_report_models.py b/tests/unit/v2/test_shared_report_models.py index 3312b02ad..031d6d281 100644 --- a/tests/unit/v2/test_shared_report_models.py +++ b/tests/unit/v2/test_shared_report_models.py @@ -413,8 +413,8 @@ def report_run(*, window_year: int | None) -> ReportRun: sqlite_session.commit() -def test_stored_runs_without_a_trigger_read_as_null() -> None: - """The run lists pass on a missing trigger as null (the route tests a label).""" +def test_stored_runs_are_read_without_a_trigger() -> None: + """A shared run has no one trigger: each user's is on their record of it.""" window_run_id = uuid4() run = ReportRun( report_id=uuid4(), @@ -422,7 +422,6 @@ def test_stored_runs_without_a_trigger_read_as_null() -> None: country_package_version="1.5.0", policyengine_version="4.2.0", status=ReportRunStatus.PENDING, - trigger=None, ) year_run = ReportRun( report_id=run.report_id, @@ -432,19 +431,17 @@ def test_stored_runs_without_a_trigger_read_as_null() -> None: country_package_version="1.5.0", policyengine_version="4.2.0", status=ReportRunStatus.PENDING, - trigger=None, ) window_run = BudgetWindowRun( id=window_run_id, report_id=run.report_id, tax_benefit_model_version_id=uuid4(), status=ReportRunStatus.PENDING, - trigger=None, ) - assert stored_report_transformations.report_run_read(run).trigger is None + assert not hasattr(stored_report_transformations.report_run_read(run), "trigger") window_read = stored_report_transformations.budget_window_run_read( window_run, [year_run] ) - assert window_read.trigger is None - assert [year.trigger for year in window_read.years] == [None] + assert not hasattr(window_read, "trigger") + assert not any(hasattr(year, "trigger") for year in window_read.years)