From 7f4d129ed614a3894bb71daad46c815724678516 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:10:09 +0400 Subject: [PATCH 1/4] Test refusing reports and simulations whose policies set missing parameters (failing) A report or simulation whose policy sets parameters the running catalog version lacks should answer 422 with code missing_parameters, naming them, before the simulation service is asked. A catalog that cannot be queried should answer 503, as the database does. Co-Authored-By: Claude Opus 5.5 --- tests/unit/v2/test_report_create_routes.py | 63 +++- ...t_simulation_submission_database_routes.py | 330 ++++++++++++++++++ tests/unit/v2/test_simulation_submissions.py | 141 +++++++- 3 files changed, 528 insertions(+), 6 deletions(-) create mode 100644 tests/unit/v2/test_simulation_submission_database_routes.py diff --git a/tests/unit/v2/test_report_create_routes.py b/tests/unit/v2/test_report_create_routes.py index 3c980d99f..e065dc1ea 100644 --- a/tests/unit/v2/test_report_create_routes.py +++ b/tests/unit/v2/test_report_create_routes.py @@ -15,7 +15,10 @@ SIMULATION_ERROR_MAPPINGS, ) from policyengine_api.libs.simulation_reports import SimulationServiceReportClient -from policyengine_api.services.v2.policies.validators import PolicyNotFoundError +from policyengine_api.services.v2.policies.validators import ( + PolicyNotFoundError, + PolicyParametersMissingError, +) from policyengine_api.services.v2.simulation_service.types import ( SimulationServiceAnswerUnreadableError, SimulationServiceInternalError, @@ -395,6 +398,64 @@ def test_a_simulation_the_unavailable_service_created_answers_its_id( assert logged_events(caplog) == ["v2_simulation_unavailable_after_creation"] +# The parameters a refused policy sets that the running catalog lacks. +REMOVED_PARAMETERS = ("gov.example.removed_amount", "gov.example.removed_rate") + + +@pytest.mark.usefixtures("enabled_route_support_logger") +@pytest.mark.parametrize( + ("client_for_error", "path", "body"), + [ + pytest.param( + lambda error: report_route_client(reports=FakeReportService(error=error)), + "/v2/reports?country_id=us", + report_body(), + id="report", + ), + pytest.param( + lambda error: report_route_client( + simulations=FakeSimulationService(error=error) + ), + "/v2/simulations?country_id=us", + simulation_body(), + id="simulation", + ), + ], +) +def test_a_policy_setting_parameters_the_running_catalog_lacks_answers_422( + client_for_error, path: str, body: dict[str, Any], caplog +) -> None: + error = PolicyParametersMissingError(POLICY_ID, REMOVED_PARAMETERS) + client = client_for_error(error) + + with caplog.at_level("ERROR", logger=ROUTE_SUPPORT_LOGGER): + response = client.post(path, json=body) + + assert response.status_code == 422 + assert response.json() == { + "status": "error", + "message": ( + f"Policy {POLICY_ID} sets parameters the running catalog version " + "does not have: gov.example.removed_amount, gov.example.removed_rate" + ), + "code": "missing_parameters", + "missing_parameters": list(REMOVED_PARAMETERS), + } + # The caller can act on the refusal, so it is not logged as a failure. + assert logged_events(caplog) == [] + + +@pytest.mark.parametrize("path", ["/v2/reports", "/v2/simulations"]) +def test_the_openapi_document_gives_the_missing_parameters_refusal(path: str) -> None: + document = report_route_client().get("/v2/openapi.json").json() + + answer = document["paths"][path]["post"]["responses"]["422"] + assert {"$ref": "#/components/schemas/V2MissingParametersErrorResponse"} in answer[ + "content" + ]["application/json"]["schema"]["anyOf"] + assert "missing_parameters" in answer["description"] + + @pytest.mark.parametrize( ("path", "status_code", "schema", "id_field"), [ diff --git a/tests/unit/v2/test_simulation_submission_database_routes.py b/tests/unit/v2/test_simulation_submission_database_routes.py new file mode 100644 index 000000000..cb9d33688 --- /dev/null +++ b/tests/unit/v2/test_simulation_submission_database_routes.py @@ -0,0 +1,330 @@ +"""Create v2 reports and simulations through the routes and database-backed checks. + +The countries and policies services read a SQLite database with the v2 +tables, so the policy check runs the real catalog query; only the simulation +service is a stand-in, which a refused submission never reaches. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any +from uuid import UUID, uuid4 + +import pytest +from sqlalchemy import text +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool +from sqlmodel import Session, create_engine + +from policyengine_api.data.v2.models import ( + V2_METADATA, + Country, + Parameter, + ParameterValue, + Policy, + TaxBenefitModel, + TaxBenefitModelVersion, +) +from policyengine_api.services.v2.countries.database_session import ( + CountryDatabaseSession, +) +from policyengine_api.services.v2.countries.services import V2CountryService +from policyengine_api.services.v2.policies.database_session import ( + PolicyDatabaseSession, +) +from policyengine_api.services.v2.policies.services import V2PolicyService +from policyengine_api.services.v2.simulation_service.services import ( + SimulationSubmissionService, +) +from policyengine_api.services.v2.simulation_service.types import ( + EconomyReportCreationInput, + EconomySimulationCreationInput, + SubmittedReport, + SubmittedSimulation, +) +from policyengine_api.services.v2.stored_reports.database_session import ( + StoredReportDatabaseSession, +) +from policyengine_api.services.v2.stored_reports.services import V2StoredReportService +from policyengine_api.services.v2.stored_simulations.database_session import ( + StoredSimulationDatabaseSession, +) +from policyengine_api.services.v2.stored_simulations.services import ( + V2StoredSimulationService, +) +from tests.unit.v2.stored_report_support import CURRENT_VERSIONS, report_route_client + +if TYPE_CHECKING: + from collections.abc import Iterator + + from fastapi.testclient import TestClient + from sqlalchemy import Engine + +RUNNING_VERSION = "6.10.0" +EARLIER_VERSION = "6.9.0" +KEPT_PARAMETER = "gov.example.rate" +REMOVED_PARAMETER = "gov.example.removed_rate" +STORED_AT = datetime(2026, 11, 10, tzinfo=UTC) +REPORTS = "/v2/reports?country_id=us" +SIMULATIONS = "/v2/simulations?country_id=us" + + +@dataclass +class RecordingSimulationService: + """The simulation service's client, recording what it is sent.""" + + reports: list[EconomyReportCreationInput] = field(default_factory=list) + simulations: list[EconomySimulationCreationInput] = field(default_factory=list) + + def submit_report(self, report: EconomyReportCreationInput) -> SubmittedReport: + self.reports.append(report) + return SubmittedReport(report_id=uuid4(), report_run_id=uuid4()) + + def submit_simulation( + self, simulation: EconomySimulationCreationInput + ) -> SubmittedSimulation: + self.simulations.append(simulation) + return SubmittedSimulation(simulation_id=uuid4(), simulation_run_id=uuid4()) + + +@pytest.fixture +def engine() -> Iterator[Engine]: + # One connection shared by every thread, so the route's worker thread + # reads the same in-memory database. + database_engine = create_engine( + "sqlite://", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + V2_METADATA.create_all(database_engine) + yield database_engine + database_engine.dispose() + + +def store_catalog_version( + session: Session, model: TaxBenefitModel, version: str, names: list[str] +) -> dict[str, Parameter]: + """Store a catalog version of ``model`` with these parameters, keyed by name.""" + model_version = TaxBenefitModelVersion( + model=model, version=version, current_law_id=1, metadata_time_periods=[2026] + ) + parameters = { + name: Parameter(name=name, tax_benefit_model_version=model_version) + for name in names + } + session.add(model_version) + session.add_all(parameters.values()) + session.flush() + return parameters + + +def store_policy( + session: Session, parameters: dict[str, Parameter], content_hash_letter: str +) -> Policy: + """Store a US policy setting each of ``parameters``, of their catalog version.""" + model_version = next(iter(parameters.values())).tax_benefit_model_version + policy = Policy( + country_id="us", + tax_benefit_model_id=model_version.model_id, + tax_benefit_model_version_id=model_version.id, + canonicalization_version=1, + content_hash=content_hash_letter * 64, + created_at=STORED_AT, + updated_at=STORED_AT, + ) + session.add(policy) + session.flush() + session.add_all( + ParameterValue( + parameter_id=parameter.id, + value_json=0.2, + start_date=STORED_AT, + policy_id=policy.id, + ) + for parameter in parameters.values() + ) + return policy + + +@pytest.fixture +def stored_policy_ids(engine: Engine) -> dict[str, UUID]: + """Store the US with two catalog versions and three policies. + + ``outdated_policy`` sets a parameter the running catalog version removed; + ``kept_policy`` was made on the earlier version too, but sets only a + parameter the running version still has; ``running_policy`` was made on + the running version. + """ + with Session(engine, expire_on_commit=False) as session: + session.add(Country(id="us", household_enabled=True, society_wide_enabled=True)) + model = TaxBenefitModel(name="policyengine-us") + earlier = store_catalog_version( + session, model, EARLIER_VERSION, [KEPT_PARAMETER, REMOVED_PARAMETER] + ) + running = store_catalog_version( + session, model, RUNNING_VERSION, [KEPT_PARAMETER] + ) + policies = { + "outdated_policy": store_policy(session, earlier, "a"), + "kept_policy": store_policy( + session, {KEPT_PARAMETER: earlier[KEPT_PARAMETER]}, "b" + ), + "running_policy": store_policy(session, running, "c"), + } + session.commit() + return {name: policy.id for name, policy in policies.items()} + + +@pytest.fixture +def simulation_service() -> RecordingSimulationService: + return RecordingSimulationService() + + +@pytest.fixture +def submissions( + engine: Engine, simulation_service: RecordingSimulationService +) -> SimulationSubmissionService: + sessions = sessionmaker(engine, class_=Session, expire_on_commit=False) + return SimulationSubmissionService( + countries=V2CountryService(CountryDatabaseSession(sessions)), + policies=V2PolicyService( + PolicyDatabaseSession(sessions), + running_policyengine_version=RUNNING_VERSION, + ), + client_factory=lambda: simulation_service, + ) + + +@pytest.fixture +def client(engine: Engine, submissions: SimulationSubmissionService) -> TestClient: + sessions = sessionmaker(engine, class_=Session, expire_on_commit=False) + return report_route_client( + reports=V2StoredReportService( + StoredReportDatabaseSession(sessions), + current_versions=lambda _country_id: CURRENT_VERSIONS, + submissions=submissions, + ), + simulations=V2StoredSimulationService( + StoredSimulationDatabaseSession(sessions), + current_versions=lambda _country_id: CURRENT_VERSIONS, + submissions=submissions, + ), + ) + + +def report_body(**changes: Any) -> dict[str, Any]: + values: dict[str, Any] = { + "country_id": "us", + "kind": "economy", + "label": "Expanded child tax credit", + "year": 2026, + "region_code": "us", + "baseline": {}, + "reform": {}, + } + return {**values, **changes} + + +def simulation_body(policy_id: UUID) -> dict[str, Any]: + return { + "country_id": "us", + "kind": "economy", + "year": 2026, + "region_code": "us", + "policy_id": str(policy_id), + } + + +@pytest.mark.parametrize("side", ["baseline", "reform"]) +def test_a_report_whose_policy_sets_parameters_the_running_catalog_lacks_is_refused( + client: TestClient, + stored_policy_ids: dict[str, UUID], + simulation_service: RecordingSimulationService, + side: str, +) -> None: + outdated_policy_id = stored_policy_ids["outdated_policy"] + + response = client.post( + REPORTS, + json=report_body(**{side: {"policy_id": str(outdated_policy_id)}}), + ) + + assert response.status_code == 422, response.text + assert response.json() == { + "status": "error", + "message": ( + f"Policy {outdated_policy_id} sets parameters the running catalog " + f"version does not have: {REMOVED_PARAMETER}" + ), + "code": "missing_parameters", + "missing_parameters": [REMOVED_PARAMETER], + } + assert simulation_service.reports == [] + + +def test_a_simulation_whose_policy_sets_parameters_the_running_catalog_lacks_is_refused( + client: TestClient, + stored_policy_ids: dict[str, UUID], + simulation_service: RecordingSimulationService, +) -> None: + response = client.post( + SIMULATIONS, json=simulation_body(stored_policy_ids["outdated_policy"]) + ) + + assert response.status_code == 422, response.text + assert response.json()["code"] == "missing_parameters" + assert response.json()["missing_parameters"] == [REMOVED_PARAMETER] + assert simulation_service.simulations == [] + + +def test_policies_whose_parameters_all_exist_in_the_running_catalog_are_sent( + submissions: SimulationSubmissionService, + stored_policy_ids: dict[str, UUID], + simulation_service: RecordingSimulationService, +) -> None: + report = EconomyReportCreationInput.model_validate( + report_body( + baseline={"policy_id": str(stored_policy_ids["kept_policy"])}, + reform={"policy_id": str(stored_policy_ids["running_policy"])}, + ) + ) + simulation = EconomySimulationCreationInput.model_validate( + simulation_body(stored_policy_ids["kept_policy"]) + ) + + submissions.submit_report(report) + submissions.submit_simulation(simulation) + + assert simulation_service.reports == [report] + assert simulation_service.simulations == [simulation] + + +@pytest.mark.parametrize("record", ["report", "simulation"]) +def test_a_catalog_that_cannot_be_queried_answers_503( + engine: Engine, + client: TestClient, + stored_policy_ids: dict[str, UUID], + simulation_service: RecordingSimulationService, + record: str, +) -> None: + running_policy_id = stored_policy_ids["running_policy"] + # The policy check's catalog query reads this table; nothing else does. + with engine.begin() as connection: + connection.execute(text("DROP TABLE tax_benefit_model_versions")) + + if record == "report": + response = client.post( + REPORTS, json=report_body(reform={"policy_id": str(running_policy_id)}) + ) + else: + response = client.post(SIMULATIONS, json=simulation_body(running_policy_id)) + + assert response.status_code == 503 + assert response.json() == { + "status": "error", + "message": "V2 storage is unavailable", + } + assert simulation_service.reports == [] + assert simulation_service.simulations == [] diff --git a/tests/unit/v2/test_simulation_submissions.py b/tests/unit/v2/test_simulation_submissions.py index b6e22bd33..bc0e5ae60 100644 --- a/tests/unit/v2/test_simulation_submissions.py +++ b/tests/unit/v2/test_simulation_submissions.py @@ -7,9 +7,16 @@ from uuid import UUID, uuid4 import pytest +from sqlalchemy.exc import OperationalError +from policyengine_api.data.v2.catalog.catalog_selection_validation import ( + MetadataCatalogUnavailableError, +) from policyengine_api.services.v2.countries.types import CountryRead -from policyengine_api.services.v2.policies.validators import PolicyNotFoundError +from policyengine_api.services.v2.policies.validators import ( + PolicyNotFoundError, + PolicyParametersMissingError, +) from policyengine_api.services.v2.simulation_service.services import ( SimulationSubmissionService, ) @@ -21,10 +28,27 @@ ) from policyengine_api.services.v2.simulation_service.validators import ( EconomyCalculationsUnsupportedError, + SubmissionStorageError, ) STORED_POLICY_ID = UUID("00000000-0000-0000-0000-00000000c001") OTHER_POLICY_ID = UUID("00000000-0000-0000-0000-00000000c002") +# Stored policies that set parameters the running catalog version lacks. +OUTDATED_POLICY_ID = UUID("00000000-0000-0000-0000-00000000c003") +SECOND_OUTDATED_POLICY_ID = UUID("00000000-0000-0000-0000-00000000c004") +# A stored policy the running catalog could not check, because the running +# PolicyEngine.py version has no published catalog for the country. +UNCHECKED_POLICY_ID = UUID("00000000-0000-0000-0000-00000000c005") +REMOVED_PARAMETERS = ("gov.example.removed_amount", "gov.example.removed_rate") +SECOND_REMOVED_PARAMETERS = ("gov.example.removed_threshold",) +# Each stored policy's parameters the running catalog version lacks, as the +# policy service checks them: empty when all exist, None when not checked. +MISSING_PARAMETERS: dict[UUID, tuple[str, ...] | None] = { + STORED_POLICY_ID: (), + OUTDATED_POLICY_ID: REMOVED_PARAMETERS, + SECOND_OUTDATED_POLICY_ID: SECOND_REMOVED_PARAMETERS, + UNCHECKED_POLICY_ID: None, +} SERVED_COUNTRIES = [ CountryRead(id="uk", household_enabled=True, society_wide_enabled=True), CountryRead(id="us", household_enabled=True, society_wide_enabled=True), @@ -44,18 +68,33 @@ def list_countries(self) -> list[CountryRead]: return self.countries +@dataclass(frozen=True) +class CheckedPolicy: + """The parts of a checked policy (``CheckedPolicyRead``) the checks read.""" + + id: UUID + missing_parameters: tuple[str, ...] | None + + @dataclass class StoredPolicies: - """The stored policies: only STORED_POLICY_ID, in the US and the UK.""" + """The policies in MISSING_PARAMETERS, stored in the US and the UK. + + With ``error``, every read raises it, as a database or catalog that + cannot be read does. + """ + error: Exception | None = None asked: list[tuple[str, UUID]] = field(default_factory=list) - def get_policy(self, *, country_id: str, policy_id: UUID) -> Any: + def get_policy(self, *, country_id: str, policy_id: UUID) -> CheckedPolicy: self.asked.append((country_id, policy_id)) - if policy_id != STORED_POLICY_ID: + if self.error is not None: + raise self.error + if policy_id not in MISSING_PARAMETERS: msg = f"Policy {policy_id} was not found" raise PolicyNotFoundError(msg) - return object() + return CheckedPolicy(policy_id, MISSING_PARAMETERS[policy_id]) @dataclass @@ -195,3 +234,95 @@ def test_a_simulation_naming_a_policy_not_stored_is_refused() -> None: with pytest.raises(PolicyNotFoundError): service.submit_simulation(simulation_input(policy_id=str(OTHER_POLICY_ID))) + + +@pytest.mark.parametrize("side", ["baseline", "reform"]) +def test_a_report_whose_policy_sets_parameters_the_running_catalog_lacks_is_refused( + side: str, +) -> None: + service = submissions(unconfigured_client) + + with pytest.raises(PolicyParametersMissingError) as refusal: + service.submit_report( + report_input(**{side: {"policy_id": str(OUTDATED_POLICY_ID)}}) + ) + + assert refusal.value.policy_id == OUTDATED_POLICY_ID + assert refusal.value.missing_parameters == REMOVED_PARAMETERS + + +def test_a_simulation_whose_policy_sets_missing_parameters_is_refused() -> None: + service = submissions(unconfigured_client) + + with pytest.raises(PolicyParametersMissingError) as refusal: + service.submit_simulation(simulation_input(policy_id=str(OUTDATED_POLICY_ID))) + + assert refusal.value.policy_id == OUTDATED_POLICY_ID + assert refusal.value.missing_parameters == REMOVED_PARAMETERS + + +def test_a_report_with_two_outdated_policies_is_refused_for_its_baseline() -> None: + service = submissions(unconfigured_client) + + with pytest.raises(PolicyParametersMissingError) as refusal: + service.submit_report( + report_input( + baseline={"policy_id": str(OUTDATED_POLICY_ID)}, + reform={"policy_id": str(SECOND_OUTDATED_POLICY_ID)}, + ) + ) + + assert refusal.value.policy_id == OUTDATED_POLICY_ID + + +@pytest.mark.parametrize("outdated_side", ["baseline", "reform"]) +def test_a_policy_not_stored_is_refused_before_another_is_checked_for_parameters( + outdated_side: str, +) -> None: + # A policy that is not stored answers 404 whichever side names it, before + # the other side's missing parameters answer 422. + other_side = "reform" if outdated_side == "baseline" else "baseline" + service = submissions(unconfigured_client) + + with pytest.raises(PolicyNotFoundError, match=str(OTHER_POLICY_ID)): + service.submit_report( + report_input( + **{ + outdated_side: {"policy_id": str(OUTDATED_POLICY_ID)}, + other_side: {"policy_id": str(OTHER_POLICY_ID)}, + } + ) + ) + + +def test_a_policy_the_running_catalog_could_not_check_is_sent() -> None: + client = RecordingSimulationService() + + submissions(lambda: client).submit_report( + report_input(reform={"policy_id": str(UNCHECKED_POLICY_ID)}) + ) + submissions(lambda: client).submit_simulation( + simulation_input(policy_id=str(UNCHECKED_POLICY_ID)) + ) + + assert len(client.reports) == 1 + assert len(client.simulations) == 1 + + +@pytest.mark.parametrize( + "error", + [ + MetadataCatalogUnavailableError("the v2 metadata catalog cannot be queried"), + OperationalError("SELECT 1", {}, Exception("connection refused")), + ], + ids=["catalog", "database"], +) +def test_a_policy_check_that_cannot_read_storage_is_storage_unavailable( + error: Exception, +) -> None: + service = submissions(unconfigured_client, StoredPolicies(error=error)) + + with pytest.raises(SubmissionStorageError) as unavailable: + service.submit_report(report_input()) + + assert unavailable.value.__cause__ is error From 0e6100d6510f7a30631cac566a6f83e3168c61b7 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:12:45 +0400 Subject: [PATCH 2/4] Refuse reports and simulations whose policies set missing parameters POST /v2/reports and POST /v2/simulations now check, before the simulation service is asked, that the running catalog version has every parameter each named policy sets. A report checks its baseline's policy, then its reform's; every policy is read first, so a policy that is not stored still answers 404 before another's missing parameters answer 422. The 422 has code missing_parameters and names the parameters, as POST /v2/household-calculations answers. require_runnable_policies holds the check, for re-runs to call too. The submission checks now answer 503 when the catalog query fails, as they do when the database cannot be read, rather than 500. Co-Authored-By: Claude Opus 5.5 --- .../v2-runs-missing-parameters.added.md | 1 + .../v2/reports/response_models.py | 30 ++++--- .../fastapi_routes/v2/reports/routes.py | 4 +- .../v2/simulation_resources_support.py | 16 +++- .../v2/simulations/response_models.py | 28 ++++--- .../fastapi_routes/v2/simulations/routes.py | 3 +- .../v2/simulation_service/services.py | 81 ++++++++++++++----- .../services/v2/simulation_service/types.py | 3 +- .../v2/simulation_service/validators.py | 2 +- 9 files changed, 122 insertions(+), 46 deletions(-) create mode 100644 changelog.d/v2-runs-missing-parameters.added.md diff --git a/changelog.d/v2-runs-missing-parameters.added.md b/changelog.d/v2-runs-missing-parameters.added.md new file mode 100644 index 000000000..955d1ec79 --- /dev/null +++ b/changelog.d/v2-runs-missing-parameters.added.md @@ -0,0 +1 @@ +`POST /v2/reports` and `POST /v2/simulations` refuse a policy that sets parameters the running catalog version no longer has, as `POST /v2/household-calculations` does: they answer 422 with code `missing_parameters`, naming those parameters, and the simulation service is not asked, so no run starts. A report checks its baseline's policy, then its reform's. A catalog that cannot be read during the check answers 503. diff --git a/policyengine_api/fastapi_routes/v2/reports/response_models.py b/policyengine_api/fastapi_routes/v2/reports/response_models.py index 10f80261a..a6843c029 100644 --- a/policyengine_api/fastapi_routes/v2/reports/response_models.py +++ b/policyengine_api/fastapi_routes/v2/reports/response_models.py @@ -8,7 +8,10 @@ from pydantic import BaseModel, ConfigDict, JsonValue -from policyengine_api.fastapi_routes.v2.errors_support import V2ErrorResponse +from policyengine_api.fastapi_routes.v2.errors_support import ( + V2ErrorResponse, + V2MissingParametersErrorResponse, +) from policyengine_api.fastapi_routes.v2.simulation_resources_support import ( error_response, ) @@ -184,14 +187,20 @@ class UnavailableAfterReportCreatedResponse(V2ErrorResponse): 400: error_response("The body and query name different countries."), 404: error_response("A policy the report names is not stored in the country."), 413: error_response("The request body exceeds 16 KiB."), - 422: error_response( - "The request does not match the schema, the country has no economy " - "reports (such as a household-only country), or the simulation service " - "cannot run the report: its 422, or its 400 for a region, dataset or " - "dynamic it does not have. The message says which; for the simulation " - "service's refusal, it gives the service's code, when it gave one, and " - "its reason, but not its status." - ), + 422: { + "model": V2MissingParametersErrorResponse | V2ErrorResponse, + "description": ( + "The request does not match the schema; the country has no economy " + "reports (such as a household-only country); a policy the report " + "names sets parameters the running catalog version does not have " + "(code missing_parameters, which names them; the baseline's policy is " + "checked first); or the simulation service cannot run the report: its " + "422, or its 400 for a region, dataset or dynamic it does not have. The " + "message says which; for the simulation service's refusal, it gives " + "the service's code, when it gave one, and its reason, but not its " + "status." + ), + }, 502: { "model": V2ErrorResponse | CreatedReportUnreadableResponse, "description": ( @@ -221,7 +230,8 @@ class UnavailableAfterReportCreatedResponse(V2ErrorResponse): "release deploys), and the message gives its code and reason; it " "answered 400 or 422 in a shape the Public API cannot read; or it answered " "401, 403, 404, 405, 413 or 429, which are sent before its route runs. Or " - "the v2 database is unavailable. Or the service created the report, then " + "the v2 database, or the catalog the policies are checked against, cannot " + "be read. Or the service created the report, then " "Modal refused the run's call and the service failed the run: its 503 " "named the report, with its status and code, and the answer gives its " "report_id." diff --git a/policyengine_api/fastapi_routes/v2/reports/routes.py b/policyengine_api/fastapi_routes/v2/reports/routes.py index 89daf7673..b57fcd6d8 100644 --- a/policyengine_api/fastapi_routes/v2/reports/routes.py +++ b/policyengine_api/fastapi_routes/v2/reports/routes.py @@ -187,7 +187,9 @@ def build_v2_report_router(dependencies: NativeRouteDependencies) -> APIRouter: summary="Create an economy report", description=( "Checks that the country has economy reports and stores each named " - "policy, then has the simulation service create the report, its two " + "policy, and that the running catalog version has every parameter " + "those policies set, then has the simulation service create the " + "report, its two " "simulations and their runs on its default worker's PolicyEngine.py " "version, and start the run. Answers the report as stored, with its " "pending run; read it again to follow the run. A created report " diff --git a/policyengine_api/fastapi_routes/v2/simulation_resources_support.py b/policyengine_api/fastapi_routes/v2/simulation_resources_support.py index bb153ccbc..971b73cb4 100644 --- a/policyengine_api/fastapi_routes/v2/simulation_resources_support.py +++ b/policyengine_api/fastapi_routes/v2/simulation_resources_support.py @@ -20,8 +20,12 @@ V2ErrorMapping, V2ErrorResponse, run_v2_operation, + v2_missing_parameters_response, +) +from policyengine_api.services.v2.policies.validators import ( + PolicyNotFoundError, + PolicyParametersMissingError, ) -from policyengine_api.services.v2.policies.validators import PolicyNotFoundError from policyengine_api.services.v2.simulation_service.types import ( SimulationServiceAnswerUnreadableError, SimulationServiceInternalError, @@ -136,13 +140,17 @@ def simulation_resource_operation( ) -> OperationT | JSONResponse: """Run one service call and answer its failure with a typed v2 error. - A failure of a type in ``refusals`` is the caller's to act on and is not - logged; any other is logged under ``event``. + A policy that sets parameters the running catalog version lacks answers + 422 with code ``missing_parameters``, naming them, whatever the route. A + failure of a type in ``refusals`` is the caller's to act on and is not + logged, nor is that 422; any other is logged under ``event``. """ - def logged() -> OperationT: + def logged() -> OperationT | JSONResponse: try: return operation() + except PolicyParametersMissingError as error: + return v2_missing_parameters_response(str(error), error.missing_parameters) except Exception as error: if not isinstance(error, refusals): _log_failure(event, error) diff --git a/policyengine_api/fastapi_routes/v2/simulations/response_models.py b/policyengine_api/fastapi_routes/v2/simulations/response_models.py index b1b3f75be..c95369c9b 100644 --- a/policyengine_api/fastapi_routes/v2/simulations/response_models.py +++ b/policyengine_api/fastapi_routes/v2/simulations/response_models.py @@ -8,7 +8,10 @@ from pydantic import BaseModel, ConfigDict -from policyengine_api.fastapi_routes.v2.errors_support import V2ErrorResponse +from policyengine_api.fastapi_routes.v2.errors_support import ( + V2ErrorResponse, + V2MissingParametersErrorResponse, +) from policyengine_api.fastapi_routes.v2.reports.response_models import ( PackageVersionsItem, ) @@ -126,14 +129,18 @@ class UnavailableAfterSimulationCreatedResponse(V2ErrorResponse): "The policy the simulation names is not stored in the country." ), 413: error_response("The request body exceeds 16 KiB."), - 422: error_response( - "The request does not match the schema, the country has no economy " - "calculations, or the simulation service cannot run the simulation: " - "its 422, or its 400 for a region, dataset or dynamic it does not " - "have. The message says which; for the simulation service's refusal, " - "it gives the service's code, when it gave one, and its reason, but " - "not its status." - ), + 422: { + "model": V2MissingParametersErrorResponse | V2ErrorResponse, + "description": ( + "The request does not match the schema; the country has no economy " + "calculations; the policy sets parameters the running catalog version " + "does not have (code missing_parameters, which names them); or the " + "simulation service cannot run the simulation: its 422, or its 400 " + "for a region, dataset or dynamic it does not have. The message says " + "which; for the simulation service's refusal, it gives the service's " + "code, when it gave one, and its reason, but not its status." + ), + }, 502: { "model": V2ErrorResponse | CreatedSimulationUnreadableResponse, "description": ( @@ -163,7 +170,8 @@ class UnavailableAfterSimulationCreatedResponse(V2ErrorResponse): "release deploys), and the message gives its code and reason; it " "answered 400 or 422 in a shape the Public API cannot read; or it answered " "401, 403, 404, 405, 413 or 429, which are sent before its route runs. Or " - "the v2 database is unavailable. Or the service created the simulation, " + "the v2 database, or the catalog the policy is checked against, cannot " + "be read. Or the service created the simulation, " "then Modal refused the run's call and the service failed the run: its 503 " "named the simulation, with its status and code, and the answer gives its " "simulation_id." diff --git a/policyengine_api/fastapi_routes/v2/simulations/routes.py b/policyengine_api/fastapi_routes/v2/simulations/routes.py index ceadeaf4c..043f61402 100644 --- a/policyengine_api/fastapi_routes/v2/simulations/routes.py +++ b/policyengine_api/fastapi_routes/v2/simulations/routes.py @@ -170,7 +170,8 @@ def build_v2_simulation_router(dependencies: NativeRouteDependencies) -> APIRout summary="Create an economy simulation", description=( "Checks that the country has economy calculations and stores the " - "named policy, then has the simulation service create the " + "named policy, and that the running catalog version has every " + "parameter it sets, then has the simulation service create the " "simulation and its first run on its default worker's " "PolicyEngine.py version, and start the run. Answers the simulation " "as stored, with its pending run. A created simulation that cannot " diff --git a/policyengine_api/services/v2/simulation_service/services.py b/policyengine_api/services/v2/simulation_service/services.py index 97f7870e1..0867553fa 100644 --- a/policyengine_api/services/v2/simulation_service/services.py +++ b/policyengine_api/services/v2/simulation_service/services.py @@ -4,26 +4,35 @@ ``POST /reports`` goes through ``SimulationSubmissionService``, so checks that apply to both live in one place. It checks, in order, that the country's ``countries`` row allows economy calculations (a household-only -country does not) and that each policy the submission names is stored in the -country, through the same service as ``/v2/policies``; current law names -none. Only then is the simulation service asked. The dynamics a submission -names, its region and its dataset are checked by the simulation service, -which reads them in the catalog version its default worker runs. +country does not), that each policy the submission names is stored in the +country, through the same service as ``/v2/policies`` (current law names +none), and that none of those policies sets a parameter the running catalog +version lacks (``require_runnable_policies``, which re-runs should call too). +Only then is the simulation service asked. The dynamics a submission names, +its region and its dataset are checked by the simulation service, which reads +them in the catalog version its default worker runs. """ from __future__ import annotations +from contextlib import contextmanager from typing import TYPE_CHECKING from sqlalchemy.exc import SQLAlchemyError +from policyengine_api.data.v2.catalog.catalog_selection_validation import ( + MetadataCatalogUnavailableError, +) +from policyengine_api.services.v2.policies.validators import ( + require_policy_parameters_present, +) from policyengine_api.services.v2.simulation_service.validators import ( SubmissionStorageError, require_economy_country, ) if TYPE_CHECKING: - from collections.abc import Callable, Iterable + from collections.abc import Callable, Iterable, Iterator from uuid import UUID from policyengine_api.services.v2.simulation_service.types import ( @@ -37,6 +46,44 @@ ) +@contextmanager +def _storage_failures() -> Iterator[None]: + """Raise SubmissionStorageError for a read that failed. + + That includes the policy check's catalog query, which the policy service + reports as MetadataCatalogUnavailableError. + """ + try: + yield + except (SQLAlchemyError, MetadataCatalogUnavailableError) as error: + msg = "V2 storage is unavailable to check the submission" + raise SubmissionStorageError(msg) from error + + +def require_runnable_policies( + policies: PolicySource, *, country_id: str, policy_ids: Iterable[UUID | None] +) -> None: + """Raise unless each named policy is stored and its parameters all exist. + + Every policy is read before any is checked, so PolicyNotFoundError for a + policy the country does not store comes before PolicyParametersMissingError + for one that sets parameters the running catalog version lacks; the + latter names the first such policy in ``policy_ids``. Current law (None) + names no policy, and a policy the running catalog could not check passes. + Raise SubmissionStorageError when the database or its catalog cannot be + read. Anything else that starts a run of stored policies, such as a re-run, + should check them here first. + """ + with _storage_failures(): + stored_policies = [ + policies.get_policy(country_id=country_id, policy_id=policy_id) + for policy_id in dict.fromkeys(policy_ids) + if policy_id is not None + ] + for policy in stored_policies: + require_policy_parameters_present(policy.id, policy.missing_parameters) + + class SimulationSubmissionService: """Check submissions, then send them to the simulation service.""" @@ -58,21 +105,17 @@ def __init__( self._client_factory = client_factory def _check(self, country_id: str, policy_ids: Iterable[UUID | None]) -> None: - """Raise unless the country has economy calculations and stores the policies. + """Raise unless the country has economy calculations and the policies can run. Raise EconomyCalculationsUnsupportedError, then PolicyNotFoundError, - and SubmissionStorageError when the database cannot be read. + then PolicyParametersMissingError, and SubmissionStorageError when the + database or its catalog cannot be read. """ - try: + with _storage_failures(): require_economy_country(self._countries.list_countries(), country_id) - for policy_id in dict.fromkeys(policy_ids): - if policy_id is not None: - self._policies.get_policy( - country_id=country_id, policy_id=policy_id - ) - except SQLAlchemyError as error: - msg = "V2 storage is unavailable to check the submission" - raise SubmissionStorageError(msg) from error + require_runnable_policies( + self._policies, country_id=country_id, policy_ids=policy_ids + ) def submit_simulation( self, simulation: EconomySimulationCreationInput @@ -87,7 +130,9 @@ def submit_simulation( def submit_report(self, report: EconomyReportCreationInput) -> SubmittedReport: """Check the report, then have the simulation service create and start it. - Raise the checks' errors, then the simulation service's. + Both the baseline's and the reform's policies are checked, the + baseline's first. Raise the checks' errors, then the simulation + service's. """ self._check( report.country_id, (report.baseline.policy_id, report.reform.policy_id) diff --git a/policyengine_api/services/v2/simulation_service/types.py b/policyengine_api/services/v2/simulation_service/types.py index da92249eb..dcfd1f00e 100644 --- a/policyengine_api/services/v2/simulation_service/types.py +++ b/policyengine_api/services/v2/simulation_service/types.py @@ -244,7 +244,8 @@ class SimulationServiceClient(Protocol): ``SimulationSubmissionService``, which checks a submission before sending it through a client, also raises EconomyCalculationsUnsupportedError, - PolicyNotFoundError and SubmissionStorageError from its checks. + PolicyNotFoundError, PolicyParametersMissingError and + SubmissionStorageError from its checks. """ def submit_simulation( diff --git a/policyengine_api/services/v2/simulation_service/validators.py b/policyengine_api/services/v2/simulation_service/validators.py index f7239ed23..1c0c84a29 100644 --- a/policyengine_api/services/v2/simulation_service/validators.py +++ b/policyengine_api/services/v2/simulation_service/validators.py @@ -19,7 +19,7 @@ class EconomyCalculationsUnsupportedError(ValueError): class SubmissionStorageError(RuntimeError): - """The countries or policies a submission is checked against could not be read.""" + """A submission's countries, policies or catalog could not be read.""" def require_economy_country(countries: Iterable[CountryRead], country_id: str) -> None: From 795116de82e3e53b60f95280473039303199a08e Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:20:41 +0400 Subject: [PATCH 3/4] Name the policy check's errors in the create protocols, and type the test policies The report and simulation create protocols now list PolicyParametersMissingError and SubmissionStorageError among the errors a create raises. The submission tests' stand-in policy source returns CheckedPolicyRead values, as the protocol it stands in for does. Co-Authored-By: Claude Opus 5.5 --- .../resource_service_protocols.py | 5 +++- tests/unit/v2/test_report_create_routes.py | 5 +++- tests/unit/v2/test_simulation_submissions.py | 27 ++++++++++++++----- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/policyengine_api/fastapi_routes/resource_service_protocols.py b/policyengine_api/fastapi_routes/resource_service_protocols.py index a82265a82..d3257a6e5 100644 --- a/policyengine_api/fastapi_routes/resource_service_protocols.py +++ b/policyengine_api/fastapi_routes/resource_service_protocols.py @@ -211,7 +211,10 @@ def create_report(self, report_input: EconomyReportCreationInput) -> ReportRead: Raise ``EconomyCalculationsUnsupportedError`` for a country without economy reports, ``PolicyNotFoundError`` for a policy the country - does not store, and the simulation service's errors. + does not store, ``PolicyParametersMissingError`` for a policy that + sets parameters the running catalog version lacks, + ``SubmissionStorageError`` when the database or its catalog cannot be + read for those checks, and the simulation service's errors. """ ... diff --git a/tests/unit/v2/test_report_create_routes.py b/tests/unit/v2/test_report_create_routes.py index e065dc1ea..6e255382a 100644 --- a/tests/unit/v2/test_report_create_routes.py +++ b/tests/unit/v2/test_report_create_routes.py @@ -423,7 +423,10 @@ def test_a_simulation_the_unavailable_service_created_answers_its_id( ], ) def test_a_policy_setting_parameters_the_running_catalog_lacks_answers_422( - client_for_error, path: str, body: dict[str, Any], caplog + client_for_error: Callable[[Exception], TestClient], + path: str, + body: dict[str, Any], + caplog: pytest.LogCaptureFixture, ) -> None: error = PolicyParametersMissingError(POLICY_ID, REMOVED_PARAMETERS) client = client_for_error(error) diff --git a/tests/unit/v2/test_simulation_submissions.py b/tests/unit/v2/test_simulation_submissions.py index bc0e5ae60..15a44f6b9 100644 --- a/tests/unit/v2/test_simulation_submissions.py +++ b/tests/unit/v2/test_simulation_submissions.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import dataclass, field +from datetime import UTC, datetime from typing import Any from uuid import UUID, uuid4 @@ -13,6 +14,7 @@ MetadataCatalogUnavailableError, ) from policyengine_api.services.v2.countries.types import CountryRead +from policyengine_api.services.v2.policies.types import CheckedPolicyRead from policyengine_api.services.v2.policies.validators import ( PolicyNotFoundError, PolicyParametersMissingError, @@ -68,12 +70,23 @@ def list_countries(self) -> list[CountryRead]: return self.countries -@dataclass(frozen=True) -class CheckedPolicy: - """The parts of a checked policy (``CheckedPolicyRead``) the checks read.""" +STORED_AT = datetime(2026, 11, 12, tzinfo=UTC) - id: UUID - missing_parameters: tuple[str, ...] | None + +def checked_policy( + policy_id: UUID, country_id: str, missing_parameters: tuple[str, ...] | None +) -> CheckedPolicyRead: + """Return a stored policy as the policy service checks it.""" + return CheckedPolicyRead( + id=policy_id, + country_id=country_id, + tax_benefit_model_id=uuid4(), + tax_benefit_model_version_id=uuid4(), + created_at=STORED_AT, + updated_at=STORED_AT, + parameter_values=(), + missing_parameters=missing_parameters, + ) @dataclass @@ -87,14 +100,14 @@ class StoredPolicies: error: Exception | None = None asked: list[tuple[str, UUID]] = field(default_factory=list) - def get_policy(self, *, country_id: str, policy_id: UUID) -> CheckedPolicy: + def get_policy(self, *, country_id: str, policy_id: UUID) -> CheckedPolicyRead: self.asked.append((country_id, policy_id)) if self.error is not None: raise self.error if policy_id not in MISSING_PARAMETERS: msg = f"Policy {policy_id} was not found" raise PolicyNotFoundError(msg) - return CheckedPolicy(policy_id, MISSING_PARAMETERS[policy_id]) + return checked_policy(policy_id, country_id, MISSING_PARAMETERS[policy_id]) @dataclass From d544556ac138878a6693007ccd1b81029ff5bdda Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:23:55 +0400 Subject: [PATCH 4/4] Name the read-back errors in the create protocols Each create protocol now names the error raised when the record the simulation service created cannot be read back, and the simulation create no longer points at the report create's read-back error. Co-Authored-By: Claude Opus 5.5 --- .../fastapi_routes/resource_service_protocols.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/policyengine_api/fastapi_routes/resource_service_protocols.py b/policyengine_api/fastapi_routes/resource_service_protocols.py index d3257a6e5..f444ab644 100644 --- a/policyengine_api/fastapi_routes/resource_service_protocols.py +++ b/policyengine_api/fastapi_routes/resource_service_protocols.py @@ -214,7 +214,9 @@ def create_report(self, report_input: EconomyReportCreationInput) -> ReportRead: does not store, ``PolicyParametersMissingError`` for a policy that sets parameters the running catalog version lacks, ``SubmissionStorageError`` when the database or its catalog cannot be - read for those checks, and the simulation service's errors. + read for those checks, the simulation service's errors, and + ``CreatedReportUnreadableError`` when the report it created cannot be + read back. """ ... @@ -234,7 +236,10 @@ def create_simulation( ) -> SimulationRead: """Create the simulation through the simulation service; return it as stored. - Raise as ``V2StoredReportResourceService.create_report`` does. + Raise the checks' and the simulation service's errors as + ``V2StoredReportResourceService.create_report`` does, for the + simulation's one policy, and ``CreatedSimulationUnreadableError`` when + the simulation it created cannot be read back. """ ...