diff --git a/changelog.d/v2-runs-missing-parameters.added.md b/changelog.d/v2-runs-missing-parameters.added.md new file mode 100644 index 000000000..955d1ec79 --- /dev/null +++ b/changelog.d/v2-runs-missing-parameters.added.md @@ -0,0 +1 @@ +`POST /v2/reports` and `POST /v2/simulations` refuse a policy that sets parameters the running catalog version no longer has, as `POST /v2/household-calculations` does: they answer 422 with code `missing_parameters`, naming those parameters, and the simulation service is not asked, so no run starts. A report checks its baseline's policy, then its reform's. A catalog that cannot be read during the check answers 503. diff --git a/policyengine_api/fastapi_routes/resource_service_protocols.py b/policyengine_api/fastapi_routes/resource_service_protocols.py index a82265a82..f444ab644 100644 --- a/policyengine_api/fastapi_routes/resource_service_protocols.py +++ b/policyengine_api/fastapi_routes/resource_service_protocols.py @@ -211,7 +211,12 @@ def create_report(self, report_input: EconomyReportCreationInput) -> ReportRead: Raise ``EconomyCalculationsUnsupportedError`` for a country without economy reports, ``PolicyNotFoundError`` for a policy the country - does not store, and the simulation service's errors. + does not store, ``PolicyParametersMissingError`` for a policy that + sets parameters the running catalog version lacks, + ``SubmissionStorageError`` when the database or its catalog cannot be + read for those checks, the simulation service's errors, and + ``CreatedReportUnreadableError`` when the report it created cannot be + read back. """ ... @@ -231,7 +236,10 @@ def create_simulation( ) -> SimulationRead: """Create the simulation through the simulation service; return it as stored. - Raise as ``V2StoredReportResourceService.create_report`` does. + Raise the checks' and the simulation service's errors as + ``V2StoredReportResourceService.create_report`` does, for the + simulation's one policy, and ``CreatedSimulationUnreadableError`` when + the simulation it created cannot be read back. """ ... diff --git a/policyengine_api/fastapi_routes/v2/reports/response_models.py b/policyengine_api/fastapi_routes/v2/reports/response_models.py index 10f80261a..a6843c029 100644 --- a/policyengine_api/fastapi_routes/v2/reports/response_models.py +++ b/policyengine_api/fastapi_routes/v2/reports/response_models.py @@ -8,7 +8,10 @@ from pydantic import BaseModel, ConfigDict, JsonValue -from policyengine_api.fastapi_routes.v2.errors_support import V2ErrorResponse +from policyengine_api.fastapi_routes.v2.errors_support import ( + V2ErrorResponse, + V2MissingParametersErrorResponse, +) from policyengine_api.fastapi_routes.v2.simulation_resources_support import ( error_response, ) @@ -184,14 +187,20 @@ class UnavailableAfterReportCreatedResponse(V2ErrorResponse): 400: error_response("The body and query name different countries."), 404: error_response("A policy the report names is not stored in the country."), 413: error_response("The request body exceeds 16 KiB."), - 422: error_response( - "The request does not match the schema, the country has no economy " - "reports (such as a household-only country), or the simulation service " - "cannot run the report: its 422, or its 400 for a region, dataset or " - "dynamic it does not have. The message says which; for the simulation " - "service's refusal, it gives the service's code, when it gave one, and " - "its reason, but not its status." - ), + 422: { + "model": V2MissingParametersErrorResponse | V2ErrorResponse, + "description": ( + "The request does not match the schema; the country has no economy " + "reports (such as a household-only country); a policy the report " + "names sets parameters the running catalog version does not have " + "(code missing_parameters, which names them; the baseline's policy is " + "checked first); or the simulation service cannot run the report: its " + "422, or its 400 for a region, dataset or dynamic it does not have. The " + "message says which; for the simulation service's refusal, it gives " + "the service's code, when it gave one, and its reason, but not its " + "status." + ), + }, 502: { "model": V2ErrorResponse | CreatedReportUnreadableResponse, "description": ( @@ -221,7 +230,8 @@ class UnavailableAfterReportCreatedResponse(V2ErrorResponse): "release deploys), and the message gives its code and reason; it " "answered 400 or 422 in a shape the Public API cannot read; or it answered " "401, 403, 404, 405, 413 or 429, which are sent before its route runs. Or " - "the v2 database is unavailable. Or the service created the report, then " + "the v2 database, or the catalog the policies are checked against, cannot " + "be read. Or the service created the report, then " "Modal refused the run's call and the service failed the run: its 503 " "named the report, with its status and code, and the answer gives its " "report_id." diff --git a/policyengine_api/fastapi_routes/v2/reports/routes.py b/policyengine_api/fastapi_routes/v2/reports/routes.py index 89daf7673..b57fcd6d8 100644 --- a/policyengine_api/fastapi_routes/v2/reports/routes.py +++ b/policyengine_api/fastapi_routes/v2/reports/routes.py @@ -187,7 +187,9 @@ def build_v2_report_router(dependencies: NativeRouteDependencies) -> APIRouter: summary="Create an economy report", description=( "Checks that the country has economy reports and stores each named " - "policy, then has the simulation service create the report, its two " + "policy, and that the running catalog version has every parameter " + "those policies set, then has the simulation service create the " + "report, its two " "simulations and their runs on its default worker's PolicyEngine.py " "version, and start the run. Answers the report as stored, with its " "pending run; read it again to follow the run. A created report " diff --git a/policyengine_api/fastapi_routes/v2/simulation_resources_support.py b/policyengine_api/fastapi_routes/v2/simulation_resources_support.py index bb153ccbc..971b73cb4 100644 --- a/policyengine_api/fastapi_routes/v2/simulation_resources_support.py +++ b/policyengine_api/fastapi_routes/v2/simulation_resources_support.py @@ -20,8 +20,12 @@ V2ErrorMapping, V2ErrorResponse, run_v2_operation, + v2_missing_parameters_response, +) +from policyengine_api.services.v2.policies.validators import ( + PolicyNotFoundError, + PolicyParametersMissingError, ) -from policyengine_api.services.v2.policies.validators import PolicyNotFoundError from policyengine_api.services.v2.simulation_service.types import ( SimulationServiceAnswerUnreadableError, SimulationServiceInternalError, @@ -136,13 +140,17 @@ def simulation_resource_operation( ) -> OperationT | JSONResponse: """Run one service call and answer its failure with a typed v2 error. - A failure of a type in ``refusals`` is the caller's to act on and is not - logged; any other is logged under ``event``. + A policy that sets parameters the running catalog version lacks answers + 422 with code ``missing_parameters``, naming them, whatever the route. A + failure of a type in ``refusals`` is the caller's to act on and is not + logged, nor is that 422; any other is logged under ``event``. """ - def logged() -> OperationT: + def logged() -> OperationT | JSONResponse: try: return operation() + except PolicyParametersMissingError as error: + return v2_missing_parameters_response(str(error), error.missing_parameters) except Exception as error: if not isinstance(error, refusals): _log_failure(event, error) diff --git a/policyengine_api/fastapi_routes/v2/simulations/response_models.py b/policyengine_api/fastapi_routes/v2/simulations/response_models.py index b1b3f75be..c95369c9b 100644 --- a/policyengine_api/fastapi_routes/v2/simulations/response_models.py +++ b/policyengine_api/fastapi_routes/v2/simulations/response_models.py @@ -8,7 +8,10 @@ from pydantic import BaseModel, ConfigDict -from policyengine_api.fastapi_routes.v2.errors_support import V2ErrorResponse +from policyengine_api.fastapi_routes.v2.errors_support import ( + V2ErrorResponse, + V2MissingParametersErrorResponse, +) from policyengine_api.fastapi_routes.v2.reports.response_models import ( PackageVersionsItem, ) @@ -126,14 +129,18 @@ class UnavailableAfterSimulationCreatedResponse(V2ErrorResponse): "The policy the simulation names is not stored in the country." ), 413: error_response("The request body exceeds 16 KiB."), - 422: error_response( - "The request does not match the schema, the country has no economy " - "calculations, or the simulation service cannot run the simulation: " - "its 422, or its 400 for a region, dataset or dynamic it does not " - "have. The message says which; for the simulation service's refusal, " - "it gives the service's code, when it gave one, and its reason, but " - "not its status." - ), + 422: { + "model": V2MissingParametersErrorResponse | V2ErrorResponse, + "description": ( + "The request does not match the schema; the country has no economy " + "calculations; the policy sets parameters the running catalog version " + "does not have (code missing_parameters, which names them); or the " + "simulation service cannot run the simulation: its 422, or its 400 " + "for a region, dataset or dynamic it does not have. The message says " + "which; for the simulation service's refusal, it gives the service's " + "code, when it gave one, and its reason, but not its status." + ), + }, 502: { "model": V2ErrorResponse | CreatedSimulationUnreadableResponse, "description": ( @@ -163,7 +170,8 @@ class UnavailableAfterSimulationCreatedResponse(V2ErrorResponse): "release deploys), and the message gives its code and reason; it " "answered 400 or 422 in a shape the Public API cannot read; or it answered " "401, 403, 404, 405, 413 or 429, which are sent before its route runs. Or " - "the v2 database is unavailable. Or the service created the simulation, " + "the v2 database, or the catalog the policy is checked against, cannot " + "be read. Or the service created the simulation, " "then Modal refused the run's call and the service failed the run: its 503 " "named the simulation, with its status and code, and the answer gives its " "simulation_id." diff --git a/policyengine_api/fastapi_routes/v2/simulations/routes.py b/policyengine_api/fastapi_routes/v2/simulations/routes.py index ceadeaf4c..043f61402 100644 --- a/policyengine_api/fastapi_routes/v2/simulations/routes.py +++ b/policyengine_api/fastapi_routes/v2/simulations/routes.py @@ -170,7 +170,8 @@ def build_v2_simulation_router(dependencies: NativeRouteDependencies) -> APIRout summary="Create an economy simulation", description=( "Checks that the country has economy calculations and stores the " - "named policy, then has the simulation service create the " + "named policy, and that the running catalog version has every " + "parameter it sets, then has the simulation service create the " "simulation and its first run on its default worker's " "PolicyEngine.py version, and start the run. Answers the simulation " "as stored, with its pending run. A created simulation that cannot " diff --git a/policyengine_api/services/v2/simulation_service/services.py b/policyengine_api/services/v2/simulation_service/services.py index 97f7870e1..0867553fa 100644 --- a/policyengine_api/services/v2/simulation_service/services.py +++ b/policyengine_api/services/v2/simulation_service/services.py @@ -4,26 +4,35 @@ ``POST /reports`` goes through ``SimulationSubmissionService``, so checks that apply to both live in one place. It checks, in order, that the country's ``countries`` row allows economy calculations (a household-only -country does not) and that each policy the submission names is stored in the -country, through the same service as ``/v2/policies``; current law names -none. Only then is the simulation service asked. The dynamics a submission -names, its region and its dataset are checked by the simulation service, -which reads them in the catalog version its default worker runs. +country does not), that each policy the submission names is stored in the +country, through the same service as ``/v2/policies`` (current law names +none), and that none of those policies sets a parameter the running catalog +version lacks (``require_runnable_policies``, which re-runs should call too). +Only then is the simulation service asked. The dynamics a submission names, +its region and its dataset are checked by the simulation service, which reads +them in the catalog version its default worker runs. """ from __future__ import annotations +from contextlib import contextmanager from typing import TYPE_CHECKING from sqlalchemy.exc import SQLAlchemyError +from policyengine_api.data.v2.catalog.catalog_selection_validation import ( + MetadataCatalogUnavailableError, +) +from policyengine_api.services.v2.policies.validators import ( + require_policy_parameters_present, +) from policyengine_api.services.v2.simulation_service.validators import ( SubmissionStorageError, require_economy_country, ) if TYPE_CHECKING: - from collections.abc import Callable, Iterable + from collections.abc import Callable, Iterable, Iterator from uuid import UUID from policyengine_api.services.v2.simulation_service.types import ( @@ -37,6 +46,44 @@ ) +@contextmanager +def _storage_failures() -> Iterator[None]: + """Raise SubmissionStorageError for a read that failed. + + That includes the policy check's catalog query, which the policy service + reports as MetadataCatalogUnavailableError. + """ + try: + yield + except (SQLAlchemyError, MetadataCatalogUnavailableError) as error: + msg = "V2 storage is unavailable to check the submission" + raise SubmissionStorageError(msg) from error + + +def require_runnable_policies( + policies: PolicySource, *, country_id: str, policy_ids: Iterable[UUID | None] +) -> None: + """Raise unless each named policy is stored and its parameters all exist. + + Every policy is read before any is checked, so PolicyNotFoundError for a + policy the country does not store comes before PolicyParametersMissingError + for one that sets parameters the running catalog version lacks; the + latter names the first such policy in ``policy_ids``. Current law (None) + names no policy, and a policy the running catalog could not check passes. + Raise SubmissionStorageError when the database or its catalog cannot be + read. Anything else that starts a run of stored policies, such as a re-run, + should check them here first. + """ + with _storage_failures(): + stored_policies = [ + policies.get_policy(country_id=country_id, policy_id=policy_id) + for policy_id in dict.fromkeys(policy_ids) + if policy_id is not None + ] + for policy in stored_policies: + require_policy_parameters_present(policy.id, policy.missing_parameters) + + class SimulationSubmissionService: """Check submissions, then send them to the simulation service.""" @@ -58,21 +105,17 @@ def __init__( self._client_factory = client_factory def _check(self, country_id: str, policy_ids: Iterable[UUID | None]) -> None: - """Raise unless the country has economy calculations and stores the policies. + """Raise unless the country has economy calculations and the policies can run. Raise EconomyCalculationsUnsupportedError, then PolicyNotFoundError, - and SubmissionStorageError when the database cannot be read. + then PolicyParametersMissingError, and SubmissionStorageError when the + database or its catalog cannot be read. """ - try: + with _storage_failures(): require_economy_country(self._countries.list_countries(), country_id) - for policy_id in dict.fromkeys(policy_ids): - if policy_id is not None: - self._policies.get_policy( - country_id=country_id, policy_id=policy_id - ) - except SQLAlchemyError as error: - msg = "V2 storage is unavailable to check the submission" - raise SubmissionStorageError(msg) from error + require_runnable_policies( + self._policies, country_id=country_id, policy_ids=policy_ids + ) def submit_simulation( self, simulation: EconomySimulationCreationInput @@ -87,7 +130,9 @@ def submit_simulation( def submit_report(self, report: EconomyReportCreationInput) -> SubmittedReport: """Check the report, then have the simulation service create and start it. - Raise the checks' errors, then the simulation service's. + Both the baseline's and the reform's policies are checked, the + baseline's first. Raise the checks' errors, then the simulation + service's. """ self._check( report.country_id, (report.baseline.policy_id, report.reform.policy_id) diff --git a/policyengine_api/services/v2/simulation_service/types.py b/policyengine_api/services/v2/simulation_service/types.py index da92249eb..dcfd1f00e 100644 --- a/policyengine_api/services/v2/simulation_service/types.py +++ b/policyengine_api/services/v2/simulation_service/types.py @@ -244,7 +244,8 @@ class SimulationServiceClient(Protocol): ``SimulationSubmissionService``, which checks a submission before sending it through a client, also raises EconomyCalculationsUnsupportedError, - PolicyNotFoundError and SubmissionStorageError from its checks. + PolicyNotFoundError, PolicyParametersMissingError and + SubmissionStorageError from its checks. """ def submit_simulation( diff --git a/policyengine_api/services/v2/simulation_service/validators.py b/policyengine_api/services/v2/simulation_service/validators.py index f7239ed23..1c0c84a29 100644 --- a/policyengine_api/services/v2/simulation_service/validators.py +++ b/policyengine_api/services/v2/simulation_service/validators.py @@ -19,7 +19,7 @@ class EconomyCalculationsUnsupportedError(ValueError): class SubmissionStorageError(RuntimeError): - """The countries or policies a submission is checked against could not be read.""" + """A submission's countries, policies or catalog could not be read.""" def require_economy_country(countries: Iterable[CountryRead], country_id: str) -> None: diff --git a/tests/unit/v2/test_report_create_routes.py b/tests/unit/v2/test_report_create_routes.py index 3c980d99f..6e255382a 100644 --- a/tests/unit/v2/test_report_create_routes.py +++ b/tests/unit/v2/test_report_create_routes.py @@ -15,7 +15,10 @@ SIMULATION_ERROR_MAPPINGS, ) from policyengine_api.libs.simulation_reports import SimulationServiceReportClient -from policyengine_api.services.v2.policies.validators import PolicyNotFoundError +from policyengine_api.services.v2.policies.validators import ( + PolicyNotFoundError, + PolicyParametersMissingError, +) from policyengine_api.services.v2.simulation_service.types import ( SimulationServiceAnswerUnreadableError, SimulationServiceInternalError, @@ -395,6 +398,67 @@ def test_a_simulation_the_unavailable_service_created_answers_its_id( assert logged_events(caplog) == ["v2_simulation_unavailable_after_creation"] +# The parameters a refused policy sets that the running catalog lacks. +REMOVED_PARAMETERS = ("gov.example.removed_amount", "gov.example.removed_rate") + + +@pytest.mark.usefixtures("enabled_route_support_logger") +@pytest.mark.parametrize( + ("client_for_error", "path", "body"), + [ + pytest.param( + lambda error: report_route_client(reports=FakeReportService(error=error)), + "/v2/reports?country_id=us", + report_body(), + id="report", + ), + pytest.param( + lambda error: report_route_client( + simulations=FakeSimulationService(error=error) + ), + "/v2/simulations?country_id=us", + simulation_body(), + id="simulation", + ), + ], +) +def test_a_policy_setting_parameters_the_running_catalog_lacks_answers_422( + client_for_error: Callable[[Exception], TestClient], + path: str, + body: dict[str, Any], + caplog: pytest.LogCaptureFixture, +) -> None: + error = PolicyParametersMissingError(POLICY_ID, REMOVED_PARAMETERS) + client = client_for_error(error) + + with caplog.at_level("ERROR", logger=ROUTE_SUPPORT_LOGGER): + response = client.post(path, json=body) + + assert response.status_code == 422 + assert response.json() == { + "status": "error", + "message": ( + f"Policy {POLICY_ID} sets parameters the running catalog version " + "does not have: gov.example.removed_amount, gov.example.removed_rate" + ), + "code": "missing_parameters", + "missing_parameters": list(REMOVED_PARAMETERS), + } + # The caller can act on the refusal, so it is not logged as a failure. + assert logged_events(caplog) == [] + + +@pytest.mark.parametrize("path", ["/v2/reports", "/v2/simulations"]) +def test_the_openapi_document_gives_the_missing_parameters_refusal(path: str) -> None: + document = report_route_client().get("/v2/openapi.json").json() + + answer = document["paths"][path]["post"]["responses"]["422"] + assert {"$ref": "#/components/schemas/V2MissingParametersErrorResponse"} in answer[ + "content" + ]["application/json"]["schema"]["anyOf"] + assert "missing_parameters" in answer["description"] + + @pytest.mark.parametrize( ("path", "status_code", "schema", "id_field"), [ diff --git a/tests/unit/v2/test_simulation_submission_database_routes.py b/tests/unit/v2/test_simulation_submission_database_routes.py new file mode 100644 index 000000000..cb9d33688 --- /dev/null +++ b/tests/unit/v2/test_simulation_submission_database_routes.py @@ -0,0 +1,330 @@ +"""Create v2 reports and simulations through the routes and database-backed checks. + +The countries and policies services read a SQLite database with the v2 +tables, so the policy check runs the real catalog query; only the simulation +service is a stand-in, which a refused submission never reaches. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any +from uuid import UUID, uuid4 + +import pytest +from sqlalchemy import text +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool +from sqlmodel import Session, create_engine + +from policyengine_api.data.v2.models import ( + V2_METADATA, + Country, + Parameter, + ParameterValue, + Policy, + TaxBenefitModel, + TaxBenefitModelVersion, +) +from policyengine_api.services.v2.countries.database_session import ( + CountryDatabaseSession, +) +from policyengine_api.services.v2.countries.services import V2CountryService +from policyengine_api.services.v2.policies.database_session import ( + PolicyDatabaseSession, +) +from policyengine_api.services.v2.policies.services import V2PolicyService +from policyengine_api.services.v2.simulation_service.services import ( + SimulationSubmissionService, +) +from policyengine_api.services.v2.simulation_service.types import ( + EconomyReportCreationInput, + EconomySimulationCreationInput, + SubmittedReport, + SubmittedSimulation, +) +from policyengine_api.services.v2.stored_reports.database_session import ( + StoredReportDatabaseSession, +) +from policyengine_api.services.v2.stored_reports.services import V2StoredReportService +from policyengine_api.services.v2.stored_simulations.database_session import ( + StoredSimulationDatabaseSession, +) +from policyengine_api.services.v2.stored_simulations.services import ( + V2StoredSimulationService, +) +from tests.unit.v2.stored_report_support import CURRENT_VERSIONS, report_route_client + +if TYPE_CHECKING: + from collections.abc import Iterator + + from fastapi.testclient import TestClient + from sqlalchemy import Engine + +RUNNING_VERSION = "6.10.0" +EARLIER_VERSION = "6.9.0" +KEPT_PARAMETER = "gov.example.rate" +REMOVED_PARAMETER = "gov.example.removed_rate" +STORED_AT = datetime(2026, 11, 10, tzinfo=UTC) +REPORTS = "/v2/reports?country_id=us" +SIMULATIONS = "/v2/simulations?country_id=us" + + +@dataclass +class RecordingSimulationService: + """The simulation service's client, recording what it is sent.""" + + reports: list[EconomyReportCreationInput] = field(default_factory=list) + simulations: list[EconomySimulationCreationInput] = field(default_factory=list) + + def submit_report(self, report: EconomyReportCreationInput) -> SubmittedReport: + self.reports.append(report) + return SubmittedReport(report_id=uuid4(), report_run_id=uuid4()) + + def submit_simulation( + self, simulation: EconomySimulationCreationInput + ) -> SubmittedSimulation: + self.simulations.append(simulation) + return SubmittedSimulation(simulation_id=uuid4(), simulation_run_id=uuid4()) + + +@pytest.fixture +def engine() -> Iterator[Engine]: + # One connection shared by every thread, so the route's worker thread + # reads the same in-memory database. + database_engine = create_engine( + "sqlite://", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + V2_METADATA.create_all(database_engine) + yield database_engine + database_engine.dispose() + + +def store_catalog_version( + session: Session, model: TaxBenefitModel, version: str, names: list[str] +) -> dict[str, Parameter]: + """Store a catalog version of ``model`` with these parameters, keyed by name.""" + model_version = TaxBenefitModelVersion( + model=model, version=version, current_law_id=1, metadata_time_periods=[2026] + ) + parameters = { + name: Parameter(name=name, tax_benefit_model_version=model_version) + for name in names + } + session.add(model_version) + session.add_all(parameters.values()) + session.flush() + return parameters + + +def store_policy( + session: Session, parameters: dict[str, Parameter], content_hash_letter: str +) -> Policy: + """Store a US policy setting each of ``parameters``, of their catalog version.""" + model_version = next(iter(parameters.values())).tax_benefit_model_version + policy = Policy( + country_id="us", + tax_benefit_model_id=model_version.model_id, + tax_benefit_model_version_id=model_version.id, + canonicalization_version=1, + content_hash=content_hash_letter * 64, + created_at=STORED_AT, + updated_at=STORED_AT, + ) + session.add(policy) + session.flush() + session.add_all( + ParameterValue( + parameter_id=parameter.id, + value_json=0.2, + start_date=STORED_AT, + policy_id=policy.id, + ) + for parameter in parameters.values() + ) + return policy + + +@pytest.fixture +def stored_policy_ids(engine: Engine) -> dict[str, UUID]: + """Store the US with two catalog versions and three policies. + + ``outdated_policy`` sets a parameter the running catalog version removed; + ``kept_policy`` was made on the earlier version too, but sets only a + parameter the running version still has; ``running_policy`` was made on + the running version. + """ + with Session(engine, expire_on_commit=False) as session: + session.add(Country(id="us", household_enabled=True, society_wide_enabled=True)) + model = TaxBenefitModel(name="policyengine-us") + earlier = store_catalog_version( + session, model, EARLIER_VERSION, [KEPT_PARAMETER, REMOVED_PARAMETER] + ) + running = store_catalog_version( + session, model, RUNNING_VERSION, [KEPT_PARAMETER] + ) + policies = { + "outdated_policy": store_policy(session, earlier, "a"), + "kept_policy": store_policy( + session, {KEPT_PARAMETER: earlier[KEPT_PARAMETER]}, "b" + ), + "running_policy": store_policy(session, running, "c"), + } + session.commit() + return {name: policy.id for name, policy in policies.items()} + + +@pytest.fixture +def simulation_service() -> RecordingSimulationService: + return RecordingSimulationService() + + +@pytest.fixture +def submissions( + engine: Engine, simulation_service: RecordingSimulationService +) -> SimulationSubmissionService: + sessions = sessionmaker(engine, class_=Session, expire_on_commit=False) + return SimulationSubmissionService( + countries=V2CountryService(CountryDatabaseSession(sessions)), + policies=V2PolicyService( + PolicyDatabaseSession(sessions), + running_policyengine_version=RUNNING_VERSION, + ), + client_factory=lambda: simulation_service, + ) + + +@pytest.fixture +def client(engine: Engine, submissions: SimulationSubmissionService) -> TestClient: + sessions = sessionmaker(engine, class_=Session, expire_on_commit=False) + return report_route_client( + reports=V2StoredReportService( + StoredReportDatabaseSession(sessions), + current_versions=lambda _country_id: CURRENT_VERSIONS, + submissions=submissions, + ), + simulations=V2StoredSimulationService( + StoredSimulationDatabaseSession(sessions), + current_versions=lambda _country_id: CURRENT_VERSIONS, + submissions=submissions, + ), + ) + + +def report_body(**changes: Any) -> dict[str, Any]: + values: dict[str, Any] = { + "country_id": "us", + "kind": "economy", + "label": "Expanded child tax credit", + "year": 2026, + "region_code": "us", + "baseline": {}, + "reform": {}, + } + return {**values, **changes} + + +def simulation_body(policy_id: UUID) -> dict[str, Any]: + return { + "country_id": "us", + "kind": "economy", + "year": 2026, + "region_code": "us", + "policy_id": str(policy_id), + } + + +@pytest.mark.parametrize("side", ["baseline", "reform"]) +def test_a_report_whose_policy_sets_parameters_the_running_catalog_lacks_is_refused( + client: TestClient, + stored_policy_ids: dict[str, UUID], + simulation_service: RecordingSimulationService, + side: str, +) -> None: + outdated_policy_id = stored_policy_ids["outdated_policy"] + + response = client.post( + REPORTS, + json=report_body(**{side: {"policy_id": str(outdated_policy_id)}}), + ) + + assert response.status_code == 422, response.text + assert response.json() == { + "status": "error", + "message": ( + f"Policy {outdated_policy_id} sets parameters the running catalog " + f"version does not have: {REMOVED_PARAMETER}" + ), + "code": "missing_parameters", + "missing_parameters": [REMOVED_PARAMETER], + } + assert simulation_service.reports == [] + + +def test_a_simulation_whose_policy_sets_parameters_the_running_catalog_lacks_is_refused( + client: TestClient, + stored_policy_ids: dict[str, UUID], + simulation_service: RecordingSimulationService, +) -> None: + response = client.post( + SIMULATIONS, json=simulation_body(stored_policy_ids["outdated_policy"]) + ) + + assert response.status_code == 422, response.text + assert response.json()["code"] == "missing_parameters" + assert response.json()["missing_parameters"] == [REMOVED_PARAMETER] + assert simulation_service.simulations == [] + + +def test_policies_whose_parameters_all_exist_in_the_running_catalog_are_sent( + submissions: SimulationSubmissionService, + stored_policy_ids: dict[str, UUID], + simulation_service: RecordingSimulationService, +) -> None: + report = EconomyReportCreationInput.model_validate( + report_body( + baseline={"policy_id": str(stored_policy_ids["kept_policy"])}, + reform={"policy_id": str(stored_policy_ids["running_policy"])}, + ) + ) + simulation = EconomySimulationCreationInput.model_validate( + simulation_body(stored_policy_ids["kept_policy"]) + ) + + submissions.submit_report(report) + submissions.submit_simulation(simulation) + + assert simulation_service.reports == [report] + assert simulation_service.simulations == [simulation] + + +@pytest.mark.parametrize("record", ["report", "simulation"]) +def test_a_catalog_that_cannot_be_queried_answers_503( + engine: Engine, + client: TestClient, + stored_policy_ids: dict[str, UUID], + simulation_service: RecordingSimulationService, + record: str, +) -> None: + running_policy_id = stored_policy_ids["running_policy"] + # The policy check's catalog query reads this table; nothing else does. + with engine.begin() as connection: + connection.execute(text("DROP TABLE tax_benefit_model_versions")) + + if record == "report": + response = client.post( + REPORTS, json=report_body(reform={"policy_id": str(running_policy_id)}) + ) + else: + response = client.post(SIMULATIONS, json=simulation_body(running_policy_id)) + + assert response.status_code == 503 + assert response.json() == { + "status": "error", + "message": "V2 storage is unavailable", + } + assert simulation_service.reports == [] + assert simulation_service.simulations == [] diff --git a/tests/unit/v2/test_simulation_submissions.py b/tests/unit/v2/test_simulation_submissions.py index b6e22bd33..15a44f6b9 100644 --- a/tests/unit/v2/test_simulation_submissions.py +++ b/tests/unit/v2/test_simulation_submissions.py @@ -3,13 +3,22 @@ from __future__ import annotations from dataclasses import dataclass, field +from datetime import UTC, datetime from typing import Any from uuid import UUID, uuid4 import pytest +from sqlalchemy.exc import OperationalError +from policyengine_api.data.v2.catalog.catalog_selection_validation import ( + MetadataCatalogUnavailableError, +) from policyengine_api.services.v2.countries.types import CountryRead -from policyengine_api.services.v2.policies.validators import PolicyNotFoundError +from policyengine_api.services.v2.policies.types import CheckedPolicyRead +from policyengine_api.services.v2.policies.validators import ( + PolicyNotFoundError, + PolicyParametersMissingError, +) from policyengine_api.services.v2.simulation_service.services import ( SimulationSubmissionService, ) @@ -21,10 +30,27 @@ ) from policyengine_api.services.v2.simulation_service.validators import ( EconomyCalculationsUnsupportedError, + SubmissionStorageError, ) STORED_POLICY_ID = UUID("00000000-0000-0000-0000-00000000c001") OTHER_POLICY_ID = UUID("00000000-0000-0000-0000-00000000c002") +# Stored policies that set parameters the running catalog version lacks. +OUTDATED_POLICY_ID = UUID("00000000-0000-0000-0000-00000000c003") +SECOND_OUTDATED_POLICY_ID = UUID("00000000-0000-0000-0000-00000000c004") +# A stored policy the running catalog could not check, because the running +# PolicyEngine.py version has no published catalog for the country. +UNCHECKED_POLICY_ID = UUID("00000000-0000-0000-0000-00000000c005") +REMOVED_PARAMETERS = ("gov.example.removed_amount", "gov.example.removed_rate") +SECOND_REMOVED_PARAMETERS = ("gov.example.removed_threshold",) +# Each stored policy's parameters the running catalog version lacks, as the +# policy service checks them: empty when all exist, None when not checked. +MISSING_PARAMETERS: dict[UUID, tuple[str, ...] | None] = { + STORED_POLICY_ID: (), + OUTDATED_POLICY_ID: REMOVED_PARAMETERS, + SECOND_OUTDATED_POLICY_ID: SECOND_REMOVED_PARAMETERS, + UNCHECKED_POLICY_ID: None, +} SERVED_COUNTRIES = [ CountryRead(id="uk", household_enabled=True, society_wide_enabled=True), CountryRead(id="us", household_enabled=True, society_wide_enabled=True), @@ -44,18 +70,44 @@ def list_countries(self) -> list[CountryRead]: return self.countries +STORED_AT = datetime(2026, 11, 12, tzinfo=UTC) + + +def checked_policy( + policy_id: UUID, country_id: str, missing_parameters: tuple[str, ...] | None +) -> CheckedPolicyRead: + """Return a stored policy as the policy service checks it.""" + return CheckedPolicyRead( + id=policy_id, + country_id=country_id, + tax_benefit_model_id=uuid4(), + tax_benefit_model_version_id=uuid4(), + created_at=STORED_AT, + updated_at=STORED_AT, + parameter_values=(), + missing_parameters=missing_parameters, + ) + + @dataclass class StoredPolicies: - """The stored policies: only STORED_POLICY_ID, in the US and the UK.""" + """The policies in MISSING_PARAMETERS, stored in the US and the UK. + With ``error``, every read raises it, as a database or catalog that + cannot be read does. + """ + + error: Exception | None = None asked: list[tuple[str, UUID]] = field(default_factory=list) - def get_policy(self, *, country_id: str, policy_id: UUID) -> Any: + def get_policy(self, *, country_id: str, policy_id: UUID) -> CheckedPolicyRead: self.asked.append((country_id, policy_id)) - if policy_id != STORED_POLICY_ID: + if self.error is not None: + raise self.error + if policy_id not in MISSING_PARAMETERS: msg = f"Policy {policy_id} was not found" raise PolicyNotFoundError(msg) - return object() + return checked_policy(policy_id, country_id, MISSING_PARAMETERS[policy_id]) @dataclass @@ -195,3 +247,95 @@ def test_a_simulation_naming_a_policy_not_stored_is_refused() -> None: with pytest.raises(PolicyNotFoundError): service.submit_simulation(simulation_input(policy_id=str(OTHER_POLICY_ID))) + + +@pytest.mark.parametrize("side", ["baseline", "reform"]) +def test_a_report_whose_policy_sets_parameters_the_running_catalog_lacks_is_refused( + side: str, +) -> None: + service = submissions(unconfigured_client) + + with pytest.raises(PolicyParametersMissingError) as refusal: + service.submit_report( + report_input(**{side: {"policy_id": str(OUTDATED_POLICY_ID)}}) + ) + + assert refusal.value.policy_id == OUTDATED_POLICY_ID + assert refusal.value.missing_parameters == REMOVED_PARAMETERS + + +def test_a_simulation_whose_policy_sets_missing_parameters_is_refused() -> None: + service = submissions(unconfigured_client) + + with pytest.raises(PolicyParametersMissingError) as refusal: + service.submit_simulation(simulation_input(policy_id=str(OUTDATED_POLICY_ID))) + + assert refusal.value.policy_id == OUTDATED_POLICY_ID + assert refusal.value.missing_parameters == REMOVED_PARAMETERS + + +def test_a_report_with_two_outdated_policies_is_refused_for_its_baseline() -> None: + service = submissions(unconfigured_client) + + with pytest.raises(PolicyParametersMissingError) as refusal: + service.submit_report( + report_input( + baseline={"policy_id": str(OUTDATED_POLICY_ID)}, + reform={"policy_id": str(SECOND_OUTDATED_POLICY_ID)}, + ) + ) + + assert refusal.value.policy_id == OUTDATED_POLICY_ID + + +@pytest.mark.parametrize("outdated_side", ["baseline", "reform"]) +def test_a_policy_not_stored_is_refused_before_another_is_checked_for_parameters( + outdated_side: str, +) -> None: + # A policy that is not stored answers 404 whichever side names it, before + # the other side's missing parameters answer 422. + other_side = "reform" if outdated_side == "baseline" else "baseline" + service = submissions(unconfigured_client) + + with pytest.raises(PolicyNotFoundError, match=str(OTHER_POLICY_ID)): + service.submit_report( + report_input( + **{ + outdated_side: {"policy_id": str(OUTDATED_POLICY_ID)}, + other_side: {"policy_id": str(OTHER_POLICY_ID)}, + } + ) + ) + + +def test_a_policy_the_running_catalog_could_not_check_is_sent() -> None: + client = RecordingSimulationService() + + submissions(lambda: client).submit_report( + report_input(reform={"policy_id": str(UNCHECKED_POLICY_ID)}) + ) + submissions(lambda: client).submit_simulation( + simulation_input(policy_id=str(UNCHECKED_POLICY_ID)) + ) + + assert len(client.reports) == 1 + assert len(client.simulations) == 1 + + +@pytest.mark.parametrize( + "error", + [ + MetadataCatalogUnavailableError("the v2 metadata catalog cannot be queried"), + OperationalError("SELECT 1", {}, Exception("connection refused")), + ], + ids=["catalog", "database"], +) +def test_a_policy_check_that_cannot_read_storage_is_storage_unavailable( + error: Exception, +) -> None: + service = submissions(unconfigured_client, StoredPolicies(error=error)) + + with pytest.raises(SubmissionStorageError) as unavailable: + service.submit_report(report_input()) + + assert unavailable.value.__cause__ is error