From eb8e42f1fe69611a9004e0abca42d2621f57e2f2 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:01:58 +0400 Subject: [PATCH 01/57] Test household calculation inputs, axis limits and pre-submission checks (failing) The request model's axis limits, the country, default year and missing-parameter checks come before anything is sent to the simulation service. The modules they test do not exist yet. Co-Authored-By: Claude Opus 5.5 --- .../v2/test_household_calculation_inputs.py | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 tests/unit/v2/test_household_calculation_inputs.py diff --git a/tests/unit/v2/test_household_calculation_inputs.py b/tests/unit/v2/test_household_calculation_inputs.py new file mode 100644 index 000000000..166885113 --- /dev/null +++ b/tests/unit/v2/test_household_calculation_inputs.py @@ -0,0 +1,154 @@ +"""A household calculation's request fields, axis limits and pre-submission checks.""" + +from __future__ import annotations + +from uuid import UUID + +import pytest +from policyengine_api.services.v2.household_calculations.types import ( + MAXIMUM_AXIS_POINTS, + HouseholdCalculationInput, +) +from policyengine_api.services.v2.household_calculations.validators import ( + HouseholdCalculationCountryError, + HouseholdCalculationYearError, + require_default_year, + require_household_country, +) +from pydantic import ValidationError + +from policyengine_api.services.v2.countries.types import CountryRead +from policyengine_api.services.v2.policies.validators import ( + PolicyParametersMissingError, + require_policy_parameters_present, +) + +HOUSEHOLD_ID = UUID("00000000-0000-4000-8000-0000000000a1") +POLICY_ID = UUID("00000000-0000-4000-8000-0000000000b1") +COUNTRIES = ( + CountryRead(id="uk", household_enabled=True, society_wide_enabled=False), + CountryRead(id="us", household_enabled=True, society_wide_enabled=True), +) + + +def _axis(count: int, name: str = "employment_income") -> dict[str, object]: + return {"name": name, "count": count, "min": 0, "max": 100_000} + + +def _calculation(**changes: object) -> HouseholdCalculationInput: + fields: dict[str, object] = {"country_id": "us", "household_id": HOUSEHOLD_ID} + fields.update(changes) + return HouseholdCalculationInput.model_validate(fields) + + +def test_a_calculation_needs_only_a_country_and_a_household() -> None: + calculation = _calculation(country_id="US") + + assert calculation.country_id == "us" + assert calculation.policy_id is None + assert calculation.axes is None + assert calculation.add_missing_variables is False + assert calculation.spm is None + + +def test_axis_groups_whose_grid_reaches_the_point_limit_are_accepted() -> None: + calculation = _calculation( + axes=[[_axis(100), _axis(100, "self_employment_income")], [_axis(100)]] + ) + + assert calculation.axes is not None + assert [len(group) for group in calculation.axes] == [2, 1] + assert MAXIMUM_AXIS_POINTS == 100 * 100 + + +@pytest.mark.parametrize( + "axes", + [ + pytest.param([[_axis(3), _axis(4, "pension_income")]], id="uneven-group"), + pytest.param([[_axis(101)], [_axis(100)]], id="grid-over-the-limit"), + pytest.param([[_axis(MAXIMUM_AXIS_POINTS + 1)]], id="axis-over-the-limit"), + pytest.param([[]], id="empty-group"), + pytest.param([[_axis(2)]] * 9, id="too-many-groups"), + pytest.param([[_axis(0)]], id="no-points"), + ], +) +def test_axes_outside_the_simulation_service_limits_are_refused(axes) -> None: + with pytest.raises(ValidationError): + _calculation(axes=axes) + + +def test_an_axis_keeps_only_the_keys_its_caller_gave() -> None: + calculation = _calculation(axes=[[_axis(5)]]) + + assert calculation.axes is not None + (group,) = calculation.axes + (axis,) = group + assert axis.model_dump(mode="json", exclude_unset=True) == _axis(5) + + +@pytest.mark.parametrize( + "changes", + [ + pytest.param({"unknown": True}, id="unknown-field"), + pytest.param({"spm": None}, id="explicit-null-spm"), + pytest.param({"spm": {"geography_kind": "metro"}}, id="metro-without-id"), + pytest.param({"add_missing_variables": "sometimes"}, id="not-a-boolean"), + pytest.param({"country_id": "fr"}, id="unsupported-country"), + ], +) +def test_a_malformed_calculation_is_refused(changes) -> None: + with pytest.raises(ValidationError): + _calculation(**changes) + + +def test_spm_settings_keep_only_the_fields_the_caller_chose() -> None: + calculation = _calculation(spm={"scenario": "ce_trend"}) + + assert calculation.spm is not None + assert calculation.spm.model_dump(mode="json") == {"scenario": "ce_trend"} + + +def test_a_household_only_country_calculates_households() -> None: + require_household_country(COUNTRIES, "uk") + require_household_country(COUNTRIES, "us") + + with pytest.raises(HouseholdCalculationCountryError, match="uk"): + require_household_country( + ( + CountryRead( + id="uk", household_enabled=False, society_wide_enabled=False + ), + ), + "uk", + ) + + +def test_a_country_without_a_countries_row_calculates_no_households() -> None: + with pytest.raises(HouseholdCalculationCountryError, match="us"): + require_household_country(COUNTRIES[:1], "us") + + +def test_a_household_without_a_default_year_is_refused() -> None: + require_default_year(2026) + + with pytest.raises(HouseholdCalculationYearError, match="default year"): + require_default_year(None) + + +def test_a_policy_with_parameters_the_running_catalog_lacks_is_refused() -> None: + missing = ("gov.example.removed", "gov.example.renamed") + + with pytest.raises(PolicyParametersMissingError) as refused: + require_policy_parameters_present(POLICY_ID, missing) + + assert refused.value.policy_id == POLICY_ID + assert refused.value.missing_parameters == missing + assert str(refused.value) == ( + f"Policy {POLICY_ID} sets parameters the running catalog version does " + "not have: gov.example.removed, gov.example.renamed" + ) + + +@pytest.mark.parametrize("missing", [(), None], ids=["none-missing", "not-checked"]) +def test_a_policy_with_nothing_missing_or_unchecked_is_accepted(missing) -> None: + assert require_policy_parameters_present(POLICY_ID, missing) is None From 92182483af46333e3a196a92cf166bab671708e2 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:04:16 +0400 Subject: [PATCH 02/57] Add household calculation inputs, axis limits and pre-submission checks HouseholdCalculationInput takes a stored household and an optional stored policy, with the simulation service's axis limits, add_missing_variables and optional SPM settings. The validators refuse a country without household calculations and a household without a default year, and the policy validators refuse a policy that sets parameters the running catalog version lacks, a check later calculation routes can share. Co-Authored-By: Claude Opus 5.5 --- .../v2/household_calculations/__init__.py | 1 + .../database_connectors/__init__.py | 1 + .../v2/household_calculations/types.py | 104 ++++++++++++++++++ .../v2/household_calculations/validators.py | 44 ++++++++ .../services/v2/policies/validators.py | 36 ++++++ .../v2/test_household_calculation_inputs.py | 6 +- 6 files changed, 189 insertions(+), 3 deletions(-) create mode 100644 policyengine_api/services/v2/household_calculations/__init__.py create mode 100644 policyengine_api/services/v2/household_calculations/database_connectors/__init__.py create mode 100644 policyengine_api/services/v2/household_calculations/types.py create mode 100644 policyengine_api/services/v2/household_calculations/validators.py diff --git a/policyengine_api/services/v2/household_calculations/__init__.py b/policyengine_api/services/v2/household_calculations/__init__.py new file mode 100644 index 000000000..6246639a5 --- /dev/null +++ b/policyengine_api/services/v2/household_calculations/__init__.py @@ -0,0 +1 @@ +"""Application services for v2 household calculations.""" diff --git a/policyengine_api/services/v2/household_calculations/database_connectors/__init__.py b/policyengine_api/services/v2/household_calculations/database_connectors/__init__.py new file mode 100644 index 000000000..71f902a7a --- /dev/null +++ b/policyengine_api/services/v2/household_calculations/database_connectors/__init__.py @@ -0,0 +1 @@ +"""SQL-facing connector functions for v2 household calculations.""" diff --git a/policyengine_api/services/v2/household_calculations/types.py b/policyengine_api/services/v2/household_calculations/types.py new file mode 100644 index 000000000..5ce6a6051 --- /dev/null +++ b/policyengine_api/services/v2/household_calculations/types.py @@ -0,0 +1,104 @@ +"""Framework-independent data exchanged by the household calculation layers.""" + +from __future__ import annotations + +import math +from typing import Annotated, Any +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic.json_schema import SkipJsonSchema + +from policyengine_api.query_parameters import CountryId +from policyengine_api.spm import SPMSelection + +# The simulation service's limits on axes, as its household contract sets +# them: at most MAXIMUM_AXIS_GROUPS groups of at most MAXIMUM_PARALLEL_AXES +# axes each, and at most MAXIMUM_AXIS_POINTS points in the grid the groups +# form (the product of their point counts). +MAXIMUM_AXIS_POINTS = 10_000 +MAXIMUM_AXIS_GROUPS = 8 +MAXIMUM_PARALLEL_AXES = 32 + + +class StrictCalculationModel(BaseModel): + """Base for household calculation input: no unknown fields, finite numbers.""" + + model_config = ConfigDict(extra="forbid", allow_inf_nan=False, frozen=True) + + +class HouseholdCalculationAxis(StrictCalculationModel): + """One input varied across points. + + The variable ``name`` takes ``count`` evenly spaced values from ``min`` to + ``max`` in ``period`` (the household's ``default_year`` when omitted), for + the entity member at position ``index`` (the first when omitted). + """ + + name: Annotated[str, Field(min_length=1, max_length=255)] + count: Annotated[int, Field(ge=1, le=MAXIMUM_AXIS_POINTS)] + min: int | float + max: int | float + period: Annotated[str, Field(min_length=1, max_length=64)] | int | None = None + index: Annotated[int, Field(ge=0)] | None = None + + +AxisGroup = Annotated[ + list[HouseholdCalculationAxis], + Field(min_length=1, max_length=MAXIMUM_PARALLEL_AXES), +] + + +def _omit_spm_default(schema: dict[str, Any]) -> None: + # Omitting spm sends no SPM settings; an explicit null is not a selection. + schema.pop("default", None) + + +class HouseholdCalculationInput(StrictCalculationModel): + """A stored household to calculate under a stored policy, or current law. + + The calculation's year is the household's default year, the year its + values name. ``axes`` vary inputs across points: the axes of one group + vary together, so they share one point count, and the groups form a grid + of at most 10,000 points. ``add_missing_variables`` adds every yearly, + monthly and eternal variable the household lacks, as ``/calculate-full`` + does. ``spm`` selects the SPM forecast scenario and geography for this + calculation; omit it, rather than sending null, to send no settings. + """ + + country_id: CountryId + household_id: UUID + policy_id: UUID | None = None + axes: Annotated[list[AxisGroup], Field(max_length=MAXIMUM_AXIS_GROUPS)] | None = ( + None + ) + add_missing_variables: bool = False + spm: SPMSelection | SkipJsonSchema[None] = Field( + default=None, json_schema_extra=_omit_spm_default + ) + + @field_validator("axes") + @classmethod + def limit_axis_points( + cls, axes: list[list[HouseholdCalculationAxis]] | None + ) -> list[list[HouseholdCalculationAxis]] | None: + """Refuse groups of uneven point counts and grids of too many points.""" + if axes is None: + return None + for group in axes: + if len({axis.count for axis in group}) != 1: + msg = "the axes of one group must share one point count" + raise ValueError(msg) + if math.prod(group[0].count for group in axes) > MAXIMUM_AXIS_POINTS: + msg = f"the axis groups may vary at most {MAXIMUM_AXIS_POINTS} points" + raise ValueError(msg) + return axes + + @field_validator("spm", mode="before") + @classmethod + def require_spm_object(cls, value: object) -> object: + """Refuse an explicit null ``spm``; omit the field to send no settings.""" + if value is None: + msg = "spm must be an object when supplied" + raise ValueError(msg) + return value diff --git a/policyengine_api/services/v2/household_calculations/validators.py b/policyengine_api/services/v2/household_calculations/validators.py new file mode 100644 index 000000000..d2656934e --- /dev/null +++ b/policyengine_api/services/v2/household_calculations/validators.py @@ -0,0 +1,44 @@ +"""Checks of a household calculation against values already loaded.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterable + + from policyengine_api.services.v2.countries.types import CountryRead + + +class HouseholdCalculationCountryError(ValueError): + """The country does not offer household calculations.""" + + +class HouseholdCalculationYearError(ValueError): + """The household has no default year to calculate.""" + + +def require_household_country( + countries: Iterable[CountryRead], country_id: str +) -> None: + """Raise HouseholdCalculationCountryError unless the country calculates households. + + A household-only country (household calculations without society-wide + reports) passes; a country without a ``countries`` row does not. + """ + for country in countries: + if country.id == country_id and country.household_enabled: + return + msg = f"Household calculations are not available for country {country_id}." + raise HouseholdCalculationCountryError(msg) + + +def require_default_year(default_year: int | None) -> None: + """Raise HouseholdCalculationYearError unless the household has a default year. + + The simulation service calculates that year, the year the household's + values name. + """ + if default_year is None: + msg = "This household has no default year to calculate." + raise HouseholdCalculationYearError(msg) diff --git a/policyengine_api/services/v2/policies/validators.py b/policyengine_api/services/v2/policies/validators.py index 0f8b1d61a..05786fdb7 100644 --- a/policyengine_api/services/v2/policies/validators.py +++ b/policyengine_api/services/v2/policies/validators.py @@ -58,6 +58,42 @@ def __init__(self, message: str, *, reason: LegacyPolicyTranslationReason) -> No self.reason = reason +class PolicyParametersMissingError(ValueError): + """Raised when a policy sets parameters the running catalog version lacks. + + A calculation refuses such a policy before anything runs, rather than + calculate it without those values; ``missing_parameters`` lists them, + sorted. + """ + + def __init__(self, policy_id: UUID, missing_parameters: tuple[str, ...]) -> None: + """Name the policy and the parameters it sets that are missing.""" + self.policy_id = policy_id + self.missing_parameters = missing_parameters + # The arguments, in order, so copy and pickle can rebuild the error. + super().__init__(policy_id, missing_parameters) + + def __str__(self) -> str: + """Name the policy and each missing parameter.""" + return ( + f"Policy {self.policy_id} sets parameters the running catalog " + f"version does not have: {', '.join(self.missing_parameters)}" + ) + + +def require_policy_parameters_present( + policy_id: UUID, missing_parameters: tuple[str, ...] | None +) -> None: + """Raise PolicyParametersMissingError when the policy sets missing parameters. + + ``missing_parameters`` is a checked policy's (``CheckedPolicyRead``): None + when the running PolicyEngine.py version has no published catalog for the + country, so nothing could be checked, which passes. + """ + if missing_parameters: + raise PolicyParametersMissingError(policy_id, missing_parameters) + + def parameters_missing_from_model( policy_parameter_names: Iterable[str], model_parameter_names: Container[str], diff --git a/tests/unit/v2/test_household_calculation_inputs.py b/tests/unit/v2/test_household_calculation_inputs.py index 166885113..6a3e5864f 100644 --- a/tests/unit/v2/test_household_calculation_inputs.py +++ b/tests/unit/v2/test_household_calculation_inputs.py @@ -5,6 +5,9 @@ from uuid import UUID import pytest +from pydantic import ValidationError + +from policyengine_api.services.v2.countries.types import CountryRead from policyengine_api.services.v2.household_calculations.types import ( MAXIMUM_AXIS_POINTS, HouseholdCalculationInput, @@ -15,9 +18,6 @@ require_default_year, require_household_country, ) -from pydantic import ValidationError - -from policyengine_api.services.v2.countries.types import CountryRead from policyengine_api.services.v2.policies.validators import ( PolicyParametersMissingError, require_policy_parameters_present, From 0fe120c873d63d80f885343e89f809ba026601ae Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:08:03 +0400 Subject: [PATCH 03/57] Test reading household calculations from household_jobs (failing) A job's status, result document and error are read into a calculation: a succeeded job's household in its stored key order, with its warnings and SPM choice and receipt; a refused job's error; a job that failed unexpectedly by its error message. The integration tests read the jobs as the runtime role, through the service. The modules do not exist yet. Co-Authored-By: Claude Opus 5.5 --- .../household_calculation_documents.py | 103 ++++++++ .../test_v2_household_calculation_reads.py | 231 ++++++++++++++++++ .../v2/test_household_calculation_results.py | 208 ++++++++++++++++ 3 files changed, 542 insertions(+) create mode 100644 tests/fixtures/household_calculation_documents.py create mode 100644 tests/integration/test_v2_household_calculation_reads.py create mode 100644 tests/unit/v2/test_household_calculation_results.py diff --git a/tests/fixtures/household_calculation_documents.py b/tests/fixtures/household_calculation_documents.py new file mode 100644 index 000000000..4ab68b1af --- /dev/null +++ b/tests/fixtures/household_calculation_documents.py @@ -0,0 +1,103 @@ +"""Household job rows' result documents, as the v2 household functions write them. + +A succeeded job's ``result`` is the simulation contract's +``HouseholdExecutionResult``: the calculated household for its one policy, +with its warnings, SPM receipt and the SPM selection applied. A failed job's +is ``HouseholdExecutionFailure``, with a one-line summary in +``error_message``; a job that failed unexpectedly has a null ``result``. +""" + +from __future__ import annotations + +from typing import Any +from uuid import UUID + +JOB_ID = UUID("00000000-0000-4000-8000-0000000000c1") +WORKER = { + "application_name": "policyengine-v2-worker-6-10-0", + "function_name": "calculate_household_us", +} +BUNDLE = { + "policyengine_version": "6.10.0", + "core_package_version": "3.33.0", + "country_package_name": "policyengine-us", + "country_package_version": "1.450.0", + "bundle_manifest_sha256": "a" * 64, +} +SPM_CONFIG = { + "forecast_content_sha256": "3d" * 32, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, +} +SPM_PROVENANCE = {"schema_version": "canonical-spm-provenance-v2", "years": ["2026"]} +# Keys in the order the household function wrote them, which is not sorted: +# the household's people come before its groups, and each person's +# variables in the order the caller sent them. +CALCULATED_HOUSEHOLD = { + "people": { + "person-1": {"employment_income": {"2026": 30000}, "age": {"2026": 40}}, + }, + "households": { + "household-1": { + "members": ["person-1"], + "state_name": {"2026": "CA"}, + "household_net_income": {"2026": 27000.5}, + } + }, + "families": {"family-1": {"members": ["person-1"]}}, + "tax_units": {"tax_unit-1": {"members": ["person-1"]}}, + "spm_units": {"spm_unit-1": {"members": ["person-1"]}}, + "marital_units": {"marital_unit-1": {"members": ["person-1"]}}, +} +WARNING = "The input `age_group` is deprecated and was ignored." +INPUT_ERROR = { + "type": "household_variable", + "name": "not_a_variable", + "path": "people.person-1.not_a_variable", + "message": "not_a_variable is not a variable of policyengine-us.", +} +FAILURE_SUMMARY = "invalid_inputs: The household names a variable the model lacks." + + +def succeeded_document(**output_changes: object) -> dict[str, Any]: + """Return a succeeded job's result document.""" + output: dict[str, Any] = { + "country": "us", + "bundle": BUNDLE, + "spm_config": SPM_CONFIG, + "results": [ + { + "policy_key": "reform", + "household": CALCULATED_HOUSEHOLD, + "warnings": [WARNING], + "spm_provenance": SPM_PROVENANCE, + } + ], + } + output.update(output_changes) + return { + "status": "succeeded", + "schema_version": 1, + "calculation_id": str(JOB_ID), + "output": output, + "worker": WORKER, + } + + +def failed_document() -> dict[str, Any]: + """Return the result document of a job whose household function refused it.""" + return { + "status": "failed", + "schema_version": 1, + "calculation_id": str(JOB_ID), + "error": { + "code": "invalid_inputs", + "message": "The household names a variable the model lacks.", + "input_errors": [INPUT_ERROR], + "policy_key": "reform", + }, + "worker": WORKER, + } diff --git a/tests/integration/test_v2_household_calculation_reads.py b/tests/integration/test_v2_household_calculation_reads.py new file mode 100644 index 000000000..a7e53ab25 --- /dev/null +++ b/tests/integration/test_v2_household_calculation_reads.py @@ -0,0 +1,231 @@ +"""Read household calculations from ``household_jobs`` as the Public API's runtime role. + +The simulation service inserts the jobs and its Modal workers record their +results; the Public API only reads them, through the household calculation +service, as ``policyengine_v2_runtime``. Its declared privileges must cover +the read, so a statement they don't allow fails here as it would in staging +and production (rule ``api.v2.runtime-privileges-tested-as-runtime-role``). +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import TYPE_CHECKING +from uuid import UUID, uuid4 + +import pytest +from alembic import command +from alembic.config import Config +from policyengine_api.services.v2.household_calculations.database_session import ( + HouseholdCalculationDatabaseSession, +) +from policyengine_api.services.v2.household_calculations.services import ( + V2HouseholdCalculationService, +) +from sqlalchemy import create_engine, text +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import NullPool +from sqlmodel import Session + +from policyengine_api.constants import REPO +from policyengine_api.services.v2.countries.database_session import ( + CountryDatabaseSession, +) +from policyengine_api.services.v2.countries.services import V2CountryService +from policyengine_api.services.v2.household_calculations.validators import ( + HouseholdCalculationNotFoundError, +) +from policyengine_api.services.v2.households.database_session import ( + HouseholdDatabaseSession, +) +from policyengine_api.services.v2.households.services import V2HouseholdService +from policyengine_api.services.v2.policies.database_session import ( + PolicyDatabaseSession, +) +from policyengine_api.services.v2.policies.services import V2PolicyService +from tests.fixtures.household_calculation_documents import ( + CALCULATED_HOUSEHOLD, + FAILURE_SUMMARY, + INPUT_ERROR, + SPM_CONFIG, + SPM_PROVENANCE, + WARNING, + failed_document, + succeeded_document, +) +from tests.integration.simulation_rows import ( + RUN_VERSIONS, + PublicApiRows, + delete_public_api_rows, + insert_public_api_rows, + insert_row, +) + +if TYPE_CHECKING: + from collections.abc import Iterator + +COMPLETED_AT = datetime(2026, 11, 10, 12, 0, 9, tzinfo=UTC) + + +@dataclass(frozen=True) +class StoredJobs: + """The jobs stored for one test, by what happened to them.""" + + public_api_rows: PublicApiRows + succeeded: UUID + refused: UUID + failed_unexpectedly: UUID + pending: UUID + + +def _insert_job( + connection, rows: PublicApiRows, job_id: UUID, status: str, **columns: object +) -> None: + insert_row( + connection, + "household_jobs", + id=job_id, + country="us", + household_id=rows.household_id, + year=2026, + request_data="{}", + tax_benefit_model_version_id=rows.version_id, + status=status, + **RUN_VERSIONS, + **columns, + ) + + +@pytest.fixture +def stored_jobs(disposable_v2_database_url, managed_v2_roles) -> Iterator[StoredJobs]: + """Commit, as the superuser, one job of each outcome; delete them afterwards. + + The service reads on connections of its own, so the rows are committed + rather than left in a transaction the test rolls back. + """ + command.upgrade(Config(str(REPO / "alembic-v2.ini")), "head") + engine = create_engine(disposable_v2_database_url, poolclass=NullPool) + with engine.begin() as connection: + rows = insert_public_api_rows(connection) + jobs = StoredJobs( + public_api_rows=rows, + succeeded=uuid4(), + refused=uuid4(), + failed_unexpectedly=uuid4(), + pending=uuid4(), + ) + _insert_job( + connection, + rows, + jobs.succeeded, + "succeeded", + # A json column keeps the document's text, so its key order too. + result=json.dumps(succeeded_document()), + completed_at=COMPLETED_AT, + ) + _insert_job( + connection, + rows, + jobs.refused, + "failed", + result=json.dumps(failed_document()), + error_message=FAILURE_SUMMARY, + completed_at=COMPLETED_AT, + ) + _insert_job( + connection, + rows, + jobs.failed_unexpectedly, + "failed", + error_message="The Modal call ended.", + completed_at=COMPLETED_AT, + ) + _insert_job(connection, rows, jobs.pending, "pending") + try: + yield jobs + finally: + with engine.begin() as connection: + connection.execute( + text("DELETE FROM household_jobs WHERE household_id = :household_id"), + {"household_id": rows.household_id}, + ) + delete_public_api_rows(connection, rows) + engine.dispose() + + +def _no_simulation_service(): + msg = "reading a household calculation never calls the simulation service" + raise AssertionError(msg) + + +@pytest.fixture +def runtime_service(managed_v2_roles, v2_engine_as_role): + """Return the household calculation service, reading as the runtime role.""" + runtime_engine = v2_engine_as_role(managed_v2_roles.runtime, poolclass=NullPool) + sessions = sessionmaker(runtime_engine, class_=Session) + yield V2HouseholdCalculationService( + countries=V2CountryService(CountryDatabaseSession(sessions)), + households=V2HouseholdService(HouseholdDatabaseSession(sessions)), + policies=V2PolicyService( + PolicyDatabaseSession(sessions), running_policyengine_version="6.10.0" + ), + database_session=HouseholdCalculationDatabaseSession(sessions), + simulation_factory=_no_simulation_service, + ) + runtime_engine.dispose() + + +def test_the_runtime_role_reads_a_succeeded_calculation_in_its_stored_order( + stored_jobs, runtime_service +) -> None: + calculation = runtime_service.get_calculation(stored_jobs.succeeded) + + assert calculation.calculation_id == stored_jobs.succeeded + assert calculation.household_id == stored_jobs.public_api_rows.household_id + assert calculation.catalog_version_id == stored_jobs.public_api_rows.version_id + assert calculation.policy_id is None + assert calculation.status == "succeeded" + assert calculation.completed_at == COMPLETED_AT + assert calculation.error is None + assert calculation.result is not None + assert json.dumps(calculation.result.household) == json.dumps(CALCULATED_HOUSEHOLD) + assert calculation.result.warnings == (WARNING,) + assert calculation.result.spm_config == SPM_CONFIG + assert calculation.result.spm_provenance == SPM_PROVENANCE + + +def test_the_runtime_role_reads_why_a_calculation_failed( + stored_jobs, runtime_service +) -> None: + refused = runtime_service.get_calculation(stored_jobs.refused) + failed_unexpectedly = runtime_service.get_calculation( + stored_jobs.failed_unexpectedly + ) + + assert refused.status == "failed" + assert refused.result is None + assert refused.error is not None + assert refused.error.code == "invalid_inputs" + assert refused.error.input_errors == (INPUT_ERROR,) + assert failed_unexpectedly.error is not None + assert failed_unexpectedly.error.code is None + assert failed_unexpectedly.error.message == "The Modal call ended." + + +def test_the_runtime_role_reads_an_unfinished_calculation( + stored_jobs, runtime_service +) -> None: + calculation = runtime_service.get_calculation(stored_jobs.pending) + + assert calculation.status == "pending" + assert calculation.started_at is None + assert calculation.completed_at is None + assert calculation.result is None + assert calculation.error is None + + +def test_an_unknown_calculation_is_not_found(stored_jobs, runtime_service) -> None: + with pytest.raises(HouseholdCalculationNotFoundError): + runtime_service.get_calculation(uuid4()) diff --git a/tests/unit/v2/test_household_calculation_results.py b/tests/unit/v2/test_household_calculation_results.py new file mode 100644 index 000000000..9af4dfcf4 --- /dev/null +++ b/tests/unit/v2/test_household_calculation_results.py @@ -0,0 +1,208 @@ +"""Read a household job's status, result document and error into a calculation.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from uuid import UUID + +import pytest +from policyengine_api.services.v2.household_calculations.transformations import ( + household_calculation_read, +) + +from policyengine_api.data.v2.models import HouseholdJob, HouseholdJobStatus +from policyengine_api.services.v2.household_calculations.validators import ( + HouseholdCalculationResultUnreadableError, + stored_household_outcome, +) +from tests.fixtures.household_calculation_documents import ( + CALCULATED_HOUSEHOLD, + FAILURE_SUMMARY, + INPUT_ERROR, + JOB_ID, + SPM_CONFIG, + SPM_PROVENANCE, + WARNING, + failed_document, + succeeded_document, +) + +HOUSEHOLD_ID = UUID("00000000-0000-4000-8000-0000000000a1") +POLICY_ID = UUID("00000000-0000-4000-8000-0000000000b1") +CATALOG_VERSION_ID = UUID("00000000-0000-4000-8000-0000000000e1") +CREATED_AT = datetime(2026, 11, 10, 12, 0, tzinfo=UTC) +STARTED_AT = datetime(2026, 11, 10, 12, 0, 5, tzinfo=UTC) +COMPLETED_AT = datetime(2026, 11, 10, 12, 0, 9, tzinfo=UTC) + + +def _job( + status: HouseholdJobStatus, *, result: object = None, **changes: object +) -> HouseholdJob: + finished = status in {HouseholdJobStatus.SUCCEEDED, HouseholdJobStatus.FAILED} + fields: dict[str, object] = { + "id": JOB_ID, + "created_at": CREATED_AT, + "updated_at": COMPLETED_AT, + "country": "us", + "household_id": HOUSEHOLD_ID, + "year": 2026, + "request_data": {}, + "policy_id": POLICY_ID, + "tax_benefit_model_version_id": CATALOG_VERSION_ID, + "country_package_version": "1.450.0", + "policyengine_version": "6.10.0", + "status": status, + "result": result, + "started_at": None if status is HouseholdJobStatus.PENDING else STARTED_AT, + "completed_at": COMPLETED_AT if finished else None, + } + fields.update(changes) + return HouseholdJob(**fields) + + +def _read(job: HouseholdJob): + return household_calculation_read( + job, stored_household_outcome(job.id, job.status, job.result) + ) + + +def test_a_succeeded_job_reads_back_its_household_warnings_and_spm() -> None: + calculation = _read(_job(HouseholdJobStatus.SUCCEEDED, result=succeeded_document())) + + assert calculation.calculation_id == JOB_ID + assert calculation.country_id == "us" + assert calculation.household_id == HOUSEHOLD_ID + assert calculation.policy_id == POLICY_ID + assert calculation.year == 2026 + assert calculation.catalog_version_id == CATALOG_VERSION_ID + assert calculation.country_package_version == "1.450.0" + assert calculation.policyengine_version == "6.10.0" + assert calculation.status == "succeeded" + assert (calculation.created_at, calculation.started_at) == (CREATED_AT, STARTED_AT) + assert calculation.completed_at == COMPLETED_AT + assert calculation.error is None + assert calculation.result is not None + assert calculation.result.household == CALCULATED_HOUSEHOLD + assert calculation.result.warnings == (WARNING,) + assert calculation.result.spm_config == SPM_CONFIG + assert calculation.result.spm_provenance == SPM_PROVENANCE + + +def test_a_calculated_household_keeps_the_key_order_it_was_stored_in() -> None: + calculation = _read(_job(HouseholdJobStatus.SUCCEEDED, result=succeeded_document())) + + assert calculation.result is not None + household = calculation.result.household + assert list(household) == list(CALCULATED_HOUSEHOLD) + assert list(household["people"]["person-1"]) == ["employment_income", "age"] + + +def test_a_country_without_spm_reads_back_no_spm_choice_or_receipt() -> None: + document = succeeded_document( + country="uk", + spm_config=None, + results=[{"policy_key": "baseline", "household": CALCULATED_HOUSEHOLD}], + ) + + calculation = _read(_job(HouseholdJobStatus.SUCCEEDED, result=document)) + + assert calculation.result is not None + assert calculation.result.warnings == () + assert calculation.result.spm_config is None + assert calculation.result.spm_provenance is None + + +def test_a_job_the_household_function_refused_reads_back_its_error() -> None: + calculation = _read( + _job( + HouseholdJobStatus.FAILED, + result=failed_document(), + error_message=FAILURE_SUMMARY, + ) + ) + + assert calculation.status == "failed" + assert calculation.result is None + assert calculation.error is not None + assert calculation.error.code == "invalid_inputs" + assert calculation.error.message == ( + "The household names a variable the model lacks." + ) + assert calculation.error.input_errors == (INPUT_ERROR,) + assert calculation.error.spm_error is None + + +def test_a_job_that_failed_unexpectedly_reads_back_its_error_message() -> None: + calculation = _read( + _job(HouseholdJobStatus.FAILED, error_message="The Modal call ended.") + ) + + assert calculation.result is None + assert calculation.error is not None + assert calculation.error.code is None + assert calculation.error.message == "The Modal call ended." + assert calculation.error.input_errors == () + assert calculation.error.spm_error is None + + +@pytest.mark.parametrize( + "document", [None, {"status": "failed"}, {"unexpected": True}, ["not", "a", "map"]] +) +def test_a_failed_job_without_a_readable_failure_says_it_failed(document) -> None: + calculation = _read(_job(HouseholdJobStatus.FAILED, result=document)) + + assert calculation.error is not None + assert calculation.error.code is None + assert calculation.error.message == "The household calculation failed." + + +@pytest.mark.parametrize( + "status", [HouseholdJobStatus.PENDING, HouseholdJobStatus.RUNNING] +) +def test_an_unfinished_job_reads_back_no_result_or_error(status) -> None: + calculation = _read( + _job(status, result=succeeded_document(), error_message="not yet") + ) + + assert calculation.status == status.value + assert calculation.completed_at is None + assert calculation.result is None + assert calculation.error is None + + +def test_a_document_with_fields_the_public_api_does_not_read_is_still_read() -> None: + document = succeeded_document(added_by_a_later_contract={"any": "value"}) + document["added_at_the_top"] = True + + calculation = _read(_job(HouseholdJobStatus.SUCCEEDED, result=document)) + + assert calculation.result is not None + assert calculation.result.household == CALCULATED_HOUSEHOLD + + +@pytest.mark.parametrize( + "document", + [ + pytest.param(None, id="no-document"), + pytest.param(failed_document(), id="a-failure-document"), + pytest.param(succeeded_document(results=[]), id="no-result"), + pytest.param( + succeeded_document( + results=[ + {"policy_key": "baseline", "household": CALCULATED_HOUSEHOLD}, + {"policy_key": "reform", "household": CALCULATED_HOUSEHOLD}, + ] + ), + id="two-results", + ), + pytest.param( + succeeded_document( + results=[{"policy_key": "reform", "household": "not a household"}] + ), + id="malformed-household", + ), + ], +) +def test_a_succeeded_job_without_one_readable_result_is_unreadable(document) -> None: + with pytest.raises(HouseholdCalculationResultUnreadableError, match=str(JOB_ID)): + _read(_job(HouseholdJobStatus.SUCCEEDED, result=document)) From dba90dab0588efcedcb96b8752a9d72f90efbc4a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:12:08 +0400 Subject: [PATCH 04/57] Read household calculations from household_jobs through the v2 layers GET's read path: a read connector loads one household job by ID, a database session module opens the read sessions, a validator reads the job's result document as its status says, and a transformation builds the calculation from the job. A succeeded job must hold one policy result, which keeps the key order the json column stored; a refused job gives the household function's error, and one that failed without a failure document its error message. Database failures become a typed storage error, so the routes need not import SQLAlchemy. The cross-database integration check runs the read as the runtime role. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/v2-integration-check.yml | 4 +- docs/engineering/skills/alembic-migrations.md | 5 +- .../database_connectors/reads.py | 17 ++ .../database_session.py | 30 +++ .../v2/household_calculations/services.py | 93 ++++++++ .../household_calculations/transformations.py | 120 ++++++++++ .../v2/household_calculations/types.py | 213 +++++++++++++++++- .../v2/household_calculations/validators.py | 46 ++++ .../test_v2_household_calculation_reads.py | 12 +- tests/unit/test_alembic_workflows.py | 1 + .../v2/test_household_calculation_results.py | 4 +- 11 files changed, 532 insertions(+), 13 deletions(-) create mode 100644 policyengine_api/services/v2/household_calculations/database_connectors/reads.py create mode 100644 policyengine_api/services/v2/household_calculations/database_session.py create mode 100644 policyengine_api/services/v2/household_calculations/services.py create mode 100644 policyengine_api/services/v2/household_calculations/transformations.py diff --git a/.github/workflows/v2-integration-check.yml b/.github/workflows/v2-integration-check.yml index 5d56e9214..581b346a5 100644 --- a/.github/workflows/v2-integration-check.yml +++ b/.github/workflows/v2-integration-check.yml @@ -81,8 +81,8 @@ jobs: run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_user_identity_persistence.py - name: Test comparison-run persistence, the simulation API's roles and privilege reconciliation as the managed roles run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_stage12_comparison_persistence.py tests/integration/test_v2_simulation_role_privileges.py tests/integration/test_v2_privilege_reconciliation.py - - name: Test v2 household persistence and immediate v1 copying - run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_household_persistence.py tests/integration/test_v1_household_dual_write.py tests/integration/test_mysql_household_spm_dual_write.py + - name: Test v2 household persistence, immediate v1 copying and household calculation reads + run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_household_persistence.py tests/integration/test_v1_household_dual_write.py tests/integration/test_mysql_household_spm_dual_write.py tests/integration/test_v2_household_calculation_reads.py - name: Test the row counts v1 ORM updates report on MySQL run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_mysql_update_row_counts.py # The real US and UK models, reduced to a few parameter folders: the diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index 5aab0e4e5..763af1373 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -275,7 +275,9 @@ check the declarations: `policyengine_stage12_test`, holding only their declared privileges. `tests/integration/test_v2_simulation_role_privileges.py` runs the simulation API's writes as its two roles; their code lives in the simulation - API's repository. + API's repository. `tests/integration/test_v2_household_calculation_reads.py` + reads the household jobs they write through the household calculation + service as `policyengine_v2_runtime`. - `api.v2.deployed-privileges-match`: every deploy runs `scripts/reconcile_v2_privileges.py apply` right after the schema upgrade, then `verify`, before the catalog is seeded. @@ -321,6 +323,7 @@ When a pull request adds or changes a v2 table, or a connector's use of one: tests/integration/test_v2_catalog_publication.py \ tests/integration/test_v2_catalog_publication_pruning.py \ tests/integration/test_v2_simulation_role_privileges.py \ + tests/integration/test_v2_household_calculation_reads.py \ tests/integration/test_v2_privilege_reconciliation.py ``` diff --git a/policyengine_api/services/v2/household_calculations/database_connectors/reads.py b/policyengine_api/services/v2/household_calculations/database_connectors/reads.py new file mode 100644 index 000000000..0f1a9e6b6 --- /dev/null +++ b/policyengine_api/services/v2/household_calculations/database_connectors/reads.py @@ -0,0 +1,17 @@ +"""Database selections of household jobs, which the simulation service writes.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from policyengine_api.data.v2.models import HouseholdJob + +if TYPE_CHECKING: + from uuid import UUID + + from sqlmodel import Session + + +def read_household_job(session: Session, job_id: UUID) -> HouseholdJob | None: + """Return the household job with this ID, or None.""" + return session.get(HouseholdJob, job_id) diff --git a/policyengine_api/services/v2/household_calculations/database_session.py b/policyengine_api/services/v2/household_calculations/database_session.py new file mode 100644 index 000000000..621c0c4fb --- /dev/null +++ b/policyengine_api/services/v2/household_calculations/database_session.py @@ -0,0 +1,30 @@ +"""Database session lifetime management for v2 household calculations.""" + +from __future__ import annotations + +from contextlib import contextmanager +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sqlalchemy.orm import sessionmaker + from sqlmodel import Session + + +class HouseholdCalculationDatabaseSession: + """Open the read sessions the household calculation service uses. + + The Public API only reads household jobs; the simulation service writes + them. + """ + + def __init__(self, session_factory: sessionmaker[Session]) -> None: + """Use ``session_factory`` to open every session.""" + self._session_factory = session_factory + + @contextmanager + def read(self) -> Iterator[Session]: + """Yield a session for reads; nothing is committed.""" + with self._session_factory() as session: + yield session diff --git a/policyengine_api/services/v2/household_calculations/services.py b/policyengine_api/services/v2/household_calculations/services.py new file mode 100644 index 000000000..d777ec509 --- /dev/null +++ b/policyengine_api/services/v2/household_calculations/services.py @@ -0,0 +1,93 @@ +"""Calculate stored households through the simulation service, and read the results. + +Reading a calculation reads its job from ``household_jobs``, which the +simulation service inserts and its household function completes; it never +calls the simulation service. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from typing import TYPE_CHECKING + +from sqlalchemy.exc import SQLAlchemyError + +from policyengine_api.services.v2.household_calculations.database_connectors import ( + reads as household_job_reads, +) +from policyengine_api.services.v2.household_calculations.transformations import ( + household_calculation_read, +) +from policyengine_api.services.v2.household_calculations.validators import ( + HouseholdCalculationNotFoundError, + HouseholdCalculationStorageError, + stored_household_outcome, +) + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator + from uuid import UUID + + from policyengine_api.services.v2.household_calculations.database_session import ( + HouseholdCalculationDatabaseSession, + ) + from policyengine_api.services.v2.household_calculations.types import ( + CountrySource, + HouseholdCalculationRead, + HouseholdCalculationSubmitter, + HouseholdSource, + PolicySource, + ) + + +@contextmanager +def _storage_failures() -> Iterator[None]: + """Raise HouseholdCalculationStorageError for a failed database read.""" + try: + yield + except SQLAlchemyError as error: + msg = "V2 household calculation storage is unavailable" + raise HouseholdCalculationStorageError(msg) from error + + +class V2HouseholdCalculationService: + """Submit household calculations to the simulation service and read them.""" + + def __init__( + self, + *, + countries: CountrySource, + households: HouseholdSource, + policies: PolicySource, + database_session: HouseholdCalculationDatabaseSession, + simulation_factory: Callable[[], HouseholdCalculationSubmitter], + ) -> None: + """Read through these services and sessions for one request. + + ``simulation_factory`` returns the simulation service's client only + once a submission is sent to it, so a request the Public API refuses + gets its own answer even when the simulation service is not + configured, and a read never needs it. + """ + self._countries = countries + self._households = households + self._policies = policies + self._database_session = database_session + self._simulation_factory = simulation_factory + + def get_calculation(self, calculation_id: UUID) -> HouseholdCalculationRead: + """Return the calculation household_jobs records. + + Raise HouseholdCalculationNotFoundError when there is no such job, + HouseholdCalculationResultUnreadableError when a succeeded job's + result cannot be read, and HouseholdCalculationStorageError when the + database cannot be read. + """ + with _storage_failures(), self._database_session.read() as session: + job = household_job_reads.read_household_job(session, calculation_id) + if job is None: + msg = f"Household calculation {calculation_id} not found." + raise HouseholdCalculationNotFoundError(msg) + return household_calculation_read( + job, stored_household_outcome(job.id, job.status, job.result) + ) diff --git a/policyengine_api/services/v2/household_calculations/transformations.py b/policyengine_api/services/v2/household_calculations/transformations.py new file mode 100644 index 000000000..f5ddf0c5a --- /dev/null +++ b/policyengine_api/services/v2/household_calculations/transformations.py @@ -0,0 +1,120 @@ +"""Conversions of household jobs and simulation answers into calculation results.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from policyengine_api.data.v2.models import HouseholdJobStatus +from policyengine_api.services.v2.household_calculations.types import ( + HouseholdCalculationFailure, + HouseholdCalculationOutput, + HouseholdCalculationRead, + HouseholdCalculationSubmitted, + StoredExecutionFailure, + StoredExecutionResult, +) + +if TYPE_CHECKING: + from policyengine_api.data.v2.models import HouseholdJob + from policyengine_api.services.v2.household_calculations.types import ( + HouseholdCalculationAccepted, + HouseholdCalculationInput, + HouseholdJobStatusName, + ) + +# The error of a failed job that records neither a failure document nor an +# error message. +UNEXPLAINED_FAILURE = "The household calculation failed." +STATUS_NAMES: dict[HouseholdJobStatus, HouseholdJobStatusName] = { + HouseholdJobStatus.PENDING: "pending", + HouseholdJobStatus.RUNNING: "running", + HouseholdJobStatus.SUCCEEDED: "succeeded", + HouseholdJobStatus.FAILED: "failed", +} + + +def _calculation_output(document: StoredExecutionResult) -> HouseholdCalculationOutput: + (policy_result,) = document.output.results + return HouseholdCalculationOutput( + household=policy_result.household, + warnings=tuple(policy_result.warnings), + spm_config=document.output.spm_config, + spm_provenance=policy_result.spm_provenance, + ) + + +def _calculation_failure( + document: StoredExecutionFailure | None, error_message: str | None +) -> HouseholdCalculationFailure: + if document is None: + return HouseholdCalculationFailure( + code=None, + message=error_message or UNEXPLAINED_FAILURE, + input_errors=(), + spm_error=None, + ) + error = document.error + return HouseholdCalculationFailure( + code=error.code, + message=error.message, + input_errors=tuple(error.input_errors), + spm_error=error.spm_error, + ) + + +def household_calculation_read( + job: HouseholdJob, + outcome: StoredExecutionResult | StoredExecutionFailure | None, +) -> HouseholdCalculationRead: + """Return the calculation a household job records. + + ``outcome`` is the job's result document, already read as its status says + (``stored_household_outcome``). A succeeded job's result is its one policy + result; a failed job's error is its failure document's error, or else its + error message. + """ + status = STATUS_NAMES[HouseholdJobStatus(job.status)] + return HouseholdCalculationRead( + calculation_id=job.id, + country_id=job.country, + household_id=job.household_id, + policy_id=job.policy_id, + year=job.year, + catalog_version_id=job.tax_benefit_model_version_id, + country_package_version=job.country_package_version, + policyengine_version=job.policyengine_version, + status=status, + created_at=job.created_at, + started_at=job.started_at, + completed_at=job.completed_at, + result=( + _calculation_output(outcome) + if status == "succeeded" and isinstance(outcome, StoredExecutionResult) + else None + ), + error=( + _calculation_failure( + outcome if isinstance(outcome, StoredExecutionFailure) else None, + job.error_message, + ) + if status == "failed" + else None + ), + ) + + +def household_calculation_submitted( + calculation: HouseholdCalculationInput, + accepted: HouseholdCalculationAccepted, + *, + year: int, +) -> HouseholdCalculationSubmitted: + """Return the accepted calculation with what it calculates.""" + return HouseholdCalculationSubmitted( + calculation_id=accepted.calculation_id, + status=accepted.status, + country_id=calculation.country_id, + household_id=calculation.household_id, + policy_id=calculation.policy_id, + year=year, + ) diff --git a/policyengine_api/services/v2/household_calculations/types.py b/policyengine_api/services/v2/household_calculations/types.py index 5ce6a6051..528f37f8d 100644 --- a/policyengine_api/services/v2/household_calculations/types.py +++ b/policyengine_api/services/v2/household_calculations/types.py @@ -3,15 +3,23 @@ from __future__ import annotations import math -from typing import Annotated, Any +from dataclasses import dataclass +from typing import TYPE_CHECKING, Annotated, Any, Literal, Protocol from uuid import UUID -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator from pydantic.json_schema import SkipJsonSchema from policyengine_api.query_parameters import CountryId from policyengine_api.spm import SPMSelection +if TYPE_CHECKING: + from datetime import datetime + + from policyengine_api.services.v2.countries.types import CountryRead + from policyengine_api.services.v2.households.types import HouseholdRead + from policyengine_api.services.v2.policies.types import CheckedPolicyRead + # The simulation service's limits on axes, as its household contract sets # them: at most MAXIMUM_AXIS_GROUPS groups of at most MAXIMUM_PARALLEL_AXES # axes each, and at most MAXIMUM_AXIS_POINTS points in the grid the groups @@ -20,6 +28,11 @@ MAXIMUM_AXIS_GROUPS = 8 MAXIMUM_PARALLEL_AXES = 32 +# A household job's status, as household_jobs stores it: the simulation +# service inserts the job pending, its household function sets it running, +# then succeeded or failed. +HouseholdJobStatusName = Literal["pending", "running", "succeeded", "failed"] + class StrictCalculationModel(BaseModel): """Base for household calculation input: no unknown fields, finite numbers.""" @@ -102,3 +115,199 @@ def require_spm_object(cls, value: object) -> object: msg = "spm must be an object when supplied" raise ValueError(msg) return value + + +@dataclass(frozen=True) +class HouseholdCalculationAccepted: + """The simulation service's answer to a submission: the new job and its status.""" + + calculation_id: UUID + status: HouseholdJobStatusName + + +@dataclass(frozen=True) +class HouseholdCalculationSubmitted: + """A calculation the simulation service accepted, and what it calculates. + + ``year`` is the household's default year, which the simulation service + calculates. + """ + + calculation_id: UUID + status: HouseholdJobStatusName + country_id: str + household_id: UUID + policy_id: UUID | None + year: int + + +@dataclass(frozen=True) +class HouseholdCalculationOutput: + """A succeeded calculation's household and its notes. + + ``household`` is the stored household in the situation shape, its people + and groups named by their record IDs, with each requested value filled + in, in the order the household function wrote it. ``warnings`` lists the + inputs the country model ignored. ``spm_config`` is the SPM selection the + calculation applied and ``spm_provenance`` its receipt, both None for a + country without SPM. + """ + + household: dict[str, Any] + warnings: tuple[str, ...] + spm_config: dict[str, Any] | None + spm_provenance: dict[str, Any] | None + + +@dataclass(frozen=True) +class HouseholdCalculationFailure: + """Why a calculation failed. + + ``code`` is the household function's error code, such as + ``invalid_inputs``, with ``input_errors`` naming each input the country + model does not recognize and ``spm_error`` an SPM error. A job that failed + without the household function's failure, such as one whose Modal call + ended, has no code, and ``message`` is the job's error message. + """ + + code: str | None + message: str + input_errors: tuple[dict[str, Any], ...] + spm_error: dict[str, Any] | None + + +@dataclass(frozen=True) +class HouseholdCalculationRead: + """A household calculation as household_jobs records it. + + ``catalog_version_id`` and the package versions are those the simulation + service ran it with. ``result`` is set once it succeeded and ``error`` + once it failed. + """ + + calculation_id: UUID + country_id: str + household_id: UUID + policy_id: UUID | None + year: int + catalog_version_id: UUID + country_package_version: str + policyengine_version: str + status: HouseholdJobStatusName + created_at: datetime + started_at: datetime | None + completed_at: datetime | None + result: HouseholdCalculationOutput | None + error: HouseholdCalculationFailure | None + + +class StoredDocumentModel(BaseModel): + """Base for the result documents the household functions store. + + The simulation service owns their shape, so fields the Public API does + not read are ignored, and a later contract may add some. + """ + + model_config = ConfigDict(extra="ignore", frozen=True) + + +class StoredPolicyResult(StoredDocumentModel): + """The calculated household under the job's one policy.""" + + household: dict[str, JsonValue] + warnings: list[str] = Field(default_factory=list) + spm_provenance: dict[str, JsonValue] | None = None + + +class StoredCalculationOutput(StoredDocumentModel): + """A finished calculation: one result, for the job's one policy.""" + + spm_config: dict[str, JsonValue] | None = None + results: Annotated[list[StoredPolicyResult], Field(min_length=1, max_length=1)] + + +class StoredExecutionResult(StoredDocumentModel): + """A succeeded job's result document (``HouseholdExecutionResult``).""" + + status: Literal["succeeded"] + output: StoredCalculationOutput + + +class StoredCalculationError(StoredDocumentModel): + """The household function's error, worded as the Public API words it.""" + + code: str + message: str + input_errors: list[dict[str, JsonValue]] = Field(default_factory=list) + spm_error: dict[str, JsonValue] | None = None + + +class StoredExecutionFailure(StoredDocumentModel): + """A refused job's result document (``HouseholdExecutionFailure``).""" + + status: Literal["failed"] + error: StoredCalculationError + + +class CountrySource(Protocol): + """The countries table (``V2CountryService``).""" + + def list_countries(self) -> list[CountryRead]: + """Return every served country and the calculations it supports.""" + ... + + +class HouseholdSource(Protocol): + """Stored households (``V2HouseholdService``).""" + + def get_household(self, *, country_id: str, household_id: UUID) -> HouseholdRead: + """Return one household of this country, or raise HouseholdNotFoundError.""" + ... + + +class PolicySource(Protocol): + """Stored policies (``V2PolicyService``).""" + + def get_policy(self, *, country_id: str, policy_id: UUID) -> CheckedPolicyRead: + """Return one policy of this country, checked against the running catalog. + + Raise PolicyNotFoundError when the country has no such policy. + """ + ... + + +class HouseholdCalculationSubmitter(Protocol): + """The simulation service's ``POST /household-calculations``. + + The simulation service reads the household, the policy's parameter values + and the catalog version from the Public API's tables itself, inserts the + job and starts its Modal call. + """ + + def submit( + self, calculation: HouseholdCalculationInput + ) -> HouseholdCalculationAccepted: + """Submit the calculation and return the job the service created. + + Raise SimulationServiceInvalidRequestError when the service refuses + the caller's data, with its reasons, and + SimulationServiceUnavailableError for any other refusal or failure. + """ + ... + + +class SimulationServiceUnavailableError(RuntimeError): + """The simulation service could not be reached, could not answer, or refused. + + It covers every refusal other than the caller's data: a household, policy + or catalog version it cannot find (which the Public API checks first), a + database or manifest it cannot read, and a Modal call it cannot start. + """ + + +class SimulationServiceInvalidRequestError(ValueError): + """The simulation service refuses the caller's data; the message says why. + + Such as a stored household or policy value its household contract cannot + take, or an option it refuses. + """ diff --git a/policyengine_api/services/v2/household_calculations/validators.py b/policyengine_api/services/v2/household_calculations/validators.py index d2656934e..e0eecd49c 100644 --- a/policyengine_api/services/v2/household_calculations/validators.py +++ b/policyengine_api/services/v2/household_calculations/validators.py @@ -4,8 +4,16 @@ from typing import TYPE_CHECKING +from pydantic import ValidationError + +from policyengine_api.services.v2.household_calculations.types import ( + StoredExecutionFailure, + StoredExecutionResult, +) + if TYPE_CHECKING: from collections.abc import Iterable + from uuid import UUID from policyengine_api.services.v2.countries.types import CountryRead @@ -18,6 +26,18 @@ class HouseholdCalculationYearError(ValueError): """The household has no default year to calculate.""" +class HouseholdCalculationNotFoundError(LookupError): + """household_jobs has no job with this ID.""" + + +class HouseholdCalculationResultUnreadableError(RuntimeError): + """A succeeded job's stored result is not one result document the API reads.""" + + +class HouseholdCalculationStorageError(RuntimeError): + """The v2 database could not be read while handling a household calculation.""" + + def require_household_country( countries: Iterable[CountryRead], country_id: str ) -> None: @@ -42,3 +62,29 @@ def require_default_year(default_year: int | None) -> None: if default_year is None: msg = "This household has no default year to calculate." raise HouseholdCalculationYearError(msg) + + +def stored_household_outcome( + job_id: UUID, status: str, document: object +) -> StoredExecutionResult | StoredExecutionFailure | None: + """Return a job's stored result document, read as its status says. + + An unfinished job's result is not read. A succeeded job must hold one + ``HouseholdExecutionResult`` with exactly one policy result, or + HouseholdCalculationResultUnreadableError is raised. A failed job holds a + ``HouseholdExecutionFailure``, or None when it failed without one; any + other document is read as none, since the job's error message still + says why it failed. + """ + if status == "succeeded": + try: + return StoredExecutionResult.model_validate(document) + except ValidationError as error: + msg = f"Household calculation {job_id} has an unreadable result." + raise HouseholdCalculationResultUnreadableError(msg) from error + if status == "failed" and document is not None: + try: + return StoredExecutionFailure.model_validate(document) + except ValidationError: + return None + return None diff --git a/tests/integration/test_v2_household_calculation_reads.py b/tests/integration/test_v2_household_calculation_reads.py index a7e53ab25..1c7e20f2f 100644 --- a/tests/integration/test_v2_household_calculation_reads.py +++ b/tests/integration/test_v2_household_calculation_reads.py @@ -18,12 +18,6 @@ import pytest from alembic import command from alembic.config import Config -from policyengine_api.services.v2.household_calculations.database_session import ( - HouseholdCalculationDatabaseSession, -) -from policyengine_api.services.v2.household_calculations.services import ( - V2HouseholdCalculationService, -) from sqlalchemy import create_engine, text from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import NullPool @@ -34,6 +28,12 @@ CountryDatabaseSession, ) from policyengine_api.services.v2.countries.services import V2CountryService +from policyengine_api.services.v2.household_calculations.database_session import ( + HouseholdCalculationDatabaseSession, +) +from policyengine_api.services.v2.household_calculations.services import ( + V2HouseholdCalculationService, +) from policyengine_api.services.v2.household_calculations.validators import ( HouseholdCalculationNotFoundError, ) diff --git a/tests/unit/test_alembic_workflows.py b/tests/unit/test_alembic_workflows.py index 6fb3de5d2..2b69ceafc 100644 --- a/tests/unit/test_alembic_workflows.py +++ b/tests/unit/test_alembic_workflows.py @@ -253,6 +253,7 @@ def test_reusable_v2_integration_check_uses_disposable_databases_and_coverage(): assert "test_v2_household_persistence.py" in workflow assert "test_v1_household_dual_write.py" in workflow assert "test_mysql_household_spm_dual_write.py" in workflow + assert "test_v2_household_calculation_reads.py" in workflow assert "test_mysql_update_row_counts.py" in workflow assert "test_runtime_cache_redis.py" in workflow # The steps run coverage and pytest, which only the dev extra declares; diff --git a/tests/unit/v2/test_household_calculation_results.py b/tests/unit/v2/test_household_calculation_results.py index 9af4dfcf4..996aee68b 100644 --- a/tests/unit/v2/test_household_calculation_results.py +++ b/tests/unit/v2/test_household_calculation_results.py @@ -6,11 +6,11 @@ from uuid import UUID import pytest + +from policyengine_api.data.v2.models import HouseholdJob, HouseholdJobStatus from policyengine_api.services.v2.household_calculations.transformations import ( household_calculation_read, ) - -from policyengine_api.data.v2.models import HouseholdJob, HouseholdJobStatus from policyengine_api.services.v2.household_calculations.validators import ( HouseholdCalculationResultUnreadableError, stored_household_outcome, From 2599d22f699ac401054638c7d55ffe89eb111e3e Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:13:46 +0400 Subject: [PATCH 05/57] Test submitting a household calculation after the Public API's checks (failing) The service checks the country's household_enabled, that the household and policy exist in that country, the household's default year and the policy's missing parameters, in that order, and calls the simulation service only when every check passes. The submission method does not exist yet. The last test, of a database failure while reading, checks the read path committed before and passes. Co-Authored-By: Claude Opus 5.5 --- .../unit/v2/household_calculation_support.py | 153 ++++++++++++ .../v2/test_household_calculation_service.py | 218 ++++++++++++++++++ 2 files changed, 371 insertions(+) create mode 100644 tests/unit/v2/household_calculation_support.py create mode 100644 tests/unit/v2/test_household_calculation_service.py diff --git a/tests/unit/v2/household_calculation_support.py b/tests/unit/v2/household_calculation_support.py new file mode 100644 index 000000000..a26486e72 --- /dev/null +++ b/tests/unit/v2/household_calculation_support.py @@ -0,0 +1,153 @@ +"""Stored rows and stand-in services shared by the household calculation tests.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import UTC, datetime +from uuid import UUID + +from policyengine_api.services.v2.countries.types import CountryRead +from policyengine_api.services.v2.household_calculations.types import ( + HouseholdCalculationAccepted, + HouseholdCalculationInput, +) +from policyengine_api.services.v2.households.types import HouseholdRead +from policyengine_api.services.v2.households.validators import HouseholdNotFoundError +from policyengine_api.services.v2.policies.types import ( + CheckedPolicyRead, + PolicyParameterValueRead, +) +from policyengine_api.services.v2.policies.validators import PolicyNotFoundError +from tests.fixtures.household_calculation_documents import JOB_ID + +HOUSEHOLD_ID = UUID("00000000-0000-4000-8000-0000000000a1") +POLICY_ID = UUID("00000000-0000-4000-8000-0000000000b1") +MODEL_ID = UUID("00000000-0000-4000-8000-0000000000d1") +CATALOG_VERSION_ID = UUID("00000000-0000-4000-8000-0000000000e1") +CREATED_AT = datetime(2026, 11, 10, 12, 0, tzinfo=UTC) +REFORMED_PARAMETER = "gov.irs.credits.ctc.amount.base[0].amount" + + +def household_read(*, default_year: int | None = 2026) -> HouseholdRead: + """Return the stored US household.""" + return HouseholdRead( + id=HOUSEHOLD_ID, + country_id="us", + default_year=default_year, + household_data={"people": [], "household": []}, + created_at=CREATED_AT, + updated_at=CREATED_AT, + ) + + +def policy_read( + *, missing_parameters: tuple[str, ...] | None = () +) -> CheckedPolicyRead: + """Return the stored US reform, checked against the running catalog.""" + return CheckedPolicyRead( + id=POLICY_ID, + country_id="us", + tax_benefit_model_id=MODEL_ID, + tax_benefit_model_version_id=CATALOG_VERSION_ID, + created_at=CREATED_AT, + updated_at=CREATED_AT, + parameter_values=( + PolicyParameterValueRead( + id=POLICY_ID, + parameter_id=POLICY_ID, + parameter_name=REFORMED_PARAMETER, + value=3000, + start_date=datetime(2026, 1, 1, tzinfo=UTC), + end_date=None, + ), + ), + missing_parameters=missing_parameters, + ) + + +def calculation_input(**changes: object) -> HouseholdCalculationInput: + """Return a calculation of the stored household under the stored reform.""" + fields: dict[str, object] = { + "country_id": "us", + "household_id": HOUSEHOLD_ID, + "policy_id": POLICY_ID, + } + fields.update(changes) + return HouseholdCalculationInput.model_validate(fields) + + +@dataclass +class FakeCountries: + """The countries table: the US with every calculation, the UK households only.""" + + rows: tuple[CountryRead, ...] = ( + CountryRead(id="uk", household_enabled=True, society_wide_enabled=False), + CountryRead(id="us", household_enabled=True, society_wide_enabled=True), + ) + error: Exception | None = None + + def list_countries(self) -> list[CountryRead]: + """Return every row, or raise the scripted error.""" + if self.error is not None: + raise self.error + return list(self.rows) + + +@dataclass +class FakeHouseholds: + """Stored households by country and ID; records each read.""" + + households: dict[tuple[str, UUID], HouseholdRead] = field( + default_factory=lambda: {("us", HOUSEHOLD_ID): household_read()} + ) + reads: list[tuple[str, UUID]] = field(default_factory=list) + + def get_household(self, *, country_id: str, household_id: UUID) -> HouseholdRead: + """Return the household or raise HouseholdNotFoundError.""" + self.reads.append((country_id, household_id)) + household = self.households.get((country_id, household_id)) + if household is None: + msg = f"Household {household_id} not found." + raise HouseholdNotFoundError(msg) + return household + + +@dataclass +class FakePolicies: + """Stored policies by country and ID; records each read.""" + + policies: dict[tuple[str, UUID], CheckedPolicyRead] = field( + default_factory=lambda: {("us", POLICY_ID): policy_read()} + ) + reads: list[tuple[str, UUID]] = field(default_factory=list) + + def get_policy(self, *, country_id: str, policy_id: UUID) -> CheckedPolicyRead: + """Return the policy or raise PolicyNotFoundError.""" + self.reads.append((country_id, policy_id)) + policy = self.policies.get((country_id, policy_id)) + if policy is None: + msg = f"Policy {policy_id} not found." + raise PolicyNotFoundError(msg) + return policy + + +@dataclass +class FakeSimulation: + """The simulation service's household submission, as a scripted answer.""" + + answer: HouseholdCalculationAccepted = field( + default_factory=lambda: HouseholdCalculationAccepted( + calculation_id=JOB_ID, status="pending" + ) + ) + error: Exception | None = None + submissions: list[HouseholdCalculationInput] = field(default_factory=list) + + def submit( + self, calculation: HouseholdCalculationInput + ) -> HouseholdCalculationAccepted: + """Record the submission and answer it, or raise the scripted error.""" + self.submissions.append(calculation) + if self.error is not None: + raise self.error + return self.answer diff --git a/tests/unit/v2/test_household_calculation_service.py b/tests/unit/v2/test_household_calculation_service.py new file mode 100644 index 000000000..6f1bc5c45 --- /dev/null +++ b/tests/unit/v2/test_household_calculation_service.py @@ -0,0 +1,218 @@ +"""Submit a stored household's calculation only after the Public API's own checks.""" + +from __future__ import annotations + +from contextlib import contextmanager +from uuid import uuid4 + +import pytest +from sqlalchemy.exc import OperationalError + +from policyengine_api.services.v2.household_calculations.services import ( + V2HouseholdCalculationService, +) +from policyengine_api.services.v2.household_calculations.types import ( + SimulationServiceInvalidRequestError, + SimulationServiceUnavailableError, +) +from policyengine_api.services.v2.household_calculations.validators import ( + HouseholdCalculationCountryError, + HouseholdCalculationStorageError, + HouseholdCalculationYearError, +) +from policyengine_api.services.v2.households.validators import HouseholdNotFoundError +from policyengine_api.services.v2.policies.validators import ( + PolicyNotFoundError, + PolicyParametersMissingError, +) +from tests.fixtures.household_calculation_documents import JOB_ID +from tests.unit.v2.household_calculation_support import ( + HOUSEHOLD_ID, + POLICY_ID, + REFORMED_PARAMETER, + FakeCountries, + FakeHouseholds, + FakePolicies, + FakeSimulation, + calculation_input, + household_read, + policy_read, +) + + +class UnreadableDatabase: + """A household job database whose every read fails.""" + + @contextmanager + def read(self): + msg = "SELECT household_jobs" + raise OperationalError(msg, {}, Exception("connection refused")) + yield + + +class Services: + """The service under test and the stand-ins it reads and calls.""" + + def __init__(self) -> None: + self.countries = FakeCountries() + self.households = FakeHouseholds() + self.policies = FakePolicies() + self.simulation = FakeSimulation() + self.simulation_requests = 0 + + def simulation_factory(self) -> FakeSimulation: + self.simulation_requests += 1 + return self.simulation + + def service(self, database=None) -> V2HouseholdCalculationService: + return V2HouseholdCalculationService( + countries=self.countries, + households=self.households, + policies=self.policies, + database_session=database or UnreadableDatabase(), + simulation_factory=self.simulation_factory, + ) + + +@pytest.fixture +def services() -> Services: + return Services() + + +def test_a_calculation_is_sent_as_given_and_returned_with_its_year(services) -> None: + calculation = calculation_input(add_missing_variables=True) + + submitted = services.service().submit_calculation(calculation) + + assert services.simulation.submissions == [calculation] + assert services.households.reads == [("us", HOUSEHOLD_ID)] + assert services.policies.reads == [("us", POLICY_ID)] + assert submitted.calculation_id == JOB_ID + assert submitted.status == "pending" + assert (submitted.country_id, submitted.household_id) == ("us", HOUSEHOLD_ID) + assert submitted.policy_id == POLICY_ID + assert submitted.year == 2026 + + +def test_current_law_reads_no_policy(services) -> None: + submitted = services.service().submit_calculation(calculation_input(policy_id=None)) + + assert services.policies.reads == [] + assert submitted.policy_id is None + (sent,) = services.simulation.submissions + assert sent.policy_id is None + + +def test_a_household_only_country_is_calculated(services) -> None: + services.households.households[("uk", HOUSEHOLD_ID)] = household_read() + + services.service().submit_calculation( + calculation_input(country_id="uk", policy_id=None) + ) + + assert services.households.reads == [("uk", HOUSEHOLD_ID)] + assert len(services.simulation.submissions) == 1 + + +def test_a_country_without_household_calculations_is_refused_first(services) -> None: + services.countries.rows = services.countries.rows[1:] + + with pytest.raises(HouseholdCalculationCountryError, match="uk"): + services.service().submit_calculation(calculation_input(country_id="uk")) + + assert services.households.reads == [] + assert services.simulation_requests == 0 + + +def test_an_unknown_household_is_not_found_before_its_policy_is_read( + services, +) -> None: + with pytest.raises(HouseholdNotFoundError): + services.service().submit_calculation(calculation_input(household_id=uuid4())) + + assert services.policies.reads == [] + assert services.simulation_requests == 0 + + +def test_an_unknown_policy_is_not_found_before_the_year_is_checked(services) -> None: + services.households.households[("us", HOUSEHOLD_ID)] = household_read( + default_year=None + ) + + with pytest.raises(PolicyNotFoundError): + services.service().submit_calculation(calculation_input(policy_id=uuid4())) + + assert services.simulation_requests == 0 + + +def test_a_policy_of_another_country_is_not_found(services) -> None: + services.households.households[("uk", HOUSEHOLD_ID)] = household_read() + + with pytest.raises(PolicyNotFoundError): + services.service().submit_calculation(calculation_input(country_id="uk")) + + assert services.policies.reads == [("uk", POLICY_ID)] + assert services.simulation_requests == 0 + + +def test_a_household_without_a_default_year_is_refused(services) -> None: + services.households.households[("us", HOUSEHOLD_ID)] = household_read( + default_year=None + ) + + with pytest.raises(HouseholdCalculationYearError): + services.service().submit_calculation(calculation_input()) + + assert services.simulation_requests == 0 + + +def test_a_policy_setting_parameters_the_catalog_lacks_is_refused(services) -> None: + services.policies.policies[("us", POLICY_ID)] = policy_read( + missing_parameters=(REFORMED_PARAMETER,) + ) + + with pytest.raises(PolicyParametersMissingError) as refused: + services.service().submit_calculation(calculation_input()) + + assert refused.value.missing_parameters == (REFORMED_PARAMETER,) + assert services.simulation_requests == 0 + + +def test_a_policy_no_published_catalog_could_check_is_sent(services) -> None: + services.policies.policies[("us", POLICY_ID)] = policy_read(missing_parameters=None) + + services.service().submit_calculation(calculation_input()) + + assert len(services.simulation.submissions) == 1 + + +@pytest.mark.parametrize( + "refusal", + [ + SimulationServiceInvalidRequestError("the contract refuses the household"), + SimulationServiceUnavailableError("The simulation service answered 503."), + ], +) +def test_the_simulation_services_refusals_reach_the_caller(services, refusal) -> None: + services.simulation.error = refusal + + with pytest.raises(type(refusal), match=str(refusal)): + services.service().submit_calculation(calculation_input()) + + +def test_an_unreadable_database_while_checking_is_a_storage_error(services) -> None: + services.countries.error = OperationalError( + "SELECT countries", {}, Exception("connection refused") + ) + + with pytest.raises(HouseholdCalculationStorageError): + services.service().submit_calculation(calculation_input()) + + assert services.simulation_requests == 0 + + +def test_an_unreadable_database_while_reading_is_a_storage_error(services) -> None: + with pytest.raises(HouseholdCalculationStorageError): + services.service(UnreadableDatabase()).get_calculation(JOB_ID) + + assert services.simulation_requests == 0 From b19c779646b912a28cde7d9820f3b3c8350fab5b Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:14:23 +0400 Subject: [PATCH 06/57] Submit a household calculation only after the Public API's checks submit_calculation checks the country, the household and policy, the household's default year and the policy's missing parameters, then sends the calculation to the simulation service through the HouseholdCalculationSubmitter protocol, and returns the job the service created with the year it calculates. require_default_year now returns the year it checked. Co-Authored-By: Claude Opus 5.5 --- .../v2/household_calculations/services.py | 49 +++++++++++++++++++ .../v2/household_calculations/validators.py | 5 +- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/policyengine_api/services/v2/household_calculations/services.py b/policyengine_api/services/v2/household_calculations/services.py index d777ec509..869f9344e 100644 --- a/policyengine_api/services/v2/household_calculations/services.py +++ b/policyengine_api/services/v2/household_calculations/services.py @@ -1,5 +1,16 @@ """Calculate stored households through the simulation service, and read the results. +Submitting a calculation checks, in order, that the country's ``countries`` +row allows household calculations (a household-only country does), that the +stored household and policy exist in that country, through the same +services as ``/v2/households`` and ``/v2/policies`` (so one of another +country, or none, is not found), that the household has a default year, and +that the policy sets no parameter the running catalog version lacks. Only +then does it send the simulation service the IDs and options. The simulation +service reads the household, the policy's parameter values and the catalog +version from the Public API's tables itself, inserts the job and calculates +the household's default year; each submission creates a new job. + Reading a calculation reads its job from ``household_jobs``, which the simulation service inserts and its household function completes; it never calls the simulation service. @@ -17,12 +28,18 @@ ) from policyengine_api.services.v2.household_calculations.transformations import ( household_calculation_read, + household_calculation_submitted, ) from policyengine_api.services.v2.household_calculations.validators import ( HouseholdCalculationNotFoundError, HouseholdCalculationStorageError, + require_default_year, + require_household_country, stored_household_outcome, ) +from policyengine_api.services.v2.policies.validators import ( + require_policy_parameters_present, +) if TYPE_CHECKING: from collections.abc import Callable, Iterator @@ -33,7 +50,9 @@ ) from policyengine_api.services.v2.household_calculations.types import ( CountrySource, + HouseholdCalculationInput, HouseholdCalculationRead, + HouseholdCalculationSubmitted, HouseholdCalculationSubmitter, HouseholdSource, PolicySource, @@ -75,6 +94,36 @@ def __init__( self._database_session = database_session self._simulation_factory = simulation_factory + def submit_calculation( + self, calculation: HouseholdCalculationInput + ) -> HouseholdCalculationSubmitted: + """Send the calculation to the simulation service and return its new job. + + Raise HouseholdCalculationCountryError, HouseholdNotFoundError, + PolicyNotFoundError, HouseholdCalculationYearError or + PolicyParametersMissingError, in that order, before anything is sent; + HouseholdCalculationStorageError when the database cannot be read; + and the simulation service's errors when it refuses the calculation. + """ + country_id = calculation.country_id + with _storage_failures(): + require_household_country(self._countries.list_countries(), country_id) + household = self._households.get_household( + country_id=country_id, household_id=calculation.household_id + ) + policy = ( + None + if calculation.policy_id is None + else self._policies.get_policy( + country_id=country_id, policy_id=calculation.policy_id + ) + ) + year = require_default_year(household.default_year) + if policy is not None: + require_policy_parameters_present(policy.id, policy.missing_parameters) + accepted = self._simulation_factory().submit(calculation) + return household_calculation_submitted(calculation, accepted, year=year) + def get_calculation(self, calculation_id: UUID) -> HouseholdCalculationRead: """Return the calculation household_jobs records. diff --git a/policyengine_api/services/v2/household_calculations/validators.py b/policyengine_api/services/v2/household_calculations/validators.py index e0eecd49c..43e647988 100644 --- a/policyengine_api/services/v2/household_calculations/validators.py +++ b/policyengine_api/services/v2/household_calculations/validators.py @@ -53,8 +53,8 @@ def require_household_country( raise HouseholdCalculationCountryError(msg) -def require_default_year(default_year: int | None) -> None: - """Raise HouseholdCalculationYearError unless the household has a default year. +def require_default_year(default_year: int | None) -> int: + """Return the household's default year, or raise HouseholdCalculationYearError. The simulation service calculates that year, the year the household's values name. @@ -62,6 +62,7 @@ def require_default_year(default_year: int | None) -> None: if default_year is None: msg = "This household has no default year to calculate." raise HouseholdCalculationYearError(msg) + return default_year def stored_household_outcome( From bb5f5742d634546bd77846b4c669706e0ec9f004 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:15:41 +0400 Subject: [PATCH 07/57] Test the simulation service submission of household calculations (failing) The client posts a calculation's IDs and options to the Cloud Run simulation service's POST /household-calculations, reads the 202's job ID and status, passes the service's 422 reasons through, and treats every other answer, no answer, an unreadable answer, a gateway token it cannot fetch and missing configuration as the service being unavailable. The module does not exist yet. Co-Authored-By: Claude Opus 5.5 --- .../test_simulation_household_calculations.py | 286 ++++++++++++++++++ 1 file changed, 286 insertions(+) create mode 100644 tests/unit/libs/test_simulation_household_calculations.py diff --git a/tests/unit/libs/test_simulation_household_calculations.py b/tests/unit/libs/test_simulation_household_calculations.py new file mode 100644 index 000000000..c8cfa355c --- /dev/null +++ b/tests/unit/libs/test_simulation_household_calculations.py @@ -0,0 +1,286 @@ +"""Submit household calculations to the simulation service and read its answers.""" + +from __future__ import annotations + +import json +from uuid import UUID + +import httpx +import pytest +from policyengine_api.libs.simulation_household_calculations import ( + HOUSEHOLD_CALCULATIONS_PATH, + SimulationAnswer, + SimulationHouseholdCalculationClient, +) + +from policyengine_api.libs.gateway_auth import GatewayAuthError +from policyengine_api.services.v2.household_calculations.types import ( + HouseholdCalculationAccepted, + SimulationServiceInvalidRequestError, + SimulationServiceUnavailableError, +) +from tests.fixtures.household_calculation_documents import JOB_ID +from tests.unit.v2.household_calculation_support import ( + HOUSEHOLD_ID, + POLICY_ID, + calculation_input, +) + +SIMULATION_URL = "https://simulation.example.test" +ENVIRONMENT_VARIABLES = ( + "GATEWAY_AUTH_ISSUER", + "GATEWAY_AUTH_AUDIENCE", + "GATEWAY_AUTH_CLIENT_ID", + "GATEWAY_AUTH_CLIENT_SECRET", + "GATEWAY_AUTH_CLIENT_SECRET_RESOURCE", + "GATEWAY_AUTH_REQUIRED", + "SIM_ENTRYPOINT", + "OLD_SIMULATION_GATEWAY_URL", + "SIMULATION_ENTRYPOINT_URL", +) +ACCEPTED = {"job_id": str(JOB_ID), "status": "pending"} + + +@pytest.fixture(autouse=True) +def configured_simulation_service(monkeypatch): + """Name the Cloud Run simulation service; set no gateway credentials.""" + for name in ENVIRONMENT_VARIABLES: + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("SIMULATION_ENTRYPOINT_URL", f"{SIMULATION_URL}/") + # The household routes exist only on the Cloud Run simulation service, + # whatever the economy routes select. + monkeypatch.setenv("SIM_ENTRYPOINT", "old_gateway_direct") + monkeypatch.setenv("OLD_SIMULATION_GATEWAY_URL", "https://old.example.test") + + +class RecordingService: + """The simulation service, answering every request with one scripted answer.""" + + def __init__(self, status_code: int = 202, body: object = ACCEPTED) -> None: + self.status_code = status_code + self.body = body + self.requests: list[httpx.Request] = [] + + def answer(self, request: httpx.Request) -> httpx.Response: + self.requests.append(request) + if isinstance(self.body, bytes): + return httpx.Response(self.status_code, content=self.body) + return httpx.Response(self.status_code, json=self.body) + + def client(self) -> SimulationHouseholdCalculationClient: + return SimulationHouseholdCalculationClient.from_environment( + transport=httpx.MockTransport(self.answer) + ) + + def sent_body(self) -> dict[str, object]: + (request,) = self.requests + return json.loads(request.content) + + +def test_a_submission_is_posted_and_its_new_job_returned() -> None: + service = RecordingService() + + accepted = service.client().submit(calculation_input()) + + assert accepted == HouseholdCalculationAccepted( + calculation_id=JOB_ID, status="pending" + ) + (request,) = service.requests + assert request.method == "POST" + assert str(request.url) == f"{SIMULATION_URL}{HOUSEHOLD_CALCULATIONS_PATH}" + assert service.sent_body() == { + "country": "us", + "household_id": str(HOUSEHOLD_ID), + "policy_id": str(POLICY_ID), + "add_missing_variables": False, + } + + +def test_current_law_sends_a_null_policy() -> None: + service = RecordingService() + + service.client().submit(calculation_input(policy_id=None)) + + assert service.sent_body()["policy_id"] is None + + +def test_axes_and_spm_settings_are_sent_only_as_the_caller_gave_them() -> None: + service = RecordingService() + axis = {"name": "employment_income", "count": 5, "min": 0, "max": 50_000} + + service.client().submit( + calculation_input( + add_missing_variables=True, + axes=[[{**axis, "period": "2026"}], [axis]], + spm={"scenario": "ce_trend", "geography_kind": "national"}, + ) + ) + + body = service.sent_body() + assert body["add_missing_variables"] is True + assert body["axes"] == [[{**axis, "period": "2026"}], [axis]] + assert body["spm"] == {"scenario": "ce_trend", "geography_kind": "national"} + + +def test_an_answer_with_fields_the_public_api_does_not_read_is_accepted() -> None: + service = RecordingService(body={**ACCEPTED, "created_at": "2026-11-10T12:00Z"}) + + accepted = service.client().submit(calculation_input()) + + assert accepted.calculation_id == JOB_ID + + +def test_caller_data_the_service_refuses_is_reported_with_its_reason() -> None: + service = RecordingService(422, {"detail": "the policy value is an object"}) + + with pytest.raises(SimulationServiceInvalidRequestError) as refused: + service.client().submit(calculation_input()) + + assert str(refused.value) == ( + "The simulation service cannot calculate this household: " + "the policy value is an object" + ) + + +def test_request_fields_the_service_refuses_are_reported_one_by_one() -> None: + service = RecordingService( + 422, + { + "detail": [ + {"loc": ["body", "axes", 0, 0, "count"], "msg": "too large"}, + {"loc": ["body", "spm"], "msg": "must be an object", "type": "x"}, + ] + }, + ) + + with pytest.raises(SimulationServiceInvalidRequestError) as refused: + service.client().submit(calculation_input()) + + assert str(refused.value) == ( + "The simulation service cannot calculate this household: " + "axes.0.0.count: too large; spm: must be an object" + ) + + +@pytest.mark.parametrize( + "body", + [ + pytest.param({"detail": {"nested": "object"}}, id="detail-object"), + pytest.param({"detail": [{"msg": "no location"}]}, id="field-without-loc"), + pytest.param({"message": "no detail"}, id="no-detail"), + pytest.param(b"not json", id="not-json"), + ], +) +def test_a_refusal_of_another_shape_is_unavailable(body) -> None: + service = RecordingService(422, body) + + with pytest.raises(SimulationServiceUnavailableError, match="could not be read"): + service.client().submit(calculation_input()) + + +def test_a_reference_the_service_cannot_find_is_unavailable_with_its_reason() -> None: + service = RecordingService(400, {"detail": f"Household {HOUSEHOLD_ID} not found"}) + + with pytest.raises(SimulationServiceUnavailableError) as refused: + service.client().submit(calculation_input()) + + assert str(refused.value) == ( + "The simulation service refused the calculation: " + f"Household {HOUSEHOLD_ID} not found" + ) + + +@pytest.mark.parametrize("status_code", [400, 401, 404, 500, 502, 503]) +def test_any_other_answer_is_unavailable(status_code) -> None: + service = RecordingService(status_code, b"") + + with pytest.raises(SimulationServiceUnavailableError, match=str(status_code)): + service.client().submit(calculation_input()) + + +@pytest.mark.parametrize( + "body", + [ + pytest.param(b"not json", id="not-json"), + pytest.param({"status": "pending"}, id="no-job-id"), + pytest.param({"job_id": "not-a-uuid", "status": "pending"}, id="bad-id"), + pytest.param({"job_id": str(JOB_ID), "status": "queued"}, id="bad-status"), + ], +) +def test_an_accepted_answer_of_another_shape_is_unavailable(body) -> None: + service = RecordingService(202, body) + + with pytest.raises(SimulationServiceUnavailableError, match="could not be read"): + service.client().submit(calculation_input()) + + +def test_an_unreachable_service_is_unavailable() -> None: + def refuse(request: httpx.Request) -> httpx.Response: + msg = "connection refused" + raise httpx.ConnectError(msg, request=request) + + client = SimulationHouseholdCalculationClient.from_environment( + transport=httpx.MockTransport(refuse) + ) + + with pytest.raises(SimulationServiceUnavailableError, match="could not be reached"): + client.submit(calculation_input()) + + +def test_a_gateway_token_that_cannot_be_fetched_is_unavailable() -> None: + def refuse_token(http_client, body) -> SimulationAnswer: + msg = "Auth0 refused the client credentials" + raise GatewayAuthError(msg) + + client = SimulationHouseholdCalculationClient( + httpx.Client(base_url=SIMULATION_URL), send=refuse_token + ) + + with pytest.raises(SimulationServiceUnavailableError, match="could not be reached"): + client.submit(calculation_input()) + + +def test_the_request_is_sent_through_the_one_replaceable_send_function() -> None: + sent: list[dict[str, object]] = [] + + def send(http_client: httpx.Client, body: dict[str, object]) -> SimulationAnswer: + assert str(http_client.base_url) == f"{SIMULATION_URL}/" + sent.append(body) + return SimulationAnswer(status_code=202, content=json.dumps(ACCEPTED).encode()) + + client = SimulationHouseholdCalculationClient( + httpx.Client(base_url=SIMULATION_URL), send=send + ) + + accepted = client.submit(calculation_input(policy_id=None)) + + assert accepted.calculation_id == UUID(ACCEPTED["job_id"]) + assert sent == [ + { + "country": "us", + "household_id": str(HOUSEHOLD_ID), + "policy_id": None, + "add_missing_variables": False, + } + ] + + +def test_a_missing_service_url_is_unavailable(monkeypatch) -> None: + monkeypatch.delenv("SIMULATION_ENTRYPOINT_URL") + + with pytest.raises(SimulationServiceUnavailableError, match="not configured"): + SimulationHouseholdCalculationClient.from_environment() + + +def test_partly_configured_gateway_credentials_are_unavailable(monkeypatch) -> None: + monkeypatch.setenv("GATEWAY_AUTH_ISSUER", "https://tenant.example.test/") + + with pytest.raises(SimulationServiceUnavailableError, match="not configured"): + SimulationHouseholdCalculationClient.from_environment() + + +def test_required_but_missing_gateway_credentials_are_unavailable(monkeypatch) -> None: + monkeypatch.setenv("GATEWAY_AUTH_REQUIRED", "1") + + with pytest.raises(SimulationServiceUnavailableError, match="not configured"): + SimulationHouseholdCalculationClient.from_environment() From a905ae35a3bb932422eb3091ddc8a0464badbde8 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:17:00 +0400 Subject: [PATCH 08/57] Submit household calculations to the simulation service behind a protocol SimulationHouseholdCalculationClient implements the service's HouseholdCalculationSubmitter. It posts to the Cloud Run simulation service's POST /household-calculations through the HTTP client SimulationEntrypointClient builds, so it shares its gateway credentials, correlation headers, transport and instrumentation. A 202's job ID and status are read with hand-written models that ignore fields they don't name; a 422's reasons reach the caller; every other answer is the service being unavailable. post_household_calculation is the only code that sends the request, so wiring the generated simulation client changes that one function and adds the dependency. Co-Authored-By: Claude Opus 5.5 --- .../libs/simulation_household_calculations.py | 258 ++++++++++++++++++ .../test_simulation_household_calculations.py | 6 +- 2 files changed, 261 insertions(+), 3 deletions(-) create mode 100644 policyengine_api/libs/simulation_household_calculations.py diff --git a/policyengine_api/libs/simulation_household_calculations.py b/policyengine_api/libs/simulation_household_calculations.py new file mode 100644 index 000000000..580f2f936 --- /dev/null +++ b/policyengine_api/libs/simulation_household_calculations.py @@ -0,0 +1,258 @@ +"""Submit household calculations to the simulation service. + +``SimulationHouseholdCalculationClient`` is the household calculation +service's ``HouseholdCalculationSubmitter``. It posts a calculation to the +Cloud Run simulation service's ``POST /household-calculations``, at +``SIMULATION_ENTRYPOINT_URL`` whatever ``SIM_ENTRYPOINT`` selects for the +economy routes, through the HTTP client ``SimulationEntrypointClient`` builds: +the same gateway credentials (``GATEWAY_AUTH_*``), request and observability +ID headers, transport and instrumentation as every other simulation call. + +The request names the country, the stored household and the stored policy +(null for current law), ``add_missing_variables``, and the axes and SPM +settings only when the caller gave them. The simulation service answers: + +- 202 with the new job's ID and status; +- 422 with its reasons when it refuses the caller's data (a malformed + request, or one its household contract refuses): the reasons reach the + caller as SimulationServiceInvalidRequestError; +- anything else (400 for a household, policy or catalog version it cannot + find, which the Public API checks first; 503 when it cannot read its + manifest or database or Modal refuses the call): the service is + unavailable (SimulationServiceUnavailableError), as it is for no answer, + an answer of another shape, a gateway token that cannot be fetched and + missing configuration. + +Wiring the generated simulation client: ``post_household_calculation`` is the +only code that sends the request, and it returns the status and body, which +the rest of this module reads. Once a ``policyengine-api-simulation-client`` +release has the household calculation route, depend on that release and make +the function call the generated operation, as its docstring says. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from functools import lru_cache +from http import HTTPStatus +from typing import TYPE_CHECKING, Any +from uuid import UUID + +import httpx +from pydantic import BaseModel, ConfigDict, ValidationError + +from policyengine_api.libs.gateway_auth import GatewayAuthError +from policyengine_api.libs.simulation_entrypoint import SimulationEntrypointClient +from policyengine_api.services.v2.household_calculations.types import ( + HouseholdCalculationAccepted, + HouseholdJobStatusName, + SimulationServiceInvalidRequestError, + SimulationServiceUnavailableError, +) + +if TYPE_CHECKING: + from collections.abc import Callable + + from policyengine_api.services.v2.household_calculations.types import ( + HouseholdCalculationInput, + ) + +HOUSEHOLD_CALCULATIONS_PATH = "/household-calculations" +CLOUD_RUN_SIMULATION_ENTRYPOINT = "cloud_run_simulation_entrypoint" +UNREADABLE_ANSWER = "The simulation service's answer could not be read." + + +@dataclass(frozen=True) +class SimulationAnswer: + """The simulation service's answer: its HTTP status and body.""" + + status_code: int + content: bytes + + +class _AnswerModel(BaseModel): + """Base for the simulation service's answers; fields it adds are ignored.""" + + model_config = ConfigDict(extra="ignore", frozen=True) + + +class _AcceptedJob(_AnswerModel): + """The 202 body: the job the simulation service inserted, and its status.""" + + job_id: UUID + status: HouseholdJobStatusName + + +class _RefusedField(_AnswerModel): + """One request field the simulation service refuses, and why.""" + + loc: list[str | int] + msg: str + + +class _Refusal(_AnswerModel): + """A refusal's body: a reason, or each refused request field.""" + + detail: str | list[_RefusedField] + + +def household_calculation_request( + calculation: HouseholdCalculationInput, +) -> dict[str, Any]: + """Return the simulation service's request body for the calculation. + + ``axes`` and ``spm`` are sent only when the caller gave them; each axis + and the SPM settings keep only the fields the caller gave. + """ + body: dict[str, Any] = { + "country": calculation.country_id, + "household_id": str(calculation.household_id), + "policy_id": ( + None if calculation.policy_id is None else str(calculation.policy_id) + ), + "add_missing_variables": calculation.add_missing_variables, + } + if calculation.axes is not None: + body["axes"] = [ + [axis.model_dump(mode="json", exclude_unset=True) for axis in group] + for group in calculation.axes + ] + if calculation.spm is not None: + body["spm"] = calculation.spm.model_dump(mode="json") + return body + + +def post_household_calculation( + http_client: httpx.Client, body: dict[str, Any] +) -> SimulationAnswer: + """Send one submission to the simulation service and return its answer. + + Raise httpx.HTTPError when no answer arrives, and GatewayAuthError when + the gateway token cannot be fetched. ``http_client`` has the service's + base URL. To send through the generated client instead, this body + becomes (the operation and model names are those the simulation + service's OpenAPI document gives the route):: + + client = Client(base_url=str(http_client.base_url)) + client.set_httpx_client(http_client) + response = submit_household_calculation.sync_detailed( + client=client, body=HouseholdCalculationRequest.from_dict(body) + ) + return SimulationAnswer(int(response.status_code), response.content) + """ + response = http_client.post(HOUSEHOLD_CALCULATIONS_PATH, json=body) + return SimulationAnswer(status_code=response.status_code, content=response.content) + + +def _invalid_request_reason(refusal: _Refusal) -> str: + """Return the service's reason, or each refused field and why. + + A field's location is given without the request's ``body`` prefix. + """ + if isinstance(refusal.detail, str): + reason = refusal.detail + else: + reasons = [] + for refused_field in refusal.detail: + location = [str(part) for part in refused_field.loc] + if location[:1] == ["body"]: + location = location[1:] + reasons.append(f"{'.'.join(location)}: {refused_field.msg}") + reason = "; ".join(reasons) or "the request is invalid" + return f"The simulation service cannot calculate this household: {reason}" + + +def _refusal(answer: SimulationAnswer) -> _Refusal | None: + try: + return _Refusal.model_validate_json(answer.content) + except ValidationError: + return None + + +class SimulationHouseholdCalculationClient: + """Submit household calculations to the simulation service.""" + + def __init__( + self, + http_client: httpx.Client, + *, + send: Callable[ + [httpx.Client, dict[str, Any]], SimulationAnswer + ] = post_household_calculation, + ) -> None: + """Send through ``send`` on ``http_client``, which has the service's URL.""" + self._http_client = http_client + self._send = send + + @classmethod + def from_environment( + cls, *, transport: httpx.BaseTransport | None = None + ) -> SimulationHouseholdCalculationClient: + """Call the configured Cloud Run simulation service with gateway auth. + + ``transport`` replaces the simulation transport, for tests. Raise + SimulationServiceUnavailableError when ``SIMULATION_ENTRYPOINT_URL`` + is missing or invalid, or gateway auth is partly configured or + required but unset. + """ + try: + entrypoint = SimulationEntrypointClient( + CLOUD_RUN_SIMULATION_ENTRYPOINT, transport=transport + ) + except (ValueError, GatewayAuthError) as error: + msg = "The simulation service is not configured." + raise SimulationServiceUnavailableError(msg) from error + http_client = entrypoint.client + http_client.base_url = httpx.URL(entrypoint.base_url) + return cls(http_client) + + def submit( + self, calculation: HouseholdCalculationInput + ) -> HouseholdCalculationAccepted: + """Submit the calculation and return the job the service created. + + Raise SimulationServiceInvalidRequestError with the service's reasons + when it refuses the caller's data, and SimulationServiceUnavailableError + otherwise. + """ + try: + answer = self._send( + self._http_client, household_calculation_request(calculation) + ) + except (httpx.HTTPError, GatewayAuthError) as error: + msg = "The simulation service could not be reached." + raise SimulationServiceUnavailableError(msg) from error + if answer.status_code == HTTPStatus.ACCEPTED: + try: + accepted = _AcceptedJob.model_validate_json(answer.content) + except ValidationError as error: + raise SimulationServiceUnavailableError(UNREADABLE_ANSWER) from error + return HouseholdCalculationAccepted( + calculation_id=accepted.job_id, status=accepted.status + ) + refusal = _refusal(answer) + if answer.status_code == HTTPStatus.UNPROCESSABLE_ENTITY: + if refusal is None: + raise SimulationServiceUnavailableError(UNREADABLE_ANSWER) + raise SimulationServiceInvalidRequestError(_invalid_request_reason(refusal)) + if ( + answer.status_code == HTTPStatus.BAD_REQUEST + and refusal is not None + and isinstance(refusal.detail, str) + ): + msg = f"The simulation service refused the calculation: {refusal.detail}" + raise SimulationServiceUnavailableError(msg) + msg = f"The simulation service answered {answer.status_code}." + raise SimulationServiceUnavailableError(msg) + + +@lru_cache(maxsize=1) +def process_simulation_household_calculation_client() -> ( + SimulationHouseholdCalculationClient +): + """Return the process's client, built from the environment on first use. + + Its connections and gateway token are reused across requests. A + configuration error is raised again on the next call, not kept. + """ + return SimulationHouseholdCalculationClient.from_environment() diff --git a/tests/unit/libs/test_simulation_household_calculations.py b/tests/unit/libs/test_simulation_household_calculations.py index c8cfa355c..c06db82d3 100644 --- a/tests/unit/libs/test_simulation_household_calculations.py +++ b/tests/unit/libs/test_simulation_household_calculations.py @@ -7,13 +7,13 @@ import httpx import pytest + +from policyengine_api.libs.gateway_auth import GatewayAuthError from policyengine_api.libs.simulation_household_calculations import ( HOUSEHOLD_CALCULATIONS_PATH, SimulationAnswer, SimulationHouseholdCalculationClient, ) - -from policyengine_api.libs.gateway_auth import GatewayAuthError from policyengine_api.services.v2.household_calculations.types import ( HouseholdCalculationAccepted, SimulationServiceInvalidRequestError, @@ -244,7 +244,7 @@ def test_the_request_is_sent_through_the_one_replaceable_send_function() -> None sent: list[dict[str, object]] = [] def send(http_client: httpx.Client, body: dict[str, object]) -> SimulationAnswer: - assert str(http_client.base_url) == f"{SIMULATION_URL}/" + assert str(http_client.base_url).rstrip("/") == SIMULATION_URL sent.append(body) return SimulationAnswer(status_code=202, content=json.dumps(ACCEPTED).encode()) From 6d3db8ac038391f0de92cb6a72fd2d91b82f98f5 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:18:24 +0400 Subject: [PATCH 09/57] Test the /v2/household-calculations routes and their error answers (failing) POST answers 202 with the new job, needs no account, and answers each refusal: 422 for a malformed request, a country without household calculations, a household without a default year and the simulation service's 422 reasons; 422 missing_parameters naming the parameters; 404 for a household or policy not stored in that country; 503 when the simulation service or v2 storage is unavailable. GET reads the job and answers 404, 500 for an unreadable result and 503. The OpenAPI test lists the two new paths. The routes do not exist yet. Co-Authored-By: Claude Opus 5.5 --- .../v2/test_household_calculation_routes.py | 432 ++++++++++++++++++ tests/unit/v2/test_metadata_routes.py | 2 + 2 files changed, 434 insertions(+) create mode 100644 tests/unit/v2/test_household_calculation_routes.py diff --git a/tests/unit/v2/test_household_calculation_routes.py b/tests/unit/v2/test_household_calculation_routes.py new file mode 100644 index 000000000..45821ba76 --- /dev/null +++ b/tests/unit/v2/test_household_calculation_routes.py @@ -0,0 +1,432 @@ +"""The native ``/v2/household-calculations`` routes and their error answers.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from uuid import uuid4 + +import pytest +from fastapi.testclient import TestClient +from flask import Flask + +from policyengine_api.asgi_factory import create_asgi_app +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.migration_flags import ( + RouteImplementation, + RouteImplementationSettings, +) +from policyengine_api.services.v2.household_calculations.types import ( + HouseholdCalculationFailure, + HouseholdCalculationOutput, + HouseholdCalculationRead, + HouseholdCalculationSubmitted, + SimulationServiceInvalidRequestError, + SimulationServiceUnavailableError, +) +from policyengine_api.services.v2.household_calculations.validators import ( + HouseholdCalculationCountryError, + HouseholdCalculationNotFoundError, + HouseholdCalculationResultUnreadableError, + HouseholdCalculationStorageError, + HouseholdCalculationYearError, +) +from policyengine_api.services.v2.households.validators import HouseholdNotFoundError +from policyengine_api.services.v2.policies.validators import ( + PolicyNotFoundError, + PolicyParametersMissingError, +) +from tests.fixtures.household_calculation_documents import ( + CALCULATED_HOUSEHOLD, + INPUT_ERROR, + JOB_ID, + SPM_CONFIG, + SPM_PROVENANCE, + WARNING, +) +from tests.unit.v2.household_calculation_support import ( + CATALOG_VERSION_ID, + CREATED_AT, + HOUSEHOLD_ID, + POLICY_ID, + REFORMED_PARAMETER, +) + +COMPLETED_AT = datetime(2026, 11, 10, 12, 0, 9, tzinfo=UTC) +CALCULATIONS = "/v2/household-calculations" + + +def _read(**changes: object) -> HouseholdCalculationRead: + fields: dict[str, object] = { + "calculation_id": JOB_ID, + "country_id": "us", + "household_id": HOUSEHOLD_ID, + "policy_id": POLICY_ID, + "year": 2026, + "catalog_version_id": CATALOG_VERSION_ID, + "country_package_version": "1.450.0", + "policyengine_version": "6.10.0", + "status": "succeeded", + "created_at": CREATED_AT, + "started_at": CREATED_AT, + "completed_at": COMPLETED_AT, + "result": HouseholdCalculationOutput( + household=CALCULATED_HOUSEHOLD, + warnings=(WARNING,), + spm_config=SPM_CONFIG, + spm_provenance=SPM_PROVENANCE, + ), + "error": None, + } + fields.update(changes) + return HouseholdCalculationRead(**fields) + + +class FakeCalculationService: + """The household calculation service, with scripted answers and errors.""" + + def __init__(self) -> None: + self.submissions: list = [] + self.reads: list = [] + self.error: Exception | None = None + self.read_answer = _read() + + def submit_calculation(self, calculation) -> HouseholdCalculationSubmitted: + self.submissions.append(calculation) + if self.error is not None: + raise self.error + return HouseholdCalculationSubmitted( + calculation_id=JOB_ID, + status="pending", + country_id=calculation.country_id, + household_id=calculation.household_id, + policy_id=calculation.policy_id, + year=2026, + ) + + def get_calculation(self, calculation_id) -> HouseholdCalculationRead: + self.reads.append(calculation_id) + if self.error is not None: + raise self.error + return self.read_answer + + +def _client(service_factory) -> TestClient: + dependencies = NativeRouteDependencies( + readiness_probe=lambda: True, + gateway_client_factory=lambda: None, + metadata_reader_factory=lambda: None, + v2_household_calculation_service_factory=service_factory, + ) + settings = RouteImplementationSettings( + health=RouteImplementation.FLASK_FALLBACK, + specification=RouteImplementation.FLASK_FALLBACK, + metadata=RouteImplementation.FLASK_FALLBACK, + ) + return TestClient( + create_asgi_app( + Flask(__name__), dependencies=dependencies, route_settings=settings + ), + raise_server_exceptions=False, + ) + + +@pytest.fixture +def service() -> FakeCalculationService: + return FakeCalculationService() + + +@pytest.fixture +def client(service) -> TestClient: + return _client(lambda: service) + + +def _body(**changes: object) -> dict[str, object]: + body: dict[str, object] = { + "country_id": "us", + "household_id": str(HOUSEHOLD_ID), + "policy_id": str(POLICY_ID), + } + body.update(changes) + return body + + +def test_a_calculation_is_accepted_without_an_account(client, service) -> None: + response = client.post(CALCULATIONS, json=_body(country_id="US")) + + assert response.status_code == 202 + assert response.json() == { + "status": "ok", + "message": None, + "result": { + "item": { + "calculation_id": str(JOB_ID), + "status": "pending", + "country_id": "us", + "household_id": str(HOUSEHOLD_ID), + "policy_id": str(POLICY_ID), + "year": 2026, + } + }, + } + (calculation,) = service.submissions + assert calculation.country_id == "us" + assert calculation.household_id == HOUSEHOLD_ID + assert calculation.policy_id == POLICY_ID + + +def test_axes_options_and_spm_settings_reach_the_service(client, service) -> None: + axis = {"name": "employment_income", "count": 3, "min": 0, "max": 9_000} + + response = client.post( + CALCULATIONS, + json=_body( + policy_id=None, + axes=[[axis]], + add_missing_variables=True, + spm={"scenario": "ce_trend"}, + ), + ) + + assert response.status_code == 202 + (calculation,) = service.submissions + assert calculation.policy_id is None + assert calculation.add_missing_variables is True + assert calculation.axes is not None + assert calculation.axes[0][0].model_dump(exclude_unset=True) == axis + assert calculation.spm.model_dump(mode="json") == {"scenario": "ce_trend"} + + +@pytest.mark.parametrize( + "changes", + [ + pytest.param({"household_id": None}, id="no-household"), + pytest.param({"unknown": 1}, id="unknown-field"), + pytest.param({"spm": None}, id="null-spm"), + pytest.param( + { + "axes": [ + [ + {"name": "a", "count": 2, "min": 0, "max": 1}, + {"name": "b", "count": 3, "min": 0, "max": 1}, + ] + ] + }, + id="uneven-axes", + ), + pytest.param({"country_id": "ca"}, id="unsupported-country"), + ], +) +def test_a_malformed_request_is_refused_before_anything_is_read( + client, service, changes +) -> None: + response = client.post(CALCULATIONS, json=_body(**changes)) + + assert response.status_code == 422 + assert response.json() == {"status": "error", "message": "Invalid API v2 request"} + assert service.submissions == [] + + +@pytest.mark.parametrize( + ("error", "status_code", "message"), + [ + ( + HouseholdCalculationCountryError("No household calculations for uk."), + 422, + "No household calculations for uk.", + ), + ( + HouseholdCalculationYearError("This household has no default year."), + 422, + "This household has no default year.", + ), + ( + SimulationServiceInvalidRequestError("The service cannot calculate it."), + 422, + "The service cannot calculate it.", + ), + (HouseholdNotFoundError("Household not found."), 404, "Household not found."), + (PolicyNotFoundError("Policy not found."), 404, "Policy not found."), + ( + SimulationServiceUnavailableError("The simulation service answered 503."), + 503, + "The simulation service answered 503.", + ), + ( + HouseholdCalculationStorageError("storage down"), + 503, + "V2 household calculation storage is unavailable", + ), + ( + RuntimeError("secret detail"), + 500, + "V2 household calculation failed", + ), + ], +) +def test_a_refused_submission_gets_its_error_answer( + client, service, error, status_code, message +) -> None: + service.error = error + + response = client.post(CALCULATIONS, json=_body()) + + assert response.status_code == status_code + assert response.json() == {"status": "error", "message": message} + + +def test_a_policy_with_missing_parameters_is_refused_with_their_names( + client, service +) -> None: + service.error = PolicyParametersMissingError(POLICY_ID, (REFORMED_PARAMETER,)) + + response = client.post(CALCULATIONS, json=_body()) + + assert response.status_code == 422 + assert response.json() == { + "status": "error", + "message": ( + f"Policy {POLICY_ID} sets parameters the running catalog version " + f"does not have: {REFORMED_PARAMETER}" + ), + "code": "missing_parameters", + "missing_parameters": [REFORMED_PARAMETER], + } + + +def test_v2_storage_that_is_not_configured_answers_503() -> None: + def unconfigured(): + msg = "V2_RUNTIME_DATABASE_URL is required" + raise V2ConfigurationError(msg) + + client = _client(unconfigured) + + submitted = client.post(CALCULATIONS, json=_body()) + read = client.get(f"{CALCULATIONS}/{JOB_ID}") + + assert [submitted.status_code, read.status_code] == [503, 503] + assert read.json() == { + "status": "error", + "message": "V2 household calculation storage is unavailable", + } + + +def test_a_succeeded_calculation_reads_back_its_household(client, service) -> None: + response = client.get(f"{CALCULATIONS}/{JOB_ID}") + + assert response.status_code == 200 + assert service.reads == [JOB_ID] + item = response.json()["result"]["item"] + assert item == { + "calculation_id": str(JOB_ID), + "country_id": "us", + "household_id": str(HOUSEHOLD_ID), + "policy_id": str(POLICY_ID), + "year": 2026, + "catalog_version_id": str(CATALOG_VERSION_ID), + "country_package_version": "1.450.0", + "policyengine_version": "6.10.0", + "status": "succeeded", + "created_at": "2026-11-10T12:00:00Z", + "started_at": "2026-11-10T12:00:00Z", + "completed_at": "2026-11-10T12:00:09Z", + "result": { + "household": CALCULATED_HOUSEHOLD, + "warnings": [WARNING], + "spm_config": SPM_CONFIG, + "spm_provenance": SPM_PROVENANCE, + }, + "error": None, + } + assert list(item["result"]["household"]["people"]["person-1"]) == [ + "employment_income", + "age", + ] + + +def test_a_failed_calculation_reads_back_its_error(client, service) -> None: + service.read_answer = _read( + status="failed", + result=None, + error=HouseholdCalculationFailure( + code="invalid_inputs", + message="The household names a variable the model lacks.", + input_errors=(INPUT_ERROR,), + spm_error=None, + ), + ) + + response = client.get(f"{CALCULATIONS}/{JOB_ID}") + + assert response.status_code == 200 + item = response.json()["result"]["item"] + assert item["status"] == "failed" + assert item["result"] is None + assert item["error"] == { + "code": "invalid_inputs", + "message": "The household names a variable the model lacks.", + "input_errors": [INPUT_ERROR], + "spm_error": None, + } + + +@pytest.mark.parametrize( + ("error", "status_code", "message"), + [ + ( + HouseholdCalculationNotFoundError("Household calculation not found."), + 404, + "Household calculation not found.", + ), + ( + HouseholdCalculationResultUnreadableError("unreadable"), + 500, + "Stored household calculation result is unreadable", + ), + ( + HouseholdCalculationStorageError("storage down"), + 503, + "V2 household calculation storage is unavailable", + ), + ], +) +def test_a_calculation_that_cannot_be_read_gets_its_error_answer( + client, service, error, status_code, message +) -> None: + service.error = error + + response = client.get(f"{CALCULATIONS}/{uuid4()}") + + assert response.status_code == status_code + assert response.json() == {"status": "error", "message": message} + + +def test_a_calculation_id_that_is_not_a_uuid_is_refused(client, service) -> None: + response = client.get(f"{CALCULATIONS}/not-a-uuid") + + assert response.status_code == 422 + assert service.reads == [] + + +def test_openapi_documents_both_routes_and_the_missing_parameter_refusal( + client, +) -> None: + schema = client.get("/v2/openapi.json").json() + + submit = schema["paths"][CALCULATIONS]["post"] + read = schema["paths"][f"{CALCULATIONS}/{{calculation_id}}"]["get"] + assert set(submit["responses"]) >= {"202", "404", "422", "500", "503"} + assert set(read["responses"]) >= {"200", "404", "422", "500", "503"} + assert "security" not in submit + refusal_schema = submit["responses"]["422"]["content"]["application/json"]["schema"] + refusals = {option["$ref"].rsplit("/", 1)[-1] for option in refusal_schema["anyOf"]} + assert refusals == {"V2ErrorResponse", "V2MissingParametersErrorResponse"} + missing_parameters = schema["components"]["schemas"][ + "V2MissingParametersErrorResponse" + ] + assert set(missing_parameters["properties"]) == { + "status", + "message", + "code", + "missing_parameters", + } diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index fbab7a0dc..edc7ba7c0 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -333,6 +333,8 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: "/v2/countries", "/v2/households", "/v2/households/{household_id}", + "/v2/household-calculations", + "/v2/household-calculations/{calculation_id}", "/v2/policies", "/v2/policies/{policy_id}", "/v2/user-households", From 4b24d1b163c9c8cab8a6b24e08f505fb4bfed48f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:21:47 +0400 Subject: [PATCH 10/57] Serve POST and GET /v2/household-calculations POST takes a stored household, an optional stored policy, axes, add_missing_variables and SPM settings, needs no account, and answers 202 with the new job's ID and status, the household and policy, and the year calculated. A policy setting parameters the running catalog version lacks gets 422 with code missing_parameters and their names, in a new error model in errors_support that later calculation routes can share. GET reads the job from household_jobs and answers its status, result and error. The service is built by a default factory that imports the simulation client only on first use, and the new modules join ruff's list of modules loaded only inside functions. Co-Authored-By: Claude Opus 5.5 --- .../fastapi_routes/dependencies.py | 33 +++ .../resource_service_protocols.py | 31 +++ .../fastapi_routes/v2/errors_support.py | 22 ++ .../v2/household_calculations/__init__.py | 1 + .../household_calculations/request_models.py | 27 +++ .../household_calculations/response_models.py | 198 ++++++++++++++++++ .../v2/household_calculations/routes.py | 169 +++++++++++++++ policyengine_api/fastapi_routes/v2/routes.py | 4 + pyproject.toml | 10 +- 9 files changed, 493 insertions(+), 2 deletions(-) create mode 100644 policyengine_api/fastapi_routes/v2/household_calculations/__init__.py create mode 100644 policyengine_api/fastapi_routes/v2/household_calculations/request_models.py create mode 100644 policyengine_api/fastapi_routes/v2/household_calculations/response_models.py create mode 100644 policyengine_api/fastapi_routes/v2/household_calculations/routes.py diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 9f4c12028..303b4374f 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -17,6 +17,7 @@ from policyengine_api.fastapi_routes.resource_service_protocols import ( V2AccountResourceService, V2CountryResourceService, + V2HouseholdCalculationResourceService, V2HouseholdResourceService, V2PolicyResourceService, V2UserHouseholdResourceService, @@ -58,6 +59,7 @@ "SimulationGatewayProbe", "V2AccountResourceService", "V2CountryResourceService", + "V2HouseholdCalculationResourceService", "V2HouseholdResourceService", "V2MetadataResourceReader", "V2PolicyResourceService", @@ -170,6 +172,31 @@ def _default_v2_household_service_factory() -> V2HouseholdResourceService: ) +def _default_v2_household_calculation_service_factory() -> ( + V2HouseholdCalculationResourceService +): + from policyengine_api.data.v2.database import ( + get_v2_session_factory, + ) + from policyengine_api.libs.simulation_household_calculations import ( + process_simulation_household_calculation_client, + ) + from policyengine_api.services.v2.household_calculations.database_session import ( + HouseholdCalculationDatabaseSession, + ) + from policyengine_api.services.v2.household_calculations.services import ( + V2HouseholdCalculationService, + ) + + return V2HouseholdCalculationService( + countries=_default_v2_country_service_factory(), + households=_default_v2_household_service_factory(), + policies=_default_v2_policy_service_factory(), + database_session=HouseholdCalculationDatabaseSession(get_v2_session_factory()), + simulation_factory=process_simulation_household_calculation_client, + ) + + _user_bearer_scheme = HTTPBearer( auto_error=False, scheme_name="UserAccessToken", @@ -331,6 +358,9 @@ class NativeRouteDependencies: v2_country_service_factory: Callable[[], V2CountryResourceService] | None = None v2_policy_service_factory: Callable[[], V2PolicyResourceService] | None = None v2_household_service_factory: Callable[[], V2HouseholdResourceService] | None = None + v2_household_calculation_service_factory: ( + Callable[[], V2HouseholdCalculationResourceService] | None + ) = None v2_user_policy_service_factory: Callable[[], V2UserPolicyResourceService] | None = ( None ) @@ -365,6 +395,9 @@ def defaults(cls) -> NativeRouteDependencies: v2_metadata_response_cache=_process_metadata_response_cache(), v2_policy_service_factory=_default_v2_policy_service_factory, v2_household_service_factory=_default_v2_household_service_factory, + v2_household_calculation_service_factory=( + _default_v2_household_calculation_service_factory + ), v2_user_policy_service_factory=(_default_v2_user_policy_service_factory), v2_user_household_service_factory=( _default_v2_user_household_service_factory diff --git a/policyengine_api/fastapi_routes/resource_service_protocols.py b/policyengine_api/fastapi_routes/resource_service_protocols.py index de03ca04a..ddae5f697 100644 --- a/policyengine_api/fastapi_routes/resource_service_protocols.py +++ b/policyengine_api/fastapi_routes/resource_service_protocols.py @@ -8,6 +8,11 @@ from uuid import UUID from policyengine_api.services.v2.countries.types import CountryRead + from policyengine_api.services.v2.household_calculations.types import ( + HouseholdCalculationInput, + HouseholdCalculationRead, + HouseholdCalculationSubmitted, + ) from policyengine_api.services.v2.households.types import ( HouseholdPage, HouseholdRead, @@ -139,6 +144,32 @@ def list_households( ... +class V2HouseholdCalculationResourceService(Protocol): + """Route-independent household calculation operations for one request.""" + + def submit_calculation( + self, calculation: HouseholdCalculationInput + ) -> HouseholdCalculationSubmitted: + """Send a stored household's calculation to the simulation service. + + Raise ``HouseholdCalculationCountryError`` for a country without + household calculations, ``HouseholdNotFoundError`` or + ``PolicyNotFoundError`` for a record not stored in that country, + ``HouseholdCalculationYearError`` for a household without a default + year, ``PolicyParametersMissingError`` for a policy setting parameters + the running catalog version lacks, and the simulation service errors + when it refuses the calculation. + """ + ... + + def get_calculation(self, calculation_id: UUID) -> HouseholdCalculationRead: + """Return the calculation household_jobs records. + + Raise ``HouseholdCalculationNotFoundError`` when there is no such job. + """ + ... + + class V2UserPolicyResourceService(Protocol): """Route-independent user-policy operations for one request.""" diff --git a/policyengine_api/fastapi_routes/v2/errors_support.py b/policyengine_api/fastapi_routes/v2/errors_support.py index 0c1f38525..f922c46f2 100644 --- a/policyengine_api/fastapi_routes/v2/errors_support.py +++ b/policyengine_api/fastapi_routes/v2/errors_support.py @@ -46,6 +46,18 @@ class V2ErrorResponse(BaseModel): message: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] +class V2MissingParametersErrorResponse(V2ErrorResponse): + """A refused calculation whose policy sets parameters the catalog lacks. + + ``missing_parameters`` names each parameter the policy sets that the + running catalog version does not have, sorted; the calculation would + otherwise run without those values. + """ + + code: Literal["missing_parameters"] = "missing_parameters" + missing_parameters: list[str] + + def v2_error_response( status_code: int, message: str, @@ -61,6 +73,16 @@ def v2_error_response( ) +def v2_missing_parameters_response( + message: str, missing_parameters: Sequence[str] +) -> JSONResponse: + """Serialize a 422 refusing a policy whose parameters the catalog lacks.""" + error = V2MissingParametersErrorResponse( + message=message, missing_parameters=list(missing_parameters) + ) + return JSONResponse(status_code=422, content=error.model_dump(mode="json")) + + OperationT = TypeVar("OperationT") diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/__init__.py b/policyengine_api/fastapi_routes/v2/household_calculations/__init__.py new file mode 100644 index 000000000..59e3798f7 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/household_calculations/__init__.py @@ -0,0 +1 @@ +"""Native FastAPI routes calculating stored v2 households.""" diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/request_models.py b/policyengine_api/fastapi_routes/v2/household_calculations/request_models.py new file mode 100644 index 000000000..4379aaa83 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/household_calculations/request_models.py @@ -0,0 +1,27 @@ +"""Strict HTTP request models for v2 household calculations.""" + +from __future__ import annotations + +from policyengine_api.services.v2.household_calculations.types import ( + HouseholdCalculationInput, +) + + +class HouseholdCalculationRequest(HouseholdCalculationInput): + """A stored household to calculate under a stored policy, or current law. + + ``household_id`` and ``policy_id`` name records of ``country_id`` stored + through ``/v2/households`` and ``/v2/policies``; without ``policy_id`` the + household is calculated under current law, for the household's + ``default_year``. Each group in ``axes`` varies its inputs together across + ``count`` points; several groups form a grid of at most 10,000 points. + ``add_missing_variables`` adds every variable the household lacks before + calculating. ``spm`` selects the SPM forecast scenario and geography for + a US household. Each submission starts a new calculation. + """ + + def to_service_input(self) -> HouseholdCalculationInput: + """Convert the request to the calculation service's input.""" + return HouseholdCalculationInput.model_validate( + self.model_dump(mode="python", exclude_unset=True) + ) diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py new file mode 100644 index 000000000..b395fd8cf --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py @@ -0,0 +1,198 @@ +"""Strict HTTP response models for v2 household calculations.""" + +from __future__ import annotations + +from datetime import datetime +from typing import TYPE_CHECKING, Any, Generic, Literal, TypeVar +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, JsonValue + +from policyengine_api.fastapi_routes.v2.errors_support import ( + V2ErrorResponse, + V2MissingParametersErrorResponse, +) + +if TYPE_CHECKING: + from policyengine_api.services.v2.household_calculations.types import ( + HouseholdCalculationRead, + HouseholdCalculationSubmitted, + ) + +CalculationStatus = Literal["pending", "running", "succeeded", "failed"] + + +class StrictHouseholdCalculationAPIModel(BaseModel): + """Base for calculation responses; rejects unknown fields and NaN or infinity.""" + + model_config = ConfigDict(extra="forbid", allow_inf_nan=False, from_attributes=True) + + +class HouseholdCalculationSubmittedItem(StrictHouseholdCalculationAPIModel): + """A calculation the simulation service accepted. + + Poll ``GET /v2/household-calculations/{calculation_id}`` until ``status`` + is ``succeeded`` or ``failed``. ``year`` is the household's default year, + which is calculated. + """ + + calculation_id: UUID + status: CalculationStatus + country_id: str + household_id: UUID + policy_id: UUID | None + year: int + + @classmethod + def from_submitted( + cls, calculation: HouseholdCalculationSubmitted + ) -> HouseholdCalculationSubmittedItem: + """Build the response item from the service's accepted calculation.""" + return cls.model_validate(calculation) + + +class HouseholdCalculationOutputItem(StrictHouseholdCalculationAPIModel): + """A succeeded calculation's household. + + ``household`` is the stored household in the situation shape (people and + groups keyed by their record IDs) with each calculated value filled in by + period. ``warnings`` lists inputs the country model ignored. In the US, + ``spm_config`` is the SPM selection the calculation applied and + ``spm_provenance`` its receipt; both are null elsewhere. + """ + + household: dict[str, JsonValue] + warnings: list[str] + spm_config: dict[str, JsonValue] | None + spm_provenance: dict[str, JsonValue] | None + + +class HouseholdCalculationErrorItem(StrictHouseholdCalculationAPIModel): + """Why a calculation failed. + + ``code`` is the household function's error code: ``invalid_inputs``, + with ``input_errors`` naming each input the country model does not + recognize; ``invalid_household``; ``invalid_spm_selection``, with + ``spm_error``; ``bundle_mismatch``; or ``calculation_failed``. It is null + when the calculation failed for another reason, such as a Modal call that + ended; ``message`` then says why, and submitting again may succeed. + """ + + code: str | None + message: str + input_errors: list[dict[str, JsonValue]] + spm_error: dict[str, JsonValue] | None + + +class HouseholdCalculationItem(StrictHouseholdCalculationAPIModel): + """A household calculation and its outcome. + + ``status`` moves from ``pending`` through ``running`` to ``succeeded``, + with ``result``, or ``failed``, with ``error``. ``catalog_version_id``, + ``country_package_version`` and ``policyengine_version`` are those the + calculation runs with. + """ + + calculation_id: UUID + country_id: str + household_id: UUID + policy_id: UUID | None + year: int + catalog_version_id: UUID + country_package_version: str + policyengine_version: str + status: CalculationStatus + created_at: datetime + started_at: datetime | None + completed_at: datetime | None + result: HouseholdCalculationOutputItem | None + error: HouseholdCalculationErrorItem | None + + @classmethod + def from_read( + cls, calculation: HouseholdCalculationRead + ) -> HouseholdCalculationItem: + """Build the response item from the service's calculation.""" + return cls.model_validate(calculation) + + +class HouseholdCalculationSubmittedResult(StrictHouseholdCalculationAPIModel): + """Result holding the accepted calculation in ``item``.""" + + item: HouseholdCalculationSubmittedItem + + +class HouseholdCalculationResult(StrictHouseholdCalculationAPIModel): + """Result holding one calculation in ``item``.""" + + item: HouseholdCalculationItem + + +ResultT = TypeVar("ResultT") + + +class HouseholdCalculationSuccessResponse( + StrictHouseholdCalculationAPIModel, Generic[ResultT] +): + """Success envelope: ``status`` "ok", a null ``message`` and the ``result``.""" + + status: Literal["ok"] = "ok" + message: None = None + result: ResultT + + +class HouseholdCalculationSubmittedResponse( + HouseholdCalculationSuccessResponse[HouseholdCalculationSubmittedResult] +): + """Success response holding the accepted calculation.""" + + +class HouseholdCalculationResponse( + HouseholdCalculationSuccessResponse[HouseholdCalculationResult] +): + """Success response holding one calculation.""" + + +HOUSEHOLD_CALCULATION_ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { + 404: { + "model": V2ErrorResponse, + "description": "The household, policy or calculation does not exist.", + }, + 422: { + "model": V2ErrorResponse, + "description": "The request does not match the schema.", + }, + 500: { + "model": V2ErrorResponse, + "description": ( + "The calculation failed unexpectedly, or a succeeded calculation's " + "stored result cannot be read." + ), + }, + 503: { + "model": V2ErrorResponse, + "description": "The simulation service or v2 storage is unavailable.", + }, +} + +SUBMISSION_REFUSAL_RESPONSES: dict[int | str, dict[str, Any]] = { + 422: { + "model": V2MissingParametersErrorResponse | V2ErrorResponse, + "description": ( + "The request does not match the schema; the country does not offer " + "household calculations; the household has no default year; the " + "policy sets parameters the running catalog version does not have " + "(code missing_parameters, which names them); or the simulation " + "service refuses the caller's data, with its reason in the message." + ), + }, + 503: { + "model": V2ErrorResponse, + "description": ( + "The simulation service is unavailable, cannot be reached, answers " + "in a shape the Public API cannot read, or refuses a household, " + "policy or catalog version the Public API has already checked; or " + "v2 storage is unavailable." + ), + }, +} diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/routes.py b/policyengine_api/fastapi_routes/v2/household_calculations/routes.py new file mode 100644 index 000000000..059ce8f7d --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/household_calculations/routes.py @@ -0,0 +1,169 @@ +"""Native FastAPI routes calculating stored households, and reading the results. + +The calculation runs on the simulation service, which records it as a job in +``household_jobs``; the routes need no account. Submitting sends the +simulation service the stored household's and policy's IDs once the Public +API's own checks pass; reading reads the job from the database and never +calls the simulation service. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, TypeVar +from uuid import UUID + +from fastapi import APIRouter +from starlette.responses import JSONResponse + +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.fastapi_routes.dependencies import ( + _default_v2_household_calculation_service_factory, +) +from policyengine_api.fastapi_routes.v2.errors_support import ( + V2ErrorMapping, + run_v2_operation, + v2_missing_parameters_response, +) +from policyengine_api.fastapi_routes.v2.household_calculations.request_models import ( + HouseholdCalculationRequest, +) +from policyengine_api.fastapi_routes.v2.household_calculations.response_models import ( + HOUSEHOLD_CALCULATION_ERROR_RESPONSES, + SUBMISSION_REFUSAL_RESPONSES, + HouseholdCalculationItem, + HouseholdCalculationResponse, + HouseholdCalculationResult, + HouseholdCalculationSubmittedItem, + HouseholdCalculationSubmittedResponse, + HouseholdCalculationSubmittedResult, +) +from policyengine_api.services.v2.household_calculations.types import ( + SimulationServiceInvalidRequestError, + SimulationServiceUnavailableError, +) +from policyengine_api.services.v2.household_calculations.validators import ( + HouseholdCalculationCountryError, + HouseholdCalculationNotFoundError, + HouseholdCalculationResultUnreadableError, + HouseholdCalculationStorageError, + HouseholdCalculationYearError, +) +from policyengine_api.services.v2.households.validators import HouseholdNotFoundError +from policyengine_api.services.v2.policies.validators import ( + PolicyNotFoundError, + PolicyParametersMissingError, +) + +if TYPE_CHECKING: + from collections.abc import Callable + + from policyengine_api.fastapi_routes.dependencies import ( + NativeRouteDependencies, + V2HouseholdCalculationResourceService, + ) + +OperationT = TypeVar("OperationT") + +STORAGE_UNAVAILABLE = "V2 household calculation storage is unavailable" +HOUSEHOLD_CALCULATION_ERROR_MAPPINGS = ( + V2ErrorMapping( + ( + HouseholdCalculationCountryError, + HouseholdCalculationYearError, + SimulationServiceInvalidRequestError, + ), + 422, + ), + V2ErrorMapping( + ( + HouseholdNotFoundError, + PolicyNotFoundError, + HouseholdCalculationNotFoundError, + ), + 404, + ), + V2ErrorMapping((SimulationServiceUnavailableError,), 503), + V2ErrorMapping( + (HouseholdCalculationResultUnreadableError,), + 500, + "Stored household calculation result is unreadable", + ), + V2ErrorMapping( + (HouseholdCalculationStorageError, V2ConfigurationError), + 503, + STORAGE_UNAVAILABLE, + ), +) + + +def _service_factory( + dependencies: NativeRouteDependencies, +) -> Callable[[], V2HouseholdCalculationResourceService]: + if dependencies.v2_household_calculation_service_factory is not None: + return dependencies.v2_household_calculation_service_factory + return _default_v2_household_calculation_service_factory + + +def _calculation_operation( + operation: Callable[[], OperationT], +) -> OperationT | JSONResponse: + return run_v2_operation( + operation, + HOUSEHOLD_CALCULATION_ERROR_MAPPINGS, + fallback_message="V2 household calculation failed", + ) + + +def build_v2_household_calculation_router( + dependencies: NativeRouteDependencies, +) -> APIRouter: + """Build the household calculation routes without opening any connection.""" + router = APIRouter(prefix="/v2", responses=HOUSEHOLD_CALCULATION_ERROR_RESPONSES) + service_factory = _service_factory(dependencies) + + @router.post( + "/household-calculations", + response_model=HouseholdCalculationSubmittedResponse, + status_code=202, + responses=SUBMISSION_REFUSAL_RESPONSES, + summary="Calculate a stored household under a stored policy", + ) + def submit_household_calculation( + body: HouseholdCalculationRequest, + ) -> HouseholdCalculationSubmittedResponse | JSONResponse: + def submit() -> HouseholdCalculationSubmittedResponse | JSONResponse: + try: + calculation = service_factory().submit_calculation( + body.to_service_input() + ) + except PolicyParametersMissingError as error: + return v2_missing_parameters_response( + str(error), error.missing_parameters + ) + return HouseholdCalculationSubmittedResponse( + result=HouseholdCalculationSubmittedResult( + item=HouseholdCalculationSubmittedItem.from_submitted(calculation) + ) + ) + + return _calculation_operation(submit) + + @router.get( + "/household-calculations/{calculation_id}", + response_model=HouseholdCalculationResponse, + summary="Read a household calculation and its result", + ) + def get_household_calculation( + calculation_id: UUID, + ) -> HouseholdCalculationResponse | JSONResponse: + def read() -> HouseholdCalculationResponse: + calculation = service_factory().get_calculation(calculation_id) + return HouseholdCalculationResponse( + result=HouseholdCalculationResult( + item=HouseholdCalculationItem.from_read(calculation) + ) + ) + + return _calculation_operation(read) + + return router diff --git a/policyengine_api/fastapi_routes/v2/routes.py b/policyengine_api/fastapi_routes/v2/routes.py index 2309feb46..e268372ac 100644 --- a/policyengine_api/fastapi_routes/v2/routes.py +++ b/policyengine_api/fastapi_routes/v2/routes.py @@ -11,6 +11,9 @@ build_v2_country_router, ) from policyengine_api.fastapi_routes.v2.errors_support import V2ErrorResponse +from policyengine_api.fastapi_routes.v2.household_calculations.routes import ( + build_v2_household_calculation_router, +) from policyengine_api.fastapi_routes.v2.households.routes import ( build_v2_household_router, ) @@ -58,6 +61,7 @@ def build_v2_router( router.include_router(build_v2_user_household_router(dependencies)) router.include_router(build_v2_user_policy_router(dependencies)) router.include_router(build_v2_household_router(dependencies)) + router.include_router(build_v2_household_calculation_router(dependencies)) router.include_router(build_v2_policy_router(dependencies)) router.include_router(build_v2_country_router(dependencies)) router.include_router(build_v2_metadata_model_router(dependencies)) diff --git a/pyproject.toml b/pyproject.toml index ae57b9d35..d07b3bf81 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -383,8 +383,10 @@ showcontent = true # never at module level (TID253 enforces it; TYPE_CHECKING imports are fine): # policyengine and spm_calculator load country models, the v2 database module # registers an exit hook at import, the v2 database sessions and services load -# persistence only when a default factory runs, and Auth0 Management and the -# Google Secret Manager client load only when a secret is read. Ruff's PLC0415 +# persistence only when a default factory runs, the household calculation +# client of the simulation service loads only when a calculation is submitted, +# and Auth0 Management and the Google Secret Manager client load only when a +# secret is read. Ruff's PLC0415 # accepts in-function imports of these without a suppression. A module belongs # here only if no application module imports it at module level. banned-module-level-imports = [ @@ -392,8 +394,11 @@ banned-module-level-imports = [ "policyengine", "policyengine_api.data.v2.database", "policyengine_api.libs.auth0_management", + "policyengine_api.libs.simulation_household_calculations", "policyengine_api.services.v2.countries.database_session", "policyengine_api.services.v2.countries.services", + "policyengine_api.services.v2.household_calculations.database_session", + "policyengine_api.services.v2.household_calculations.services", "policyengine_api.services.v2.households.database_session", "policyengine_api.services.v2.households.services", "policyengine_api.services.v2.legacy_refs.database_session", @@ -449,6 +454,7 @@ runtime-evaluated-base-classes = [ "policyengine_api.query_parameters.PaginationQuery", "policyengine_api.query_parameters.StrictQueryParameters", "policyengine_api.services.v2.associations.types.StrictAssociationModel", + "policyengine_api.services.v2.household_calculations.types.HouseholdCalculationInput", "policyengine_api.services.v2.households.types.HouseholdCreationInput", "policyengine_api.services.v2.households.types.StrictHouseholdInput", "policyengine_api.services.v2.metadata.types.MetadataModelVersion", From b13d762538ca6f949eed732a42b960dd34b4bf6a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:23:43 +0400 Subject: [PATCH 11/57] Test household calculations' route group, request log and contracts (failing) /v2/household-calculations belongs to the household route group. Its request log records what it uses whatever the household flags select: a submission reads Supabase and computes on the Cloud Run simulation service's v2 household functions, a read reads Supabase only. The contract registry and the generated contract documents gain the two routes. None of this exists yet. Co-Authored-By: Claude Opus 5.5 --- .../test_app_v2_workflow_contracts.py | 2 + .../routes/test_migration_context_logging.py | 76 +++++++++++++++++++ .../unit/test_migration_contract_artifacts.py | 5 +- tests/unit/test_migration_flags.py | 5 ++ 4 files changed, 86 insertions(+), 2 deletions(-) diff --git a/tests/contract/test_app_v2_workflow_contracts.py b/tests/contract/test_app_v2_workflow_contracts.py index 6be078e41..20c6733d3 100644 --- a/tests/contract/test_app_v2_workflow_contracts.py +++ b/tests/contract/test_app_v2_workflow_contracts.py @@ -13,6 +13,7 @@ def test_app_v2_workflow_contract_registry_is_complete(): "user_policy_associations_v2", "household_save_read", "household_resources_v2", + "household_calculations_v2", "user_household_associations_v2", "user_simulation_associations_v2", "user_report_associations_v2", @@ -36,6 +37,7 @@ def test_app_v2_workflow_contract_registry_is_complete(): "policy_resources_v2", "user_policy_associations_v2", "household_resources_v2", + "household_calculations_v2", "user_household_associations_v2", "user_simulation_associations_v2", "user_report_associations_v2", diff --git a/tests/unit/routes/test_migration_context_logging.py b/tests/unit/routes/test_migration_context_logging.py index 97d280da7..fd5ca1142 100644 --- a/tests/unit/routes/test_migration_context_logging.py +++ b/tests/unit/routes/test_migration_context_logging.py @@ -723,3 +723,79 @@ def test_compute_routes_in_the_same_groups_keep_their_compute_labels(): assert context["route_group"] == "simulation" assert context["sim_flow"] == "economy" assert context["sim_compute"] is not None + + +def _log_household_calculation(method: str, path: str) -> dict: + with patch("policyengine_api.migration_logging.logger") as mock_logger: + log_migration_request( + ServedRequest( + request_id=f"request-{method}", + method=method, + path=path, + status_code=202 if method == "POST" else 200, + started_at=None, + route_impl=RouteImplementation.FASTAPI_NATIVE, + ) + ) + return mock_logger.log_struct.call_args.args[0]["migration"] + + +def test_household_calculations_log_the_sources_they_use_whatever_the_flags_say( + monkeypatch, +): + # None of these flags applies to /v2/household-calculations, so invalid + # values must not reach its log record. + monkeypatch.setenv("DB_READ_HOUSEHOLD", "invalid-for-household-calculations") + monkeypatch.setenv("DB_WRITE_HOUSEHOLD", "invalid-for-household-calculations") + monkeypatch.setenv("SIM_ENTRYPOINT", "invalid-for-household-calculations") + monkeypatch.setenv("SIM_COMPUTE_HOUSEHOLD", "invalid-for-household-calculations") + + submitted = _log_household_calculation("POST", "/v2/household-calculations") + read = _log_household_calculation( + "GET", "/v2/household-calculations/00000000-0000-0000-0000-000000000001" + ) + + # A submission reads the household and policy from Supabase and computes + # on the Cloud Run simulation service's v2 household functions. + assert submitted == { + "route_group": "household", + "route_impl": "fastapi_native", + "db_entity": "household", + "db_write": None, + "db_read": "supabase", + "sim_flow": "household", + "sim_entrypoint": "cloud_run_simulation_entrypoint", + "sim_compute": "v2_primary", + } + # A read reads the job from Supabase and calls nothing. + assert read == { + "route_group": "household", + "route_impl": "fastapi_native", + "db_entity": "household", + "db_write": None, + "db_read": "supabase", + "sim_flow": None, + "sim_entrypoint": "cloud_run_simulation_entrypoint", + "sim_compute": None, + } + + +def test_household_calculations_report_an_invalid_route_implementation( + monkeypatch, +): + monkeypatch.setenv("ROUTE_IMPL_HOUSEHOLD", "invalid") + + with patch("policyengine_api.migration_logging.logger") as mock_logger: + log_migration_request( + ServedRequest( + request_id="request", + method="POST", + path="/v2/household-calculations", + status_code=202, + started_at=None, + ) + ) + + context = mock_logger.log_struct.call_args.args[0]["migration"] + assert context["route_group"] == "household" + assert "ROUTE_IMPL_HOUSEHOLD" in context["migration_flag_error"] diff --git a/tests/unit/test_migration_contract_artifacts.py b/tests/unit/test_migration_contract_artifacts.py index 497d5489b..12b96afae 100644 --- a/tests/unit/test_migration_contract_artifacts.py +++ b/tests/unit/test_migration_contract_artifacts.py @@ -13,8 +13,8 @@ def test_migration_contract_payload_summarizes_route_contracts(): assert payload["version"] == 1 assert payload["metadata"] == { "route_group_count": 9, - "workflow_count": 16, - "request_count": 90, + "workflow_count": 17, + "request_count": 92, "db_entity_count": 6, "sim_flow_count": 3, } @@ -24,6 +24,7 @@ def test_migration_contract_payload_summarizes_route_contracts(): "user_policy_associations_v2", "household_save_read", "household_resources_v2", + "household_calculations_v2", "user_household_associations_v2", "user_simulation_associations_v2", "user_report_associations_v2", diff --git a/tests/unit/test_migration_flags.py b/tests/unit/test_migration_flags.py index e110da879..4ca5dd68f 100644 --- a/tests/unit/test_migration_flags.py +++ b/tests/unit/test_migration_flags.py @@ -181,6 +181,11 @@ def test_explicit_migration_context_rejects_invalid_database_sources( "policy", ), ("/v2/user-households", "household"), + ("/v2/household-calculations", "household"), + ( + "/v2/household-calculations/00000000-0000-0000-0000-000000000001", + "household", + ), ("/v2/user-simulations", "simulation"), ("/v2/user-reports/00000000-0000-0000-0000-000000000001", "report"), ("/v2/me", "user_profile"), From 291e230c58323e23ed2919450774fe72e1d7e0c0 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:24:45 +0400 Subject: [PATCH 12/57] Log household calculations' sources and add them to the migration contracts /v2/household-calculations joins the household route group. Its request log records the sources it uses, not the household group's database, entrypoint and compute flags: both routes read Supabase, and a submission computes on the Cloud Run simulation service's v2 household functions. The contract registry gains the two routes, and the generated contract documents are regenerated. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/migration-contracts.md | 16 ++++++-- docs/generated/migration_contracts.json | 54 ++++++++++++++++++++++++- policyengine_api/migration_logging.py | 38 +++++++++++++++++ policyengine_api/migration_registry.py | 1 + tests/contract/registry.py | 47 +++++++++++++++++++++ 5 files changed, 151 insertions(+), 5 deletions(-) diff --git a/docs/engineering/migration-contracts.md b/docs/engineering/migration-contracts.md index 556de0a1f..1974f6c08 100644 --- a/docs/engineering/migration-contracts.md +++ b/docs/engineering/migration-contracts.md @@ -7,8 +7,8 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | Metric | Count | | --- | ---: | | route group count | 9 | -| workflow count | 16 | -| request count | 90 | +| workflow count | 17 | +| request count | 92 | | db entity count | 6 | | sim flow count | 3 | @@ -20,7 +20,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `specification` | `specification` | `none` | `none` | | `metadata` | `metadata`, `countries`, `datasets`, `economy-options`, `model-catalog`, `parameter-values`, `parameters`, `regions`, `tax-benefit-model-versions`, `tax-benefit-models`, `variables` | `metadata` | `none` | | `policy` | `policy`, `policies`, `user-policy`, `user-policies` | `policy` | `none` | -| `household` | `household`, `households`, `user-household`, `user-households`, `calculate`, `calculate-full` | `household` | `household` | +| `household` | `household`, `households`, `household-calculations`, `user-household`, `user-households`, `calculate`, `calculate-full` | `household` | `household` | | `economy` | `economy` | `simulation` | `economy` | | `simulation` | `simulation`, `simulations`, `user-simulations` | `simulation` | `economy` | | `report` | `report`, `user-reports`, `shared-reports` | `report` | `report` | @@ -88,6 +88,16 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `GET` | `/v2/households/{household_id}?country_id=us` | 200 | `household` | none | `status`, `message`, `result.item.id`, `result.item.country_id`, `result.item.default_year`, `result.item.household_data`, `result.item.created_at`, `result.item.updated_at` | | `GET` | `/v2/households?country_id=us` | 200 | `household` | none | `status`, `message`, `result.items`, `result.offset`, `result.limit`, `result.has_more` | +### `household_calculations_v2` + +- Current contract: `typed_v2_resources` +- Future owner: Stored-household calculations through the simulation service + +| Method | Path | Status | Route group | Required headers | Stable response fields | +| --- | --- | ---: | --- | --- | --- | +| `POST` | `/v2/household-calculations` | 202 | `household` | none | `status`, `message`, `result.item.calculation_id`, `result.item.status`, `result.item.country_id`, `result.item.household_id`, `result.item.policy_id`, `result.item.year` | +| `GET` | `/v2/household-calculations/{calculation_id}` | 200 | `household` | none | `status`, `message`, `result.item.calculation_id`, `result.item.country_id`, `result.item.household_id`, `result.item.policy_id`, `result.item.year`, `result.item.catalog_version_id`, `result.item.country_package_version`, `result.item.policyengine_version`, `result.item.status`, `result.item.created_at`, `result.item.started_at`, `result.item.completed_at`, `result.item.result`, `result.item.error` | + ### `user_household_associations_v2` - Current contract: `typed_v2_resources` diff --git a/docs/generated/migration_contracts.json b/docs/generated/migration_contracts.json index 26b037711..dcf1697d0 100644 --- a/docs/generated/migration_contracts.json +++ b/docs/generated/migration_contracts.json @@ -1,10 +1,10 @@ { "metadata": { "db_entity_count": 6, - "request_count": 90, + "request_count": 92, "route_group_count": 9, "sim_flow_count": 3, - "workflow_count": 16 + "workflow_count": 17 }, "route_groups": [ { @@ -62,6 +62,7 @@ "path_segments": [ "household", "households", + "household-calculations", "user-household", "user-households", "calculate", @@ -484,6 +485,55 @@ } ] }, + { + "current_contract": "typed_v2_resources", + "future_owner_pr": "Stored-household calculations through the simulation service", + "name": "household_calculations_v2", + "requests": [ + { + "expected_status": 202, + "headers": [], + "method": "POST", + "path": "/v2/household-calculations", + "route_group": "household", + "stable_response_fields": [ + "status", + "message", + "result.item.calculation_id", + "result.item.status", + "result.item.country_id", + "result.item.household_id", + "result.item.policy_id", + "result.item.year" + ] + }, + { + "expected_status": 200, + "headers": [], + "method": "GET", + "path": "/v2/household-calculations/{calculation_id}", + "route_group": "household", + "stable_response_fields": [ + "status", + "message", + "result.item.calculation_id", + "result.item.country_id", + "result.item.household_id", + "result.item.policy_id", + "result.item.year", + "result.item.catalog_version_id", + "result.item.country_package_version", + "result.item.policyengine_version", + "result.item.status", + "result.item.created_at", + "result.item.started_at", + "result.item.completed_at", + "result.item.result", + "result.item.error" + ] + } + ] + }, { "current_contract": "typed_v2_resources", "future_owner_pr": "PR 12A: User Login and v1-Anchored User Associations", diff --git a/policyengine_api/migration_logging.py b/policyengine_api/migration_logging.py index 774b42641..668729324 100644 --- a/policyengine_api/migration_logging.py +++ b/policyengine_api/migration_logging.py @@ -13,10 +13,13 @@ from policyengine_api.gcp_logging import logger from policyengine_api.migration_flags import ( ExplicitDatabaseSources, + MigrationContext, RouteImplementation, get_migration_log_context, + get_route_impl, infer_route_group, ) +from policyengine_api.migration_registry import ROUTE_GROUP_CONFIG_BY_NAME from policyengine_api.observability.identifiers import ( OBSERVABILITY_ID_HEADER, normalize_observability_id, @@ -62,6 +65,15 @@ V2_USER_RESOURCE_SEGMENTS = frozenset( {"user-policies", "user-households", "user-simulations", "user-reports", "me"} ) +# /v2/household-calculations reads only Supabase and computes only on the +# Cloud Run simulation service's v2 household functions, whatever the +# household route group's database, entrypoint and compute flags select: a +# submission reads the countries row, household and policy and sends the +# calculation there, and a read reads the job and calls nothing. It reads +# none of those flags, so none of them is checked for its log record. +V2_HOUSEHOLD_CALCULATION_SEGMENT = "household-calculations" +V2_HOUSEHOLD_CALCULATION_ENTRYPOINT = "cloud_run_simulation_entrypoint" +V2_HOUSEHOLD_CALCULATION_COMPUTE = "v2_primary" V2_RESOURCE_METHODS = frozenset({"GET", "POST", "PATCH", "DELETE"}) # A v2 resource path has at least the "v2" segment and the resource segment. V2_RESOURCE_PATH_MINIMUM_SEGMENTS = 2 @@ -245,6 +257,28 @@ def log_migration_request(served_request: ServedRequest) -> None: ) +def _household_calculation_context( + route_group: str, *, method: str, route_impl: RouteImplementation | None +) -> dict[str, str | None]: + """Return a household calculation's log record from what it actually uses.""" + try: + selected_route_impl = route_impl or get_route_impl(route_group) + except ValueError as error: + return {"route_group": route_group, "migration_flag_error": str(error)} + route_config = ROUTE_GROUP_CONFIG_BY_NAME[route_group] + submits = method == "POST" + return MigrationContext( + route_group=route_group, + route_impl=selected_route_impl, + db_entity=route_config.db_entity, + db_write=None, + db_read="supabase", + sim_flow=route_config.sim_flow if submits else None, + sim_entrypoint=V2_HOUSEHOLD_CALCULATION_ENTRYPOINT, + sim_compute=V2_HOUSEHOLD_CALCULATION_COMPUTE if submits else None, + ).to_log_dict() + + def _migration_context( *, method: str, @@ -252,6 +286,10 @@ def _migration_context( route_impl: RouteImplementation | None, ) -> dict[str, str | None]: route_group = infer_route_group(path) + if _v2_resource_segment(path) == V2_HOUSEHOLD_CALCULATION_SEGMENT: + return _household_calculation_context( + route_group, method=method, route_impl=route_impl + ) is_v2_metadata_read = _is_v2_metadata_resource_read(method, path) is_v2_supabase_resource = _is_v2_supabase_resource(method, path) uses_explicit_v2_source = is_v2_metadata_read or is_v2_supabase_resource diff --git a/policyengine_api/migration_registry.py b/policyengine_api/migration_registry.py index 3afb5efe0..eafe090f9 100644 --- a/policyengine_api/migration_registry.py +++ b/policyengine_api/migration_registry.py @@ -64,6 +64,7 @@ class RouteGroupConfig: path_segments=( "household", "households", + "household-calculations", "user-household", "user-households", "calculate", diff --git a/tests/contract/registry.py b/tests/contract/registry.py index c0e70f0f1..8f03910cd 100644 --- a/tests/contract/registry.py +++ b/tests/contract/registry.py @@ -310,6 +310,53 @@ class WorkflowContract: ), ), ), + WorkflowContract( + name="household_calculations_v2", + current_contract="typed_v2_resources", + future_owner_pr="Stored-household calculations through the simulation service", + requests=( + ContractRequest( + method="POST", + path="/v2/household-calculations", + expected_status=202, + stable_response_fields=( + "status", + "message", + "result.item.calculation_id", + "result.item.status", + "result.item.country_id", + "result.item.household_id", + "result.item.policy_id", + "result.item.year", + ), + route_group="household", + ), + ContractRequest( + method="GET", + path="/v2/household-calculations/{calculation_id}", + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.item.calculation_id", + "result.item.country_id", + "result.item.household_id", + "result.item.policy_id", + "result.item.year", + "result.item.catalog_version_id", + "result.item.country_package_version", + "result.item.policyengine_version", + "result.item.status", + "result.item.created_at", + "result.item.started_at", + "result.item.completed_at", + "result.item.result", + "result.item.error", + ), + route_group="household", + ), + ), + ), WorkflowContract( name="user_household_associations_v2", current_contract="typed_v2_resources", From 7af81ff387c76961671b8343d66441d71e9ebe29 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:28:41 +0400 Subject: [PATCH 13/57] Test reading a succeeded job's single-policy result document (failing) The simulation service's household contract names exactly one policy, so a succeeded job's HouseholdExecutionResult holds that policy's household, warnings and SPM receipt in its output, beside the SPM selection applied, instead of a list of policy results. The failure document keeps its shape. The reader still expects the list, so these tests fail with an unreadable result. Co-Authored-By: Claude Opus 5.5 --- .../household_calculation_documents.py | 24 ++++++++----------- .../v2/test_household_calculation_results.py | 21 +++++----------- 2 files changed, 16 insertions(+), 29 deletions(-) diff --git a/tests/fixtures/household_calculation_documents.py b/tests/fixtures/household_calculation_documents.py index 4ab68b1af..1ee0a65d3 100644 --- a/tests/fixtures/household_calculation_documents.py +++ b/tests/fixtures/household_calculation_documents.py @@ -1,10 +1,11 @@ """Household job rows' result documents, as the v2 household functions write them. A succeeded job's ``result`` is the simulation contract's -``HouseholdExecutionResult``: the calculated household for its one policy, -with its warnings, SPM receipt and the SPM selection applied. A failed job's -is ``HouseholdExecutionFailure``, with a one-line summary in -``error_message``; a job that failed unexpectedly has a null ``result``. +``HouseholdExecutionResult``. The contract names exactly one policy, so its +``output`` holds that policy's calculated household, warnings and SPM receipt +beside the SPM selection applied. A failed job's is +``HouseholdExecutionFailure``, with a one-line summary in ``error_message``; +a job that failed unexpectedly has a null ``result``. """ from __future__ import annotations @@ -63,21 +64,17 @@ def succeeded_document(**output_changes: object) -> dict[str, Any]: - """Return a succeeded job's result document.""" + """Return a succeeded job's result document; ``None`` changes drop a field.""" output: dict[str, Any] = { "country": "us", "bundle": BUNDLE, "spm_config": SPM_CONFIG, - "results": [ - { - "policy_key": "reform", - "household": CALCULATED_HOUSEHOLD, - "warnings": [WARNING], - "spm_provenance": SPM_PROVENANCE, - } - ], + "household": CALCULATED_HOUSEHOLD, + "warnings": [WARNING], + "spm_provenance": SPM_PROVENANCE, } output.update(output_changes) + output = {name: value for name, value in output.items() if value is not None} return { "status": "succeeded", "schema_version": 1, @@ -97,7 +94,6 @@ def failed_document() -> dict[str, Any]: "code": "invalid_inputs", "message": "The household names a variable the model lacks.", "input_errors": [INPUT_ERROR], - "policy_key": "reform", }, "worker": WORKER, } diff --git a/tests/unit/v2/test_household_calculation_results.py b/tests/unit/v2/test_household_calculation_results.py index 996aee68b..730574e57 100644 --- a/tests/unit/v2/test_household_calculation_results.py +++ b/tests/unit/v2/test_household_calculation_results.py @@ -99,9 +99,7 @@ def test_a_calculated_household_keeps_the_key_order_it_was_stored_in() -> None: def test_a_country_without_spm_reads_back_no_spm_choice_or_receipt() -> None: document = succeeded_document( - country="uk", - spm_config=None, - results=[{"policy_key": "baseline", "household": CALCULATED_HOUSEHOLD}], + country="uk", spm_config=None, warnings=None, spm_provenance=None ) calculation = _read(_job(HouseholdJobStatus.SUCCEEDED, result=document)) @@ -185,21 +183,14 @@ def test_a_document_with_fields_the_public_api_does_not_read_is_still_read() -> [ pytest.param(None, id="no-document"), pytest.param(failed_document(), id="a-failure-document"), - pytest.param(succeeded_document(results=[]), id="no-result"), + pytest.param({"status": "succeeded"}, id="no-output"), + pytest.param(succeeded_document(household=None), id="no-household"), pytest.param( - succeeded_document( - results=[ - {"policy_key": "baseline", "household": CALCULATED_HOUSEHOLD}, - {"policy_key": "reform", "household": CALCULATED_HOUSEHOLD}, - ] - ), - id="two-results", + succeeded_document(household="not a household"), + id="malformed-household", ), pytest.param( - succeeded_document( - results=[{"policy_key": "reform", "household": "not a household"}] - ), - id="malformed-household", + succeeded_document(warnings="not a list"), id="malformed-warnings" ), ], ) From 365fd86cd8bcb1d5e5382c45fcba7c089a3489da Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:29:22 +0400 Subject: [PATCH 14/57] Read a succeeded job's single-policy result document The stored document models now read the household, warnings and SPM receipt from the output of a HouseholdExecutionResult that names one policy. The two document models convert themselves into the calculation's output and failure, so they are the only code that names the simulation contract's fields, and a change to those names stays in one place. Co-Authored-By: Claude Opus 5.5 --- .../household_calculations/transformations.py | 35 ++++----------- .../v2/household_calculations/types.py | 44 ++++++++++++++----- .../v2/household_calculations/validators.py | 4 +- 3 files changed, 44 insertions(+), 39 deletions(-) diff --git a/policyengine_api/services/v2/household_calculations/transformations.py b/policyengine_api/services/v2/household_calculations/transformations.py index f5ddf0c5a..06d4cc64e 100644 --- a/policyengine_api/services/v2/household_calculations/transformations.py +++ b/policyengine_api/services/v2/household_calculations/transformations.py @@ -7,7 +7,6 @@ from policyengine_api.data.v2.models import HouseholdJobStatus from policyengine_api.services.v2.household_calculations.types import ( HouseholdCalculationFailure, - HouseholdCalculationOutput, HouseholdCalculationRead, HouseholdCalculationSubmitted, StoredExecutionFailure, @@ -33,32 +32,16 @@ } -def _calculation_output(document: StoredExecutionResult) -> HouseholdCalculationOutput: - (policy_result,) = document.output.results - return HouseholdCalculationOutput( - household=policy_result.household, - warnings=tuple(policy_result.warnings), - spm_config=document.output.spm_config, - spm_provenance=policy_result.spm_provenance, - ) - - def _calculation_failure( document: StoredExecutionFailure | None, error_message: str | None ) -> HouseholdCalculationFailure: - if document is None: - return HouseholdCalculationFailure( - code=None, - message=error_message or UNEXPLAINED_FAILURE, - input_errors=(), - spm_error=None, - ) - error = document.error + if document is not None: + return document.calculation_failure() return HouseholdCalculationFailure( - code=error.code, - message=error.message, - input_errors=tuple(error.input_errors), - spm_error=error.spm_error, + code=None, + message=error_message or UNEXPLAINED_FAILURE, + input_errors=(), + spm_error=None, ) @@ -69,8 +52,8 @@ def household_calculation_read( """Return the calculation a household job records. ``outcome`` is the job's result document, already read as its status says - (``stored_household_outcome``). A succeeded job's result is its one policy - result; a failed job's error is its failure document's error, or else its + (``stored_household_outcome``). A succeeded job's result is its document's + output; a failed job's error is its failure document's error, or else its error message. """ status = STATUS_NAMES[HouseholdJobStatus(job.status)] @@ -88,7 +71,7 @@ def household_calculation_read( started_at=job.started_at, completed_at=job.completed_at, result=( - _calculation_output(outcome) + outcome.calculation_output() if status == "succeeded" and isinstance(outcome, StoredExecutionResult) else None ), diff --git a/policyengine_api/services/v2/household_calculations/types.py b/policyengine_api/services/v2/household_calculations/types.py index 528f37f8d..4d1a66703 100644 --- a/policyengine_api/services/v2/household_calculations/types.py +++ b/policyengine_api/services/v2/household_calculations/types.py @@ -201,29 +201,33 @@ class HouseholdCalculationRead: error: HouseholdCalculationFailure | None +# The result documents the household functions store in household_jobs.result. +# The simulation service's household contract owns their shape; these models +# and their two conversions are the only code that names its fields. + + class StoredDocumentModel(BaseModel): """Base for the result documents the household functions store. - The simulation service owns their shape, so fields the Public API does - not read are ignored, and a later contract may add some. + Fields the Public API does not read are ignored, so a later contract may + add some. """ model_config = ConfigDict(extra="ignore", frozen=True) -class StoredPolicyResult(StoredDocumentModel): - """The calculated household under the job's one policy.""" +class StoredCalculationOutput(StoredDocumentModel): + """A finished calculation of the job's one policy. + + ``household`` is the calculated household, ``warnings`` the inputs the + country model ignored, ``spm_provenance`` the SPM receipt and + ``spm_config`` the SPM selection applied, both absent or null without SPM. + """ household: dict[str, JsonValue] warnings: list[str] = Field(default_factory=list) - spm_provenance: dict[str, JsonValue] | None = None - - -class StoredCalculationOutput(StoredDocumentModel): - """A finished calculation: one result, for the job's one policy.""" - spm_config: dict[str, JsonValue] | None = None - results: Annotated[list[StoredPolicyResult], Field(min_length=1, max_length=1)] + spm_provenance: dict[str, JsonValue] | None = None class StoredExecutionResult(StoredDocumentModel): @@ -232,6 +236,15 @@ class StoredExecutionResult(StoredDocumentModel): status: Literal["succeeded"] output: StoredCalculationOutput + def calculation_output(self) -> HouseholdCalculationOutput: + """Return the calculated household and its notes.""" + return HouseholdCalculationOutput( + household=self.output.household, + warnings=tuple(self.output.warnings), + spm_config=self.output.spm_config, + spm_provenance=self.output.spm_provenance, + ) + class StoredCalculationError(StoredDocumentModel): """The household function's error, worded as the Public API words it.""" @@ -248,6 +261,15 @@ class StoredExecutionFailure(StoredDocumentModel): status: Literal["failed"] error: StoredCalculationError + def calculation_failure(self) -> HouseholdCalculationFailure: + """Return the household function's error.""" + return HouseholdCalculationFailure( + code=self.error.code, + message=self.error.message, + input_errors=tuple(self.error.input_errors), + spm_error=self.error.spm_error, + ) + class CountrySource(Protocol): """The countries table (``V2CountryService``).""" diff --git a/policyengine_api/services/v2/household_calculations/validators.py b/policyengine_api/services/v2/household_calculations/validators.py index 43e647988..5fe353931 100644 --- a/policyengine_api/services/v2/household_calculations/validators.py +++ b/policyengine_api/services/v2/household_calculations/validators.py @@ -70,8 +70,8 @@ def stored_household_outcome( ) -> StoredExecutionResult | StoredExecutionFailure | None: """Return a job's stored result document, read as its status says. - An unfinished job's result is not read. A succeeded job must hold one - ``HouseholdExecutionResult`` with exactly one policy result, or + An unfinished job's result is not read. A succeeded job must hold a + ``HouseholdExecutionResult`` with its policy's calculated household, or HouseholdCalculationResultUnreadableError is raised. A failed job holds a ``HouseholdExecutionFailure``, or None when it failed without one; any other document is read as none, since the job's error message still From 161cf4b4ab0bfa1093284f0283a415365982290b Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:31:11 +0400 Subject: [PATCH 15/57] Test recording SPM availability at publication and serving it (failing) Catalog publication records, for each US catalog version, whether the simulation service's public GET /versions lists a certified SPM capability for its PolicyEngine.py version, with the settings schema and defaults, in a new nullable tax_benefit_model_versions.spm_availability column added by a revision on the saved-item keys revision. /v2/model-catalog serves it as spm. Publication needs SIMULATION_ENTRYPOINT_URL, which the seeding workflow passes, and the catalog writer role records the column. None of this exists yet. Co-Authored-By: Claude Opus 5.5 --- tests/fixtures/v2_metadata_routes.py | 6 + .../test_v2_catalog_spm_availability.py | 89 +++++++ tests/unit/test_alembic_workflows.py | 1 + tests/unit/v2/test_alembic_v2.py | 45 ++++ tests/unit/v2/test_catalog_initialization.py | 33 ++- .../unit/v2/test_catalog_spm_availability.py | 243 ++++++++++++++++++ tests/unit/v2/test_metadata_deployment.py | 6 + tests/unit/v2/test_metadata_routes.py | 5 + tests/unit/v2/test_metadata_service.py | 32 +++ tests/unit/v2/test_models.py | 9 + 10 files changed, 465 insertions(+), 4 deletions(-) create mode 100644 tests/integration/test_v2_catalog_spm_availability.py create mode 100644 tests/unit/v2/test_catalog_spm_availability.py diff --git a/tests/fixtures/v2_metadata_routes.py b/tests/fixtures/v2_metadata_routes.py index 0a865fd1f..2ececbb89 100644 --- a/tests/fixtures/v2_metadata_routes.py +++ b/tests/fixtures/v2_metadata_routes.py @@ -35,6 +35,7 @@ MetadataParameterSummary, MetadataRegion, MetadataRegionOption, + MetadataSPMAvailability, MetadataTimePeriodOption, MetadataVariable, ) @@ -252,6 +253,11 @@ def detail(item: object) -> MetadataDetailResult: ) ], ), + spm=MetadataSPMAvailability( + available=True, + settings_schema={"type": "object"}, + defaults={"scenario": "ce_trend"}, + ), ), "get_economy_options": MetadataEconomyOptionsResult( policyengine_version=version, diff --git a/tests/integration/test_v2_catalog_spm_availability.py b/tests/integration/test_v2_catalog_spm_availability.py new file mode 100644 index 000000000..eeb325578 --- /dev/null +++ b/tests/integration/test_v2_catalog_spm_availability.py @@ -0,0 +1,89 @@ +"""Record SPM availability on published catalog versions in disposable Postgres. + +Catalog publication records it as ``policyengine_v2_catalog_writer``, whose +declared privileges must cover the update (rule +``api.v2.runtime-privileges-tested-as-runtime-role``). +""" + +from __future__ import annotations + +from uuid import uuid4 + +import sqlalchemy as sa +from alembic import command +from alembic.config import Config +from policyengine_api.data.v2.catalog.spm_availability import ( + MODEL_VERSIONS, + MODELS, + record_spm_availability, +) +from sqlalchemy import create_engine +from sqlalchemy.pool import NullPool + +from policyengine_api.constants import REPO +from policyengine_api.data.v2.privileges import CATALOG_WRITER_ROLE + +AVAILABLE = {"available": True, "settings_schema": {}, "defaults": {"scenario": "x"}} +UNAVAILABLE = {"available": False} + + +def test_availability_fills_only_the_named_version_and_only_once( + disposable_v2_database_url: str, managed_v2_roles, v2_engine_as_role +) -> None: + command.upgrade(Config(str(REPO / "alembic-v2.ini")), "head") + engine = create_engine(disposable_v2_database_url, poolclass=NullPool) + writer_engine = v2_engine_as_role(CATALOG_WRITER_ROLE, poolclass=NullPool) + model_id, version_id, other_version_id = uuid4(), uuid4(), uuid4() + model_name = f"policyengine-us-{model_id.hex[:8]}" + try: + with engine.begin() as connection: + connection.execute(sa.insert(MODELS).values(id=model_id, name=model_name)) + connection.execute( + sa.insert(MODEL_VERSIONS), + [ + { + "id": identifier, + "model_id": model_id, + "version": version, + "current_law_id": 2, + "metadata_time_periods": [2026], + } + for identifier, version in ( + (version_id, "1.450.0"), + (other_version_id, "1.449.0"), + ) + ], + ) + with writer_engine.begin() as connection: + first = record_spm_availability( + connection, + model_name=model_name, + version="1.450.0", + availability=AVAILABLE, + ) + again = record_spm_availability( + connection, + model_name=model_name, + version="1.450.0", + availability=UNAVAILABLE, + ) + with engine.connect() as connection: + stored = dict( + connection.execute( + sa.select( + MODEL_VERSIONS.c.id, MODEL_VERSIONS.c.spm_availability + ).where(MODEL_VERSIONS.c.model_id == model_id) + ).all() + ) + finally: + with engine.begin() as connection: + connection.execute( + sa.delete(MODEL_VERSIONS).where(MODEL_VERSIONS.c.model_id == model_id) + ) + connection.execute(sa.delete(MODELS).where(MODELS.c.id == model_id)) + writer_engine.dispose() + engine.dispose() + + assert first + assert not again + assert stored == {version_id: AVAILABLE, other_version_id: None} diff --git a/tests/unit/test_alembic_workflows.py b/tests/unit/test_alembic_workflows.py index 2b69ceafc..3a1a7d1b3 100644 --- a/tests/unit/test_alembic_workflows.py +++ b/tests/unit/test_alembic_workflows.py @@ -239,6 +239,7 @@ def test_reusable_v2_integration_check_uses_disposable_databases_and_coverage(): assert "test_v2_catalog_publication.py" in workflow assert "test_v2_catalog_publication_time_periods.py" in workflow assert "test_v2_catalog_publication_pruning.py" in workflow + assert "test_v2_catalog_spm_availability.py" in workflow assert "test_v2_metadata_routes.py" in workflow assert "test_v2_country_routes.py" in workflow assert "test_v2_policy_persistence.py" in workflow diff --git a/tests/unit/v2/test_alembic_v2.py b/tests/unit/v2/test_alembic_v2.py index 86f03d731..6359b1c00 100644 --- a/tests/unit/v2/test_alembic_v2.py +++ b/tests/unit/v2/test_alembic_v2.py @@ -71,6 +71,7 @@ CATALOG_PRUNING_REVISION = "334859ef4fec" # The simulation runs revision, parent of the saved-item keys revision. SIMULATION_RUNS_REVISION = "1356f658347d" +# The saved-item keys revision, parent of the SPM availability revision. SAVED_ITEM_KEYS_REVISION = "81b728a088e7" DISPOSABLE_URL = ( "postgresql+psycopg://postgres:test-password@127.0.0.1:5432/" @@ -253,6 +254,7 @@ def test_v2_revision_chain_has_generated_policy_and_user_identity_changes() -> N assert script.get_heads() == [V2_ALEMBIC_HEAD_REVISION] assert [revision.revision for revision in script.walk_revisions()] == [ V2_ALEMBIC_HEAD_REVISION, + SAVED_ITEM_KEYS_REVISION, SIMULATION_RUNS_REVISION, CATALOG_PRUNING_REVISION, PARAMETER_SEARCH_FILE_REVISION, @@ -1178,3 +1180,46 @@ def test_saved_item_keys_revision_adds_only_two_foreign_keys() -> None: assert "op.drop_table(" not in body assert "op.add_column(" not in body assert "op.drop_column(" not in body + + +def test_spm_availability_revision_adds_only_one_nullable_column() -> None: + """The revision after the saved-item keys records SPM availability. + + It is generated, adds the nullable JSON ``spm_availability`` column to + ``tax_benefit_model_versions`` and drops it on downgrade, and writes no + data: catalog publication records it. + """ + script = ScriptDirectory.from_config(Config(str(REPO / "alembic-v2.ini"))) + children = [ + revision + for revision in script.walk_revisions() + if revision.down_revision == SAVED_ITEM_KEYS_REVISION + ] + assert len(children) == 1 + assert children[0].revision == V2_ALEMBIC_HEAD_REVISION + revision = Path(children[0].path).read_text(encoding="utf-8") + upgrade = revision[ + revision.index("def upgrade()") : revision.index("def downgrade()") + ] + downgrade = revision[revision.index("def downgrade()") :] + + assert ( + "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" + in revision + ) + assert upgrade.count("op.add_column(") == 1 + assert re.search( + r'op\.add_column\(\s*"tax_benefit_model_versions",\s*' + r'sa\.Column\(\s*"spm_availability",\s*sa\.JSON\(\),\s*' + r"nullable=True,?\s*\),?\s*\)", + upgrade, + ) + assert downgrade.count("op.drop_column(") == 1 + assert 'op.drop_column("tax_benefit_model_versions", "spm_availability")' in ( + downgrade + ) + for body in (upgrade, downgrade): + assert "op.execute(" not in body + assert "op.bulk_insert(" not in body + assert "op.create_table(" not in body + assert "op.drop_table(" not in body diff --git a/tests/unit/v2/test_catalog_initialization.py b/tests/unit/v2/test_catalog_initialization.py index 5d494449f..79881718f 100644 --- a/tests/unit/v2/test_catalog_initialization.py +++ b/tests/unit/v2/test_catalog_initialization.py @@ -35,6 +35,7 @@ V2_SUPABASE_ENVIRONMENT: "test-foundation", "V2_RUNTIME_DATABASE_URL": "invalid-runtime-value", "V2_MIGRATION_DATABASE_URL": "invalid-migration-value", + "SIMULATION_ENTRYPOINT_URL": "https://simulation.example", } @@ -79,12 +80,19 @@ def verify(selected_engine, catalog, environ): environ, ) + def record_spm(selected_engine, catalog, environ): + assert "engine" in observed + observed["spm"] = (selected_engine, catalog.policyengine_version, environ) + result = initialization.initialize_catalog( ENVIRONMENT, extractor=normalized_catalog, engine_builder=build, - publisher=publish, - region_data_verifier=verify, + steps=initialization.PublicationSteps( + verify_region_data=verify, + publish=publish, + record_spm_availability=record_spm, + ), ) assert result == _evidence() @@ -93,6 +101,7 @@ def verify(selected_engine, catalog, environ): "verified": (engine, POLICYENGINE_VERSION, ENVIRONMENT), "engine": engine, "version": POLICYENGINE_VERSION, + "spm": (engine, POLICYENGINE_VERSION, ENVIRONMENT), } assert engine.disposed @@ -113,8 +122,9 @@ def refuse_publication(_engine, _catalog): ENVIRONMENT, extractor=normalized_catalog, engine_builder=lambda _settings: engine, - publisher=refuse_publication, - region_data_verifier=reject, + steps=initialization.PublicationSteps( + verify_region_data=reject, publish=refuse_publication + ), ) assert engine.disposed @@ -383,3 +393,18 @@ def verify_without_token(): assert "HUGGING_FACE_TOKEN must be set to read the certified uk data" in ( failure.err ) + + +def test_a_missing_simulation_service_url_stops_before_extraction() -> None: + def refuse_extraction(): + msg = "extraction must not start" + raise AssertionError(msg) + + environment = { + name: value + for name, value in ENVIRONMENT.items() + if name != "SIMULATION_ENTRYPOINT_URL" + } + + with pytest.raises(V2ConfigurationError, match="SIMULATION_ENTRYPOINT_URL"): + initialization.initialize_catalog(environment, extractor=refuse_extraction) diff --git a/tests/unit/v2/test_catalog_spm_availability.py b/tests/unit/v2/test_catalog_spm_availability.py new file mode 100644 index 000000000..c8cd1ac1b --- /dev/null +++ b/tests/unit/v2/test_catalog_spm_availability.py @@ -0,0 +1,243 @@ +"""Each catalog version's SPM availability, from the simulation service's versions.""" + +from __future__ import annotations + +from contextlib import contextmanager +from types import SimpleNamespace +from typing import TYPE_CHECKING + +import httpx +import pytest + +from policyengine_api.data.v2.catalog import spm_availability +from policyengine_api.data.v2.catalog.publication_types import CatalogPublicationError +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.spm import SPMSelection + +if TYPE_CHECKING: + from collections.abc import Iterator + +POLICYENGINE_VERSION = "6.10.0" +DEFAULTS = { + "forecast_content_sha256": "3d" * 32, + "scenario": "ce_trend", + "geography_kind": "county", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, +} +ROUTES = {"latest": POLICYENGINE_VERSION, POLICYENGINE_VERSION: "policyengine-v6-10-0"} +VERSIONS = { + "policyengine": ROUTES, + "spm_capabilities": { + POLICYENGINE_VERSION: { + "contract_version": "canonical-spm-v1", + "defaults": DEFAULTS, + } + }, +} + + +def test_a_us_version_with_a_certified_capability_offers_spm() -> None: + availability = spm_availability.spm_availability( + "us", POLICYENGINE_VERSION, VERSIONS + ) + + assert availability == { + "available": True, + "settings_schema": SPMSelection.model_json_schema(), + "defaults": DEFAULTS, + } + + +@pytest.mark.parametrize( + ("country_id", "policyengine_version", "versions"), + [ + ("uk", POLICYENGINE_VERSION, VERSIONS), + ("us", POLICYENGINE_VERSION, {"policyengine": ROUTES}), + ("us", POLICYENGINE_VERSION, {"policyengine": ROUTES, "spm_capabilities": []}), + ( + "us", + POLICYENGINE_VERSION, + { + "policyengine": ROUTES, + "spm_capabilities": { + POLICYENGINE_VERSION: { + "contract_version": "canonical-spm-v0", + "defaults": DEFAULTS, + } + }, + }, + ), + ( + "us", + POLICYENGINE_VERSION, + { + "policyengine": ROUTES, + "spm_capabilities": { + POLICYENGINE_VERSION: { + "contract_version": "canonical-spm-v1", + "defaults": {**DEFAULTS, "scenario": None}, + } + }, + }, + ), + ], + ids=[ + "uk", + "no-capability", + "malformed-capabilities", + "other-contract", + "unpinned-defaults", + ], +) +def test_anything_else_offers_no_spm( + country_id: str, policyengine_version: str, versions: object +) -> None: + assert spm_availability.spm_availability( + country_id, policyengine_version, versions + ) == {"available": False} + + +@pytest.mark.parametrize( + "versions", + [ + {**VERSIONS, "policyengine": {"latest": "6.11.0", "6.11.0": "app"}}, + {"spm_capabilities": VERSIONS["spm_capabilities"]}, + [], + ], + ids=["version-not-served", "no-route-map", "not-an-object"], +) +def test_a_us_version_the_service_does_not_serve_is_not_known_yet( + versions: object, +) -> None: + assert ( + spm_availability.spm_availability("us", POLICYENGINE_VERSION, versions) is None + ) + + +class RecordingEngine: + """An engine whose transactions are only counted.""" + + def __init__(self) -> None: + """Start with no transaction.""" + self.transactions = 0 + + @contextmanager + def begin(self) -> Iterator[str]: + """Count one transaction and hand out a placeholder connection.""" + self.transactions += 1 + yield "connection" + + +def test_publication_records_only_what_is_known( + monkeypatch: pytest.MonkeyPatch, +) -> None: + recorded: list[tuple[str, dict]] = [] + monkeypatch.setattr( + spm_availability, + "record_spm_availability", + lambda _connection, *, model_name, version, availability: recorded.append( + (model_name, dict(availability)) + ), + ) + catalog = SimpleNamespace( + policyengine_version=POLICYENGINE_VERSION, + countries=[ + SimpleNamespace( + country_id=country_id, + model=SimpleNamespace(name=f"policyengine-{country_id}"), + model_version=SimpleNamespace(version="1.0.0"), + ) + for country_id in ("uk", "us") + ], + ) + not_served = {"policyengine": {"latest": "6.11.0", "6.11.0": "app"}} + + spm_availability.record_published_spm_availability( + RecordingEngine(), + catalog, + {"SIMULATION_ENTRYPOINT_URL": "https://simulation.example"}, + fetch_versions=lambda _url: not_served, + ) + + assert recorded == [("policyengine-uk", {"available": False})] + + +def test_the_versions_come_from_the_simulation_services_public_route() -> None: + requests: list[httpx.Request] = [] + + def answer(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=VERSIONS) + + with httpx.Client(transport=httpx.MockTransport(answer)) as client: + versions = spm_availability.fetch_simulation_versions( + "https://simulation.example/", client=client + ) + + assert versions == VERSIONS + assert [str(request.url) for request in requests] == [ + "https://simulation.example/versions" + ] + assert "authorization" not in requests[0].headers + + +def test_an_unreadable_versions_answer_stops_the_recording() -> None: + with ( + httpx.Client( + transport=httpx.MockTransport(lambda _request: httpx.Response(503)) + ) as client, + pytest.raises(CatalogPublicationError, match="versions"), + ): + spm_availability.fetch_simulation_versions( + "https://simulation.example", client=client + ) + + +def test_recording_needs_the_simulation_service_url() -> None: + with pytest.raises(V2ConfigurationError, match="SIMULATION_ENTRYPOINT_URL"): + spm_availability.simulation_entrypoint_url({}) + + assert ( + spm_availability.simulation_entrypoint_url( + {"SIMULATION_ENTRYPOINT_URL": " https://simulation.example/ "} + ) + == "https://simulation.example" + ) + + +def test_an_unreadable_versions_answer_leaves_every_version_unrecorded( + monkeypatch: pytest.MonkeyPatch, +) -> None: + recorded: list[str] = [] + monkeypatch.setattr( + spm_availability, + "record_spm_availability", + lambda _connection, *, model_name, version, availability: recorded.append( + model_name + ), + ) + catalog = SimpleNamespace( + policyengine_version=POLICYENGINE_VERSION, + countries=[ + SimpleNamespace( + country_id="us", + model=SimpleNamespace(name="policyengine-us"), + model_version=SimpleNamespace(version="1.0.0"), + ) + ], + ) + + def unreadable(_url: str) -> object: + msg = "the simulation service's versions could not be read" + raise CatalogPublicationError(msg) + + spm_availability.record_published_spm_availability( + RecordingEngine(), + catalog, + {"SIMULATION_ENTRYPOINT_URL": "https://simulation.example"}, + fetch_versions=unreadable, + ) + + assert recorded == [] diff --git a/tests/unit/v2/test_metadata_deployment.py b/tests/unit/v2/test_metadata_deployment.py index 2972f62b0..12388568d 100644 --- a/tests/unit/v2/test_metadata_deployment.py +++ b/tests/unit/v2/test_metadata_deployment.py @@ -74,6 +74,12 @@ def test_reusable_seeding_workflow_separates_database_credentials() -> None: assert "V2_MIGRATION_DATABASE_URL" not in publication assert "V2_RUNTIME_DATABASE_URL" not in workflow assert "scripts/publish_v2_metadata_catalog.py" in publication + # Publication records each version's SPM availability from the + # environment's own simulation service. + assert ( + "SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }}" + in publication + ) assert (REPO / "scripts/publish_v2_metadata_catalog.py").is_file() assert not (REPO / "scripts/initialize_v2_metadata.py").exists() diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index edc7ba7c0..82ce0fbc7 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -393,6 +393,11 @@ def test_model_catalog_route_serves_the_small_parts_of_the_selected_version() -> assert result["basic_inputs"] == ["employment_income"] assert [region["code"] for region in result["regions"]] == ["us"] assert result["economy_options"]["datasets"][0]["default"] is True + assert result["spm"] == { + "available": True, + "settings_schema": {"type": "object"}, + "defaults": {"scenario": "ce_trend"}, + } assert reader.calls == [("get_model_catalog", ("us", None), {})] assert reader.closed diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index 2fa5f0d93..d351a8cf6 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -440,6 +440,38 @@ def test_model_catalog_published_before_format_2_has_empty_new_fields( assert result.basic_inputs == [] +SPM_AVAILABLE = { + "available": True, + "settings_schema": {"title": "SPMSelection", "type": "object"}, + "defaults": {"scenario": "ce_trend", "geography_kind": "county"}, +} + + +def test_model_catalog_serves_the_versions_recorded_spm_availability( + catalog_session: Session, +) -> None: + model_version = stored_model_version(catalog_session, "policyengine-us") + model_version.spm_availability = SPM_AVAILABLE + catalog_session.add(model_version) + catalog_session.commit() + + result = catalog_service(catalog_session).get_model_catalog("us") + + assert result.spm.model_dump(mode="json") == SPM_AVAILABLE + + +def test_model_catalog_without_recorded_spm_availability_offers_none( + catalog_session: Session, +) -> None: + result = catalog_service(catalog_session).get_model_catalog("uk") + + assert result.spm.model_dump(mode="json") == { + "available": False, + "settings_schema": None, + "defaults": None, + } + + def test_model_catalog_with_regions_requires_the_national_region( catalog_session: Session, ) -> None: diff --git a/tests/unit/v2/test_models.py b/tests/unit/v2/test_models.py index db421b65f..b4426f890 100644 --- a/tests/unit/v2/test_models.py +++ b/tests/unit/v2/test_models.py @@ -747,6 +747,15 @@ def test_v1_copy_mappings_keep_their_us_and_uk_checks() -> None: } +def test_spm_availability_is_nullable_for_versions_not_yet_recorded() -> None: + """Publication records it after the catalog, so a version may lack it.""" + column = V2_METADATA.tables["tax_benefit_model_versions"].c["spm_availability"] + + assert column.nullable + assert column.server_default is None + assert isinstance(column.type, sa.JSON) + + def test_catalog_format_2_columns_are_nullable_for_earlier_versions() -> None: """Catalog format 2 stores v3's metadata; versions published before it keep null. From 1f7b70124a8aa166804995517bb42f982a8035ab Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:36:38 +0400 Subject: [PATCH 16/57] Record each catalog version's SPM availability and serve it as spm tax_benefit_model_versions gains the nullable JSON column spm_availability, in revision 0c136a278f29, autogenerated on the saved-item keys revision 81b728a088e7, which is now V2_ALEMBIC_HEAD_REVISION. After publishing the catalog, publication records for each US version of the installed PolicyEngine.py whether the simulation service's public GET /versions lists a certified SPM capability for it, with the settings schema and defaults; a version it does not serve yet, or an unreadable answer, stays unrecorded for the next run. /v2/model-catalog serves it as spm. Publication needs SIMULATION_ENTRYPOINT_URL, which the seeding workflow now passes from the environment's secret. The documents that list what changes on a version after publication name the new column, including its effect on each process's kept responses. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/seed-v2-database.yml | 3 + .github/workflows/v2-integration-check.yml | 2 +- docs/engineering/migration-contracts.md | 2 +- docs/engineering/skills/alembic-migrations.md | 21 ++ .../skills/v2-code-organization.md | 5 +- docs/generated/migration_contracts.json | 3 +- docs/migration/stage-9-v2-metadata.md | 15 +- ...6a278f29_store_catalog_spm_availability.py | 36 ++++ .../data/v2/catalog/initialization.py | 47 ++++- .../data/v2/catalog/spm_availability.py | 194 ++++++++++++++++++ policyengine_api/data/v2/database.py | 5 +- policyengine_api/data/v2/migration_target.py | 2 +- policyengine_api/data/v2/models/metadata.py | 7 + policyengine_api/data/v2/privileges.py | 4 +- .../v2/metadata/response_cache_support.py | 7 +- .../v2/metadata/route_support.py | 6 +- .../services/v2/metadata/process_state.py | 11 +- .../services/v2/metadata/transformations.py | 7 +- .../services/v2/metadata/types.py | 21 +- tests/contract/registry.py | 1 + .../test_v2_catalog_spm_availability.py | 12 +- tests/unit/v2/test_alembic_v2.py | 11 +- 22 files changed, 382 insertions(+), 40 deletions(-) create mode 100644 migrations/v2/versions/0c136a278f29_store_catalog_spm_availability.py create mode 100644 policyengine_api/data/v2/catalog/spm_availability.py diff --git a/.github/workflows/seed-v2-database.yml b/.github/workflows/seed-v2-database.yml index c83175f68..8c622a079 100644 --- a/.github/workflows/seed-v2-database.yml +++ b/.github/workflows/seed-v2-database.yml @@ -73,3 +73,6 @@ jobs: # Reads the certified UK data release, whose repository is private, # to check each new catalog version's region filters. HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} + # Each version's SPM availability comes from this environment's + # simulation service's public versions answer. + SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }} diff --git a/.github/workflows/v2-integration-check.yml b/.github/workflows/v2-integration-check.yml index 581b346a5..34b8540e8 100644 --- a/.github/workflows/v2-integration-check.yml +++ b/.github/workflows/v2-integration-check.yml @@ -74,7 +74,7 @@ jobs: - name: Verify the managed v2 roles hold their declared privileges run: uv run --frozen --extra dev python scripts/reconcile_v2_privileges.py verify - name: Test v2 metadata publication, resource routes and the countries list - run: uv run --frozen --extra dev coverage run --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_catalog_publication_time_periods.py tests/integration/test_v2_catalog_publication_pruning.py tests/integration/test_v2_metadata_routes.py tests/integration/test_v2_country_routes.py + run: uv run --frozen --extra dev coverage run --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_catalog_publication_time_periods.py tests/integration/test_v2_catalog_publication_pruning.py tests/integration/test_v2_catalog_spm_availability.py tests/integration/test_v2_metadata_routes.py tests/integration/test_v2_country_routes.py - name: Test v2 policy persistence and immediate v1 mirroring run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_policy_persistence.py tests/integration/test_v1_policy_dual_write.py tests/integration/test_mysql_policy_dual_write.py tests/integration/test_v2_user_policy_mirroring.py tests/integration/test_v1_user_policy_dual_write.py - name: Test signed-in identity provisioning and user associations diff --git a/docs/engineering/migration-contracts.md b/docs/engineering/migration-contracts.md index 1974f6c08..e064aaf9f 100644 --- a/docs/engineering/migration-contracts.md +++ b/docs/engineering/migration-contracts.md @@ -223,7 +223,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `GET` | `/v2/parameters/search-file?country_id=us` | 200 | `metadata` | none | `format`, `country_id`, `policyengine_version`, `parameters`, `other_parameters`, `nodes` | | `GET` | `/v2/tax-benefit-models/by-country/us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.model`, `result.model_version` | | `GET` | `/v2/economy-options?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.current_law_id`, `result.region`, `result.time_period`, `result.datasets` | -| `GET` | `/v2/model-catalog?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.entities`, `result.basic_inputs`, `result.regions`, `result.datasets`, `result.economy_options` | +| `GET` | `/v2/model-catalog?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.entities`, `result.basic_inputs`, `result.regions`, `result.datasets`, `result.economy_options`, `result.spm` | | `GET` | `/v2/model-catalog/variables?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.variables`, `result.part_parameter_values` | | `GET` | `/v2/model-catalog/parameters?country_id=us&name=gov.irs.income.bracket.rates.1` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.parameters`, `result.node_labels`, `result.missing_names` | diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index 763af1373..94125aa9d 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -739,3 +739,24 @@ simulations' `simulation_id` (deleting a saved simulation that has no `legacy_simulation_id`, which its check requires). `tests/integration/test_alembic_v2_saved_item_keys.py` checks the refused inserts and deletes, the refused upgrade and the downgrade. + +### API v2 catalog SPM availability + +Revision `0c136a278f29` (parent `81b728a088e7`) was autogenerated, with no +post-generation corrections, after `tax_benefit_model_versions` gained the +nullable JSON column `spm_availability`. Catalog publication records it for +each catalog version of the installed PolicyEngine.py version that has none +yet, after publishing the catalog +(`policyengine_api/data/v2/catalog/spm_availability.py`): whether the +version offers SPM settings, from the simulation service's public +`GET /versions` (at `SIMULATION_ENTRYPOINT_URL`), with the settings' JSON +schema and the certified defaults. A US version the simulation service does +not serve yet, and every version when its answer cannot be read, stays +unrecorded for the next publication run. `/v2/model-catalog` serves it as +`spm`, and a version without it offers none. The catalog writer role already +holds `UPDATE` on the table, so no privilege changes. + +The revision writes no data, and the previous Public API revision, which +doesn't map the column, keeps working at it. The downgrade drops the column +with its values, which the next publication run of each PolicyEngine.py +version records again. diff --git a/docs/engineering/skills/v2-code-organization.md b/docs/engineering/skills/v2-code-organization.md index b48b7ed5a..c9cf4d0dc 100644 --- a/docs/engineering/skills/v2-code-organization.md +++ b/docs/engineering/skills/v2-code-organization.md @@ -128,8 +128,9 @@ A resource may also add a `process_state.py` module for process-wide state its service reuses across requests, such as the metadata package's (each catalog version a request has selected; after publication its row changes only in its time-period options, its `catalog_last_published_at`, which no read uses, -and its `catalog_values_pruned_at`, which the service checks again before or -after the reads of the rows pruning deletes). Such a module holds plain data, constructs no +its `catalog_values_pruned_at`, which the service checks again before or +after the reads of the rows pruning deletes, and its `spm_availability`, +which publication records once after publishing it). Such a module holds plain data, constructs no SQL and owns no session; the composition code creates one instance per process and injects it into the service. diff --git a/docs/generated/migration_contracts.json b/docs/generated/migration_contracts.json index dcf1697d0..ccf123f82 100644 --- a/docs/generated/migration_contracts.json +++ b/docs/generated/migration_contracts.json @@ -1662,7 +1662,8 @@ "result.basic_inputs", "result.regions", "result.datasets", - "result.economy_options" + "result.economy_options", + "result.spm" ] }, { diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 2e4052864..b7e1d6c27 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -36,7 +36,14 @@ version (with the current-law policy id and the years offered), the entities input dataset, and the economy choices. A catalog without regions, such as a household-only country's, has empty region, dataset and economy-choice lists rather than an error. A version published before catalog format 2 has a null -`catalog_format` and empty entities and basic inputs. The result names the +`catalog_format` and empty entities and basic inputs. `spm` says whether the +version offers SPM settings for a household: `available`, and when it does, +`settings_schema` (the JSON schema of a household's `spm` selection) and +`defaults` (the certified selection for every field a household leaves out). +Catalog publication records it for each US version from the simulation +service's public `GET /versions`, after publishing the version; any other +country, a bundle without a certified capability, and a version not recorded +yet offer none. The result names the selected PolicyEngine.py version, so a client that starts without a version learns the running one and requests every other part for that exact version. The query uses the shared `CatalogQuery` model: unknown and repeated @@ -159,7 +166,11 @@ dropped first; the 6.10.0 US variables body is 4,190,824 bytes uncompressed, so eight entries hold about 12 MB for both countries' two routes at two versions). A response is kept under the exact version it was read from, so a request without `policyengine_version` (the process's running version, -which never changes while it runs) and one naming that version share it. Only +which never changes while it runs) and one naming that version share it. A +version's SPM availability is the one part recorded after publication: a +process that built a version's `/v2/model-catalog` body before publication +recorded it keeps offering no SPM settings until it restarts, normally at the +deployment that follows the publication run. Only the first successful request per process reads the database. Requests that arrive while that read is in progress wait for it and all receive its outcome, the kept response or a copy of its error response, so one failing read is not diff --git a/migrations/v2/versions/0c136a278f29_store_catalog_spm_availability.py b/migrations/v2/versions/0c136a278f29_store_catalog_spm_availability.py new file mode 100644 index 000000000..d1274cb17 --- /dev/null +++ b/migrations/v2/versions/0c136a278f29_store_catalog_spm_availability.py @@ -0,0 +1,36 @@ +"""store catalog spm availability + +Revision ID: 0c136a278f29 +Revises: 81b728a088e7 +Create Date: 2026-11-10 23:31:42.564586 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate + +No post-generation corrections. It adds one nullable column, which catalog +publication fills after publishing each version, so it writes no data, and +the previous Public API revision, which doesn't map the column, keeps +working. The downgrade drops the column with its values. +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +revision: str = '0c136a278f29' +down_revision: Union[str, None] = '81b728a088e7' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('tax_benefit_model_versions', sa.Column('spm_availability', sa.JSON(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('tax_benefit_model_versions', 'spm_availability') + # ### end Alembic commands ### diff --git a/policyengine_api/data/v2/catalog/initialization.py b/policyengine_api/data/v2/catalog/initialization.py index cafaefc33..9b3702d6e 100644 --- a/policyengine_api/data/v2/catalog/initialization.py +++ b/policyengine_api/data/v2/catalog/initialization.py @@ -5,6 +5,7 @@ import json import os import sys +from dataclasses import dataclass from typing import TYPE_CHECKING from sqlalchemy import Engine, create_engine @@ -27,6 +28,10 @@ RegionDataVerificationError, verify_region_data, ) +from policyengine_api.data.v2.catalog.spm_availability import ( + record_published_spm_availability, + simulation_entrypoint_url, +) from policyengine_api.data.v2.settings import ( V2ConfigurationError, V2DatabaseSettings, @@ -96,25 +101,49 @@ def verify_unpublished_region_data( ) +@dataclass(frozen=True) +class PublicationSteps: + """What initialization does with the database, in order, after extraction. + + ``verify_region_data`` checks the regions' certified data before anything + is written, ``publish`` writes the catalog, and ``record_spm_availability`` + then records each version's SPM availability from the simulation service + (``spm_availability``). + """ + + verify_region_data: Callable[ + [Engine, NormalizedCatalog, Mapping[str, str] | None], None + ] = verify_unpublished_region_data + publish: Callable[[Engine, NormalizedCatalog], PublicationEvidence] = ( + publish_catalog + ) + record_spm_availability: Callable[ + [Engine, NormalizedCatalog, Mapping[str, str] | None], None + ] = record_published_spm_availability + + def initialize_catalog( environ: Mapping[str, str] | None = None, *, extractor: Callable[[], NormalizedCatalog] = extract_installed_catalog, engine_builder: Callable[[V2DatabaseSettings], Engine] = build_data_write_engine, - publisher: Callable[[Engine, NormalizedCatalog], PublicationEvidence] = ( - publish_catalog - ), - region_data_verifier: Callable[ - [Engine, NormalizedCatalog, Mapping[str, str] | None], None - ] = verify_unpublished_region_data, + steps: PublicationSteps | None = None, ) -> PublicationEvidence: - """Load only row-write settings, extract fully, verify, then publish explicitly.""" + """Load only row-write settings, extract fully, verify, then publish explicitly. + + After publishing, record each version's SPM availability. The simulation + service's URL, which that needs, is checked before anything else. + """ + selected_steps = steps or PublicationSteps() settings = load_v2_data_write_database_settings(environ) + simulation_entrypoint_url(os.environ if environ is None else environ) catalog = extractor() engine = engine_builder(settings) try: - region_data_verifier(engine, catalog, environ) - return publisher(engine, catalog) + selected_steps.verify_region_data(engine, catalog, environ) + evidence = selected_steps.publish(engine, catalog) + selected_steps.record_spm_availability(engine, catalog, environ) + return evidence finally: engine.dispose() diff --git a/policyengine_api/data/v2/catalog/spm_availability.py b/policyengine_api/data/v2/catalog/spm_availability.py new file mode 100644 index 000000000..2a2149dda --- /dev/null +++ b/policyengine_api/data/v2/catalog/spm_availability.py @@ -0,0 +1,194 @@ +"""Record whether each published catalog version offers SPM settings. + +The simulation service's public ``GET /versions`` lists, under +``spm_capabilities``, the certified SPM capability of each PolicyEngine.py +bundle it runs: its contract version and pinned defaults. The Public API +already checks economy requests' SPM selections against it. A US catalog +version offers SPM when the service reports a canonical capability for its +PolicyEngine.py version; other countries never do. + +Publication records it after the catalog itself, for the installed +PolicyEngine.py version's catalog versions that have none yet, so a version +published before this existed gets it on the next publication run of that +PolicyEngine.py version; versions of other PolicyEngine.py versions keep none. +A bundle pins its SPM release, so a recorded value never changes. A US version +the service does not serve yet (its ``policyengine`` routes do not name it) +is left unrecorded rather than recorded as unavailable, and the next +publication run tries again; ``/v2/model-catalog`` offers no SPM settings for +it meanwhile. So is every version when the service's versions cannot be +read: an unreadable answer never fails the publication. +""" + +from __future__ import annotations + +import logging +import os +from typing import TYPE_CHECKING, Any + +import httpx +import sqlalchemy as sa +from pydantic import ValidationError + +from policyengine_api.data.v2.catalog.publication_tables import ( + MODEL_VERSIONS, + MODELS, +) +from policyengine_api.data.v2.catalog.publication_types import ( + CatalogPublicationError, +) +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.spm import SPMSelection + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping + + from sqlalchemy import Connection, Engine + + from policyengine_api.data.v2.catalog.records import NormalizedCatalog + +LOGGER = logging.getLogger(__name__) +SIMULATION_ENTRYPOINT_URL = "SIMULATION_ENTRYPOINT_URL" +SPM_COUNTRY_ID = "us" +SPM_CONTRACT_VERSION = "canonical-spm-v1" +VERSIONS_TIMEOUT_SECONDS = 30.0 +UNAVAILABLE: dict[str, Any] = {"available": False} + + +def _served(policyengine_version: str, versions: dict[str, Any]) -> bool: + """Return whether the service's ``policyengine`` routes name the version.""" + routes = versions.get("policyengine") + if not isinstance(routes, dict): + return False + return policyengine_version != "latest" and ( + policyengine_version in routes or routes.get("latest") == policyengine_version + ) + + +def spm_availability( + country_id: str, policyengine_version: str, versions: object +) -> dict[str, Any] | None: + """Return ``{available, settings_schema, defaults}`` for one catalog version. + + Only a US version whose PolicyEngine.py version has a canonical capability + with pinned defaults (forecast artifact and scenario) offers SPM; any other + country, and a US version the service serves without one, is + ``{"available": False}``. A US version the service does not serve yet is + None: not known. + """ + if country_id != SPM_COUNTRY_ID: + return dict(UNAVAILABLE) + if not isinstance(versions, dict) or not _served(policyengine_version, versions): + return None + capabilities = versions.get("spm_capabilities") + capability = ( + capabilities.get(policyengine_version) + if isinstance(capabilities, dict) + else None + ) + if ( + not isinstance(capability, dict) + or set(capability) != {"contract_version", "defaults"} + or capability["contract_version"] != SPM_CONTRACT_VERSION + ): + return dict(UNAVAILABLE) + try: + defaults = SPMSelection.model_validate(capability["defaults"]) + except ValidationError: + return dict(UNAVAILABLE) + if not defaults.forecast_content_sha256 or not defaults.scenario: + return dict(UNAVAILABLE) + return { + "available": True, + "settings_schema": SPMSelection.model_json_schema(), + "defaults": { + name: getattr(defaults, name) for name in SPMSelection.model_fields + }, + } + + +def simulation_entrypoint_url(environ: Mapping[str, str]) -> str: + """Return the simulation service's base URL; raise when it is not set.""" + url = environ.get(SIMULATION_ENTRYPOINT_URL, "").strip().rstrip("/") + if not url: + msg = f"{SIMULATION_ENTRYPOINT_URL} is required to record SPM availability" + raise V2ConfigurationError(msg) + return url + + +def fetch_simulation_versions( + base_url: str, *, client: httpx.Client | None = None +) -> object: + """Read the simulation service's public ``GET /versions`` answer.""" + url = f"{base_url.rstrip('/')}/versions" + try: + if client is None: + response = httpx.get(url, timeout=VERSIONS_TIMEOUT_SECONDS) + else: + response = client.get(url, timeout=VERSIONS_TIMEOUT_SECONDS) + response.raise_for_status() + return response.json() + except (httpx.HTTPError, ValueError) as error: + msg = "the simulation service's versions could not be read" + raise CatalogPublicationError(msg) from error + + +def record_spm_availability( + connection: Connection, + *, + model_name: str, + version: str, + availability: Mapping[str, Any], +) -> bool: + """Store a catalog version's SPM availability unless it already has one. + + Return whether the version was changed. + """ + model_ids = sa.select(MODELS.c.id).where(MODELS.c.name == model_name) + result = connection.execute( + sa.update(MODEL_VERSIONS) + .where( + MODEL_VERSIONS.c.model_id.in_(model_ids), + MODEL_VERSIONS.c.version == version, + MODEL_VERSIONS.c.spm_availability.is_(None), + ) + .values(spm_availability=dict(availability)) + ) + return bool(result.rowcount) + + +def record_published_spm_availability( + engine: Engine, + catalog: NormalizedCatalog, + environ: Mapping[str, str] | None = None, + *, + fetch_versions: Callable[[str], object] = fetch_simulation_versions, +) -> None: + """Record each published country's SPM availability for the catalog version. + + A US version the service does not serve yet stays unrecorded, and so does + every version when the service's versions cannot be read; the next + publication run tries again. Raise V2ConfigurationError without + ``SIMULATION_ENTRYPOINT_URL``. + """ + url = simulation_entrypoint_url(os.environ if environ is None else environ) + try: + versions = fetch_versions(url) + except CatalogPublicationError as error: + LOGGER.warning( + "SPM availability was not recorded; the next publication run " + "tries again: %s", + error, + ) + return + with engine.begin() as connection: + for country in catalog.countries: + availability = spm_availability( + country.country_id, catalog.policyengine_version, versions + ) + if availability is not None: + record_spm_availability( + connection, + model_name=country.model.name, + version=country.model_version.version, + availability=availability, + ) diff --git a/policyengine_api/data/v2/database.py b/policyengine_api/data/v2/database.py index 3d3149948..f72524f5c 100644 --- a/policyengine_api/data/v2/database.py +++ b/policyengine_api/data/v2/database.py @@ -101,8 +101,9 @@ def build_v2_read_session_factory(engine: Engine) -> sessionmaker[Session]: when the session closes. Only the metadata reader uses them: each of its reads is one statement on a published catalog version. After publication the version's row changes only in its time-period options, its - ``catalog_last_published_at`` (set by each publication of it) and its - ``catalog_values_pruned_at`` (set by pruning, cleared by restoration), and + ``catalog_last_published_at`` (set by each publication of it), its + ``catalog_values_pruned_at`` (set by pruning, cleared by restoration) and + its ``spm_availability`` (recorded once after publishing it), and its other rows only when pruning deletes its current-law values, tree rows and search file, which restoration writes back identically. A read that finds none of those rows reads ``catalog_values_pruned_at`` again, so no diff --git a/policyengine_api/data/v2/migration_target.py b/policyengine_api/data/v2/migration_target.py index 61165b0cc..f52dc5987 100644 --- a/policyengine_api/data/v2/migration_target.py +++ b/policyengine_api/data/v2/migration_target.py @@ -24,7 +24,7 @@ from collections.abc import Mapping V2_ALEMBIC_DISPOSABLE_TEST = "V2_ALEMBIC_DISPOSABLE_TEST" -V2_ALEMBIC_HEAD_REVISION = "81b728a088e7" +V2_ALEMBIC_HEAD_REVISION = "0c136a278f29" DISPOSABLE_DATABASE_NAME = "policyengine_v2_alembic_test" DISPOSABLE_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "postgres"}) MIGRATION_ROLE = "policyengine_v2_migrator" diff --git a/policyengine_api/data/v2/models/metadata.py b/policyengine_api/data/v2/models/metadata.py index 4fd1c0c8b..30115f15a 100644 --- a/policyengine_api/data/v2/models/metadata.py +++ b/policyengine_api/data/v2/models/metadata.py @@ -94,6 +94,7 @@ class TaxBenefitModelVersion(IdentifiedModel, table=True): version: it deleted the version's current-law parameter values, parameter tree rows and parameter search file, which publishing the version again restores from the installed packages, clearing the marker. + ``spm_availability`` records whether the version offers SPM settings. """ __tablename__ = "tax_benefit_model_versions" @@ -129,6 +130,12 @@ class TaxBenefitModelVersion(IdentifiedModel, table=True): catalog_values_pruned_at: datetime | None = Field( default=None, sa_type=sa.DateTime(timezone=True) ) + # Whether this version offers SPM settings, with their schema and the + # certified defaults, as the simulation service reports its bundle's SPM + # support: {available, settings_schema, defaults}. Catalog publication + # records it once, after publishing the version, so a version has none + # until then. + spm_availability: dict[str, Any] | None = Field(default=None, sa_type=sa.JSON) model: TaxBenefitModel = Relationship(back_populates="versions") variables: list["Variable"] = Relationship( diff --git a/policyengine_api/data/v2/privileges.py b/policyengine_api/data/v2/privileges.py index e2286aad4..a85b5e3cb 100644 --- a/policyengine_api/data/v2/privileges.py +++ b/policyengine_api/data/v2/privileges.py @@ -255,8 +255,8 @@ def _run_table( "regions": _access( runtime=READ, catalog_writer=READ_INSERT_UPDATE, **_RUN_INPUT_READS ), - # Catalog publication updates a version's time periods and - # catalog_last_published_at, and catalog pruning its + # Catalog publication updates a version's time periods, + # catalog_last_published_at and spm_availability, and catalog pruning its # catalog_values_pruned_at. "tax_benefit_model_versions": _access( runtime=READ, catalog_writer=READ_INSERT_UPDATE, **_RUN_INPUT_READS diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_cache_support.py b/policyengine_api/fastapi_routes/v2/metadata/response_cache_support.py index a197971da..fc21da9be 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/response_cache_support.py +++ b/policyengine_api/fastapi_routes/v2/metadata/response_cache_support.py @@ -84,8 +84,11 @@ class MetadataResponseCache: Of what these bodies hold, a published catalog version changes only when a release's catalog publication replaces its time-period options, before - that release's processes start, or when pruning deletes its current-law - values, which restoration writes back identically; publication also sets + that release's processes start, when pruning deletes its current-law + values, which restoration writes back identically, or when publication + records its SPM availability once after publishing it, so a body built + before then offers no SPM settings for the life of the process; + publication also sets the version's ``catalog_last_published_at`` and pruning its ``catalog_values_pruned_at``, which no body holds. So a body built for a route, country and version stays valid for the life of the process; a diff --git a/policyengine_api/fastapi_routes/v2/metadata/route_support.py b/policyengine_api/fastapi_routes/v2/metadata/route_support.py index 819c3eb6a..72ad595a0 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/route_support.py +++ b/policyengine_api/fastapi_routes/v2/metadata/route_support.py @@ -231,7 +231,11 @@ def policyengine_version(self) -> str: the ETag lets a client revalidate it cheaply after that. Catalog pruning deletes a version's current-law values, which publishing it again restores identically, and the 404 answered for a pruned version (503 for the running -one) is never stored (``Cache-Control: no-store``). +one) is never stored (``Cache-Control: no-store``). A version's SPM +availability in ``/v2/model-catalog`` is recorded once after publication: +until a publication run records it, a response says the version offers no +SPM settings, a process keeps saying so until it restarts, and a client may +reuse such a response for up to this long after that. """ RUNNING_VERSION_MAX_AGE_SECONDS = 300 """How long a response for the running catalog version may be reused. diff --git a/policyengine_api/services/v2/metadata/process_state.py b/policyengine_api/services/v2/metadata/process_state.py index 3403b3e4c..f52b9be44 100644 --- a/policyengine_api/services/v2/metadata/process_state.py +++ b/policyengine_api/services/v2/metadata/process_state.py @@ -24,10 +24,13 @@ class SelectedCatalogStore: time-period options, which a release's publication replaces before that release's processes start, its ``catalog_last_published_at``, which each publication of it sets and no read uses, and its - ``catalog_values_pruned_at``, which pruning sets and restoration clears. - So once a request has found the version, later requests for the same - country and version need not query for it again; a process of the - previous release keeps the time-period options it read. + ``catalog_values_pruned_at``, which pruning sets and restoration clears, + and its ``spm_availability``, which publication records once after + publishing it. So once a request has found the version, later requests + for the same country and version need not query for it again; a process + of the previous release keeps the time-period options it read, and a + process that found a version before its SPM availability was recorded + offers no SPM settings for it until it restarts. A kept ``catalog_values_pruned_at`` can be out of date in either direction, so ``MetadataCatalogSelection`` checks it for the reads of the diff --git a/policyengine_api/services/v2/metadata/transformations.py b/policyengine_api/services/v2/metadata/transformations.py index f2586c653..f69e484d6 100644 --- a/policyengine_api/services/v2/metadata/transformations.py +++ b/policyengine_api/services/v2/metadata/transformations.py @@ -30,6 +30,7 @@ MetadataRegion, MetadataRegionOption, MetadataRegionType, + MetadataSPMAvailability, MetadataTimePeriodOption, MetadataVariable, ParameterNodeLabelRow, @@ -427,7 +428,8 @@ def metadata_model_catalog( Without a national dataset (a catalog without regions) the economy choices are empty. Entities and basic inputs a version published before catalog - format 2 lacks are empty lists. + format 2 lacks are empty lists, and a version whose SPM availability is + not recorded yet offers none. """ model_version = selected.model_version return MetadataModelCatalogResult( @@ -449,4 +451,7 @@ def metadata_model_catalog( if national_dataset is not None else MetadataEconomyOptions(region=[], time_period=[], datasets=[]) ), + spm=MetadataSPMAvailability.model_validate( + model_version.spm_availability or {"available": False} + ), ) diff --git a/policyengine_api/services/v2/metadata/types.py b/policyengine_api/services/v2/metadata/types.py index c0b03a818..6be355ec6 100644 --- a/policyengine_api/services/v2/metadata/types.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -356,14 +356,30 @@ class MetadataEconomyOptions(StrictResponseModel): datasets: list[MetadataDatasetOption] +class MetadataSPMAvailability(StrictResponseModel): + """Whether a catalog version offers SPM settings for a household. + + When ``available``, ``settings_schema`` is the JSON schema of a + household's ``spm`` selection and ``defaults`` the certified selection + that applies to every field a household leaves out. A version without + them (any country but the US, a bundle without certified SPM support, or + a version whose availability publication has not recorded yet) is not + available, and both are null. + """ + + available: bool + settings_schema: dict[str, JsonValue] | None = None + defaults: dict[str, JsonValue] | None = None + + class MetadataModelCatalogResult(StrictResponseModel): """The small parts of one catalog version, read when a client starts. policyengine_version names the selected catalog version, so a client can request its other parts for that exact version. catalog_format is null, and entities and basic_inputs are empty, for a version published before - catalog format 2. Variables, parameters and their values are served by - their own routes. + catalog format 2. ``spm`` says whether the version offers SPM settings. + Variables, parameters and their values are served by their own routes. """ policyengine_version: str @@ -375,6 +391,7 @@ class MetadataModelCatalogResult(StrictResponseModel): regions: list[MetadataRegion] datasets: list[MetadataDataset] economy_options: MetadataEconomyOptions + spm: MetadataSPMAvailability class MetadataDatedValue(StrictResponseModel): diff --git a/tests/contract/registry.py b/tests/contract/registry.py index 8f03910cd..5c2df6b72 100644 --- a/tests/contract/registry.py +++ b/tests/contract/registry.py @@ -719,6 +719,7 @@ class WorkflowContract: "result.regions", "result.datasets", "result.economy_options", + "result.spm", ), route_group="metadata", ), diff --git a/tests/integration/test_v2_catalog_spm_availability.py b/tests/integration/test_v2_catalog_spm_availability.py index eeb325578..d3c98b48d 100644 --- a/tests/integration/test_v2_catalog_spm_availability.py +++ b/tests/integration/test_v2_catalog_spm_availability.py @@ -12,15 +12,17 @@ import sqlalchemy as sa from alembic import command from alembic.config import Config -from policyengine_api.data.v2.catalog.spm_availability import ( - MODEL_VERSIONS, - MODELS, - record_spm_availability, -) from sqlalchemy import create_engine from sqlalchemy.pool import NullPool from policyengine_api.constants import REPO +from policyengine_api.data.v2.catalog.publication_tables import ( + MODEL_VERSIONS, + MODELS, +) +from policyengine_api.data.v2.catalog.spm_availability import ( + record_spm_availability, +) from policyengine_api.data.v2.privileges import CATALOG_WRITER_ROLE AVAILABLE = {"available": True, "settings_schema": {}, "defaults": {"scenario": "x"}} diff --git a/tests/unit/v2/test_alembic_v2.py b/tests/unit/v2/test_alembic_v2.py index 6359b1c00..bf3d67d52 100644 --- a/tests/unit/v2/test_alembic_v2.py +++ b/tests/unit/v2/test_alembic_v2.py @@ -1207,16 +1207,19 @@ def test_spm_availability_revision_adds_only_one_nullable_column() -> None: "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" in revision ) + # The revision stays as Alembic wrote it, in either quote style. assert upgrade.count("op.add_column(") == 1 assert re.search( - r'op\.add_column\(\s*"tax_benefit_model_versions",\s*' - r'sa\.Column\(\s*"spm_availability",\s*sa\.JSON\(\),\s*' + r"""op\.add_column\(\s*["']tax_benefit_model_versions["'],\s*""" + r"""sa\.Column\(\s*["']spm_availability["'],\s*sa\.JSON\(\),\s*""" r"nullable=True,?\s*\),?\s*\)", upgrade, ) assert downgrade.count("op.drop_column(") == 1 - assert 'op.drop_column("tax_benefit_model_versions", "spm_availability")' in ( - downgrade + assert re.search( + r"""op\.drop_column\(\s*["']tax_benefit_model_versions["'],\s*""" + r"""["']spm_availability["'],?\s*\)""", + downgrade, ) for body in (upgrade, downgrade): assert "op.execute(" not in body From d35e69735ad8f3ce18f048bfd6935004507a6751 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:39:33 +0400 Subject: [PATCH 17/57] Test household calculations live against the staging candidate (failing) The live checks store households and reforms through /v2/households and /v2/policies, submit them to /v2/household-calculations and read each job until it finishes: a submit and read cycle, six calculations submitted at once within their latency limits, a household across 401 axis points, and a parity set of three US and three UK households compared value by value with the in-process /calculate and /calculate-full. A unit test checks the parity households that add variables calculate one year on both sides. The workflow test expects the household checks to run after the other live tests with fewer workers, and the job's limit to be 30 minutes, which the script and workflow don't do yet. Co-Authored-By: Claude Opus 5.5 --- .../live_household_calculations.py | 320 ++++++++++++++++++ .../test_live_v2_household_calculations.py | 169 +++++++++ .../test_live_v2_household_parity.py | 187 ++++++++++ tests/unit/test_cloud_run_push_workflow.py | 15 + .../test_live_household_parity_households.py | 64 ++++ 5 files changed, 755 insertions(+) create mode 100644 tests/fixtures/integration/live_household_calculations.py create mode 100644 tests/integration/test_live_v2_household_calculations.py create mode 100644 tests/integration/test_live_v2_household_parity.py create mode 100644 tests/unit/v2/test_live_household_parity_households.py diff --git a/tests/fixtures/integration/live_household_calculations.py b/tests/fixtures/integration/live_household_calculations.py new file mode 100644 index 000000000..c26c54225 --- /dev/null +++ b/tests/fixtures/integration/live_household_calculations.py @@ -0,0 +1,320 @@ +"""Households and helpers for the live staging household calculation checks. + +They need only httpx and the standard library, because the parallel live test +job installs nothing else. +""" + +from __future__ import annotations + +import math +import os +import time +from dataclasses import dataclass, field +from functools import partial +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Callable + + import httpx + +CALCULATION_PATH = "/v2/household-calculations" +CALCULATION_TIMEOUT_SECONDS = float( + os.environ.get("STAGING_HOUSEHOLD_CALCULATION_TIMEOUT_SECONDS", "600") +) +POLL_INTERVAL_SECONDS = float( + os.environ.get("STAGING_API_TEST_POLL_INTERVAL_SECONDS", "5") +) +# A 502, 503 or 504 can be a short outage, such as the simulation service +# failing to read its database or version manifest for a moment, or the +# Public API failing to read household_jobs; a +# calculation's submission and each poll are sent again while such answers +# last at most TRANSIENT_LIMIT_SECONDS in a row. Any other failure, and a +# longer run of such answers, fails the check with the answer's body. +TRANSIENT_STATUS_CODES = {502, 503, 504} +TRANSIENT_LIMIT_SECONDS = float( + os.environ.get("STAGING_HOUSEHOLD_TRANSIENT_LIMIT_SECONDS", "90") +) +FINISHED_STATUSES = {"succeeded", "failed"} +YEAR = "2026" +REFORM_START = f"{YEAR}-01-01" +OPEN_END = "2100-12-31" +COLLECTIONS = { + "us": { + "household": "households", + "family": "families", + "tax_unit": "tax_units", + "spm_unit": "spm_units", + "marital_unit": "marital_units", + }, + "uk": {"household": "households", "benunit": "benunits"}, +} + + +@dataclass(frozen=True) +class LiveHousehold: + """A household whose people all share one group of each kind. + + ``people`` and ``groups`` map variable names to values by period; a null + value asks for the variable to be calculated. ``axes`` vary inputs across + points, as in a ``/calculate`` situation. ``reform`` sets each named + parameter to a value from the start of ``YEAR`` with no end, through a + policy stored with ``/v2/policies``. ``add_missing_variables`` calculates + every variable the household lacks, as ``/calculate-full`` does. + ``expects_warnings`` says the household has a deprecated input, which the + calculation drops with a warning before the country model runs. + """ + + name: str + country_id: str + people: dict[str, dict[str, Any]] + groups: dict[str, dict[str, Any]] = field(default_factory=dict) + axes: list[list[dict[str, Any]]] | None = None + reform: dict[str, int | float] | None = None + add_missing_variables: bool = False + expects_warnings: bool = False + + def document(self) -> dict[str, Any]: + """Return the household as ``/v2/households`` stores it.""" + collections = COLLECTIONS[self.country_id] + document: dict[str, Any] = { + "people": [ + { + "id": person_id, + "values": values, + "memberships": { + collection: f"{collection}-1" for collection in collections + }, + } + for person_id, values in self.people.items() + ] + } + for collection in collections: + document[collection] = [ + {"id": f"{collection}-1", "values": self.groups.get(collection, {})} + ] + return document + + def situation(self) -> dict[str, Any]: + """Return the same household as a ``/calculate`` situation.""" + situation: dict[str, Any] = {"people": self.people} + for collection, plural in COLLECTIONS[self.country_id].items(): + situation[plural] = { + f"{collection}-1": { + "members": list(self.people), + **self.groups.get(collection, {}), + } + } + if self.axes is not None: + situation["axes"] = self.axes + return situation + + +@dataclass(frozen=True) +class FinishedCalculation: + """A calculation's final answer and how long each step took.""" + + item: dict[str, Any] + submit_seconds: float + total_seconds: float + + +def probe_amount(integration_probe_id: str) -> int: + """Return a small run-specific amount that makes each household new.""" + return int(integration_probe_id.rsplit("-", 1)[-1], 16) % 997 + + +def store_household(client: httpx.Client, household: LiveHousehold) -> str: + """Store the household through ``/v2/households`` and return its ID.""" + response = client.post( + "/v2/households", + params={"country_id": household.country_id}, + json={ + "country_id": household.country_id, + "default_year": int(YEAR), + "household_data": household.document(), + }, + ) + assert response.status_code in {200, 201}, response.text + return response.json()["result"]["item"]["id"] + + +def _one_catalog_item( + client: httpx.Client, resource: str, country_id: str, **filters: str +) -> dict[str, Any]: + response = client.get( + f"/v2/{resource}", params={"country_id": country_id, "limit": 1, **filters} + ) + assert response.status_code == 200, response.text[:500] + (item,) = response.json()["result"]["items"] + return item + + +def store_reform(client: httpx.Client, household: LiveHousehold) -> str | None: + """Store the household's reform through ``/v2/policies``; return its ID.""" + if household.reform is None: + return None + country_id = household.country_id + model = _one_catalog_item(client, "tax-benefit-models", country_id) + parameter_values = [ + { + "parameter_id": _one_catalog_item( + client, "parameters", country_id, name=parameter_name + )["id"], + "value": value, + "start_date": f"{REFORM_START}T00:00:00Z", + } + for parameter_name, value in household.reform.items() + ] + response = client.post( + "/v2/policies", + params={"country_id": country_id}, + json={ + "country_id": country_id, + "tax_benefit_model_id": model["id"], + "parameter_values": parameter_values, + }, + ) + assert response.status_code in {200, 201}, response.text[:500] + return response.json()["result"]["item"]["id"] + + +def _send_through_short_outages(send: Callable[[], httpx.Response]) -> httpx.Response: + """Send the request again while it gets 502, 503 or 504 for a short time.""" + outage_started: float | None = None + while True: + response = send() + if response.status_code not in TRANSIENT_STATUS_CODES: + return response + outage_started = outage_started or time.monotonic() + if time.monotonic() - outage_started >= TRANSIENT_LIMIT_SECONDS: + msg = ( + f"HTTP {response.status_code} for {TRANSIENT_LIMIT_SECONDS:.0f} s: " + f"{response.text[:500]}" + ) + raise AssertionError(msg) + time.sleep(POLL_INTERVAL_SECONDS) + + +def calculate( + client: httpx.Client, + household: LiveHousehold, + household_id: str, + policy_id: str | None = None, +) -> FinishedCalculation: + """Submit the stored household's calculation and poll until it finishes.""" + started = time.monotonic() + body: dict[str, Any] = { + "country_id": household.country_id, + "household_id": household_id, + "policy_id": policy_id, + "add_missing_variables": household.add_missing_variables, + } + if household.axes is not None: + body["axes"] = household.axes + deadline = started + CALCULATION_TIMEOUT_SECONDS + response = _send_through_short_outages( + lambda: client.post(CALCULATION_PATH, json=body) + ) + submit_seconds = time.monotonic() - started + assert response.status_code == 202, response.text + accepted = response.json()["result"]["item"] + assert accepted["household_id"] == household_id, accepted + calculation_path = f"{CALCULATION_PATH}/{accepted['calculation_id']}" + # The submission answers with the new job only; its result is read from + # household_jobs, so the calculation is read at least once. + while True: + polled = _send_through_short_outages(partial(client.get, calculation_path)) + assert polled.status_code == 200, polled.text + item = polled.json()["result"]["item"] + if item["status"] in FINISHED_STATUSES: + break + if time.monotonic() >= deadline: + msg = ( + f"Calculation {item['calculation_id']} did not finish within " + f"{CALCULATION_TIMEOUT_SECONDS:.0f} s; last status {item['status']}" + ) + raise AssertionError(msg) + time.sleep(POLL_INTERVAL_SECONDS) + return FinishedCalculation( + item=item, + submit_seconds=submit_seconds, + total_seconds=time.monotonic() - started, + ) + + +def calculate_with_v1(client: httpx.Client, household: LiveHousehold) -> dict: + """Return the Public API's own answer for the same household and reform. + + ``/calculate-full`` answers when the household adds missing variables, + ``/calculate`` otherwise. + """ + route = "calculate-full" if household.add_missing_variables else "calculate" + request: dict[str, Any] = {"household": household.situation()} + if household.reform is not None: + request["policy"] = { + parameter_name: {f"{REFORM_START}.{OPEN_END}": value} + for parameter_name, value in household.reform.items() + } + response = client.post(f"/{household.country_id}/{route}", json=request) + assert response.status_code == 200, response.text + payload = response.json() + assert payload["status"] == "ok", payload + return payload + + +def _same_value(v1_value: object, v2_value: object) -> bool: + if isinstance(v1_value, list) and isinstance(v2_value, list): + return len(v1_value) == len(v2_value) and all( + _same_value(v1_item, v2_item) + for v1_item, v2_item in zip(v1_value, v2_value, strict=True) + ) + # The Public API's /calculate writes an undefined number as a bare NaN; + # the v2 household function writes "NaN". + if isinstance(v1_value, float) and math.isnan(v1_value): + return v2_value == "NaN" + if isinstance(v1_value, bool) or isinstance(v2_value, bool): + return v1_value == v2_value + if isinstance(v1_value, int | float) and isinstance(v2_value, int | float): + return math.isclose(v1_value, v2_value, rel_tol=1e-9, abs_tol=1e-6) + return v1_value == v2_value + + +def household_differences( + v1_household: dict[str, Any], v2_household: dict[str, Any] +) -> list[str]: + """Return every value that differs between the two calculated households. + + Each entity, variable and period in either household is compared, so a + value only one of them has is a difference too. + """ + differences: list[str] = [] + for plural in sorted(set(v1_household) | set(v2_household)): + v1_entities = v1_household.get(plural) + v2_entities = v2_household.get(plural) + if not isinstance(v1_entities, dict) or not isinstance(v2_entities, dict): + if v1_entities != v2_entities: + differences.append(f"{plural}: {v1_entities!r} != {v2_entities!r}") + continue + for entity_id in sorted(set(v1_entities) | set(v2_entities)): + v1_variables = v1_entities.get(entity_id) or {} + v2_variables = v2_entities.get(entity_id) or {} + for variable in sorted(set(v1_variables) | set(v2_variables)): + v1_value = v1_variables.get(variable) + v2_value = v2_variables.get(variable) + if variable == "members" or not ( + isinstance(v1_value, dict) and isinstance(v2_value, dict) + ): + if not _same_value(v1_value, v2_value): + differences.append( + f"{plural}.{entity_id}.{variable}: " + f"{v1_value!r} != {v2_value!r}" + ) + continue + differences.extend( + f"{plural}.{entity_id}.{variable}.{period}: " + f"{v1_value.get(period)!r} != {v2_value.get(period)!r}" + for period in sorted(set(v1_value) | set(v2_value)) + if not _same_value(v1_value.get(period), v2_value.get(period)) + ) + return differences diff --git a/tests/integration/test_live_v2_household_calculations.py b/tests/integration/test_live_v2_household_calculations.py new file mode 100644 index 000000000..af918b27b --- /dev/null +++ b/tests/integration/test_live_v2_household_calculations.py @@ -0,0 +1,169 @@ +"""Live staging checks of ``/v2/household-calculations``. + +These run against the tagged staging candidate, through the staging Public +API, the staging simulation service, the v2 household functions on Modal and +the staging v2 database, where the simulation service records each +calculation as a household job: + +- a submit, poll and result cycle for a new US household; +- a latency check: several calculations submitted at once, so the simulation + service serves them in parallel, each accepted within + ``STAGING_HOUSEHOLD_SUBMIT_LIMIT_SECONDS`` and finished within + ``STAGING_HOUSEHOLD_FINISH_LIMIT_SECONDS``; +- a memory check: a household varied across 401 points of employment income, + with its net income calculated at each point, which must finish without + failing; a household function that runs out of memory fails it. + +The measured times are reported as a warning, so the job log keeps them. +""" + +from __future__ import annotations + +import os +import statistics +import warnings +from concurrent.futures import ThreadPoolExecutor + +import httpx + +from tests.fixtures.integration.live_household_calculations import ( + YEAR, + FinishedCalculation, + LiveHousehold, + calculate, + probe_amount, + store_household, +) + +PARALLEL_CALCULATIONS = int(os.environ.get("STAGING_HOUSEHOLD_PARALLEL_COUNT", "6")) +SUBMIT_LIMIT_SECONDS = float( + os.environ.get("STAGING_HOUSEHOLD_SUBMIT_LIMIT_SECONDS", "10") +) +FINISH_LIMIT_SECONDS = float( + os.environ.get("STAGING_HOUSEHOLD_FINISH_LIMIT_SECONDS", "300") +) +MEMORY_CHECK_POINTS = 401 + + +class HouseholdCalculationMeasurement(UserWarning): + """Times measured by the live household calculation checks.""" + + +def _us_household(name: str, employment_income: int) -> LiveHousehold: + return LiveHousehold( + name=name, + country_id="us", + people={ + "adult": { + "age": {YEAR: 40}, + "employment_income": {YEAR: employment_income}, + } + }, + groups={ + "household": { + "state_name": {YEAR: "NY"}, + "household_net_income": {YEAR: None}, + } + }, + ) + + +def test_a_us_household_is_submitted_polled_and_read(api_client, integration_probe_id): + household = _us_household("cycle", 51_000 + probe_amount(integration_probe_id)) + household_id = store_household(api_client, household) + + finished = calculate(api_client, household, household_id) + + item = finished.item + assert item["status"] == "succeeded", item["error"] + assert item["household_id"] == household_id + assert item["completed_at"] is not None + result = item["result"] + net_income = result["household"]["households"]["household-1"][ + "household_net_income" + ][YEAR] + assert isinstance(net_income, int | float) + # A US calculation applies the bundle's certified SPM selection. + assert result["spm_config"] is not None + assert item["policyengine_version"] + assert item["country_package_version"] + read_again = api_client.get( + f"/v2/household-calculations/{item['calculation_id']}" + ).json()["result"]["item"] + assert read_again == item + + +def test_parallel_calculations_finish_within_the_latency_limits( + api_base_url, integration_probe_id +): + run_amount = probe_amount(integration_probe_id) + households = [ + _us_household(f"parallel-{index}", 20_000 + 1_000 * index + run_amount) + for index in range(PARALLEL_CALCULATIONS) + ] + + def run(household: LiveHousehold) -> FinishedCalculation: + with httpx.Client(base_url=api_base_url, timeout=90.0) as client: + return calculate(client, household, store_household(client, household)) + + with ThreadPoolExecutor(max_workers=PARALLEL_CALCULATIONS) as pool: + outcomes = list(pool.map(run, households)) + + submit_times = [outcome.submit_seconds for outcome in outcomes] + finish_times = [outcome.total_seconds for outcome in outcomes] + warnings.warn( + HouseholdCalculationMeasurement( + f"{PARALLEL_CALCULATIONS} parallel household calculations: submit " + f"median {statistics.median(submit_times):.2f} s, longest " + f"{max(submit_times):.2f} s; finish median " + f"{statistics.median(finish_times):.1f} s, longest " + f"{max(finish_times):.1f} s" + ), + stacklevel=1, + ) + assert [outcome.item["status"] for outcome in outcomes] == ["succeeded"] * len( + outcomes + ), [outcome.item["error"] for outcome in outcomes] + assert max(submit_times) <= SUBMIT_LIMIT_SECONDS, submit_times + assert max(finish_times) <= FINISH_LIMIT_SECONDS, finish_times + + +def test_a_household_across_many_points_finishes(api_client, integration_probe_id): + household = LiveHousehold( + name="memory", + country_id="us", + people={"adult": {"age": {YEAR: 45}}}, + groups={ + "household": { + "state_name": {YEAR: "CA"}, + "household_net_income": {YEAR: None}, + } + }, + axes=[ + [ + { + "name": "employment_income", + "count": MEMORY_CHECK_POINTS, + "min": 0, + "max": 400_000 + probe_amount(integration_probe_id), + "period": YEAR, + } + ] + ], + ) + household_id = store_household(api_client, household) + + finished = calculate(api_client, household, household_id) + + warnings.warn( + HouseholdCalculationMeasurement( + f"Household across {MEMORY_CHECK_POINTS} points: submit " + f"{finished.submit_seconds:.2f} s, finish {finished.total_seconds:.1f} s" + ), + stacklevel=1, + ) + assert finished.item["status"] == "succeeded", finished.item["error"] + net_income = finished.item["result"]["household"]["households"]["household-1"][ + "household_net_income" + ][YEAR] + assert len(net_income) == MEMORY_CHECK_POINTS diff --git a/tests/integration/test_live_v2_household_parity.py b/tests/integration/test_live_v2_household_parity.py new file mode 100644 index 000000000..e8758e0ac --- /dev/null +++ b/tests/integration/test_live_v2_household_parity.py @@ -0,0 +1,187 @@ +"""Live staging parity of v2 household calculations with the in-process ``/calculate``. + +Each household is stored through ``/v2/households`` (and its reform through +``/v2/policies``), calculated through ``/v2/household-calculations`` (the v2 +household function on Modal) and, as the same situation and policy, through +the Public API's in-process ``/calculate`` (``/calculate-full`` when it adds +missing variables) on the same candidate. Every calculated value must match, +axes, reforms and added variables included. A run-specific input makes each +stored household new. +""" + +from __future__ import annotations + +import pytest + +from tests.fixtures.integration.live_household_calculations import ( + YEAR, + LiveHousehold, + calculate, + calculate_with_v1, + household_differences, + probe_amount, + store_household, + store_reform, +) + +CALCULATED = {YEAR: None} + + +def parity_households(run_amount: int) -> list[LiveHousehold]: + return [ + LiveHousehold( + name="us-single-california", + country_id="us", + people={ + "adult": { + "age": {YEAR: 35}, + "employment_income": {YEAR: 42_000 + run_amount}, + # Deprecated: dropped with a warning by both calculations. + "medical_out_of_pocket_expenses": {YEAR: 1_000}, + "income_tax": CALCULATED, + } + }, + groups={ + "household": { + "state_name": {YEAR: "CA"}, + "household_net_income": CALCULATED, + }, + "tax_unit": {"eitc": CALCULATED, "ctc": CALCULATED}, + "spm_unit": {"snap": CALCULATED}, + }, + expects_warnings=True, + ), + LiveHousehold( + name="us-couple-two-children-texas-axes", + country_id="us", + people={ + "first-adult": {"age": {YEAR: 38}}, + "second-adult": { + "age": {YEAR: 36}, + "employment_income": {YEAR: 15_000 + run_amount}, + }, + "first-child": {"age": {YEAR: 7}}, + "second-child": {"age": {YEAR: 3}}, + }, + groups={ + "household": { + "state_name": {YEAR: "TX"}, + "household_net_income": CALCULATED, + }, + "tax_unit": {"eitc": CALCULATED, "ctc": CALCULATED}, + "spm_unit": {"snap": CALCULATED}, + }, + axes=[ + [ + { + "name": "employment_income", + "count": 5, + "min": 0, + "max": 100_000, + "period": YEAR, + } + ] + ], + ), + LiveHousehold( + name="us-single-reform-added-variables", + country_id="us", + # /calculate-full adds variables for the age year of a person + # named ``you``; the v2 path for the household's default year. + people={ + "you": { + "age": {YEAR: 29}, + "employment_income": {YEAR: 61_000 + run_amount}, + } + }, + groups={"household": {"state_name": {YEAR: "NY"}}}, + reform={"gov.irs.deductions.standard.amount.SINGLE": 20_000}, + add_missing_variables=True, + ), + LiveHousehold( + name="uk-single", + country_id="uk", + people={ + "adult": { + "age": {YEAR: 30}, + "employment_income": {YEAR: 28_000 + run_amount}, + "income_tax": CALCULATED, + "national_insurance": CALCULATED, + } + }, + groups={"household": {"household_net_income": CALCULATED}}, + ), + LiveHousehold( + name="uk-single-reform", + country_id="uk", + people={ + "adult": { + "age": {YEAR: 41}, + "employment_income": {YEAR: 36_000 + run_amount}, + "income_tax": CALCULATED, + } + }, + groups={"household": {"household_net_income": CALCULATED}}, + reform={"gov.hmrc.income_tax.allowances.personal_allowance.amount": 15_000}, + ), + LiveHousehold( + name="uk-couple-child-axes", + country_id="uk", + people={ + "first-adult": {"age": {YEAR: 34}}, + "second-adult": { + "age": {YEAR: 33}, + "employment_income": {YEAR: 9_000 + run_amount}, + }, + "child": {"age": {YEAR: 4}}, + }, + groups={ + "household": {"household_net_income": CALCULATED}, + "benunit": { + "universal_credit": CALCULATED, + "child_benefit": CALCULATED, + }, + }, + axes=[ + [ + { + "name": "employment_income", + "count": 5, + "min": 0, + "max": 60_000, + "period": YEAR, + } + ] + ], + ), + ] + + +PARITY_NAMES = [household.name for household in parity_households(0)] + + +@pytest.mark.parametrize("household_name", PARITY_NAMES) +def test_a_v2_household_calculation_matches_api_v1( + api_client, integration_probe_id, household_name +): + (household,) = [ + candidate + for candidate in parity_households(probe_amount(integration_probe_id)) + if candidate.name == household_name + ] + household_id = store_household(api_client, household) + policy_id = store_reform(api_client, household) + + finished = calculate(api_client, household, household_id, policy_id) + v1_answer = calculate_with_v1(api_client, household) + + assert finished.item["status"] == "succeeded", finished.item["error"] + result = finished.item["result"] + differences = household_differences(v1_answer["result"], result["household"]) + assert differences == [], "\n".join(differences[:50]) + # The v2 database orders each stored record's keys itself, so warnings, + # listed in the household's key order, may come in another order. + assert sorted(result["warnings"]) == sorted(v1_answer.get("warnings", [])) + assert bool(result["warnings"]) == household.expects_warnings + if household.country_id == "us": + assert result["spm_config"] is not None diff --git a/tests/unit/test_cloud_run_push_workflow.py b/tests/unit/test_cloud_run_push_workflow.py index 2d3eb12d6..f0a5ba83b 100644 --- a/tests/unit/test_cloud_run_push_workflow.py +++ b/tests/unit/test_cloud_run_push_workflow.py @@ -232,6 +232,21 @@ def _assert_live_tests_run_in_parallel_after_readiness( assert "tests/integration/test_live_budget_window_cache.py" in live_test_script assert "tests/integration/test_live_calculate.py" in live_test_script assert "tests/integration/test_live_economy.py" in live_test_script + # The household calculation checks run after the other live tests, with + # fewer workers, so the parallel submissions never overlap them. + household_checks = live_test_script.index( + "tests/integration/test_live_v2_household_calculations.py" + ) + assert live_test_script.index("tests/integration/test_live_economy.py") < ( + household_checks + ) + assert ( + "tests/integration/test_live_v2_household_parity.py" + in (live_test_script[household_checks:]) + ) + assert "-n 4" in live_test_script[household_checks - 200 : household_checks] + # They wait on Modal; 30 minutes leaves room for both runs. + assert "timeout-minutes: 30" in parallel_live_tests assert "tests/integration/test_live_v2_policies.py" not in parallel_live_tests assert ( "API_BASE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }}" diff --git a/tests/unit/v2/test_live_household_parity_households.py b/tests/unit/v2/test_live_household_parity_households.py new file mode 100644 index 000000000..af01dc107 --- /dev/null +++ b/tests/unit/v2/test_live_household_parity_households.py @@ -0,0 +1,64 @@ +"""The staging parity set's households calculate the same year on both sides. + +The Public API's ``/calculate-full`` adds missing variables for the first +age period of a person named ``you``, or else the current year; the v2 +household function adds them for the stored household's default year. A +parity household that adds missing variables must therefore name its person +``you``, or the two sides differ once the calendar year moves on. +""" + +from __future__ import annotations + +import json +from datetime import date + +import httpx + +from policyengine_api.services import household_calculation_service +from tests.fixtures.integration.live_household_calculations import store_household +from tests.integration.test_live_v2_household_parity import parity_households + + +class DateIn2027(date): + """Today is a day in 2027, a year after the households' year.""" + + @classmethod + def today(cls) -> DateIn2027: + """Return 2027-06-01.""" + return cls(2027, 6, 1) + + +def test_households_that_add_variables_use_one_year_on_both_sides( + monkeypatch, +) -> None: + monkeypatch.setattr(household_calculation_service, "date", DateIn2027) + adding = [ + household + for household in parity_households(run_amount=0) + if household.add_missing_variables + ] + + assert adding + for household in adding: + v1_year = household_calculation_service.get_household_year( + household.situation() + ) + # The v2 side calculates the default year the household is stored with. + assert str(v1_year) == str(_stored_default_year(household)), household.name + + +def _stored_default_year(household) -> int: + """Return the default year the live check stores the household with.""" + sent: list[dict] = [] + + def answer(request: httpx.Request) -> httpx.Response: + sent.append(json.loads(request.content)) + return httpx.Response(201, json={"result": {"item": {"id": "stored"}}}) + + with httpx.Client( + base_url="https://public-api.example", + transport=httpx.MockTransport(answer), + ) as client: + store_household(client, household) + (request,) = sent + return request["default_year"] From 188832be0ae137d55e7d0154752cef6169255eb2 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:40:07 +0400 Subject: [PATCH 18/57] Run the household calculation checks after the other live staging tests run_live_staging_tests.sh runs the household calculation and parity checks in a second pytest run, on 4 workers, once the other live tests finish, so the six submissions sent at once never share the candidate's six request slots with them. The live job's limit rises from 20 to 30 minutes; it has taken about 10 minutes before this change. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/run_live_staging_tests.sh | 13 +++++++++++++ .github/workflows/push.yml | 4 +++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/scripts/run_live_staging_tests.sh b/.github/scripts/run_live_staging_tests.sh index 0d0907436..1b101c637 100755 --- a/.github/scripts/run_live_staging_tests.sh +++ b/.github/scripts/run_live_staging_tests.sh @@ -21,3 +21,16 @@ uv run --frozen --extra dev python -m pytest \ tests/integration/test_live_calculate.py \ tests/integration/test_live_economy.py \ -v + +# The household calculation checks run afterwards, on 4 workers, so they never +# share the candidate with the tests above. Their requests are short (a +# submission, then a poll every few seconds) apart from the parity tests' +# /calculate requests, but the parallel check submits 6 calculations at once: +# with the 3 other workers' requests, at most 3 wait for a free slot, as above. +uv run --frozen --extra dev python -m pytest \ + -n 4 \ + --dist load \ + --maxschedchunk=1 \ + tests/integration/test_live_v2_household_calculations.py \ + tests/integration/test_live_v2_household_parity.py \ + -v diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 015b710b2..cf77ec721 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -630,7 +630,9 @@ jobs: parallel-live-simulation-tests-staging-cloud-run: name: Run live simulations against the staging candidate runs-on: ubuntu-latest - timeout-minutes: 20 + # The household calculation checks run after the other live tests and + # wait on Modal. + timeout-minutes: 30 needs: deploy-cloud-run-staging if: | !cancelled() From f687a6f7067872ce566c848b05a593985c90fc78 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:40:52 +0400 Subject: [PATCH 19/57] Document household calculations and add a changelog fragment The Stage 11 household deployment notes describe both routes: the checks before submission and their answers, how the simulation service's answers are mapped, how GET reads household_jobs, where the simulation call is made, and the live staging checks. Co-Authored-By: Claude Opus 5.5 --- .../v2-household-calculations.added.md | 1 + docs/migration/stage-11-v2-households.md | 53 +++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 changelog.d/v2-household-calculations.added.md diff --git a/changelog.d/v2-household-calculations.added.md b/changelog.d/v2-household-calculations.added.md new file mode 100644 index 000000000..ffbb544e0 --- /dev/null +++ b/changelog.d/v2-household-calculations.added.md @@ -0,0 +1 @@ +Add `POST /v2/household-calculations`, which calculates a stored household through the simulation service after checking the country, the household and policy, the household's default year and the policy's parameters, and `GET /v2/household-calculations/{calculation_id}`, which reads the calculation's status, result and error. `/v2/model-catalog` now says whether each catalog version offers SPM settings, as `spm`. diff --git a/docs/migration/stage-11-v2-households.md b/docs/migration/stage-11-v2-households.md index 57c90ccc5..88bce25f8 100644 --- a/docs/migration/stage-11-v2-households.md +++ b/docs/migration/stage-11-v2-households.md @@ -296,3 +296,56 @@ packages are no longer installed. Their v1 household requests are rejected as unknown countries before any write, so the Cloud SQL-only path described above for those countries no longer runs. Existing Cloud SQL rows for them are kept. + +## Household Calculations + +`POST /v2/household-calculations` calculates a household stored through +`/v2/households`, under a policy stored through `/v2/policies` or under +current law, on the simulation service's v2 household functions. It needs no +account. The body names `country_id`, `household_id`, an optional +`policy_id`, optional `axes` (at most 8 groups of at most 32 axes, one point +count per group, at most 10,000 points in all), `add_missing_variables` and +optional `spm` settings. The year calculated is the household's +`default_year`. Before anything is sent, the Public API checks, in order: + +1. that the country's `countries` row has `household_enabled` (422 otherwise); +2. that the household, and the policy when one is named, exist in that + country (404 otherwise); +3. that the household has a `default_year` (422 otherwise); +4. that the policy sets no parameter the running catalog version lacks (422 + with `code` `missing_parameters` and the parameters' names in + `missing_parameters`). A policy no published catalog of the running + PolicyEngine.py version can check passes. + +It then sends the simulation service's `POST /household-calculations` the +IDs and options. The answer is 202 with the new job's `calculation_id` and +`status`; each submission creates a new job. The simulation service's 422 +reasons reach the caller as 422; any other refusal, no answer, an answer the +Public API can't read, missing configuration or a gateway token it can't +fetch is 503. + +`GET /v2/household-calculations/{calculation_id}` reads the job from +`household_jobs`, which the simulation service inserts and its household +function completes, as the runtime role, and never calls the simulation +service. It answers the job's status, catalog and package versions and +times; once it succeeded, `result` (the calculated household in its stored +key order, its warnings, the SPM selection applied and the SPM receipt); +once it failed, `error` (the household function's error code and message, +with the inputs it doesn't recognize and its SPM error, or only the job's +error message when it failed without one). An unknown ID is 404, and a +succeeded job whose result can't be read is 500. + +The call to the simulation service goes through +`policyengine_api/libs/simulation_household_calculations.py`, at +`SIMULATION_ENTRYPOINT_URL` with the gateway credentials of every other +simulation call. `post_household_calculation` is the only code that sends +the request, so moving to the generated simulation client changes that +function and the dependency. + +The live staging job runs, after the other live tests: a submit and read +cycle, six calculations submitted at once (each accepted within +`STAGING_HOUSEHOLD_SUBMIT_LIMIT_SECONDS`, 10 by default, and finished within +`STAGING_HOUSEHOLD_FINISH_LIMIT_SECONDS`, 300), a household across 401 axis +points, and a parity set of three US and three UK households compared value +by value with `/calculate` and `/calculate-full`. The checks need the +simulation service's household calculation route deployed to staging first. From ed03c25e2977a4ac16c209c13a1f9db0bd1688bb Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:44:26 +0400 Subject: [PATCH 20/57] Name the household functions' unexpected-error messages in the error docs The simulation contract's household functions record a job that raised or was cancelled with a null result and an error message starting household_function_error or household_function_cancelled. The error model's documentation names them, and the tests use that format. Nothing else changes: the single-policy result and the failure document already match the contract. Co-Authored-By: Claude Opus 5.5 --- .../v2/household_calculations/response_models.py | 6 ++++-- .../services/v2/household_calculations/types.py | 5 +++-- tests/integration/test_v2_household_calculation_reads.py | 7 +++++-- tests/unit/v2/test_household_calculation_results.py | 7 +++++-- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py index b395fd8cf..cebfe22f2 100644 --- a/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py +++ b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py @@ -74,8 +74,10 @@ class HouseholdCalculationErrorItem(StrictHouseholdCalculationAPIModel): with ``input_errors`` naming each input the country model does not recognize; ``invalid_household``; ``invalid_spm_selection``, with ``spm_error``; ``bundle_mismatch``; or ``calculation_failed``. It is null - when the calculation failed for another reason, such as a Modal call that - ended; ``message`` then says why, and submitting again may succeed. + when the calculation failed for another reason, such as a household + function that raised or was cancelled; ``message`` is then the job's + one-line error, such as ``household_function_error: …`` or + ``household_function_cancelled: …``, and submitting again may succeed. """ code: str | None diff --git a/policyengine_api/services/v2/household_calculations/types.py b/policyengine_api/services/v2/household_calculations/types.py index 4d1a66703..1f4bed6d9 100644 --- a/policyengine_api/services/v2/household_calculations/types.py +++ b/policyengine_api/services/v2/household_calculations/types.py @@ -166,8 +166,9 @@ class HouseholdCalculationFailure: ``code`` is the household function's error code, such as ``invalid_inputs``, with ``input_errors`` naming each input the country model does not recognize and ``spm_error`` an SPM error. A job that failed - without the household function's failure, such as one whose Modal call - ended, has no code, and ``message`` is the job's error message. + without the household function's failure document (one whose function + raised or was cancelled) has no code, and ``message`` is the job's error + message, such as ``household_function_error: …``. """ code: str | None diff --git a/tests/integration/test_v2_household_calculation_reads.py b/tests/integration/test_v2_household_calculation_reads.py index 1c7e20f2f..d5abc7ece 100644 --- a/tests/integration/test_v2_household_calculation_reads.py +++ b/tests/integration/test_v2_household_calculation_reads.py @@ -139,7 +139,7 @@ def stored_jobs(disposable_v2_database_url, managed_v2_roles) -> Iterator[Stored rows, jobs.failed_unexpectedly, "failed", - error_message="The Modal call ended.", + error_message="household_function_error: the Modal call ended", completed_at=COMPLETED_AT, ) _insert_job(connection, rows, jobs.pending, "pending") @@ -211,7 +211,10 @@ def test_the_runtime_role_reads_why_a_calculation_failed( assert refused.error.input_errors == (INPUT_ERROR,) assert failed_unexpectedly.error is not None assert failed_unexpectedly.error.code is None - assert failed_unexpectedly.error.message == "The Modal call ended." + assert ( + failed_unexpectedly.error.message + == "household_function_error: the Modal call ended" + ) def test_the_runtime_role_reads_an_unfinished_calculation( diff --git a/tests/unit/v2/test_household_calculation_results.py b/tests/unit/v2/test_household_calculation_results.py index 730574e57..7feff0da0 100644 --- a/tests/unit/v2/test_household_calculation_results.py +++ b/tests/unit/v2/test_household_calculation_results.py @@ -132,13 +132,16 @@ def test_a_job_the_household_function_refused_reads_back_its_error() -> None: def test_a_job_that_failed_unexpectedly_reads_back_its_error_message() -> None: calculation = _read( - _job(HouseholdJobStatus.FAILED, error_message="The Modal call ended.") + _job( + HouseholdJobStatus.FAILED, + error_message="household_function_error: the Modal call ended", + ) ) assert calculation.result is None assert calculation.error is not None assert calculation.error.code is None - assert calculation.error.message == "The Modal call ended." + assert calculation.error.message == "household_function_error: the Modal call ended" assert calculation.error.input_errors == () assert calculation.error.spm_error is None From 813ca7c253c1b8116021159e9a48387604a3cf0b Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:12:12 +0400 Subject: [PATCH 21/57] Test submitting through the database-backed services (failing) The route runs with the real countries, households and policies services over the v2 tables in SQLite, so the policy check runs the real catalog query. A submission, a missing-parameter refusal and an unknown household pass. When the catalog query fails, the policy service raises MetadataCatalogUnavailableError, which the route answers 500 instead of 503, as /v2/policies answers it. Co-Authored-By: Claude Opus 5.5 --- ...t_household_calculation_database_routes.py | 243 ++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 tests/unit/v2/test_household_calculation_database_routes.py diff --git a/tests/unit/v2/test_household_calculation_database_routes.py b/tests/unit/v2/test_household_calculation_database_routes.py new file mode 100644 index 000000000..d0e85a8d8 --- /dev/null +++ b/tests/unit/v2/test_household_calculation_database_routes.py @@ -0,0 +1,243 @@ +"""Submit household calculations through the route and the database-backed services. + +The countries, households and policies services read a SQLite database with +the v2 tables, so the policy check runs the real catalog query; only the +simulation service is a stand-in. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from uuid import uuid4 + +import pytest +from fastapi.testclient import TestClient +from flask import Flask +from sqlalchemy import text +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool +from sqlmodel import Session, create_engine + +from policyengine_api.asgi_factory import create_asgi_app +from policyengine_api.data.v2.models import ( + V2_METADATA, + Country, + Household, + Parameter, + ParameterValue, + Policy, + TaxBenefitModel, + TaxBenefitModelVersion, +) +from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.migration_flags import ( + RouteImplementation, + RouteImplementationSettings, +) +from policyengine_api.services.v2.countries.database_session import ( + CountryDatabaseSession, +) +from policyengine_api.services.v2.countries.services import V2CountryService +from policyengine_api.services.v2.household_calculations.database_session import ( + HouseholdCalculationDatabaseSession, +) +from policyengine_api.services.v2.household_calculations.services import ( + V2HouseholdCalculationService, +) +from policyengine_api.services.v2.households.database_session import ( + HouseholdDatabaseSession, +) +from policyengine_api.services.v2.households.services import V2HouseholdService +from policyengine_api.services.v2.policies.database_session import ( + PolicyDatabaseSession, +) +from policyengine_api.services.v2.policies.services import V2PolicyService +from tests.fixtures.household_calculation_documents import JOB_ID +from tests.unit.v2.household_calculation_support import FakeSimulation + +RUNNING_VERSION = "6.10.0" +EARLIER_VERSION = "6.9.0" +KEPT_PARAMETER = "gov.example.rate" +REMOVED_PARAMETER = "gov.example.removed_rate" +CALCULATIONS = "/v2/household-calculations" +STORED_AT = datetime(2026, 11, 10, tzinfo=UTC) + + +@pytest.fixture +def engine(): + # One connection shared by every thread, so the route's worker thread + # reads the same in-memory database. + database_engine = create_engine( + "sqlite://", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + V2_METADATA.create_all(database_engine) + yield database_engine + database_engine.dispose() + + +def _catalog_version( + session: Session, model: TaxBenefitModel, version: str, names: list[str] +) -> dict[str, Parameter]: + model_version = TaxBenefitModelVersion( + model=model, version=version, current_law_id=1, metadata_time_periods=[2026] + ) + parameters = { + name: Parameter(name=name, tax_benefit_model_version=model_version) + for name in names + } + session.add(model_version) + session.add_all(parameters.values()) + session.flush() + return parameters + + +def _policy(session: Session, parameters: dict[str, Parameter], hash_seed: str): + model_version = next(iter(parameters.values())).tax_benefit_model_version + policy = Policy( + country_id="us", + tax_benefit_model_id=model_version.model_id, + tax_benefit_model_version_id=model_version.id, + canonicalization_version=1, + content_hash=hash_seed * 64, + created_at=STORED_AT, + updated_at=STORED_AT, + ) + session.add(policy) + session.flush() + session.add_all( + ParameterValue( + parameter_id=parameter.id, + value_json=0.2, + start_date=STORED_AT, + policy_id=policy.id, + ) + for parameter in parameters.values() + ) + return policy + + +@pytest.fixture +def stored(engine) -> dict[str, object]: + """Store the US, a household, and a policy on each of two catalog versions.""" + with Session(engine, expire_on_commit=False) as session: + session.add(Country(id="us", household_enabled=True, society_wide_enabled=True)) + model = TaxBenefitModel(name="policyengine-us") + earlier = _catalog_version( + session, model, EARLIER_VERSION, [KEPT_PARAMETER, REMOVED_PARAMETER] + ) + running = _catalog_version(session, model, RUNNING_VERSION, [KEPT_PARAMETER]) + household = Household( + country_id="us", + default_year=2026, + household_data={"people": []}, + canonicalization_version=1, + content_hash="c" * 64, + ) + session.add(household) + rows = { + "household": household, + "earlier_policy": _policy(session, earlier, "a"), + "running_policy": _policy(session, running, "b"), + } + session.commit() + return {name: row.id for name, row in rows.items()} + + +@pytest.fixture +def simulation() -> FakeSimulation: + return FakeSimulation() + + +@pytest.fixture +def client(engine, simulation) -> TestClient: + sessions = sessionmaker(engine, class_=Session, expire_on_commit=False) + + def service() -> V2HouseholdCalculationService: + return V2HouseholdCalculationService( + countries=V2CountryService(CountryDatabaseSession(sessions)), + households=V2HouseholdService(HouseholdDatabaseSession(sessions)), + policies=V2PolicyService( + PolicyDatabaseSession(sessions), + running_policyengine_version=RUNNING_VERSION, + ), + database_session=HouseholdCalculationDatabaseSession(sessions), + simulation_factory=lambda: simulation, + ) + + dependencies = NativeRouteDependencies( + readiness_probe=lambda: True, + gateway_client_factory=lambda: None, + metadata_reader_factory=lambda: None, + v2_household_calculation_service_factory=service, + ) + settings = RouteImplementationSettings( + health=RouteImplementation.FLASK_FALLBACK, + specification=RouteImplementation.FLASK_FALLBACK, + metadata=RouteImplementation.FLASK_FALLBACK, + ) + return TestClient( + create_asgi_app( + Flask(__name__), dependencies=dependencies, route_settings=settings + ), + raise_server_exceptions=False, + ) + + +def _body(stored, policy: str | None) -> dict[str, object]: + return { + "country_id": "us", + "household_id": str(stored["household"]), + "policy_id": None if policy is None else str(stored[policy]), + } + + +def test_a_stored_household_is_submitted_after_the_database_backed_checks( + client, stored, simulation +) -> None: + response = client.post(CALCULATIONS, json=_body(stored, "running_policy")) + + assert response.status_code == 202 + assert response.json()["result"]["item"]["calculation_id"] == str(JOB_ID) + (sent,) = simulation.submissions + assert sent.policy_id == stored["running_policy"] + + +def test_a_policy_setting_parameters_the_running_catalog_lacks_is_refused( + client, stored, simulation +) -> None: + response = client.post(CALCULATIONS, json=_body(stored, "earlier_policy")) + + assert response.status_code == 422 + assert response.json()["code"] == "missing_parameters" + assert response.json()["missing_parameters"] == [REMOVED_PARAMETER] + assert simulation.submissions == [] + + +def test_an_unknown_household_is_not_found_in_the_database( + client, stored, simulation +) -> None: + body = {**_body(stored, None), "household_id": str(uuid4())} + + response = client.post(CALCULATIONS, json=body) + + assert response.status_code == 404 + assert simulation.submissions == [] + + +def test_a_catalog_that_cannot_be_queried_answers_503( + engine, client, stored, simulation +) -> None: + # The policy check's catalog query reads this table; nothing else does. + with engine.begin() as connection: + connection.execute(text("DROP TABLE tax_benefit_model_versions")) + + response = client.post(CALCULATIONS, json=_body(stored, "running_policy")) + + assert response.status_code == 503 + assert response.json() == { + "status": "error", + "message": "V2 household calculation storage is unavailable", + } + assert simulation.submissions == [] From e0008859c9d0317f59b5a29417d555f2fd044e37 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:12:46 +0400 Subject: [PATCH 22/57] Answer 503 when the policy check's catalog query fails The policy service reports a catalog it cannot query as MetadataCatalogUnavailableError. The household calculation service now turns it, like a SQLAlchemy error, into its storage error, which the route answers 503, as /v2/policies does. Co-Authored-By: Claude Opus 5.5 --- .../services/v2/household_calculations/services.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/policyengine_api/services/v2/household_calculations/services.py b/policyengine_api/services/v2/household_calculations/services.py index 869f9344e..3360040f0 100644 --- a/policyengine_api/services/v2/household_calculations/services.py +++ b/policyengine_api/services/v2/household_calculations/services.py @@ -23,6 +23,9 @@ from sqlalchemy.exc import SQLAlchemyError +from policyengine_api.data.v2.catalog.catalog_selection_validation import ( + MetadataCatalogUnavailableError, +) from policyengine_api.services.v2.household_calculations.database_connectors import ( reads as household_job_reads, ) @@ -61,10 +64,14 @@ @contextmanager def _storage_failures() -> Iterator[None]: - """Raise HouseholdCalculationStorageError for a failed database read.""" + """Raise HouseholdCalculationStorageError for a failed database read. + + That includes the policy check's catalog query, which the policy service + reports as MetadataCatalogUnavailableError. + """ try: yield - except SQLAlchemyError as error: + except (SQLAlchemyError, MetadataCatalogUnavailableError) as error: msg = "V2 household calculation storage is unavailable" raise HouseholdCalculationStorageError(msg) from error From e559ef77f446963a9ea87776e4c4ba2628bbc6a4 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:14:14 +0400 Subject: [PATCH 23/57] Test the simulation submission's IDs, sim's codes and an unreadable 202 (failing) Through the route and the real simulation client, the submission carries the request ID and an observability ID: the caller's, or a new one. Sim's 400 and 503 answers name their code, and a 503 after Modal refused the call names the household job it failed. A 202 whose body cannot be read means sim has already created the job, so it is its own error, which the route answers 502 rather than the 503 that invites a retry. The submission sends no observability ID yet, and the error does not exist. Co-Authored-By: Claude Opus 5.5 --- .../test_simulation_household_calculations.py | 53 ++++++++- ...t_household_calculation_database_routes.py | 110 +++++++++++++++++- .../v2/test_household_calculation_routes.py | 12 +- 3 files changed, 169 insertions(+), 6 deletions(-) diff --git a/tests/unit/libs/test_simulation_household_calculations.py b/tests/unit/libs/test_simulation_household_calculations.py index c06db82d3..971943f23 100644 --- a/tests/unit/libs/test_simulation_household_calculations.py +++ b/tests/unit/libs/test_simulation_household_calculations.py @@ -16,6 +16,7 @@ ) from policyengine_api.services.v2.household_calculations.types import ( HouseholdCalculationAccepted, + SimulationServiceAnswerUnreadableError, SimulationServiceInvalidRequestError, SimulationServiceUnavailableError, ) @@ -179,17 +180,53 @@ def test_a_refusal_of_another_shape_is_unavailable(body) -> None: def test_a_reference_the_service_cannot_find_is_unavailable_with_its_reason() -> None: - service = RecordingService(400, {"detail": f"Household {HOUSEHOLD_ID} not found"}) + service = RecordingService( + 400, + {"code": "unknown_household", "detail": f"Household {HOUSEHOLD_ID} not found"}, + ) with pytest.raises(SimulationServiceUnavailableError) as refused: service.client().submit(calculation_input()) assert str(refused.value) == ( - "The simulation service refused the calculation: " + "The simulation service refused the calculation (unknown_household): " f"Household {HOUSEHOLD_ID} not found" ) +def test_a_modal_call_the_service_could_not_start_names_the_failed_job() -> None: + service = RecordingService( + 503, + { + "code": "modal_call_refused", + "detail": "Modal refused the household function's call", + "household_job_id": str(JOB_ID), + }, + ) + + with pytest.raises(SimulationServiceUnavailableError) as refused: + service.client().submit(calculation_input()) + + assert str(refused.value) == ( + "The simulation service is unavailable (modal_call_refused): Modal " + f"refused the household function's call; it failed household job {JOB_ID}" + ) + + +def test_a_dependency_the_service_cannot_read_names_its_code() -> None: + service = RecordingService( + 503, {"code": "database_unavailable", "detail": "The database is down"} + ) + + with pytest.raises(SimulationServiceUnavailableError) as refused: + service.client().submit(calculation_input()) + + assert str(refused.value) == ( + "The simulation service is unavailable (database_unavailable): " + "The database is down" + ) + + @pytest.mark.parametrize("status_code", [400, 401, 404, 500, 502, 503]) def test_any_other_answer_is_unavailable(status_code) -> None: service = RecordingService(status_code, b"") @@ -207,12 +244,20 @@ def test_any_other_answer_is_unavailable(status_code) -> None: pytest.param({"job_id": str(JOB_ID), "status": "queued"}, id="bad-status"), ], ) -def test_an_accepted_answer_of_another_shape_is_unavailable(body) -> None: +def test_an_accepted_answer_that_cannot_be_read_is_not_unavailable(body) -> None: + # The service has created the job and started its call, so the caller is + # told the answer was unreadable rather than to try again. service = RecordingService(202, body) - with pytest.raises(SimulationServiceUnavailableError, match="could not be read"): + with pytest.raises(SimulationServiceAnswerUnreadableError) as unreadable: service.client().submit(calculation_input()) + assert not isinstance(unreadable.value, SimulationServiceUnavailableError) + assert str(unreadable.value) == ( + "The simulation service accepted the calculation, but its answer " + "could not be read." + ) + def test_an_unreachable_service_is_unavailable() -> None: def refuse(request: httpx.Request) -> httpx.Response: diff --git a/tests/unit/v2/test_household_calculation_database_routes.py b/tests/unit/v2/test_household_calculation_database_routes.py index d0e85a8d8..f6ca70dd9 100644 --- a/tests/unit/v2/test_household_calculation_database_routes.py +++ b/tests/unit/v2/test_household_calculation_database_routes.py @@ -8,8 +8,9 @@ from __future__ import annotations from datetime import UTC, datetime -from uuid import uuid4 +from uuid import UUID, uuid4 +import httpx import pytest from fastapi.testclient import TestClient from flask import Flask @@ -30,10 +31,15 @@ TaxBenefitModelVersion, ) from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.libs.simulation_household_calculations import ( + SimulationHouseholdCalculationClient, +) from policyengine_api.migration_flags import ( RouteImplementation, RouteImplementationSettings, ) +from policyengine_api.observability.identifiers import OBSERVABILITY_ID_HEADER +from policyengine_api.request_context import REQUEST_ID_HEADER from policyengine_api.services.v2.countries.database_session import ( CountryDatabaseSession, ) @@ -241,3 +247,105 @@ def test_a_catalog_that_cannot_be_queried_answers_503( "message": "V2 household calculation storage is unavailable", } assert simulation.submissions == [] + + +GATEWAY_AND_ENTRYPOINT_VARIABLES = ( + "GATEWAY_AUTH_ISSUER", + "GATEWAY_AUTH_AUDIENCE", + "GATEWAY_AUTH_CLIENT_ID", + "GATEWAY_AUTH_CLIENT_SECRET", + "GATEWAY_AUTH_CLIENT_SECRET_RESOURCE", + "GATEWAY_AUTH_REQUIRED", + "SIM_ENTRYPOINT", +) + + +class SimulationRecorder: + """The simulation service, answering every submission 202 and recording it.""" + + def __init__(self) -> None: + self.requests: list[httpx.Request] = [] + + def answer(self, request: httpx.Request) -> httpx.Response: + self.requests.append(request) + return httpx.Response(202, json={"job_id": str(JOB_ID), "status": "pending"}) + + +@pytest.fixture +def simulation_recorder(monkeypatch) -> SimulationRecorder: + """Configure the Cloud Run simulation service, with no gateway credentials.""" + for name in GATEWAY_AND_ENTRYPOINT_VARIABLES: + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("SIMULATION_ENTRYPOINT_URL", "https://simulation.example.test") + return SimulationRecorder() + + +@pytest.fixture +def client_with_simulation_client(engine, simulation_recorder) -> TestClient: + """Serve the route with the real simulation client over a recording transport.""" + sessions = sessionmaker(engine, class_=Session, expire_on_commit=False) + transport = httpx.MockTransport(simulation_recorder.answer) + + def service() -> V2HouseholdCalculationService: + return V2HouseholdCalculationService( + countries=V2CountryService(CountryDatabaseSession(sessions)), + households=V2HouseholdService(HouseholdDatabaseSession(sessions)), + policies=V2PolicyService( + PolicyDatabaseSession(sessions), + running_policyengine_version=RUNNING_VERSION, + ), + database_session=HouseholdCalculationDatabaseSession(sessions), + simulation_factory=lambda: ( + SimulationHouseholdCalculationClient.from_environment( + transport=transport + ) + ), + ) + + dependencies = NativeRouteDependencies( + readiness_probe=lambda: True, + gateway_client_factory=lambda: None, + metadata_reader_factory=lambda: None, + v2_household_calculation_service_factory=service, + ) + settings = RouteImplementationSettings( + health=RouteImplementation.FLASK_FALLBACK, + specification=RouteImplementation.FLASK_FALLBACK, + metadata=RouteImplementation.FLASK_FALLBACK, + ) + return TestClient( + create_asgi_app( + Flask(__name__), dependencies=dependencies, route_settings=settings + ), + raise_server_exceptions=False, + ) + + +def test_the_submission_carries_the_request_and_callers_observability_ids( + client_with_simulation_client, stored, simulation_recorder +) -> None: + observability_id = str(uuid4()) + + response = client_with_simulation_client.post( + CALCULATIONS, + json=_body(stored, "running_policy"), + headers={OBSERVABILITY_ID_HEADER: observability_id}, + ) + + assert response.status_code == 202 + (sent,) = simulation_recorder.requests + assert str(sent.url) == "https://simulation.example.test/household-calculations" + assert sent.headers[REQUEST_ID_HEADER] == response.headers[REQUEST_ID_HEADER] + assert sent.headers[OBSERVABILITY_ID_HEADER] == observability_id + + +def test_a_submission_without_an_observability_id_is_given_one( + client_with_simulation_client, stored, simulation_recorder +) -> None: + response = client_with_simulation_client.post( + CALCULATIONS, json=_body(stored, None) + ) + + assert response.status_code == 202 + (sent,) = simulation_recorder.requests + assert UUID(sent.headers[OBSERVABILITY_ID_HEADER]) diff --git a/tests/unit/v2/test_household_calculation_routes.py b/tests/unit/v2/test_household_calculation_routes.py index 45821ba76..d618b3227 100644 --- a/tests/unit/v2/test_household_calculation_routes.py +++ b/tests/unit/v2/test_household_calculation_routes.py @@ -21,6 +21,7 @@ HouseholdCalculationOutput, HouseholdCalculationRead, HouseholdCalculationSubmitted, + SimulationServiceAnswerUnreadableError, SimulationServiceInvalidRequestError, SimulationServiceUnavailableError, ) @@ -257,6 +258,15 @@ def test_a_malformed_request_is_refused_before_anything_is_read( 503, "V2 household calculation storage is unavailable", ), + ( + SimulationServiceAnswerUnreadableError( + "The simulation service accepted the calculation, but its " + "answer could not be read." + ), + 502, + "The simulation service accepted the calculation, but its " + "answer could not be read.", + ), ( RuntimeError("secret detail"), 500, @@ -415,7 +425,7 @@ def test_openapi_documents_both_routes_and_the_missing_parameter_refusal( submit = schema["paths"][CALCULATIONS]["post"] read = schema["paths"][f"{CALCULATIONS}/{{calculation_id}}"]["get"] - assert set(submit["responses"]) >= {"202", "404", "422", "500", "503"} + assert set(submit["responses"]) >= {"202", "404", "422", "500", "502", "503"} assert set(read["responses"]) >= {"200", "404", "422", "500", "503"} assert "security" not in submit refusal_schema = submit["responses"]["422"]["content"]["application/json"]["schema"] From cfc74603a82b60aa89fbcbaea1cc878abe1e2bbf Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:15:45 +0400 Subject: [PATCH 24/57] Send the observability ID to sim, name sim's codes, and answer 502 for an unreadable 202 The submission binds the request's observability ID (the caller's, or a new one) before it calls the simulation service, so the shared request hook sends it with the request ID, and sim passes it to the household function. A 400 or 503 from sim gives its code and reason in the 503's message, and a 503 after Modal refused the call names the job sim failed. A 202 whose body cannot be read raises SimulationServiceAnswerUnreadableError, answered 502: the job already exists, so the caller is not invited to submit again. Co-Authored-By: Claude Opus 5.5 --- .../household_calculations/response_models.py | 16 +++-- .../v2/household_calculations/routes.py | 4 ++ .../libs/simulation_household_calculations.py | 71 ++++++++++++++----- .../v2/household_calculations/types.py | 12 +++- .../test_simulation_household_calculations.py | 9 +++ 5 files changed, 88 insertions(+), 24 deletions(-) diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py index cebfe22f2..0233dff07 100644 --- a/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py +++ b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py @@ -178,6 +178,13 @@ class HouseholdCalculationResponse( } SUBMISSION_REFUSAL_RESPONSES: dict[int | str, dict[str, Any]] = { + 502: { + "model": V2ErrorResponse, + "description": ( + "The simulation service accepted the calculation, so its job exists, " + "but its answer cannot be read. Submitting again creates another job." + ), + }, 422: { "model": V2MissingParametersErrorResponse | V2ErrorResponse, "description": ( @@ -191,10 +198,11 @@ class HouseholdCalculationResponse( 503: { "model": V2ErrorResponse, "description": ( - "The simulation service is unavailable, cannot be reached, answers " - "in a shape the Public API cannot read, or refuses a household, " - "policy or catalog version the Public API has already checked; or " - "v2 storage is unavailable." + "The simulation service is unavailable or cannot be reached, " + "refuses in a shape the Public API cannot read, or refuses a " + "household, policy or catalog version the Public API has already " + "checked; or v2 storage is unavailable. The message gives the " + "simulation service's code and reason when it states them." ), }, } diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/routes.py b/policyengine_api/fastapi_routes/v2/household_calculations/routes.py index 059ce8f7d..be8ab8738 100644 --- a/policyengine_api/fastapi_routes/v2/household_calculations/routes.py +++ b/policyengine_api/fastapi_routes/v2/household_calculations/routes.py @@ -38,6 +38,7 @@ HouseholdCalculationSubmittedResult, ) from policyengine_api.services.v2.household_calculations.types import ( + SimulationServiceAnswerUnreadableError, SimulationServiceInvalidRequestError, SimulationServiceUnavailableError, ) @@ -83,6 +84,9 @@ 404, ), V2ErrorMapping((SimulationServiceUnavailableError,), 503), + # The simulation service created the job, so this is not a 503, which + # invites the caller to submit again. + V2ErrorMapping((SimulationServiceAnswerUnreadableError,), 502), V2ErrorMapping( (HouseholdCalculationResultUnreadableError,), 500, diff --git a/policyengine_api/libs/simulation_household_calculations.py b/policyengine_api/libs/simulation_household_calculations.py index 580f2f936..89b4036de 100644 --- a/policyengine_api/libs/simulation_household_calculations.py +++ b/policyengine_api/libs/simulation_household_calculations.py @@ -12,17 +12,27 @@ (null for current law), ``add_missing_variables``, and the axes and SPM settings only when the caller gave them. The simulation service answers: -- 202 with the new job's ID and status; +- 202 with the new job's ID and status. A 202 whose body cannot be read + is SimulationServiceAnswerUnreadableError: the job exists, so the caller + must not be invited to submit again; - 422 with its reasons when it refuses the caller's data (a malformed - request, or one its household contract refuses): the reasons reach the - caller as SimulationServiceInvalidRequestError; + request, or one its household contract refuses), one per field as + FastAPI writes them: the reasons reach the caller as + SimulationServiceInvalidRequestError; - anything else (400 for a household, policy or catalog version it cannot find, which the Public API checks first; 503 when it cannot read its - manifest or database or Modal refuses the call): the service is - unavailable (SimulationServiceUnavailableError), as it is for no answer, - an answer of another shape, a gateway token that cannot be fetched and + manifest or database or Modal refuses the call, failing the job it + names): the service is unavailable (SimulationServiceUnavailableError), + with its ``code`` and reason when it gives them, as it is for no answer, + a refusal of another shape, a gateway token that cannot be fetched and missing configuration. +Each submission carries the request's ID and observability ID headers. The +observability ID is the request's own, else the caller's, else a new one, +bound for the rest of the request, as the Public API's own household +calculations bind one once they accept a calculation; the simulation service +passes it to the household function. + Wiring the generated simulation client: ``post_household_calculation`` is the only code that sends the request, and it returns the status and body, which the rest of this module reads. Once a ``policyengine-api-simulation-client`` @@ -43,9 +53,11 @@ from policyengine_api.libs.gateway_auth import GatewayAuthError from policyengine_api.libs.simulation_entrypoint import SimulationEntrypointClient +from policyengine_api.request_context import start_observability_id from policyengine_api.services.v2.household_calculations.types import ( HouseholdCalculationAccepted, HouseholdJobStatusName, + SimulationServiceAnswerUnreadableError, SimulationServiceInvalidRequestError, SimulationServiceUnavailableError, ) @@ -60,6 +72,9 @@ HOUSEHOLD_CALCULATIONS_PATH = "/household-calculations" CLOUD_RUN_SIMULATION_ENTRYPOINT = "cloud_run_simulation_entrypoint" UNREADABLE_ANSWER = "The simulation service's answer could not be read." +UNREADABLE_ACCEPTANCE = ( + "The simulation service accepted the calculation, but its answer could not be read." +) @dataclass(frozen=True) @@ -91,9 +106,15 @@ class _RefusedField(_AnswerModel): class _Refusal(_AnswerModel): - """A refusal's body: a reason, or each refused request field.""" + """A refusal's body: a reason, or each refused request field. + + A 400 or 503 also names its ``code``, and a 503 after Modal refused the + call names the household job it failed. + """ detail: str | list[_RefusedField] + code: str | None = None + household_job_id: UUID | None = None def household_calculation_request( @@ -169,6 +190,22 @@ def _refusal(answer: SimulationAnswer) -> _Refusal | None: return None +def _unavailable(answer: SimulationAnswer, refusal: _Refusal | None) -> str: + """Describe an answer that means the service cannot calculate now.""" + if refusal is None or not isinstance(refusal.detail, str): + return f"The simulation service answered {answer.status_code}." + stated = ( + "refused the calculation" + if answer.status_code == HTTPStatus.BAD_REQUEST + else "is unavailable" + ) + code = "" if refusal.code is None else f" ({refusal.code})" + message = f"The simulation service {stated}{code}: {refusal.detail}" + if refusal.household_job_id is not None: + message += f"; it failed household job {refusal.household_job_id}" + return message + + class SimulationHouseholdCalculationClient: """Submit household calculations to the simulation service.""" @@ -212,9 +249,11 @@ def submit( """Submit the calculation and return the job the service created. Raise SimulationServiceInvalidRequestError with the service's reasons - when it refuses the caller's data, and SimulationServiceUnavailableError - otherwise. + when it refuses the caller's data, SimulationServiceAnswerUnreadableError + when it accepted the calculation but its answer cannot be read, and + SimulationServiceUnavailableError otherwise. """ + start_observability_id() try: answer = self._send( self._http_client, household_calculation_request(calculation) @@ -226,7 +265,9 @@ def submit( try: accepted = _AcceptedJob.model_validate_json(answer.content) except ValidationError as error: - raise SimulationServiceUnavailableError(UNREADABLE_ANSWER) from error + raise SimulationServiceAnswerUnreadableError( + UNREADABLE_ACCEPTANCE + ) from error return HouseholdCalculationAccepted( calculation_id=accepted.job_id, status=accepted.status ) @@ -235,15 +276,7 @@ def submit( if refusal is None: raise SimulationServiceUnavailableError(UNREADABLE_ANSWER) raise SimulationServiceInvalidRequestError(_invalid_request_reason(refusal)) - if ( - answer.status_code == HTTPStatus.BAD_REQUEST - and refusal is not None - and isinstance(refusal.detail, str) - ): - msg = f"The simulation service refused the calculation: {refusal.detail}" - raise SimulationServiceUnavailableError(msg) - msg = f"The simulation service answered {answer.status_code}." - raise SimulationServiceUnavailableError(msg) + raise SimulationServiceUnavailableError(_unavailable(answer, refusal)) @lru_cache(maxsize=1) diff --git a/policyengine_api/services/v2/household_calculations/types.py b/policyengine_api/services/v2/household_calculations/types.py index 1f4bed6d9..f955abf4f 100644 --- a/policyengine_api/services/v2/household_calculations/types.py +++ b/policyengine_api/services/v2/household_calculations/types.py @@ -313,7 +313,9 @@ def submit( """Submit the calculation and return the job the service created. Raise SimulationServiceInvalidRequestError when the service refuses - the caller's data, with its reasons, and + the caller's data, with its reasons; + SimulationServiceAnswerUnreadableError when it accepted the + calculation but its answer cannot be read; and SimulationServiceUnavailableError for any other refusal or failure. """ ... @@ -328,6 +330,14 @@ class SimulationServiceUnavailableError(RuntimeError): """ +class SimulationServiceAnswerUnreadableError(RuntimeError): + """The simulation service accepted the calculation, but its answer is unreadable. + + The service answered 202, so it has already created the job and started + its call; submitting again would create another job. + """ + + class SimulationServiceInvalidRequestError(ValueError): """The simulation service refuses the caller's data; the message says why. diff --git a/tests/unit/libs/test_simulation_household_calculations.py b/tests/unit/libs/test_simulation_household_calculations.py index 971943f23..815ad7f2a 100644 --- a/tests/unit/libs/test_simulation_household_calculations.py +++ b/tests/unit/libs/test_simulation_household_calculations.py @@ -14,6 +14,7 @@ SimulationAnswer, SimulationHouseholdCalculationClient, ) +from policyengine_api.request_context import _asgi_observability_id from policyengine_api.services.v2.household_calculations.types import ( HouseholdCalculationAccepted, SimulationServiceAnswerUnreadableError, @@ -54,6 +55,14 @@ def configured_simulation_service(monkeypatch): monkeypatch.setenv("OLD_SIMULATION_GATEWAY_URL", "https://old.example.test") +@pytest.fixture(autouse=True) +def restored_observability_id(): + """Undo the observability ID each submission binds in the test's context.""" + token = _asgi_observability_id.set(None) + yield + _asgi_observability_id.reset(token) + + class RecordingService: """The simulation service, answering every request with one scripted answer.""" From 9ca15aee624c92ff344b0ecf7c3aaf21d7c9217b Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:16:32 +0400 Subject: [PATCH 25/57] Test which answers the live household checks send again (failing) A submission should be sent again only after a 503 or a failed connection, never after a 502, which now means the simulation service created the job; a read may be sent again after a 502, 503 or 504 or a failed connection. The live fixture sends a submission again after any of 502, 503 and 504, and never after a failed connection. Co-Authored-By: Claude Opus 5.5 --- ...est_live_household_calculation_requests.py | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 tests/unit/v2/test_live_household_calculation_requests.py diff --git a/tests/unit/v2/test_live_household_calculation_requests.py b/tests/unit/v2/test_live_household_calculation_requests.py new file mode 100644 index 000000000..0441c4248 --- /dev/null +++ b/tests/unit/v2/test_live_household_calculation_requests.py @@ -0,0 +1,118 @@ +"""Which answers the live household calculation checks send again. + +A submission is sent again only after a 503 or a failed connection: a 502 +means the simulation service accepted it, so the job exists. A read is sent +again after a 502, 503 or 504 or a failed connection, since reading creates +nothing. +""" + +from __future__ import annotations + +import json +from uuid import uuid4 + +import httpx +import pytest + +from tests.fixtures.integration import live_household_calculations +from tests.fixtures.integration.live_household_calculations import ( + LiveHousehold, + calculate, +) + +HOUSEHOLD_ID = str(uuid4()) +CALCULATION_ID = str(uuid4()) +HOUSEHOLD = LiveHousehold(name="retries", country_id="us", people={"adult": {}}) + + +@pytest.fixture(autouse=True) +def no_waiting(monkeypatch): + monkeypatch.setattr( + live_household_calculations.time, "sleep", lambda _seconds: None + ) + + +class ScriptedPublicApi: + """The Public API's household calculation routes, answering from a script. + + Each script entry is an HTTP status, or an exception class to raise. + """ + + def __init__(self, submissions: list, reads: list) -> None: + self.submissions = list(submissions) + self.reads = list(reads) + self.sent: list[str] = [] + + def answer(self, request: httpx.Request) -> httpx.Response: + self.sent.append(request.method) + script = self.submissions if request.method == "POST" else self.reads + outcome = script.pop(0) + if isinstance(outcome, type): + msg = "connection refused" + raise outcome(msg, request=request) + if outcome == 202: + item = { + "calculation_id": CALCULATION_ID, + "status": "pending", + "household_id": HOUSEHOLD_ID, + } + return httpx.Response(202, json={"result": {"item": item}}) + if outcome == 200: + item = {"calculation_id": CALCULATION_ID, "status": "succeeded"} + return httpx.Response(200, json={"result": {"item": item}}) + return httpx.Response(outcome, content=json.dumps({"status": "error"})) + + def calculate(self): + with httpx.Client( + base_url="https://public-api.example", + transport=httpx.MockTransport(self.answer), + ) as client: + return calculate(client, HOUSEHOLD, HOUSEHOLD_ID) + + +def test_a_submission_is_sent_again_after_a_503() -> None: + public_api = ScriptedPublicApi(submissions=[503, 202], reads=[200]) + + finished = public_api.calculate() + + assert finished.item["status"] == "succeeded" + assert public_api.sent == ["POST", "POST", "GET"] + + +def test_a_submission_is_sent_again_after_a_failed_connection() -> None: + public_api = ScriptedPublicApi(submissions=[httpx.ConnectError, 202], reads=[200]) + + finished = public_api.calculate() + + assert finished.item["status"] == "succeeded" + assert public_api.sent == ["POST", "POST", "GET"] + + +@pytest.mark.parametrize("status_code", [502, 504, 500]) +def test_a_submission_is_never_sent_again_after_another_failure(status_code) -> None: + public_api = ScriptedPublicApi(submissions=[status_code, 202], reads=[200]) + + with pytest.raises(AssertionError): + public_api.calculate() + + assert public_api.sent == ["POST"] + + +@pytest.mark.parametrize("outcome", [502, 503, 504, httpx.ReadError]) +def test_a_read_is_sent_again_after_a_short_outage(outcome) -> None: + public_api = ScriptedPublicApi(submissions=[202], reads=[outcome, 200]) + + finished = public_api.calculate() + + assert finished.item["status"] == "succeeded" + assert public_api.sent == ["POST", "GET", "GET"] + + +def test_a_lasting_outage_fails_the_check(monkeypatch) -> None: + monkeypatch.setattr(live_household_calculations, "TRANSIENT_LIMIT_SECONDS", 0) + public_api = ScriptedPublicApi(submissions=[503, 202], reads=[200]) + + with pytest.raises(AssertionError, match="HTTP 503"): + public_api.calculate() + + assert public_api.sent == ["POST"] From c6b7cc0ab74a1a8c292108b114a064b42d82c325 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:17:24 +0400 Subject: [PATCH 26/57] Send a live household submission again only after a 503 or a failed connection The live fixture now sends a submission again only after a 503 or a failed connection, never after a 502, which means the simulation service created the job, and sends a read again after a 502, 503 or 504 or a failed connection. The live script and modules say the checks pass only once the simulation service's household calculation route is on staging, so the change that adds them merges only after that deployment. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/run_live_staging_tests.sh | 5 +- .../live_household_calculations.py | 64 +++++++++++++------ .../test_live_v2_household_calculations.py | 2 + .../test_live_v2_household_parity.py | 3 +- 4 files changed, 51 insertions(+), 23 deletions(-) diff --git a/.github/scripts/run_live_staging_tests.sh b/.github/scripts/run_live_staging_tests.sh index 1b101c637..0a2d04ebb 100755 --- a/.github/scripts/run_live_staging_tests.sh +++ b/.github/scripts/run_live_staging_tests.sh @@ -23,7 +23,10 @@ uv run --frozen --extra dev python -m pytest \ -v # The household calculation checks run afterwards, on 4 workers, so they never -# share the candidate with the tests above. Their requests are short (a +# share the candidate with the tests above. They call the simulation service's +# household calculation route, so they pass only once that route is deployed +# to staging; staging promotion waits on them, so the change that adds them +# merges only after that deployment. Their requests are short (a # submission, then a poll every few seconds) apart from the parity tests' # /calculate requests, but the parallel check submits 6 calculations at once: # with the 3 other workers' requests, at most 3 wait for a free slot, as above. diff --git a/tests/fixtures/integration/live_household_calculations.py b/tests/fixtures/integration/live_household_calculations.py index c26c54225..3ac6179be 100644 --- a/tests/fixtures/integration/live_household_calculations.py +++ b/tests/fixtures/integration/live_household_calculations.py @@ -2,6 +2,12 @@ They need only httpx and the standard library, because the parallel live test job installs nothing else. + +The checks call the simulation service's household calculation route through +the staging candidate, so they pass only once that route (``POST +/household-calculations``, answering ``job_id`` and accepting ``axes``) is +deployed to staging. Every staging promotion waits on them, so the change that +adds them merges only after that deployment. """ from __future__ import annotations @@ -13,11 +19,11 @@ from functools import partial from typing import TYPE_CHECKING, Any +import httpx + if TYPE_CHECKING: from collections.abc import Callable - import httpx - CALCULATION_PATH = "/v2/household-calculations" CALCULATION_TIMEOUT_SECONDS = float( os.environ.get("STAGING_HOUSEHOLD_CALCULATION_TIMEOUT_SECONDS", "600") @@ -25,13 +31,18 @@ POLL_INTERVAL_SECONDS = float( os.environ.get("STAGING_API_TEST_POLL_INTERVAL_SECONDS", "5") ) -# A 502, 503 or 504 can be a short outage, such as the simulation service -# failing to read its database or version manifest for a moment, or the -# Public API failing to read household_jobs; a -# calculation's submission and each poll are sent again while such answers -# last at most TRANSIENT_LIMIT_SECONDS in a row. Any other failure, and a -# longer run of such answers, fails the check with the answer's body. -TRANSIENT_STATUS_CODES = {502, 503, 504} +# A short outage, such as the simulation service failing to read its database +# or version manifest for a moment, or the Public API failing to read +# household_jobs, is sent through again while it lasts at most +# TRANSIENT_LIMIT_SECONDS in a row. A submission is sent again only after a +# 503 or a failed connection: the Public API answers 502 when the simulation +# service accepted the calculation but its answer could not be read, so the +# job exists, and sending it again would create another. A read is sent again +# after a 502, 503 or 504 or a failed connection, since reading creates +# nothing. Any other failure, and a longer outage, fails the check with the +# answer's body. +SUBMISSION_RETRY_STATUS_CODES = frozenset({503}) +READ_RETRY_STATUS_CODES = frozenset({502, 503, 504}) TRANSIENT_LIMIT_SECONDS = float( os.environ.get("STAGING_HOUSEHOLD_TRANSIENT_LIMIT_SECONDS", "90") ) @@ -179,19 +190,27 @@ def store_reform(client: httpx.Client, household: LiveHousehold) -> str | None: return response.json()["result"]["item"]["id"] -def _send_through_short_outages(send: Callable[[], httpx.Response]) -> httpx.Response: - """Send the request again while it gets 502, 503 or 504 for a short time.""" +def _send_through_short_outages( + send: Callable[[], httpx.Response], retry_status_codes: frozenset[int] +) -> httpx.Response: + """Send the request again while it fails to connect or gets one of the codes. + + Raise AssertionError once that has lasted TRANSIENT_LIMIT_SECONDS. + """ outage_started: float | None = None while True: - response = send() - if response.status_code not in TRANSIENT_STATUS_CODES: - return response - outage_started = outage_started or time.monotonic() + try: + response = send() + except httpx.TransportError as error: + failure = f"{type(error).__name__}: {error}" + else: + if response.status_code not in retry_status_codes: + return response + failure = f"HTTP {response.status_code}: {response.text[:500]}" + if outage_started is None: + outage_started = time.monotonic() if time.monotonic() - outage_started >= TRANSIENT_LIMIT_SECONDS: - msg = ( - f"HTTP {response.status_code} for {TRANSIENT_LIMIT_SECONDS:.0f} s: " - f"{response.text[:500]}" - ) + msg = f"{failure}, for {TRANSIENT_LIMIT_SECONDS:.0f} s" raise AssertionError(msg) time.sleep(POLL_INTERVAL_SECONDS) @@ -214,7 +233,8 @@ def calculate( body["axes"] = household.axes deadline = started + CALCULATION_TIMEOUT_SECONDS response = _send_through_short_outages( - lambda: client.post(CALCULATION_PATH, json=body) + lambda: client.post(CALCULATION_PATH, json=body), + SUBMISSION_RETRY_STATUS_CODES, ) submit_seconds = time.monotonic() - started assert response.status_code == 202, response.text @@ -224,7 +244,9 @@ def calculate( # The submission answers with the new job only; its result is read from # household_jobs, so the calculation is read at least once. while True: - polled = _send_through_short_outages(partial(client.get, calculation_path)) + polled = _send_through_short_outages( + partial(client.get, calculation_path), READ_RETRY_STATUS_CODES + ) assert polled.status_code == 200, polled.text item = polled.json()["result"]["item"] if item["status"] in FINISHED_STATUSES: diff --git a/tests/integration/test_live_v2_household_calculations.py b/tests/integration/test_live_v2_household_calculations.py index af918b27b..0a6dd12c1 100644 --- a/tests/integration/test_live_v2_household_calculations.py +++ b/tests/integration/test_live_v2_household_calculations.py @@ -15,6 +15,8 @@ failing; a household function that runs out of memory fails it. The measured times are reported as a warning, so the job log keeps them. +They need the simulation service's household calculation route on staging, +so the change that adds them merges only after that route is deployed there. """ from __future__ import annotations diff --git a/tests/integration/test_live_v2_household_parity.py b/tests/integration/test_live_v2_household_parity.py index e8758e0ac..c8085bff5 100644 --- a/tests/integration/test_live_v2_household_parity.py +++ b/tests/integration/test_live_v2_household_parity.py @@ -6,7 +6,8 @@ the Public API's in-process ``/calculate`` (``/calculate-full`` when it adds missing variables) on the same candidate. Every calculated value must match, axes, reforms and added variables included. A run-specific input makes each -stored household new. +stored household new. Like the other household calculation checks, these need +the simulation service's household calculation route on staging. """ from __future__ import annotations From 76b5e076efcdb47df01b89ab6666ff4145f54a8e Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:18:44 +0400 Subject: [PATCH 27/57] Test reporting SPM availability publication left unknown (failing) Publication's evidence should list the versions it recorded SPM availability for and those it left unknown, with the reason when the simulation service's versions couldn't be read, which the command also prints to stderr. /v2/model-catalog should serve a version without a recorded availability as available null, unknown, apart from a recorded absence, available false. Publication now reports nothing, and the catalog serves both as false. Co-Authored-By: Claude Opus 5.5 --- tests/unit/v2/test_catalog_initialization.py | 35 +++++++- tests/unit/v2/test_catalog_publication.py | 9 ++ .../unit/v2/test_catalog_spm_availability.py | 86 ++++++++++++------- tests/unit/v2/test_metadata_service.py | 19 +++- 4 files changed, 115 insertions(+), 34 deletions(-) diff --git a/tests/unit/v2/test_catalog_initialization.py b/tests/unit/v2/test_catalog_initialization.py index 79881718f..f8b8d684d 100644 --- a/tests/unit/v2/test_catalog_initialization.py +++ b/tests/unit/v2/test_catalog_initialization.py @@ -4,6 +4,7 @@ import ast import json +from dataclasses import replace from pathlib import Path import pytest @@ -80,9 +81,14 @@ def verify(selected_engine, catalog, environ): environ, ) + recording = publication_types.SPMAvailabilityRecording( + recorded=(("us", "1.450.0", True),) + ) + def record_spm(selected_engine, catalog, environ): assert "engine" in observed observed["spm"] = (selected_engine, catalog.policyengine_version, environ) + return recording result = initialization.initialize_catalog( ENVIRONMENT, @@ -95,7 +101,7 @@ def record_spm(selected_engine, catalog, environ): ), ) - assert result == _evidence() + assert result == replace(_evidence(), spm_availability=recording) assert observed == { "username": "data-writer", "verified": (engine, POLICYENGINE_VERSION, ENVIRONMENT), @@ -408,3 +414,30 @@ def refuse_extraction(): with pytest.raises(V2ConfigurationError, match="SIMULATION_ENTRYPOINT_URL"): initialization.initialize_catalog(environment, extractor=refuse_extraction) + + +def test_main_says_on_stderr_when_spm_availability_could_not_be_recorded( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """The evidence names the versions left unknown, and stderr says why.""" + recording = publication_types.SPMAvailabilityRecording( + left_unknown=(("us", "1.450.0"),), + versions_unreadable="the simulation service's versions could not be read", + ) + evidence = replace(_evidence(), spm_availability=recording) + monkeypatch.setattr(initialization, "initialize_catalog", lambda: evidence) + + assert initialization.main() == 0 + + output = capsys.readouterr() + assert json.loads(output.out)["spm_availability"] == { + "recorded": [], + "left_unknown": [{"country_id": "us", "version": "1.450.0"}], + "versions_unreadable": ("the simulation service's versions could not be read"), + } + assert output.err.splitlines() == [ + "SPM availability was not recorded: the simulation service's versions " + "could not be read; /v2/model-catalog reports it as unknown until a " + "publication records it" + ] diff --git a/tests/unit/v2/test_catalog_publication.py b/tests/unit/v2/test_catalog_publication.py index a646bcd03..9f4a22c45 100644 --- a/tests/unit/v2/test_catalog_publication.py +++ b/tests/unit/v2/test_catalog_publication.py @@ -321,6 +321,10 @@ def test_completion_evidence_contains_only_reviewed_non_secret_fields() -> None: ), ), catalog_versions_left_to_prune=3, + spm_availability=publication_types.SPMAvailabilityRecording( + recorded=(("uk", "2.50.0", False),), + left_unknown=(("us", "1.450.0"),), + ), ).as_dict() assert evidence == { @@ -395,6 +399,11 @@ def test_completion_evidence_contains_only_reviewed_non_secret_fields() -> None: } ], "catalog_versions_left_to_prune": 3, + "spm_availability": { + "recorded": [{"country_id": "uk", "version": "2.50.0", "available": False}], + "left_unknown": [{"country_id": "us", "version": "1.450.0"}], + "versions_unreadable": None, + }, "elapsed_seconds": 2.346, } assert ( diff --git a/tests/unit/v2/test_catalog_spm_availability.py b/tests/unit/v2/test_catalog_spm_availability.py index c8cd1ac1b..6d3e79687 100644 --- a/tests/unit/v2/test_catalog_spm_availability.py +++ b/tests/unit/v2/test_catalog_spm_availability.py @@ -10,7 +10,10 @@ import pytest from policyengine_api.data.v2.catalog import spm_availability -from policyengine_api.data.v2.catalog.publication_types import CatalogPublicationError +from policyengine_api.data.v2.catalog.publication_types import ( + CatalogPublicationError, + SPMAvailabilityRecording, +) from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.spm import SPMSelection @@ -116,6 +119,21 @@ def test_a_us_version_the_service_does_not_serve_is_not_known_yet( ) +def _catalog(*country_ids: str) -> SimpleNamespace: + """Return a published catalog of version 1.0.0 of each country's model.""" + return SimpleNamespace( + policyengine_version=POLICYENGINE_VERSION, + countries=[ + SimpleNamespace( + country_id=country_id, + model=SimpleNamespace(name=f"policyengine-{country_id}"), + model_version=SimpleNamespace(version="1.0.0"), + ) + for country_id in country_ids + ], + ) + + class RecordingEngine: """An engine whose transactions are only counted.""" @@ -134,34 +152,44 @@ def test_publication_records_only_what_is_known( monkeypatch: pytest.MonkeyPatch, ) -> None: recorded: list[tuple[str, dict]] = [] + + def record(_connection, *, model_name, version, availability) -> bool: + recorded.append((model_name, dict(availability))) + return True + + monkeypatch.setattr(spm_availability, "record_spm_availability", record) + not_served = {"policyengine": {"latest": "6.11.0", "6.11.0": "app"}} + + recording = spm_availability.record_published_spm_availability( + RecordingEngine(), + _catalog("uk", "us"), + {"SIMULATION_ENTRYPOINT_URL": "https://simulation.example"}, + fetch_versions=lambda _url: not_served, + ) + + assert recorded == [("policyengine-uk", {"available": False})] + assert recording == SPMAvailabilityRecording( + recorded=(("uk", "1.0.0", False),), left_unknown=(("us", "1.0.0"),) + ) + + +def test_a_version_recorded_by_an_earlier_publication_is_not_listed( + monkeypatch: pytest.MonkeyPatch, +) -> None: monkeypatch.setattr( spm_availability, "record_spm_availability", - lambda _connection, *, model_name, version, availability: recorded.append( - (model_name, dict(availability)) - ), - ) - catalog = SimpleNamespace( - policyengine_version=POLICYENGINE_VERSION, - countries=[ - SimpleNamespace( - country_id=country_id, - model=SimpleNamespace(name=f"policyengine-{country_id}"), - model_version=SimpleNamespace(version="1.0.0"), - ) - for country_id in ("uk", "us") - ], + lambda _connection, *, model_name, version, availability: False, ) - not_served = {"policyengine": {"latest": "6.11.0", "6.11.0": "app"}} - spm_availability.record_published_spm_availability( + recording = spm_availability.record_published_spm_availability( RecordingEngine(), - catalog, + _catalog("us"), {"SIMULATION_ENTRYPOINT_URL": "https://simulation.example"}, - fetch_versions=lambda _url: not_served, + fetch_versions=lambda _url: VERSIONS, ) - assert recorded == [("policyengine-uk", {"available": False})] + assert recording == SPMAvailabilityRecording() def test_the_versions_come_from_the_simulation_services_public_route() -> None: @@ -218,26 +246,20 @@ def test_an_unreadable_versions_answer_leaves_every_version_unrecorded( model_name ), ) - catalog = SimpleNamespace( - policyengine_version=POLICYENGINE_VERSION, - countries=[ - SimpleNamespace( - country_id="us", - model=SimpleNamespace(name="policyengine-us"), - model_version=SimpleNamespace(version="1.0.0"), - ) - ], - ) def unreadable(_url: str) -> object: msg = "the simulation service's versions could not be read" raise CatalogPublicationError(msg) - spm_availability.record_published_spm_availability( + recording = spm_availability.record_published_spm_availability( RecordingEngine(), - catalog, + _catalog("uk", "us"), {"SIMULATION_ENTRYPOINT_URL": "https://simulation.example"}, fetch_versions=unreadable, ) assert recorded == [] + assert recording == SPMAvailabilityRecording( + left_unknown=(("uk", "1.0.0"), ("us", "1.0.0")), + versions_unreadable="the simulation service's versions could not be read", + ) diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index d351a8cf6..44ae5b956 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -460,11 +460,28 @@ def test_model_catalog_serves_the_versions_recorded_spm_availability( assert result.spm.model_dump(mode="json") == SPM_AVAILABLE -def test_model_catalog_without_recorded_spm_availability_offers_none( +def test_model_catalog_without_recorded_spm_availability_reports_it_unknown( catalog_session: Session, ) -> None: result = catalog_service(catalog_session).get_model_catalog("uk") + assert result.spm.model_dump(mode="json") == { + "available": None, + "settings_schema": None, + "defaults": None, + } + + +def test_model_catalog_serves_a_recorded_absence_of_spm( + catalog_session: Session, +) -> None: + model_version = stored_model_version(catalog_session, "policyengine-uk") + model_version.spm_availability = {"available": False} + catalog_session.add(model_version) + catalog_session.commit() + + result = catalog_service(catalog_session).get_model_catalog("uk") + assert result.spm.model_dump(mode="json") == { "available": False, "settings_schema": None, From 3ecd5f830a18c014a8d805bc494069105b8c4df9 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:20:38 +0400 Subject: [PATCH 28/57] Report SPM availability publication left unknown, and serve it as unknown record_published_spm_availability returns what it recorded and which versions it didn't, with the reason when the simulation service's versions couldn't be read. Initialization adds that to the publication evidence, which the command prints, and the command also prints the reason to stderr. /v2/model-catalog serves a version without a recorded availability as available null, unknown, apart from a recorded absence, available false. The Stage 9 runbook's publication step names SIMULATION_ENTRYPOINT_URL, which publication now needs, and says how a failed versions read is reported. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/skills/alembic-migrations.md | 3 +- docs/migration/stage-9-v2-metadata.md | 31 +++++++----- .../data/v2/catalog/initialization.py | 33 +++++++++---- .../data/v2/catalog/publication_types.py | 41 ++++++++++++++++ .../data/v2/catalog/spm_availability.py | 49 +++++++++++++------ .../v2/metadata/response_cache_support.py | 3 +- .../v2/metadata/route_support.py | 4 +- .../services/v2/metadata/process_state.py | 2 +- .../services/v2/metadata/transformations.py | 4 +- .../services/v2/metadata/types.py | 11 +++-- tests/unit/v2/test_catalog_initialization.py | 10 ++-- tests/unit/v2/test_catalog_publication.py | 4 +- .../unit/v2/test_catalog_spm_availability.py | 4 +- 13 files changed, 144 insertions(+), 55 deletions(-) diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index a814daaa0..f5abccd2c 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -817,7 +817,8 @@ version offers SPM settings, from the simulation service's public schema and the certified defaults. A US version the simulation service does not serve yet, and every version when its answer cannot be read, stays unrecorded for the next publication run. `/v2/model-catalog` serves it as -`spm`, and a version without it offers none. The catalog writer role already +`spm`, and a version without it reports `available` null, unknown, which a +recorded absence (`available` false) is not. The catalog writer role already holds `UPDATE` on the table, so no privilege changes. The revision writes no data, and the previous Public API revision, which diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 2ebd46ef7..48a07b84f 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -41,9 +41,11 @@ version offers SPM settings for a household: `available`, and when it does, `settings_schema` (the JSON schema of a household's `spm` selection) and `defaults` (the certified selection for every field a household leaves out). Catalog publication records it for each US version from the simulation -service's public `GET /versions`, after publishing the version; any other -country, a bundle without a certified capability, and a version not recorded -yet offer none. The result names the +service's public `GET /versions`, after publishing the version. Any other +country, and a bundle without a certified capability, has `available` false; +a version whose availability no publication has recorded yet (the simulation +service didn't serve its PolicyEngine.py version, or its versions couldn't be +read) has `available` null, unknown. The result names the selected PolicyEngine.py version, so a client that starts without a version learns the running one and requests every other part for that exact version. The query uses the shared `CatalogQuery` model: unknown and repeated @@ -169,7 +171,7 @@ so a request without `policyengine_version` (the process's running version, which never changes while it runs) and one naming that version share it. A version's SPM availability is the one part recorded after publication: a process that built a version's `/v2/model-catalog` body before publication -recorded it keeps offering no SPM settings until it restarts, normally at the +recorded it keeps reporting its availability as unknown until it restarts, normally at the deployment that follows the publication run. Only the first successful request per process reads the database. Requests that arrive while that read is in progress wait for it and all receive its outcome, @@ -216,14 +218,21 @@ must execute in this order: Alembic chain, confirms all heads are current, and detects ungenerated model changes. 3. Remove the migration URL from the command environment. -4. Supply only `V2_DATA_WRITE_DATABASE_URL`, the same target identity and +4. Supply only `V2_DATA_WRITE_DATABASE_URL`, the same target identity, `HUGGING_FACE_TOKEN` (from the organization secret - `PE_UK_PRIVATE_HF_READ_TOKEN`), then run - `uv run python scripts/publish_v2_metadata_catalog.py`. Before publishing a - catalog version that is not stored yet, the command reads the certified data - artifacts its regions' filters use and stops if one cannot select a region. - It stops before reading anything if a UK version needs checking and - `HUGGING_FACE_TOKEN` is unset, because the UK data repository is private. + `PE_UK_PRIVATE_HF_READ_TOKEN`) and `SIMULATION_ENTRYPOINT_URL` (the + environment's secret, the same simulation service the candidate calls), + then run `uv run python scripts/publish_v2_metadata_catalog.py`. It stops + before reading anything if `SIMULATION_ENTRYPOINT_URL` is unset. Before + publishing a catalog version that is not stored yet, the command reads the + certified data artifacts its regions' filters use and stops if one cannot + select a region. It stops before reading anything if a UK version needs + checking and `HUGGING_FACE_TOKEN` is unset, because the UK data repository + is private. After publishing, it records each version's SPM availability + from the simulation service's public `GET /versions`; when that can't be + read, the command still succeeds, its evidence's `spm_availability` lists + the versions not recorded with the reason, and it prints the reason to + stderr. 5. Require the command to finish successfully and retain its non-secret JSON evidence before creating the candidate revision. 6. Deploy the Cloud Run candidate with the target identity and diff --git a/policyengine_api/data/v2/catalog/initialization.py b/policyengine_api/data/v2/catalog/initialization.py index 9b3702d6e..3585b0f40 100644 --- a/policyengine_api/data/v2/catalog/initialization.py +++ b/policyengine_api/data/v2/catalog/initialization.py @@ -5,7 +5,7 @@ import json import os import sys -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import TYPE_CHECKING from sqlalchemy import Engine, create_engine @@ -41,6 +41,9 @@ if TYPE_CHECKING: from collections.abc import Callable, Mapping, Sequence + from policyengine_api.data.v2.catalog.publication_types import ( + SPMAvailabilityRecording, + ) from policyengine_api.data.v2.catalog.records import NormalizedCatalog @@ -118,7 +121,8 @@ class PublicationSteps: publish_catalog ) record_spm_availability: Callable[ - [Engine, NormalizedCatalog, Mapping[str, str] | None], None + [Engine, NormalizedCatalog, Mapping[str, str] | None], + SPMAvailabilityRecording, ] = record_published_spm_availability @@ -131,8 +135,9 @@ def initialize_catalog( ) -> PublicationEvidence: """Load only row-write settings, extract fully, verify, then publish explicitly. - After publishing, record each version's SPM availability. The simulation - service's URL, which that needs, is checked before anything else. + After publishing, record each version's SPM availability, which the + evidence returned reports. The simulation service's URL, which that + needs, is checked before anything else. """ selected_steps = steps or PublicationSteps() settings = load_v2_data_write_database_settings(environ) @@ -142,8 +147,8 @@ def initialize_catalog( try: selected_steps.verify_region_data(engine, catalog, environ) evidence = selected_steps.publish(engine, catalog) - selected_steps.record_spm_availability(engine, catalog, environ) - return evidence + recording = selected_steps.record_spm_availability(engine, catalog, environ) + return replace(evidence, spm_availability=recording) finally: engine.dispose() @@ -172,9 +177,10 @@ def _error_payload(error: Exception) -> dict[str, object]: def main() -> int: """Run explicit initialization and return a shell-compatible status. - A failure to prune an older catalog version leaves the status successful, - since the publication had committed; each one is in the evidence and is - also printed to stderr. + A failure to prune an older catalog version, or to read the simulation + service's versions to record SPM availability, leaves the status + successful, since the publication had committed; each one is in the + evidence and is also printed to stderr. """ try: evidence = initialize_catalog() @@ -193,6 +199,15 @@ def main() -> int: f"{failure.error_type}: {failure.message}", file=sys.stderr, ) + spm_recording = evidence.spm_availability + if spm_recording is not None and spm_recording.versions_unreadable is not None: + print( + "SPM availability was not recorded: " + f"{spm_recording.versions_unreadable}; /v2/model-catalog reports a " + "version without a recorded availability as unknown until a " + "publication records it", + file=sys.stderr, + ) return 0 diff --git a/policyengine_api/data/v2/catalog/publication_types.py b/policyengine_api/data/v2/catalog/publication_types.py index 636b09cbb..62e92be06 100644 --- a/policyengine_api/data/v2/catalog/publication_types.py +++ b/policyengine_api/data/v2/catalog/publication_types.py @@ -41,6 +41,39 @@ class CatalogPruningFailure: message: str +@dataclass(frozen=True, slots=True) +class SPMAvailabilityRecording: + """What one publication recorded of its catalog versions' SPM availability. + + ``recorded`` lists each version it recorded: country, model version and + whether SPM settings are available. ``not_recorded`` lists each version + whose availability it doesn't know: one the simulation service doesn't + serve yet, or every version when ``versions_unreadable`` says why the + service's versions couldn't be read. A version is unknown to + ``/v2/model-catalog`` until some publication records it, and a version + an earlier publication recorded keeps its value and is listed in + neither. + """ + + recorded: tuple[tuple[str, str, bool], ...] = () + not_recorded: tuple[tuple[str, str], ...] = () + versions_unreadable: str | None = None + + def as_dict(self) -> dict[str, object]: + """Return the recording as it appears in the publication evidence.""" + return { + "recorded": [ + {"country_id": country_id, "version": version, "available": available} + for country_id, version, available in self.recorded + ], + "not_recorded": [ + {"country_id": country_id, "version": version} + for country_id, version in self.not_recorded + ], + "versions_unreadable": self.versions_unreadable, + } + + @dataclass(frozen=True, slots=True) class PublicationEvidence: """Non-secret facts emitted after a successful publication.""" @@ -78,6 +111,9 @@ class PublicationEvidence: catalog_pruning_failures: tuple[CatalogPruningFailure, ...] = () # Chosen versions beyond the run's limit, which the next publications prune. catalog_versions_left_to_prune: int = 0 + # The SPM availability recorded after the publication committed; None + # until initialization records it. + spm_availability: SPMAvailabilityRecording | None = None def as_dict(self) -> dict[str, object]: """Return the evidence as the JSON object printed after a publication.""" @@ -167,5 +203,10 @@ def as_dict(self) -> dict[str, object]: for failure in self.catalog_pruning_failures ], "catalog_versions_left_to_prune": self.catalog_versions_left_to_prune, + "spm_availability": ( + None + if self.spm_availability is None + else self.spm_availability.as_dict() + ), "elapsed_seconds": round(self.elapsed_seconds, 3), } diff --git a/policyengine_api/data/v2/catalog/spm_availability.py b/policyengine_api/data/v2/catalog/spm_availability.py index 2a2149dda..a73fe8729 100644 --- a/policyengine_api/data/v2/catalog/spm_availability.py +++ b/policyengine_api/data/v2/catalog/spm_availability.py @@ -14,9 +14,11 @@ A bundle pins its SPM release, so a recorded value never changes. A US version the service does not serve yet (its ``policyengine`` routes do not name it) is left unrecorded rather than recorded as unavailable, and the next -publication run tries again; ``/v2/model-catalog`` offers no SPM settings for -it meanwhile. So is every version when the service's versions cannot be -read: an unreadable answer never fails the publication. +publication run tries again; ``/v2/model-catalog`` reports its availability +as unknown meanwhile. So is every version when the service's versions cannot +be read: an unreadable answer never fails the publication, but the +publication evidence lists the versions not recorded, with the reason, and +the command prints it to stderr. """ from __future__ import annotations @@ -35,6 +37,7 @@ ) from policyengine_api.data.v2.catalog.publication_types import ( CatalogPublicationError, + SPMAvailabilityRecording, ) from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.spm import SPMSelection @@ -162,13 +165,13 @@ def record_published_spm_availability( environ: Mapping[str, str] | None = None, *, fetch_versions: Callable[[str], object] = fetch_simulation_versions, -) -> None: +) -> SPMAvailabilityRecording: """Record each published country's SPM availability for the catalog version. - A US version the service does not serve yet stays unrecorded, and so does - every version when the service's versions cannot be read; the next - publication run tries again. Raise V2ConfigurationError without - ``SIMULATION_ENTRYPOINT_URL``. + Return what was recorded and what was not. A US version the service does + not serve yet stays unrecorded, and so does every version when the + service's versions cannot be read; the next publication run tries again. + Raise V2ConfigurationError without ``SIMULATION_ENTRYPOINT_URL``. """ url = simulation_entrypoint_url(os.environ if environ is None else environ) try: @@ -179,16 +182,32 @@ def record_published_spm_availability( "tries again: %s", error, ) - return + return SPMAvailabilityRecording( + not_recorded=tuple( + (country.country_id, country.model_version.version) + for country in catalog.countries + ), + versions_unreadable=str(error), + ) + recorded: list[tuple[str, str, bool]] = [] + not_recorded: list[tuple[str, str]] = [] with engine.begin() as connection: for country in catalog.countries: + version = country.model_version.version availability = spm_availability( country.country_id, catalog.policyengine_version, versions ) - if availability is not None: - record_spm_availability( - connection, - model_name=country.model.name, - version=country.model_version.version, - availability=availability, + if availability is None: + not_recorded.append((country.country_id, version)) + elif record_spm_availability( + connection, + model_name=country.model.name, + version=version, + availability=availability, + ): + recorded.append( + (country.country_id, version, bool(availability["available"])) ) + return SPMAvailabilityRecording( + recorded=tuple(recorded), not_recorded=tuple(not_recorded) + ) diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_cache_support.py b/policyengine_api/fastapi_routes/v2/metadata/response_cache_support.py index fc21da9be..f52161a0f 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/response_cache_support.py +++ b/policyengine_api/fastapi_routes/v2/metadata/response_cache_support.py @@ -87,7 +87,8 @@ class MetadataResponseCache: that release's processes start, when pruning deletes its current-law values, which restoration writes back identically, or when publication records its SPM availability once after publishing it, so a body built - before then offers no SPM settings for the life of the process; + before then reports its SPM availability as unknown for the life of the + process; publication also sets the version's ``catalog_last_published_at`` and pruning its ``catalog_values_pruned_at``, which no body holds. So a body built for a diff --git a/policyengine_api/fastapi_routes/v2/metadata/route_support.py b/policyengine_api/fastapi_routes/v2/metadata/route_support.py index 72ad595a0..e8a5d005f 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/route_support.py +++ b/policyengine_api/fastapi_routes/v2/metadata/route_support.py @@ -233,8 +233,8 @@ def policyengine_version(self) -> str: identically, and the 404 answered for a pruned version (503 for the running one) is never stored (``Cache-Control: no-store``). A version's SPM availability in ``/v2/model-catalog`` is recorded once after publication: -until a publication run records it, a response says the version offers no -SPM settings, a process keeps saying so until it restarts, and a client may +until a publication run records it, a response reports it as unknown, a +process keeps reporting so until it restarts, and a client may reuse such a response for up to this long after that. """ RUNNING_VERSION_MAX_AGE_SECONDS = 300 diff --git a/policyengine_api/services/v2/metadata/process_state.py b/policyengine_api/services/v2/metadata/process_state.py index f52b9be44..afbcd63fc 100644 --- a/policyengine_api/services/v2/metadata/process_state.py +++ b/policyengine_api/services/v2/metadata/process_state.py @@ -30,7 +30,7 @@ class SelectedCatalogStore: for the same country and version need not query for it again; a process of the previous release keeps the time-period options it read, and a process that found a version before its SPM availability was recorded - offers no SPM settings for it until it restarts. + reports its SPM availability as unknown until it restarts. A kept ``catalog_values_pruned_at`` can be out of date in either direction, so ``MetadataCatalogSelection`` checks it for the reads of the diff --git a/policyengine_api/services/v2/metadata/transformations.py b/policyengine_api/services/v2/metadata/transformations.py index f69e484d6..cc6f0e443 100644 --- a/policyengine_api/services/v2/metadata/transformations.py +++ b/policyengine_api/services/v2/metadata/transformations.py @@ -429,7 +429,7 @@ def metadata_model_catalog( Without a national dataset (a catalog without regions) the economy choices are empty. Entities and basic inputs a version published before catalog format 2 lacks are empty lists, and a version whose SPM availability is - not recorded yet offers none. + not recorded yet reports it as unknown. """ model_version = selected.model_version return MetadataModelCatalogResult( @@ -452,6 +452,6 @@ def metadata_model_catalog( else MetadataEconomyOptions(region=[], time_period=[], datasets=[]) ), spm=MetadataSPMAvailability.model_validate( - model_version.spm_availability or {"available": False} + model_version.spm_availability or {"available": None} ), ) diff --git a/policyengine_api/services/v2/metadata/types.py b/policyengine_api/services/v2/metadata/types.py index 6be355ec6..1d75635d1 100644 --- a/policyengine_api/services/v2/metadata/types.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -362,12 +362,15 @@ class MetadataSPMAvailability(StrictResponseModel): When ``available``, ``settings_schema`` is the JSON schema of a household's ``spm`` selection and ``defaults`` the certified selection that applies to every field a household leaves out. A version without - them (any country but the US, a bundle without certified SPM support, or - a version whose availability publication has not recorded yet) is not - available, and both are null. + them (any country but the US, or a bundle without certified SPM + support) is not available, and both are null. ``available`` is null + while catalog publication has not recorded the version's availability, + because the simulation service didn't serve its PolicyEngine.py version + or its versions couldn't be read: whether it offers SPM settings is not + known yet. """ - available: bool + available: bool | None settings_schema: dict[str, JsonValue] | None = None defaults: dict[str, JsonValue] | None = None diff --git a/tests/unit/v2/test_catalog_initialization.py b/tests/unit/v2/test_catalog_initialization.py index f8b8d684d..d5c4bb1b2 100644 --- a/tests/unit/v2/test_catalog_initialization.py +++ b/tests/unit/v2/test_catalog_initialization.py @@ -420,9 +420,9 @@ def test_main_says_on_stderr_when_spm_availability_could_not_be_recorded( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: - """The evidence names the versions left unknown, and stderr says why.""" + """The evidence names the versions not recorded, and stderr says why.""" recording = publication_types.SPMAvailabilityRecording( - left_unknown=(("us", "1.450.0"),), + not_recorded=(("us", "1.450.0"),), versions_unreadable="the simulation service's versions could not be read", ) evidence = replace(_evidence(), spm_availability=recording) @@ -433,11 +433,11 @@ def test_main_says_on_stderr_when_spm_availability_could_not_be_recorded( output = capsys.readouterr() assert json.loads(output.out)["spm_availability"] == { "recorded": [], - "left_unknown": [{"country_id": "us", "version": "1.450.0"}], + "not_recorded": [{"country_id": "us", "version": "1.450.0"}], "versions_unreadable": ("the simulation service's versions could not be read"), } assert output.err.splitlines() == [ "SPM availability was not recorded: the simulation service's versions " - "could not be read; /v2/model-catalog reports it as unknown until a " - "publication records it" + "could not be read; /v2/model-catalog reports a version without a " + "recorded availability as unknown until a publication records it" ] diff --git a/tests/unit/v2/test_catalog_publication.py b/tests/unit/v2/test_catalog_publication.py index 9f4a22c45..338a6d97c 100644 --- a/tests/unit/v2/test_catalog_publication.py +++ b/tests/unit/v2/test_catalog_publication.py @@ -323,7 +323,7 @@ def test_completion_evidence_contains_only_reviewed_non_secret_fields() -> None: catalog_versions_left_to_prune=3, spm_availability=publication_types.SPMAvailabilityRecording( recorded=(("uk", "2.50.0", False),), - left_unknown=(("us", "1.450.0"),), + not_recorded=(("us", "1.450.0"),), ), ).as_dict() @@ -401,7 +401,7 @@ def test_completion_evidence_contains_only_reviewed_non_secret_fields() -> None: "catalog_versions_left_to_prune": 3, "spm_availability": { "recorded": [{"country_id": "uk", "version": "2.50.0", "available": False}], - "left_unknown": [{"country_id": "us", "version": "1.450.0"}], + "not_recorded": [{"country_id": "us", "version": "1.450.0"}], "versions_unreadable": None, }, "elapsed_seconds": 2.346, diff --git a/tests/unit/v2/test_catalog_spm_availability.py b/tests/unit/v2/test_catalog_spm_availability.py index 6d3e79687..447df6e92 100644 --- a/tests/unit/v2/test_catalog_spm_availability.py +++ b/tests/unit/v2/test_catalog_spm_availability.py @@ -169,7 +169,7 @@ def record(_connection, *, model_name, version, availability) -> bool: assert recorded == [("policyengine-uk", {"available": False})] assert recording == SPMAvailabilityRecording( - recorded=(("uk", "1.0.0", False),), left_unknown=(("us", "1.0.0"),) + recorded=(("uk", "1.0.0", False),), not_recorded=(("us", "1.0.0"),) ) @@ -260,6 +260,6 @@ def unreadable(_url: str) -> object: assert recorded == [] assert recording == SPMAvailabilityRecording( - left_unknown=(("uk", "1.0.0"), ("us", "1.0.0")), + not_recorded=(("uk", "1.0.0"), ("us", "1.0.0")), versions_unreadable="the simulation service's versions could not be read", ) From 37f0c9115db08fcadf220a0b15ee2f330c917389 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:21:05 +0400 Subject: [PATCH 29/57] Test that only version 1 result documents are read (failing) A succeeded job's document whose schema_version is missing or isn't 1 should be unreadable, and a failure document of another version should be read as none, so the job's error message says why it failed. The document models ignore schema_version, so both are read as version 1. Co-Authored-By: Claude Opus 5.5 --- .../v2/test_household_calculation_results.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/unit/v2/test_household_calculation_results.py b/tests/unit/v2/test_household_calculation_results.py index 7feff0da0..e61d4a0f0 100644 --- a/tests/unit/v2/test_household_calculation_results.py +++ b/tests/unit/v2/test_household_calculation_results.py @@ -200,3 +200,31 @@ def test_a_document_with_fields_the_public_api_does_not_read_is_still_read() -> def test_a_succeeded_job_without_one_readable_result_is_unreadable(document) -> None: with pytest.raises(HouseholdCalculationResultUnreadableError, match=str(JOB_ID)): _read(_job(HouseholdJobStatus.SUCCEEDED, result=document)) + + +@pytest.mark.parametrize( + "schema_version", [2, None, "1"], ids=["later", "missing", "a-string"] +) +def test_a_succeeded_document_of_another_contract_version_is_unreadable( + schema_version, +) -> None: + document = succeeded_document() + if schema_version is None: + del document["schema_version"] + else: + document["schema_version"] = schema_version + + with pytest.raises(HouseholdCalculationResultUnreadableError): + _read(_job(HouseholdJobStatus.SUCCEEDED, result=document)) + + +def test_a_failure_document_of_another_contract_version_is_not_read() -> None: + document = {**failed_document(), "schema_version": 2} + + calculation = _read( + _job(HouseholdJobStatus.FAILED, result=document, error_message=FAILURE_SUMMARY) + ) + + assert calculation.error is not None + assert calculation.error.code is None + assert calculation.error.message == FAILURE_SUMMARY From cfd8be911ae4a5863c7a5d603b3262abc8a30729 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:21:40 +0400 Subject: [PATCH 30/57] Read household result documents only at contract version 1 The stored result and failure document models require schema_version 1. A succeeded job's document of another version, or of none, is unreadable (500), and a failure document of another version is read as none, so the job's error message says why it failed, rather than either being read as version 1. Co-Authored-By: Claude Opus 5.5 --- .../services/v2/household_calculations/types.py | 8 ++++++-- .../services/v2/household_calculations/validators.py | 10 +++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/policyengine_api/services/v2/household_calculations/types.py b/policyengine_api/services/v2/household_calculations/types.py index f955abf4f..48f6e6957 100644 --- a/policyengine_api/services/v2/household_calculations/types.py +++ b/policyengine_api/services/v2/household_calculations/types.py @@ -210,8 +210,10 @@ class HouseholdCalculationRead: class StoredDocumentModel(BaseModel): """Base for the result documents the household functions store. - Fields the Public API does not read are ignored, so a later contract may - add some. + The documents are read only at the household contract's version 1 + (``schema_version``); one of another version, or of none, is refused + rather than read as version 1. Fields the Public API does not read are + ignored, so the contract may add some within its version. """ model_config = ConfigDict(extra="ignore", frozen=True) @@ -235,6 +237,7 @@ class StoredExecutionResult(StoredDocumentModel): """A succeeded job's result document (``HouseholdExecutionResult``).""" status: Literal["succeeded"] + schema_version: Literal[1] output: StoredCalculationOutput def calculation_output(self) -> HouseholdCalculationOutput: @@ -260,6 +263,7 @@ class StoredExecutionFailure(StoredDocumentModel): """A refused job's result document (``HouseholdExecutionFailure``).""" status: Literal["failed"] + schema_version: Literal[1] error: StoredCalculationError def calculation_failure(self) -> HouseholdCalculationFailure: diff --git a/policyengine_api/services/v2/household_calculations/validators.py b/policyengine_api/services/v2/household_calculations/validators.py index 5fe353931..3c8b164d2 100644 --- a/policyengine_api/services/v2/household_calculations/validators.py +++ b/policyengine_api/services/v2/household_calculations/validators.py @@ -71,11 +71,11 @@ def stored_household_outcome( """Return a job's stored result document, read as its status says. An unfinished job's result is not read. A succeeded job must hold a - ``HouseholdExecutionResult`` with its policy's calculated household, or - HouseholdCalculationResultUnreadableError is raised. A failed job holds a - ``HouseholdExecutionFailure``, or None when it failed without one; any - other document is read as none, since the job's error message still - says why it failed. + version 1 ``HouseholdExecutionResult`` with its policy's calculated + household, or HouseholdCalculationResultUnreadableError is raised. A + failed job holds a version 1 ``HouseholdExecutionFailure``, or None when + it failed without one; any other document is read as none, since the + job's error message still says why it failed. """ if status == "succeeded": try: From c74667c28ea1424569484c8caeb1b7990cef880f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:22:12 +0400 Subject: [PATCH 31/57] Name the generated client's household modules and document the new answers post_household_calculation's docstring names the generated client's modules for the household route, as the simulation service's OpenAPI document gives them. The Stage 11 notes describe the 502 for an unreadable 202, sim's codes in the 503 messages, the correlation headers and the 503 for a failed catalog query. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-11-v2-households.md | 14 +++++++++----- .../libs/simulation_household_calculations.py | 6 ++++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/migration/stage-11-v2-households.md b/docs/migration/stage-11-v2-households.md index 88bce25f8..0030648f1 100644 --- a/docs/migration/stage-11-v2-households.md +++ b/docs/migration/stage-11-v2-households.md @@ -318,11 +318,15 @@ optional `spm` settings. The year calculated is the household's PolicyEngine.py version can check passes. It then sends the simulation service's `POST /household-calculations` the -IDs and options. The answer is 202 with the new job's `calculation_id` and -`status`; each submission creates a new job. The simulation service's 422 -reasons reach the caller as 422; any other refusal, no answer, an answer the -Public API can't read, missing configuration or a gateway token it can't -fetch is 503. +IDs and options, with the request's ID and observability ID headers. The +answer is 202 with the new job's `calculation_id` and `status`; each +submission creates a new job. The simulation service's 422 reasons reach the +caller as 422. A 202 whose body the Public API can't read is 502: the job +exists, so submitting again would create another. Any other refusal (with +the simulation service's `code` and reason in the message, and the failed +job's ID when Modal refused the call), no answer, a refusal the Public API +can't read, missing configuration or a gateway token it can't fetch is 503. +A failed catalog query in the policy check is 503 too. `GET /v2/household-calculations/{calculation_id}` reads the job from `household_jobs`, which the simulation service inserts and its household diff --git a/policyengine_api/libs/simulation_household_calculations.py b/policyengine_api/libs/simulation_household_calculations.py index 89b4036de..3abe3b8d8 100644 --- a/policyengine_api/libs/simulation_household_calculations.py +++ b/policyengine_api/libs/simulation_household_calculations.py @@ -151,8 +151,10 @@ def post_household_calculation( Raise httpx.HTTPError when no answer arrives, and GatewayAuthError when the gateway token cannot be fetched. ``http_client`` has the service's base URL. To send through the generated client instead, this body - becomes (the operation and model names are those the simulation - service's OpenAPI document gives the route):: + becomes, with ``Client`` from ``policyengine_api_simulation_client``, + ``submit_household_calculation`` from its ``api.default`` package and + ``HouseholdCalculationRequest`` from its ``models``, the names the + simulation service's OpenAPI document gives the route:: client = Client(base_url=str(http_client.base_url)) client.set_httpx_client(http_client) From 75afad4c2f0ca18e1d9cbf708f55efaf8d518277 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:40:50 +0400 Subject: [PATCH 32/57] Test that a 500 from the simulation service is answered 502 (failing) Sim answers 500 internal_error when it fails, possibly after creating the job, so a 500 should raise its own error, answered 502, rather than the unavailable error answered 503, which invites a resubmission. The error doesn't exist yet. Co-Authored-By: Claude Opus 5.5 --- .../test_simulation_household_calculations.py | 37 ++++++++++++++++++- .../v2/test_household_calculation_routes.py | 6 +++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/tests/unit/libs/test_simulation_household_calculations.py b/tests/unit/libs/test_simulation_household_calculations.py index 815ad7f2a..5b9b0a67e 100644 --- a/tests/unit/libs/test_simulation_household_calculations.py +++ b/tests/unit/libs/test_simulation_household_calculations.py @@ -18,6 +18,7 @@ from policyengine_api.services.v2.household_calculations.types import ( HouseholdCalculationAccepted, SimulationServiceAnswerUnreadableError, + SimulationServiceInternalError, SimulationServiceInvalidRequestError, SimulationServiceUnavailableError, ) @@ -236,7 +237,41 @@ def test_a_dependency_the_service_cannot_read_names_its_code() -> None: ) -@pytest.mark.parametrize("status_code", [400, 401, 404, 500, 502, 503]) +@pytest.mark.parametrize( + ("body", "message"), + [ + pytest.param( + { + "code": "internal_error", + "detail": "The service could not build the household function's " + "request.", + }, + "The simulation service failed (internal_error): The service could " + "not build the household function's request. It may have created " + "the job, so submitting again could create another.", + id="internal-error", + ), + pytest.param( + b"", + "The simulation service answered 500. It may have created the job, " + "so submitting again could create another.", + id="unreadable", + ), + ], +) +def test_an_internal_error_is_not_unavailable(body, message) -> None: + # A 500 may come after the service created the job, so the caller is not + # invited to submit again. + service = RecordingService(500, body) + + with pytest.raises(SimulationServiceInternalError) as failed: + service.client().submit(calculation_input()) + + assert not isinstance(failed.value, SimulationServiceUnavailableError) + assert str(failed.value) == message + + +@pytest.mark.parametrize("status_code", [400, 401, 404, 502, 503]) def test_any_other_answer_is_unavailable(status_code) -> None: service = RecordingService(status_code, b"") diff --git a/tests/unit/v2/test_household_calculation_routes.py b/tests/unit/v2/test_household_calculation_routes.py index d618b3227..c052a549f 100644 --- a/tests/unit/v2/test_household_calculation_routes.py +++ b/tests/unit/v2/test_household_calculation_routes.py @@ -22,6 +22,7 @@ HouseholdCalculationRead, HouseholdCalculationSubmitted, SimulationServiceAnswerUnreadableError, + SimulationServiceInternalError, SimulationServiceInvalidRequestError, SimulationServiceUnavailableError, ) @@ -267,6 +268,11 @@ def test_a_malformed_request_is_refused_before_anything_is_read( "The simulation service accepted the calculation, but its " "answer could not be read.", ), + ( + SimulationServiceInternalError("The simulation service failed."), + 502, + "The simulation service failed.", + ), ( RuntimeError("secret detail"), 500, From 2be97a798b1c55e635b25ee9a57b89ad197e78fc Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:41:48 +0400 Subject: [PATCH 33/57] Answer the simulation service's 500 with 502 A 500 from sim (internal_error) may come after it created the job, so the adapter raises SimulationServiceInternalError, with sim's code and reason when it gives them, and the route answers it 502, as it answers an unreadable 202, instead of the 503 that invites a resubmission. The Stage 11 notes say so. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-11-v2-households.md | 4 +++- .../household_calculations/response_models.py | 3 ++- .../v2/household_calculations/routes.py | 9 +++++--- .../libs/simulation_household_calculations.py | 23 ++++++++++++++++++- .../v2/household_calculations/types.py | 12 +++++++++- 5 files changed, 44 insertions(+), 7 deletions(-) diff --git a/docs/migration/stage-11-v2-households.md b/docs/migration/stage-11-v2-households.md index 0030648f1..46bfeea37 100644 --- a/docs/migration/stage-11-v2-households.md +++ b/docs/migration/stage-11-v2-households.md @@ -322,7 +322,9 @@ IDs and options, with the request's ID and observability ID headers. The answer is 202 with the new job's `calculation_id` and `status`; each submission creates a new job. The simulation service's 422 reasons reach the caller as 422. A 202 whose body the Public API can't read is 502: the job -exists, so submitting again would create another. Any other refusal (with +exists, so submitting again would create another. So is a 500 from the +simulation service (`internal_error`), which may come after it created the +job. Any other refusal (with the simulation service's `code` and reason in the message, and the failed job's ID when Modal refused the call), no answer, a refusal the Public API can't read, missing configuration or a gateway token it can't fetch is 503. diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py index 0233dff07..e924ea8d2 100644 --- a/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py +++ b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py @@ -182,7 +182,8 @@ class HouseholdCalculationResponse( "model": V2ErrorResponse, "description": ( "The simulation service accepted the calculation, so its job exists, " - "but its answer cannot be read. Submitting again creates another job." + "but its answer cannot be read; or it failed (500), possibly after " + "creating the job. Submitting again could create another job." ), }, 422: { diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/routes.py b/policyengine_api/fastapi_routes/v2/household_calculations/routes.py index be8ab8738..4aa3714a0 100644 --- a/policyengine_api/fastapi_routes/v2/household_calculations/routes.py +++ b/policyengine_api/fastapi_routes/v2/household_calculations/routes.py @@ -39,6 +39,7 @@ ) from policyengine_api.services.v2.household_calculations.types import ( SimulationServiceAnswerUnreadableError, + SimulationServiceInternalError, SimulationServiceInvalidRequestError, SimulationServiceUnavailableError, ) @@ -84,9 +85,11 @@ 404, ), V2ErrorMapping((SimulationServiceUnavailableError,), 503), - # The simulation service created the job, so this is not a 503, which - # invites the caller to submit again. - V2ErrorMapping((SimulationServiceAnswerUnreadableError,), 502), + # The simulation service created the job, or may have, so this is not a + # 503, which invites the caller to submit again. + V2ErrorMapping( + (SimulationServiceAnswerUnreadableError, SimulationServiceInternalError), 502 + ), V2ErrorMapping( (HouseholdCalculationResultUnreadableError,), 500, diff --git a/policyengine_api/libs/simulation_household_calculations.py b/policyengine_api/libs/simulation_household_calculations.py index 3abe3b8d8..949de0efa 100644 --- a/policyengine_api/libs/simulation_household_calculations.py +++ b/policyengine_api/libs/simulation_household_calculations.py @@ -19,6 +19,8 @@ request, or one its household contract refuses), one per field as FastAPI writes them: the reasons reach the caller as SimulationServiceInvalidRequestError; +- 500 (``internal_error``) when it failed, possibly after creating the job: + SimulationServiceInternalError, which the caller must not resubmit after; - anything else (400 for a household, policy or catalog version it cannot find, which the Public API checks first; 503 when it cannot read its manifest or database or Modal refuses the call, failing the job it @@ -58,6 +60,7 @@ HouseholdCalculationAccepted, HouseholdJobStatusName, SimulationServiceAnswerUnreadableError, + SimulationServiceInternalError, SimulationServiceInvalidRequestError, SimulationServiceUnavailableError, ) @@ -192,6 +195,21 @@ def _refusal(answer: SimulationAnswer) -> _Refusal | None: return None +RESUBMISSION_WARNING = ( + "It may have created the job, so submitting again could create another." +) + + +def _internal_error(refusal: _Refusal | None) -> str: + """Describe the service's failure, which may have created the job.""" + if refusal is None or not isinstance(refusal.detail, str): + return f"The simulation service answered 500. {RESUBMISSION_WARNING}" + code = "" if refusal.code is None else f" ({refusal.code})" + return ( + f"The simulation service failed{code}: {refusal.detail} {RESUBMISSION_WARNING}" + ) + + def _unavailable(answer: SimulationAnswer, refusal: _Refusal | None) -> str: """Describe an answer that means the service cannot calculate now.""" if refusal is None or not isinstance(refusal.detail, str): @@ -252,7 +270,8 @@ def submit( Raise SimulationServiceInvalidRequestError with the service's reasons when it refuses the caller's data, SimulationServiceAnswerUnreadableError - when it accepted the calculation but its answer cannot be read, and + when it accepted the calculation but its answer cannot be read, + SimulationServiceInternalError when it failed (500), and SimulationServiceUnavailableError otherwise. """ start_observability_id() @@ -278,6 +297,8 @@ def submit( if refusal is None: raise SimulationServiceUnavailableError(UNREADABLE_ANSWER) raise SimulationServiceInvalidRequestError(_invalid_request_reason(refusal)) + if answer.status_code == HTTPStatus.INTERNAL_SERVER_ERROR: + raise SimulationServiceInternalError(_internal_error(refusal)) raise SimulationServiceUnavailableError(_unavailable(answer, refusal)) diff --git a/policyengine_api/services/v2/household_calculations/types.py b/policyengine_api/services/v2/household_calculations/types.py index 48f6e6957..713fe2e5c 100644 --- a/policyengine_api/services/v2/household_calculations/types.py +++ b/policyengine_api/services/v2/household_calculations/types.py @@ -319,7 +319,9 @@ def submit( Raise SimulationServiceInvalidRequestError when the service refuses the caller's data, with its reasons; SimulationServiceAnswerUnreadableError when it accepted the - calculation but its answer cannot be read; and + calculation but its answer cannot be read; + SimulationServiceInternalError when it failed (500), possibly after + creating the job; and SimulationServiceUnavailableError for any other refusal or failure. """ ... @@ -342,6 +344,14 @@ class SimulationServiceAnswerUnreadableError(RuntimeError): """ +class SimulationServiceInternalError(RuntimeError): + """The simulation service failed (500), possibly after creating the job. + + Submitting again could create another job, so this is not reported as + the service being unavailable. + """ + + class SimulationServiceInvalidRequestError(ValueError): """The simulation service refuses the caller's data; the message says why. From ad1e18db905b0dcd38fee7fd5a4e426c5b5c868a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:45:11 +0400 Subject: [PATCH 34/57] Test that the 202 returns the observability ID sent to sim (failing) The submission's 202 should carry X-PolicyEngine-Observability-Id with the ID the simulation service received, the caller's or a new one, as the Flask household routes return theirs, and a submission refused before sim is called should carry none. The ID is bound in the synchronous route's copied context, so the response never carries it. Co-Authored-By: Claude Opus 5.5 --- ...st_household_calculation_database_routes.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/unit/v2/test_household_calculation_database_routes.py b/tests/unit/v2/test_household_calculation_database_routes.py index f6ca70dd9..4fa1074a2 100644 --- a/tests/unit/v2/test_household_calculation_database_routes.py +++ b/tests/unit/v2/test_household_calculation_database_routes.py @@ -337,6 +337,8 @@ def test_the_submission_carries_the_request_and_callers_observability_ids( assert str(sent.url) == "https://simulation.example.test/household-calculations" assert sent.headers[REQUEST_ID_HEADER] == response.headers[REQUEST_ID_HEADER] assert sent.headers[OBSERVABILITY_ID_HEADER] == observability_id + # The 202 returns the ID the simulation service received. + assert response.headers[OBSERVABILITY_ID_HEADER] == observability_id def test_a_submission_without_an_observability_id_is_given_one( @@ -349,3 +351,19 @@ def test_a_submission_without_an_observability_id_is_given_one( assert response.status_code == 202 (sent,) = simulation_recorder.requests assert UUID(sent.headers[OBSERVABILITY_ID_HEADER]) + assert ( + response.headers[OBSERVABILITY_ID_HEADER] + == (sent.headers[OBSERVABILITY_ID_HEADER]) + ) + + +def test_a_refused_submission_returns_no_observability_id( + client_with_simulation_client, stored, simulation_recorder +) -> None: + body = {**_body(stored, None), "household_id": str(uuid4())} + + response = client_with_simulation_client.post(CALCULATIONS, json=body) + + assert response.status_code == 404 + assert simulation_recorder.requests == [] + assert OBSERVABILITY_ID_HEADER not in response.headers From f497ec508503be3d2a43b0e766a5c25d47851478 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:47:09 +0400 Subject: [PATCH 35/57] Return the observability ID sent to sim on the 202 The submission binds its observability ID in the synchronous route's copy of the request context, which the response middleware doesn't see. The route now reads it from that context after a successful submission and sets X-PolicyEngine-Observability-Id on the 202, which the middleware keeps, so the caller gets the ID the simulation service received. The adapter's docstring says where the ID is bound and how it reaches the response. Co-Authored-By: Claude Opus 5.5 --- .../v2/household_calculations/routes.py | 11 ++++++++++- .../libs/simulation_household_calculations.py | 9 ++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/routes.py b/policyengine_api/fastapi_routes/v2/household_calculations/routes.py index 4aa3714a0..5d8423680 100644 --- a/policyengine_api/fastapi_routes/v2/household_calculations/routes.py +++ b/policyengine_api/fastapi_routes/v2/household_calculations/routes.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, TypeVar from uuid import UUID -from fastapi import APIRouter +from fastapi import APIRouter, Response from starlette.responses import JSONResponse from policyengine_api.data.v2.settings import V2ConfigurationError @@ -37,6 +37,8 @@ HouseholdCalculationSubmittedResponse, HouseholdCalculationSubmittedResult, ) +from policyengine_api.observability.identifiers import OBSERVABILITY_ID_HEADER +from policyengine_api.request_context import current_observability_id from policyengine_api.services.v2.household_calculations.types import ( SimulationServiceAnswerUnreadableError, SimulationServiceInternalError, @@ -137,6 +139,7 @@ def build_v2_household_calculation_router( ) def submit_household_calculation( body: HouseholdCalculationRequest, + response: Response, ) -> HouseholdCalculationSubmittedResponse | JSONResponse: def submit() -> HouseholdCalculationSubmittedResponse | JSONResponse: try: @@ -147,6 +150,12 @@ def submit() -> HouseholdCalculationSubmittedResponse | JSONResponse: return v2_missing_parameters_response( str(error), error.missing_parameters ) + # The submission bound the observability ID it sent the simulation + # service in this worker thread's copy of the request context, + # which the response middleware doesn't see, so the 202 names it. + observability_id = current_observability_id() + if observability_id is not None: + response.headers[OBSERVABILITY_ID_HEADER] = observability_id return HouseholdCalculationSubmittedResponse( result=HouseholdCalculationSubmittedResult( item=HouseholdCalculationSubmittedItem.from_submitted(calculation) diff --git a/policyengine_api/libs/simulation_household_calculations.py b/policyengine_api/libs/simulation_household_calculations.py index 949de0efa..ec3d529d3 100644 --- a/policyengine_api/libs/simulation_household_calculations.py +++ b/policyengine_api/libs/simulation_household_calculations.py @@ -31,9 +31,12 @@ Each submission carries the request's ID and observability ID headers. The observability ID is the request's own, else the caller's, else a new one, -bound for the rest of the request, as the Public API's own household -calculations bind one once they accept a calculation; the simulation service -passes it to the household function. +which ``submit`` binds in the current context once the Public API's checks +have passed, as the Public API's own household calculations bind one when +they accept a calculation; the simulation service passes it to the household +function. The route reads it from that context and returns it in the 202's +``X-PolicyEngine-Observability-Id`` header, since a synchronous route runs in +a copy of the request context that the response middleware doesn't see. Wiring the generated simulation client: ``post_household_calculation`` is the only code that sends the request, and it returns the status and body, which From 031b249fd139960778ee96992526fb6e330d0874 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:58:49 +0400 Subject: [PATCH 36/57] Test listing only versions still unrecorded after a publication (failing) The publication evidence should list as not recorded only the versions whose SPM availability is still unrecorded after it, so a version an earlier publication recorded is never listed, whether the simulation service's versions couldn't be read or don't serve it now. The integration test runs it as the catalog writer role. Today every such version is listed. Co-Authored-By: Claude Opus 5.5 --- .../test_v2_catalog_spm_availability.py | 83 +++++++++++++++++++ .../unit/v2/test_catalog_spm_availability.py | 30 ++++++- 2 files changed, 109 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_v2_catalog_spm_availability.py b/tests/integration/test_v2_catalog_spm_availability.py index d3c98b48d..a38b07abe 100644 --- a/tests/integration/test_v2_catalog_spm_availability.py +++ b/tests/integration/test_v2_catalog_spm_availability.py @@ -7,6 +7,7 @@ from __future__ import annotations +from types import SimpleNamespace from uuid import uuid4 import sqlalchemy as sa @@ -20,7 +21,12 @@ MODEL_VERSIONS, MODELS, ) +from policyengine_api.data.v2.catalog.publication_types import ( + CatalogPublicationError, + SPMAvailabilityRecording, +) from policyengine_api.data.v2.catalog.spm_availability import ( + record_published_spm_availability, record_spm_availability, ) from policyengine_api.data.v2.privileges import CATALOG_WRITER_ROLE @@ -89,3 +95,80 @@ def test_availability_fills_only_the_named_version_and_only_once( assert first assert not again assert stored == {version_id: AVAILABLE, other_version_id: None} + + +def _unreadable_versions(_url: str) -> object: + msg = "the simulation service's versions could not be read" + raise CatalogPublicationError(msg) + + +def test_publication_lists_only_versions_still_unrecorded( + disposable_v2_database_url: str, managed_v2_roles, v2_engine_as_role +) -> None: + command.upgrade(Config(str(REPO / "alembic-v2.ini")), "head") + engine = create_engine(disposable_v2_database_url, poolclass=NullPool) + writer_engine = v2_engine_as_role(CATALOG_WRITER_ROLE, poolclass=NullPool) + model_ids = {"us": uuid4(), "uk": uuid4()} + model_names = { + country_id: f"policyengine-{country_id}-{model_id.hex[:8]}" + for country_id, model_id in model_ids.items() + } + catalog = SimpleNamespace( + policyengine_version="6.10.0", + countries=[ + SimpleNamespace( + country_id=country_id, + model=SimpleNamespace(name=model_names[country_id]), + model_version=SimpleNamespace(version="1.450.0"), + ) + for country_id in ("uk", "us") + ], + ) + try: + with engine.begin() as connection: + for country_id, model_id in model_ids.items(): + connection.execute( + sa.insert(MODELS).values(id=model_id, name=model_names[country_id]) + ) + connection.execute( + sa.insert(MODEL_VERSIONS).values( + id=uuid4(), + model_id=model_id, + version="1.450.0", + current_law_id=2, + metadata_time_periods=[2026], + # An earlier publication recorded the US version. + spm_availability=AVAILABLE if country_id == "us" else None, + ) + ) + unreadable = record_published_spm_availability( + writer_engine, + catalog, + {"SIMULATION_ENTRYPOINT_URL": "https://simulation.example"}, + fetch_versions=_unreadable_versions, + ) + # The simulation service serves no PolicyEngine.py version yet. + not_served = record_published_spm_availability( + writer_engine, + catalog, + {"SIMULATION_ENTRYPOINT_URL": "https://simulation.example"}, + fetch_versions=lambda _url: {"policyengine": {}}, + ) + finally: + with engine.begin() as connection: + for model_id in model_ids.values(): + connection.execute( + sa.delete(MODEL_VERSIONS).where( + MODEL_VERSIONS.c.model_id == model_id + ) + ) + connection.execute(sa.delete(MODELS).where(MODELS.c.id == model_id)) + writer_engine.dispose() + engine.dispose() + + assert unreadable == SPMAvailabilityRecording( + not_recorded=(("uk", "1.450.0"),), + versions_unreadable="the simulation service's versions could not be read", + ) + # The UK version is recorded unavailable; the US one keeps its value. + assert not_served == SPMAvailabilityRecording(recorded=(("uk", "1.450.0", False),)) diff --git a/tests/unit/v2/test_catalog_spm_availability.py b/tests/unit/v2/test_catalog_spm_availability.py index 447df6e92..abe78a507 100644 --- a/tests/unit/v2/test_catalog_spm_availability.py +++ b/tests/unit/v2/test_catalog_spm_availability.py @@ -134,6 +134,19 @@ def _catalog(*country_ids: str) -> SimpleNamespace: ) +def _store_unrecorded(monkeypatch: pytest.MonkeyPatch, *model_names: str) -> None: + """Make version 1.0.0 of these models the only ones still unrecorded.""" + monkeypatch.setattr( + spm_availability, + "read_unrecorded_versions", + lambda _connection, versions: { + (model_name, version) + for model_name, version in versions + if model_name in model_names + }, + ) + + class RecordingEngine: """An engine whose transactions are only counted.""" @@ -158,6 +171,7 @@ def record(_connection, *, model_name, version, availability) -> bool: return True monkeypatch.setattr(spm_availability, "record_spm_availability", record) + _store_unrecorded(monkeypatch, "policyengine-us") not_served = {"policyengine": {"latest": "6.11.0", "6.11.0": "app"}} recording = spm_availability.record_published_spm_availability( @@ -173,20 +187,26 @@ def record(_connection, *, model_name, version, availability) -> bool: ) +@pytest.mark.parametrize( + "versions", + [VERSIONS, {"policyengine": {"latest": "6.11.0", "6.11.0": "app"}}], + ids=["served", "not-served-now"], +) def test_a_version_recorded_by_an_earlier_publication_is_not_listed( - monkeypatch: pytest.MonkeyPatch, + monkeypatch: pytest.MonkeyPatch, versions: object ) -> None: monkeypatch.setattr( spm_availability, "record_spm_availability", lambda _connection, *, model_name, version, availability: False, ) + _store_unrecorded(monkeypatch) recording = spm_availability.record_published_spm_availability( RecordingEngine(), _catalog("us"), {"SIMULATION_ENTRYPOINT_URL": "https://simulation.example"}, - fetch_versions=lambda _url: VERSIONS, + fetch_versions=lambda _url: versions, ) assert recording == SPMAvailabilityRecording() @@ -235,7 +255,7 @@ def test_recording_needs_the_simulation_service_url() -> None: ) -def test_an_unreadable_versions_answer_leaves_every_version_unrecorded( +def test_an_unreadable_versions_answer_lists_only_versions_still_unrecorded( monkeypatch: pytest.MonkeyPatch, ) -> None: recorded: list[str] = [] @@ -246,6 +266,8 @@ def test_an_unreadable_versions_answer_leaves_every_version_unrecorded( model_name ), ) + # An earlier publication recorded the US version. + _store_unrecorded(monkeypatch, "policyengine-uk") def unreadable(_url: str) -> object: msg = "the simulation service's versions could not be read" @@ -260,6 +282,6 @@ def unreadable(_url: str) -> object: assert recorded == [] assert recording == SPMAvailabilityRecording( - not_recorded=(("uk", "1.0.0"), ("us", "1.0.0")), + not_recorded=(("uk", "1.0.0"),), versions_unreadable="the simulation service's versions could not be read", ) From 86d3b1a670283a81fa6a45524e595eade57ad6ad Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 01:03:07 +0400 Subject: [PATCH 37/57] List as not recorded only versions still unrecorded after a publication record_published_spm_availability now reads which of the publication's versions still have no SPM availability, after recording, or at once when the simulation service's versions couldn't be read, and lists only those. A version an earlier publication recorded is never listed, as SPMAvailabilityRecording's docstring now says. The integration test now leaves the UK version's column out, as publication's insert does, so it holds SQL NULL rather than JSON null. Co-Authored-By: Claude Opus 5.5 --- .../data/v2/catalog/publication_types.py | 13 ++-- .../data/v2/catalog/spm_availability.py | 66 ++++++++++++++----- .../test_v2_catalog_spm_availability.py | 9 ++- 3 files changed, 62 insertions(+), 26 deletions(-) diff --git a/policyengine_api/data/v2/catalog/publication_types.py b/policyengine_api/data/v2/catalog/publication_types.py index 62e92be06..e46f478c1 100644 --- a/policyengine_api/data/v2/catalog/publication_types.py +++ b/policyengine_api/data/v2/catalog/publication_types.py @@ -46,13 +46,12 @@ class SPMAvailabilityRecording: """What one publication recorded of its catalog versions' SPM availability. ``recorded`` lists each version it recorded: country, model version and - whether SPM settings are available. ``not_recorded`` lists each version - whose availability it doesn't know: one the simulation service doesn't - serve yet, or every version when ``versions_unreadable`` says why the - service's versions couldn't be read. A version is unknown to - ``/v2/model-catalog`` until some publication records it, and a version - an earlier publication recorded keeps its value and is listed in - neither. + whether SPM settings are available. ``not_recorded`` lists each of its + versions whose availability is still unrecorded after it, so unknown to + ``/v2/model-catalog``: one the simulation service doesn't serve yet, or + any when ``versions_unreadable`` says why the service's versions + couldn't be read. A version an earlier publication recorded keeps its + value and is listed in neither. """ recorded: tuple[tuple[str, str, bool], ...] = () diff --git a/policyengine_api/data/v2/catalog/spm_availability.py b/policyengine_api/data/v2/catalog/spm_availability.py index a73fe8729..41ec54df9 100644 --- a/policyengine_api/data/v2/catalog/spm_availability.py +++ b/policyengine_api/data/v2/catalog/spm_availability.py @@ -43,7 +43,7 @@ from policyengine_api.spm import SPMSelection if TYPE_CHECKING: - from collections.abc import Callable, Mapping + from collections.abc import Callable, Iterable, Mapping from sqlalchemy import Connection, Engine @@ -159,6 +159,42 @@ def record_spm_availability( return bool(result.rowcount) +def read_unrecorded_versions( + connection: Connection, versions: Iterable[tuple[str, str]] +) -> set[tuple[str, str]]: + """Return the (model name, version) pairs whose SPM availability is unrecorded.""" + requested = list(versions) + if not requested: + return set() + rows = connection.execute( + sa.select(MODELS.c.name, MODEL_VERSIONS.c.version) + .join(MODEL_VERSIONS, MODEL_VERSIONS.c.model_id == MODELS.c.id) + .where( + sa.tuple_(MODELS.c.name, MODEL_VERSIONS.c.version).in_(requested), + MODEL_VERSIONS.c.spm_availability.is_(None), + ) + ).all() + return {(row.name, row.version) for row in rows} + + +def _still_unrecorded( + connection: Connection, catalog: NormalizedCatalog +) -> tuple[tuple[str, str], ...]: + """Return the catalog's versions, by country, whose availability is unrecorded.""" + unrecorded = read_unrecorded_versions( + connection, + [ + (country.model.name, country.model_version.version) + for country in catalog.countries + ], + ) + return tuple( + (country.country_id, country.model_version.version) + for country in catalog.countries + if (country.model.name, country.model_version.version) in unrecorded + ) + + def record_published_spm_availability( engine: Engine, catalog: NormalizedCatalog, @@ -168,10 +204,12 @@ def record_published_spm_availability( ) -> SPMAvailabilityRecording: """Record each published country's SPM availability for the catalog version. - Return what was recorded and what was not. A US version the service does - not serve yet stays unrecorded, and so does every version when the - service's versions cannot be read; the next publication run tries again. - Raise V2ConfigurationError without ``SIMULATION_ENTRYPOINT_URL``. + Return what was recorded, and the versions still unrecorded after it. A + US version the service does not serve yet stays unrecorded, and so does + every version when the service's versions cannot be read; the next + publication run tries again. A version an earlier publication recorded + keeps its value and is in neither list. Raise V2ConfigurationError + without ``SIMULATION_ENTRYPOINT_URL``. """ url = simulation_entrypoint_url(os.environ if environ is None else environ) try: @@ -182,24 +220,19 @@ def record_published_spm_availability( "tries again: %s", error, ) + with engine.begin() as connection: + not_recorded = _still_unrecorded(connection, catalog) return SPMAvailabilityRecording( - not_recorded=tuple( - (country.country_id, country.model_version.version) - for country in catalog.countries - ), - versions_unreadable=str(error), + not_recorded=not_recorded, versions_unreadable=str(error) ) recorded: list[tuple[str, str, bool]] = [] - not_recorded: list[tuple[str, str]] = [] with engine.begin() as connection: for country in catalog.countries: version = country.model_version.version availability = spm_availability( country.country_id, catalog.policyengine_version, versions ) - if availability is None: - not_recorded.append((country.country_id, version)) - elif record_spm_availability( + if availability is not None and record_spm_availability( connection, model_name=country.model.name, version=version, @@ -208,6 +241,5 @@ def record_published_spm_availability( recorded.append( (country.country_id, version, bool(availability["available"])) ) - return SPMAvailabilityRecording( - recorded=tuple(recorded), not_recorded=tuple(not_recorded) - ) + not_recorded = _still_unrecorded(connection, catalog) + return SPMAvailabilityRecording(recorded=tuple(recorded), not_recorded=not_recorded) diff --git a/tests/integration/test_v2_catalog_spm_availability.py b/tests/integration/test_v2_catalog_spm_availability.py index a38b07abe..672cd8147 100644 --- a/tests/integration/test_v2_catalog_spm_availability.py +++ b/tests/integration/test_v2_catalog_spm_availability.py @@ -130,6 +130,12 @@ def test_publication_lists_only_versions_still_unrecorded( connection.execute( sa.insert(MODELS).values(id=model_id, name=model_names[country_id]) ) + # An earlier publication recorded the US version. The UK one + # leaves the column out, as publication's insert does, so it + # holds SQL NULL rather than JSON null. + recorded_earlier = ( + {"spm_availability": AVAILABLE} if country_id == "us" else {} + ) connection.execute( sa.insert(MODEL_VERSIONS).values( id=uuid4(), @@ -137,8 +143,7 @@ def test_publication_lists_only_versions_still_unrecorded( version="1.450.0", current_law_id=2, metadata_time_periods=[2026], - # An earlier publication recorded the US version. - spm_availability=AVAILABLE if country_id == "us" else None, + **recorded_earlier, ) ) unreadable = record_published_spm_availability( From 77d13fa3a262ab2f1d49ff995ac1b98ea93bbca2 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 01:05:11 +0400 Subject: [PATCH 38/57] Test that a live submission isn't sent again once it may have arrived (failing) After a read timeout, a dropped response or a read error, the submission may have reached the Public API and the job may exist, so the live fixture should raise the error rather than send it again. It sends it again after any transport error. Co-Authored-By: Claude Opus 5.5 --- ...est_live_household_calculation_requests.py | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/unit/v2/test_live_household_calculation_requests.py b/tests/unit/v2/test_live_household_calculation_requests.py index 0441c4248..b8505f1b2 100644 --- a/tests/unit/v2/test_live_household_calculation_requests.py +++ b/tests/unit/v2/test_live_household_calculation_requests.py @@ -1,9 +1,9 @@ """Which answers the live household calculation checks send again. -A submission is sent again only after a 503 or a failed connection: a 502 -means the simulation service accepted it, so the job exists. A read is sent -again after a 502, 503 or 504 or a failed connection, since reading creates -nothing. +A submission is sent again only after a 503 or a connection that was never +made: a 502 means the simulation service accepted it, and after a read +timeout or a dropped response the job may exist. A read is sent again after +a 502, 503 or 504 or any transport error, since reading creates nothing. """ from __future__ import annotations @@ -88,6 +88,20 @@ def test_a_submission_is_sent_again_after_a_failed_connection() -> None: assert public_api.sent == ["POST", "POST", "GET"] +@pytest.mark.parametrize( + "transport_error", [httpx.ReadTimeout, httpx.RemoteProtocolError, httpx.ReadError] +) +def test_a_submission_is_never_sent_again_once_it_may_have_arrived( + transport_error, +) -> None: + public_api = ScriptedPublicApi(submissions=[transport_error, 202], reads=[200]) + + with pytest.raises(transport_error): + public_api.calculate() + + assert public_api.sent == ["POST"] + + @pytest.mark.parametrize("status_code", [502, 504, 500]) def test_a_submission_is_never_sent_again_after_another_failure(status_code) -> None: public_api = ScriptedPublicApi(submissions=[status_code, 202], reads=[200]) From 51daa7de096175d2dacf55b4c1bcf711a7b141f0 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 01:05:54 +0400 Subject: [PATCH 39/57] Send a live submission again only after a 503 or a connection never made The live fixture now sends a submission again only after a 503 or an httpx.ConnectError, and raises any other transport error, such as a read timeout or a dropped response, after which the job may exist. A read is still sent again after any transport error or a 502, 503 or 504. Co-Authored-By: Claude Opus 5.5 --- .../live_household_calculations.py | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/tests/fixtures/integration/live_household_calculations.py b/tests/fixtures/integration/live_household_calculations.py index 3ac6179be..2917dc0cd 100644 --- a/tests/fixtures/integration/live_household_calculations.py +++ b/tests/fixtures/integration/live_household_calculations.py @@ -35,14 +35,16 @@ # or version manifest for a moment, or the Public API failing to read # household_jobs, is sent through again while it lasts at most # TRANSIENT_LIMIT_SECONDS in a row. A submission is sent again only after a -# 503 or a failed connection: the Public API answers 502 when the simulation -# service accepted the calculation but its answer could not be read, so the -# job exists, and sending it again would create another. A read is sent again -# after a 502, 503 or 504 or a failed connection, since reading creates -# nothing. Any other failure, and a longer outage, fails the check with the -# answer's body. +# 503 or a connection that was never made (httpx.ConnectError): the Public API +# answers 502 when the simulation service may have created the job, and after +# a read timeout or a dropped response the job may exist too, so sending it +# again could create another. A read is sent again after a 502, 503 or 504 or +# any transport error, since reading creates nothing. Any other failure, and a +# longer outage, fails the check with the answer's body or the error. SUBMISSION_RETRY_STATUS_CODES = frozenset({503}) +SUBMISSION_RETRY_ERRORS: tuple[type[httpx.TransportError], ...] = (httpx.ConnectError,) READ_RETRY_STATUS_CODES = frozenset({502, 503, 504}) +READ_RETRY_ERRORS: tuple[type[httpx.TransportError], ...] = (httpx.TransportError,) TRANSIENT_LIMIT_SECONDS = float( os.environ.get("STAGING_HOUSEHOLD_TRANSIENT_LIMIT_SECONDS", "90") ) @@ -191,17 +193,20 @@ def store_reform(client: httpx.Client, household: LiveHousehold) -> str | None: def _send_through_short_outages( - send: Callable[[], httpx.Response], retry_status_codes: frozenset[int] + send: Callable[[], httpx.Response], + retry_status_codes: frozenset[int], + retry_errors: tuple[type[httpx.TransportError], ...], ) -> httpx.Response: - """Send the request again while it fails to connect or gets one of the codes. + """Send the request again while it raises one of the errors or gets a code. - Raise AssertionError once that has lasted TRANSIENT_LIMIT_SECONDS. + Any other error is raised. Raise AssertionError once the outage has + lasted TRANSIENT_LIMIT_SECONDS. """ outage_started: float | None = None while True: try: response = send() - except httpx.TransportError as error: + except retry_errors as error: failure = f"{type(error).__name__}: {error}" else: if response.status_code not in retry_status_codes: @@ -235,6 +240,7 @@ def calculate( response = _send_through_short_outages( lambda: client.post(CALCULATION_PATH, json=body), SUBMISSION_RETRY_STATUS_CODES, + SUBMISSION_RETRY_ERRORS, ) submit_seconds = time.monotonic() - started assert response.status_code == 202, response.text @@ -245,7 +251,9 @@ def calculate( # household_jobs, so the calculation is read at least once. while True: polled = _send_through_short_outages( - partial(client.get, calculation_path), READ_RETRY_STATUS_CODES + partial(client.get, calculation_path), + READ_RETRY_STATUS_CODES, + READ_RETRY_ERRORS, ) assert polled.status_code == 200, polled.text item = polled.json()["result"]["item"] From 15829233e825771a028b153d0bcc241d15e232c6 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:03:12 +0400 Subject: [PATCH 40/57] Test lost answers, observability IDs on errors and quiet SPM output (failing) - A connection never made (ConnectError, ConnectTimeout, PoolTimeout) should stay unavailable (503). An answer lost after the submission may have been sent (ReadTimeout, WriteTimeout, ReadError, WriteError, RemoteProtocolError) should raise the 502 error, because sim may have created the job. Today every transport error answers 503. - Every answer given after sim was called (its 422, 503, 500 as 502 and an unreadable 202 as 502) should carry the observability ID sent to sim. A refusal before sim is called should echo no ID, even the caller's; that test now sends one. - Publication's command should say SPM availability wasn't recorded only when some version is still unrecorded. Co-Authored-By: Claude Opus 5.5 --- .../test_simulation_household_calculations.py | 43 +++++++++++-- tests/unit/v2/test_catalog_initialization.py | 18 ++++++ ...t_household_calculation_database_routes.py | 64 +++++++++++++++++-- 3 files changed, 114 insertions(+), 11 deletions(-) diff --git a/tests/unit/libs/test_simulation_household_calculations.py b/tests/unit/libs/test_simulation_household_calculations.py index 5b9b0a67e..c7ecc4f58 100644 --- a/tests/unit/libs/test_simulation_household_calculations.py +++ b/tests/unit/libs/test_simulation_household_calculations.py @@ -303,17 +303,46 @@ def test_an_accepted_answer_that_cannot_be_read_is_not_unavailable(body) -> None ) -def test_an_unreachable_service_is_unavailable() -> None: - def refuse(request: httpx.Request) -> httpx.Response: - msg = "connection refused" - raise httpx.ConnectError(msg, request=request) +def _client_raising(error_type: type[httpx.TransportError]): + def fail(request: httpx.Request) -> httpx.Response: + msg = "the transport failed" + raise error_type(msg, request=request) - client = SimulationHouseholdCalculationClient.from_environment( - transport=httpx.MockTransport(refuse) + return SimulationHouseholdCalculationClient.from_environment( + transport=httpx.MockTransport(fail) ) + +@pytest.mark.parametrize( + "error_type", [httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout] +) +def test_a_connection_never_made_is_unavailable(error_type) -> None: with pytest.raises(SimulationServiceUnavailableError, match="could not be reached"): - client.submit(calculation_input()) + _client_raising(error_type).submit(calculation_input()) + + +@pytest.mark.parametrize( + "error_type", + [ + httpx.ReadTimeout, + httpx.WriteTimeout, + httpx.ReadError, + httpx.WriteError, + httpx.RemoteProtocolError, + ], +) +def test_an_answer_lost_after_sending_is_not_unavailable(error_type) -> None: + # The service may have received the submission and created the job, so + # the caller is not invited to submit again. + with pytest.raises(SimulationServiceAnswerUnreadableError) as lost: + _client_raising(error_type).submit(calculation_input()) + + assert not isinstance(lost.value, SimulationServiceUnavailableError) + assert str(lost.value) == ( + "The simulation service may have received the submission, but its " + f"answer did not arrive ({error_type.__name__}). Check whether it " + "created the job before submitting again." + ) def test_a_gateway_token_that_cannot_be_fetched_is_unavailable() -> None: diff --git a/tests/unit/v2/test_catalog_initialization.py b/tests/unit/v2/test_catalog_initialization.py index d5c4bb1b2..7543dfc75 100644 --- a/tests/unit/v2/test_catalog_initialization.py +++ b/tests/unit/v2/test_catalog_initialization.py @@ -441,3 +441,21 @@ def test_main_says_on_stderr_when_spm_availability_could_not_be_recorded( "could not be read; /v2/model-catalog reports a version without a " "recorded availability as unknown until a publication records it" ] + + +def test_main_says_nothing_when_no_version_needed_recording( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """Every version had its availability from an earlier publication.""" + recording = publication_types.SPMAvailabilityRecording( + versions_unreadable="the simulation service's versions could not be read" + ) + evidence = replace(_evidence(), spm_availability=recording) + monkeypatch.setattr(initialization, "initialize_catalog", lambda: evidence) + + assert initialization.main() == 0 + + output = capsys.readouterr() + assert json.loads(output.out)["spm_availability"]["not_recorded"] == [] + assert output.err == "" diff --git a/tests/unit/v2/test_household_calculation_database_routes.py b/tests/unit/v2/test_household_calculation_database_routes.py index 4fa1074a2..abdb689a5 100644 --- a/tests/unit/v2/test_household_calculation_database_routes.py +++ b/tests/unit/v2/test_household_calculation_database_routes.py @@ -7,6 +7,7 @@ from __future__ import annotations +import json from datetime import UTC, datetime from uuid import UUID, uuid4 @@ -261,14 +262,20 @@ def test_a_catalog_that_cannot_be_queried_answers_503( class SimulationRecorder: - """The simulation service, answering every submission 202 and recording it.""" + """The simulation service, answering every submission as scripted. + + It answers 202 with the new job unless ``status_code`` and ``content`` + say otherwise, and records each request. + """ def __init__(self) -> None: self.requests: list[httpx.Request] = [] + self.status_code = 202 + self.content = json.dumps({"job_id": str(JOB_ID), "status": "pending"}) def answer(self, request: httpx.Request) -> httpx.Response: self.requests.append(request) - return httpx.Response(202, json={"job_id": str(JOB_ID), "status": "pending"}) + return httpx.Response(self.status_code, content=self.content) @pytest.fixture @@ -357,13 +364,62 @@ def test_a_submission_without_an_observability_id_is_given_one( ) -def test_a_refused_submission_returns_no_observability_id( +def test_a_refusal_before_sim_is_called_echoes_no_observability_id( client_with_simulation_client, stored, simulation_recorder ) -> None: + # As the Flask household routes do, a refused request binds no ID, so even + # the caller's is not returned. body = {**_body(stored, None), "household_id": str(uuid4())} - response = client_with_simulation_client.post(CALCULATIONS, json=body) + response = client_with_simulation_client.post( + CALCULATIONS, json=body, headers={OBSERVABILITY_ID_HEADER: str(uuid4())} + ) assert response.status_code == 404 assert simulation_recorder.requests == [] assert OBSERVABILITY_ID_HEADER not in response.headers + + +@pytest.mark.parametrize( + ("simulation_answer", "status_code"), + [ + pytest.param( + ( + 422, + {"detail": [{"loc": ["body", "spm"], "msg": "refused", "type": "x"}]}, + ), + 422, + id="refused-data", + ), + pytest.param( + (503, {"code": "database_unavailable", "detail": "down"}), + 503, + id="unavailable", + ), + pytest.param( + (500, {"code": "internal_error", "detail": "failed"}), 502, id="failed" + ), + pytest.param((202, {"unexpected": True}), 502, id="unreadable-202"), + ], +) +def test_an_answer_after_sim_was_called_returns_the_observability_id( + client_with_simulation_client, + stored, + simulation_recorder, + simulation_answer, + status_code, +) -> None: + simulation_status, simulation_body = simulation_answer + simulation_recorder.status_code = simulation_status + simulation_recorder.content = json.dumps(simulation_body) + + response = client_with_simulation_client.post( + CALCULATIONS, json=_body(stored, None) + ) + + assert response.status_code == status_code + (sent,) = simulation_recorder.requests + assert ( + response.headers[OBSERVABILITY_ID_HEADER] + == (sent.headers[OBSERVABILITY_ID_HEADER]) + ) From fe0d0e3ed6368af224ee6a3e1ecc6774862d4a34 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:04:51 +0400 Subject: [PATCH 41/57] Answer a lost answer 502, name the observability ID on every answer after sim, and quiet the SPM output - The adapter answers only a connection never made (ConnectError, ConnectTimeout, PoolTimeout) or a gateway token failure as unavailable (503). Any other transport error, after which sim may have received the submission, raises SimulationServiceAnswerUnreadableError (502), saying so and to check before submitting again. - The route sets X-PolicyEngine-Observability-Id on every answer once the submission bound the ID it sent to sim, its 422, 502 and 503 as well as its 202. A refusal before the call binds none, so it returns none, not even the caller's, as the Flask household routes do. - Publication's command says SPM availability wasn't recorded only when some version is still unrecorded. The Stage 11 notes describe the new answers. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-11-v2-households.md | 8 +++-- .../data/v2/catalog/initialization.py | 6 +++- .../v2/household_calculations/routes.py | 19 +++++++---- .../libs/simulation_household_calculations.py | 34 ++++++++++++++----- .../v2/household_calculations/types.py | 7 ++-- 5 files changed, 52 insertions(+), 22 deletions(-) diff --git a/docs/migration/stage-11-v2-households.md b/docs/migration/stage-11-v2-households.md index 46bfeea37..f79b88273 100644 --- a/docs/migration/stage-11-v2-households.md +++ b/docs/migration/stage-11-v2-households.md @@ -323,8 +323,12 @@ answer is 202 with the new job's `calculation_id` and `status`; each submission creates a new job. The simulation service's 422 reasons reach the caller as 422. A 202 whose body the Public API can't read is 502: the job exists, so submitting again would create another. So is a 500 from the -simulation service (`internal_error`), which may come after it created the -job. Any other refusal (with +simulation service (`internal_error`), and an answer lost after the +submission may have been sent (a read timeout or a dropped connection), +since either may come after it created the job; a connection never made is +503. Every answer given once the call was attempted carries +`X-PolicyEngine-Observability-Id` with the ID sent to the simulation service; +a refusal before it carries none, not even the caller's. Any other refusal (with the simulation service's `code` and reason in the message, and the failed job's ID when Modal refused the call), no answer, a refusal the Public API can't read, missing configuration or a gateway token it can't fetch is 503. diff --git a/policyengine_api/data/v2/catalog/initialization.py b/policyengine_api/data/v2/catalog/initialization.py index 3585b0f40..a73fcc0a7 100644 --- a/policyengine_api/data/v2/catalog/initialization.py +++ b/policyengine_api/data/v2/catalog/initialization.py @@ -200,7 +200,11 @@ def main() -> int: file=sys.stderr, ) spm_recording = evidence.spm_availability - if spm_recording is not None and spm_recording.versions_unreadable is not None: + if ( + spm_recording is not None + and spm_recording.versions_unreadable is not None + and spm_recording.not_recorded + ): print( "SPM availability was not recorded: " f"{spm_recording.versions_unreadable}; /v2/model-catalog reports a " diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/routes.py b/policyengine_api/fastapi_routes/v2/household_calculations/routes.py index 5d8423680..2304c802e 100644 --- a/policyengine_api/fastapi_routes/v2/household_calculations/routes.py +++ b/policyengine_api/fastapi_routes/v2/household_calculations/routes.py @@ -150,19 +150,24 @@ def submit() -> HouseholdCalculationSubmittedResponse | JSONResponse: return v2_missing_parameters_response( str(error), error.missing_parameters ) - # The submission bound the observability ID it sent the simulation - # service in this worker thread's copy of the request context, - # which the response middleware doesn't see, so the 202 names it. - observability_id = current_observability_id() - if observability_id is not None: - response.headers[OBSERVABILITY_ID_HEADER] = observability_id return HouseholdCalculationSubmittedResponse( result=HouseholdCalculationSubmittedResult( item=HouseholdCalculationSubmittedItem.from_submitted(calculation) ) ) - return _calculation_operation(submit) + outcome = _calculation_operation(submit) + # A submission that reached the simulation call bound the observability + # ID it sent in this worker thread's copy of the request context, which + # the response middleware doesn't see, so every answer from then on + # names it. A refusal before the call binds none. + observability_id = current_observability_id() + if observability_id is not None: + headers = ( + outcome.headers if isinstance(outcome, Response) else response.headers + ) + headers[OBSERVABILITY_ID_HEADER] = observability_id + return outcome @router.get( "/household-calculations/{calculation_id}", diff --git a/policyengine_api/libs/simulation_household_calculations.py b/policyengine_api/libs/simulation_household_calculations.py index ec3d529d3..1f01fcb01 100644 --- a/policyengine_api/libs/simulation_household_calculations.py +++ b/policyengine_api/libs/simulation_household_calculations.py @@ -27,16 +27,20 @@ names): the service is unavailable (SimulationServiceUnavailableError), with its ``code`` and reason when it gives them, as it is for no answer, a refusal of another shape, a gateway token that cannot be fetched and - missing configuration. + missing configuration. No answer means a connection never made + (``ConnectError``, ``ConnectTimeout``, ``PoolTimeout``); once the request + may have been sent, a lost answer (a read timeout, a dropped connection) is + SimulationServiceAnswerUnreadableError, since the job may exist. Each submission carries the request's ID and observability ID headers. The observability ID is the request's own, else the caller's, else a new one, which ``submit`` binds in the current context once the Public API's checks have passed, as the Public API's own household calculations bind one when they accept a calculation; the simulation service passes it to the household -function. The route reads it from that context and returns it in the 202's -``X-PolicyEngine-Observability-Id`` header, since a synchronous route runs in -a copy of the request context that the response middleware doesn't see. +function. The route reads it from that context and returns it in the +``X-PolicyEngine-Observability-Id`` header of every answer given once the +call was attempted, since a synchronous route runs in a copy of the request +context that the response middleware doesn't see. Wiring the generated simulation client: ``post_household_calculation`` is the only code that sends the request, and it returns the status and body, which @@ -81,6 +85,9 @@ UNREADABLE_ACCEPTANCE = ( "The simulation service accepted the calculation, but its answer could not be read." ) +# Transport errors raised before the request could reach the service. After +# any other, the service may have received the submission and created the job. +CONNECTION_NEVER_MADE = (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) @dataclass(frozen=True) @@ -272,19 +279,28 @@ def submit( """Submit the calculation and return the job the service created. Raise SimulationServiceInvalidRequestError with the service's reasons - when it refuses the caller's data, SimulationServiceAnswerUnreadableError - when it accepted the calculation but its answer cannot be read, - SimulationServiceInternalError when it failed (500), and - SimulationServiceUnavailableError otherwise. + when it refuses the caller's data; SimulationServiceAnswerUnreadableError + when it accepted the calculation but its answer cannot be read, or its + answer was lost after the submission may have reached it; + SimulationServiceInternalError when it failed (500); and + SimulationServiceUnavailableError otherwise, including a connection + never made. """ start_observability_id() try: answer = self._send( self._http_client, household_calculation_request(calculation) ) - except (httpx.HTTPError, GatewayAuthError) as error: + except (*CONNECTION_NEVER_MADE, GatewayAuthError) as error: msg = "The simulation service could not be reached." raise SimulationServiceUnavailableError(msg) from error + except httpx.HTTPError as error: + msg = ( + "The simulation service may have received the submission, but " + f"its answer did not arrive ({type(error).__name__}). Check " + "whether it created the job before submitting again." + ) + raise SimulationServiceAnswerUnreadableError(msg) from error if answer.status_code == HTTPStatus.ACCEPTED: try: accepted = _AcceptedJob.model_validate_json(answer.content) diff --git a/policyengine_api/services/v2/household_calculations/types.py b/policyengine_api/services/v2/household_calculations/types.py index 713fe2e5c..74e2ddafe 100644 --- a/policyengine_api/services/v2/household_calculations/types.py +++ b/policyengine_api/services/v2/household_calculations/types.py @@ -337,10 +337,11 @@ class SimulationServiceUnavailableError(RuntimeError): class SimulationServiceAnswerUnreadableError(RuntimeError): - """The simulation service accepted the calculation, but its answer is unreadable. + """The simulation service received the calculation, or may have, but no answer. - The service answered 202, so it has already created the job and started - its call; submitting again would create another job. + Either it answered 202, so it has already created the job and started its + call, but its answer is unreadable; or its answer was lost after the + submission may have reached it. Submitting again could create another job. """ From 7e746792ebf2a2e4763462824ac49bfc7294b804 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:17:02 +0400 Subject: [PATCH 42/57] Test the SPM warning, proxy answers and transport-error headers (failing) - With the simulation service's versions unreadable, publication should log its "not recorded" warning only when some version is still unrecorded; it logs it whenever the versions can't be read. - A 502 or 504 from a proxy in front of sim may stand in for a lost answer after the job exists, so it should raise the 502 error rather than the 503 one. - The "is not unavailable" assertions, which could never fail, now check the status the route answers each error with. - The route returns the attempt's observability ID on the 503 for a connection never made and the 502 for a lost answer; those two tests check the existing behaviour and pass. Co-Authored-By: Claude Opus 5.5 --- .../test_simulation_household_calculations.py | 45 ++++++++++++++++--- .../unit/v2/test_catalog_spm_availability.py | 33 ++++++++++++++ ...t_household_calculation_database_routes.py | 30 +++++++++++++ 3 files changed, 103 insertions(+), 5 deletions(-) diff --git a/tests/unit/libs/test_simulation_household_calculations.py b/tests/unit/libs/test_simulation_household_calculations.py index c7ecc4f58..0734312cd 100644 --- a/tests/unit/libs/test_simulation_household_calculations.py +++ b/tests/unit/libs/test_simulation_household_calculations.py @@ -8,6 +8,9 @@ import httpx import pytest +from policyengine_api.fastapi_routes.v2.household_calculations.routes import ( + HOUSEHOLD_CALCULATION_ERROR_MAPPINGS, +) from policyengine_api.libs.gateway_auth import GatewayAuthError from policyengine_api.libs.simulation_household_calculations import ( HOUSEHOLD_CALCULATIONS_PATH, @@ -56,6 +59,15 @@ def configured_simulation_service(monkeypatch): monkeypatch.setenv("OLD_SIMULATION_GATEWAY_URL", "https://old.example.test") +def _route_status(error: Exception) -> int: + """Return the status the household calculation route answers ``error`` with.""" + return next( + mapping.status_code + for mapping in HOUSEHOLD_CALCULATION_ERROR_MAPPINGS + if isinstance(error, mapping.error_types) + ) + + @pytest.fixture(autouse=True) def restored_observability_id(): """Undo the observability ID each submission binds in the test's context.""" @@ -267,17 +279,40 @@ def test_an_internal_error_is_not_unavailable(body, message) -> None: with pytest.raises(SimulationServiceInternalError) as failed: service.client().submit(calculation_input()) - assert not isinstance(failed.value, SimulationServiceUnavailableError) + assert _route_status(failed.value) == 502 assert str(failed.value) == message -@pytest.mark.parametrize("status_code", [400, 401, 404, 502, 503]) +@pytest.mark.parametrize("status_code", [502, 504]) +def test_a_proxys_answer_in_place_of_the_services_is_not_unavailable( + status_code, +) -> None: + # A proxy in front of the service, such as Cloud Run's front end, may + # answer 502 or 504 after the service created the job. + service = RecordingService(status_code, b"upstream request timeout") + + with pytest.raises(SimulationServiceAnswerUnreadableError) as lost: + service.client().submit(calculation_input()) + + assert _route_status(lost.value) == 502 + assert str(lost.value) == ( + "The simulation service may have received the submission, but " + f"{status_code} was answered in its place. Check whether it created the " + "job before submitting again." + ) + + +@pytest.mark.parametrize("status_code", [400, 401, 404, 503]) def test_any_other_answer_is_unavailable(status_code) -> None: service = RecordingService(status_code, b"") - with pytest.raises(SimulationServiceUnavailableError, match=str(status_code)): + with pytest.raises( + SimulationServiceUnavailableError, match=str(status_code) + ) as unavailable: service.client().submit(calculation_input()) + assert _route_status(unavailable.value) == 503 + @pytest.mark.parametrize( "body", @@ -296,7 +331,7 @@ def test_an_accepted_answer_that_cannot_be_read_is_not_unavailable(body) -> None with pytest.raises(SimulationServiceAnswerUnreadableError) as unreadable: service.client().submit(calculation_input()) - assert not isinstance(unreadable.value, SimulationServiceUnavailableError) + assert _route_status(unreadable.value) == 502 assert str(unreadable.value) == ( "The simulation service accepted the calculation, but its answer " "could not be read." @@ -337,7 +372,7 @@ def test_an_answer_lost_after_sending_is_not_unavailable(error_type) -> None: with pytest.raises(SimulationServiceAnswerUnreadableError) as lost: _client_raising(error_type).submit(calculation_input()) - assert not isinstance(lost.value, SimulationServiceUnavailableError) + assert _route_status(lost.value) == 502 assert str(lost.value) == ( "The simulation service may have received the submission, but its " f"answer did not arrive ({error_type.__name__}). Check whether it " diff --git a/tests/unit/v2/test_catalog_spm_availability.py b/tests/unit/v2/test_catalog_spm_availability.py index abe78a507..d9eb9bb2a 100644 --- a/tests/unit/v2/test_catalog_spm_availability.py +++ b/tests/unit/v2/test_catalog_spm_availability.py @@ -285,3 +285,36 @@ def unreadable(_url: str) -> object: not_recorded=(("uk", "1.0.0"),), versions_unreadable="the simulation service's versions could not be read", ) + + +@pytest.mark.parametrize( + ("still_unrecorded", "warnings_expected"), + [(("policyengine-uk",), 1), ((), 0)], + ids=["some-unrecorded", "all-recorded-earlier"], +) +def test_an_unreadable_versions_answer_warns_only_when_versions_are_unrecorded( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + still_unrecorded: tuple[str, ...], + warnings_expected: int, +) -> None: + _store_unrecorded(monkeypatch, *still_unrecorded) + + def unreadable(_url: str) -> object: + msg = "the simulation service's versions could not be read" + raise CatalogPublicationError(msg) + + with caplog.at_level("WARNING", logger=spm_availability.LOGGER.name): + spm_availability.record_published_spm_availability( + RecordingEngine(), + _catalog("uk", "us"), + {"SIMULATION_ENTRYPOINT_URL": "https://simulation.example"}, + fetch_versions=unreadable, + ) + + warnings = [ + record.getMessage() + for record in caplog.records + if record.name == spm_availability.LOGGER.name + ] + assert len(warnings) == warnings_expected diff --git a/tests/unit/v2/test_household_calculation_database_routes.py b/tests/unit/v2/test_household_calculation_database_routes.py index abdb689a5..7173b9b0b 100644 --- a/tests/unit/v2/test_household_calculation_database_routes.py +++ b/tests/unit/v2/test_household_calculation_database_routes.py @@ -272,9 +272,13 @@ def __init__(self) -> None: self.requests: list[httpx.Request] = [] self.status_code = 202 self.content = json.dumps({"job_id": str(JOB_ID), "status": "pending"}) + self.transport_error: type[httpx.TransportError] | None = None def answer(self, request: httpx.Request) -> httpx.Response: self.requests.append(request) + if self.transport_error is not None: + msg = "the transport failed" + raise self.transport_error(msg, request=request) return httpx.Response(self.status_code, content=self.content) @@ -423,3 +427,29 @@ def test_an_answer_after_sim_was_called_returns_the_observability_id( response.headers[OBSERVABILITY_ID_HEADER] == (sent.headers[OBSERVABILITY_ID_HEADER]) ) + + +@pytest.mark.parametrize( + ("transport_error", "status_code"), + [(httpx.ConnectError, 503), (httpx.ReadTimeout, 502)], + ids=["connection-never-made", "answer-lost"], +) +def test_a_failed_attempt_returns_the_attempts_observability_id( + client_with_simulation_client, + stored, + simulation_recorder, + transport_error, + status_code, +) -> None: + simulation_recorder.transport_error = transport_error + + response = client_with_simulation_client.post( + CALCULATIONS, json=_body(stored, None) + ) + + assert response.status_code == status_code + (attempted,) = simulation_recorder.requests + assert ( + response.headers[OBSERVABILITY_ID_HEADER] + == (attempted.headers[OBSERVABILITY_ID_HEADER]) + ) From f4bd858a29c59a8bbab0b11e167214b80248c062 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:19:28 +0400 Subject: [PATCH 43/57] Warn only when SPM availability stays unrecorded, and answer proxy 502s and 504s with 502 - Publication logs that SPM availability was not recorded only when some version is still unrecorded, after reading which are. - A 502 or 504 from sim, which only a proxy in front of it answers, now raises SimulationServiceAnswerUnreadableError (502): the job may exist. - The OpenAPI 502 description and the Stage 11 notes list the lost-answer and proxy cases, the Stage 11 notes no longer list "no answer" as 503, and the header text says it carries the attempt's ID, which sim never received after a connection never made or a token failure. - The warning test keeps the module's logger enabled, since Alembic's logging setup, which another test may run in the same process, disables loggers that existed before it. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-11-v2-households.md | 23 +++++++++++-------- .../data/v2/catalog/spm_availability.py | 11 +++++---- .../household_calculations/response_models.py | 6 +++-- .../libs/simulation_household_calculations.py | 20 ++++++++++++++-- .../unit/v2/test_catalog_spm_availability.py | 3 +++ 5 files changed, 44 insertions(+), 19 deletions(-) diff --git a/docs/migration/stage-11-v2-households.md b/docs/migration/stage-11-v2-households.md index f79b88273..52cffccc6 100644 --- a/docs/migration/stage-11-v2-households.md +++ b/docs/migration/stage-11-v2-households.md @@ -323,16 +323,19 @@ answer is 202 with the new job's `calculation_id` and `status`; each submission creates a new job. The simulation service's 422 reasons reach the caller as 422. A 202 whose body the Public API can't read is 502: the job exists, so submitting again would create another. So is a 500 from the -simulation service (`internal_error`), and an answer lost after the -submission may have been sent (a read timeout or a dropped connection), -since either may come after it created the job; a connection never made is -503. Every answer given once the call was attempted carries -`X-PolicyEngine-Observability-Id` with the ID sent to the simulation service; -a refusal before it carries none, not even the caller's. Any other refusal (with -the simulation service's `code` and reason in the message, and the failed -job's ID when Modal refused the call), no answer, a refusal the Public API -can't read, missing configuration or a gateway token it can't fetch is 503. -A failed catalog query in the policy check is 503 too. +simulation service (`internal_error`), an answer lost after the submission +may have been sent (a read timeout or a dropped connection), and a 502 or +504 a proxy in front of the simulation service answered in its place, since +each may come after it created the job. Any other refusal (with the +simulation service's `code` and reason in the message, and the failed job's +ID when Modal refused the call), a connection never made, a refusal the +Public API can't read, missing configuration or a gateway token it can't +fetch is 503. A failed catalog query in the policy check is 503 too. Every +answer given once the call was attempted carries +`X-PolicyEngine-Observability-Id` with the attempt's ID, which the +simulation service never received after a connection never made or a token +that couldn't be fetched; a refusal before the attempt carries none, not even +the caller's. `GET /v2/household-calculations/{calculation_id}` reads the job from `household_jobs`, which the simulation service inserts and its household diff --git a/policyengine_api/data/v2/catalog/spm_availability.py b/policyengine_api/data/v2/catalog/spm_availability.py index 41ec54df9..acc30ef9c 100644 --- a/policyengine_api/data/v2/catalog/spm_availability.py +++ b/policyengine_api/data/v2/catalog/spm_availability.py @@ -215,13 +215,14 @@ def record_published_spm_availability( try: versions = fetch_versions(url) except CatalogPublicationError as error: - LOGGER.warning( - "SPM availability was not recorded; the next publication run " - "tries again: %s", - error, - ) with engine.begin() as connection: not_recorded = _still_unrecorded(connection, catalog) + if not_recorded: + LOGGER.warning( + "SPM availability was not recorded; the next publication run " + "tries again: %s", + error, + ) return SPMAvailabilityRecording( not_recorded=not_recorded, versions_unreadable=str(error) ) diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py index e924ea8d2..0bce3e64e 100644 --- a/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py +++ b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py @@ -182,8 +182,10 @@ class HouseholdCalculationResponse( "model": V2ErrorResponse, "description": ( "The simulation service accepted the calculation, so its job exists, " - "but its answer cannot be read; or it failed (500), possibly after " - "creating the job. Submitting again could create another job." + "but its answer cannot be read; it failed (500); its answer was lost " + "after the submission may have reached it; or a proxy in front of it " + "answered 502 or 504 in its place. In each case it may have created " + "the job, so submitting again could create another." ), }, 422: { diff --git a/policyengine_api/libs/simulation_household_calculations.py b/policyengine_api/libs/simulation_household_calculations.py index 1f01fcb01..dee43e6a9 100644 --- a/policyengine_api/libs/simulation_household_calculations.py +++ b/policyengine_api/libs/simulation_household_calculations.py @@ -30,7 +30,8 @@ missing configuration. No answer means a connection never made (``ConnectError``, ``ConnectTimeout``, ``PoolTimeout``); once the request may have been sent, a lost answer (a read timeout, a dropped connection) is - SimulationServiceAnswerUnreadableError, since the job may exist. + SimulationServiceAnswerUnreadableError, since the job may exist, and so is + a 502 or 504 a proxy in front of the service answers in its place. Each submission carries the request's ID and observability ID headers. The observability ID is the request's own, else the caller's, else a new one, @@ -40,7 +41,9 @@ function. The route reads it from that context and returns it in the ``X-PolicyEngine-Observability-Id`` header of every answer given once the call was attempted, since a synchronous route runs in a copy of the request -context that the response middleware doesn't see. +context that the response middleware doesn't see. It is the attempt's ID, +which the service never received after a connection never made or a gateway +token that couldn't be fetched. Wiring the generated simulation client: ``post_household_calculation`` is the only code that sends the request, and it returns the status and body, which @@ -85,6 +88,12 @@ UNREADABLE_ACCEPTANCE = ( "The simulation service accepted the calculation, but its answer could not be read." ) +# The service never answers 502 or 504 itself: a proxy in front of it, such as +# Cloud Run's front end, answers them in its place, possibly after the +# service created the job. +PROXY_STAND_IN_STATUSES = frozenset( + {HTTPStatus.BAD_GATEWAY, HTTPStatus.GATEWAY_TIMEOUT} +) # Transport errors raised before the request could reach the service. After # any other, the service may have received the submission and created the job. CONNECTION_NEVER_MADE = (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) @@ -318,6 +327,13 @@ def submit( raise SimulationServiceInvalidRequestError(_invalid_request_reason(refusal)) if answer.status_code == HTTPStatus.INTERNAL_SERVER_ERROR: raise SimulationServiceInternalError(_internal_error(refusal)) + if answer.status_code in PROXY_STAND_IN_STATUSES: + msg = ( + "The simulation service may have received the submission, but " + f"{answer.status_code} was answered in its place. Check whether " + "it created the job before submitting again." + ) + raise SimulationServiceAnswerUnreadableError(msg) raise SimulationServiceUnavailableError(_unavailable(answer, refusal)) diff --git a/tests/unit/v2/test_catalog_spm_availability.py b/tests/unit/v2/test_catalog_spm_availability.py index d9eb9bb2a..89254a874 100644 --- a/tests/unit/v2/test_catalog_spm_availability.py +++ b/tests/unit/v2/test_catalog_spm_availability.py @@ -299,6 +299,9 @@ def test_an_unreadable_versions_answer_warns_only_when_versions_are_unrecorded( warnings_expected: int, ) -> None: _store_unrecorded(monkeypatch, *still_unrecorded) + # Alembic's logging setup, which another test may have run in this + # process, disables loggers that existed before it. + monkeypatch.setattr(spm_availability.LOGGER, "disabled", False) def unreadable(_url: str) -> object: msg = "the simulation service's versions could not be read" From f51e6589db896a6e56cb5c2547a2bbfd685712c9 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:27:29 +0400 Subject: [PATCH 44/57] Test that every 502 asks the caller to submit the household calculation again (failing) The Public API won't choose household job IDs, and no route finds a job without its ID, so a 502 can't ask the caller to check whether the job exists. The tests expect each 502 to ask for the calculation to be submitted again and to say this creates a second job if the first request went through, keeping each case's detail: the transport error, the proxy's status, sim's 500 code and reason, or the unreadable 202. Co-Authored-By: Claude Opus 5.5 --- .../test_simulation_household_calculations.py | 53 ++++++++++--------- .../v2/test_household_calculation_routes.py | 11 ++-- 2 files changed, 35 insertions(+), 29 deletions(-) diff --git a/tests/unit/libs/test_simulation_household_calculations.py b/tests/unit/libs/test_simulation_household_calculations.py index 0734312cd..843c7f5d3 100644 --- a/tests/unit/libs/test_simulation_household_calculations.py +++ b/tests/unit/libs/test_simulation_household_calculations.py @@ -47,6 +47,19 @@ ACCEPTED = {"job_id": str(JOB_ID), "status": "pending"} +def lost_answer_message(what_happened: str) -> str: + """Return the 502 message for an answer lost after the request may have arrived. + + No route finds a household job without its ID, so the caller is asked to + submit again and told that this may create a second job. + """ + return ( + "The simulation service may have received the request, but its answer " + f"was lost ({what_happened}). Submit the household calculation again; " + "if the first request went through, this creates a second job." + ) + + @pytest.fixture(autouse=True) def configured_simulation_service(monkeypatch): """Name the Cloud Run simulation service; set no gateway credentials.""" @@ -258,22 +271,23 @@ def test_a_dependency_the_service_cannot_read_names_its_code() -> None: "detail": "The service could not build the household function's " "request.", }, - "The simulation service failed (internal_error): The service could " - "not build the household function's request. It may have created " - "the job, so submitting again could create another.", + lost_answer_message( + "500 internal_error: The service could not build the household " + "function's request" + ), id="internal-error", ), pytest.param( - b"", - "The simulation service answered 500. It may have created the job, " - "so submitting again could create another.", - id="unreadable", + {"detail": "The service stopped."}, + lost_answer_message("500: The service stopped"), + id="no-code", ), + pytest.param(b"", lost_answer_message("500"), id="unreadable"), ], ) def test_an_internal_error_is_not_unavailable(body, message) -> None: - # A 500 may come after the service created the job, so the caller is not - # invited to submit again. + # A 500 may come after the service created the job, so the caller is told + # that submitting again may create a second job. service = RecordingService(500, body) with pytest.raises(SimulationServiceInternalError) as failed: @@ -295,11 +309,7 @@ def test_a_proxys_answer_in_place_of_the_services_is_not_unavailable( service.client().submit(calculation_input()) assert _route_status(lost.value) == 502 - assert str(lost.value) == ( - "The simulation service may have received the submission, but " - f"{status_code} was answered in its place. Check whether it created the " - "job before submitting again." - ) + assert str(lost.value) == lost_answer_message(f"a proxy answered {status_code}") @pytest.mark.parametrize("status_code", [400, 401, 404, 503]) @@ -325,17 +335,14 @@ def test_any_other_answer_is_unavailable(status_code) -> None: ) def test_an_accepted_answer_that_cannot_be_read_is_not_unavailable(body) -> None: # The service has created the job and started its call, so the caller is - # told the answer was unreadable rather than to try again. + # told that submitting again creates a second job. service = RecordingService(202, body) with pytest.raises(SimulationServiceAnswerUnreadableError) as unreadable: service.client().submit(calculation_input()) assert _route_status(unreadable.value) == 502 - assert str(unreadable.value) == ( - "The simulation service accepted the calculation, but its answer " - "could not be read." - ) + assert str(unreadable.value) == lost_answer_message("202 with an unreadable body") def _client_raising(error_type: type[httpx.TransportError]): @@ -368,16 +375,12 @@ def test_a_connection_never_made_is_unavailable(error_type) -> None: ) def test_an_answer_lost_after_sending_is_not_unavailable(error_type) -> None: # The service may have received the submission and created the job, so - # the caller is not invited to submit again. + # the caller is told that submitting again may create a second job. with pytest.raises(SimulationServiceAnswerUnreadableError) as lost: _client_raising(error_type).submit(calculation_input()) assert _route_status(lost.value) == 502 - assert str(lost.value) == ( - "The simulation service may have received the submission, but its " - f"answer did not arrive ({error_type.__name__}). Check whether it " - "created the job before submitting again." - ) + assert str(lost.value) == lost_answer_message(error_type.__name__) def test_a_gateway_token_that_cannot_be_fetched_is_unavailable() -> None: diff --git a/tests/unit/v2/test_household_calculation_routes.py b/tests/unit/v2/test_household_calculation_routes.py index c052a549f..2d4ac507d 100644 --- a/tests/unit/v2/test_household_calculation_routes.py +++ b/tests/unit/v2/test_household_calculation_routes.py @@ -261,12 +261,12 @@ def test_a_malformed_request_is_refused_before_anything_is_read( ), ( SimulationServiceAnswerUnreadableError( - "The simulation service accepted the calculation, but its " - "answer could not be read." + "The simulation service may have received the request, but its " + "answer was lost (ReadTimeout)." ), 502, - "The simulation service accepted the calculation, but its " - "answer could not be read.", + "The simulation service may have received the request, but its " + "answer was lost (ReadTimeout).", ), ( SimulationServiceInternalError("The simulation service failed."), @@ -433,6 +433,9 @@ def test_openapi_documents_both_routes_and_the_missing_parameter_refusal( read = schema["paths"][f"{CALCULATIONS}/{{calculation_id}}"]["get"] assert set(submit["responses"]) >= {"202", "404", "422", "500", "502", "503"} assert set(read["responses"]) >= {"200", "404", "422", "500", "503"} + lost_answer = submit["responses"]["502"]["description"] + assert "Submit the household calculation again" in lost_answer + assert "creates a second job" in lost_answer assert "security" not in submit refusal_schema = submit["responses"]["422"]["content"]["application/json"]["schema"] refusals = {option["$ref"].rsplit("/", 1)[-1] for option in refusal_schema["anyOf"]} From 200b5279a275dbb4651154dad77d45648e8917a7 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:29:08 +0400 Subject: [PATCH 45/57] Ask the caller to submit the household calculation again after a 502 No route finds a household job without its ID, so the 502 can no longer ask the caller to check whether the job exists. After an unreadable 202, sim's 500, an answer lost after sending, or a proxy's 502 or 504, the message now reads "The simulation service may have received the request, but its answer was lost (). Submit the household calculation again; if the first request went through, this creates a second job." The parentheses keep each case's detail: the transport error, the proxy's status, sim's 500 code and reason, or the unreadable 202. The OpenAPI 502 description, the error types' docstrings and the Stage 11 notes say the same. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-11-v2-households.md | 24 ++++--- .../household_calculations/response_models.py | 4 +- .../v2/household_calculations/routes.py | 3 +- .../libs/simulation_household_calculations.py | 69 +++++++++++-------- .../v2/household_calculations/types.py | 10 ++- 5 files changed, 65 insertions(+), 45 deletions(-) diff --git a/docs/migration/stage-11-v2-households.md b/docs/migration/stage-11-v2-households.md index 52cffccc6..00c7e6a64 100644 --- a/docs/migration/stage-11-v2-households.md +++ b/docs/migration/stage-11-v2-households.md @@ -321,16 +321,20 @@ It then sends the simulation service's `POST /household-calculations` the IDs and options, with the request's ID and observability ID headers. The answer is 202 with the new job's `calculation_id` and `status`; each submission creates a new job. The simulation service's 422 reasons reach the -caller as 422. A 202 whose body the Public API can't read is 502: the job -exists, so submitting again would create another. So is a 500 from the -simulation service (`internal_error`), an answer lost after the submission -may have been sent (a read timeout or a dropped connection), and a 502 or -504 a proxy in front of the simulation service answered in its place, since -each may come after it created the job. Any other refusal (with the -simulation service's `code` and reason in the message, and the failed job's -ID when Modal refused the call), a connection never made, a refusal the -Public API can't read, missing configuration or a gateway token it can't -fetch is 503. A failed catalog query in the policy check is 503 too. Every +caller as 422. A 202 whose body the Public API can't read is 502, since the +job exists. So is a 500 from the simulation service (`internal_error`), an +answer lost after the submission may have been sent (a read timeout or a +dropped connection), and a 502 or 504 a proxy in front of the simulation +service answered in its place, since each may come after it created the job. +No route finds a household job without its ID, so the 502's message asks the +caller to submit the household calculation again, and says that this +creates a second job if the first request went through; in parentheses it +names what happened (the transport error, the proxy's status, the simulation +service's 500 code and reason, or the unreadable 202). Any other refusal +(with the simulation service's `code` and reason in the message, and the +failed job's ID when Modal refused the call), a connection never made, a +refusal the Public API can't read, missing configuration or a gateway token +it can't fetch is 503. A failed catalog query in the policy check is 503 too. Every answer given once the call was attempted carries `X-PolicyEngine-Observability-Id` with the attempt's ID, which the simulation service never received after a connection never made or a token diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py index 0bce3e64e..3059f2087 100644 --- a/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py +++ b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py @@ -185,7 +185,9 @@ class HouseholdCalculationResponse( "but its answer cannot be read; it failed (500); its answer was lost " "after the submission may have reached it; or a proxy in front of it " "answered 502 or 504 in its place. In each case it may have created " - "the job, so submitting again could create another." + "the job, and no route finds a job without its ID. Submit the " + "household calculation again; if the first request went through, " + "this creates a second job. The message says what happened." ), }, 422: { diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/routes.py b/policyengine_api/fastapi_routes/v2/household_calculations/routes.py index 2304c802e..f56edd625 100644 --- a/policyengine_api/fastapi_routes/v2/household_calculations/routes.py +++ b/policyengine_api/fastapi_routes/v2/household_calculations/routes.py @@ -88,7 +88,8 @@ ), V2ErrorMapping((SimulationServiceUnavailableError,), 503), # The simulation service created the job, or may have, so this is not a - # 503, which invites the caller to submit again. + # 503: the message asks the caller to submit again and says this creates + # a second job if the first request went through. V2ErrorMapping( (SimulationServiceAnswerUnreadableError, SimulationServiceInternalError), 502 ), diff --git a/policyengine_api/libs/simulation_household_calculations.py b/policyengine_api/libs/simulation_household_calculations.py index dee43e6a9..6fc09cd58 100644 --- a/policyengine_api/libs/simulation_household_calculations.py +++ b/policyengine_api/libs/simulation_household_calculations.py @@ -13,14 +13,13 @@ settings only when the caller gave them. The simulation service answers: - 202 with the new job's ID and status. A 202 whose body cannot be read - is SimulationServiceAnswerUnreadableError: the job exists, so the caller - must not be invited to submit again; + is SimulationServiceAnswerUnreadableError: the job exists; - 422 with its reasons when it refuses the caller's data (a malformed request, or one its household contract refuses), one per field as FastAPI writes them: the reasons reach the caller as SimulationServiceInvalidRequestError; - 500 (``internal_error``) when it failed, possibly after creating the job: - SimulationServiceInternalError, which the caller must not resubmit after; + SimulationServiceInternalError; - anything else (400 for a household, policy or catalog version it cannot find, which the Public API checks first; 503 when it cannot read its manifest or database or Modal refuses the call, failing the job it @@ -33,6 +32,12 @@ SimulationServiceAnswerUnreadableError, since the job may exist, and so is a 502 or 504 a proxy in front of the service answers in its place. +No route finds a household job without its ID, so after an unreadable 202, a +500, a lost answer or a proxy's 502 or 504 the message asks the caller to +submit the calculation again, and says that this creates a second job if the +first request went through. Each message names what happened in place of a +readable answer. + Each submission carries the request's ID and observability ID headers. The observability ID is the request's own, else the caller's, else a new one, which ``submit`` binds in the current context once the Public API's checks @@ -85,9 +90,6 @@ HOUSEHOLD_CALCULATIONS_PATH = "/household-calculations" CLOUD_RUN_SIMULATION_ENTRYPOINT = "cloud_run_simulation_entrypoint" UNREADABLE_ANSWER = "The simulation service's answer could not be read." -UNREADABLE_ACCEPTANCE = ( - "The simulation service accepted the calculation, but its answer could not be read." -) # The service never answers 502 or 504 itself: a proxy in front of it, such as # Cloud Run's front end, answers them in its place, possibly after the # service created the job. @@ -99,6 +101,21 @@ CONNECTION_NEVER_MADE = (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) +def _lost_answer(what_happened: str) -> str: + """Return the message for an answer lost after the request may have arrived. + + The service may have created the job, and no route finds a household job + without its ID, so the message says that submitting again creates a + second job if the first request went through. ``what_happened`` names + what came in place of a readable answer. + """ + return ( + "The simulation service may have received the request, but its answer " + f"was lost ({what_happened}). Submit the household calculation again; " + "if the first request went through, this creates a second job." + ) + + @dataclass(frozen=True) class SimulationAnswer: """The simulation service's answer: its HTTP status and body.""" @@ -214,19 +231,17 @@ def _refusal(answer: SimulationAnswer) -> _Refusal | None: return None -RESUBMISSION_WARNING = ( - "It may have created the job, so submitting again could create another." -) - +def _internal_error(answer: SimulationAnswer, refusal: _Refusal | None) -> str: + """Describe the service's failure, which may have created the job. -def _internal_error(refusal: _Refusal | None) -> str: - """Describe the service's failure, which may have created the job.""" + The message names the status, with the service's code and reason when it + gives them. + """ if refusal is None or not isinstance(refusal.detail, str): - return f"The simulation service answered 500. {RESUBMISSION_WARNING}" - code = "" if refusal.code is None else f" ({refusal.code})" - return ( - f"The simulation service failed{code}: {refusal.detail} {RESUBMISSION_WARNING}" - ) + return _lost_answer(str(answer.status_code)) + code = "" if refusal.code is None else f" {refusal.code}" + reason = refusal.detail.strip().rstrip(".") + return _lost_answer(f"{answer.status_code}{code}: {reason}") def _unavailable(answer: SimulationAnswer, refusal: _Refusal | None) -> str: @@ -304,18 +319,15 @@ def submit( msg = "The simulation service could not be reached." raise SimulationServiceUnavailableError(msg) from error except httpx.HTTPError as error: - msg = ( - "The simulation service may have received the submission, but " - f"its answer did not arrive ({type(error).__name__}). Check " - "whether it created the job before submitting again." - ) - raise SimulationServiceAnswerUnreadableError(msg) from error + raise SimulationServiceAnswerUnreadableError( + _lost_answer(type(error).__name__) + ) from error if answer.status_code == HTTPStatus.ACCEPTED: try: accepted = _AcceptedJob.model_validate_json(answer.content) except ValidationError as error: raise SimulationServiceAnswerUnreadableError( - UNREADABLE_ACCEPTANCE + _lost_answer(f"{answer.status_code} with an unreadable body") ) from error return HouseholdCalculationAccepted( calculation_id=accepted.job_id, status=accepted.status @@ -326,14 +338,11 @@ def submit( raise SimulationServiceUnavailableError(UNREADABLE_ANSWER) raise SimulationServiceInvalidRequestError(_invalid_request_reason(refusal)) if answer.status_code == HTTPStatus.INTERNAL_SERVER_ERROR: - raise SimulationServiceInternalError(_internal_error(refusal)) + raise SimulationServiceInternalError(_internal_error(answer, refusal)) if answer.status_code in PROXY_STAND_IN_STATUSES: - msg = ( - "The simulation service may have received the submission, but " - f"{answer.status_code} was answered in its place. Check whether " - "it created the job before submitting again." + raise SimulationServiceAnswerUnreadableError( + _lost_answer(f"a proxy answered {answer.status_code}") ) - raise SimulationServiceAnswerUnreadableError(msg) raise SimulationServiceUnavailableError(_unavailable(answer, refusal)) diff --git a/policyengine_api/services/v2/household_calculations/types.py b/policyengine_api/services/v2/household_calculations/types.py index 74e2ddafe..928d01f4d 100644 --- a/policyengine_api/services/v2/household_calculations/types.py +++ b/policyengine_api/services/v2/household_calculations/types.py @@ -341,15 +341,19 @@ class SimulationServiceAnswerUnreadableError(RuntimeError): Either it answered 202, so it has already created the job and started its call, but its answer is unreadable; or its answer was lost after the - submission may have reached it. Submitting again could create another job. + submission may have reached it. The message asks the caller to submit + again, and says this creates a second job if the first request went + through. """ class SimulationServiceInternalError(RuntimeError): """The simulation service failed (500), possibly after creating the job. - Submitting again could create another job, so this is not reported as - the service being unavailable. + It is not reported as the service being unavailable: like + SimulationServiceAnswerUnreadableError, its message asks the caller to + submit again and says this creates a second job if the first request went + through. """ From ad91764f8a087cae48a7bd6f4e6b8dfad02e0c34 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:34:29 +0400 Subject: [PATCH 46/57] Name every 502 case in the submit docstrings, and say why they aren't 503s Both submit docstrings now list a proxy's 502 or 504 with the unreadable 202 and the lost answer, and the protocol's docstring no longer leaves those to "any other refusal or failure". The 500 error's docstring and the route's comment say directly that a 502 is used because the job may already exist, so submitting again may create a second one. Co-Authored-By: Claude Opus 5.5 --- .../fastapi_routes/v2/household_calculations/routes.py | 7 ++++--- .../libs/simulation_household_calculations.py | 5 +++-- .../services/v2/household_calculations/types.py | 10 +++++++--- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/routes.py b/policyengine_api/fastapi_routes/v2/household_calculations/routes.py index f56edd625..74ed0ebec 100644 --- a/policyengine_api/fastapi_routes/v2/household_calculations/routes.py +++ b/policyengine_api/fastapi_routes/v2/household_calculations/routes.py @@ -87,9 +87,10 @@ 404, ), V2ErrorMapping((SimulationServiceUnavailableError,), 503), - # The simulation service created the job, or may have, so this is not a - # 503: the message asks the caller to submit again and says this creates - # a second job if the first request went through. + # The simulation service created the job, or may have, so submitting + # again may create a second one; this is why it is not a 503. The message + # asks the caller to submit again and says this creates a second job if + # the first request went through. V2ErrorMapping( (SimulationServiceAnswerUnreadableError, SimulationServiceInternalError), 502 ), diff --git a/policyengine_api/libs/simulation_household_calculations.py b/policyengine_api/libs/simulation_household_calculations.py index 6fc09cd58..9144deeba 100644 --- a/policyengine_api/libs/simulation_household_calculations.py +++ b/policyengine_api/libs/simulation_household_calculations.py @@ -304,8 +304,9 @@ def submit( Raise SimulationServiceInvalidRequestError with the service's reasons when it refuses the caller's data; SimulationServiceAnswerUnreadableError - when it accepted the calculation but its answer cannot be read, or its - answer was lost after the submission may have reached it; + when it accepted the calculation but its answer cannot be read, its + answer was lost after the submission may have reached it, or a proxy in + front of it answered 502 or 504 in its place; SimulationServiceInternalError when it failed (500); and SimulationServiceUnavailableError otherwise, including a connection never made. diff --git a/policyengine_api/services/v2/household_calculations/types.py b/policyengine_api/services/v2/household_calculations/types.py index 928d01f4d..d6fce4557 100644 --- a/policyengine_api/services/v2/household_calculations/types.py +++ b/policyengine_api/services/v2/household_calculations/types.py @@ -319,10 +319,13 @@ def submit( Raise SimulationServiceInvalidRequestError when the service refuses the caller's data, with its reasons; SimulationServiceAnswerUnreadableError when it accepted the - calculation but its answer cannot be read; + calculation but its answer cannot be read, its answer was lost after + the submission may have reached it, or a proxy in front of it + answered 502 or 504 in its place; SimulationServiceInternalError when it failed (500), possibly after creating the job; and - SimulationServiceUnavailableError for any other refusal or failure. + SimulationServiceUnavailableError for any other refusal or failure, + including a connection never made. """ ... @@ -350,7 +353,8 @@ class SimulationServiceAnswerUnreadableError(RuntimeError): class SimulationServiceInternalError(RuntimeError): """The simulation service failed (500), possibly after creating the job. - It is not reported as the service being unavailable: like + The job may already exist, so submitting again may create a second one; + this is why it is not reported as the service being unavailable. Like SimulationServiceAnswerUnreadableError, its message asks the caller to submit again and says this creates a second job if the first request went through. From a10a733dc3254df7090c843483536322ff263e71 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:17:57 +0400 Subject: [PATCH 47/57] Test that the parallel live check's own clients send the CI server credential (failing) Master's deployed-API tests now send the CI server credential through the api_client fixture, since the Public API refuses requests without it once PUBLIC_API_SERVER_CREDENTIAL_REQUIRED=1. The parallel household check opens a client per calculation, so the tests expect a client of the deployed Public API that sends CI_SERVER_ACCESS_TOKEN, and that fails without it rather than skipping. Co-Authored-By: Claude Opus 5.5 --- ...est_live_household_calculation_requests.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/unit/v2/test_live_household_calculation_requests.py b/tests/unit/v2/test_live_household_calculation_requests.py index b8505f1b2..cbc0392cf 100644 --- a/tests/unit/v2/test_live_household_calculation_requests.py +++ b/tests/unit/v2/test_live_household_calculation_requests.py @@ -4,6 +4,9 @@ made: a 502 means the simulation service accepted it, and after a read timeout or a dropped response the job may exist. A read is sent again after a 502, 503 or 504 or any transport error, since reading creates nothing. + +A check that opens a client of its own sends the CI server credential, as the +``api_client`` fixture does. """ from __future__ import annotations @@ -19,6 +22,7 @@ LiveHousehold, calculate, ) +from tests.integration.deployed_api_credentials import MissingServerCredentialError HOUSEHOLD_ID = str(uuid4()) CALCULATION_ID = str(uuid4()) @@ -42,9 +46,11 @@ def __init__(self, submissions: list, reads: list) -> None: self.submissions = list(submissions) self.reads = list(reads) self.sent: list[str] = [] + self.authorizations: list[str | None] = [] def answer(self, request: httpx.Request) -> httpx.Response: self.sent.append(request.method) + self.authorizations.append(request.headers.get("Authorization")) script = self.submissions if request.method == "POST" else self.reads outcome = script.pop(0) if isinstance(outcome, type): @@ -130,3 +136,27 @@ def test_a_lasting_outage_fails_the_check(monkeypatch) -> None: public_api.calculate() assert public_api.sent == ["POST"] + + +def test_a_client_of_its_own_sends_the_ci_server_credential(monkeypatch) -> None: + # The parallel check gives each calculation a client of its own. Once the + # Public API requires a server credential, each must send it. + monkeypatch.setenv("CI_SERVER_ACCESS_TOKEN", "ci-server-token") + public_api = ScriptedPublicApi(submissions=[202], reads=[200]) + + with live_household_calculations.deployed_api_client( + "https://public-api.example", transport=httpx.MockTransport(public_api.answer) + ) as client: + calculate(client, HOUSEHOLD, HOUSEHOLD_ID) + + assert public_api.sent == ["POST", "GET"] + assert public_api.authorizations == ["Bearer ci-server-token"] * 2 + + +def test_a_client_of_its_own_fails_without_the_ci_server_credential( + monkeypatch, +) -> None: + monkeypatch.delenv("CI_SERVER_ACCESS_TOKEN", raising=False) + + with pytest.raises(MissingServerCredentialError): + live_household_calculations.deployed_api_client("https://public-api.example") From 40e55db9190ed534e77083a7d1d507a6d7bafb10 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:19:14 +0400 Subject: [PATCH 48/57] Send the CI server credential from the parallel live check's own clients The parallel household check opens a client per calculation, outside the api_client fixture that now sends the CI server credential. A new deployed_api_client in the live fixture builds the same client: the credential from CI_SERVER_ACCESS_TOKEN in Authorization, failing rather than skipping without it, the same timeout and redirects. The parallel check uses it, so it keeps passing once the Public API requires a server credential. Co-Authored-By: Claude Opus 5.5 --- .../live_household_calculations.py | 26 +++++++++++++++++++ .../test_live_v2_household_calculations.py | 5 ++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/tests/fixtures/integration/live_household_calculations.py b/tests/fixtures/integration/live_household_calculations.py index 2917dc0cd..8e4be3461 100644 --- a/tests/fixtures/integration/live_household_calculations.py +++ b/tests/fixtures/integration/live_household_calculations.py @@ -8,6 +8,10 @@ /household-calculations``, answering ``job_id`` and accepting ``axes``) is deployed to staging. Every staging promotion waits on them, so the change that adds them merges only after that deployment. + +Every request sends the CI server credential: through the ``api_client`` +fixture, or through ``deployed_api_client`` for a check that needs a client of +its own. """ from __future__ import annotations @@ -21,6 +25,8 @@ import httpx +from tests.integration.deployed_api_credentials import server_credential_headers + if TYPE_CHECKING: from collections.abc import Callable @@ -132,6 +138,26 @@ class FinishedCalculation: total_seconds: float +def deployed_api_client( + api_base_url: str, *, transport: httpx.BaseTransport | None = None +) -> httpx.Client: + """Return a client of the deployed Public API that sends the CI server credential. + + It sends what the ``api_client`` fixture's client sends, for a check that + needs a client of its own, such as one per parallel calculation. Raise + MissingServerCredentialError without ``CI_SERVER_ACCESS_TOKEN``, so the + check fails rather than skips. ``transport`` replaces the network, for + tests. + """ + return httpx.Client( + base_url=api_base_url, + headers=server_credential_headers(), + timeout=90.0, + follow_redirects=True, + transport=transport, + ) + + def probe_amount(integration_probe_id: str) -> int: """Return a small run-specific amount that makes each household new.""" return int(integration_probe_id.rsplit("-", 1)[-1], 16) % 997 diff --git a/tests/integration/test_live_v2_household_calculations.py b/tests/integration/test_live_v2_household_calculations.py index 0a6dd12c1..63b86d61f 100644 --- a/tests/integration/test_live_v2_household_calculations.py +++ b/tests/integration/test_live_v2_household_calculations.py @@ -26,13 +26,12 @@ import warnings from concurrent.futures import ThreadPoolExecutor -import httpx - from tests.fixtures.integration.live_household_calculations import ( YEAR, FinishedCalculation, LiveHousehold, calculate, + deployed_api_client, probe_amount, store_household, ) @@ -105,7 +104,7 @@ def test_parallel_calculations_finish_within_the_latency_limits( ] def run(household: LiveHousehold) -> FinishedCalculation: - with httpx.Client(base_url=api_base_url, timeout=90.0) as client: + with deployed_api_client(api_base_url) as client: return calculate(client, household, store_household(client, household)) with ThreadPoolExecutor(max_workers=PARALLEL_CALCULATIONS) as pool: From f995a64f018bd2541133e39a7f0521d6d00e23a2 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:26:15 +0400 Subject: [PATCH 49/57] Test that the credentialed client is built where the api_client fixture builds it (failing) The review found that deployed_api_client copied the api_client fixture's settings. The tests now expect the builder in tests/integration/deployed_api_credentials.py, which the fixture will use too, so both clients are built the same way. Co-Authored-By: Claude Opus 5.5 --- .../unit/v2/test_live_household_calculation_requests.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/unit/v2/test_live_household_calculation_requests.py b/tests/unit/v2/test_live_household_calculation_requests.py index cbc0392cf..5d7b4ec7c 100644 --- a/tests/unit/v2/test_live_household_calculation_requests.py +++ b/tests/unit/v2/test_live_household_calculation_requests.py @@ -5,8 +5,8 @@ timeout or a dropped response the job may exist. A read is sent again after a 502, 503 or 504 or any transport error, since reading creates nothing. -A check that opens a client of its own sends the CI server credential, as the -``api_client`` fixture does. +A check that opens a client of its own builds it as the ``api_client`` +fixture does, so it sends the CI server credential. """ from __future__ import annotations @@ -22,6 +22,7 @@ LiveHousehold, calculate, ) +from tests.integration import deployed_api_credentials from tests.integration.deployed_api_credentials import MissingServerCredentialError HOUSEHOLD_ID = str(uuid4()) @@ -144,7 +145,7 @@ def test_a_client_of_its_own_sends_the_ci_server_credential(monkeypatch) -> None monkeypatch.setenv("CI_SERVER_ACCESS_TOKEN", "ci-server-token") public_api = ScriptedPublicApi(submissions=[202], reads=[200]) - with live_household_calculations.deployed_api_client( + with deployed_api_credentials.deployed_api_client( "https://public-api.example", transport=httpx.MockTransport(public_api.answer) ) as client: calculate(client, HOUSEHOLD, HOUSEHOLD_ID) @@ -159,4 +160,4 @@ def test_a_client_of_its_own_fails_without_the_ci_server_credential( monkeypatch.delenv("CI_SERVER_ACCESS_TOKEN", raising=False) with pytest.raises(MissingServerCredentialError): - live_household_calculations.deployed_api_client("https://public-api.example") + deployed_api_credentials.deployed_api_client("https://public-api.example") From cf0590185272066dcbad9896a8d6d8586a4f5ebe Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:27:36 +0400 Subject: [PATCH 50/57] Build the api_client fixture's client and the parallel check's clients in one place deployed_api_client moves to tests/integration/deployed_api_credentials.py, and the api_client fixture builds its client with it, so a check that needs clients of its own sends exactly what the fixture sends. The live fixture's docstring no longer says it needs only httpx, since the live job installs the locked dev environment, and the testing skill names deployed_api_client for checks that need their own clients. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/skills/testing.md | 8 +++-- .../live_household_calculations.py | 30 ++----------------- tests/integration/conftest.py | 11 ++----- tests/integration/deployed_api_credentials.py | 24 +++++++++++++++ .../test_live_v2_household_calculations.py | 2 +- 5 files changed, 36 insertions(+), 39 deletions(-) diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index 0702fab0b..5e7b58ead 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -443,9 +443,11 @@ avoid depending on specific production data fixtures. The deployed-API tests send the CI server credential from `CI_SERVER_ACCESS_TOKEN` through the `api_client` fixture, and fail without it: a client-credentials token of a client on that environment's `PUBLIC_API_SERVER_CLIENT_IDS` -(`tests/integration/deployed_api_credentials.py`). Only refusal checks use -`uncredentialed_api_client`, which sends no credential. In push.yml each job that -sets `API_BASE_URL` requests one with +(`tests/integration/deployed_api_credentials.py`). A check that needs a client +of its own, such as one per parallel request, builds it with that module's +`deployed_api_client`, which also builds the fixture's client. Only refusal +checks use `uncredentialed_api_client`, which sends no credential. In push.yml +each job that sets `API_BASE_URL` requests one with `.github/scripts/request_ci_server_token.sh`, and `tests/unit/test_ci_server_credential_workflow.py` fails for a job that doesn't: diff --git a/tests/fixtures/integration/live_household_calculations.py b/tests/fixtures/integration/live_household_calculations.py index 8e4be3461..1e8d755d2 100644 --- a/tests/fixtures/integration/live_household_calculations.py +++ b/tests/fixtures/integration/live_household_calculations.py @@ -1,8 +1,5 @@ """Households and helpers for the live staging household calculation checks. -They need only httpx and the standard library, because the parallel live test -job installs nothing else. - The checks call the simulation service's household calculation route through the staging candidate, so they pass only once that route (``POST /household-calculations``, answering ``job_id`` and accepting ``axes``) is @@ -10,8 +7,9 @@ adds them merges only after that deployment. Every request sends the CI server credential: through the ``api_client`` -fixture, or through ``deployed_api_client`` for a check that needs a client of -its own. +fixture, or through ``deployed_api_client`` from +``tests/integration/deployed_api_credentials.py``, which builds that fixture's +client, for a check that needs a client of its own. """ from __future__ import annotations @@ -25,8 +23,6 @@ import httpx -from tests.integration.deployed_api_credentials import server_credential_headers - if TYPE_CHECKING: from collections.abc import Callable @@ -138,26 +134,6 @@ class FinishedCalculation: total_seconds: float -def deployed_api_client( - api_base_url: str, *, transport: httpx.BaseTransport | None = None -) -> httpx.Client: - """Return a client of the deployed Public API that sends the CI server credential. - - It sends what the ``api_client`` fixture's client sends, for a check that - needs a client of its own, such as one per parallel calculation. Raise - MissingServerCredentialError without ``CI_SERVER_ACCESS_TOKEN``, so the - check fails rather than skips. ``transport`` replaces the network, for - tests. - """ - return httpx.Client( - base_url=api_base_url, - headers=server_credential_headers(), - timeout=90.0, - follow_redirects=True, - transport=transport, - ) - - def probe_amount(integration_probe_id: str) -> int: """Return a small run-specific amount that makes each household new.""" return int(integration_probe_id.rsplit("-", 1)[-1], 16) % 997 diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 7f78de26e..2792eaeb3 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -8,7 +8,7 @@ from tests.integration.deployed_api_credentials import ( MissingServerCredentialError, - server_credential_headers, + deployed_api_client, ) INTEGRATION_TIMEOUT_SECONDS = float( @@ -169,15 +169,10 @@ def api_client(api_base_url: str): ``Authorization`` header on a redirect to another host. """ try: - credential_headers = server_credential_headers() + client = deployed_api_client(api_base_url) except MissingServerCredentialError as error: pytest.fail(str(error), pytrace=False) - with httpx.Client( - base_url=api_base_url, - headers=credential_headers, - timeout=90.0, - follow_redirects=True, - ) as client: + with client: yield client diff --git a/tests/integration/deployed_api_credentials.py b/tests/integration/deployed_api_credentials.py index bc11fb800..cbb2d047c 100644 --- a/tests/integration/deployed_api_credentials.py +++ b/tests/integration/deployed_api_credentials.py @@ -14,6 +14,10 @@ Since ``Authorization`` holds the server's own token, a signed-in user's token is forwarded in ``PolicyEngine-User-Authorization``, as a PolicyEngine server forwards a browser's (:func:`forwarded_user_headers`). + +:func:`deployed_api_client` builds the ``api_client`` fixture's client, and +the client of any check that needs one of its own, such as one per parallel +request. """ from __future__ import annotations @@ -21,6 +25,8 @@ import os from typing import TYPE_CHECKING +import httpx + if TYPE_CHECKING: from collections.abc import Mapping @@ -53,6 +59,24 @@ def server_credential_headers( return {"Authorization": f"Bearer {token}"} +def deployed_api_client( + api_base_url: str, *, transport: httpx.BaseTransport | None = None +) -> httpx.Client: + """Return a client of the deployed Public API that sends the CI server credential. + + Raise :class:`MissingServerCredentialError` without + ``CI_SERVER_ACCESS_TOKEN``. httpx drops the ``Authorization`` header on a + redirect to another host. ``transport`` replaces the network, for tests. + """ + return httpx.Client( + base_url=api_base_url, + headers=server_credential_headers(), + timeout=90.0, + follow_redirects=True, + transport=transport, + ) + + def forwarded_user_headers(user_token: str) -> dict[str, str]: """Return the header that forwards a signed-in user's access token.""" return {FORWARDED_USER_AUTHORIZATION_HEADER: f"Bearer {user_token}"} diff --git a/tests/integration/test_live_v2_household_calculations.py b/tests/integration/test_live_v2_household_calculations.py index 63b86d61f..a1159c630 100644 --- a/tests/integration/test_live_v2_household_calculations.py +++ b/tests/integration/test_live_v2_household_calculations.py @@ -31,10 +31,10 @@ FinishedCalculation, LiveHousehold, calculate, - deployed_api_client, probe_amount, store_household, ) +from tests.integration.deployed_api_credentials import deployed_api_client PARALLEL_CALCULATIONS = int(os.environ.get("STAGING_HOUSEHOLD_PARALLEL_COUNT", "6")) SUBMIT_LIMIT_SECONDS = float( From 1bccf663c9acbc7a2cb74f1a14478f9c925479f3 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:32:11 +0400 Subject: [PATCH 51/57] Say the parallel check opens one client per thread, and state the redirect rule once Co-Authored-By: Claude Opus 5.5 --- docs/engineering/skills/testing.md | 10 +++++----- tests/integration/conftest.py | 6 +++--- tests/integration/deployed_api_credentials.py | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index 5e7b58ead..52987b34a 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -443,11 +443,11 @@ avoid depending on specific production data fixtures. The deployed-API tests send the CI server credential from `CI_SERVER_ACCESS_TOKEN` through the `api_client` fixture, and fail without it: a client-credentials token of a client on that environment's `PUBLIC_API_SERVER_CLIENT_IDS` -(`tests/integration/deployed_api_credentials.py`). A check that needs a client -of its own, such as one per parallel request, builds it with that module's -`deployed_api_client`, which also builds the fixture's client. Only refusal -checks use `uncredentialed_api_client`, which sends no credential. In push.yml -each job that sets `API_BASE_URL` requests one with +(`tests/integration/deployed_api_credentials.py`). A check that needs clients +of its own, such as one per thread of a parallel check, builds them with that +module's `deployed_api_client`, which also builds the fixture's client. Only +refusal checks use `uncredentialed_api_client`, which sends no credential. In +push.yml each job that sets `API_BASE_URL` requests one with `.github/scripts/request_ci_server_token.sh`, and `tests/unit/test_ci_server_credential_workflow.py` fails for a job that doesn't: diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 2792eaeb3..9444a5af1 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -164,9 +164,9 @@ def api_base_url() -> str: def api_client(api_base_url: str): """Call the deployed Public API with the CI server credential on every request. - The test fails, rather than skips, without ``CI_SERVER_ACCESS_TOKEN`` - (``tests/integration/deployed_api_credentials.py``). httpx drops the - ``Authorization`` header on a redirect to another host. + The client is ``deployed_api_client``'s + (``tests/integration/deployed_api_credentials.py``). The test fails, rather + than skips, without ``CI_SERVER_ACCESS_TOKEN``. """ try: client = deployed_api_client(api_base_url) diff --git a/tests/integration/deployed_api_credentials.py b/tests/integration/deployed_api_credentials.py index cbb2d047c..f2bf140d8 100644 --- a/tests/integration/deployed_api_credentials.py +++ b/tests/integration/deployed_api_credentials.py @@ -16,8 +16,8 @@ forwards a browser's (:func:`forwarded_user_headers`). :func:`deployed_api_client` builds the ``api_client`` fixture's client, and -the client of any check that needs one of its own, such as one per parallel -request. +the client of any check that needs clients of its own, such as one per +thread of a parallel check. """ from __future__ import annotations From e324a2be5d06f055c76645b45e81fd039354bff3 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:36:52 +0400 Subject: [PATCH 52/57] Test that unexpected statuses answer 502 and that sim's 500 and 503 text stays out (failing) Sim may create the household job before answering a status the Public API doesn't otherwise handle (200, a redirect, 408, 409, 501 and the like), so the tests expect those to answer 502: "The simulation service may have created the household job, but it failed ( ).", as an unreadable 202 and sim's 500 do. Only answers sim gives before its route creates a job (401, 403, 404, 405, 413, 429 and 503), its 400, a connection never made and a gateway token that can't be fetched stay 503. Sim's 500 and 503 text may hold stored data, so only the status and a code that is a fixed identifier reach the caller; the 400 and 422 refusals keep their reasons. Co-Authored-By: Claude Opus 5.5 --- .../test_simulation_household_calculations.py | 131 +++++++++++++++--- ...t_household_calculation_database_routes.py | 5 + .../v2/test_household_calculation_routes.py | 1 + 3 files changed, 119 insertions(+), 18 deletions(-) diff --git a/tests/unit/libs/test_simulation_household_calculations.py b/tests/unit/libs/test_simulation_household_calculations.py index 843c7f5d3..4f96d307b 100644 --- a/tests/unit/libs/test_simulation_household_calculations.py +++ b/tests/unit/libs/test_simulation_household_calculations.py @@ -60,6 +60,19 @@ def lost_answer_message(what_happened: str) -> str: ) +def created_but_failed_message(status_and_code: str) -> str: + """Return the 502 message for an answer sent after the job may exist. + + It names the service's status, and its code when that is a fixed + identifier, never the service's text. + """ + return ( + "The simulation service may have created the household job, but it " + f"failed ({status_and_code}). Submit the household calculation again; " + "if the first request went through, this creates a second job." + ) + + @pytest.fixture(autouse=True) def configured_simulation_service(monkeypatch): """Name the Cloud Run simulation service; set no gateway credentials.""" @@ -229,7 +242,7 @@ def test_a_reference_the_service_cannot_find_is_unavailable_with_its_reason() -> ) -def test_a_modal_call_the_service_could_not_start_names_the_failed_job() -> None: +def test_a_refused_modal_call_gives_only_its_status_and_code() -> None: service = RecordingService( 503, { @@ -243,8 +256,7 @@ def test_a_modal_call_the_service_could_not_start_names_the_failed_job() -> None service.client().submit(calculation_input()) assert str(refused.value) == ( - "The simulation service is unavailable (modal_call_refused): Modal " - f"refused the household function's call; it failed household job {JOB_ID}" + "The simulation service is unavailable (503 modal_call_refused)." ) @@ -253,12 +265,49 @@ def test_a_dependency_the_service_cannot_read_names_its_code() -> None: 503, {"code": "database_unavailable", "detail": "The database is down"} ) + with pytest.raises(SimulationServiceUnavailableError) as refused: + service.client().submit(calculation_input()) + + # The service's text may hold stored data, so only its status and code + # reach the caller. + assert str(refused.value) == ( + "The simulation service is unavailable (503 database_unavailable)." + ) + + +@pytest.mark.parametrize( + ("code", "shown"), + [ + pytest.param("database_unavailable", "503 database_unavailable", id="kept"), + pytest.param("a" * 64, f"503 {'a' * 64}", id="64-characters"), + pytest.param("a" * 65, "503", id="65-characters"), + pytest.param("Database_unavailable", "503", id="capital"), + pytest.param("1st_failure", "503", id="leading-digit"), + pytest.param("_failure", "503", id="leading-underscore"), + pytest.param("database-unavailable", "503", id="hyphen"), + pytest.param("database unavailable: row 7", "503", id="text"), + pytest.param("", "503", id="empty"), + ], +) +def test_only_a_code_that_is_a_fixed_identifier_is_shown(code, shown) -> None: + service = RecordingService(503, {"code": code, "detail": "The database is down"}) + + with pytest.raises(SimulationServiceUnavailableError) as refused: + service.client().submit(calculation_input()) + + assert str(refused.value) == f"The simulation service is unavailable ({shown})." + + +def test_a_refusal_code_that_is_not_a_fixed_identifier_is_left_out() -> None: + service = RecordingService( + 400, {"code": "Unknown Household", "detail": "Household not found"} + ) + with pytest.raises(SimulationServiceUnavailableError) as refused: service.client().submit(calculation_input()) assert str(refused.value) == ( - "The simulation service is unavailable (database_unavailable): " - "The database is down" + "The simulation service refused the calculation: Household not found" ) @@ -271,23 +320,21 @@ def test_a_dependency_the_service_cannot_read_names_its_code() -> None: "detail": "The service could not build the household function's " "request.", }, - lost_answer_message( - "500 internal_error: The service could not build the household " - "function's request" - ), + created_but_failed_message("500 internal_error"), id="internal-error", ), pytest.param( {"detail": "The service stopped."}, - lost_answer_message("500: The service stopped"), + created_but_failed_message("500"), id="no-code", ), - pytest.param(b"", lost_answer_message("500"), id="unreadable"), + pytest.param(b"", created_but_failed_message("500"), id="unreadable"), ], ) def test_an_internal_error_is_not_unavailable(body, message) -> None: # A 500 may come after the service created the job, so the caller is told - # that submitting again may create a second job. + # that submitting again may create a second job. The service's text may + # hold stored data, so only its status and code reach the caller. service = RecordingService(500, body) with pytest.raises(SimulationServiceInternalError) as failed: @@ -312,16 +359,64 @@ def test_a_proxys_answer_in_place_of_the_services_is_not_unavailable( assert str(lost.value) == lost_answer_message(f"a proxy answered {status_code}") -@pytest.mark.parametrize("status_code", [400, 401, 404, 503]) -def test_any_other_answer_is_unavailable(status_code) -> None: +@pytest.mark.parametrize( + ("status_code", "message"), + [ + # The service's authentication, routing, body-size and rate limits + # answer these before its route creates a job. + *( + pytest.param( + status_code, + f"The simulation service is unavailable ({status_code}).", + id=str(status_code), + ) + for status_code in (401, 403, 404, 405, 413, 429, 503) + ), + pytest.param( + 400, "The simulation service refused the calculation (400).", id="400" + ), + ], +) +def test_an_answer_given_before_a_job_is_created_is_unavailable( + status_code, message +) -> None: service = RecordingService(status_code, b"") - with pytest.raises( - SimulationServiceUnavailableError, match=str(status_code) - ) as unavailable: + with pytest.raises(SimulationServiceUnavailableError) as unavailable: service.client().submit(calculation_input()) assert _route_status(unavailable.value) == 503 + assert str(unavailable.value) == message + + +@pytest.mark.parametrize( + ("status_code", "body", "status_and_code"), + [ + *( + pytest.param(status_code, b"", str(status_code), id=str(status_code)) + for status_code in (200, 201, 204, 302, 307, 408, 409, 410, 501) + ), + pytest.param( + 409, + {"code": "job_exists", "detail": "The household's income is 52000"}, + "409 job_exists", + id="409-with-code", + ), + ], +) +def test_an_unexpected_answer_may_have_created_the_job( + status_code, body, status_and_code +) -> None: + # The service may have created the job before answering a status the + # Public API does not expect, so the caller is not told that nothing was + # created. + service = RecordingService(status_code, body) + + with pytest.raises(SimulationServiceAnswerUnreadableError) as unexpected: + service.client().submit(calculation_input()) + + assert _route_status(unexpected.value) == 502 + assert str(unexpected.value) == created_but_failed_message(status_and_code) @pytest.mark.parametrize( @@ -342,7 +437,7 @@ def test_an_accepted_answer_that_cannot_be_read_is_not_unavailable(body) -> None service.client().submit(calculation_input()) assert _route_status(unreadable.value) == 502 - assert str(unreadable.value) == lost_answer_message("202 with an unreadable body") + assert str(unreadable.value) == created_but_failed_message("202") def _client_raising(error_type: type[httpx.TransportError]): diff --git a/tests/unit/v2/test_household_calculation_database_routes.py b/tests/unit/v2/test_household_calculation_database_routes.py index 7173b9b0b..215804a80 100644 --- a/tests/unit/v2/test_household_calculation_database_routes.py +++ b/tests/unit/v2/test_household_calculation_database_routes.py @@ -404,6 +404,11 @@ def test_a_refusal_before_sim_is_called_echoes_no_observability_id( (500, {"code": "internal_error", "detail": "failed"}), 502, id="failed" ), pytest.param((202, {"unexpected": True}), 502, id="unreadable-202"), + pytest.param( + (409, {"code": "job_exists", "detail": "exists"}), + 502, + id="unexpected-status", + ), ], ) def test_an_answer_after_sim_was_called_returns_the_observability_id( diff --git a/tests/unit/v2/test_household_calculation_routes.py b/tests/unit/v2/test_household_calculation_routes.py index 2d4ac507d..8d0c7b5a0 100644 --- a/tests/unit/v2/test_household_calculation_routes.py +++ b/tests/unit/v2/test_household_calculation_routes.py @@ -436,6 +436,7 @@ def test_openapi_documents_both_routes_and_the_missing_parameter_refusal( lost_answer = submit["responses"]["502"]["description"] assert "Submit the household calculation again" in lost_answer assert "creates a second job" in lost_answer + assert "a status the Public API does not expect" in lost_answer assert "security" not in submit refusal_schema = submit["responses"]["422"]["content"]["application/json"]["schema"] refusals = {option["$ref"].rsplit("/", 1)[-1] for option in refusal_schema["anyOf"]} From 756dddd32fce423999f523e34200e21dfa929829 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:39:33 +0400 Subject: [PATCH 53/57] Answer unexpected statuses 502, and keep sim's 500 and 503 text from the caller Sim may create the household job before answering a status the Public API doesn't otherwise handle, so any status other than 202, 400, 422, 500, a proxy's 502 or 504, and the ones sim answers before its route runs (401, 403, 404, 405, 413, 429 and 503) now answers 502: "The simulation service may have created the household job, but it failed ( ). Submit the household calculation again; if the first request went through, this creates a second job." An unreadable 202 and sim's 500 use the same first sentence; lost answers and a proxy's 502 or 504 keep "its answer was lost". Sim's text may hold stored data and the route needs no account, so after its 500 or 503, or an unexpected status, the message gives only the status and a code that is a fixed identifier (lower-case letters, digits and underscores, starting with a letter, at most 64 characters); the failed job's ID is no longer named either. The 400 and 422 refusals keep their reasons. The answer reading moves out of submit into one function. The OpenAPI 502 description, the error docstrings and the Stage 11 notes describe the new classes of answer. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-11-v2-households.md | 42 ++-- .../household_calculations/response_models.py | 15 +- .../libs/simulation_household_calculations.py | 232 ++++++++++++------ .../v2/household_calculations/types.py | 31 +-- 4 files changed, 208 insertions(+), 112 deletions(-) diff --git a/docs/migration/stage-11-v2-households.md b/docs/migration/stage-11-v2-households.md index 00c7e6a64..053febd73 100644 --- a/docs/migration/stage-11-v2-households.md +++ b/docs/migration/stage-11-v2-households.md @@ -321,20 +321,34 @@ It then sends the simulation service's `POST /household-calculations` the IDs and options, with the request's ID and observability ID headers. The answer is 202 with the new job's `calculation_id` and `status`; each submission creates a new job. The simulation service's 422 reasons reach the -caller as 422. A 202 whose body the Public API can't read is 502, since the -job exists. So is a 500 from the simulation service (`internal_error`), an -answer lost after the submission may have been sent (a read timeout or a -dropped connection), and a 502 or 504 a proxy in front of the simulation -service answered in its place, since each may come after it created the job. -No route finds a household job without its ID, so the 502's message asks the -caller to submit the household calculation again, and says that this -creates a second job if the first request went through; in parentheses it -names what happened (the transport error, the proxy's status, the simulation -service's 500 code and reason, or the unreadable 202). Any other refusal -(with the simulation service's `code` and reason in the message, and the -failed job's ID when Modal refused the call), a connection never made, a -refusal the Public API can't read, missing configuration or a gateway token -it can't fetch is 503. A failed catalog query in the policy check is 503 too. Every +caller as 422. + +Answers given before the simulation service's route creates a job are 503: +its 400 (a household, policy or catalog version it can't find, with its +reason), its 401, 403, 404, 405, 413 and 429 (its authentication, routing, +body-size and rate limits), its 503 (its manifest or database can't be read, +or Modal refused the call), a refusal the Public API can't read, a +connection never made, missing configuration and a gateway token it can't +fetch. A failed catalog query in the policy check is 503 too. + +Answers that may come after it created the job are 502: a 202 whose body the +Public API can't read, its 500 (`internal_error`), any status not named +above (such as 200, a redirect, 408, 409 or 501), an answer lost after the +submission may have been sent (a read timeout or a dropped connection), and +a 502 or 504 a proxy in front of it answered in its place. No route finds a +household job without its ID, so the 502's message asks the caller to submit +the household calculation again, and says that this creates a second job if +the first request went through. + +The simulation service's text may hold stored data, and the route needs no +account, so after its 500 or 503, or a status not named above, the message +gives only its status and its `code` when that is a fixed identifier +(lower-case letters, digits and underscores, starting with a letter, at most +64 characters), for example "The simulation service may have created the +household job, but it failed (500 internal_error)." The 400 and 422 refusals +keep its reasons, which name the caller's own household, policy and fields. +After a lost answer or a proxy's status, the message says "its answer was +lost" and names the transport error or the proxy's status. Every answer given once the call was attempted carries `X-PolicyEngine-Observability-Id` with the attempt's ID, which the simulation service never received after a connection never made or a token diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py index 3059f2087..e9bad6f4a 100644 --- a/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py +++ b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py @@ -182,12 +182,15 @@ class HouseholdCalculationResponse( "model": V2ErrorResponse, "description": ( "The simulation service accepted the calculation, so its job exists, " - "but its answer cannot be read; it failed (500); its answer was lost " - "after the submission may have reached it; or a proxy in front of it " - "answered 502 or 504 in its place. In each case it may have created " - "the job, and no route finds a job without its ID. Submit the " - "household calculation again; if the first request went through, " - "this creates a second job. The message says what happened." + "but its answer cannot be read; it failed (500); it answered a " + "status the Public API does not expect, such as 200, a redirect, " + "408, 409 or 501; its answer was lost after the submission may have " + "reached it; or a proxy in front of it answered 502 or 504 in its " + "place. In each case it may have created the job, and no route " + "finds a job without its ID. Submit the household calculation " + "again; if the first request went through, this creates a second " + "job. The message gives the simulation service's status and code, " + "or what came in place of its answer." ), }, 422: { diff --git a/policyengine_api/libs/simulation_household_calculations.py b/policyengine_api/libs/simulation_household_calculations.py index 9144deeba..0079c2a7f 100644 --- a/policyengine_api/libs/simulation_household_calculations.py +++ b/policyengine_api/libs/simulation_household_calculations.py @@ -10,33 +10,53 @@ The request names the country, the stored household and the stored policy (null for current law), ``add_missing_variables``, and the axes and SPM -settings only when the caller gave them. The simulation service answers: - -- 202 with the new job's ID and status. A 202 whose body cannot be read - is SimulationServiceAnswerUnreadableError: the job exists; -- 422 with its reasons when it refuses the caller's data (a malformed - request, or one its household contract refuses), one per field as - FastAPI writes them: the reasons reach the caller as - SimulationServiceInvalidRequestError; -- 500 (``internal_error``) when it failed, possibly after creating the job: - SimulationServiceInternalError; -- anything else (400 for a household, policy or catalog version it cannot - find, which the Public API checks first; 503 when it cannot read its - manifest or database or Modal refuses the call, failing the job it - names): the service is unavailable (SimulationServiceUnavailableError), - with its ``code`` and reason when it gives them, as it is for no answer, - a refusal of another shape, a gateway token that cannot be fetched and - missing configuration. No answer means a connection never made - (``ConnectError``, ``ConnectTimeout``, ``PoolTimeout``); once the request - may have been sent, a lost answer (a read timeout, a dropped connection) is - SimulationServiceAnswerUnreadableError, since the job may exist, and so is - a 502 or 504 a proxy in front of the service answers in its place. - -No route finds a household job without its ID, so after an unreadable 202, a -500, a lost answer or a proxy's 502 or 504 the message asks the caller to -submit the calculation again, and says that this creates a second job if the -first request went through. Each message names what happened in place of a -readable answer. +settings only when the caller gave them. The simulation service's answers +are read as follows. + +The job was created: + +- 202 with the new job's ID and status is the accepted job. + +Nothing was created, so the service is unavailable +(SimulationServiceUnavailableError, which the route answers 503): + +- 400 when a household, policy or catalog version it names does not exist + (the Public API checks the first two first), with the service's reason; +- 401, 403, 404, 405, 413 and 429, which its authentication, routing, + body-size and rate limits answer before its route runs; +- 503 when it cannot read its manifest or database, or Modal refused the + household function's call (failing the job it had created); +- a connection never made (``ConnectError``, ``ConnectTimeout``, + ``PoolTimeout``), a gateway token that cannot be fetched, and missing + configuration. + +The caller's data was refused: + +- 422 with its reasons (a malformed request, or one its household contract + refuses), one per field as FastAPI writes them, which reach the caller as + SimulationServiceInvalidRequestError. A 422 of another shape is + unavailable. + +The job may exist, and no route finds a household job without its ID, so the +message asks the caller to submit the calculation again and says that this +creates a second job if the first request went through (the route answers +502): + +- the service failed after it may have created the job: its 500 + (SimulationServiceInternalError), a 202 whose body cannot be read, and any + status not named above, such as 200, a redirect, 408, 409 or 501 + (SimulationServiceAnswerUnreadableError); +- its answer was lost once the request may have been sent (any other + transport error, such as a read timeout or a dropped connection), or a + proxy in front of it answered 502 or 504 in its place + (SimulationServiceAnswerUnreadableError). + +The service's own text may hold stored data, and the route needs no account, +so after a 500 or 503, or a status not named above, a message gives only the +service's status and its ``code`` when that is a fixed identifier (lower-case +letters, digits and underscores, starting with a letter, at most 64 +characters). The 400 and 422 refusals keep the service's reasons, which name +the caller's own household, policy and fields. Each submission carries the request's ID and observability ID headers. The observability ID is the request's own, else the caller's, else a new one, @@ -59,6 +79,7 @@ from __future__ import annotations +import re from dataclasses import dataclass from functools import lru_cache from http import HTTPStatus @@ -99,6 +120,27 @@ # Transport errors raised before the request could reach the service. After # any other, the service may have received the submission and created the job. CONNECTION_NEVER_MADE = (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) +# Statuses the service answers before its route creates a job: its +# authentication, routing, body-size and rate limits, and its 503 when it +# cannot read its manifest or database or Modal refused the call. +ANSWERED_BEFORE_A_JOB_EXISTS = frozenset( + { + HTTPStatus.UNAUTHORIZED, + HTTPStatus.FORBIDDEN, + HTTPStatus.NOT_FOUND, + HTTPStatus.METHOD_NOT_ALLOWED, + HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + HTTPStatus.TOO_MANY_REQUESTS, + HTTPStatus.SERVICE_UNAVAILABLE, + } +) +# A code the service names may reach the caller only when it is a fixed +# identifier, never text that could hold stored data. +FIXED_IDENTIFIER = re.compile(r"[a-z][a-z0-9_]{0,63}") +SUBMIT_AGAIN = ( + "Submit the household calculation again; if the first request went " + "through, this creates a second job." +) def _lost_answer(what_happened: str) -> str: @@ -111,8 +153,19 @@ def _lost_answer(what_happened: str) -> str: """ return ( "The simulation service may have received the request, but its answer " - f"was lost ({what_happened}). Submit the household calculation again; " - "if the first request went through, this creates a second job." + f"was lost ({what_happened}). {SUBMIT_AGAIN}" + ) + + +def _created_but_failed(status_and_code: str) -> str: + """Return the message for an answer the service sent once the job may exist. + + ``status_and_code`` is the service's status, and its code when that is + a fixed identifier. + """ + return ( + "The simulation service may have created the household job, but it " + f"failed ({status_and_code}). {SUBMIT_AGAIN}" ) @@ -147,13 +200,11 @@ class _RefusedField(_AnswerModel): class _Refusal(_AnswerModel): """A refusal's body: a reason, or each refused request field. - A 400 or 503 also names its ``code``, and a 503 after Modal refused the - call names the household job it failed. + A 400, 500 or 503 also names its ``code``. """ detail: str | list[_RefusedField] code: str | None = None - household_job_id: UUID | None = None def household_calculation_request( @@ -231,33 +282,42 @@ def _refusal(answer: SimulationAnswer) -> _Refusal | None: return None -def _internal_error(answer: SimulationAnswer, refusal: _Refusal | None) -> str: - """Describe the service's failure, which may have created the job. +def _fixed_code(refusal: _Refusal | None) -> str | None: + """Return the service's code when it is a fixed identifier, else None.""" + if refusal is None or refusal.code is None: + return None + return refusal.code if FIXED_IDENTIFIER.fullmatch(refusal.code) else None + + +def _status_and_code(answer: SimulationAnswer, refusal: _Refusal | None) -> str: + """Return the service's status, and its code when that is a fixed identifier. - The message names the status, with the service's code and reason when it - gives them. + The service's text is left out: it may hold stored data. """ - if refusal is None or not isinstance(refusal.detail, str): - return _lost_answer(str(answer.status_code)) - code = "" if refusal.code is None else f" {refusal.code}" - reason = refusal.detail.strip().rstrip(".") - return _lost_answer(f"{answer.status_code}{code}: {reason}") + code = _fixed_code(refusal) + return str(answer.status_code) if code is None else f"{answer.status_code} {code}" def _unavailable(answer: SimulationAnswer, refusal: _Refusal | None) -> str: - """Describe an answer that means the service cannot calculate now.""" + """Describe an answer the service gives before its route creates a job.""" + return ( + f"The simulation service is unavailable ({_status_and_code(answer, refusal)})." + ) + + +def _refused(answer: SimulationAnswer, refusal: _Refusal | None) -> str: + """Describe a 400: a row the request names, or a function it needs, is missing. + + The service's reason names the caller's own household or policy, so it is + kept. + """ if refusal is None or not isinstance(refusal.detail, str): - return f"The simulation service answered {answer.status_code}." - stated = ( - "refused the calculation" - if answer.status_code == HTTPStatus.BAD_REQUEST - else "is unavailable" + return f"The simulation service refused the calculation ({answer.status_code})." + code = _fixed_code(refusal) + named_code = "" if code is None else f" ({code})" + return ( + f"The simulation service refused the calculation{named_code}: {refusal.detail}" ) - code = "" if refusal.code is None else f" ({refusal.code})" - message = f"The simulation service {stated}{code}: {refusal.detail}" - if refusal.household_job_id is not None: - message += f"; it failed household job {refusal.household_job_id}" - return message class SimulationHouseholdCalculationClient: @@ -303,13 +363,15 @@ def submit( """Submit the calculation and return the job the service created. Raise SimulationServiceInvalidRequestError with the service's reasons - when it refuses the caller's data; SimulationServiceAnswerUnreadableError - when it accepted the calculation but its answer cannot be read, its - answer was lost after the submission may have reached it, or a proxy in - front of it answered 502 or 504 in its place; + when it refuses the caller's data (422); + SimulationServiceUnavailableError when nothing was created (its 400, + the statuses it answers before its route runs, its 503, a connection + never made or a gateway token that cannot be fetched); SimulationServiceInternalError when it failed (500); and - SimulationServiceUnavailableError otherwise, including a connection - never made. + SimulationServiceAnswerUnreadableError when it accepted the + calculation but its answer cannot be read, it answered any other + status, its answer was lost after the submission may have reached it, + or a proxy in front of it answered 502 or 504 in its place. """ start_observability_id() try: @@ -323,28 +385,42 @@ def submit( raise SimulationServiceAnswerUnreadableError( _lost_answer(type(error).__name__) ) from error - if answer.status_code == HTTPStatus.ACCEPTED: - try: - accepted = _AcceptedJob.model_validate_json(answer.content) - except ValidationError as error: - raise SimulationServiceAnswerUnreadableError( - _lost_answer(f"{answer.status_code} with an unreadable body") - ) from error - return HouseholdCalculationAccepted( - calculation_id=accepted.job_id, status=accepted.status - ) - refusal = _refusal(answer) - if answer.status_code == HTTPStatus.UNPROCESSABLE_ENTITY: - if refusal is None: - raise SimulationServiceUnavailableError(UNREADABLE_ANSWER) - raise SimulationServiceInvalidRequestError(_invalid_request_reason(refusal)) - if answer.status_code == HTTPStatus.INTERNAL_SERVER_ERROR: - raise SimulationServiceInternalError(_internal_error(answer, refusal)) - if answer.status_code in PROXY_STAND_IN_STATUSES: + return _accepted_job(answer) + + +def _accepted_job(answer: SimulationAnswer) -> HouseholdCalculationAccepted: + """Return the job the service's answer accepts, or raise what it means. + + The errors are those ``SimulationHouseholdCalculationClient.submit`` + raises, apart from the transport errors it handles itself. + """ + if answer.status_code == HTTPStatus.ACCEPTED: + try: + accepted = _AcceptedJob.model_validate_json(answer.content) + except ValidationError as error: raise SimulationServiceAnswerUnreadableError( - _lost_answer(f"a proxy answered {answer.status_code}") - ) + _created_but_failed(str(answer.status_code)) + ) from error + return HouseholdCalculationAccepted( + calculation_id=accepted.job_id, status=accepted.status + ) + if answer.status_code in PROXY_STAND_IN_STATUSES: + raise SimulationServiceAnswerUnreadableError( + _lost_answer(f"a proxy answered {answer.status_code}") + ) + refusal = _refusal(answer) + if answer.status_code == HTTPStatus.UNPROCESSABLE_ENTITY: + if refusal is None: + raise SimulationServiceUnavailableError(UNREADABLE_ANSWER) + raise SimulationServiceInvalidRequestError(_invalid_request_reason(refusal)) + if answer.status_code == HTTPStatus.BAD_REQUEST: + raise SimulationServiceUnavailableError(_refused(answer, refusal)) + if answer.status_code in ANSWERED_BEFORE_A_JOB_EXISTS: raise SimulationServiceUnavailableError(_unavailable(answer, refusal)) + failure = _created_but_failed(_status_and_code(answer, refusal)) + if answer.status_code == HTTPStatus.INTERNAL_SERVER_ERROR: + raise SimulationServiceInternalError(failure) + raise SimulationServiceAnswerUnreadableError(failure) @lru_cache(maxsize=1) diff --git a/policyengine_api/services/v2/household_calculations/types.py b/policyengine_api/services/v2/household_calculations/types.py index d6fce4557..d3b487a18 100644 --- a/policyengine_api/services/v2/household_calculations/types.py +++ b/policyengine_api/services/v2/household_calculations/types.py @@ -318,32 +318,35 @@ def submit( Raise SimulationServiceInvalidRequestError when the service refuses the caller's data, with its reasons; - SimulationServiceAnswerUnreadableError when it accepted the - calculation but its answer cannot be read, its answer was lost after - the submission may have reached it, or a proxy in front of it - answered 502 or 504 in its place; + SimulationServiceUnavailableError when nothing was created; SimulationServiceInternalError when it failed (500), possibly after - creating the job; and - SimulationServiceUnavailableError for any other refusal or failure, - including a connection never made. + creating the job; and SimulationServiceAnswerUnreadableError when it + accepted the calculation but its answer cannot be read, it answered a + status the Public API does not expect, its answer was lost after the + submission may have reached it, or a proxy in front of it answered + 502 or 504 in its place. """ ... class SimulationServiceUnavailableError(RuntimeError): - """The simulation service could not be reached, could not answer, or refused. - - It covers every refusal other than the caller's data: a household, policy - or catalog version it cannot find (which the Public API checks first), a - database or manifest it cannot read, and a Modal call it cannot start. + """The simulation service could not be reached, or refused before any job. + + It covers a connection never made, a gateway token that cannot be + fetched, missing configuration, and every answer the service gives before + its route creates a job: a household, policy or catalog version it cannot + find (which the Public API checks first), its authentication, routing, + body-size and rate limits, and a database or manifest it cannot read or a + Modal call it cannot start (503). """ class SimulationServiceAnswerUnreadableError(RuntimeError): """The simulation service received the calculation, or may have, but no answer. - Either it answered 202, so it has already created the job and started its - call, but its answer is unreadable; or its answer was lost after the + It answered 202, so it has already created the job and started its call, + but its answer is unreadable; it answered a status the Public API does not + expect, possibly after creating the job; or its answer was lost after the submission may have reached it. The message asks the caller to submit again, and says this creates a second job if the first request went through. From e04a04eaa70331c9f101f6ee52a04c88b5f5a3bc Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:51:09 +0400 Subject: [PATCH 54/57] Test that a 503 without sim's code is a proxy's, and that codes are read apart from reasons (failing) Sim's own 503 always names its code; without one, Cloud Run's front end answered in its place, possibly after the job was created, so the tests expect 502 "its answer was lost (a proxy answered 503)". A code that isn't text no longer makes a 400's reason or a 422's field list unreadable, a 400 without a textual reason gives its status and code, the submit route's OpenAPI 503 says only a 400 keeps sim's reason, and the tests cover a 403 with FastAPI's body, statuses outside 200 to 599, and the exact unreadable-422 message. Co-Authored-By: Claude Opus 5.5 --- .../test_simulation_household_calculations.py | 116 ++++++++++++++---- ...t_household_calculation_database_routes.py | 1 + .../v2/test_household_calculation_routes.py | 2 + 3 files changed, 98 insertions(+), 21 deletions(-) diff --git a/tests/unit/libs/test_simulation_household_calculations.py b/tests/unit/libs/test_simulation_household_calculations.py index 4f96d307b..96b6c486a 100644 --- a/tests/unit/libs/test_simulation_household_calculations.py +++ b/tests/unit/libs/test_simulation_household_calculations.py @@ -223,9 +223,27 @@ def test_request_fields_the_service_refuses_are_reported_one_by_one() -> None: def test_a_refusal_of_another_shape_is_unavailable(body) -> None: service = RecordingService(422, body) - with pytest.raises(SimulationServiceUnavailableError, match="could not be read"): + with pytest.raises(SimulationServiceUnavailableError) as unreadable: service.client().submit(calculation_input()) + assert _route_status(unreadable.value) == 503 + assert str(unreadable.value) == ( + "The simulation service's answer could not be read." + ) + + +def test_a_field_refusal_with_a_code_that_is_not_text_keeps_its_reasons() -> None: + service = RecordingService( + 422, {"code": 5, "detail": [{"loc": ["body", "spm"], "msg": "refused"}]} + ) + + with pytest.raises(SimulationServiceInvalidRequestError) as refused: + service.client().submit(calculation_input()) + + assert str(refused.value) == ( + "The simulation service cannot calculate this household: spm: refused" + ) + def test_a_reference_the_service_cannot_find_is_unavailable_with_its_reason() -> None: service = RecordingService( @@ -278,36 +296,59 @@ def test_a_dependency_the_service_cannot_read_names_its_code() -> None: @pytest.mark.parametrize( ("code", "shown"), [ - pytest.param("database_unavailable", "503 database_unavailable", id="kept"), - pytest.param("a" * 64, f"503 {'a' * 64}", id="64-characters"), - pytest.param("a" * 65, "503", id="65-characters"), - pytest.param("Database_unavailable", "503", id="capital"), - pytest.param("1st_failure", "503", id="leading-digit"), - pytest.param("_failure", "503", id="leading-underscore"), - pytest.param("database-unavailable", "503", id="hyphen"), - pytest.param("database unavailable: row 7", "503", id="text"), - pytest.param("", "503", id="empty"), + pytest.param("internal_error", "500 internal_error", id="kept"), + pytest.param("a" * 64, f"500 {'a' * 64}", id="64-characters"), + pytest.param("a" * 65, "500", id="65-characters"), + pytest.param("Internal_error", "500", id="capital"), + pytest.param("1st_failure", "500", id="leading-digit"), + pytest.param("_failure", "500", id="leading-underscore"), + pytest.param("internal-error", "500", id="hyphen"), + pytest.param("internal error: row 7", "500", id="text"), + pytest.param("internal_error\n", "500", id="trailing-newline"), + pytest.param("", "500", id="empty"), + pytest.param(5, "500", id="number"), + pytest.param(None, "500", id="null"), ], ) def test_only_a_code_that_is_a_fixed_identifier_is_shown(code, shown) -> None: - service = RecordingService(503, {"code": code, "detail": "The database is down"}) + service = RecordingService(500, {"code": code, "detail": "The household is 52000"}) + + with pytest.raises(SimulationServiceInternalError) as failed: + service.client().submit(calculation_input()) + + assert str(failed.value) == created_but_failed_message(shown) + + +@pytest.mark.parametrize( + "code", + [ + pytest.param("Unknown Household", id="not-an-identifier"), + pytest.param(5, id="number"), + pytest.param(["unknown_household"], id="list"), + ], +) +def test_a_refusal_code_that_is_not_a_fixed_identifier_is_left_out(code) -> None: + service = RecordingService(400, {"code": code, "detail": "Household not found"}) with pytest.raises(SimulationServiceUnavailableError) as refused: service.client().submit(calculation_input()) - assert str(refused.value) == f"The simulation service is unavailable ({shown})." + assert str(refused.value) == ( + "The simulation service refused the calculation: Household not found" + ) -def test_a_refusal_code_that_is_not_a_fixed_identifier_is_left_out() -> None: +def test_a_refusal_without_a_textual_reason_gives_its_status_and_code() -> None: service = RecordingService( - 400, {"code": "Unknown Household", "detail": "Household not found"} + 400, + {"code": "unknown_household", "detail": [{"loc": ["body"], "msg": "x"}]}, ) with pytest.raises(SimulationServiceUnavailableError) as refused: service.client().submit(calculation_input()) assert str(refused.value) == ( - "The simulation service refused the calculation: Household not found" + "The simulation service refused the calculation (400 unknown_household)." ) @@ -360,27 +401,37 @@ def test_a_proxys_answer_in_place_of_the_services_is_not_unavailable( @pytest.mark.parametrize( - ("status_code", "message"), + ("status_code", "body", "message"), [ # The service's authentication, routing, body-size and rate limits # answer these before its route creates a job. *( pytest.param( status_code, + b"", f"The simulation service is unavailable ({status_code}).", id=str(status_code), ) - for status_code in (401, 403, 404, 405, 413, 429, 503) + for status_code in (401, 403, 404, 405, 413, 429) ), pytest.param( - 400, "The simulation service refused the calculation (400).", id="400" + 403, + {"detail": "Forbidden"}, + "The simulation service is unavailable (403).", + id="403-from-fastapi", + ), + pytest.param( + 400, + b"", + "The simulation service refused the calculation (400).", + id="400", ), ], ) def test_an_answer_given_before_a_job_is_created_is_unavailable( - status_code, message + status_code, body, message ) -> None: - service = RecordingService(status_code, b"") + service = RecordingService(status_code, body) with pytest.raises(SimulationServiceUnavailableError) as unavailable: service.client().submit(calculation_input()) @@ -389,12 +440,35 @@ def test_an_answer_given_before_a_job_is_created_is_unavailable( assert str(unavailable.value) == message +@pytest.mark.parametrize( + "body", + [ + pytest.param(b"", id="empty"), + pytest.param(b"Service Unavailable", id="html"), + pytest.param({"detail": "Service Unavailable"}, id="no-code"), + pytest.param({"code": "Not An Identifier", "detail": "x"}, id="bad-code"), + pytest.param({"code": 7, "detail": "x"}, id="number-code"), + ], +) +def test_a_503_without_the_services_code_came_from_a_proxy(body) -> None: + # The service's own 503 always names its code. Without one, Cloud Run's + # front end answered in its place, possibly after the service created the + # job. + service = RecordingService(503, body) + + with pytest.raises(SimulationServiceAnswerUnreadableError) as lost: + service.client().submit(calculation_input()) + + assert _route_status(lost.value) == 502 + assert str(lost.value) == lost_answer_message("a proxy answered 503") + + @pytest.mark.parametrize( ("status_code", "body", "status_and_code"), [ *( pytest.param(status_code, b"", str(status_code), id=str(status_code)) - for status_code in (200, 201, 204, 302, 307, 408, 409, 410, 501) + for status_code in (199, 200, 201, 204, 302, 307, 408, 409, 410, 501, 600) ), pytest.param( 409, diff --git a/tests/unit/v2/test_household_calculation_database_routes.py b/tests/unit/v2/test_household_calculation_database_routes.py index 215804a80..29d56801d 100644 --- a/tests/unit/v2/test_household_calculation_database_routes.py +++ b/tests/unit/v2/test_household_calculation_database_routes.py @@ -409,6 +409,7 @@ def test_a_refusal_before_sim_is_called_echoes_no_observability_id( 502, id="unexpected-status", ), + pytest.param((503, {"detail": "Service Unavailable"}), 502, id="proxy-503"), ], ) def test_an_answer_after_sim_was_called_returns_the_observability_id( diff --git a/tests/unit/v2/test_household_calculation_routes.py b/tests/unit/v2/test_household_calculation_routes.py index 8d0c7b5a0..5006f5f39 100644 --- a/tests/unit/v2/test_household_calculation_routes.py +++ b/tests/unit/v2/test_household_calculation_routes.py @@ -437,6 +437,8 @@ def test_openapi_documents_both_routes_and_the_missing_parameter_refusal( assert "Submit the household calculation again" in lost_answer assert "creates a second job" in lost_answer assert "a status the Public API does not expect" in lost_answer + unavailable = submit["responses"]["503"]["description"] + assert "otherwise only its status and code" in unavailable assert "security" not in submit refusal_schema = submit["responses"]["422"]["content"]["application/json"]["schema"] refusals = {option["$ref"].rsplit("/", 1)[-1] for option in refusal_schema["anyOf"]} From a9e31752d374784d0b6a5012b41dac339891316f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:54:17 +0400 Subject: [PATCH 55/57] Answer a 503 without sim's code as a proxy's, and read codes apart from reasons Sim's own 503 always names its code. A 503 without one came from Cloud Run's front end, possibly after sim created the job, so it now answers 502 "its answer was lost (a proxy answered 503)", like a proxy's 502 or 504. The code is read from the answer's JSON on its own, so a code that isn't text no longer makes a 400's reason or a 422's field list unreadable; a 400 without a textual reason gives its status and code. The submit route's OpenAPI 503 no longer promises sim's reason: only a 400 keeps it. The docs and names now say that no job of the submission runs after a 503-class answer, rather than that none was created, since sim fails the job it created when Modal refuses the call. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-11-v2-households.md | 14 ++-- .../household_calculations/response_models.py | 21 +++-- .../libs/simulation_household_calculations.py | 84 ++++++++++++------- .../v2/household_calculations/types.py | 26 +++--- 4 files changed, 89 insertions(+), 56 deletions(-) diff --git a/docs/migration/stage-11-v2-households.md b/docs/migration/stage-11-v2-households.md index 053febd73..2245cfae7 100644 --- a/docs/migration/stage-11-v2-households.md +++ b/docs/migration/stage-11-v2-households.md @@ -323,11 +323,12 @@ answer is 202 with the new job's `calculation_id` and `status`; each submission creates a new job. The simulation service's 422 reasons reach the caller as 422. -Answers given before the simulation service's route creates a job are 503: -its 400 (a household, policy or catalog version it can't find, with its -reason), its 401, 403, 404, 405, 413 and 429 (its authentication, routing, -body-size and rate limits), its 503 (its manifest or database can't be read, -or Modal refused the call), a refusal the Public API can't read, a +Answers after which no job of the submission runs are 503: its 400 (a +household, policy or catalog version it can't find, with its reason), its +401, 403, 404, 405, 413 and 429 (its authentication, routing, body-size and +rate limits, before its route runs), its own 503, which names its code (its +manifest or database can't be read, or Modal refused the call, after which +it fails the job it had created), a refusal the Public API can't read, a connection never made, missing configuration and a gateway token it can't fetch. A failed catalog query in the policy check is 503 too. @@ -335,7 +336,8 @@ Answers that may come after it created the job are 502: a 202 whose body the Public API can't read, its 500 (`internal_error`), any status not named above (such as 200, a redirect, 408, 409 or 501), an answer lost after the submission may have been sent (a read timeout or a dropped connection), and -a 502 or 504 a proxy in front of it answered in its place. No route finds a +an answer a proxy in front of it gave in its place: a 502 or 504, which it +never answers itself, or a 503 without its code. No route finds a household job without its ID, so the 502's message asks the caller to submit the household calculation again, and says that this creates a second job if the first request went through. diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py index e9bad6f4a..fd138d1c5 100644 --- a/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py +++ b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py @@ -185,9 +185,10 @@ class HouseholdCalculationResponse( "but its answer cannot be read; it failed (500); it answered a " "status the Public API does not expect, such as 200, a redirect, " "408, 409 or 501; its answer was lost after the submission may have " - "reached it; or a proxy in front of it answered 502 or 504 in its " - "place. In each case it may have created the job, and no route " - "finds a job without its ID. Submit the household calculation " + "reached it; or a proxy in front of it answered 502, 504, or a 503 " + "without the simulation service's code, in its place. In each case " + "it may have created the job, and no route finds a job without its " + "ID. Submit the household calculation " "again; if the first request went through, this creates a second " "job. The message gives the simulation service's status and code, " "or what came in place of its answer." @@ -206,11 +207,15 @@ class HouseholdCalculationResponse( 503: { "model": V2ErrorResponse, "description": ( - "The simulation service is unavailable or cannot be reached, " - "refuses in a shape the Public API cannot read, or refuses a " - "household, policy or catalog version the Public API has already " - "checked; or v2 storage is unavailable. The message gives the " - "simulation service's code and reason when it states them." + "No job of the submission will run: the simulation service cannot " + "be reached or is not configured; it refuses a household, policy or " + "catalog version the Public API has already checked (400); its " + "authentication, routing, body-size or rate limits refuse the " + "request; it answers its own 503, naming its code, because it cannot " + "read its manifest or database or Modal refused the call; or it " + "refuses in a shape the Public API cannot read. Or v2 storage is " + "unavailable. The message gives the simulation service's reason " + "only for a 400, otherwise only its status and code." ), }, } diff --git a/policyengine_api/libs/simulation_household_calculations.py b/policyengine_api/libs/simulation_household_calculations.py index 0079c2a7f..7b5d50d9d 100644 --- a/policyengine_api/libs/simulation_household_calculations.py +++ b/policyengine_api/libs/simulation_household_calculations.py @@ -17,15 +17,16 @@ - 202 with the new job's ID and status is the accepted job. -Nothing was created, so the service is unavailable +No job of the submission will run, so the service is unavailable (SimulationServiceUnavailableError, which the route answers 503): - 400 when a household, policy or catalog version it names does not exist (the Public API checks the first two first), with the service's reason; - 401, 403, 404, 405, 413 and 429, which its authentication, routing, body-size and rate limits answer before its route runs; -- 503 when it cannot read its manifest or database, or Modal refused the - household function's call (failing the job it had created); +- its own 503, which names its code, when it cannot read its manifest or + database, or Modal refused the household function's call, after which it + failed the job it had created at once; - a connection never made (``ConnectError``, ``ConnectTimeout``, ``PoolTimeout``), a gateway token that cannot be fetched, and missing configuration. @@ -48,7 +49,8 @@ (SimulationServiceAnswerUnreadableError); - its answer was lost once the request may have been sent (any other transport error, such as a read timeout or a dropped connection), or a - proxy in front of it answered 502 or 504 in its place + proxy in front of it answered in its place: a 502 or 504, which the + service never answers itself, or a 503 without the service's code (SimulationServiceAnswerUnreadableError). The service's own text may hold stored data, and the route needs no account, @@ -79,6 +81,7 @@ from __future__ import annotations +import json import re from dataclasses import dataclass from functools import lru_cache @@ -120,10 +123,11 @@ # Transport errors raised before the request could reach the service. After # any other, the service may have received the submission and created the job. CONNECTION_NEVER_MADE = (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) -# Statuses the service answers before its route creates a job: its -# authentication, routing, body-size and rate limits, and its 503 when it -# cannot read its manifest or database or Modal refused the call. -ANSWERED_BEFORE_A_JOB_EXISTS = frozenset( +# Statuses after which no job of the submission runs: the service's +# authentication, routing, body-size and rate limits answer them before its +# route runs, and its own 503 comes before it creates the job, or after it +# failed the job whose call Modal refused. +STATUSES_AFTER_WHICH_NO_JOB_RUNS = frozenset( { HTTPStatus.UNAUTHORIZED, HTTPStatus.FORBIDDEN, @@ -200,11 +204,11 @@ class _RefusedField(_AnswerModel): class _Refusal(_AnswerModel): """A refusal's body: a reason, or each refused request field. - A 400, 500 or 503 also names its ``code``. + Its ``code`` is read on its own (``_fixed_code``), so a code of another + type never makes the reason unreadable. """ detail: str | list[_RefusedField] - code: str | None = None def household_calculation_request( @@ -282,29 +286,46 @@ def _refusal(answer: SimulationAnswer) -> _Refusal | None: return None -def _fixed_code(refusal: _Refusal | None) -> str | None: - """Return the service's code when it is a fixed identifier, else None.""" - if refusal is None or refusal.code is None: +def _fixed_code(answer: SimulationAnswer) -> str | None: + """Return the code the answer's body names when it is a fixed identifier.""" + try: + body = json.loads(answer.content) + except ValueError: return None - return refusal.code if FIXED_IDENTIFIER.fullmatch(refusal.code) else None + code = body.get("code") if isinstance(body, dict) else None + if isinstance(code, str) and FIXED_IDENTIFIER.fullmatch(code): + return code + return None -def _status_and_code(answer: SimulationAnswer, refusal: _Refusal | None) -> str: +def _status_and_code(answer: SimulationAnswer) -> str: """Return the service's status, and its code when that is a fixed identifier. The service's text is left out: it may hold stored data. """ - code = _fixed_code(refusal) + code = _fixed_code(answer) return str(answer.status_code) if code is None else f"{answer.status_code} {code}" -def _unavailable(answer: SimulationAnswer, refusal: _Refusal | None) -> str: - """Describe an answer the service gives before its route creates a job.""" +def _answered_by_a_proxy(answer: SimulationAnswer) -> bool: + """Whether a proxy in front of the service answered in its place. + + The service never answers 502 or 504 itself, and its own 503 always + names its code. + """ + if answer.status_code in PROXY_STAND_IN_STATUSES: + return True return ( - f"The simulation service is unavailable ({_status_and_code(answer, refusal)})." + answer.status_code == HTTPStatus.SERVICE_UNAVAILABLE + and _fixed_code(answer) is None ) +def _unavailable(answer: SimulationAnswer) -> str: + """Describe an answer after which no job of the submission runs.""" + return f"The simulation service is unavailable ({_status_and_code(answer)})." + + def _refused(answer: SimulationAnswer, refusal: _Refusal | None) -> str: """Describe a 400: a row the request names, or a function it needs, is missing. @@ -312,8 +333,11 @@ def _refused(answer: SimulationAnswer, refusal: _Refusal | None) -> str: kept. """ if refusal is None or not isinstance(refusal.detail, str): - return f"The simulation service refused the calculation ({answer.status_code})." - code = _fixed_code(refusal) + return ( + "The simulation service refused the calculation " + f"({_status_and_code(answer)})." + ) + code = _fixed_code(answer) named_code = "" if code is None else f" ({code})" return ( f"The simulation service refused the calculation{named_code}: {refusal.detail}" @@ -364,14 +388,14 @@ def submit( Raise SimulationServiceInvalidRequestError with the service's reasons when it refuses the caller's data (422); - SimulationServiceUnavailableError when nothing was created (its 400, - the statuses it answers before its route runs, its 503, a connection - never made or a gateway token that cannot be fetched); - SimulationServiceInternalError when it failed (500); and + SimulationServiceUnavailableError when no job of the submission will + run (its 400, the statuses it answers before its route runs, its own + 503, a connection never made or a gateway token that cannot be + fetched); SimulationServiceInternalError when it failed (500); and SimulationServiceAnswerUnreadableError when it accepted the calculation but its answer cannot be read, it answered any other status, its answer was lost after the submission may have reached it, - or a proxy in front of it answered 502 or 504 in its place. + or a proxy in front of it answered in its place. """ start_observability_id() try: @@ -404,7 +428,7 @@ def _accepted_job(answer: SimulationAnswer) -> HouseholdCalculationAccepted: return HouseholdCalculationAccepted( calculation_id=accepted.job_id, status=accepted.status ) - if answer.status_code in PROXY_STAND_IN_STATUSES: + if _answered_by_a_proxy(answer): raise SimulationServiceAnswerUnreadableError( _lost_answer(f"a proxy answered {answer.status_code}") ) @@ -415,9 +439,9 @@ def _accepted_job(answer: SimulationAnswer) -> HouseholdCalculationAccepted: raise SimulationServiceInvalidRequestError(_invalid_request_reason(refusal)) if answer.status_code == HTTPStatus.BAD_REQUEST: raise SimulationServiceUnavailableError(_refused(answer, refusal)) - if answer.status_code in ANSWERED_BEFORE_A_JOB_EXISTS: - raise SimulationServiceUnavailableError(_unavailable(answer, refusal)) - failure = _created_but_failed(_status_and_code(answer, refusal)) + if answer.status_code in STATUSES_AFTER_WHICH_NO_JOB_RUNS: + raise SimulationServiceUnavailableError(_unavailable(answer)) + failure = _created_but_failed(_status_and_code(answer)) if answer.status_code == HTTPStatus.INTERNAL_SERVER_ERROR: raise SimulationServiceInternalError(failure) raise SimulationServiceAnswerUnreadableError(failure) diff --git a/policyengine_api/services/v2/household_calculations/types.py b/policyengine_api/services/v2/household_calculations/types.py index d3b487a18..444801963 100644 --- a/policyengine_api/services/v2/household_calculations/types.py +++ b/policyengine_api/services/v2/household_calculations/types.py @@ -318,38 +318,40 @@ def submit( Raise SimulationServiceInvalidRequestError when the service refuses the caller's data, with its reasons; - SimulationServiceUnavailableError when nothing was created; + SimulationServiceUnavailableError when no job of the submission will + run; SimulationServiceInternalError when it failed (500), possibly after creating the job; and SimulationServiceAnswerUnreadableError when it accepted the calculation but its answer cannot be read, it answered a status the Public API does not expect, its answer was lost after the submission may have reached it, or a proxy in front of it answered - 502 or 504 in its place. + in its place. """ ... class SimulationServiceUnavailableError(RuntimeError): - """The simulation service could not be reached, or refused before any job. + """The simulation service could not be reached, or refused so no job runs. It covers a connection never made, a gateway token that cannot be - fetched, missing configuration, and every answer the service gives before - its route creates a job: a household, policy or catalog version it cannot + fetched, missing configuration, and every answer after which no job of + the submission runs: a household, policy or catalog version it cannot find (which the Public API checks first), its authentication, routing, - body-size and rate limits, and a database or manifest it cannot read or a - Modal call it cannot start (503). + body-size and rate limits, and its own 503 for a database or manifest it + cannot read, or a Modal call it cannot start, after which it fails the + job it had created. """ class SimulationServiceAnswerUnreadableError(RuntimeError): - """The simulation service received the calculation, or may have, but no answer. + """The simulation service may have created the job, but no usable answer came. It answered 202, so it has already created the job and started its call, but its answer is unreadable; it answered a status the Public API does not - expect, possibly after creating the job; or its answer was lost after the - submission may have reached it. The message asks the caller to submit - again, and says this creates a second job if the first request went - through. + expect, possibly after creating the job; its answer was lost after the + submission may have reached it; or a proxy answered in its place. The + message asks the caller to submit again, and says this creates a second + job if the first request went through. """ From 9f5291cf2ea951fed2b56181cf1dd0e2f683b464 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 21:07:22 +0400 Subject: [PATCH 56/57] Test that a deeply nested or non-object body reads as unreadable, not as a 500 (failing) The standard-library parser that reads sim's code raises RecursionError on a deeply nested body, which the route answers 500 without the "submit again" message. The tests expect such a body, a top-level JSON value that isn't an object and bytes that aren't UTF-8 to read as no body, and the OpenAPI 503 to list sim's not_configured 503. Co-Authored-By: Claude Opus 5.5 --- .../test_simulation_household_calculations.py | 44 +++++++++++++++++++ .../v2/test_household_calculation_routes.py | 1 + 2 files changed, 45 insertions(+) diff --git a/tests/unit/libs/test_simulation_household_calculations.py b/tests/unit/libs/test_simulation_household_calculations.py index 96b6c486a..c343a2f9a 100644 --- a/tests/unit/libs/test_simulation_household_calculations.py +++ b/tests/unit/libs/test_simulation_household_calculations.py @@ -60,6 +60,10 @@ def lost_answer_message(what_happened: str) -> str: ) +# Deeper than any JSON parser's recursion or depth limit. +DEEPLY_NESTED_BODY = b"[" * 100_000 + b"]" * 100_000 + + def created_but_failed_message(status_and_code: str) -> str: """Return the 502 message for an answer sent after the job may exist. @@ -440,6 +444,40 @@ def test_an_answer_given_before_a_job_is_created_is_unavailable( assert str(unavailable.value) == message +@pytest.mark.parametrize( + ("status_code", "error_type", "message"), + [ + pytest.param( + 400, + SimulationServiceUnavailableError, + "The simulation service refused the calculation (400).", + id="400", + ), + pytest.param( + 409, + SimulationServiceAnswerUnreadableError, + created_but_failed_message("409"), + id="409", + ), + pytest.param( + 500, + SimulationServiceInternalError, + created_but_failed_message("500"), + id="500", + ), + ], +) +def test_a_deeply_nested_body_is_read_as_an_unreadable_one( + status_code, error_type, message +) -> None: + service = RecordingService(status_code, DEEPLY_NESTED_BODY) + + with pytest.raises(error_type) as unreadable: + service.client().submit(calculation_input()) + + assert str(unreadable.value) == message + + @pytest.mark.parametrize( "body", [ @@ -448,6 +486,12 @@ def test_an_answer_given_before_a_job_is_created_is_unavailable( pytest.param({"detail": "Service Unavailable"}, id="no-code"), pytest.param({"code": "Not An Identifier", "detail": "x"}, id="bad-code"), pytest.param({"code": 7, "detail": "x"}, id="number-code"), + pytest.param(b"[]", id="array"), + pytest.param(b"5", id="number"), + pytest.param(b"null", id="null"), + pytest.param(b'"database_unavailable"', id="string"), + pytest.param(b"\xff\xfe{}", id="not-utf-8"), + pytest.param(DEEPLY_NESTED_BODY, id="deeply-nested"), ], ) def test_a_503_without_the_services_code_came_from_a_proxy(body) -> None: diff --git a/tests/unit/v2/test_household_calculation_routes.py b/tests/unit/v2/test_household_calculation_routes.py index 5006f5f39..36f5c9ee8 100644 --- a/tests/unit/v2/test_household_calculation_routes.py +++ b/tests/unit/v2/test_household_calculation_routes.py @@ -439,6 +439,7 @@ def test_openapi_documents_both_routes_and_the_missing_parameter_refusal( assert "a status the Public API does not expect" in lost_answer unavailable = submit["responses"]["503"]["description"] assert "otherwise only its status and code" in unavailable + assert "its own 503, naming its code, because it is not configured" in (unavailable) assert "security" not in submit refusal_schema = submit["responses"]["422"]["content"]["application/json"]["schema"] refusals = {option["$ref"].rsplit("/", 1)[-1] for option in refusal_schema["anyOf"]} From 3dc1f8b7935dd69d2a5c45b82bd599daabecbf19 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 11 Nov 2026 21:10:22 +0400 Subject: [PATCH 57/57] Parse each answer once with pydantic, so no body can turn a 502 or 503 into a 500 The code was read with the standard-library parser, which raises RecursionError on a deeply nested body; the route answered that 500, without the "submit again" message. Each answer other than 202 is now parsed once, by pydantic's parser, which reports its depth limit and bytes that aren't UTF-8 as a ValidationError, into a record of its status, its fixed-identifier code and its refusal, from which every message is built. The OpenAPI 503, the error docstring and the Stage 11 notes list sim's not_configured 503, and the constant of statuses after which no job runs says it holds only sim's own coded 503. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-11-v2-households.md | 10 +- .../household_calculations/response_models.py | 17 +-- .../libs/simulation_household_calculations.py | 123 ++++++++++-------- .../v2/household_calculations/types.py | 6 +- 4 files changed, 86 insertions(+), 70 deletions(-) diff --git a/docs/migration/stage-11-v2-households.md b/docs/migration/stage-11-v2-households.md index 2245cfae7..2f6a8c784 100644 --- a/docs/migration/stage-11-v2-households.md +++ b/docs/migration/stage-11-v2-households.md @@ -326,11 +326,11 @@ caller as 422. Answers after which no job of the submission runs are 503: its 400 (a household, policy or catalog version it can't find, with its reason), its 401, 403, 404, 405, 413 and 429 (its authentication, routing, body-size and -rate limits, before its route runs), its own 503, which names its code (its -manifest or database can't be read, or Modal refused the call, after which -it fails the job it had created), a refusal the Public API can't read, a -connection never made, missing configuration and a gateway token it can't -fetch. A failed catalog query in the policy check is 503 too. +rate limits, before its route runs), its own 503, which names its code (it +isn't configured, its manifest or database can't be read, or Modal refused +the call, after which it fails the job it had created), a refusal the Public +API can't read, a connection never made, missing configuration and a gateway +token it can't fetch. A failed catalog query in the policy check is 503 too. Answers that may come after it created the job are 502: a 202 whose body the Public API can't read, its 500 (`internal_error`), any status not named diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py index fd138d1c5..a73f71491 100644 --- a/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py +++ b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py @@ -208,14 +208,15 @@ class HouseholdCalculationResponse( "model": V2ErrorResponse, "description": ( "No job of the submission will run: the simulation service cannot " - "be reached or is not configured; it refuses a household, policy or " - "catalog version the Public API has already checked (400); its " - "authentication, routing, body-size or rate limits refuse the " - "request; it answers its own 503, naming its code, because it cannot " - "read its manifest or database or Modal refused the call; or it " - "refuses in a shape the Public API cannot read. Or v2 storage is " - "unavailable. The message gives the simulation service's reason " - "only for a 400, otherwise only its status and code." + "be reached, or the Public API has no URL or credentials for it; it " + "refuses a household, policy or catalog version the Public API has " + "already checked (400); its authentication, routing, body-size or " + "rate limits refuse the request; it answers its own 503, naming its " + "code, because it is not configured, cannot read its manifest or " + "database, or Modal refused the call; or it refuses in a shape the " + "Public API cannot read. Or v2 storage is unavailable. Of the " + "simulation service's answers, the message gives the reason only " + "for a 400, otherwise only its status and code." ), }, } diff --git a/policyengine_api/libs/simulation_household_calculations.py b/policyengine_api/libs/simulation_household_calculations.py index 7b5d50d9d..da32ba573 100644 --- a/policyengine_api/libs/simulation_household_calculations.py +++ b/policyengine_api/libs/simulation_household_calculations.py @@ -24,9 +24,9 @@ (the Public API checks the first two first), with the service's reason; - 401, 403, 404, 405, 413 and 429, which its authentication, routing, body-size and rate limits answer before its route runs; -- its own 503, which names its code, when it cannot read its manifest or - database, or Modal refused the household function's call, after which it - failed the job it had created at once; +- its own 503, which names its code, when it is not configured, cannot read + its manifest or database, or Modal refused the household function's call, + after which it failed the job it had created at once; - a connection never made (``ConnectError``, ``ConnectTimeout``, ``PoolTimeout``), a gateway token that cannot be fetched, and missing configuration. @@ -81,7 +81,6 @@ from __future__ import annotations -import json import re from dataclasses import dataclass from functools import lru_cache @@ -90,7 +89,7 @@ from uuid import UUID import httpx -from pydantic import BaseModel, ConfigDict, ValidationError +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from policyengine_api.libs.gateway_auth import GatewayAuthError from policyengine_api.libs.simulation_entrypoint import SimulationEntrypointClient @@ -126,7 +125,9 @@ # Statuses after which no job of the submission runs: the service's # authentication, routing, body-size and rate limits answer them before its # route runs, and its own 503 comes before it creates the job, or after it -# failed the job whose call Modal refused. +# failed the job whose call Modal refused. Only the service's own 503, which +# names its code, reaches this set: ``_accepted_job`` first answers a 503 +# without one as a proxy's. STATUSES_AFTER_WHICH_NO_JOB_RUNS = frozenset( { HTTPStatus.UNAUTHORIZED, @@ -141,6 +142,8 @@ # A code the service names may reach the caller only when it is a fixed # identifier, never text that could hold stored data. FIXED_IDENTIFIER = re.compile(r"[a-z][a-z0-9_]{0,63}") +# Any JSON value, parsed by pydantic's parser, which has a depth limit. +ANY_JSON: TypeAdapter[Any] = TypeAdapter(Any) SUBMIT_AGAIN = ( "Submit the household calculation again; if the first request went " "through, this creates a second job." @@ -204,7 +207,7 @@ class _RefusedField(_AnswerModel): class _Refusal(_AnswerModel): """A refusal's body: a reason, or each refused request field. - Its ``code`` is read on its own (``_fixed_code``), so a code of another + Its ``code`` is read on its own (``_read_answer``), so a code of another type never makes the reason unreadable. """ @@ -279,68 +282,78 @@ def _invalid_request_reason(refusal: _Refusal) -> str: return f"The simulation service cannot calculate this household: {reason}" -def _refusal(answer: SimulationAnswer) -> _Refusal | None: - try: - return _Refusal.model_validate_json(answer.content) - except ValidationError: - return None +@dataclass(frozen=True) +class _ReadAnswer: + """An answer other than 202, its body parsed once. + ``code`` is the code the body names when it is a fixed identifier, and + ``refusal`` its reason or refused fields when the body has that shape. + """ -def _fixed_code(answer: SimulationAnswer) -> str | None: - """Return the code the answer's body names when it is a fixed identifier.""" - try: - body = json.loads(answer.content) - except ValueError: - return None - code = body.get("code") if isinstance(body, dict) else None - if isinstance(code, str) and FIXED_IDENTIFIER.fullmatch(code): - return code - return None + status_code: int + code: str | None + refusal: _Refusal | None + @property + def status_and_code(self) -> str: + """The status, and the code when there is one; never the service's text.""" + if self.code is None: + return str(self.status_code) + return f"{self.status_code} {self.code}" -def _status_and_code(answer: SimulationAnswer) -> str: - """Return the service's status, and its code when that is a fixed identifier. - The service's text is left out: it may hold stored data. +def _read_answer(answer: SimulationAnswer) -> _ReadAnswer: + """Parse the answer's body once; a body that is not JSON reads as none. + + pydantic's parser reports a body nested past its depth limit, or bytes + that are not UTF-8, as a ValidationError, so no body can raise anything + else. """ - code = _fixed_code(answer) - return str(answer.status_code) if code is None else f"{answer.status_code} {code}" + try: + body = ANY_JSON.validate_json(answer.content) + except ValidationError: + body = None + code = body.get("code") if isinstance(body, dict) else None + fixed_code = ( + code if isinstance(code, str) and FIXED_IDENTIFIER.fullmatch(code) else None + ) + try: + refusal = _Refusal.model_validate(body) + except ValidationError: + refusal = None + return _ReadAnswer(answer.status_code, fixed_code, refusal) -def _answered_by_a_proxy(answer: SimulationAnswer) -> bool: +def _answered_by_a_proxy(read: _ReadAnswer) -> bool: """Whether a proxy in front of the service answered in its place. The service never answers 502 or 504 itself, and its own 503 always names its code. """ - if answer.status_code in PROXY_STAND_IN_STATUSES: + if read.status_code in PROXY_STAND_IN_STATUSES: return True - return ( - answer.status_code == HTTPStatus.SERVICE_UNAVAILABLE - and _fixed_code(answer) is None - ) + return read.status_code == HTTPStatus.SERVICE_UNAVAILABLE and read.code is None -def _unavailable(answer: SimulationAnswer) -> str: +def _unavailable(read: _ReadAnswer) -> str: """Describe an answer after which no job of the submission runs.""" - return f"The simulation service is unavailable ({_status_and_code(answer)})." + return f"The simulation service is unavailable ({read.status_and_code})." -def _refused(answer: SimulationAnswer, refusal: _Refusal | None) -> str: +def _refused(read: _ReadAnswer) -> str: """Describe a 400: a row the request names, or a function it needs, is missing. The service's reason names the caller's own household or policy, so it is kept. """ - if refusal is None or not isinstance(refusal.detail, str): + if read.refusal is None or not isinstance(read.refusal.detail, str): return ( - "The simulation service refused the calculation " - f"({_status_and_code(answer)})." + f"The simulation service refused the calculation ({read.status_and_code})." ) - code = _fixed_code(answer) - named_code = "" if code is None else f" ({code})" + named_code = "" if read.code is None else f" ({read.code})" return ( - f"The simulation service refused the calculation{named_code}: {refusal.detail}" + f"The simulation service refused the calculation{named_code}: " + f"{read.refusal.detail}" ) @@ -428,21 +441,23 @@ def _accepted_job(answer: SimulationAnswer) -> HouseholdCalculationAccepted: return HouseholdCalculationAccepted( calculation_id=accepted.job_id, status=accepted.status ) - if _answered_by_a_proxy(answer): + read = _read_answer(answer) + if _answered_by_a_proxy(read): raise SimulationServiceAnswerUnreadableError( - _lost_answer(f"a proxy answered {answer.status_code}") + _lost_answer(f"a proxy answered {read.status_code}") ) - refusal = _refusal(answer) - if answer.status_code == HTTPStatus.UNPROCESSABLE_ENTITY: - if refusal is None: + if read.status_code == HTTPStatus.UNPROCESSABLE_ENTITY: + if read.refusal is None: raise SimulationServiceUnavailableError(UNREADABLE_ANSWER) - raise SimulationServiceInvalidRequestError(_invalid_request_reason(refusal)) - if answer.status_code == HTTPStatus.BAD_REQUEST: - raise SimulationServiceUnavailableError(_refused(answer, refusal)) - if answer.status_code in STATUSES_AFTER_WHICH_NO_JOB_RUNS: - raise SimulationServiceUnavailableError(_unavailable(answer)) - failure = _created_but_failed(_status_and_code(answer)) - if answer.status_code == HTTPStatus.INTERNAL_SERVER_ERROR: + raise SimulationServiceInvalidRequestError( + _invalid_request_reason(read.refusal) + ) + if read.status_code == HTTPStatus.BAD_REQUEST: + raise SimulationServiceUnavailableError(_refused(read)) + if read.status_code in STATUSES_AFTER_WHICH_NO_JOB_RUNS: + raise SimulationServiceUnavailableError(_unavailable(read)) + failure = _created_but_failed(read.status_and_code) + if read.status_code == HTTPStatus.INTERNAL_SERVER_ERROR: raise SimulationServiceInternalError(failure) raise SimulationServiceAnswerUnreadableError(failure) diff --git a/policyengine_api/services/v2/household_calculations/types.py b/policyengine_api/services/v2/household_calculations/types.py index 444801963..fe76edd33 100644 --- a/policyengine_api/services/v2/household_calculations/types.py +++ b/policyengine_api/services/v2/household_calculations/types.py @@ -337,9 +337,9 @@ class SimulationServiceUnavailableError(RuntimeError): fetched, missing configuration, and every answer after which no job of the submission runs: a household, policy or catalog version it cannot find (which the Public API checks first), its authentication, routing, - body-size and rate limits, and its own 503 for a database or manifest it - cannot read, or a Modal call it cannot start, after which it fails the - job it had created. + body-size and rate limits, and its own 503 when it is not configured, + cannot read a database or manifest, or cannot start a Modal call, after + which it fails the job it had created. """