From 4aba7b499778591a771fb9d94587f494038faa05 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:00:30 +0400 Subject: [PATCH 1/4] Test that the Public API accepts each live parity household and is given its request timeout (failing) The live parity check's US household puts income_tax on the person, which /calculate refuses, and the check gives /calculate-full 90 s although the candidate lets a request run for 300 s. Both failed the release's staging checks. These tests send each parity household through the Public API's own input checks with the installed country models, and require the check's read timeout to be longer than CLOUD_RUN_TIMEOUT. Co-Authored-By: Claude Opus 5.5 --- .../v2/test_live_household_parity_requests.py | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 tests/unit/v2/test_live_household_parity_requests.py diff --git a/tests/unit/v2/test_live_household_parity_requests.py b/tests/unit/v2/test_live_household_parity_requests.py new file mode 100644 index 000000000..48aa10032 --- /dev/null +++ b/tests/unit/v2/test_live_household_parity_requests.py @@ -0,0 +1,120 @@ +"""The live parity checks' requests to the Public API's own calculation routes. + +Each parity household goes to ``/calculate`` (``/calculate-full`` when it adds +missing variables) as the check sends it, and must pass those routes' input +checks with the installed country models, so a household the routes refuse +fails here rather than in a release. The check waits longer than the +candidate's Cloud Run request timeout (``CLOUD_RUN_TIMEOUT`` in +``.github/scripts/cloud_run_env.sh``) for the answer: ``/calculate-full`` can +take most of it, and the check should get the candidate's own answer. +""" + +from __future__ import annotations + +import json + +import httpx +import pytest + +from policyengine_api.country import COUNTRIES +from policyengine_api.services.household_calculation_service import ( + HouseholdCalculationOptions, + HouseholdCalculationService, + InvalidHouseholdInputsError, +) +from policyengine_api.utils.input_validation import format_invalid_inputs_message +from tests.fixtures.integration.live_household_calculations import ( + LiveHousehold, + calculate_with_v1, +) +from tests.integration.test_live_v2_household_parity import parity_households +from tests.unit.cloud_run_deploy_support import run_bash, script_env + +PARITY_HOUSEHOLDS = parity_households(0) + + +class PublicApiInputChecks: + """The Public API's calculation routes, answering once their input checks pass. + + A refused request gets the routes' 400; an accepted one gets its household + back uncalculated. The read timeout of every request is recorded. + """ + + def __init__(self) -> None: + self.service = HouseholdCalculationService(country_provider=lambda: COUNTRIES) + self.read_timeouts: list[float | None] = [] + + def answer(self, request: httpx.Request) -> httpx.Response: + self.read_timeouts.append(request.extensions["timeout"]["read"]) + country_id, route = request.url.path.strip("/").split("/") + payload = json.loads(request.content) + try: + self.service.prepare_household_calculation( + country_id, + payload["household"], + payload.get("policy", {}), + options=HouseholdCalculationOptions( + add_missing=route == "calculate-full" + ), + ) + except InvalidHouseholdInputsError as error: + message = format_invalid_inputs_message(error.invalid_inputs) + return httpx.Response(400, json={"status": "error", "message": message}) + return httpx.Response( + 200, json={"status": "ok", "result": payload["household"]} + ) + + def calculate(self, household: LiveHousehold) -> dict: + with httpx.Client( + base_url="https://public-api.example", + transport=httpx.MockTransport(self.answer), + ) as client: + return calculate_with_v1(client, household) + + +def candidate_request_timeout_seconds() -> float: + """Return how long Cloud Run lets a request to the candidate run.""" + result = run_bash( + "-c", + "source .github/scripts/cloud_run_env.sh && cloud_run_set_defaults" + ' && printf "%s" "$CLOUD_RUN_TIMEOUT"', + env=script_env(), + ) + assert result.returncode == 0, result.stderr + return float(result.stdout) + + +@pytest.mark.parametrize( + "household", + PARITY_HOUSEHOLDS, + ids=[parity_household.name for parity_household in PARITY_HOUSEHOLDS], +) +def test_the_public_api_accepts_each_parity_household(household) -> None: + answer = PublicApiInputChecks().calculate(household) + + assert answer["status"] == "ok" + + +@pytest.mark.parametrize( + "household", + [ + LiveHousehold(name="calculate", country_id="us", people={"you": {}}), + LiveHousehold( + name="calculate-full", + country_id="us", + people={"you": {}}, + add_missing_variables=True, + ), + ], + ids=["calculate", "calculate-full"], +) +def test_the_check_waits_longer_than_the_candidate_lets_a_request_run( + household, +) -> None: + public_api = PublicApiInputChecks() + + public_api.calculate(household) + + (read_timeout,) = public_api.read_timeouts + assert read_timeout is not None + assert read_timeout > candidate_request_timeout_seconds() From bbee41fc642797dd5bc8ffcc4ba72555109ee356 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:03:19 +0400 Subject: [PATCH 2/4] Put the parity household's income_tax on its tax unit, and wait out /calculate-full The release's live staging checks failed twice: - us-single-california: /calculate answered 400, "Household variable `income_tax` belongs on `tax_units`, not `people`". policyengine_us defines income_tax on the tax unit, and /calculate has refused a variable on the wrong entity since May; the simulation service's household function refuses it the same way. The household now asks for it on its tax unit. - us-single-reform-added-variables: httpx.ReadTimeout after 90 s on /us/calculate-full. The candidate's request log shows that request answered 200 after 223.8 s; production's /calculate-full takes from 50 s to Cloud Run's 300 s request timeout. The check now waits 320 s for the Public API's own calculation, so it gets the candidate's answer, a 504 included. The check also reads the v2 job's status before calling the Public API, so a failed job is reported without waiting minutes for /calculate-full. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/run_live_staging_tests.sh | 5 ++++- .../live-household-parity-checks.fixed.md | 1 + docs/migration/stage-11-v2-households.md | 10 ++++++++-- .../integration/live_household_calculations.py | 16 ++++++++++++++-- .../integration/test_live_v2_household_parity.py | 11 ++++++++--- 5 files changed, 35 insertions(+), 8 deletions(-) create mode 100644 changelog.d/live-household-parity-checks.fixed.md diff --git a/.github/scripts/run_live_staging_tests.sh b/.github/scripts/run_live_staging_tests.sh index 0a2d04ebb..b0f45a3dd 100755 --- a/.github/scripts/run_live_staging_tests.sh +++ b/.github/scripts/run_live_staging_tests.sh @@ -28,7 +28,10 @@ uv run --frozen --extra dev python -m pytest \ # to staging; staging promotion waits on them, so the change that adds them # merges only after that deployment. Their requests are short (a # submission, then a poll every few seconds) apart from the parity tests' -# /calculate requests, but the parallel check submits 6 calculations at once: +# /calculate and /calculate-full requests, which the tests wait for longer +# than the candidate's 300 s request timeout (V1_CALCULATION_TIMEOUT_SECONDS in +# tests/fixtures/integration/live_household_calculations.py): /calculate-full +# took 224 s on staging. The parallel check submits 6 calculations at once: # with the 3 other workers' requests, at most 3 wait for a free slot, as above. uv run --frozen --extra dev python -m pytest \ -n 4 \ diff --git a/changelog.d/live-household-parity-checks.fixed.md b/changelog.d/live-household-parity-checks.fixed.md new file mode 100644 index 000000000..1f2763e4f --- /dev/null +++ b/changelog.d/live-household-parity-checks.fixed.md @@ -0,0 +1 @@ +Fix the live staging household parity checks: the US household's `income_tax` is on its tax unit, where `/calculate` expects it, and the checks wait longer than the candidate's 300 s request timeout for `/calculate-full`. diff --git a/docs/migration/stage-11-v2-households.md b/docs/migration/stage-11-v2-households.md index 2f6a8c784..b6627b4bf 100644 --- a/docs/migration/stage-11-v2-households.md +++ b/docs/migration/stage-11-v2-households.md @@ -380,5 +380,11 @@ cycle, six calculations submitted at once (each accepted within `STAGING_HOUSEHOLD_SUBMIT_LIMIT_SECONDS`, 10 by default, and finished within `STAGING_HOUSEHOLD_FINISH_LIMIT_SECONDS`, 300), a household across 401 axis points, and a parity set of three US and three UK households compared value -by value with `/calculate` and `/calculate-full`. The checks need the -simulation service's household calculation route deployed to staging first. +by value with `/calculate` and `/calculate-full`. The parity set waits 320 s +for each `/calculate` or `/calculate-full` answer, longer than the candidate's +300 s Cloud Run request timeout: `/calculate-full` calculates every variable +the household leaves out, which took 224 s on staging. A unit test sends each +parity household through those routes' input checks with the installed +country models, so a household they refuse fails before a release. The checks +need the simulation service's household calculation route deployed to staging +first. diff --git a/tests/fixtures/integration/live_household_calculations.py b/tests/fixtures/integration/live_household_calculations.py index 1e8d755d2..3f5951fe0 100644 --- a/tests/fixtures/integration/live_household_calculations.py +++ b/tests/fixtures/integration/live_household_calculations.py @@ -50,6 +50,13 @@ TRANSIENT_LIMIT_SECONDS = float( os.environ.get("STAGING_HOUSEHOLD_TRANSIENT_LIMIT_SECONDS", "90") ) +# How long the check waits for the Public API's own /calculate or +# /calculate-full answer: longer than the candidate's Cloud Run request timeout +# (CLOUD_RUN_TIMEOUT in .github/scripts/cloud_run_env.sh, 300 s), so the check +# gets the candidate's own answer, a 504 included. /calculate-full calculates +# every variable the household leaves out and can take most of that: 224 s on +# staging, and up to 300 s in production. +V1_CALCULATION_TIMEOUT_SECONDS = 320.0 FINISHED_STATUSES = {"succeeded", "failed"} YEAR = "2026" REFORM_START = f"{YEAR}-01-01" @@ -279,7 +286,8 @@ def calculate_with_v1(client: httpx.Client, household: LiveHousehold) -> dict: """Return the Public API's own answer for the same household and reform. ``/calculate-full`` answers when the household adds missing variables, - ``/calculate`` otherwise. + ``/calculate`` otherwise. The answer is awaited for + ``V1_CALCULATION_TIMEOUT_SECONDS``. """ route = "calculate-full" if household.add_missing_variables else "calculate" request: dict[str, Any] = {"household": household.situation()} @@ -288,7 +296,11 @@ def calculate_with_v1(client: httpx.Client, household: LiveHousehold) -> dict: parameter_name: {f"{REFORM_START}.{OPEN_END}": value} for parameter_name, value in household.reform.items() } - response = client.post(f"/{household.country_id}/{route}", json=request) + response = client.post( + f"/{household.country_id}/{route}", + json=request, + timeout=V1_CALCULATION_TIMEOUT_SECONDS, + ) assert response.status_code == 200, response.text payload = response.json() assert payload["status"] == "ok", payload diff --git a/tests/integration/test_live_v2_household_parity.py b/tests/integration/test_live_v2_household_parity.py index c8085bff5..decc860f0 100644 --- a/tests/integration/test_live_v2_household_parity.py +++ b/tests/integration/test_live_v2_household_parity.py @@ -39,7 +39,6 @@ def parity_households(run_amount: int) -> list[LiveHousehold]: "employment_income": {YEAR: 42_000 + run_amount}, # Deprecated: dropped with a warning by both calculations. "medical_out_of_pocket_expenses": {YEAR: 1_000}, - "income_tax": CALCULATED, } }, groups={ @@ -47,7 +46,11 @@ def parity_households(run_amount: int) -> list[LiveHousehold]: "state_name": {YEAR: "CA"}, "household_net_income": CALCULATED, }, - "tax_unit": {"eitc": CALCULATED, "ctc": CALCULATED}, + "tax_unit": { + "income_tax": CALCULATED, + "eitc": CALCULATED, + "ctc": CALCULATED, + }, "spm_unit": {"snap": CALCULATED}, }, expects_warnings=True, @@ -174,9 +177,11 @@ def test_a_v2_household_calculation_matches_api_v1( policy_id = store_reform(api_client, household) finished = calculate(api_client, household, household_id, policy_id) + # Checked first, so a failed job is reported without waiting for the + # Public API's own calculation, which can take minutes. + assert finished.item["status"] == "succeeded", finished.item["error"] v1_answer = calculate_with_v1(api_client, household) - assert finished.item["status"] == "succeeded", finished.item["error"] result = finished.item["result"] differences = household_differences(v1_answer["result"], result["household"]) assert differences == [], "\n".join(differences[:50]) From cecc99a442d5822b6a67a9b236fd9bfad4c7be0a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:20:13 +0400 Subject: [PATCH 3/4] Send each parity household through the Public API's real calculation routes The unit test called prepare_household_calculation itself, so it skipped the situation parsing /calculate and /calculate-full run before calculating, and any check added to the routes later. It now registers the household routes on a test Flask app with the installed country models and replaces only the calculation, which takes minutes for /calculate-full. On the old us-single-california household it fails with the body staging answered. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-11-v2-households.md | 5 +- .../v2/test_live_household_parity_requests.py | 101 ++++++++++-------- 2 files changed, 58 insertions(+), 48 deletions(-) diff --git a/docs/migration/stage-11-v2-households.md b/docs/migration/stage-11-v2-households.md index b6627b4bf..56e0d5db9 100644 --- a/docs/migration/stage-11-v2-households.md +++ b/docs/migration/stage-11-v2-households.md @@ -384,7 +384,8 @@ by value with `/calculate` and `/calculate-full`. The parity set waits 320 s for each `/calculate` or `/calculate-full` answer, longer than the candidate's 300 s Cloud Run request timeout: `/calculate-full` calculates every variable the household leaves out, which took 224 s on staging. A unit test sends each -parity household through those routes' input checks with the installed -country models, so a household they refuse fails before a release. The checks +parity household through those routes, which check its inputs and parse its +situation with the installed country models, leaving out only the +calculation, so a household they refuse fails before a release. The checks need the simulation service's household calculation route deployed to staging first. diff --git a/tests/unit/v2/test_live_household_parity_requests.py b/tests/unit/v2/test_live_household_parity_requests.py index 48aa10032..3c2a66cca 100644 --- a/tests/unit/v2/test_live_household_parity_requests.py +++ b/tests/unit/v2/test_live_household_parity_requests.py @@ -1,28 +1,31 @@ """The live parity checks' requests to the Public API's own calculation routes. Each parity household goes to ``/calculate`` (``/calculate-full`` when it adds -missing variables) as the check sends it, and must pass those routes' input -checks with the installed country models, so a household the routes refuse -fails here rather than in a release. The check waits longer than the -candidate's Cloud Run request timeout (``CLOUD_RUN_TIMEOUT`` in +missing variables) as the check sends it. The real routes check its inputs and +parse its situation with the installed country models, and only the +calculation itself is left out, so a household the routes refuse fails here +rather than in a release. The check waits longer than the candidate's Cloud +Run request timeout (``CLOUD_RUN_TIMEOUT`` in ``.github/scripts/cloud_run_env.sh``) for the answer: ``/calculate-full`` can take most of it, and the check should get the candidate's own answer. """ from __future__ import annotations -import json +from typing import TYPE_CHECKING import httpx import pytest +from flask import Flask from policyengine_api.country import COUNTRIES +from policyengine_api.extensions import cache +from policyengine_api.routes import household_routes from policyengine_api.services.household_calculation_service import ( - HouseholdCalculationOptions, + HouseholdCalculationResult, HouseholdCalculationService, - InvalidHouseholdInputsError, + PreparedHouseholdCalculation, ) -from policyengine_api.utils.input_validation import format_invalid_inputs_message from tests.fixtures.integration.live_household_calculations import ( LiveHousehold, calculate_with_v1, @@ -30,46 +33,43 @@ from tests.integration.test_live_v2_household_parity import parity_households from tests.unit.cloud_run_deploy_support import run_bash, script_env +if TYPE_CHECKING: + from collections.abc import Iterator + PARITY_HOUSEHOLDS = parity_households(0) -class PublicApiInputChecks: - """The Public API's calculation routes, answering once their input checks pass. +class UncalculatedHouseholds(HouseholdCalculationService): + """The routes' household service, answering each accepted household as sent. - A refused request gets the routes' 400; an accepted one gets its household - back uncalculated. The read timeout of every request is recorded. + Input checks and situation parsing run as in the Public API; the + calculation, which takes minutes for ``/calculate-full``, does not. """ - def __init__(self) -> None: - self.service = HouseholdCalculationService(country_provider=lambda: COUNTRIES) - self.read_timeouts: list[float | None] = [] - - def answer(self, request: httpx.Request) -> httpx.Response: - self.read_timeouts.append(request.extensions["timeout"]["read"]) - country_id, route = request.url.path.strip("/").split("/") - payload = json.loads(request.content) - try: - self.service.prepare_household_calculation( - country_id, - payload["household"], - payload.get("policy", {}), - options=HouseholdCalculationOptions( - add_missing=route == "calculate-full" - ), - ) - except InvalidHouseholdInputsError as error: - message = format_invalid_inputs_message(error.invalid_inputs) - return httpx.Response(400, json={"status": "error", "message": message}) - return httpx.Response( - 200, json={"status": "ok", "result": payload["household"]} + def calculate_prepared_household( + self, prepared: PreparedHouseholdCalculation + ) -> HouseholdCalculationResult: + return HouseholdCalculationResult( + household=prepared.household_json, warnings=prepared.warnings ) - def calculate(self, household: LiveHousehold) -> dict: - with httpx.Client( - base_url="https://public-api.example", - transport=httpx.MockTransport(self.answer), - ) as client: - return calculate_with_v1(client, household) + +@pytest.fixture +def public_api(monkeypatch: pytest.MonkeyPatch) -> Iterator[httpx.Client]: + """Call the Public API's calculation routes with the installed country models.""" + monkeypatch.setattr( + household_routes, + "household_calculation_service", + UncalculatedHouseholds(country_provider=lambda: COUNTRIES), + ) + app = Flask(__name__) + app.config.update(TESTING=True, CACHE_TYPE="NullCache") + cache.init_app(app) + app.register_blueprint(household_routes.household_bp) + with httpx.Client( + base_url="http://public-api.test", transport=httpx.WSGITransport(app=app) + ) as client: + yield client def candidate_request_timeout_seconds() -> float: @@ -89,8 +89,10 @@ def candidate_request_timeout_seconds() -> float: PARITY_HOUSEHOLDS, ids=[parity_household.name for parity_household in PARITY_HOUSEHOLDS], ) -def test_the_public_api_accepts_each_parity_household(household) -> None: - answer = PublicApiInputChecks().calculate(household) +def test_the_public_api_accepts_each_parity_household( + public_api: httpx.Client, household: LiveHousehold +) -> None: + answer = calculate_with_v1(public_api, household) assert answer["status"] == "ok" @@ -109,12 +111,19 @@ def test_the_public_api_accepts_each_parity_household(household) -> None: ids=["calculate", "calculate-full"], ) def test_the_check_waits_longer_than_the_candidate_lets_a_request_run( - household, + household: LiveHousehold, ) -> None: - public_api = PublicApiInputChecks() + read_timeouts: list[float | None] = [] + + def answer(request: httpx.Request) -> httpx.Response: + read_timeouts.append(request.extensions["timeout"]["read"]) + return httpx.Response(200, json={"status": "ok", "result": {}}) - public_api.calculate(household) + with httpx.Client( + base_url="https://public-api.example", transport=httpx.MockTransport(answer) + ) as client: + calculate_with_v1(client, household) - (read_timeout,) = public_api.read_timeouts + (read_timeout,) = read_timeouts assert read_timeout is not None assert read_timeout > candidate_request_timeout_seconds() From 46cf2f4918f26ad4eb73d2f2be684bfd4267e454 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:37:28 +0400 Subject: [PATCH 4/4] Say which refusals the parity household test covers The routes' answer under test is the prepared household, with deprecated inputs dropped and missing variables added, and a refusal during the calculation itself, such as an SPM error, is outside the test. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-11-v2-households.md | 7 ++++--- tests/unit/v2/test_live_household_parity_requests.py | 12 ++++++++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/migration/stage-11-v2-households.md b/docs/migration/stage-11-v2-households.md index 56e0d5db9..39b3bb3f0 100644 --- a/docs/migration/stage-11-v2-households.md +++ b/docs/migration/stage-11-v2-households.md @@ -384,8 +384,9 @@ by value with `/calculate` and `/calculate-full`. The parity set waits 320 s for each `/calculate` or `/calculate-full` answer, longer than the candidate's 300 s Cloud Run request timeout: `/calculate-full` calculates every variable the household leaves out, which took 224 s on staging. A unit test sends each -parity household through those routes, which check its inputs and parse its -situation with the installed country models, leaving out only the -calculation, so a household they refuse fails before a release. The checks +parity household through those routes with the installed country models and +replaces only the calculation, so a household the routes refuse before +calculating, in their input checks or situation parsing, fails before a +release. The checks need the simulation service's household calculation route deployed to staging first. diff --git a/tests/unit/v2/test_live_household_parity_requests.py b/tests/unit/v2/test_live_household_parity_requests.py index 3c2a66cca..ec9973705 100644 --- a/tests/unit/v2/test_live_household_parity_requests.py +++ b/tests/unit/v2/test_live_household_parity_requests.py @@ -3,8 +3,10 @@ Each parity household goes to ``/calculate`` (``/calculate-full`` when it adds missing variables) as the check sends it. The real routes check its inputs and parse its situation with the installed country models, and only the -calculation itself is left out, so a household the routes refuse fails here -rather than in a release. The check waits longer than the candidate's Cloud +calculation itself is left out, so a household the routes refuse before +calculating fails here rather than in a release. A refusal during the +calculation, such as an SPM error, is not covered. The check waits longer than +the candidate's Cloud Run request timeout (``CLOUD_RUN_TIMEOUT`` in ``.github/scripts/cloud_run_env.sh``) for the answer: ``/calculate-full`` can take most of it, and the check should get the candidate's own answer. @@ -40,10 +42,12 @@ class UncalculatedHouseholds(HouseholdCalculationService): - """The routes' household service, answering each accepted household as sent. + """The routes' household service, answering each accepted household uncalculated. Input checks and situation parsing run as in the Public API; the - calculation, which takes minutes for ``/calculate-full``, does not. + calculation, which takes minutes for ``/calculate-full``, does not. The + answer is the household as the routes prepared it: deprecated inputs + dropped and, for ``/calculate-full``, the missing variables added. """ def calculate_prepared_household(