From cc2fa863a1f1a9e2d658e7ea45172e63de04b07a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 08:12:27 -0400 Subject: [PATCH 1/3] Isolate economy caches and verify worker SPM capability --- .../check-policyengine-bundle-supported.sh | 23 ++++- .github/request-simulation-model-versions.sh | 12 ++- .github/workflows/pr.yml | 2 + .github/workflows/push.yml | 4 + .../spm-cache-worker-capability.fixed.md | 1 + docs/canonical-spm.md | 27 +++++ policyengine_api/services/economy_service.py | 8 ++ policyengine_api/worker_spm_release.py | 63 ++++++++++++ tests/fixtures/services/economy_service.py | 2 +- .../test_budget_window_in_flight_dedupe.py | 3 + tests/integration/test_cloud_run_candidate.py | 49 ++++++++++ .../routes/test_economy_failure_replay.py | 40 ++++++++ .../routes/test_spm_year_worker_polling.py | 40 +++++--- tests/unit/test_candidate_economy_smoke.py | 98 +++++++++++++++++++ tests/unit/test_cloud_run_deploy_scripts.py | 96 ++++++++++++++++++ tests/unit/test_worker_spm_release.py | 88 +++++++++++++++++ 16 files changed, 538 insertions(+), 18 deletions(-) create mode 100644 changelog.d/spm-cache-worker-capability.fixed.md create mode 100644 policyengine_api/worker_spm_release.py create mode 100644 tests/unit/test_candidate_economy_smoke.py create mode 100644 tests/unit/test_worker_spm_release.py diff --git a/.github/check-policyengine-bundle-supported.sh b/.github/check-policyengine-bundle-supported.sh index f48d1f076..696b526af 100755 --- a/.github/check-policyengine-bundle-supported.sh +++ b/.github/check-policyengine-bundle-supported.sh @@ -32,7 +32,12 @@ extract_policyengine_version() { sed -n 's/.*policyengine\[models\]==\([0-9.][0-9.]*\).*/\1/p' | head -n 1 } +extract_calculator_version() { + sed -n 's/.*spm-calculator==\([0-9.][0-9.]*\).*/\1/p' | head -n 1 +} + current_version="$(extract_policyengine_version < pyproject.toml)" +current_calculator="$(extract_calculator_version < pyproject.toml)" if [ -z "$current_version" ]; then echo "ERROR: policyengine[models] pin not found in pyproject.toml" exit 1 @@ -51,10 +56,22 @@ if [ "$CHECK_ONLY_IF_CHANGED" = "1" ]; then || true )" - if [ "$current_version" = "$base_version" ]; then - echo "PolicyEngine .py bundle pin is unchanged; skipping simulation API support check." + base_calculator="$( + git show "origin/${BASE_REF}:pyproject.toml" \ + | extract_calculator_version \ + || true + )" + + if [ "$current_version" = "$base_version" ] \ + && [ "$current_calculator" = "$base_calculator" ] \ + && git diff --quiet "origin/${BASE_REF}" -- \ + policyengine_api/spm.py policyengine_api/worker_spm.py \ + policyengine_api/worker_spm_release.py policyengine_api/constants.py \ + policyengine_api/country.py .github/check-policyengine-bundle-supported.sh \ + .github/request-simulation-model-versions.sh; then + echo "Bundle/calculator pins and SPM integration are unchanged; skipping simulation API support check." exit 0 fi fi -bash "$VERSION_GUARD_SCRIPT" -py "$current_version" +bash "$VERSION_GUARD_SCRIPT" -py "$current_version" --check-installed-spm diff --git a/.github/request-simulation-model-versions.sh b/.github/request-simulation-model-versions.sh index 27a68062c..bbb65cc7f 100755 --- a/.github/request-simulation-model-versions.sh +++ b/.github/request-simulation-model-versions.sh @@ -23,12 +23,14 @@ usage() { echo "Optional compatibility checks:" echo " -us us_version Expected bundled policyengine-us version" echo " -uk uk_version Expected bundled policyengine-uk version" + echo " --check-installed-spm Validate the installed API bundle's SPM capability" exit 1 } POLICYENGINE_VERSION="" US_VERSION="" UK_VERSION="" +CHECK_INSTALLED_SPM=0 while [ $# -gt 0 ]; do case "$1" in @@ -44,6 +46,10 @@ while [ $# -gt 0 ]; do UK_VERSION="$2" shift 2 ;; + --check-installed-spm) + CHECK_INSTALLED_SPM=1 + shift + ;; -h|--help) usage ;; @@ -70,7 +76,7 @@ if [ -n "$UK_VERSION" ]; then fi echo "" -VERSIONS_RESPONSE=$(curl -s "${GATEWAY_URL}/versions") +VERSIONS_RESPONSE=$(curl --fail --silent --show-error --connect-timeout 10 --max-time 60 "${GATEWAY_URL}/versions") if [ -z "$VERSIONS_RESPONSE" ]; then echo "ERROR: Failed to fetch versions from gateway" @@ -114,6 +120,10 @@ check_country_route() { check_country_route "us" "$US_VERSION" check_country_route "uk" "$UK_VERSION" +if [ "$CHECK_INSTALLED_SPM" = "1" ]; then + printf '%s' "$VERSIONS_RESPONSE" | uv run --frozen python -m policyengine_api.worker_spm_release "$POLICYENGINE_VERSION" +fi + echo "" echo "SUCCESS: PolicyEngine bundle route is deployed and ready" exit 0 diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 3cc22e682..b193772f6 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -23,6 +23,8 @@ jobs: fetch-depth: 0 - name: Install jq run: sudo apt-get install -y jq + - name: Setup uv for the installed bundle capability check + uses: astral-sh/setup-uv@v6 - name: Check simulation API supports updated PolicyEngine bundle run: bash .github/check-policyengine-bundle-supported.sh --if-changed-from-base "${{ github.base_ref }}" diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 1390bb8cb..e735f5a73 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -60,6 +60,8 @@ jobs: uses: actions/checkout@v4 - name: Install jq run: sudo apt-get install -y jq + - name: Setup uv for the installed bundle capability check + uses: astral-sh/setup-uv@v6 - name: Check simulation API supports PolicyEngine bundle run: bash .github/check-policyengine-bundle-supported.sh @@ -391,6 +393,8 @@ jobs: uses: actions/checkout@v4 - name: Install jq run: sudo apt-get install -y jq + - name: Setup uv for the installed bundle capability check + uses: astral-sh/setup-uv@v6 - name: Check simulation API supports PolicyEngine bundle run: bash .github/check-policyengine-bundle-supported.sh diff --git a/changelog.d/spm-cache-worker-capability.fixed.md b/changelog.d/spm-cache-worker-capability.fixed.md new file mode 100644 index 000000000..2521e0beb --- /dev/null +++ b/changelog.d/spm-cache-worker-capability.fixed.md @@ -0,0 +1 @@ +Isolate economy caches by calculation target and budget-window worker application, and verify the installed API bundle's SPM capability and forecast against the selected worker before deployment. Exercise current-law economy results and canonical receipts on each Cloud Run candidate. diff --git a/docs/canonical-spm.md b/docs/canonical-spm.md index 524746218..185f76bd4 100644 --- a/docs/canonical-spm.md +++ b/docs/canonical-spm.md @@ -281,3 +281,30 @@ bundle defaults. The selection schema deliberately supplies no wire defaults, so generated clients preserve that distinction. Resolved settings freeze all six fields. Responses may omit null geography_id/as_of values, but must include every non-null resolved field; receipt replay never substitutes current defaults. + + +## Economy deployment checks and cache transitions + +Deployment alignment validates the installed wrapper version and the runtime's +manifest-selected version, then applies the request-time SPM capability and +forecast-hash checks to the registered bundle. Legacy bundles require bundle +registration but do not advertise canonical SPM. A canonical promotion must +coordinate the wrapper and calculator pins and pass qualification with their +published wheels; the legacy dependency tuple does not qualify that path. + +Annual economy cache identities include the calculation target. Budget-window +identities include the resolved worker application before errors, completed +results, or running batch handles are read. The budget-window registry lookup +must succeed even for a cached response: a registry outage fails the request +rather than replaying an unverified predecessor's result. A replacement worker +receives a new cache key; the same worker's terminal error remains terminal. + +The first deployment of these identities makes older economy cache entries +ineligible. Their ordinary expiry remains in place. Active jobs under the old +keys may finish, but requests under the new keys can submit replacement jobs; +account for this one-time recomputation window when scheduling promotion. + +Both tagged Cloud Run candidates run a Utah current-law/current-law economy +probe without creating a policy. It requires zero budget change and, for a +canonical bundle, matching settings and baseline/reform forecast receipts. +National numerical acceptance remains a separate release qualification. diff --git a/policyengine_api/services/economy_service.py b/policyengine_api/services/economy_service.py index fc3947f84..7c7f7f50d 100644 --- a/policyengine_api/services/economy_service.py +++ b/policyengine_api/services/economy_service.py @@ -387,6 +387,9 @@ def get_budget_window_economic_impact( api_version=api_version, target=target, ) + # Terminal failures and completed batches belong to the worker that + # produced them. A replacement application must get a fresh key. + self._resolve_runtime_bundle_for_setup_options(setup_options) cache_key = self._build_budget_window_cache_key(setup_options) cached_error = self._budget_window_cache.get_terminal_error(cache_key) @@ -707,6 +710,7 @@ def _build_economic_impact_setup_options( dataset=resolved_dataset, data_version=resolved_data_version, policyengine_version=policyengine_version, + target=target, ) return EconomicImpactSetupOptions.model_validate( @@ -844,6 +848,7 @@ def _resolve_runtime_bundle_for_setup_options( data_version=setup_options.data_version, policyengine_version=setup_options.policyengine_version, runtime_app_name=setup_options.runtime_app_name, + target=setup_options.target, ) def _reform_impact_start_claim_arguments( @@ -1288,11 +1293,13 @@ def _build_options_hash( runtime_app_name: str | None = None, data_version: str | None = None, policyengine_version: str | None = None, + target: Literal["general", "cliff"] = "general", ) -> str: option_pairs = "&".join(f"{key}={options[key]}" for key in sorted(options)) bundle_parts = [ f"dataset={dataset}", f"model_version={model_version}", + f"target={target}", ] if data_version: bundle_parts.append(f"data_version={data_version}") @@ -1353,6 +1360,7 @@ def _should_refresh_cached_impact( data_version=setup_options.data_version, policyengine_version=setup_options.policyengine_version, runtime_app_name=runtime_app_name, + target=setup_options.target, ) if ( not isinstance(cached_resolved_app_name, str) diff --git a/policyengine_api/worker_spm_release.py b/policyengine_api/worker_spm_release.py new file mode 100644 index 000000000..a9b41fb0b --- /dev/null +++ b/policyengine_api/worker_spm_release.py @@ -0,0 +1,63 @@ +"""Validate the installed API bundle against the selected worker release registry.""" + +from importlib.metadata import version +import json +import sys + +from policyengine_api.worker_spm import validate_worker_spm + + +class _RegistryCapability: + def __init__(self, registry: dict): + self.registry = registry + + def get_spm_capability(self, country, model_version, *, policyengine_version): + capabilities = self.registry.get("spm_capabilities") + return ( + capabilities.get(policyengine_version) + if isinstance(capabilities, dict) + else None + ) + + +def validate_installed_worker(registry: dict, expected_bundle: str) -> dict | None: + """Fail before promotion if the selected worker cannot run this API bundle. + + Use the request validator directly: a legacy country can omit capability, + while a canonical country requires certification and a matching forecast. + """ + if version("policyengine") != expected_bundle: + raise ValueError( + "The installed PolicyEngine bundle differs from the release pin" + ) + from policyengine_api.constants import POLICYENGINE_VERSION + + if POLICYENGINE_VERSION != expected_bundle: + raise ValueError("The runtime-selected bundle differs from the release pin") + routes = registry.get("policyengine") if isinstance(registry, dict) else None + app = routes.get(expected_bundle) if isinstance(routes, dict) else None + if not isinstance(app, str) or not app: + raise ValueError("The selected bundle has no registered worker application") + return validate_worker_spm( + "us", + gateway=_RegistryCapability(registry), + policyengine_version=expected_bundle, + ) + + +def main() -> None: + if len(sys.argv) != 2: + raise SystemExit( + "Usage: python -m policyengine_api.worker_spm_release BUNDLE_VERSION" + ) + selection = validate_installed_worker(json.load(sys.stdin), sys.argv[1]) + if selection is None: + print( + "Legacy API bundle is registered; canonical SPM capability is not required" + ) + else: + print("Installed API bundle and registered bundle's SPM capability agree") + + +if __name__ == "__main__": + main() diff --git a/tests/fixtures/services/economy_service.py b/tests/fixtures/services/economy_service.py index 02553bf39..68656726e 100644 --- a/tests/fixtures/services/economy_service.py +++ b/tests/fixtures/services/economy_service.py @@ -26,7 +26,7 @@ "[option1=value1&option2=value2" "&dataset=hf://policyengine/faux-populace-us/faux_populace_us_2099.h5@" "faux-populace-us-2099-test-release" - "&model_version=1.2.3" + "&model_version=1.2.3&target=general" "&data_version=faux-populace-us-2099-test-release" "&policyengine_version=3.4.0]" ) diff --git a/tests/integration/test_budget_window_in_flight_dedupe.py b/tests/integration/test_budget_window_in_flight_dedupe.py index c944224b6..9672f1331 100644 --- a/tests/integration/test_budget_window_in_flight_dedupe.py +++ b/tests/integration/test_budget_window_in_flight_dedupe.py @@ -31,6 +31,9 @@ def test_budget_window_in_flight_dedupe_uses_existing_batch_without_live_db( fake_cache = BudgetWindowCache(client=FakeRedis()) simulation_entrypoint = MagicMock() + simulation_entrypoint.resolve_app_name.side_effect = ( + lambda country, version, **kwargs: ("test-budget-window-worker", version) + ) reform_impacts_service = MagicMock() simulation_entrypoint.run_budget_window_batch.return_value = ( diff --git a/tests/integration/test_cloud_run_candidate.py b/tests/integration/test_cloud_run_candidate.py index 81fb8df8f..2d5509a04 100644 --- a/tests/integration/test_cloud_run_candidate.py +++ b/tests/integration/test_cloud_run_candidate.py @@ -1,3 +1,7 @@ +from datetime import date +import math + + def test_cloud_run_candidate_health_routes(api_client): health_response = api_client.get("/health") assert health_response.status_code == 200, health_response.text @@ -66,3 +70,48 @@ def test_cloud_run_candidate_stage6_read_route_contracts(api_client): "Country zz not found. Available countries are: uk, us, ca, ng, il" ), } + + +def test_cloud_run_candidate_current_law_economy(api_client, poll_live_endpoint): + """Exercise the candidate's selected worker without creating a reform policy.""" + response = api_client.get("/us/metadata") + response.raise_for_status() + metadata = response.json()["result"] + current_law = metadata["current_law_id"] + years = [ + int(period["name"]) + for period in metadata["economy_options"]["time_period"] + if str(period["name"]).isdigit() + ] + assert years, "Metadata exposes no economy years" + current_year = date.today().year + year = str( + max((value for value in years if value <= current_year), default=min(years)) + ) + payload = poll_live_endpoint( + api_client, + f"/us/economy/{current_law}/over/{current_law}", + {"region": "ut", "time_period": year}, + route_name="Cloud Run candidate current-law economy", + ) + assert payload["status"] == "ok", payload + result = payload["result"] + impact = result["budget"]["budgetary_impact"] + assert isinstance(impact, int | float) and math.isfinite(impact), result + assert impact == 0, "Comparing current law with itself must have zero budget impact" + spm = metadata.get("spm", {}) + if spm.get("available"): + defaults = spm["defaults"] + config = result["spm_config"] + assert isinstance(config, dict) and set(config) <= set(defaults) + # HTTP transports may omit optional null settings; they may never + # inherit a missing non-null default or introduce an unknown setting. + assert all(config.get(key) == value for key, value in defaults.items()) + for side in ("baseline", "reform"): + receipts = result["spm_provenance"][side] + assert receipts, f"Missing {side} SPM receipts" + for receipt in receipts: + assert receipt["forecast_sha256"] == defaults["forecast_content_sha256"] + assert receipt["scenario"] == defaults["scenario"] + assert receipt["geography_kind"] == defaults["geography_kind"] + assert year in receipt["years"] diff --git a/tests/unit/routes/test_economy_failure_replay.py b/tests/unit/routes/test_economy_failure_replay.py index a1889f603..bc3a1338f 100644 --- a/tests/unit/routes/test_economy_failure_replay.py +++ b/tests/unit/routes/test_economy_failure_replay.py @@ -131,3 +131,43 @@ def test_successful_cached_job_still_requires_canonical_receipts(failed_job_harn assert response.json["errors"][0]["code"] == "SPM_CONFIGURATION_UNAVAILABLE" gateway.get_execution_by_id.assert_not_called() gateway.run.assert_not_called() + + +def test_annual_cliff_request_does_not_replay_general_failure( + failed_job_harness, monkeypatch +): + client, _, gateway, _, _ = failed_job_harness + url = "/us/economy/1/over/2?region=us&time_period=2026&version=1.0.0" + assert client.get(url).status_code == 502 + service = economy_routes.economy_service + monkeypatch.setattr(service, "_get_policy_jsons", lambda *_: ({}, {})) + gateway.run.return_value = SimpleNamespace(execution_id="new-cliff-job") + gateway.get_execution_id.return_value = "new-cliff-job" + response = client.get(url + "&target=cliff") + assert response.status_code == 200, response.json + assert response.json["status"] == "computing" + assert gateway.run.call_count == 1 + assert gateway.run.call_args.args[0]["include_cliffs"] is True + + +@pytest.mark.parametrize("failed_job_harness", [None], indirect=True, ids=["legacy"]) +def test_cliff_request_does_not_reuse_completed_general_result( + failed_job_harness, monkeypatch +): + client, cache, gateway, _, _ = failed_job_harness + cache.update( + EXECUTION_ID, + status="ok", + message="Completed", + reform_impact_json={"cliff_impact": None, "resolved_app_name": "test-worker"}, + ) + url = "/us/economy/1/over/2?region=us&time_period=2026&version=1.0.0" + assert client.get(url).status_code == 200 + service = economy_routes.economy_service + monkeypatch.setattr(service, "_get_policy_jsons", lambda *_: ({}, {})) + gateway.run.return_value = SimpleNamespace(execution_id="new-cliff-job") + gateway.get_execution_id.return_value = "new-cliff-job" + response = client.get(url + "&target=cliff") + assert response.status_code == 200, response.json + assert response.json["status"] == "computing" + assert gateway.run.call_args.args[0]["include_cliffs"] is True diff --git a/tests/unit/routes/test_spm_year_worker_polling.py b/tests/unit/routes/test_spm_year_worker_polling.py index 797a57097..c0e575c7c 100644 --- a/tests/unit/routes/test_spm_year_worker_polling.py +++ b/tests/unit/routes/test_spm_year_worker_polling.py @@ -94,9 +94,19 @@ def test_typed_worker_error_is_terminal_and_replays_after_service_recreation( def transport(request): received.append((request.method, request.url.path)) if request.method == "POST" and worker_app == "replacement-worker": - assert request.url.path == "/simulate/economy/comparison" + expected_path = ( + "/simulate/economy/budget-window" + if budget_window + else "/simulate/economy/comparison" + ) + assert request.url.path == expected_path return httpx.Response( - 200, json={"job_id": "replacement-job", "status": "submitted"} + 200, + json={ + "job_id": "replacement-job", + "batch_job_id": "replacement-job", + "status": "submitted", + }, ) assert request.method == "GET", "Polling must not submit a replacement job" if request.url.path == "/versions": @@ -155,6 +165,7 @@ def transport(request): api_version="1.0.0", ) if budget_window: + service._resolve_runtime_bundle_for_setup_options(setup) cache_key = service._build_budget_window_cache_key(setup) window_cache.store_batch_job_id(cache_key, job_id) url = "/us/economy/1/over/2/budget-window" @@ -219,17 +230,19 @@ def transport(request): ) monkeypatch.setattr(economy_routes, "economy_service", service) - if not budget_window: - # Preserve the existing retry policy: only a newly resolved worker - # creates a fresh canonical cache identity and permits submission. - worker_app = "replacement-worker" - monkeypatch.setattr(service, "_get_policy_jsons", lambda *_: ({}, {})) - response = app.test_client().get( - url, - query_string={"region": "us", "version": "1.0.0", **query}, - ) - assert response.status_code == 200, response.json - assert response.json["status"] == "computing" + # Only a newly resolved worker permits a replacement submission. + worker_app = "replacement-worker" + monkeypatch.setattr(service, "_get_policy_jsons", lambda *_: ({}, {})) + response = app.test_client().get( + url, + query_string={"region": "us", "version": "1.0.0", **query}, + ) + assert response.status_code == 200, response.json + assert response.json["status"] == "computing" + if budget_window: + assert window_cache.get_terminal_error(cache_key) == typed_error + assert received.count(("POST", "/simulate/economy/budget-window")) == 1 + else: replacement = annual_cache.get_by_execution_id("replacement-job") assert replacement.options_hash != setup.options_hash assert replacement.options_json == setup.options @@ -307,6 +320,7 @@ def transport(request): api_version="1.0.0", ) if budget_window: + service._resolve_runtime_bundle_for_setup_options(setup) cache_key = service._build_budget_window_cache_key(setup) window_cache.set_completed_result( cache_key, diff --git a/tests/unit/test_candidate_economy_smoke.py b/tests/unit/test_candidate_economy_smoke.py new file mode 100644 index 000000000..2388eb289 --- /dev/null +++ b/tests/unit/test_candidate_economy_smoke.py @@ -0,0 +1,98 @@ +"""The deployment smoke rejects canonical results without matching receipts.""" + +from copy import deepcopy +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from tests.integration.test_cloud_run_candidate import ( + test_cloud_run_candidate_current_law_economy as run_smoke, +) + + +@pytest.fixture +def smoke(): + defaults = { + "forecast_content_sha256": "a" * 64, + "scenario": "baseline", + "geography_kind": "county", + } + metadata = { + "current_law_id": 2, + "economy_options": {"time_period": [{"name": "2025"}]}, + "spm": {"available": True, "defaults": defaults}, + } + receipt = { + "forecast_sha256": "a" * 64, + "scenario": "baseline", + "geography_kind": "county", + "years": {"2025": {}}, + } + result = { + "budget": {"budgetary_impact": 0}, + "spm_config": defaults, + "spm_provenance": { + "baseline": [deepcopy(receipt)], + "reform": [deepcopy(receipt)], + }, + } + client = SimpleNamespace( + get=Mock( + return_value=SimpleNamespace( + raise_for_status=lambda: None, json=lambda: {"result": metadata} + ) + ) + ) + poll = Mock(return_value={"status": "ok", "result": result}) + return client, poll, metadata, result + + +def test_current_law_smoke_uses_no_policy_write(smoke): + client, poll, _, _ = smoke + run_smoke(client, poll) + client.get.assert_called_once_with("/us/metadata") + assert poll.call_args.args[1] == "/us/economy/2/over/2" + assert poll.call_args.args[2] == {"region": "ut", "time_period": "2025"} + + +@pytest.mark.parametrize( + "error", ["missing", "hash", "year", "budget", "nonzero-budget"] +) +def test_current_law_smoke_rejects_incomplete_worker_evidence(smoke, error): + client, poll, _, result = smoke + if error == "missing": + result["spm_provenance"]["reform"] = [] + elif error == "hash": + result["spm_provenance"]["reform"][0]["forecast_sha256"] = "b" * 64 + elif error == "year": + result["spm_provenance"]["reform"][0]["years"] = {"2024": {}} + elif error == "nonzero-budget": + result["budget"]["budgetary_impact"] = 1 + else: + result["budget"]["budgetary_impact"] = float("nan") + with pytest.raises(AssertionError): + run_smoke(client, poll) + + +def test_current_law_smoke_accepts_legacy_without_receipts(smoke): + client, poll, metadata, result = smoke + metadata["spm"] = {"available": False} + del result["spm_provenance"] + run_smoke(client, poll) + + +def test_current_law_smoke_accepts_only_null_setting_omissions(smoke): + client, poll, metadata, result = smoke + metadata["spm"]["defaults"] = { + **metadata["spm"]["defaults"], + "as_of": None, + "geography_id": None, + } + run_smoke(client, poll) + result["spm_config"] = { + "forecast_content_sha256": "a" * 64, + "geography_kind": "county", + } + with pytest.raises(AssertionError): + run_smoke(client, poll) diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index c9ef51054..479567428 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -260,6 +260,7 @@ def _run_simulation_version_guard( versions_response: dict, *args: str, entrypoint: str = "old_gateway_direct", + extra_env: dict | None = None, ) -> subprocess.CompletedProcess[str]: versions_json = json.dumps(versions_response) command = ( @@ -274,6 +275,7 @@ def _run_simulation_version_guard( SIM_ENTRYPOINT=entrypoint, SIMULATION_ENTRYPOINT_URL="https://simulation.example.test", OLD_SIMULATION_GATEWAY_URL="https://old-gateway.example.test", + **(extra_env or {}), ), text=True, capture_output=True, @@ -482,6 +484,7 @@ def test_policyengine_bundle_support_check_passes_pyproject_pin_to_guard(tmp_pat assert capture_path.read_text(encoding="utf-8").splitlines() == [ "-py", current_version, + "--check-installed-spm", ] @@ -2125,3 +2128,96 @@ def test_push_workflow_never_uses_latest_or_tag_alias_for_cloud_run_traffic(): assert "--to-latest" not in workflow.lower() assert "--to-revisions" in promotion_script assert '"${target_revision}=100"' in promotion_script + + +def test_bundle_alignment_jobs_install_uv_before_capability_check(): + workflows = [ + (_push_workflow(), "ensure-staging-model-version-aligns-with-sim-api"), + (_push_workflow(), "ensure-production-model-version-aligns-with-sim-api"), + ( + (REPO / ".github/workflows/pr.yml").read_text(), + "ensure-policyengine-bundle-supported-by-simulation-api", + ), + ] + for workflow, name in workflows: + job = _workflow_job_block(workflow, name) + assert job.index("uses: astral-sh/setup-uv@v6") < job.index( + "bash .github/check-policyengine-bundle-supported.sh" + ) + + +@pytest.mark.parametrize("exit_code", [0, 23]) +def test_alignment_guard_preserves_installed_capability_check_result( + tmp_path, exit_code +): + capture = tmp_path / "registry.json" + args = tmp_path / "uv-args.txt" + uv = tmp_path / "uv" + uv.write_text( + '#!/bin/sh\nprintf "%s\\n" "$@" > "$CAPTURE_ARGS"\ncat > "$CAPTURE_REGISTRY"\nexit "$CHECK_EXIT"\n' + ) + uv.chmod(0o755) + registry = {"policyengine": {"6.0.0": "canonical-worker"}, "spm_capabilities": {}} + result = _run_simulation_version_guard( + registry, + "-py", + "6.0.0", + "--check-installed-spm", + extra_env={ + "PATH": f"{tmp_path}:{os.environ['PATH']}", + "CAPTURE_ARGS": str(args), + "CAPTURE_REGISTRY": str(capture), + "CHECK_EXIT": str(exit_code), + }, + ) + assert result.returncode == exit_code, result.stderr + assert json.loads(capture.read_text()) == registry + assert args.read_text().splitlines() == [ + "run", + "--frozen", + "python", + "-m", + "policyengine_api.worker_spm_release", + "6.0.0", + ] + assert ("SUCCESS:" in result.stdout) == (exit_code == 0) + + +@pytest.mark.parametrize("change", ["none", "calculator", "spm-source"]) +def test_bundle_alignment_skip_includes_calculator_and_spm_source(tmp_path, change): + base_project = tmp_path / "base-pyproject.toml" + project = (REPO / "pyproject.toml").read_text() + if change == "calculator": + project = re.sub(r"spm-calculator==[0-9.]+", "spm-calculator==0.0.0", project) + base_project.write_text(project) + git = tmp_path / "git" + git.write_text( + '#!/bin/sh\ncase "$1" in\nfetch) exit 0;;\nshow) cat "$BASE_PROJECT";;\ndiff) exit "$DIFF_STATUS";;\n*) exit 2;;\nesac\n' + ) + git.chmod(0o755) + capture = tmp_path / "guard-args.txt" + guard = tmp_path / "guard.sh" + guard.write_text('printf "%s\\n" "$@" > "$CAPTURE_ARGS"\n') + result = subprocess.run( + [ + "bash", + ".github/check-policyengine-bundle-supported.sh", + "--if-changed-from-base", + "master", + ], + cwd=REPO, + env=_script_env( + PATH=f"{tmp_path}:{os.environ['PATH']}", + BASE_PROJECT=str(base_project), + DIFF_STATUS="1" if change == "spm-source" else "0", + SIMULATION_VERSION_GUARD_SCRIPT=str(guard), + CAPTURE_ARGS=str(capture), + ), + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + assert capture.exists() == (change != "none") + if capture.exists(): + assert "--check-installed-spm" in capture.read_text().splitlines() diff --git a/tests/unit/test_worker_spm_release.py b/tests/unit/test_worker_spm_release.py new file mode 100644 index 000000000..37139585e --- /dev/null +++ b/tests/unit/test_worker_spm_release.py @@ -0,0 +1,88 @@ +"""Release checks use the same certified selection boundary as HTTP requests.""" + +from copy import deepcopy + +import pytest + +from policyengine_api import constants, worker_spm, worker_spm_release +from policyengine_api.spm import SPMValidationError + + +DEFAULTS = { + "forecast_content_sha256": "a" * 64, + "scenario": "baseline", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, +} + + +@pytest.fixture +def registry(monkeypatch): + monkeypatch.setattr(worker_spm_release, "version", lambda _: "6.0.0") + monkeypatch.setattr(constants, "POLICYENGINE_VERSION", "6.0.0") + monkeypatch.setattr(worker_spm, "normalize_spm_selection", lambda *_: DEFAULTS) + return { + "policyengine": {"6.0.0": "canonical-worker"}, + "spm_capabilities": { + "6.0.0": {"contract_version": "canonical-spm-v1", "defaults": DEFAULTS} + }, + } + + +def test_release_accepts_matching_certified_worker(registry): + assert worker_spm_release.validate_installed_worker(registry, "6.0.0") == DEFAULTS + + +@pytest.mark.parametrize( + "mutation", ["absent", "wrong-contract", "wrong-hash", "wrong-version"] +) +def test_release_rejects_unusable_registered_worker(registry, mutation): + registry = deepcopy(registry) + capability = registry["spm_capabilities"]["6.0.0"] + if mutation == "absent": + registry["spm_capabilities"] = {} + elif mutation == "wrong-contract": + capability["contract_version"] = "other-contract" + elif mutation == "wrong-hash": + capability["defaults"]["forecast_content_sha256"] = "b" * 64 + else: + registry["spm_capabilities"] = {"5.0.0": capability} + with pytest.raises(SPMValidationError, match=""): + worker_spm_release.validate_installed_worker(registry, "6.0.0") + + +def test_release_rejects_different_installed_bundle(registry): + with pytest.raises(ValueError, match="installed"): + worker_spm_release.validate_installed_worker(registry, "7.0.0") + + +@pytest.mark.parametrize("app", [None, "", {}, 123]) +def test_release_rejects_invalid_registered_application(registry, app): + registry["policyengine"]["6.0.0"] = app + with pytest.raises(ValueError, match="application"): + worker_spm_release.validate_installed_worker(registry, "6.0.0") + + +def test_release_accepts_legacy_without_capability(registry, monkeypatch): + monkeypatch.setattr(worker_spm, "normalize_spm_selection", lambda *_: None) + registry["spm_capabilities"] = {} + assert worker_spm_release.validate_installed_worker(registry, "6.0.0") is None + + +def test_release_does_not_hide_uncertified_bundle(registry, monkeypatch): + def unavailable(*_): + raise SPMValidationError( + "SPM_CONFIGURATION_UNAVAILABLE", "Missing certification" + ) + + monkeypatch.setattr(worker_spm, "normalize_spm_selection", unavailable) + with pytest.raises(SPMValidationError): + worker_spm_release.validate_installed_worker(registry, "6.0.0") + + +def test_release_rejects_different_runtime_selected_bundle(registry, monkeypatch): + monkeypatch.setattr(constants, "POLICYENGINE_VERSION", "5.0.0") + with pytest.raises(ValueError, match="runtime-selected"): + worker_spm_release.validate_installed_worker(registry, "6.0.0") From 21d5080154b347fe871a3cffa8ac43918339e894 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 08:37:04 -0400 Subject: [PATCH 2/3] Bind budget submissions and qualify fresh economy runs --- .../spm-cache-worker-capability.fixed.md | 2 +- docs/canonical-spm.md | 11 +- .../libs/simulation_entrypoint.py | 2 + policyengine_api/query_parameters.py | 13 + policyengine_api/services/economy_service.py | 12 + tests/fixtures/libs/simulation_entrypoint.py | 1 + tests/fixtures/services/economy_service.py | 1 + .../test_budget_window_in_flight_dedupe.py | 1 + tests/integration/test_cloud_run_candidate.py | 3 +- tests/unit/libs/test_simulation_entrypoint.py | 1 + .../test_economy_submission_identity.py | 234 ++++++++++++++++++ .../routes/test_spm_year_worker_polling.py | 1 + tests/unit/test_candidate_economy_smoke.py | 16 +- tests/unit/test_openapi_spm.py | 6 + 14 files changed, 300 insertions(+), 4 deletions(-) create mode 100644 tests/unit/routes/test_economy_submission_identity.py diff --git a/changelog.d/spm-cache-worker-capability.fixed.md b/changelog.d/spm-cache-worker-capability.fixed.md index 2521e0beb..630d46c94 100644 --- a/changelog.d/spm-cache-worker-capability.fixed.md +++ b/changelog.d/spm-cache-worker-capability.fixed.md @@ -1 +1 @@ -Isolate economy caches by calculation target and budget-window worker application, and verify the installed API bundle's SPM capability and forecast against the selected worker before deployment. Exercise current-law economy results and canonical receipts on each Cloud Run candidate. +Isolate economy caches by calculation target and verified budget-window submission worker, and verify the installed API bundle's SPM capability and forecast against the selected worker before deployment. Add an optional annual economy cache_nonce UUID to qualify fresh current-law calculations and canonical receipts on each Cloud Run candidate. diff --git a/docs/canonical-spm.md b/docs/canonical-spm.md index 185f76bd4..e03d343f9 100644 --- a/docs/canonical-spm.md +++ b/docs/canonical-spm.md @@ -298,6 +298,10 @@ results, or running batch handles are read. The budget-window registry lookup must succeed even for a cached response: a registry outage fails the request rather than replaying an unverified predecessor's result. A replacement worker receives a new cache key; the same worker's terminal error remains terminal. +Submission also checks the gateway's returned `resolved_app_name` against the +application used for that key. If the registry changes between lookup and +submission, or the response omits its identity, the API releases the starting +claim and refuses to cache the handle. Retrying resolves the current worker. The first deployment of these identities makes older economy cache entries ineligible. Their ordinary expiry remains in place. Active jobs under the old @@ -305,6 +309,11 @@ keys may finish, but requests under the new keys can submit replacement jobs; account for this one-time recomputation window when scheduling promotion. Both tagged Cloud Run candidates run a Utah current-law/current-law economy -probe without creating a policy. It requires zero budget change and, for a +probe without creating a policy. Each run supplies a fresh `cache_nonce` UUID +to the annual economy route, using the same UUID throughout polling. This +isolates the job from earlier deployments' results and exercises submission +from the candidate. The nonce affects only API cache identity; it is not a +worker calculation input. Requests that omit it retain ordinary shared caching. +The probe requires zero budget change and, for a canonical bundle, matching settings and baseline/reform forecast receipts. National numerical acceptance remains a separate release qualification. diff --git a/policyengine_api/libs/simulation_entrypoint.py b/policyengine_api/libs/simulation_entrypoint.py index 009d76c57..90e836848 100644 --- a/policyengine_api/libs/simulation_entrypoint.py +++ b/policyengine_api/libs/simulation_entrypoint.py @@ -97,6 +97,7 @@ class ModalBudgetWindowBatchExecution: failed_years: list[str] = field(default_factory=list) result: Optional[dict] = None error: Optional[str] = None + resolved_app_name: Optional[str] = None @property def name(self) -> str: @@ -281,6 +282,7 @@ def run_budget_window_batch(self, payload: dict) -> ModalBudgetWindowBatchExecut return ModalBudgetWindowBatchExecution( batch_job_id=data["batch_job_id"], status=data["status"], + resolved_app_name=data.get("resolved_app_name"), ) except httpx.HTTPStatusError as e: diff --git a/policyengine_api/query_parameters.py b/policyengine_api/query_parameters.py index 01a45ed11..cbd7b8d94 100644 --- a/policyengine_api/query_parameters.py +++ b/policyengine_api/query_parameters.py @@ -172,6 +172,19 @@ class AnnualEconomyQuery(EconomyQuery): time_period: EconomyYear target: Literal["general", "cliff"] = "general" + cache_nonce: UUID | None = Field( + default=None, + description=( + "Optional UUID isolating a fresh calculation from prior cached jobs; " + "reuse the same UUID when polling. Does not change calculation inputs." + ), + ) + + def calculation_options(self) -> dict[str, Any]: + options = super().calculation_options() + if self.cache_nonce is not None: + options["cache_nonce"] = str(self.cache_nonce) + return options class BudgetWindowEconomyQuery(EconomyQuery): diff --git a/policyengine_api/services/economy_service.py b/policyengine_api/services/economy_service.py index 7c7f7f50d..2d4c0ed60 100644 --- a/policyengine_api/services/economy_service.py +++ b/policyengine_api/services/economy_service.py @@ -442,6 +442,18 @@ def get_budget_window_economic_impact( window_size=window_size, max_parallel=max_active_years, ) + if ( + not setup_options.runtime_app_name + or batch_execution.resolved_app_name + != setup_options.runtime_app_name + ): + # The registry may change between resolution and POST. + # Never attach another worker's handle to this key, + # including when the gateway omits identity evidence. + raise RuntimeError( + "Budget-window submission worker identity does not " + "match the resolved worker; retry the request" + ) self._budget_window_cache.store_batch_job_id( cache_key, batch_execution.batch_job_id ) diff --git a/tests/fixtures/libs/simulation_entrypoint.py b/tests/fixtures/libs/simulation_entrypoint.py index b196559bd..2cf4ed371 100644 --- a/tests/fixtures/libs/simulation_entrypoint.py +++ b/tests/fixtures/libs/simulation_entrypoint.py @@ -100,6 +100,7 @@ "poll_url": f"/budget-window-jobs/{MOCK_BATCH_JOB_ID}", "country": "us", "version": "1.459.0", + "resolved_app_name": "test-budget-window-worker", } MOCK_BATCH_POLL_RESPONSE_RUNNING = { diff --git a/tests/fixtures/services/economy_service.py b/tests/fixtures/services/economy_service.py index 68656726e..9c2bcbf22 100644 --- a/tests/fixtures/services/economy_service.py +++ b/tests/fixtures/services/economy_service.py @@ -299,6 +299,7 @@ def create_mock_budget_window_batch_execution( """Helper function to create mock batch execution objects.""" mock_execution = MagicMock() mock_execution.batch_job_id = batch_job_id + mock_execution.resolved_app_name = MOCK_RESOLVED_APP_NAME mock_execution.name = batch_job_id mock_execution.status = status mock_execution.progress = progress diff --git a/tests/integration/test_budget_window_in_flight_dedupe.py b/tests/integration/test_budget_window_in_flight_dedupe.py index 9672f1331..0529473aa 100644 --- a/tests/integration/test_budget_window_in_flight_dedupe.py +++ b/tests/integration/test_budget_window_in_flight_dedupe.py @@ -40,6 +40,7 @@ def test_budget_window_in_flight_dedupe_uses_existing_batch_without_live_db( ModalBudgetWindowBatchExecution( batch_job_id="fc-budget-window-parent", status="submitted", + resolved_app_name="test-budget-window-worker", ) ) simulation_entrypoint.get_budget_window_batch_by_id.return_value = ( diff --git a/tests/integration/test_cloud_run_candidate.py b/tests/integration/test_cloud_run_candidate.py index 2d5509a04..657642eaf 100644 --- a/tests/integration/test_cloud_run_candidate.py +++ b/tests/integration/test_cloud_run_candidate.py @@ -1,5 +1,6 @@ from datetime import date import math +from uuid import uuid4 def test_cloud_run_candidate_health_routes(api_client): @@ -91,7 +92,7 @@ def test_cloud_run_candidate_current_law_economy(api_client, poll_live_endpoint) payload = poll_live_endpoint( api_client, f"/us/economy/{current_law}/over/{current_law}", - {"region": "ut", "time_period": year}, + {"region": "ut", "time_period": year, "cache_nonce": str(uuid4())}, route_name="Cloud Run candidate current-law economy", ) assert payload["status"] == "ok", payload diff --git a/tests/unit/libs/test_simulation_entrypoint.py b/tests/unit/libs/test_simulation_entrypoint.py index b1deb55ad..2f29ad609 100644 --- a/tests/unit/libs/test_simulation_entrypoint.py +++ b/tests/unit/libs/test_simulation_entrypoint.py @@ -745,6 +745,7 @@ def test__given_valid_payload__then_returns_batch_execution( assert execution.batch_job_id == MOCK_BATCH_JOB_ID assert execution.status == MODAL_EXECUTION_STATUS_SUBMITTED + assert execution.resolved_app_name == "test-budget-window-worker" call_args = mock_httpx_client.post.call_args assert "/simulate/economy/budget-window" in call_args[0][0] diff --git a/tests/unit/routes/test_economy_submission_identity.py b/tests/unit/routes/test_economy_submission_identity.py new file mode 100644 index 000000000..eac066811 --- /dev/null +++ b/tests/unit/routes/test_economy_submission_identity.py @@ -0,0 +1,234 @@ +"""Cache identity must describe the worker execution actually being qualified.""" + +from types import SimpleNamespace +import json +from uuid import uuid4 + +from flask import Flask +import httpx +import pytest + +from policyengine_api.libs.simulation_entrypoint import SimulationEntrypointClient +from policyengine_api.routes import economy_routes +from policyengine_api.runtime_cache.core import CacheNamespace +from policyengine_api.runtime_cache.fake import InMemoryCacheBackend +from policyengine_api.runtime_cache.reform_impacts import ReformImpactCache +from policyengine_api.services import economy_service as economy_module +from policyengine_api.services.budget_window_cache import BudgetWindowCache +from policyengine_api.services.economy_service import EconomyService +from policyengine_api.services.reform_impacts_service import ReformImpactsService +from tests.integration.test_cloud_run_candidate import ( + test_cloud_run_candidate_current_law_economy as run_smoke, +) + + +@pytest.fixture +def economy(monkeypatch): + # The transport/cache boundary needs no real bundle or population compute. + monkeypatch.setattr(economy_module, "validate_worker_spm", lambda *a, **k: None) + monkeypatch.setattr(economy_module, "COUNTRY_PACKAGE_VERSIONS", {"us": "1.0.0"}) + monkeypatch.setattr(economy_routes, "COUNTRY_PACKAGE_VERSIONS", {"us": "1.0.0"}) + monkeypatch.setattr(economy_module, "POLICYENGINE_VERSION", "5.2.0") + backend = InMemoryCacheBackend() + namespace = CacheNamespace("test", "submission-identity") + impacts = ReformImpactsService(ReformImpactCache(backend, namespace)) + window = BudgetWindowCache(backend, namespace) + gateway = object.__new__(SimulationEntrypointClient) + gateway.base_url = "https://worker.invalid" + service = EconomyService( + reform_impacts_service_=impacts, + budget_window_cache_=window, + simulation_entrypoint_=gateway, + ) + monkeypatch.setattr(service, "_get_policy_jsons", lambda *_: ({}, {})) + monkeypatch.setattr(economy_routes, "economy_service", service) + app = Flask(__name__) + app.config["TESTING"] = True + app.register_blueprint(economy_routes.economy_bp) + return service, impacts, window, gateway, app.test_client() + + +@pytest.mark.parametrize("submitted_app", ["worker-B", None, ""]) +def test_budget_submission_identity_is_verified_before_caching(economy, submitted_app): + service, _, window, gateway, client = economy + active_app = "worker-A" + posts = [] + + def transport(request): + nonlocal active_app + if request.url.path == "/versions/policyengine": + return httpx.Response(200, json={"5.2.0": active_app}) + assert request.method == "POST" + assert request.url.path == "/simulate/economy/budget-window" + posts.append(request) + # The registry changes after A was resolved, before B was submitted. + active_app = "worker-B" + return httpx.Response( + 200, + json={ + "batch_job_id": "batch-B", + "status": "submitted", + "resolved_app_name": submitted_app, + }, + ) + + with httpx.Client(transport=httpx.MockTransport(transport)) as gateway.client: + setup = service._build_economic_impact_setup_options( + country_id="us", + policy_id=2, + baseline_policy_id=2, + region="us", + dataset="default", + time_period="budget_window:2025:2", + options={}, + api_version="1.0.0", + ) + service._resolve_runtime_bundle_for_setup_options(setup) + key_a = service._build_budget_window_cache_key(setup) + for _ in range(2): + # A rollback must never replay B's handle under A's cache identity. + active_app = "worker-A" + with pytest.raises(RuntimeError, match="worker identity"): + client.get( + "/us/economy/2/over/2/budget-window", + query_string={ + "region": "us", + "start_year": "2025", + "window_size": "2", + }, + ) + assert window.get_batch_job_id(key_a) is None + assert window.get_completed_result(key_a) is None + assert window.get_terminal_error(key_a) is None + assert len(posts) == 2 + + +def test_candidate_smoke_cannot_pass_from_prior_result_when_submission_is_broken( + economy, +): + service, impacts, _, gateway, client = economy + posts = [] + result = {"budget": {"budgetary_impact": 0}, "resolved_app_name": "worker-A"} + metadata = { + "current_law_id": 2, + "economy_options": {"time_period": [{"name": "2025"}]}, + "spm": {"available": False}, + } + + def transport(request): + if request.url.path == "/versions/policyengine": + return httpx.Response(200, json={"5.2.0": "worker-A"}) + assert request.method == "POST" + assert request.url.path == "/simulate/economy/comparison" + posts.append(request) + raise httpx.ConnectError("candidate submission is broken", request=request) + + with httpx.Client(transport=httpx.MockTransport(transport)) as gateway.client: + setup = service._build_economic_impact_setup_options( + country_id="us", + policy_id=2, + baseline_policy_id=2, + region="state/ut", + dataset="default", + time_period="2025", + options={}, + api_version="1.0.0", + ) + service._resolve_runtime_bundle_for_setup_options(setup) + impacts.set_reform_impact( + country_id=setup.country_id, + policy_id=2, + baseline_policy_id=2, + region=setup.region, + dataset=setup.dataset, + time_period=setup.time_period, + options=setup.options, + options_hash=setup.options_hash, + status="ok", + api_version=setup.api_version, + reform_impact_json=result, + start_time=None, + execution_id="prior-deployment-job", + ) + response = client.get( + "/us/economy/2/over/2", query_string={"region": "ut", "time_period": "2025"} + ) + assert response.json["status"] == "ok" + assert not posts + + metadata_client = SimpleNamespace( + get=lambda _: SimpleNamespace( + raise_for_status=lambda: None, json=lambda: {"result": metadata} + ) + ) + + def poll(_client, path, query, **kwargs): + response = client.get(path, query_string=query) + assert response.status_code == 200, response.json + return response.json + + with pytest.raises(httpx.ConnectError, match="candidate submission is broken"): + run_smoke(metadata_client, poll) + assert len(posts) == 1 + assert "cache_nonce" not in json.loads(posts[0].content) + + +def test_nonce_polling_reuses_its_job_but_another_nonce_submits_again(economy): + _, _, _, gateway, client = economy + submissions = [] + + def transport(request): + if request.url.path == "/versions/policyengine": + return httpx.Response(200, json={"5.2.0": "worker-A"}) + if request.method == "POST": + submissions.append(json.loads(request.content)) + return httpx.Response( + 200, + json={ + "job_id": f"job-{len(submissions)}", + "status": "submitted", + "resolved_app_name": "worker-A", + }, + ) + assert request.url.path == "/jobs/job-1" + return httpx.Response( + 200, + json={ + "status": "complete", + "result": {"budget": {"budgetary_impact": 0}}, + "resolved_app_name": "worker-A", + }, + ) + + path = "/us/economy/2/over/2" + query = {"region": "ut", "time_period": "2025", "cache_nonce": str(uuid4())} + with httpx.Client(transport=httpx.MockTransport(transport)) as gateway.client: + for expected in ("computing", "ok", "ok"): + response = client.get(path, query_string=query) + assert response.status_code == 200, response.json + assert response.json["status"] == expected + assert len(submissions) == 1 + query["cache_nonce"] = str(uuid4()) + response = client.get(path, query_string=query) + assert response.json["status"] == "computing" + assert len(submissions) == 2 + # Job/correlation metadata varies; the worker's calculation is identical. + computation_payloads = [ + {key: value for key, value in payload.items() if not key.startswith("_")} + for payload in submissions + ] + assert computation_payloads[0] == computation_payloads[1] + assert "cache_nonce" not in submissions[0] + + +@pytest.mark.parametrize( + "nonce_values", [["invalid"], [""], [str(uuid4()), str(uuid4())]] +) +def test_invalid_or_duplicate_nonce_never_reaches_the_worker(economy, nonce_values): + _, _, _, _, client = economy + response = client.get( + "/us/economy/2/over/2", + query_string=[("region", "ut"), ("time_period", "2025")] + + [("cache_nonce", value) for value in nonce_values], + ) + assert response.status_code == 400, response.json diff --git a/tests/unit/routes/test_spm_year_worker_polling.py b/tests/unit/routes/test_spm_year_worker_polling.py index c0e575c7c..092fa657e 100644 --- a/tests/unit/routes/test_spm_year_worker_polling.py +++ b/tests/unit/routes/test_spm_year_worker_polling.py @@ -106,6 +106,7 @@ def transport(request): "job_id": "replacement-job", "batch_job_id": "replacement-job", "status": "submitted", + "resolved_app_name": worker_app, }, ) assert request.method == "GET", "Polling must not submit a replacement job" diff --git a/tests/unit/test_candidate_economy_smoke.py b/tests/unit/test_candidate_economy_smoke.py index 2388eb289..e4cd8637f 100644 --- a/tests/unit/test_candidate_economy_smoke.py +++ b/tests/unit/test_candidate_economy_smoke.py @@ -3,6 +3,7 @@ from copy import deepcopy from types import SimpleNamespace from unittest.mock import Mock +from uuid import UUID import pytest @@ -53,7 +54,20 @@ def test_current_law_smoke_uses_no_policy_write(smoke): run_smoke(client, poll) client.get.assert_called_once_with("/us/metadata") assert poll.call_args.args[1] == "/us/economy/2/over/2" - assert poll.call_args.args[2] == {"region": "ut", "time_period": "2025"} + query = poll.call_args.args[2] + assert query["region"] == "ut" + assert query["time_period"] == "2025" + assert UUID(query["cache_nonce"]).version == 4 + + +def test_each_current_law_smoke_has_a_fresh_cache_identity(smoke): + client, poll, _, _ = smoke + run_smoke(client, poll) + run_smoke(client, poll) + assert ( + poll.call_args_list[0].args[2]["cache_nonce"] + != poll.call_args_list[1].args[2]["cache_nonce"] + ) @pytest.mark.parametrize( diff --git a/tests/unit/test_openapi_spm.py b/tests/unit/test_openapi_spm.py index 807af4d0e..669521664 100644 --- a/tests/unit/test_openapi_spm.py +++ b/tests/unit/test_openapi_spm.py @@ -41,6 +41,12 @@ def test_economy_query_and_receipt_schemas_match_public_http_contract(): if suffix: assert parameters["window_size"]["schema"]["minimum"] == 1 assert parameters["window_size"]["schema"]["maximum"] == 75 + else: + nonce = parameters["cache_nonce"] + assert nonce["required"] is False + assert nonce["schema"]["type"] == "string" + assert nonce["schema"]["format"] == "uuid" + assert nonce["schema"]["default"] is None result = operation["responses"]["200"]["content"]["application/json"]["schema"][ "properties" ]["result"] From 466740d589017df0eb76eb09c59672b806047058 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 15 Sep 2026 09:14:48 -0400 Subject: [PATCH 3/3] Answer economy dependency failures with 503 and stop orphaning batches A budget-window submission whose worker identity did not match the resolved one raised a bare RuntimeError after the gateway's POST had already spawned the Modal batch. The route's `except ValueError` did not catch it, so the caller received 500, which polling clients retry immediately, and every retry spawned another batch that nothing pointed at. The mismatch cannot be caught before the spawn. The gateway resolves the route and calls spawn inside the submit request and reports resolved_app_name only in that request's response, and its BudgetWindowBatchRequest forbids extra fields and declares no expected-application field to pin against, so this API's registry read and the gateway's cannot be made atomic. The batch is therefore retained rather than abandoned: its handle is filed under the cache key built from the application the gateway reported, which is exactly the key a later request computes once the registry serves that application, so the next poll adopts the running batch. Filing uses a set-if-absent so it can never displace another request's handle or starting claim. The requested key keeps its own starting claim, so retries under the old identity return computing instead of submitting again, and the request that saw the mismatch answers 503 with a short Retry-After. A response that omits its identity leaves nothing to file the batch under; that case is logged and documented. resolve_app_name raises ValueError when the registry publishes no worker for this runtime's bundle, and the route mapped that to 400. Both versions it can resolve here come from the installed distribution and the runtime manifest, never from the query, so the caller has nothing to correct; an unreachable registry likewise produced a 500. Both are now the same typed error answered with 503 and Retry-After. SPMValidationError is re-raised first, so genuine settings errors keep their 400 and their typed detail. cache_nonce results were written into the shared per-scope lookup index, which every write trims to its newest 1,000 members, so roughly a thousand nonce'd requests could evict the entry every other caller reads for a chosen policy, region and year. A nonce'd record's options hash can only ever be matched by the same nonce, so those records now index under a key that includes it. Records without a nonce keep the index key they already had, so this adds no cache rotation beyond the one already disclosed. M3 from the same review is not fixed and is documented as a known operational limitation: a worker repaired by redeploying the same wrapper version keeps its application name, so its terminal error replays for the remainder of its retention. The gateway's /versions responses carry only version-to-application maps and SPM capabilities, with no image digest, deployment revision or registration timestamp to fold into the key, and this API has no authenticated operator surface to hang an invalidation route on. The contract suite's fake economy-service namespace gains a stub exception class because the route now imports one by name. Co-Authored-By: Claude Fable 5.1 --- .../spm-cache-worker-capability.fixed.md | 2 +- docs/canonical-spm.md | 48 ++- policyengine_api/routes/economy_routes.py | 27 ++ .../runtime_cache/reform_impacts.py | 43 ++- .../services/budget_window_cache.py | 34 ++ policyengine_api/services/economy_service.py | 222 ++++++++++-- .../services/reform_impacts_service.py | 10 + tests/contract/test_v1_route_contracts.py | 11 + .../test_economy_submission_identity.py | 317 ++++++++++++++++-- .../unit/runtime_cache/test_reform_impacts.py | 79 +++++ .../unit/services/test_budget_window_cache.py | 26 ++ tests/unit/services/test_economy_service.py | 2 + 12 files changed, 760 insertions(+), 61 deletions(-) diff --git a/changelog.d/spm-cache-worker-capability.fixed.md b/changelog.d/spm-cache-worker-capability.fixed.md index 630d46c94..39ab9ca71 100644 --- a/changelog.d/spm-cache-worker-capability.fixed.md +++ b/changelog.d/spm-cache-worker-capability.fixed.md @@ -1 +1 @@ -Isolate economy caches by calculation target and verified budget-window submission worker, and verify the installed API bundle's SPM capability and forecast against the selected worker before deployment. Add an optional annual economy cache_nonce UUID to qualify fresh current-law calculations and canonical receipts on each Cloud Run candidate. +Isolate economy caches by calculation target and verified budget-window submission worker, and verify the installed API bundle's SPM capability and forecast against the selected worker before deployment. Add an optional annual economy cache_nonce UUID to qualify fresh current-law calculations and canonical receipts on each Cloud Run candidate. Keep cache_nonce results out of the shared lookup index so isolated requests cannot evict a shared economy cache entry. Answer worker-registry and submission-routing failures with 503 and Retry-After instead of 400 or 500, and retain a budget-window batch under the worker that ran it so a registry change during submission cannot orphan it or spawn a batch on every retry. diff --git a/docs/canonical-spm.md b/docs/canonical-spm.md index e03d343f9..0e36bee8b 100644 --- a/docs/canonical-spm.md +++ b/docs/canonical-spm.md @@ -296,12 +296,48 @@ Annual economy cache identities include the calculation target. Budget-window identities include the resolved worker application before errors, completed results, or running batch handles are read. The budget-window registry lookup must succeed even for a cached response: a registry outage fails the request -rather than replaying an unverified predecessor's result. A replacement worker -receives a new cache key; the same worker's terminal error remains terminal. +rather than replaying an unverified predecessor's result. Both economy routes +answer a registry failure — a missing entry for this bundle, or an unreachable +registry — with 503 and a `Retry-After` header, not 400 and not 500. The +versions being resolved come from the installed distribution and the runtime +manifest, never from the query, so the caller has nothing to correct; and 500 +is a status polling clients retry immediately. A replacement worker receives a +new cache key; the same worker's terminal error remains terminal. + Submission also checks the gateway's returned `resolved_app_name` against the -application used for that key. If the registry changes between lookup and -submission, or the response omits its identity, the API releases the starting -claim and refuses to cache the handle. Retrying resolves the current worker. +application used for that key. The gateway spawns the batch inside the submit +call and reports the application it routed to only in that call's response, so +a registry change between this request's lookup and its POST cannot be caught +before the work starts. On a mismatch — including a response that omits its +identity — the API never attaches the handle to the requested key. Instead it +files the handle under the cache key of the application the gateway reported, +which is the key a later request computes once the registry serves that +application, so the next poll adopts the running batch rather than spawning a +second one. Filing never overwrites an existing handle or starting claim under +that key. The requested key keeps its own starting claim for the claim's +lifetime, so retries under the old identity return `computing` instead of +submitting again. A gateway response that omits its identity leaves nothing to +file the batch under; that batch is abandoned and logged, and the retained +claim still bounds resubmission. The request that saw the mismatch answers 503 +with a short `Retry-After` rather than 500, because 500 is a status polling +clients retry immediately. + +A budget-window terminal error is keyed on the worker application, and the +application name is a pure function of the PolicyEngine wrapper version. A +worker replaced by a wrapper-version bump therefore gets a fresh key and clears +its predecessor's terminal error, but a worker repaired by redeploying the same +wrapper version keeps the same name, so its terminal error replays for the +remainder of its retention. This is a known operational limitation. The +gateway's `/versions` responses carry version-to-application maps and SPM +capabilities and expose no per-deployment identifier — no image digest, +deployment revision, or registration timestamp — so there is nothing to fold +into the key that a same-version redeploy would change, and this API has no +authenticated operator surface to hang an invalidation route on. Until one of +those exists, clear such an error by bumping the wrapper version or by removing +the key from the shared cache out of band. The annual route's `cache_nonce` +does not apply here: the budget-window query contract has no nonce, and adding +a client-chosen cache identity to the route that fans out across years would +put the most expensive economy path behind a value any caller can vary. The first deployment of these identities makes older economy cache entries ineligible. Their ordinary expiry remains in place. Active jobs under the old @@ -314,6 +350,8 @@ to the annual economy route, using the same UUID throughout polling. This isolates the job from earlier deployments' results and exercises submission from the candidate. The nonce affects only API cache identity; it is not a worker calculation input. Requests that omit it retain ordinary shared caching. +Nonce-bearing results are also indexed apart from the shared scope they belong +to, so no volume of nonce'd requests can evict the entry ordinary callers read. The probe requires zero budget change and, for a canonical bundle, matching settings and baseline/reform forecast receipts. National numerical acceptance remains a separate release qualification. diff --git a/policyengine_api/routes/economy_routes.py b/policyengine_api/routes/economy_routes.py index 0adbcb74d..e50f77ea6 100644 --- a/policyengine_api/routes/economy_routes.py +++ b/policyengine_api/routes/economy_routes.py @@ -7,6 +7,7 @@ parse_multidict_query, ) from policyengine_api.services.economy_service import ( + EconomyDependencyUnavailableError, EconomyService, EconomicImpactResult, BudgetWindowEconomicImpactResult, @@ -63,6 +64,28 @@ def _bad_request_response(error: str | ValueError) -> Response: return _make_error_response(error, 400, result=None, **fields) +def _dependency_unavailable_response( + error: EconomyDependencyUnavailableError, +) -> Response: + """Answer a server-side economy dependency failure with 503 + Retry-After. + + These failures are not the caller's fault, so 400 would misdescribe them, + and they are not opaque server faults either: 500 sits in the transient set + that polling clients retry immediately, which is how one unlucky request + turns into a retry storm. 503 with an explicit `Retry-After` names the + condition as temporary and gives every client the same back-off, whether or + not it reads the body. + """ + + response = _make_error_response( + error, + HTTPStatus.SERVICE_UNAVAILABLE, + result=None, + ) + response.headers["Retry-After"] = str(error.retry_after_seconds) + return response + + @economy_bp.route( "//economy//over/", methods=["GET"], @@ -89,6 +112,8 @@ def get_economic_impact(country_id: str, policy_id: int, baseline_policy_id: int target=query.target, ) ) + except EconomyDependencyUnavailableError as error: + return _dependency_unavailable_response(error) except ValueError as error: return _bad_request_response(error) @@ -139,6 +164,8 @@ def get_budget_window_economic_impact( target=query.target, ) ) + except EconomyDependencyUnavailableError as error: + return _dependency_unavailable_response(error) except ValueError as error: return _bad_request_response(error) diff --git a/policyengine_api/runtime_cache/reform_impacts.py b/policyengine_api/runtime_cache/reform_impacts.py index 920b4f422..20cb00611 100644 --- a/policyengine_api/runtime_cache/reform_impacts.py +++ b/policyengine_api/runtime_cache/reform_impacts.py @@ -102,6 +102,23 @@ def _impact_from_wire(payload: Any) -> CachedReformImpact | None: return None +def _isolation_token(options_json: Any) -> str | None: + """Return the client-isolation token that scopes a record's lookup index. + + A ``cache_nonce`` request is private to the caller that chose the value: + the nonce is part of the options hash, so no other caller's hash or prefix + can ever match its records. Indexing those records beside shared ones would + still let them evict shared ones, because every write trims the scope index + to ``REFORM_IMPACT_INDEX_LIMIT`` newest members. Isolated records therefore + get their own index and exert no eviction pressure on the shared scope. + """ + + if not isinstance(options_json, dict): + return None + token = options_json.get("cache_nonce") + return token if isinstance(token, str) and token else None + + def _like_matches(value: str, pattern: str) -> bool: expression: list[str] = ["^"] escaped = False @@ -233,18 +250,23 @@ def _record_key(self, execution_id: str) -> str: ) def _scope_index(self, impact: CachedReformImpact) -> str: + inputs: dict[str, Any] = { + "api_version": impact.api_version, + "baseline_policy_id": impact.baseline_policy_id, + "country_id": impact.country_id, + "dataset": impact.dataset, + "reform_policy_id": impact.reform_policy_id, + "region": impact.region, + "time_period": impact.time_period, + } + isolation_token = _isolation_token(impact.options_json) + if isolation_token is not None: + # Only isolated records move; the shared scope keeps its own key. + inputs["cache_nonce"] = isolation_token return self.namespace.key( "reform-impact-index", REFORM_IMPACT_SCHEMA_VERSION, - { - "api_version": impact.api_version, - "baseline_policy_id": impact.baseline_policy_id, - "country_id": impact.country_id, - "dataset": impact.dataset, - "reform_policy_id": impact.reform_policy_id, - "region": impact.region, - "time_period": impact.time_period, - }, + inputs, ) def _recent_index(self) -> str: @@ -353,6 +375,7 @@ def matching( api_version: str, options_hash: str, options_hash_pattern: str | None = None, + cache_nonce: str | None = None, ) -> list[CachedReformImpact]: probe = CachedReformImpact( reform_impact_id=0, @@ -362,7 +385,7 @@ def matching( region=region, dataset=dataset, time_period=time_period, - options_json=None, + options_json={"cache_nonce": cache_nonce} if cache_nonce else None, options_hash=options_hash, api_version=api_version, reform_impact_json={}, diff --git a/policyengine_api/services/budget_window_cache.py b/policyengine_api/services/budget_window_cache.py index ae2e0cf6b..6beebbb79 100644 --- a/policyengine_api/services/budget_window_cache.py +++ b/policyengine_api/services/budget_window_cache.py @@ -261,6 +261,40 @@ def store_batch_job_id(self, cache_key: str, batch_job_id: str) -> None: started_at=started_at, ) + def adopt_batch_job_id(self, cache_key: str, batch_job_id: str) -> bool: + """Record a handle under another key's identity without displacing it. + + Used when the gateway reports that it ran a batch on a different worker + application than the one this request resolved. The batch is real and + already running, so its handle belongs under the identity that actually + ran it. Unlike `store_batch_job_id` this never overwrites: an existing + handle or an in-progress starting claim under that identity owns it, + and clobbering either would orphan that request's batch instead. + """ + + started_at = time.perf_counter() + try: + stored = self.client.set( + self._batch_key(cache_key), + batch_job_id, + ex=BUDGET_WINDOW_BATCH_TTL_SECONDS, + nx=True, + ) + except Exception: + self._handle_cache_error( + "adopt-batch-id", + event="coordination-failed", + started_at=started_at, + ) + return False + record_cache_event( + family=BUDGET_WINDOW_CACHE_FAMILY, + event="coordination-write" if stored else "claim-contended", + operation="adopt-batch-id", + started_at=started_at, + ) + return bool(stored) + def clear_starting_claim(self, cache_key: str, claim_token: str) -> None: try: self._claims.release( diff --git a/policyengine_api/services/economy_service.py b/policyengine_api/services/economy_service.py index 2d4c0ed60..796aebabd 100644 --- a/policyengine_api/services/economy_service.py +++ b/policyengine_api/services/economy_service.py @@ -42,6 +42,50 @@ budget_window_cache = BudgetWindowCache() +class EconomyDependencyUnavailableError(Exception): + """A server-side economy dependency failed; the request was never invalid. + + Deliberately not a `ValueError`. The economy routes map `ValueError` to + 400, which is the wrong answer for a worker-registry or submission-routing + failure: the caller sent a well-formed request and cannot fix anything. + These are answered with 503 and a `Retry-After`, so a polling client backs + off on a documented schedule instead of retrying as fast as it can. + """ + + #: Seconds a client should wait before repeating the request. + retry_after_seconds: int = 30 + + def __init__(self, message: str, *, retry_after_seconds: int | None = None): + super().__init__(message) + if retry_after_seconds is not None: + self.retry_after_seconds = retry_after_seconds + + +class WorkerRegistryUnavailableError(EconomyDependencyUnavailableError): + """The gateway registry publishes no worker for this runtime's bundle. + + The bundle version is resolved from the installed distribution and the + runtime manifest, never from the request, so a missing entry is a + deployment or registry problem. It usually needs an operator, hence the + longer default `Retry-After`. + """ + + +class BudgetWindowSubmissionIdentityError(EconomyDependencyUnavailableError): + """The gateway ran the batch on a worker other than the resolved one. + + The registry read and the submission POST cannot be made atomic, so a + worker rollout between them routes the batch to a different application + than the one this request's cache key names. The handle is retained under + the identity the gateway reported (see + `EconomyService._retain_budget_window_batch_under_resolved_identity`), so + the next poll — which re-reads the registry — adopts the running batch + instead of spawning another one. + """ + + retry_after_seconds = 5 + + class ImpactAction(Enum): """ Enum for the action to take based on the status of an economic impact calculation. @@ -442,21 +486,6 @@ def get_budget_window_economic_impact( window_size=window_size, max_parallel=max_active_years, ) - if ( - not setup_options.runtime_app_name - or batch_execution.resolved_app_name - != setup_options.runtime_app_name - ): - # The registry may change between resolution and POST. - # Never attach another worker's handle to this key, - # including when the gateway omits identity evidence. - raise RuntimeError( - "Budget-window submission worker identity does not " - "match the resolved worker; retry the request" - ) - self._budget_window_cache.store_batch_job_id( - cache_key, batch_execution.batch_job_id - ) except httpx.HTTPStatusError as error: self._budget_window_cache.clear_starting_claim( cache_key, claim_token @@ -477,6 +506,37 @@ def get_budget_window_economic_impact( ) raise + if ( + not setup_options.runtime_app_name + or batch_execution.resolved_app_name + != setup_options.runtime_app_name + ): + # The registry may change between resolution and POST. + # Never attach another worker's handle to this key, + # including when the gateway omits identity evidence. The + # POST already spawned the batch, so hand it to the key + # that names the worker running it and keep this key's + # starting claim: both stop a retry from spawning again. + self._retain_budget_window_batch_under_resolved_identity( + setup_options=setup_options, + batch_execution=batch_execution, + ) + raise BudgetWindowSubmissionIdentityError( + "The simulation worker registry changed while this " + "budget window was being submitted; the calculation is " + "running and the next request will pick it up" + ) + + try: + self._budget_window_cache.store_batch_job_id( + cache_key, batch_execution.batch_job_id + ) + except Exception: + self._budget_window_cache.clear_starting_claim( + cache_key, claim_token + ) + raise + return self._build_budget_window_computing_result( total_years=len(years), completed_years=[], @@ -489,6 +549,78 @@ def get_budget_window_economic_impact( print(f"Error getting budget-window economic impact: {str(e)}") raise e + def _retain_budget_window_batch_under_resolved_identity( + self, + *, + setup_options: EconomicImpactSetupOptions, + batch_execution: Any, + ) -> str | None: + """Key an already-spawned batch under the worker that actually ran it. + + The gateway spawns the Modal batch inside the submit call and only then + reports which application it routed to, so a registry change between + this request's lookup and its POST cannot be caught before the work + starts. Discarding the handle would leave that batch running with + nothing pointing at it, and every retry would spawn another. + + The only component of the cache key that differs is the worker + application: `model_version` comes from the installed country package + and the remaining components are this request's own. Re-deriving the + key with the reported application therefore produces exactly the key a + later request computes once the registry serves that application, so + its poll finds this batch. Returns that key, or None when the gateway + reported no identity to file it under. + """ + + resolved_app_name = getattr(batch_execution, "resolved_app_name", None) + batch_job_id = getattr(batch_execution, "batch_job_id", None) + if ( + not isinstance(resolved_app_name, str) + or not resolved_app_name + or not batch_job_id + ): + logger.log_struct( + { + "message": ( + "Budget-window submission reported no worker identity; " + "the spawned batch cannot be filed under a cache key" + ), + **setup_options.model_dump(), + }, + severity="ERROR", + ) + return None + + resolved_setup_options = setup_options.model_copy(deep=True) + resolved_setup_options.runtime_app_name = resolved_app_name + resolved_setup_options.options_hash = self._build_options_hash( + options=resolved_setup_options.options, + model_version=resolved_setup_options.model_version, + dataset=resolved_setup_options.dataset, + data_version=resolved_setup_options.data_version, + policyengine_version=resolved_setup_options.policyengine_version, + runtime_app_name=resolved_app_name, + target=resolved_setup_options.target, + ) + resolved_cache_key = self._build_budget_window_cache_key(resolved_setup_options) + adopted = self._budget_window_cache.adopt_batch_job_id( + resolved_cache_key, batch_job_id + ) + logger.log_struct( + { + "message": ( + "Budget-window batch retained under the submitting worker" + if adopted + else "Budget-window batch identity is already claimed by " + "another request; this batch is not retained" + ), + **setup_options.model_dump(), + "submitted_app_name": resolved_app_name, + }, + severity="WARNING", + ) + return resolved_cache_key if adopted else None + def _build_budget_window_cache_key( self, setup_options: EconomicImpactSetupOptions, @@ -844,14 +976,39 @@ def _resolve_runtime_bundle_for_setup_options( setup_options: EconomicImpactSetupOptions, ) -> None: if setup_options.runtime_app_name is None: - ( - setup_options.runtime_app_name, - setup_options.model_version, - ) = self._simulation_gateway.resolve_app_name( - setup_options.country_id, - setup_options.model_version, - policyengine_version=setup_options.policyengine_version, - ) + try: + ( + setup_options.runtime_app_name, + setup_options.model_version, + ) = self._simulation_gateway.resolve_app_name( + setup_options.country_id, + setup_options.model_version, + policyengine_version=setup_options.policyengine_version, + ) + except SPMValidationError: + # A typed SPM error really is about the request's settings. + raise + except ValueError as error: + # `resolve_app_name` raises ValueError when the registry has no + # entry for the version it was asked about. Both versions it can + # be asked about here are server-side: `policyengine_version` is + # the runtime's own bundle and `model_version` is the installed + # country package. Neither comes from the query, so a missing + # entry is a dependency failure, not a bad request. + raise WorkerRegistryUnavailableError( + "No simulation worker is currently registered for this " + "API's model bundle; the request is valid and can be " + "retried once the worker registry publishes it" + ) from error + except httpx.HTTPError as error: + # The registry itself is unreachable or answering with an + # error. Same class of failure, same answer: the request was + # fine and only a dependency is missing. + raise WorkerRegistryUnavailableError( + "The simulation worker registry is unavailable; the " + "request is valid and can be retried once the registry " + "responds again" + ) from error setup_options.options_hash = self._build_options_hash( options=setup_options.options, @@ -918,6 +1075,22 @@ def _build_budget_window_progress_message( return f"Scoring budget window ({completed_count} of {total_years} complete)..." + @staticmethod + def _cache_isolation_token( + setup_options: EconomicImpactSetupOptions, + ) -> str | None: + """Return the request's client-isolation token, if it supplied one. + + `cache_nonce` is a client-chosen cache identity on an unauthenticated + route. Its records are only ever matched by the same nonce, so they are + indexed apart from the shared scope; otherwise a flood of nonces would + trim the shared scope's bounded index and evict an entry every other + caller reads. + """ + + token = setup_options.options.get("cache_nonce") + return token if isinstance(token, str) and token else None + def _get_previous_impacts( self, country_id: str, @@ -928,6 +1101,7 @@ def _get_previous_impacts( time_period: str, options_hash: str, api_version: str, + cache_nonce: str | None = None, ): """ Fetch any previous simulation runs for the given policy reform. @@ -945,6 +1119,7 @@ def _get_previous_impacts( options_hash, self._build_options_hash_lookup_pattern(options_hash), api_version, + cache_nonce=cache_nonce, ) ) return previous_impacts @@ -966,6 +1141,7 @@ def _get_most_recent_impact( time_period=setup_options.time_period, options_hash=setup_options.options_hash, api_version=setup_options.api_version, + cache_nonce=self._cache_isolation_token(setup_options), ) if not previous_impacts: diff --git a/policyengine_api/services/reform_impacts_service.py b/policyengine_api/services/reform_impacts_service.py index 81e66cad2..a9ef73493 100644 --- a/policyengine_api/services/reform_impacts_service.py +++ b/policyengine_api/services/reform_impacts_service.py @@ -41,6 +41,7 @@ def get_all_reform_impacts( time_period, options_hash, api_version, + cache_nonce: str | None = None, ) -> list[CachedReformImpact]: return self._cache.matching( country_id=country_id, @@ -51,6 +52,7 @@ def get_all_reform_impacts( time_period=time_period, api_version=api_version, options_hash=options_hash, + cache_nonce=cache_nonce, ) def get_all_reform_impacts_by_options_hash_prefix( @@ -64,7 +66,14 @@ def get_all_reform_impacts_by_options_hash_prefix( options_hash, options_hash_prefix, api_version, + cache_nonce: str | None = None, ) -> list[CachedReformImpact]: + """Look up a scope's records, isolated by ``cache_nonce`` when supplied. + + A nonce'd request reads its own index, so it never sees — and its + writes never displace — the entries shared by ordinary callers. + """ + return self._cache.matching( country_id=country_id, reform_policy_id=policy_id, @@ -75,6 +84,7 @@ def get_all_reform_impacts_by_options_hash_prefix( api_version=api_version, options_hash=options_hash, options_hash_pattern=options_hash_prefix, + cache_nonce=cache_nonce, ) def claim_reform_impact_start( diff --git a/tests/contract/test_v1_route_contracts.py b/tests/contract/test_v1_route_contracts.py index db1457ff5..e26b78da0 100644 --- a/tests/contract/test_v1_route_contracts.py +++ b/tests/contract/test_v1_route_contracts.py @@ -87,6 +87,16 @@ def to_dict(self): } +class _EconomyDependencyUnavailableError(Exception): + """Stand-in for the route's 503 dependency-failure type. + + The route only needs an exception class to name in an `except` clause and + a back-off to publish; the contract fixtures never raise one. + """ + + retry_after_seconds = 30 + + class _EconomyService: def get_budget_window_economic_impact(self, **kwargs): return _BudgetWindowEconomicImpactResult.completed( @@ -144,6 +154,7 @@ def _load_contract_economy_blueprint(): service_module_name="policyengine_api.services.economy_service", route_module_name="policyengine_api.routes.economy_routes", fake_service_module=SimpleNamespace( + EconomyDependencyUnavailableError=_EconomyDependencyUnavailableError, EconomyService=_EconomyService, EconomicImpactResult=object, BudgetWindowEconomicImpactResult=_BudgetWindowEconomicImpactResult, diff --git a/tests/unit/routes/test_economy_submission_identity.py b/tests/unit/routes/test_economy_submission_identity.py index eac066811..9b0de13c7 100644 --- a/tests/unit/routes/test_economy_submission_identity.py +++ b/tests/unit/routes/test_economy_submission_identity.py @@ -12,7 +12,10 @@ from policyengine_api.routes import economy_routes from policyengine_api.runtime_cache.core import CacheNamespace from policyengine_api.runtime_cache.fake import InMemoryCacheBackend -from policyengine_api.runtime_cache.reform_impacts import ReformImpactCache +from policyengine_api.runtime_cache.reform_impacts import ( + REFORM_IMPACT_INDEX_LIMIT, + ReformImpactCache, +) from policyengine_api.services import economy_service as economy_module from policyengine_api.services.budget_window_cache import BudgetWindowCache from policyengine_api.services.economy_service import EconomyService @@ -48,6 +51,38 @@ def economy(monkeypatch): return service, impacts, window, gateway, app.test_client() +BUDGET_WINDOW_QUERY = {"region": "us", "start_year": "2025", "window_size": "2"} + + +def _budget_window_keys(service): + """Return the budget-window cache keys for worker A and worker B.""" + + keys = {} + for app_name in ("worker-A", "worker-B"): + setup = service._build_economic_impact_setup_options( + country_id="us", + policy_id=2, + baseline_policy_id=2, + region="us", + dataset="default", + time_period="budget_window:2025:2", + options={}, + api_version="1.0.0", + ) + setup.runtime_app_name = app_name + setup.options_hash = service._build_options_hash( + options=setup.options, + model_version=setup.model_version, + dataset=setup.dataset, + data_version=setup.data_version, + policyengine_version=setup.policyengine_version, + runtime_app_name=app_name, + target=setup.target, + ) + keys[app_name] = service._build_budget_window_cache_key(setup) + return keys + + @pytest.mark.parametrize("submitted_app", ["worker-B", None, ""]) def test_budget_submission_identity_is_verified_before_caching(economy, submitted_app): service, _, window, gateway, client = economy @@ -73,34 +108,272 @@ def transport(request): ) with httpx.Client(transport=httpx.MockTransport(transport)) as gateway.client: - setup = service._build_economic_impact_setup_options( + keys = _budget_window_keys(service) + key_a = keys["worker-A"] + for attempt in range(2): + # A rollback must never replay B's handle under A's cache identity. + active_app = "worker-A" + response = client.get( + "/us/economy/2/over/2/budget-window", + query_string=BUDGET_WINDOW_QUERY, + ) + if attempt == 0: + # The request that saw the mismatch says "come back shortly", + # not "you sent a bad request" and not "something broke". + assert response.status_code == 503, response.json + assert response.headers["Retry-After"] == "5" + assert "registry changed" in response.json["message"] + else: + # A's starting claim still stands, so the retry waits rather + # than paying for a second batch. + assert response.status_code == 200, response.json + assert response.json["status"] == "computing" + assert window.get_batch_job_id(key_a) is None + assert window.get_completed_result(key_a) is None + assert window.get_terminal_error(key_a) is None + # One POST, one spawned batch: the retry never reached the gateway. + assert len(posts) == 1 + + +def test_budget_submission_identity_mismatch_retains_the_batch_for_the_next_poll( + economy, +): + service, _, window, gateway, client = economy + active_app = "worker-A" + posts = [] + + def transport(request): + nonlocal active_app + if request.url.path == "/versions/policyengine": + return httpx.Response(200, json={"5.2.0": active_app}) + if request.method == "POST": + posts.append(request) + # The rollout completes between this request's lookup and its POST. + active_app = "worker-B" + return httpx.Response( + 200, + json={ + "batch_job_id": "batch-B", + "status": "submitted", + "resolved_app_name": "worker-B", + }, + ) + assert request.url.path == "/budget-window-jobs/batch-B" + return httpx.Response( + 200, + json={ + "batch_job_id": "batch-B", + "status": "running", + "progress": 0, + "completed_years": [], + "running_years": ["2025"], + "queued_years": ["2026"], + "failed_years": [], + }, + ) + + with httpx.Client(transport=httpx.MockTransport(transport)) as gateway.client: + keys = _budget_window_keys(service) + response = client.get( + "/us/economy/2/over/2/budget-window", + query_string=BUDGET_WINDOW_QUERY, + ) + assert response.status_code == 503, response.json + # The spawned batch is filed under the worker that actually ran it, + # never under the identity this request resolved. + assert window.get_batch_job_id(keys["worker-A"]) is None + assert window.get_batch_job_id(keys["worker-B"]) == "batch-B" + + # The next poll re-reads the registry, which now serves worker B, and + # adopts the running batch instead of spawning a second one. + response = client.get( + "/us/economy/2/over/2/budget-window", + query_string=BUDGET_WINDOW_QUERY, + ) + assert response.status_code == 200, response.json + assert response.json["status"] == "computing" + assert response.headers["X-PolicyEngine-Budget-Window-Cache"] == "batch-id-hit" + assert len(posts) == 1 + + +def test_budget_submission_identity_mismatch_never_overwrites_another_claim(economy): + service, _, window, gateway, client = economy + active_app = "worker-A" + + def transport(request): + if request.url.path == "/versions/policyengine": + return httpx.Response(200, json={"5.2.0": active_app}) + return httpx.Response( + 200, + json={ + "batch_job_id": "batch-late", + "status": "submitted", + "resolved_app_name": "worker-B", + }, + ) + + with httpx.Client(transport=httpx.MockTransport(transport)) as gateway.client: + keys = _budget_window_keys(service) + window.store_batch_job_id(keys["worker-B"], "batch-already-running") + response = client.get( + "/us/economy/2/over/2/budget-window", + query_string=BUDGET_WINDOW_QUERY, + ) + assert response.status_code == 503, response.json + # Worker B's key already points at a batch; displacing it would orphan + # that one instead. + assert window.get_batch_job_id(keys["worker-B"]) == "batch-already-running" + + +@pytest.mark.parametrize( + "path,query", + [ + ("/us/economy/2/over/2", {"region": "ut", "time_period": "2025"}), + ("/us/economy/2/over/2/budget-window", BUDGET_WINDOW_QUERY), + ], +) +def test_missing_registry_entry_is_a_dependency_failure_not_a_bad_request( + economy, path, query +): + _, _, _, gateway, client = economy + + def transport(request): + assert request.url.path == "/versions/policyengine" + # The registry is reachable but publishes no worker for this bundle. + return httpx.Response(200, json={"4.0.0": "worker-old"}) + + with httpx.Client(transport=httpx.MockTransport(transport)) as gateway.client: + response = client.get(path, query_string=query) + assert response.status_code == 503, response.json + assert response.headers["Retry-After"] == "30" + assert ( + "No simulation worker is currently registered" in response.json["message"] + ) + + +@pytest.mark.parametrize( + "path,query", + [ + ("/us/economy/2/over/2", {"region": "ut", "time_period": "2025"}), + ("/us/economy/2/over/2/budget-window", BUDGET_WINDOW_QUERY), + ], +) +def test_unreachable_registry_is_a_dependency_failure_not_a_server_fault( + economy, path, query +): + _, _, _, gateway, client = economy + + def transport(request): + raise httpx.ConnectError("registry unreachable", request=request) + + with httpx.Client(transport=httpx.MockTransport(transport)) as gateway.client: + response = client.get(path, query_string=query) + # 500 is a status polling clients retry immediately; 503 carries an + # explicit back-off for the same condition. + assert response.status_code == 503, response.json + assert response.headers["Retry-After"] == "30" + assert "registry is unavailable" in response.json["message"] + + +def test_invalid_query_is_still_a_bad_request_when_the_registry_is_empty(economy): + _, _, _, gateway, client = economy + + def transport(request): + return httpx.Response(200, json={}) + + with httpx.Client(transport=httpx.MockTransport(transport)) as gateway.client: + response = client.get( + "/us/economy/2/over/2/budget-window", + query_string={"region": "us", "start_year": "2025", "window_size": "0"}, + ) + assert response.status_code == 400, response.json + + +def test_nonce_floods_cannot_evict_the_shared_scope_entry(economy): + service, impacts, _, gateway, client = economy + posts = [] + shared_result = {"budget": {"budgetary_impact": 0}, "resolved_app_name": "worker-A"} + + def transport(request): + if request.url.path == "/versions/policyengine": + return httpx.Response(200, json={"5.2.0": "worker-A"}) + posts.append(request) + return httpx.Response( + 200, + json={ + "job_id": f"job-{len(posts)}", + "status": "submitted", + "resolved_app_name": "worker-A", + }, + ) + + with httpx.Client(transport=httpx.MockTransport(transport)) as gateway.client: + shared = service._build_economic_impact_setup_options( country_id="us", policy_id=2, baseline_policy_id=2, - region="us", + region="state/ut", dataset="default", - time_period="budget_window:2025:2", + time_period="2025", options={}, api_version="1.0.0", ) - service._resolve_runtime_bundle_for_setup_options(setup) - key_a = service._build_budget_window_cache_key(setup) - for _ in range(2): - # A rollback must never replay B's handle under A's cache identity. - active_app = "worker-A" - with pytest.raises(RuntimeError, match="worker identity"): - client.get( - "/us/economy/2/over/2/budget-window", - query_string={ - "region": "us", - "start_year": "2025", - "window_size": "2", - }, - ) - assert window.get_batch_job_id(key_a) is None - assert window.get_completed_result(key_a) is None - assert window.get_terminal_error(key_a) is None - assert len(posts) == 2 + service._resolve_runtime_bundle_for_setup_options(shared) + impacts.set_reform_impact( + country_id=shared.country_id, + policy_id=2, + baseline_policy_id=2, + region=shared.region, + dataset=shared.dataset, + time_period=shared.time_period, + options=shared.options, + options_hash=shared.options_hash, + status="ok", + api_version=shared.api_version, + reform_impact_json=shared_result, + start_time=None, + execution_id="shared-job", + ) + + # One more than the shared scope index holds. Every one of these is + # newer than the shared entry, so an unpartitioned index would have + # trimmed the shared entry out of its own scope. + for index in range(REFORM_IMPACT_INDEX_LIMIT + 1): + nonce = str(uuid4()) + isolated = service._build_economic_impact_setup_options( + country_id="us", + policy_id=2, + baseline_policy_id=2, + region="state/ut", + dataset="default", + time_period="2025", + options={"cache_nonce": nonce}, + api_version="1.0.0", + ) + service._resolve_runtime_bundle_for_setup_options(isolated) + impacts.set_reform_impact( + country_id=isolated.country_id, + policy_id=2, + baseline_policy_id=2, + region=isolated.region, + dataset=isolated.dataset, + time_period=isolated.time_period, + options=isolated.options, + options_hash=isolated.options_hash, + status="ok", + api_version=isolated.api_version, + reform_impact_json={"budget": {"budgetary_impact": 0}}, + start_time=None, + execution_id=f"isolated-job-{index}", + ) + + response = client.get( + "/us/economy/2/over/2", query_string={"region": "ut", "time_period": "2025"} + ) + assert response.status_code == 200, response.json + assert response.json["status"] == "ok" + # The shared entry answered; nothing was resubmitted to the worker. + assert not posts def test_candidate_smoke_cannot_pass_from_prior_result_when_submission_is_broken( diff --git a/tests/unit/runtime_cache/test_reform_impacts.py b/tests/unit/runtime_cache/test_reform_impacts.py index 98b134b4f..3e42a3d05 100644 --- a/tests/unit/runtime_cache/test_reform_impacts.py +++ b/tests/unit/runtime_cache/test_reform_impacts.py @@ -1,5 +1,6 @@ """Reform-impact cache tests.""" +from dataclasses import replace from datetime import datetime import pytest @@ -204,3 +205,81 @@ def test_failed_terminal_error_write_keeps_existing_worker_handle(monkeypatch): unchanged = cache.get_by_execution_id("job") assert unchanged.status == "computing" assert unchanged.execution_id == "job" + + +def test_client_isolated_records_index_apart_from_the_shared_scope( + monkeypatch, +) -> None: + import policyengine_api.runtime_cache.reform_impacts as module + + monkeypatch.setattr(module, "REFORM_IMPACT_INDEX_LIMIT", 2) + backend = InMemoryCacheBackend() + cache = ReformImpactCache(backend, _namespace()) + shared = _impact("shared", "hash-shared", 1) + assert cache.set(shared) + + lookup = { + "country_id": "us", + "reform_policy_id": 2, + "baseline_policy_id": 1, + "region": "us", + "dataset": "default", + "time_period": "2026", + "api_version": "v1", + } + + # More isolated records than the index holds, each newer than the shared + # one: an index they shared would have trimmed the shared record away. + for day in range(2, 6): + nonce = f"nonce-{day}" + isolated = _impact(f"isolated-{day}", f"hash-{nonce}", day) + isolated = replace(isolated, options_json={"cache_nonce": nonce}) + assert cache.set(isolated) + + assert [ + value.execution_id + for value in cache.matching(**lookup, options_hash="hash-shared") + ] == ["shared"] + + # Each isolated request still finds its own record, and only its own. + assert [ + value.execution_id + for value in cache.matching( + **lookup, + options_hash="hash-nonce-5", + cache_nonce="nonce-5", + ) + ] == ["isolated-5"] + assert ( + cache.matching( + **lookup, + options_hash="hash-shared", + cache_nonce="nonce-5", + ) + == [] + ) + + +def test_shared_scope_index_key_is_unchanged_by_isolation_support() -> None: + backend = InMemoryCacheBackend() + cache = ReformImpactCache(backend, _namespace()) + shared = _impact("shared", "hash-shared", 1) + + # Records with no nonce keep the scope key they already had, so supporting + # isolation does not rotate the shared index on deployment. + assert cache._scope_index(shared) == _namespace().key( + "reform-impact-index", + 1, + { + "api_version": "v1", + "baseline_policy_id": 1, + "country_id": "us", + "dataset": "default", + "reform_policy_id": 2, + "region": "us", + "time_period": "2026", + }, + ) + assert cache._scope_index( + replace(shared, options_json={"cache_nonce": "nonce"}) + ) != cache._scope_index(shared) diff --git a/tests/unit/services/test_budget_window_cache.py b/tests/unit/services/test_budget_window_cache.py index 0bb51ee9d..ca5f9cd77 100644 --- a/tests/unit/services/test_budget_window_cache.py +++ b/tests/unit/services/test_budget_window_cache.py @@ -294,3 +294,29 @@ def test_clear_batch_job_id_logs_and_swallows_errors(monkeypatch): cache.clear_batch_job_id("budget_window:v1:us:key") assert mock_logger.log_struct.call_args.kwargs["severity"] == "WARNING" + + +def test_adopt_batch_job_id_files_a_handle_without_displacing_one(): + backend = FakeRedis() + cache = BudgetWindowCache(client=backend) + key = "budget_window:v1:us:key" + + assert cache.adopt_batch_job_id(key, "batch-1") is True + assert cache.get_batch_job_id(key) == "batch-1" + assert set(backend._expires.values()) == {BUDGET_WINDOW_BATCH_TTL_SECONDS} + + # Adoption is for a batch nobody has claimed: it must never take a key + # another request already owns, which would orphan that request's work. + assert cache.adopt_batch_job_id(key, "batch-2") is False + assert cache.get_batch_job_id(key) == "batch-1" + + claimed = "budget_window:v1:us:claimed" + assert cache.claim_batch_start(claimed, "process-1") is True + assert cache.adopt_batch_job_id(claimed, "batch-3") is False + assert cache.get_batch_job_id(claimed) is None + + +def test_adopt_batch_job_id_reports_failure_instead_of_raising(): + cache = BudgetWindowCache(client=RaisingRedis(method="set")) + + assert cache.adopt_batch_job_id("budget_window:v1:us:key", "batch-1") is False diff --git a/tests/unit/services/test_economy_service.py b/tests/unit/services/test_economy_service.py index 97c942623..17a2b687b 100644 --- a/tests/unit/services/test_economy_service.py +++ b/tests/unit/services/test_economy_service.py @@ -654,6 +654,7 @@ def test__given_runtime_cache_version__uses_versioned_economy_cache_key( MOCK_LOOKUP_OPTIONS_HASH ), cache_version, + cache_nonce=None, ) def test__given_alias_dataset__queries_previous_impacts_with_resolved_bundle( @@ -1571,6 +1572,7 @@ def test_given_valid_parameters_calls_service_correctly( MOCK_OPTIONS_HASH, economy_service._build_options_hash_lookup_pattern(MOCK_OPTIONS_HASH), MOCK_API_VERSION, + cache_nonce=None, ) class TestGetMostRecentImpact: