diff --git a/.github/check-policyengine-bundle-supported.sh b/.github/check-policyengine-bundle-supported.sh index f48d1f076..696b526af 100755 --- a/.github/check-policyengine-bundle-supported.sh +++ b/.github/check-policyengine-bundle-supported.sh @@ -32,7 +32,12 @@ extract_policyengine_version() { sed -n 's/.*policyengine\[models\]==\([0-9.][0-9.]*\).*/\1/p' | head -n 1 } +extract_calculator_version() { + sed -n 's/.*spm-calculator==\([0-9.][0-9.]*\).*/\1/p' | head -n 1 +} + current_version="$(extract_policyengine_version < pyproject.toml)" +current_calculator="$(extract_calculator_version < pyproject.toml)" if [ -z "$current_version" ]; then echo "ERROR: policyengine[models] pin not found in pyproject.toml" exit 1 @@ -51,10 +56,22 @@ if [ "$CHECK_ONLY_IF_CHANGED" = "1" ]; then || true )" - if [ "$current_version" = "$base_version" ]; then - echo "PolicyEngine .py bundle pin is unchanged; skipping simulation API support check." + base_calculator="$( + git show "origin/${BASE_REF}:pyproject.toml" \ + | extract_calculator_version \ + || true + )" + + if [ "$current_version" = "$base_version" ] \ + && [ "$current_calculator" = "$base_calculator" ] \ + && git diff --quiet "origin/${BASE_REF}" -- \ + policyengine_api/spm.py policyengine_api/worker_spm.py \ + policyengine_api/worker_spm_release.py policyengine_api/constants.py \ + policyengine_api/country.py .github/check-policyengine-bundle-supported.sh \ + .github/request-simulation-model-versions.sh; then + echo "Bundle/calculator pins and SPM integration are unchanged; skipping simulation API support check." exit 0 fi fi -bash "$VERSION_GUARD_SCRIPT" -py "$current_version" +bash "$VERSION_GUARD_SCRIPT" -py "$current_version" --check-installed-spm diff --git a/.github/request-simulation-model-versions.sh b/.github/request-simulation-model-versions.sh index 27a68062c..bbb65cc7f 100755 --- a/.github/request-simulation-model-versions.sh +++ b/.github/request-simulation-model-versions.sh @@ -23,12 +23,14 @@ usage() { echo "Optional compatibility checks:" echo " -us us_version Expected bundled policyengine-us version" echo " -uk uk_version Expected bundled policyengine-uk version" + echo " --check-installed-spm Validate the installed API bundle's SPM capability" exit 1 } POLICYENGINE_VERSION="" US_VERSION="" UK_VERSION="" +CHECK_INSTALLED_SPM=0 while [ $# -gt 0 ]; do case "$1" in @@ -44,6 +46,10 @@ while [ $# -gt 0 ]; do UK_VERSION="$2" shift 2 ;; + --check-installed-spm) + CHECK_INSTALLED_SPM=1 + shift + ;; -h|--help) usage ;; @@ -70,7 +76,7 @@ if [ -n "$UK_VERSION" ]; then fi echo "" -VERSIONS_RESPONSE=$(curl -s "${GATEWAY_URL}/versions") +VERSIONS_RESPONSE=$(curl --fail --silent --show-error --connect-timeout 10 --max-time 60 "${GATEWAY_URL}/versions") if [ -z "$VERSIONS_RESPONSE" ]; then echo "ERROR: Failed to fetch versions from gateway" @@ -114,6 +120,10 @@ check_country_route() { check_country_route "us" "$US_VERSION" check_country_route "uk" "$UK_VERSION" +if [ "$CHECK_INSTALLED_SPM" = "1" ]; then + printf '%s' "$VERSIONS_RESPONSE" | uv run --frozen python -m policyengine_api.worker_spm_release "$POLICYENGINE_VERSION" +fi + echo "" echo "SUCCESS: PolicyEngine bundle route is deployed and ready" exit 0 diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 3cc22e682..b193772f6 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -23,6 +23,8 @@ jobs: fetch-depth: 0 - name: Install jq run: sudo apt-get install -y jq + - name: Setup uv for the installed bundle capability check + uses: astral-sh/setup-uv@v6 - name: Check simulation API supports updated PolicyEngine bundle run: bash .github/check-policyengine-bundle-supported.sh --if-changed-from-base "${{ github.base_ref }}" diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 1390bb8cb..e735f5a73 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -60,6 +60,8 @@ jobs: uses: actions/checkout@v4 - name: Install jq run: sudo apt-get install -y jq + - name: Setup uv for the installed bundle capability check + uses: astral-sh/setup-uv@v6 - name: Check simulation API supports PolicyEngine bundle run: bash .github/check-policyengine-bundle-supported.sh @@ -391,6 +393,8 @@ jobs: uses: actions/checkout@v4 - name: Install jq run: sudo apt-get install -y jq + - name: Setup uv for the installed bundle capability check + uses: astral-sh/setup-uv@v6 - name: Check simulation API supports PolicyEngine bundle run: bash .github/check-policyengine-bundle-supported.sh diff --git a/changelog.d/spm-cache-worker-capability.fixed.md b/changelog.d/spm-cache-worker-capability.fixed.md new file mode 100644 index 000000000..39ab9ca71 --- /dev/null +++ b/changelog.d/spm-cache-worker-capability.fixed.md @@ -0,0 +1 @@ +Isolate economy caches by calculation target and verified budget-window submission worker, and verify the installed API bundle's SPM capability and forecast against the selected worker before deployment. Add an optional annual economy cache_nonce UUID to qualify fresh current-law calculations and canonical receipts on each Cloud Run candidate. Keep cache_nonce results out of the shared lookup index so isolated requests cannot evict a shared economy cache entry. Answer worker-registry and submission-routing failures with 503 and Retry-After instead of 400 or 500, and retain a budget-window batch under the worker that ran it so a registry change during submission cannot orphan it or spawn a batch on every retry. diff --git a/docs/canonical-spm.md b/docs/canonical-spm.md index 524746218..0e36bee8b 100644 --- a/docs/canonical-spm.md +++ b/docs/canonical-spm.md @@ -281,3 +281,77 @@ bundle defaults. The selection schema deliberately supplies no wire defaults, so generated clients preserve that distinction. Resolved settings freeze all six fields. Responses may omit null geography_id/as_of values, but must include every non-null resolved field; receipt replay never substitutes current defaults. + + +## Economy deployment checks and cache transitions + +Deployment alignment validates the installed wrapper version and the runtime's +manifest-selected version, then applies the request-time SPM capability and +forecast-hash checks to the registered bundle. Legacy bundles require bundle +registration but do not advertise canonical SPM. A canonical promotion must +coordinate the wrapper and calculator pins and pass qualification with their +published wheels; the legacy dependency tuple does not qualify that path. + +Annual economy cache identities include the calculation target. Budget-window +identities include the resolved worker application before errors, completed +results, or running batch handles are read. The budget-window registry lookup +must succeed even for a cached response: a registry outage fails the request +rather than replaying an unverified predecessor's result. Both economy routes +answer a registry failure — a missing entry for this bundle, or an unreachable +registry — with 503 and a `Retry-After` header, not 400 and not 500. The +versions being resolved come from the installed distribution and the runtime +manifest, never from the query, so the caller has nothing to correct; and 500 +is a status polling clients retry immediately. A replacement worker receives a +new cache key; the same worker's terminal error remains terminal. + +Submission also checks the gateway's returned `resolved_app_name` against the +application used for that key. The gateway spawns the batch inside the submit +call and reports the application it routed to only in that call's response, so +a registry change between this request's lookup and its POST cannot be caught +before the work starts. On a mismatch — including a response that omits its +identity — the API never attaches the handle to the requested key. Instead it +files the handle under the cache key of the application the gateway reported, +which is the key a later request computes once the registry serves that +application, so the next poll adopts the running batch rather than spawning a +second one. Filing never overwrites an existing handle or starting claim under +that key. The requested key keeps its own starting claim for the claim's +lifetime, so retries under the old identity return `computing` instead of +submitting again. A gateway response that omits its identity leaves nothing to +file the batch under; that batch is abandoned and logged, and the retained +claim still bounds resubmission. The request that saw the mismatch answers 503 +with a short `Retry-After` rather than 500, because 500 is a status polling +clients retry immediately. + +A budget-window terminal error is keyed on the worker application, and the +application name is a pure function of the PolicyEngine wrapper version. A +worker replaced by a wrapper-version bump therefore gets a fresh key and clears +its predecessor's terminal error, but a worker repaired by redeploying the same +wrapper version keeps the same name, so its terminal error replays for the +remainder of its retention. This is a known operational limitation. The +gateway's `/versions` responses carry version-to-application maps and SPM +capabilities and expose no per-deployment identifier — no image digest, +deployment revision, or registration timestamp — so there is nothing to fold +into the key that a same-version redeploy would change, and this API has no +authenticated operator surface to hang an invalidation route on. Until one of +those exists, clear such an error by bumping the wrapper version or by removing +the key from the shared cache out of band. The annual route's `cache_nonce` +does not apply here: the budget-window query contract has no nonce, and adding +a client-chosen cache identity to the route that fans out across years would +put the most expensive economy path behind a value any caller can vary. + +The first deployment of these identities makes older economy cache entries +ineligible. Their ordinary expiry remains in place. Active jobs under the old +keys may finish, but requests under the new keys can submit replacement jobs; +account for this one-time recomputation window when scheduling promotion. + +Both tagged Cloud Run candidates run a Utah current-law/current-law economy +probe without creating a policy. Each run supplies a fresh `cache_nonce` UUID +to the annual economy route, using the same UUID throughout polling. This +isolates the job from earlier deployments' results and exercises submission +from the candidate. The nonce affects only API cache identity; it is not a +worker calculation input. Requests that omit it retain ordinary shared caching. +Nonce-bearing results are also indexed apart from the shared scope they belong +to, so no volume of nonce'd requests can evict the entry ordinary callers read. +The probe requires zero budget change and, for a +canonical bundle, matching settings and baseline/reform forecast receipts. +National numerical acceptance remains a separate release qualification. diff --git a/policyengine_api/libs/simulation_entrypoint.py b/policyengine_api/libs/simulation_entrypoint.py index 009d76c57..90e836848 100644 --- a/policyengine_api/libs/simulation_entrypoint.py +++ b/policyengine_api/libs/simulation_entrypoint.py @@ -97,6 +97,7 @@ class ModalBudgetWindowBatchExecution: failed_years: list[str] = field(default_factory=list) result: Optional[dict] = None error: Optional[str] = None + resolved_app_name: Optional[str] = None @property def name(self) -> str: @@ -281,6 +282,7 @@ def run_budget_window_batch(self, payload: dict) -> ModalBudgetWindowBatchExecut return ModalBudgetWindowBatchExecution( batch_job_id=data["batch_job_id"], status=data["status"], + resolved_app_name=data.get("resolved_app_name"), ) except httpx.HTTPStatusError as e: diff --git a/policyengine_api/query_parameters.py b/policyengine_api/query_parameters.py index 01a45ed11..cbd7b8d94 100644 --- a/policyengine_api/query_parameters.py +++ b/policyengine_api/query_parameters.py @@ -172,6 +172,19 @@ class AnnualEconomyQuery(EconomyQuery): time_period: EconomyYear target: Literal["general", "cliff"] = "general" + cache_nonce: UUID | None = Field( + default=None, + description=( + "Optional UUID isolating a fresh calculation from prior cached jobs; " + "reuse the same UUID when polling. Does not change calculation inputs." + ), + ) + + def calculation_options(self) -> dict[str, Any]: + options = super().calculation_options() + if self.cache_nonce is not None: + options["cache_nonce"] = str(self.cache_nonce) + return options class BudgetWindowEconomyQuery(EconomyQuery): diff --git a/policyengine_api/routes/economy_routes.py b/policyengine_api/routes/economy_routes.py index 0adbcb74d..e50f77ea6 100644 --- a/policyengine_api/routes/economy_routes.py +++ b/policyengine_api/routes/economy_routes.py @@ -7,6 +7,7 @@ parse_multidict_query, ) from policyengine_api.services.economy_service import ( + EconomyDependencyUnavailableError, EconomyService, EconomicImpactResult, BudgetWindowEconomicImpactResult, @@ -63,6 +64,28 @@ def _bad_request_response(error: str | ValueError) -> Response: return _make_error_response(error, 400, result=None, **fields) +def _dependency_unavailable_response( + error: EconomyDependencyUnavailableError, +) -> Response: + """Answer a server-side economy dependency failure with 503 + Retry-After. + + These failures are not the caller's fault, so 400 would misdescribe them, + and they are not opaque server faults either: 500 sits in the transient set + that polling clients retry immediately, which is how one unlucky request + turns into a retry storm. 503 with an explicit `Retry-After` names the + condition as temporary and gives every client the same back-off, whether or + not it reads the body. + """ + + response = _make_error_response( + error, + HTTPStatus.SERVICE_UNAVAILABLE, + result=None, + ) + response.headers["Retry-After"] = str(error.retry_after_seconds) + return response + + @economy_bp.route( "//economy//over/", methods=["GET"], @@ -89,6 +112,8 @@ def get_economic_impact(country_id: str, policy_id: int, baseline_policy_id: int target=query.target, ) ) + except EconomyDependencyUnavailableError as error: + return _dependency_unavailable_response(error) except ValueError as error: return _bad_request_response(error) @@ -139,6 +164,8 @@ def get_budget_window_economic_impact( target=query.target, ) ) + except EconomyDependencyUnavailableError as error: + return _dependency_unavailable_response(error) except ValueError as error: return _bad_request_response(error) diff --git a/policyengine_api/runtime_cache/reform_impacts.py b/policyengine_api/runtime_cache/reform_impacts.py index 920b4f422..20cb00611 100644 --- a/policyengine_api/runtime_cache/reform_impacts.py +++ b/policyengine_api/runtime_cache/reform_impacts.py @@ -102,6 +102,23 @@ def _impact_from_wire(payload: Any) -> CachedReformImpact | None: return None +def _isolation_token(options_json: Any) -> str | None: + """Return the client-isolation token that scopes a record's lookup index. + + A ``cache_nonce`` request is private to the caller that chose the value: + the nonce is part of the options hash, so no other caller's hash or prefix + can ever match its records. Indexing those records beside shared ones would + still let them evict shared ones, because every write trims the scope index + to ``REFORM_IMPACT_INDEX_LIMIT`` newest members. Isolated records therefore + get their own index and exert no eviction pressure on the shared scope. + """ + + if not isinstance(options_json, dict): + return None + token = options_json.get("cache_nonce") + return token if isinstance(token, str) and token else None + + def _like_matches(value: str, pattern: str) -> bool: expression: list[str] = ["^"] escaped = False @@ -233,18 +250,23 @@ def _record_key(self, execution_id: str) -> str: ) def _scope_index(self, impact: CachedReformImpact) -> str: + inputs: dict[str, Any] = { + "api_version": impact.api_version, + "baseline_policy_id": impact.baseline_policy_id, + "country_id": impact.country_id, + "dataset": impact.dataset, + "reform_policy_id": impact.reform_policy_id, + "region": impact.region, + "time_period": impact.time_period, + } + isolation_token = _isolation_token(impact.options_json) + if isolation_token is not None: + # Only isolated records move; the shared scope keeps its own key. + inputs["cache_nonce"] = isolation_token return self.namespace.key( "reform-impact-index", REFORM_IMPACT_SCHEMA_VERSION, - { - "api_version": impact.api_version, - "baseline_policy_id": impact.baseline_policy_id, - "country_id": impact.country_id, - "dataset": impact.dataset, - "reform_policy_id": impact.reform_policy_id, - "region": impact.region, - "time_period": impact.time_period, - }, + inputs, ) def _recent_index(self) -> str: @@ -353,6 +375,7 @@ def matching( api_version: str, options_hash: str, options_hash_pattern: str | None = None, + cache_nonce: str | None = None, ) -> list[CachedReformImpact]: probe = CachedReformImpact( reform_impact_id=0, @@ -362,7 +385,7 @@ def matching( region=region, dataset=dataset, time_period=time_period, - options_json=None, + options_json={"cache_nonce": cache_nonce} if cache_nonce else None, options_hash=options_hash, api_version=api_version, reform_impact_json={}, diff --git a/policyengine_api/services/budget_window_cache.py b/policyengine_api/services/budget_window_cache.py index ae2e0cf6b..6beebbb79 100644 --- a/policyengine_api/services/budget_window_cache.py +++ b/policyengine_api/services/budget_window_cache.py @@ -261,6 +261,40 @@ def store_batch_job_id(self, cache_key: str, batch_job_id: str) -> None: started_at=started_at, ) + def adopt_batch_job_id(self, cache_key: str, batch_job_id: str) -> bool: + """Record a handle under another key's identity without displacing it. + + Used when the gateway reports that it ran a batch on a different worker + application than the one this request resolved. The batch is real and + already running, so its handle belongs under the identity that actually + ran it. Unlike `store_batch_job_id` this never overwrites: an existing + handle or an in-progress starting claim under that identity owns it, + and clobbering either would orphan that request's batch instead. + """ + + started_at = time.perf_counter() + try: + stored = self.client.set( + self._batch_key(cache_key), + batch_job_id, + ex=BUDGET_WINDOW_BATCH_TTL_SECONDS, + nx=True, + ) + except Exception: + self._handle_cache_error( + "adopt-batch-id", + event="coordination-failed", + started_at=started_at, + ) + return False + record_cache_event( + family=BUDGET_WINDOW_CACHE_FAMILY, + event="coordination-write" if stored else "claim-contended", + operation="adopt-batch-id", + started_at=started_at, + ) + return bool(stored) + def clear_starting_claim(self, cache_key: str, claim_token: str) -> None: try: self._claims.release( diff --git a/policyengine_api/services/economy_service.py b/policyengine_api/services/economy_service.py index fc3947f84..796aebabd 100644 --- a/policyengine_api/services/economy_service.py +++ b/policyengine_api/services/economy_service.py @@ -42,6 +42,50 @@ budget_window_cache = BudgetWindowCache() +class EconomyDependencyUnavailableError(Exception): + """A server-side economy dependency failed; the request was never invalid. + + Deliberately not a `ValueError`. The economy routes map `ValueError` to + 400, which is the wrong answer for a worker-registry or submission-routing + failure: the caller sent a well-formed request and cannot fix anything. + These are answered with 503 and a `Retry-After`, so a polling client backs + off on a documented schedule instead of retrying as fast as it can. + """ + + #: Seconds a client should wait before repeating the request. + retry_after_seconds: int = 30 + + def __init__(self, message: str, *, retry_after_seconds: int | None = None): + super().__init__(message) + if retry_after_seconds is not None: + self.retry_after_seconds = retry_after_seconds + + +class WorkerRegistryUnavailableError(EconomyDependencyUnavailableError): + """The gateway registry publishes no worker for this runtime's bundle. + + The bundle version is resolved from the installed distribution and the + runtime manifest, never from the request, so a missing entry is a + deployment or registry problem. It usually needs an operator, hence the + longer default `Retry-After`. + """ + + +class BudgetWindowSubmissionIdentityError(EconomyDependencyUnavailableError): + """The gateway ran the batch on a worker other than the resolved one. + + The registry read and the submission POST cannot be made atomic, so a + worker rollout between them routes the batch to a different application + than the one this request's cache key names. The handle is retained under + the identity the gateway reported (see + `EconomyService._retain_budget_window_batch_under_resolved_identity`), so + the next poll — which re-reads the registry — adopts the running batch + instead of spawning another one. + """ + + retry_after_seconds = 5 + + class ImpactAction(Enum): """ Enum for the action to take based on the status of an economic impact calculation. @@ -387,6 +431,9 @@ def get_budget_window_economic_impact( api_version=api_version, target=target, ) + # Terminal failures and completed batches belong to the worker that + # produced them. A replacement application must get a fresh key. + self._resolve_runtime_bundle_for_setup_options(setup_options) cache_key = self._build_budget_window_cache_key(setup_options) cached_error = self._budget_window_cache.get_terminal_error(cache_key) @@ -439,9 +486,6 @@ def get_budget_window_economic_impact( window_size=window_size, max_parallel=max_active_years, ) - self._budget_window_cache.store_batch_job_id( - cache_key, batch_execution.batch_job_id - ) except httpx.HTTPStatusError as error: self._budget_window_cache.clear_starting_claim( cache_key, claim_token @@ -462,6 +506,37 @@ def get_budget_window_economic_impact( ) raise + if ( + not setup_options.runtime_app_name + or batch_execution.resolved_app_name + != setup_options.runtime_app_name + ): + # The registry may change between resolution and POST. + # Never attach another worker's handle to this key, + # including when the gateway omits identity evidence. The + # POST already spawned the batch, so hand it to the key + # that names the worker running it and keep this key's + # starting claim: both stop a retry from spawning again. + self._retain_budget_window_batch_under_resolved_identity( + setup_options=setup_options, + batch_execution=batch_execution, + ) + raise BudgetWindowSubmissionIdentityError( + "The simulation worker registry changed while this " + "budget window was being submitted; the calculation is " + "running and the next request will pick it up" + ) + + try: + self._budget_window_cache.store_batch_job_id( + cache_key, batch_execution.batch_job_id + ) + except Exception: + self._budget_window_cache.clear_starting_claim( + cache_key, claim_token + ) + raise + return self._build_budget_window_computing_result( total_years=len(years), completed_years=[], @@ -474,6 +549,78 @@ def get_budget_window_economic_impact( print(f"Error getting budget-window economic impact: {str(e)}") raise e + def _retain_budget_window_batch_under_resolved_identity( + self, + *, + setup_options: EconomicImpactSetupOptions, + batch_execution: Any, + ) -> str | None: + """Key an already-spawned batch under the worker that actually ran it. + + The gateway spawns the Modal batch inside the submit call and only then + reports which application it routed to, so a registry change between + this request's lookup and its POST cannot be caught before the work + starts. Discarding the handle would leave that batch running with + nothing pointing at it, and every retry would spawn another. + + The only component of the cache key that differs is the worker + application: `model_version` comes from the installed country package + and the remaining components are this request's own. Re-deriving the + key with the reported application therefore produces exactly the key a + later request computes once the registry serves that application, so + its poll finds this batch. Returns that key, or None when the gateway + reported no identity to file it under. + """ + + resolved_app_name = getattr(batch_execution, "resolved_app_name", None) + batch_job_id = getattr(batch_execution, "batch_job_id", None) + if ( + not isinstance(resolved_app_name, str) + or not resolved_app_name + or not batch_job_id + ): + logger.log_struct( + { + "message": ( + "Budget-window submission reported no worker identity; " + "the spawned batch cannot be filed under a cache key" + ), + **setup_options.model_dump(), + }, + severity="ERROR", + ) + return None + + resolved_setup_options = setup_options.model_copy(deep=True) + resolved_setup_options.runtime_app_name = resolved_app_name + resolved_setup_options.options_hash = self._build_options_hash( + options=resolved_setup_options.options, + model_version=resolved_setup_options.model_version, + dataset=resolved_setup_options.dataset, + data_version=resolved_setup_options.data_version, + policyengine_version=resolved_setup_options.policyengine_version, + runtime_app_name=resolved_app_name, + target=resolved_setup_options.target, + ) + resolved_cache_key = self._build_budget_window_cache_key(resolved_setup_options) + adopted = self._budget_window_cache.adopt_batch_job_id( + resolved_cache_key, batch_job_id + ) + logger.log_struct( + { + "message": ( + "Budget-window batch retained under the submitting worker" + if adopted + else "Budget-window batch identity is already claimed by " + "another request; this batch is not retained" + ), + **setup_options.model_dump(), + "submitted_app_name": resolved_app_name, + }, + severity="WARNING", + ) + return resolved_cache_key if adopted else None + def _build_budget_window_cache_key( self, setup_options: EconomicImpactSetupOptions, @@ -707,6 +854,7 @@ def _build_economic_impact_setup_options( dataset=resolved_dataset, data_version=resolved_data_version, policyengine_version=policyengine_version, + target=target, ) return EconomicImpactSetupOptions.model_validate( @@ -828,14 +976,39 @@ def _resolve_runtime_bundle_for_setup_options( setup_options: EconomicImpactSetupOptions, ) -> None: if setup_options.runtime_app_name is None: - ( - setup_options.runtime_app_name, - setup_options.model_version, - ) = self._simulation_gateway.resolve_app_name( - setup_options.country_id, - setup_options.model_version, - policyengine_version=setup_options.policyengine_version, - ) + try: + ( + setup_options.runtime_app_name, + setup_options.model_version, + ) = self._simulation_gateway.resolve_app_name( + setup_options.country_id, + setup_options.model_version, + policyengine_version=setup_options.policyengine_version, + ) + except SPMValidationError: + # A typed SPM error really is about the request's settings. + raise + except ValueError as error: + # `resolve_app_name` raises ValueError when the registry has no + # entry for the version it was asked about. Both versions it can + # be asked about here are server-side: `policyengine_version` is + # the runtime's own bundle and `model_version` is the installed + # country package. Neither comes from the query, so a missing + # entry is a dependency failure, not a bad request. + raise WorkerRegistryUnavailableError( + "No simulation worker is currently registered for this " + "API's model bundle; the request is valid and can be " + "retried once the worker registry publishes it" + ) from error + except httpx.HTTPError as error: + # The registry itself is unreachable or answering with an + # error. Same class of failure, same answer: the request was + # fine and only a dependency is missing. + raise WorkerRegistryUnavailableError( + "The simulation worker registry is unavailable; the " + "request is valid and can be retried once the registry " + "responds again" + ) from error setup_options.options_hash = self._build_options_hash( options=setup_options.options, @@ -844,6 +1017,7 @@ def _resolve_runtime_bundle_for_setup_options( data_version=setup_options.data_version, policyengine_version=setup_options.policyengine_version, runtime_app_name=setup_options.runtime_app_name, + target=setup_options.target, ) def _reform_impact_start_claim_arguments( @@ -901,6 +1075,22 @@ def _build_budget_window_progress_message( return f"Scoring budget window ({completed_count} of {total_years} complete)..." + @staticmethod + def _cache_isolation_token( + setup_options: EconomicImpactSetupOptions, + ) -> str | None: + """Return the request's client-isolation token, if it supplied one. + + `cache_nonce` is a client-chosen cache identity on an unauthenticated + route. Its records are only ever matched by the same nonce, so they are + indexed apart from the shared scope; otherwise a flood of nonces would + trim the shared scope's bounded index and evict an entry every other + caller reads. + """ + + token = setup_options.options.get("cache_nonce") + return token if isinstance(token, str) and token else None + def _get_previous_impacts( self, country_id: str, @@ -911,6 +1101,7 @@ def _get_previous_impacts( time_period: str, options_hash: str, api_version: str, + cache_nonce: str | None = None, ): """ Fetch any previous simulation runs for the given policy reform. @@ -928,6 +1119,7 @@ def _get_previous_impacts( options_hash, self._build_options_hash_lookup_pattern(options_hash), api_version, + cache_nonce=cache_nonce, ) ) return previous_impacts @@ -949,6 +1141,7 @@ def _get_most_recent_impact( time_period=setup_options.time_period, options_hash=setup_options.options_hash, api_version=setup_options.api_version, + cache_nonce=self._cache_isolation_token(setup_options), ) if not previous_impacts: @@ -1288,11 +1481,13 @@ def _build_options_hash( runtime_app_name: str | None = None, data_version: str | None = None, policyengine_version: str | None = None, + target: Literal["general", "cliff"] = "general", ) -> str: option_pairs = "&".join(f"{key}={options[key]}" for key in sorted(options)) bundle_parts = [ f"dataset={dataset}", f"model_version={model_version}", + f"target={target}", ] if data_version: bundle_parts.append(f"data_version={data_version}") @@ -1353,6 +1548,7 @@ def _should_refresh_cached_impact( data_version=setup_options.data_version, policyengine_version=setup_options.policyengine_version, runtime_app_name=runtime_app_name, + target=setup_options.target, ) if ( not isinstance(cached_resolved_app_name, str) diff --git a/policyengine_api/services/reform_impacts_service.py b/policyengine_api/services/reform_impacts_service.py index 81e66cad2..a9ef73493 100644 --- a/policyengine_api/services/reform_impacts_service.py +++ b/policyengine_api/services/reform_impacts_service.py @@ -41,6 +41,7 @@ def get_all_reform_impacts( time_period, options_hash, api_version, + cache_nonce: str | None = None, ) -> list[CachedReformImpact]: return self._cache.matching( country_id=country_id, @@ -51,6 +52,7 @@ def get_all_reform_impacts( time_period=time_period, api_version=api_version, options_hash=options_hash, + cache_nonce=cache_nonce, ) def get_all_reform_impacts_by_options_hash_prefix( @@ -64,7 +66,14 @@ def get_all_reform_impacts_by_options_hash_prefix( options_hash, options_hash_prefix, api_version, + cache_nonce: str | None = None, ) -> list[CachedReformImpact]: + """Look up a scope's records, isolated by ``cache_nonce`` when supplied. + + A nonce'd request reads its own index, so it never sees — and its + writes never displace — the entries shared by ordinary callers. + """ + return self._cache.matching( country_id=country_id, reform_policy_id=policy_id, @@ -75,6 +84,7 @@ def get_all_reform_impacts_by_options_hash_prefix( api_version=api_version, options_hash=options_hash, options_hash_pattern=options_hash_prefix, + cache_nonce=cache_nonce, ) def claim_reform_impact_start( diff --git a/policyengine_api/worker_spm_release.py b/policyengine_api/worker_spm_release.py new file mode 100644 index 000000000..a9b41fb0b --- /dev/null +++ b/policyengine_api/worker_spm_release.py @@ -0,0 +1,63 @@ +"""Validate the installed API bundle against the selected worker release registry.""" + +from importlib.metadata import version +import json +import sys + +from policyengine_api.worker_spm import validate_worker_spm + + +class _RegistryCapability: + def __init__(self, registry: dict): + self.registry = registry + + def get_spm_capability(self, country, model_version, *, policyengine_version): + capabilities = self.registry.get("spm_capabilities") + return ( + capabilities.get(policyengine_version) + if isinstance(capabilities, dict) + else None + ) + + +def validate_installed_worker(registry: dict, expected_bundle: str) -> dict | None: + """Fail before promotion if the selected worker cannot run this API bundle. + + Use the request validator directly: a legacy country can omit capability, + while a canonical country requires certification and a matching forecast. + """ + if version("policyengine") != expected_bundle: + raise ValueError( + "The installed PolicyEngine bundle differs from the release pin" + ) + from policyengine_api.constants import POLICYENGINE_VERSION + + if POLICYENGINE_VERSION != expected_bundle: + raise ValueError("The runtime-selected bundle differs from the release pin") + routes = registry.get("policyengine") if isinstance(registry, dict) else None + app = routes.get(expected_bundle) if isinstance(routes, dict) else None + if not isinstance(app, str) or not app: + raise ValueError("The selected bundle has no registered worker application") + return validate_worker_spm( + "us", + gateway=_RegistryCapability(registry), + policyengine_version=expected_bundle, + ) + + +def main() -> None: + if len(sys.argv) != 2: + raise SystemExit( + "Usage: python -m policyengine_api.worker_spm_release BUNDLE_VERSION" + ) + selection = validate_installed_worker(json.load(sys.stdin), sys.argv[1]) + if selection is None: + print( + "Legacy API bundle is registered; canonical SPM capability is not required" + ) + else: + print("Installed API bundle and registered bundle's SPM capability agree") + + +if __name__ == "__main__": + main() diff --git a/tests/contract/test_v1_route_contracts.py b/tests/contract/test_v1_route_contracts.py index db1457ff5..e26b78da0 100644 --- a/tests/contract/test_v1_route_contracts.py +++ b/tests/contract/test_v1_route_contracts.py @@ -87,6 +87,16 @@ def to_dict(self): } +class _EconomyDependencyUnavailableError(Exception): + """Stand-in for the route's 503 dependency-failure type. + + The route only needs an exception class to name in an `except` clause and + a back-off to publish; the contract fixtures never raise one. + """ + + retry_after_seconds = 30 + + class _EconomyService: def get_budget_window_economic_impact(self, **kwargs): return _BudgetWindowEconomicImpactResult.completed( @@ -144,6 +154,7 @@ def _load_contract_economy_blueprint(): service_module_name="policyengine_api.services.economy_service", route_module_name="policyengine_api.routes.economy_routes", fake_service_module=SimpleNamespace( + EconomyDependencyUnavailableError=_EconomyDependencyUnavailableError, EconomyService=_EconomyService, EconomicImpactResult=object, BudgetWindowEconomicImpactResult=_BudgetWindowEconomicImpactResult, diff --git a/tests/fixtures/libs/simulation_entrypoint.py b/tests/fixtures/libs/simulation_entrypoint.py index b196559bd..2cf4ed371 100644 --- a/tests/fixtures/libs/simulation_entrypoint.py +++ b/tests/fixtures/libs/simulation_entrypoint.py @@ -100,6 +100,7 @@ "poll_url": f"/budget-window-jobs/{MOCK_BATCH_JOB_ID}", "country": "us", "version": "1.459.0", + "resolved_app_name": "test-budget-window-worker", } MOCK_BATCH_POLL_RESPONSE_RUNNING = { diff --git a/tests/fixtures/services/economy_service.py b/tests/fixtures/services/economy_service.py index 02553bf39..9c2bcbf22 100644 --- a/tests/fixtures/services/economy_service.py +++ b/tests/fixtures/services/economy_service.py @@ -26,7 +26,7 @@ "[option1=value1&option2=value2" "&dataset=hf://policyengine/faux-populace-us/faux_populace_us_2099.h5@" "faux-populace-us-2099-test-release" - "&model_version=1.2.3" + "&model_version=1.2.3&target=general" "&data_version=faux-populace-us-2099-test-release" "&policyengine_version=3.4.0]" ) @@ -299,6 +299,7 @@ def create_mock_budget_window_batch_execution( """Helper function to create mock batch execution objects.""" mock_execution = MagicMock() mock_execution.batch_job_id = batch_job_id + mock_execution.resolved_app_name = MOCK_RESOLVED_APP_NAME mock_execution.name = batch_job_id mock_execution.status = status mock_execution.progress = progress diff --git a/tests/integration/test_budget_window_in_flight_dedupe.py b/tests/integration/test_budget_window_in_flight_dedupe.py index c944224b6..0529473aa 100644 --- a/tests/integration/test_budget_window_in_flight_dedupe.py +++ b/tests/integration/test_budget_window_in_flight_dedupe.py @@ -31,12 +31,16 @@ def test_budget_window_in_flight_dedupe_uses_existing_batch_without_live_db( fake_cache = BudgetWindowCache(client=FakeRedis()) simulation_entrypoint = MagicMock() + simulation_entrypoint.resolve_app_name.side_effect = ( + lambda country, version, **kwargs: ("test-budget-window-worker", version) + ) reform_impacts_service = MagicMock() simulation_entrypoint.run_budget_window_batch.return_value = ( ModalBudgetWindowBatchExecution( batch_job_id="fc-budget-window-parent", status="submitted", + resolved_app_name="test-budget-window-worker", ) ) simulation_entrypoint.get_budget_window_batch_by_id.return_value = ( diff --git a/tests/integration/test_cloud_run_candidate.py b/tests/integration/test_cloud_run_candidate.py index 81fb8df8f..657642eaf 100644 --- a/tests/integration/test_cloud_run_candidate.py +++ b/tests/integration/test_cloud_run_candidate.py @@ -1,3 +1,8 @@ +from datetime import date +import math +from uuid import uuid4 + + def test_cloud_run_candidate_health_routes(api_client): health_response = api_client.get("/health") assert health_response.status_code == 200, health_response.text @@ -66,3 +71,48 @@ def test_cloud_run_candidate_stage6_read_route_contracts(api_client): "Country zz not found. Available countries are: uk, us, ca, ng, il" ), } + + +def test_cloud_run_candidate_current_law_economy(api_client, poll_live_endpoint): + """Exercise the candidate's selected worker without creating a reform policy.""" + response = api_client.get("/us/metadata") + response.raise_for_status() + metadata = response.json()["result"] + current_law = metadata["current_law_id"] + years = [ + int(period["name"]) + for period in metadata["economy_options"]["time_period"] + if str(period["name"]).isdigit() + ] + assert years, "Metadata exposes no economy years" + current_year = date.today().year + year = str( + max((value for value in years if value <= current_year), default=min(years)) + ) + payload = poll_live_endpoint( + api_client, + f"/us/economy/{current_law}/over/{current_law}", + {"region": "ut", "time_period": year, "cache_nonce": str(uuid4())}, + route_name="Cloud Run candidate current-law economy", + ) + assert payload["status"] == "ok", payload + result = payload["result"] + impact = result["budget"]["budgetary_impact"] + assert isinstance(impact, int | float) and math.isfinite(impact), result + assert impact == 0, "Comparing current law with itself must have zero budget impact" + spm = metadata.get("spm", {}) + if spm.get("available"): + defaults = spm["defaults"] + config = result["spm_config"] + assert isinstance(config, dict) and set(config) <= set(defaults) + # HTTP transports may omit optional null settings; they may never + # inherit a missing non-null default or introduce an unknown setting. + assert all(config.get(key) == value for key, value in defaults.items()) + for side in ("baseline", "reform"): + receipts = result["spm_provenance"][side] + assert receipts, f"Missing {side} SPM receipts" + for receipt in receipts: + assert receipt["forecast_sha256"] == defaults["forecast_content_sha256"] + assert receipt["scenario"] == defaults["scenario"] + assert receipt["geography_kind"] == defaults["geography_kind"] + assert year in receipt["years"] diff --git a/tests/unit/libs/test_simulation_entrypoint.py b/tests/unit/libs/test_simulation_entrypoint.py index b1deb55ad..2f29ad609 100644 --- a/tests/unit/libs/test_simulation_entrypoint.py +++ b/tests/unit/libs/test_simulation_entrypoint.py @@ -745,6 +745,7 @@ def test__given_valid_payload__then_returns_batch_execution( assert execution.batch_job_id == MOCK_BATCH_JOB_ID assert execution.status == MODAL_EXECUTION_STATUS_SUBMITTED + assert execution.resolved_app_name == "test-budget-window-worker" call_args = mock_httpx_client.post.call_args assert "/simulate/economy/budget-window" in call_args[0][0] diff --git a/tests/unit/routes/test_economy_failure_replay.py b/tests/unit/routes/test_economy_failure_replay.py index a1889f603..bc3a1338f 100644 --- a/tests/unit/routes/test_economy_failure_replay.py +++ b/tests/unit/routes/test_economy_failure_replay.py @@ -131,3 +131,43 @@ def test_successful_cached_job_still_requires_canonical_receipts(failed_job_harn assert response.json["errors"][0]["code"] == "SPM_CONFIGURATION_UNAVAILABLE" gateway.get_execution_by_id.assert_not_called() gateway.run.assert_not_called() + + +def test_annual_cliff_request_does_not_replay_general_failure( + failed_job_harness, monkeypatch +): + client, _, gateway, _, _ = failed_job_harness + url = "/us/economy/1/over/2?region=us&time_period=2026&version=1.0.0" + assert client.get(url).status_code == 502 + service = economy_routes.economy_service + monkeypatch.setattr(service, "_get_policy_jsons", lambda *_: ({}, {})) + gateway.run.return_value = SimpleNamespace(execution_id="new-cliff-job") + gateway.get_execution_id.return_value = "new-cliff-job" + response = client.get(url + "&target=cliff") + assert response.status_code == 200, response.json + assert response.json["status"] == "computing" + assert gateway.run.call_count == 1 + assert gateway.run.call_args.args[0]["include_cliffs"] is True + + +@pytest.mark.parametrize("failed_job_harness", [None], indirect=True, ids=["legacy"]) +def test_cliff_request_does_not_reuse_completed_general_result( + failed_job_harness, monkeypatch +): + client, cache, gateway, _, _ = failed_job_harness + cache.update( + EXECUTION_ID, + status="ok", + message="Completed", + reform_impact_json={"cliff_impact": None, "resolved_app_name": "test-worker"}, + ) + url = "/us/economy/1/over/2?region=us&time_period=2026&version=1.0.0" + assert client.get(url).status_code == 200 + service = economy_routes.economy_service + monkeypatch.setattr(service, "_get_policy_jsons", lambda *_: ({}, {})) + gateway.run.return_value = SimpleNamespace(execution_id="new-cliff-job") + gateway.get_execution_id.return_value = "new-cliff-job" + response = client.get(url + "&target=cliff") + assert response.status_code == 200, response.json + assert response.json["status"] == "computing" + assert gateway.run.call_args.args[0]["include_cliffs"] is True diff --git a/tests/unit/routes/test_economy_submission_identity.py b/tests/unit/routes/test_economy_submission_identity.py new file mode 100644 index 000000000..9b0de13c7 --- /dev/null +++ b/tests/unit/routes/test_economy_submission_identity.py @@ -0,0 +1,507 @@ +"""Cache identity must describe the worker execution actually being qualified.""" + +from types import SimpleNamespace +import json +from uuid import uuid4 + +from flask import Flask +import httpx +import pytest + +from policyengine_api.libs.simulation_entrypoint import SimulationEntrypointClient +from policyengine_api.routes import economy_routes +from policyengine_api.runtime_cache.core import CacheNamespace +from policyengine_api.runtime_cache.fake import InMemoryCacheBackend +from policyengine_api.runtime_cache.reform_impacts import ( + REFORM_IMPACT_INDEX_LIMIT, + ReformImpactCache, +) +from policyengine_api.services import economy_service as economy_module +from policyengine_api.services.budget_window_cache import BudgetWindowCache +from policyengine_api.services.economy_service import EconomyService +from policyengine_api.services.reform_impacts_service import ReformImpactsService +from tests.integration.test_cloud_run_candidate import ( + test_cloud_run_candidate_current_law_economy as run_smoke, +) + + +@pytest.fixture +def economy(monkeypatch): + # The transport/cache boundary needs no real bundle or population compute. + monkeypatch.setattr(economy_module, "validate_worker_spm", lambda *a, **k: None) + monkeypatch.setattr(economy_module, "COUNTRY_PACKAGE_VERSIONS", {"us": "1.0.0"}) + monkeypatch.setattr(economy_routes, "COUNTRY_PACKAGE_VERSIONS", {"us": "1.0.0"}) + monkeypatch.setattr(economy_module, "POLICYENGINE_VERSION", "5.2.0") + backend = InMemoryCacheBackend() + namespace = CacheNamespace("test", "submission-identity") + impacts = ReformImpactsService(ReformImpactCache(backend, namespace)) + window = BudgetWindowCache(backend, namespace) + gateway = object.__new__(SimulationEntrypointClient) + gateway.base_url = "https://worker.invalid" + service = EconomyService( + reform_impacts_service_=impacts, + budget_window_cache_=window, + simulation_entrypoint_=gateway, + ) + monkeypatch.setattr(service, "_get_policy_jsons", lambda *_: ({}, {})) + monkeypatch.setattr(economy_routes, "economy_service", service) + app = Flask(__name__) + app.config["TESTING"] = True + app.register_blueprint(economy_routes.economy_bp) + return service, impacts, window, gateway, app.test_client() + + +BUDGET_WINDOW_QUERY = {"region": "us", "start_year": "2025", "window_size": "2"} + + +def _budget_window_keys(service): + """Return the budget-window cache keys for worker A and worker B.""" + + keys = {} + for app_name in ("worker-A", "worker-B"): + setup = service._build_economic_impact_setup_options( + country_id="us", + policy_id=2, + baseline_policy_id=2, + region="us", + dataset="default", + time_period="budget_window:2025:2", + options={}, + api_version="1.0.0", + ) + setup.runtime_app_name = app_name + setup.options_hash = service._build_options_hash( + options=setup.options, + model_version=setup.model_version, + dataset=setup.dataset, + data_version=setup.data_version, + policyengine_version=setup.policyengine_version, + runtime_app_name=app_name, + target=setup.target, + ) + keys[app_name] = service._build_budget_window_cache_key(setup) + return keys + + +@pytest.mark.parametrize("submitted_app", ["worker-B", None, ""]) +def test_budget_submission_identity_is_verified_before_caching(economy, submitted_app): + service, _, window, gateway, client = economy + active_app = "worker-A" + posts = [] + + def transport(request): + nonlocal active_app + if request.url.path == "/versions/policyengine": + return httpx.Response(200, json={"5.2.0": active_app}) + assert request.method == "POST" + assert request.url.path == "/simulate/economy/budget-window" + posts.append(request) + # The registry changes after A was resolved, before B was submitted. + active_app = "worker-B" + return httpx.Response( + 200, + json={ + "batch_job_id": "batch-B", + "status": "submitted", + "resolved_app_name": submitted_app, + }, + ) + + with httpx.Client(transport=httpx.MockTransport(transport)) as gateway.client: + keys = _budget_window_keys(service) + key_a = keys["worker-A"] + for attempt in range(2): + # A rollback must never replay B's handle under A's cache identity. + active_app = "worker-A" + response = client.get( + "/us/economy/2/over/2/budget-window", + query_string=BUDGET_WINDOW_QUERY, + ) + if attempt == 0: + # The request that saw the mismatch says "come back shortly", + # not "you sent a bad request" and not "something broke". + assert response.status_code == 503, response.json + assert response.headers["Retry-After"] == "5" + assert "registry changed" in response.json["message"] + else: + # A's starting claim still stands, so the retry waits rather + # than paying for a second batch. + assert response.status_code == 200, response.json + assert response.json["status"] == "computing" + assert window.get_batch_job_id(key_a) is None + assert window.get_completed_result(key_a) is None + assert window.get_terminal_error(key_a) is None + # One POST, one spawned batch: the retry never reached the gateway. + assert len(posts) == 1 + + +def test_budget_submission_identity_mismatch_retains_the_batch_for_the_next_poll( + economy, +): + service, _, window, gateway, client = economy + active_app = "worker-A" + posts = [] + + def transport(request): + nonlocal active_app + if request.url.path == "/versions/policyengine": + return httpx.Response(200, json={"5.2.0": active_app}) + if request.method == "POST": + posts.append(request) + # The rollout completes between this request's lookup and its POST. + active_app = "worker-B" + return httpx.Response( + 200, + json={ + "batch_job_id": "batch-B", + "status": "submitted", + "resolved_app_name": "worker-B", + }, + ) + assert request.url.path == "/budget-window-jobs/batch-B" + return httpx.Response( + 200, + json={ + "batch_job_id": "batch-B", + "status": "running", + "progress": 0, + "completed_years": [], + "running_years": ["2025"], + "queued_years": ["2026"], + "failed_years": [], + }, + ) + + with httpx.Client(transport=httpx.MockTransport(transport)) as gateway.client: + keys = _budget_window_keys(service) + response = client.get( + "/us/economy/2/over/2/budget-window", + query_string=BUDGET_WINDOW_QUERY, + ) + assert response.status_code == 503, response.json + # The spawned batch is filed under the worker that actually ran it, + # never under the identity this request resolved. + assert window.get_batch_job_id(keys["worker-A"]) is None + assert window.get_batch_job_id(keys["worker-B"]) == "batch-B" + + # The next poll re-reads the registry, which now serves worker B, and + # adopts the running batch instead of spawning a second one. + response = client.get( + "/us/economy/2/over/2/budget-window", + query_string=BUDGET_WINDOW_QUERY, + ) + assert response.status_code == 200, response.json + assert response.json["status"] == "computing" + assert response.headers["X-PolicyEngine-Budget-Window-Cache"] == "batch-id-hit" + assert len(posts) == 1 + + +def test_budget_submission_identity_mismatch_never_overwrites_another_claim(economy): + service, _, window, gateway, client = economy + active_app = "worker-A" + + def transport(request): + if request.url.path == "/versions/policyengine": + return httpx.Response(200, json={"5.2.0": active_app}) + return httpx.Response( + 200, + json={ + "batch_job_id": "batch-late", + "status": "submitted", + "resolved_app_name": "worker-B", + }, + ) + + with httpx.Client(transport=httpx.MockTransport(transport)) as gateway.client: + keys = _budget_window_keys(service) + window.store_batch_job_id(keys["worker-B"], "batch-already-running") + response = client.get( + "/us/economy/2/over/2/budget-window", + query_string=BUDGET_WINDOW_QUERY, + ) + assert response.status_code == 503, response.json + # Worker B's key already points at a batch; displacing it would orphan + # that one instead. + assert window.get_batch_job_id(keys["worker-B"]) == "batch-already-running" + + +@pytest.mark.parametrize( + "path,query", + [ + ("/us/economy/2/over/2", {"region": "ut", "time_period": "2025"}), + ("/us/economy/2/over/2/budget-window", BUDGET_WINDOW_QUERY), + ], +) +def test_missing_registry_entry_is_a_dependency_failure_not_a_bad_request( + economy, path, query +): + _, _, _, gateway, client = economy + + def transport(request): + assert request.url.path == "/versions/policyengine" + # The registry is reachable but publishes no worker for this bundle. + return httpx.Response(200, json={"4.0.0": "worker-old"}) + + with httpx.Client(transport=httpx.MockTransport(transport)) as gateway.client: + response = client.get(path, query_string=query) + assert response.status_code == 503, response.json + assert response.headers["Retry-After"] == "30" + assert ( + "No simulation worker is currently registered" in response.json["message"] + ) + + +@pytest.mark.parametrize( + "path,query", + [ + ("/us/economy/2/over/2", {"region": "ut", "time_period": "2025"}), + ("/us/economy/2/over/2/budget-window", BUDGET_WINDOW_QUERY), + ], +) +def test_unreachable_registry_is_a_dependency_failure_not_a_server_fault( + economy, path, query +): + _, _, _, gateway, client = economy + + def transport(request): + raise httpx.ConnectError("registry unreachable", request=request) + + with httpx.Client(transport=httpx.MockTransport(transport)) as gateway.client: + response = client.get(path, query_string=query) + # 500 is a status polling clients retry immediately; 503 carries an + # explicit back-off for the same condition. + assert response.status_code == 503, response.json + assert response.headers["Retry-After"] == "30" + assert "registry is unavailable" in response.json["message"] + + +def test_invalid_query_is_still_a_bad_request_when_the_registry_is_empty(economy): + _, _, _, gateway, client = economy + + def transport(request): + return httpx.Response(200, json={}) + + with httpx.Client(transport=httpx.MockTransport(transport)) as gateway.client: + response = client.get( + "/us/economy/2/over/2/budget-window", + query_string={"region": "us", "start_year": "2025", "window_size": "0"}, + ) + assert response.status_code == 400, response.json + + +def test_nonce_floods_cannot_evict_the_shared_scope_entry(economy): + service, impacts, _, gateway, client = economy + posts = [] + shared_result = {"budget": {"budgetary_impact": 0}, "resolved_app_name": "worker-A"} + + def transport(request): + if request.url.path == "/versions/policyengine": + return httpx.Response(200, json={"5.2.0": "worker-A"}) + posts.append(request) + return httpx.Response( + 200, + json={ + "job_id": f"job-{len(posts)}", + "status": "submitted", + "resolved_app_name": "worker-A", + }, + ) + + with httpx.Client(transport=httpx.MockTransport(transport)) as gateway.client: + shared = service._build_economic_impact_setup_options( + country_id="us", + policy_id=2, + baseline_policy_id=2, + region="state/ut", + dataset="default", + time_period="2025", + options={}, + api_version="1.0.0", + ) + service._resolve_runtime_bundle_for_setup_options(shared) + impacts.set_reform_impact( + country_id=shared.country_id, + policy_id=2, + baseline_policy_id=2, + region=shared.region, + dataset=shared.dataset, + time_period=shared.time_period, + options=shared.options, + options_hash=shared.options_hash, + status="ok", + api_version=shared.api_version, + reform_impact_json=shared_result, + start_time=None, + execution_id="shared-job", + ) + + # One more than the shared scope index holds. Every one of these is + # newer than the shared entry, so an unpartitioned index would have + # trimmed the shared entry out of its own scope. + for index in range(REFORM_IMPACT_INDEX_LIMIT + 1): + nonce = str(uuid4()) + isolated = service._build_economic_impact_setup_options( + country_id="us", + policy_id=2, + baseline_policy_id=2, + region="state/ut", + dataset="default", + time_period="2025", + options={"cache_nonce": nonce}, + api_version="1.0.0", + ) + service._resolve_runtime_bundle_for_setup_options(isolated) + impacts.set_reform_impact( + country_id=isolated.country_id, + policy_id=2, + baseline_policy_id=2, + region=isolated.region, + dataset=isolated.dataset, + time_period=isolated.time_period, + options=isolated.options, + options_hash=isolated.options_hash, + status="ok", + api_version=isolated.api_version, + reform_impact_json={"budget": {"budgetary_impact": 0}}, + start_time=None, + execution_id=f"isolated-job-{index}", + ) + + response = client.get( + "/us/economy/2/over/2", query_string={"region": "ut", "time_period": "2025"} + ) + assert response.status_code == 200, response.json + assert response.json["status"] == "ok" + # The shared entry answered; nothing was resubmitted to the worker. + assert not posts + + +def test_candidate_smoke_cannot_pass_from_prior_result_when_submission_is_broken( + economy, +): + service, impacts, _, gateway, client = economy + posts = [] + result = {"budget": {"budgetary_impact": 0}, "resolved_app_name": "worker-A"} + metadata = { + "current_law_id": 2, + "economy_options": {"time_period": [{"name": "2025"}]}, + "spm": {"available": False}, + } + + def transport(request): + if request.url.path == "/versions/policyengine": + return httpx.Response(200, json={"5.2.0": "worker-A"}) + assert request.method == "POST" + assert request.url.path == "/simulate/economy/comparison" + posts.append(request) + raise httpx.ConnectError("candidate submission is broken", request=request) + + with httpx.Client(transport=httpx.MockTransport(transport)) as gateway.client: + setup = service._build_economic_impact_setup_options( + country_id="us", + policy_id=2, + baseline_policy_id=2, + region="state/ut", + dataset="default", + time_period="2025", + options={}, + api_version="1.0.0", + ) + service._resolve_runtime_bundle_for_setup_options(setup) + impacts.set_reform_impact( + country_id=setup.country_id, + policy_id=2, + baseline_policy_id=2, + region=setup.region, + dataset=setup.dataset, + time_period=setup.time_period, + options=setup.options, + options_hash=setup.options_hash, + status="ok", + api_version=setup.api_version, + reform_impact_json=result, + start_time=None, + execution_id="prior-deployment-job", + ) + response = client.get( + "/us/economy/2/over/2", query_string={"region": "ut", "time_period": "2025"} + ) + assert response.json["status"] == "ok" + assert not posts + + metadata_client = SimpleNamespace( + get=lambda _: SimpleNamespace( + raise_for_status=lambda: None, json=lambda: {"result": metadata} + ) + ) + + def poll(_client, path, query, **kwargs): + response = client.get(path, query_string=query) + assert response.status_code == 200, response.json + return response.json + + with pytest.raises(httpx.ConnectError, match="candidate submission is broken"): + run_smoke(metadata_client, poll) + assert len(posts) == 1 + assert "cache_nonce" not in json.loads(posts[0].content) + + +def test_nonce_polling_reuses_its_job_but_another_nonce_submits_again(economy): + _, _, _, gateway, client = economy + submissions = [] + + def transport(request): + if request.url.path == "/versions/policyengine": + return httpx.Response(200, json={"5.2.0": "worker-A"}) + if request.method == "POST": + submissions.append(json.loads(request.content)) + return httpx.Response( + 200, + json={ + "job_id": f"job-{len(submissions)}", + "status": "submitted", + "resolved_app_name": "worker-A", + }, + ) + assert request.url.path == "/jobs/job-1" + return httpx.Response( + 200, + json={ + "status": "complete", + "result": {"budget": {"budgetary_impact": 0}}, + "resolved_app_name": "worker-A", + }, + ) + + path = "/us/economy/2/over/2" + query = {"region": "ut", "time_period": "2025", "cache_nonce": str(uuid4())} + with httpx.Client(transport=httpx.MockTransport(transport)) as gateway.client: + for expected in ("computing", "ok", "ok"): + response = client.get(path, query_string=query) + assert response.status_code == 200, response.json + assert response.json["status"] == expected + assert len(submissions) == 1 + query["cache_nonce"] = str(uuid4()) + response = client.get(path, query_string=query) + assert response.json["status"] == "computing" + assert len(submissions) == 2 + # Job/correlation metadata varies; the worker's calculation is identical. + computation_payloads = [ + {key: value for key, value in payload.items() if not key.startswith("_")} + for payload in submissions + ] + assert computation_payloads[0] == computation_payloads[1] + assert "cache_nonce" not in submissions[0] + + +@pytest.mark.parametrize( + "nonce_values", [["invalid"], [""], [str(uuid4()), str(uuid4())]] +) +def test_invalid_or_duplicate_nonce_never_reaches_the_worker(economy, nonce_values): + _, _, _, _, client = economy + response = client.get( + "/us/economy/2/over/2", + query_string=[("region", "ut"), ("time_period", "2025")] + + [("cache_nonce", value) for value in nonce_values], + ) + assert response.status_code == 400, response.json diff --git a/tests/unit/routes/test_spm_year_worker_polling.py b/tests/unit/routes/test_spm_year_worker_polling.py index 797a57097..092fa657e 100644 --- a/tests/unit/routes/test_spm_year_worker_polling.py +++ b/tests/unit/routes/test_spm_year_worker_polling.py @@ -94,9 +94,20 @@ def test_typed_worker_error_is_terminal_and_replays_after_service_recreation( def transport(request): received.append((request.method, request.url.path)) if request.method == "POST" and worker_app == "replacement-worker": - assert request.url.path == "/simulate/economy/comparison" + expected_path = ( + "/simulate/economy/budget-window" + if budget_window + else "/simulate/economy/comparison" + ) + assert request.url.path == expected_path return httpx.Response( - 200, json={"job_id": "replacement-job", "status": "submitted"} + 200, + json={ + "job_id": "replacement-job", + "batch_job_id": "replacement-job", + "status": "submitted", + "resolved_app_name": worker_app, + }, ) assert request.method == "GET", "Polling must not submit a replacement job" if request.url.path == "/versions": @@ -155,6 +166,7 @@ def transport(request): api_version="1.0.0", ) if budget_window: + service._resolve_runtime_bundle_for_setup_options(setup) cache_key = service._build_budget_window_cache_key(setup) window_cache.store_batch_job_id(cache_key, job_id) url = "/us/economy/1/over/2/budget-window" @@ -219,17 +231,19 @@ def transport(request): ) monkeypatch.setattr(economy_routes, "economy_service", service) - if not budget_window: - # Preserve the existing retry policy: only a newly resolved worker - # creates a fresh canonical cache identity and permits submission. - worker_app = "replacement-worker" - monkeypatch.setattr(service, "_get_policy_jsons", lambda *_: ({}, {})) - response = app.test_client().get( - url, - query_string={"region": "us", "version": "1.0.0", **query}, - ) - assert response.status_code == 200, response.json - assert response.json["status"] == "computing" + # Only a newly resolved worker permits a replacement submission. + worker_app = "replacement-worker" + monkeypatch.setattr(service, "_get_policy_jsons", lambda *_: ({}, {})) + response = app.test_client().get( + url, + query_string={"region": "us", "version": "1.0.0", **query}, + ) + assert response.status_code == 200, response.json + assert response.json["status"] == "computing" + if budget_window: + assert window_cache.get_terminal_error(cache_key) == typed_error + assert received.count(("POST", "/simulate/economy/budget-window")) == 1 + else: replacement = annual_cache.get_by_execution_id("replacement-job") assert replacement.options_hash != setup.options_hash assert replacement.options_json == setup.options @@ -307,6 +321,7 @@ def transport(request): api_version="1.0.0", ) if budget_window: + service._resolve_runtime_bundle_for_setup_options(setup) cache_key = service._build_budget_window_cache_key(setup) window_cache.set_completed_result( cache_key, diff --git a/tests/unit/runtime_cache/test_reform_impacts.py b/tests/unit/runtime_cache/test_reform_impacts.py index 98b134b4f..3e42a3d05 100644 --- a/tests/unit/runtime_cache/test_reform_impacts.py +++ b/tests/unit/runtime_cache/test_reform_impacts.py @@ -1,5 +1,6 @@ """Reform-impact cache tests.""" +from dataclasses import replace from datetime import datetime import pytest @@ -204,3 +205,81 @@ def test_failed_terminal_error_write_keeps_existing_worker_handle(monkeypatch): unchanged = cache.get_by_execution_id("job") assert unchanged.status == "computing" assert unchanged.execution_id == "job" + + +def test_client_isolated_records_index_apart_from_the_shared_scope( + monkeypatch, +) -> None: + import policyengine_api.runtime_cache.reform_impacts as module + + monkeypatch.setattr(module, "REFORM_IMPACT_INDEX_LIMIT", 2) + backend = InMemoryCacheBackend() + cache = ReformImpactCache(backend, _namespace()) + shared = _impact("shared", "hash-shared", 1) + assert cache.set(shared) + + lookup = { + "country_id": "us", + "reform_policy_id": 2, + "baseline_policy_id": 1, + "region": "us", + "dataset": "default", + "time_period": "2026", + "api_version": "v1", + } + + # More isolated records than the index holds, each newer than the shared + # one: an index they shared would have trimmed the shared record away. + for day in range(2, 6): + nonce = f"nonce-{day}" + isolated = _impact(f"isolated-{day}", f"hash-{nonce}", day) + isolated = replace(isolated, options_json={"cache_nonce": nonce}) + assert cache.set(isolated) + + assert [ + value.execution_id + for value in cache.matching(**lookup, options_hash="hash-shared") + ] == ["shared"] + + # Each isolated request still finds its own record, and only its own. + assert [ + value.execution_id + for value in cache.matching( + **lookup, + options_hash="hash-nonce-5", + cache_nonce="nonce-5", + ) + ] == ["isolated-5"] + assert ( + cache.matching( + **lookup, + options_hash="hash-shared", + cache_nonce="nonce-5", + ) + == [] + ) + + +def test_shared_scope_index_key_is_unchanged_by_isolation_support() -> None: + backend = InMemoryCacheBackend() + cache = ReformImpactCache(backend, _namespace()) + shared = _impact("shared", "hash-shared", 1) + + # Records with no nonce keep the scope key they already had, so supporting + # isolation does not rotate the shared index on deployment. + assert cache._scope_index(shared) == _namespace().key( + "reform-impact-index", + 1, + { + "api_version": "v1", + "baseline_policy_id": 1, + "country_id": "us", + "dataset": "default", + "reform_policy_id": 2, + "region": "us", + "time_period": "2026", + }, + ) + assert cache._scope_index( + replace(shared, options_json={"cache_nonce": "nonce"}) + ) != cache._scope_index(shared) diff --git a/tests/unit/services/test_budget_window_cache.py b/tests/unit/services/test_budget_window_cache.py index 0bb51ee9d..ca5f9cd77 100644 --- a/tests/unit/services/test_budget_window_cache.py +++ b/tests/unit/services/test_budget_window_cache.py @@ -294,3 +294,29 @@ def test_clear_batch_job_id_logs_and_swallows_errors(monkeypatch): cache.clear_batch_job_id("budget_window:v1:us:key") assert mock_logger.log_struct.call_args.kwargs["severity"] == "WARNING" + + +def test_adopt_batch_job_id_files_a_handle_without_displacing_one(): + backend = FakeRedis() + cache = BudgetWindowCache(client=backend) + key = "budget_window:v1:us:key" + + assert cache.adopt_batch_job_id(key, "batch-1") is True + assert cache.get_batch_job_id(key) == "batch-1" + assert set(backend._expires.values()) == {BUDGET_WINDOW_BATCH_TTL_SECONDS} + + # Adoption is for a batch nobody has claimed: it must never take a key + # another request already owns, which would orphan that request's work. + assert cache.adopt_batch_job_id(key, "batch-2") is False + assert cache.get_batch_job_id(key) == "batch-1" + + claimed = "budget_window:v1:us:claimed" + assert cache.claim_batch_start(claimed, "process-1") is True + assert cache.adopt_batch_job_id(claimed, "batch-3") is False + assert cache.get_batch_job_id(claimed) is None + + +def test_adopt_batch_job_id_reports_failure_instead_of_raising(): + cache = BudgetWindowCache(client=RaisingRedis(method="set")) + + assert cache.adopt_batch_job_id("budget_window:v1:us:key", "batch-1") is False diff --git a/tests/unit/services/test_economy_service.py b/tests/unit/services/test_economy_service.py index 97c942623..17a2b687b 100644 --- a/tests/unit/services/test_economy_service.py +++ b/tests/unit/services/test_economy_service.py @@ -654,6 +654,7 @@ def test__given_runtime_cache_version__uses_versioned_economy_cache_key( MOCK_LOOKUP_OPTIONS_HASH ), cache_version, + cache_nonce=None, ) def test__given_alias_dataset__queries_previous_impacts_with_resolved_bundle( @@ -1571,6 +1572,7 @@ def test_given_valid_parameters_calls_service_correctly( MOCK_OPTIONS_HASH, economy_service._build_options_hash_lookup_pattern(MOCK_OPTIONS_HASH), MOCK_API_VERSION, + cache_nonce=None, ) class TestGetMostRecentImpact: diff --git a/tests/unit/test_candidate_economy_smoke.py b/tests/unit/test_candidate_economy_smoke.py new file mode 100644 index 000000000..e4cd8637f --- /dev/null +++ b/tests/unit/test_candidate_economy_smoke.py @@ -0,0 +1,112 @@ +"""The deployment smoke rejects canonical results without matching receipts.""" + +from copy import deepcopy +from types import SimpleNamespace +from unittest.mock import Mock +from uuid import UUID + +import pytest + +from tests.integration.test_cloud_run_candidate import ( + test_cloud_run_candidate_current_law_economy as run_smoke, +) + + +@pytest.fixture +def smoke(): + defaults = { + "forecast_content_sha256": "a" * 64, + "scenario": "baseline", + "geography_kind": "county", + } + metadata = { + "current_law_id": 2, + "economy_options": {"time_period": [{"name": "2025"}]}, + "spm": {"available": True, "defaults": defaults}, + } + receipt = { + "forecast_sha256": "a" * 64, + "scenario": "baseline", + "geography_kind": "county", + "years": {"2025": {}}, + } + result = { + "budget": {"budgetary_impact": 0}, + "spm_config": defaults, + "spm_provenance": { + "baseline": [deepcopy(receipt)], + "reform": [deepcopy(receipt)], + }, + } + client = SimpleNamespace( + get=Mock( + return_value=SimpleNamespace( + raise_for_status=lambda: None, json=lambda: {"result": metadata} + ) + ) + ) + poll = Mock(return_value={"status": "ok", "result": result}) + return client, poll, metadata, result + + +def test_current_law_smoke_uses_no_policy_write(smoke): + client, poll, _, _ = smoke + run_smoke(client, poll) + client.get.assert_called_once_with("/us/metadata") + assert poll.call_args.args[1] == "/us/economy/2/over/2" + query = poll.call_args.args[2] + assert query["region"] == "ut" + assert query["time_period"] == "2025" + assert UUID(query["cache_nonce"]).version == 4 + + +def test_each_current_law_smoke_has_a_fresh_cache_identity(smoke): + client, poll, _, _ = smoke + run_smoke(client, poll) + run_smoke(client, poll) + assert ( + poll.call_args_list[0].args[2]["cache_nonce"] + != poll.call_args_list[1].args[2]["cache_nonce"] + ) + + +@pytest.mark.parametrize( + "error", ["missing", "hash", "year", "budget", "nonzero-budget"] +) +def test_current_law_smoke_rejects_incomplete_worker_evidence(smoke, error): + client, poll, _, result = smoke + if error == "missing": + result["spm_provenance"]["reform"] = [] + elif error == "hash": + result["spm_provenance"]["reform"][0]["forecast_sha256"] = "b" * 64 + elif error == "year": + result["spm_provenance"]["reform"][0]["years"] = {"2024": {}} + elif error == "nonzero-budget": + result["budget"]["budgetary_impact"] = 1 + else: + result["budget"]["budgetary_impact"] = float("nan") + with pytest.raises(AssertionError): + run_smoke(client, poll) + + +def test_current_law_smoke_accepts_legacy_without_receipts(smoke): + client, poll, metadata, result = smoke + metadata["spm"] = {"available": False} + del result["spm_provenance"] + run_smoke(client, poll) + + +def test_current_law_smoke_accepts_only_null_setting_omissions(smoke): + client, poll, metadata, result = smoke + metadata["spm"]["defaults"] = { + **metadata["spm"]["defaults"], + "as_of": None, + "geography_id": None, + } + run_smoke(client, poll) + result["spm_config"] = { + "forecast_content_sha256": "a" * 64, + "geography_kind": "county", + } + with pytest.raises(AssertionError): + run_smoke(client, poll) diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index c9ef51054..479567428 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -260,6 +260,7 @@ def _run_simulation_version_guard( versions_response: dict, *args: str, entrypoint: str = "old_gateway_direct", + extra_env: dict | None = None, ) -> subprocess.CompletedProcess[str]: versions_json = json.dumps(versions_response) command = ( @@ -274,6 +275,7 @@ def _run_simulation_version_guard( SIM_ENTRYPOINT=entrypoint, SIMULATION_ENTRYPOINT_URL="https://simulation.example.test", OLD_SIMULATION_GATEWAY_URL="https://old-gateway.example.test", + **(extra_env or {}), ), text=True, capture_output=True, @@ -482,6 +484,7 @@ def test_policyengine_bundle_support_check_passes_pyproject_pin_to_guard(tmp_pat assert capture_path.read_text(encoding="utf-8").splitlines() == [ "-py", current_version, + "--check-installed-spm", ] @@ -2125,3 +2128,96 @@ def test_push_workflow_never_uses_latest_or_tag_alias_for_cloud_run_traffic(): assert "--to-latest" not in workflow.lower() assert "--to-revisions" in promotion_script assert '"${target_revision}=100"' in promotion_script + + +def test_bundle_alignment_jobs_install_uv_before_capability_check(): + workflows = [ + (_push_workflow(), "ensure-staging-model-version-aligns-with-sim-api"), + (_push_workflow(), "ensure-production-model-version-aligns-with-sim-api"), + ( + (REPO / ".github/workflows/pr.yml").read_text(), + "ensure-policyengine-bundle-supported-by-simulation-api", + ), + ] + for workflow, name in workflows: + job = _workflow_job_block(workflow, name) + assert job.index("uses: astral-sh/setup-uv@v6") < job.index( + "bash .github/check-policyengine-bundle-supported.sh" + ) + + +@pytest.mark.parametrize("exit_code", [0, 23]) +def test_alignment_guard_preserves_installed_capability_check_result( + tmp_path, exit_code +): + capture = tmp_path / "registry.json" + args = tmp_path / "uv-args.txt" + uv = tmp_path / "uv" + uv.write_text( + '#!/bin/sh\nprintf "%s\\n" "$@" > "$CAPTURE_ARGS"\ncat > "$CAPTURE_REGISTRY"\nexit "$CHECK_EXIT"\n' + ) + uv.chmod(0o755) + registry = {"policyengine": {"6.0.0": "canonical-worker"}, "spm_capabilities": {}} + result = _run_simulation_version_guard( + registry, + "-py", + "6.0.0", + "--check-installed-spm", + extra_env={ + "PATH": f"{tmp_path}:{os.environ['PATH']}", + "CAPTURE_ARGS": str(args), + "CAPTURE_REGISTRY": str(capture), + "CHECK_EXIT": str(exit_code), + }, + ) + assert result.returncode == exit_code, result.stderr + assert json.loads(capture.read_text()) == registry + assert args.read_text().splitlines() == [ + "run", + "--frozen", + "python", + "-m", + "policyengine_api.worker_spm_release", + "6.0.0", + ] + assert ("SUCCESS:" in result.stdout) == (exit_code == 0) + + +@pytest.mark.parametrize("change", ["none", "calculator", "spm-source"]) +def test_bundle_alignment_skip_includes_calculator_and_spm_source(tmp_path, change): + base_project = tmp_path / "base-pyproject.toml" + project = (REPO / "pyproject.toml").read_text() + if change == "calculator": + project = re.sub(r"spm-calculator==[0-9.]+", "spm-calculator==0.0.0", project) + base_project.write_text(project) + git = tmp_path / "git" + git.write_text( + '#!/bin/sh\ncase "$1" in\nfetch) exit 0;;\nshow) cat "$BASE_PROJECT";;\ndiff) exit "$DIFF_STATUS";;\n*) exit 2;;\nesac\n' + ) + git.chmod(0o755) + capture = tmp_path / "guard-args.txt" + guard = tmp_path / "guard.sh" + guard.write_text('printf "%s\\n" "$@" > "$CAPTURE_ARGS"\n') + result = subprocess.run( + [ + "bash", + ".github/check-policyengine-bundle-supported.sh", + "--if-changed-from-base", + "master", + ], + cwd=REPO, + env=_script_env( + PATH=f"{tmp_path}:{os.environ['PATH']}", + BASE_PROJECT=str(base_project), + DIFF_STATUS="1" if change == "spm-source" else "0", + SIMULATION_VERSION_GUARD_SCRIPT=str(guard), + CAPTURE_ARGS=str(capture), + ), + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + assert capture.exists() == (change != "none") + if capture.exists(): + assert "--check-installed-spm" in capture.read_text().splitlines() diff --git a/tests/unit/test_openapi_spm.py b/tests/unit/test_openapi_spm.py index 807af4d0e..669521664 100644 --- a/tests/unit/test_openapi_spm.py +++ b/tests/unit/test_openapi_spm.py @@ -41,6 +41,12 @@ def test_economy_query_and_receipt_schemas_match_public_http_contract(): if suffix: assert parameters["window_size"]["schema"]["minimum"] == 1 assert parameters["window_size"]["schema"]["maximum"] == 75 + else: + nonce = parameters["cache_nonce"] + assert nonce["required"] is False + assert nonce["schema"]["type"] == "string" + assert nonce["schema"]["format"] == "uuid" + assert nonce["schema"]["default"] is None result = operation["responses"]["200"]["content"]["application/json"]["schema"][ "properties" ]["result"] diff --git a/tests/unit/test_worker_spm_release.py b/tests/unit/test_worker_spm_release.py new file mode 100644 index 000000000..37139585e --- /dev/null +++ b/tests/unit/test_worker_spm_release.py @@ -0,0 +1,88 @@ +"""Release checks use the same certified selection boundary as HTTP requests.""" + +from copy import deepcopy + +import pytest + +from policyengine_api import constants, worker_spm, worker_spm_release +from policyengine_api.spm import SPMValidationError + + +DEFAULTS = { + "forecast_content_sha256": "a" * 64, + "scenario": "baseline", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, +} + + +@pytest.fixture +def registry(monkeypatch): + monkeypatch.setattr(worker_spm_release, "version", lambda _: "6.0.0") + monkeypatch.setattr(constants, "POLICYENGINE_VERSION", "6.0.0") + monkeypatch.setattr(worker_spm, "normalize_spm_selection", lambda *_: DEFAULTS) + return { + "policyengine": {"6.0.0": "canonical-worker"}, + "spm_capabilities": { + "6.0.0": {"contract_version": "canonical-spm-v1", "defaults": DEFAULTS} + }, + } + + +def test_release_accepts_matching_certified_worker(registry): + assert worker_spm_release.validate_installed_worker(registry, "6.0.0") == DEFAULTS + + +@pytest.mark.parametrize( + "mutation", ["absent", "wrong-contract", "wrong-hash", "wrong-version"] +) +def test_release_rejects_unusable_registered_worker(registry, mutation): + registry = deepcopy(registry) + capability = registry["spm_capabilities"]["6.0.0"] + if mutation == "absent": + registry["spm_capabilities"] = {} + elif mutation == "wrong-contract": + capability["contract_version"] = "other-contract" + elif mutation == "wrong-hash": + capability["defaults"]["forecast_content_sha256"] = "b" * 64 + else: + registry["spm_capabilities"] = {"5.0.0": capability} + with pytest.raises(SPMValidationError, match=""): + worker_spm_release.validate_installed_worker(registry, "6.0.0") + + +def test_release_rejects_different_installed_bundle(registry): + with pytest.raises(ValueError, match="installed"): + worker_spm_release.validate_installed_worker(registry, "7.0.0") + + +@pytest.mark.parametrize("app", [None, "", {}, 123]) +def test_release_rejects_invalid_registered_application(registry, app): + registry["policyengine"]["6.0.0"] = app + with pytest.raises(ValueError, match="application"): + worker_spm_release.validate_installed_worker(registry, "6.0.0") + + +def test_release_accepts_legacy_without_capability(registry, monkeypatch): + monkeypatch.setattr(worker_spm, "normalize_spm_selection", lambda *_: None) + registry["spm_capabilities"] = {} + assert worker_spm_release.validate_installed_worker(registry, "6.0.0") is None + + +def test_release_does_not_hide_uncertified_bundle(registry, monkeypatch): + def unavailable(*_): + raise SPMValidationError( + "SPM_CONFIGURATION_UNAVAILABLE", "Missing certification" + ) + + monkeypatch.setattr(worker_spm, "normalize_spm_selection", unavailable) + with pytest.raises(SPMValidationError): + worker_spm_release.validate_installed_worker(registry, "6.0.0") + + +def test_release_rejects_different_runtime_selected_bundle(registry, monkeypatch): + monkeypatch.setattr(constants, "POLICYENGINE_VERSION", "5.0.0") + with pytest.raises(ValueError, match="runtime-selected"): + worker_spm_release.validate_installed_worker(registry, "6.0.0")