diff --git a/.github/scripts/deploy-cloud-run-simulation-entry.sh b/.github/scripts/deploy-cloud-run-simulation-entry.sh index 482788d87..cbe60559e 100755 --- a/.github/scripts/deploy-cloud-run-simulation-entry.sh +++ b/.github/scripts/deploy-cloud-run-simulation-entry.sh @@ -100,6 +100,8 @@ required_variables=( OLD_GATEWAY_AUTH_CLIENT_ID_VALUE OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME SIMULATION_SERVICE_DATABASE_URL_SECRET_NAME + MODAL_TOKEN_ID_SECRET_NAME + MODAL_TOKEN_SECRET_SECRET_NAME OBSERVABILITY_SERVICE_NAMESPACE OBSERVABILITY_TRACE_PROJECT_ID OTEL_EXPORTER_OTLP_ENDPOINT @@ -144,13 +146,17 @@ add_runtime_secret OLD_GATEWAY_AUTH_CLIENT_SECRET \ # API's tables, on every deploy whatever the Stage 12 setting. add_runtime_secret SIMULATION_SERVICE_DATABASE_URL \ "${SIMULATION_SERVICE_DATABASE_URL_SECRET_NAME}" +# The Modal token the household routes call the v2 workers' household +# functions with, on every deploy whatever the Stage 12 setting. The Modal +# client reads it from MODAL_TOKEN_ID and MODAL_TOKEN_SECRET, so the Stage 12 +# comparison path, in the same process, uses this mount too. +add_runtime_secret MODAL_TOKEN_ID "${MODAL_TOKEN_ID_SECRET_NAME}" +add_runtime_secret MODAL_TOKEN_SECRET "${MODAL_TOKEN_SECRET_SECRET_NAME}" if [[ "${DEPLOY_STAGE12_V2}" == "true" ]]; then stage12_variables=( STAGE12_ENABLED_VALUE STAGE12_ARTIFACT_BUCKET_VALUE - MODAL_TOKEN_ID_SECRET_NAME - MODAL_TOKEN_SECRET_SECRET_NAME STAGE12_DATABASE_URL_SECRET_NAME ) for variable_name in "${stage12_variables[@]}"; do @@ -162,8 +168,6 @@ if [[ "${DEPLOY_STAGE12_V2}" == "true" ]]; then || fail "STAGE12_ARTIFACT_BUCKET_VALUE is invalid" validate_secret_reference "${STAGE12_DATABASE_URL_SECRET_NAME}" - add_stage12_runtime_secret MODAL_TOKEN_ID "${MODAL_TOKEN_ID_SECRET_NAME}" - add_stage12_runtime_secret MODAL_TOKEN_SECRET "${MODAL_TOKEN_SECRET_SECRET_NAME}" add_stage12_runtime_secret STAGE12_DATABASE_URL "${STAGE12_DATABASE_URL_SECRET_NAME}" fi @@ -189,6 +193,8 @@ jq -n ' OTEL_METRICS_EXPORTER: "otlp", OTEL_TRACES_SAMPLER_ARG: "1.0", POLICYENGINE_OTEL_GOOGLE_AUDIENCE: env.POLICYENGINE_OTEL_GOOGLE_AUDIENCE, + SIMULATION_V2_MODAL_ENVIRONMENT: env.MODAL_ENVIRONMENT, + SIMULATION_V2_MANIFEST_NAME: "simulation-api-v2-version-manifest", STAGE12_ENABLED: ( if env.DEPLOY_STAGE12_V2 == "true" then env.STAGE12_ENABLED_VALUE else "0" end ) diff --git a/.github/scripts/simulation-staging-household-check.sh b/.github/scripts/simulation-staging-household-check.sh new file mode 100755 index 000000000..f5632c4e2 --- /dev/null +++ b/.github/scripts/simulation-staging-household-check.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# Calculate the fixed staging household through a deployed Simulation +# Entrypoint, and require its household job to succeed. +# Usage: simulation-staging-household-check.sh +# Required env vars: STAGING_HOUSEHOLD_ID (the ID of a US household stored in +# the staging Public API, from the beta variable +# SIMULATION_STAGING_HOUSEHOLD_ID), and the test client's +# SIMULATION_TEST_AUTH_ISSUER, SIMULATION_TEST_AUTH_AUDIENCE, +# SIMULATION_TEST_AUTH_CLIENT_ID and SIMULATION_TEST_AUTH_CLIENT_SECRET. +# Optional: HOUSEHOLD_CHECK_DEADLINE_SECONDS (600) and +# HOUSEHOLD_CHECK_POLL_SECONDS (5). +# The access token is sent from a file and the client secret on standard +# input, so neither appears on a command line. + +set -euo pipefail + +fail() { + printf 'Staging household check failed: %s\n' "$1" >&2 + exit 1 +} + +base_url="${1:?Simulation Entrypoint base URL is required}" +base_url="${base_url%/}" +household_id="${STAGING_HOUSEHOLD_ID:-}" +deadline_seconds="${HOUSEHOLD_CHECK_DEADLINE_SECONDS:-600}" +poll_seconds="${HOUSEHOLD_CHECK_POLL_SECONDS:-5}" +uuid_pattern='^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + +[[ -n "${household_id}" ]] \ + || fail "STAGING_HOUSEHOLD_ID is empty: set the beta environment variable SIMULATION_STAGING_HOUSEHOLD_ID to the ID of a US household stored in the staging Public API" +[[ "${household_id}" =~ ${uuid_pattern} ]] \ + || fail "STAGING_HOUSEHOLD_ID is not a UUID in lowercase hexadecimal" +for variable_name in SIMULATION_TEST_AUTH_ISSUER SIMULATION_TEST_AUTH_AUDIENCE \ + SIMULATION_TEST_AUTH_CLIENT_ID SIMULATION_TEST_AUTH_CLIENT_SECRET; do + [[ -n "${!variable_name:-}" ]] || fail "${variable_name} must be set" +done +[[ "${deadline_seconds}" =~ ^[0-9]+$ ]] \ + || fail "HOUSEHOLD_CHECK_DEADLINE_SECONDS must be a whole number of seconds" +[[ "${poll_seconds}" =~ ^[0-9]+$ ]] \ + || fail "HOUSEHOLD_CHECK_POLL_SECONDS must be a whole number of seconds" + +work_directory="$(mktemp -d)" +trap 'rm -rf "${work_directory}"' EXIT +chmod 700 "${work_directory}" + +# Send one request; write its body to .json and print its HTTP status. +send() { + local name="$1" + shift + curl --silent --show-error --output "${work_directory}/${name}.json" \ + --write-out '%{http_code}' "$@" +} + +token_status="$( + jq -n \ + --arg client_id "${SIMULATION_TEST_AUTH_CLIENT_ID}" \ + --arg client_secret "${SIMULATION_TEST_AUTH_CLIENT_SECRET}" \ + --arg audience "${SIMULATION_TEST_AUTH_AUDIENCE}" \ + '{client_id: $client_id, client_secret: $client_secret, audience: $audience, grant_type: "client_credentials"}' \ + | send token --request POST --header 'Content-Type: application/json' \ + --data @- "${SIMULATION_TEST_AUTH_ISSUER%/}/oauth/token" +)" +[[ "${token_status}" == "200" ]] || fail "the token request answered ${token_status}" +access_token="$(jq -r '.access_token // empty' "${work_directory}/token.json")" +[[ -n "${access_token}" ]] || fail "the token response has no access_token" +authorization_header="${work_directory}/authorization-header" +printf 'Authorization: Bearer %s\n' "${access_token}" > "${authorization_header}" + +submission_status="$( + jq -n --arg household_id "${household_id}" \ + '{country: "us", household_id: $household_id}' \ + | send submission --request POST --header "@${authorization_header}" \ + --header 'Content-Type: application/json' --data @- \ + "${base_url}/household-calculations" +)" +[[ "${submission_status}" == "202" ]] \ + || fail "POST /household-calculations answered ${submission_status}: $(cat "${work_directory}/submission.json")" +job_id="$(jq -r '.job_id // empty' "${work_directory}/submission.json")" +[[ "${job_id}" =~ ${uuid_pattern} ]] || fail "the submission named no household job" +printf 'Submitted household job %s for staging household %s\n' "${job_id}" "${household_id}" + +started_at="${SECONDS}" +while true; do + job_status_code="$( + send job --header "@${authorization_header}" \ + "${base_url}/household-calculations/${job_id}" + )" + job_body="${work_directory}/job.json" + if [[ "${job_status_code}" == "200" ]]; then + job_status="$(jq -r '.status' "${job_body}")" + case "${job_status}" in + succeeded) + [[ "$(jq -r '.result.status // empty' "${job_body}")" == "succeeded" ]] \ + || fail "household job ${job_id} succeeded without a succeeded result document" + printf 'Household job %s succeeded on .py %s with %s\n' "${job_id}" \ + "$(jq -r '.policyengine_version' "${job_body}")" \ + "$(jq -r '.country_package_version' "${job_body}")" + exit 0 + ;; + failed) + fail "household job ${job_id} failed: $(jq -r '.error_message // "no error message"' "${job_body}")" + ;; + pending | running) ;; + *) fail "household job ${job_id} has an unknown status: ${job_status}" ;; + esac + elif [[ "${job_status_code}" != "503" ]]; then + fail "GET /household-calculations/${job_id} answered ${job_status_code}: $(cat "${job_body}")" + fi + ((SECONDS - started_at < deadline_seconds)) \ + || fail "household job ${job_id} did not finish within ${deadline_seconds} seconds" + sleep "${poll_seconds}" +done diff --git a/.github/scripts/simulation-staging-household-ready.sh b/.github/scripts/simulation-staging-household-ready.sh new file mode 100755 index 000000000..a691a4994 --- /dev/null +++ b/.github/scripts/simulation-staging-household-ready.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Require the staging household check to have succeeded in a staging +# deployment, and to have been skipped in any other. +# Required env vars: REQUESTED ("true" in a staging deployment) and +# CHECK_RESULT (the staging_household_check job result). + +set -euo pipefail + +if [[ "${REQUESTED}" == "true" ]]; then + test "${CHECK_RESULT}" = "success" +else + test "${CHECK_RESULT}" = "skipped" +fi diff --git a/.github/workflows/simulation-deploy.reusable.yml b/.github/workflows/simulation-deploy.reusable.yml index 7c735c746..187e708ee 100644 --- a/.github/workflows/simulation-deploy.reusable.yml +++ b/.github/workflows/simulation-deploy.reusable.yml @@ -1215,6 +1215,41 @@ jobs: MODAL_ENVIRONMENT: ${{ inputs.modal_environment }} run: ../../.github/scripts/stage12-v2-default-restore.sh + staging_household_check: + name: Calculate the staging household through the candidate + # Every staging deployment calculates the stored staging household that + # the beta variable SIMULATION_STAGING_HOUSEHOLD_ID names, through the + # candidate's household routes, and requires its household job to + # succeed. The routes use only the v2 manifest's default worker, so the + # check runs after the default has moved to the release's worker (or, in + # a deployment without Stage 12, with the default as it is). It calls the + # tagged candidate URL, which serves this release's revision whether or + # not it was promoted. The job's call writes a household_jobs row in the + # staging database, which the household function completes. + if: >- + ${{ + !cancelled() && + inputs.deployment_environment == 'staging' && + needs.deploy_entrypoint.result == 'success' && + needs.authenticated_test.result == 'success' && + ((inputs.deploy_stage12_v2 && needs.publish_stage12_v2_default.result == 'success') || + (!inputs.deploy_stage12_v2 && needs.publish_stage12_v2_default.result == 'skipped')) + }} + needs: [deploy_entrypoint, authenticated_test, publish_stage12_v2_default] + runs-on: ubuntu-latest + environment: ${{ inputs.release_environment }} + steps: + - uses: actions/checkout@v6 + + - name: Calculate the staging household and require it to succeed + env: + STAGING_HOUSEHOLD_ID: ${{ vars.SIMULATION_STAGING_HOUSEHOLD_ID }} + SIMULATION_TEST_AUTH_ISSUER: ${{ secrets.SIMULATION_ENTRYPOINT_AUTH_ISSUER }} + SIMULATION_TEST_AUTH_AUDIENCE: ${{ secrets.SIMULATION_ENTRYPOINT_AUTH_AUDIENCE }} + SIMULATION_TEST_AUTH_CLIENT_ID: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_ID }} + SIMULATION_TEST_AUTH_CLIENT_SECRET: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_SECRET }} + run: .github/scripts/simulation-staging-household-check.sh "${{ needs.deploy_entrypoint.outputs.candidate_url }}" + finalize_routing: name: Finalize the Modal routing publication # Finish the durable journal after every gate, including failed uploads @@ -1273,7 +1308,7 @@ jobs: # The reporter only reads job results. Journal completion happens before # it, so a failure or cancellation here cannot strand a publication. if: ${{ !cancelled() }} - needs: [deploy_entrypoint, update_routing, integration, authenticated_test, promote_entrypoint, publish_latest, finalize_routing, publish_stage12_v2_default] + needs: [deploy_entrypoint, update_routing, integration, authenticated_test, promote_entrypoint, publish_latest, finalize_routing, publish_stage12_v2_default, staging_household_check] runs-on: ubuntu-latest defaults: run: @@ -1302,6 +1337,12 @@ jobs: DEFAULT_RESULT: ${{ needs.publish_stage12_v2_default.result }} run: .github/scripts/stage12-v2-default-ready.sh + - name: Check the staging household check result + env: + REQUESTED: ${{ inputs.deployment_environment == 'staging' }} + CHECK_RESULT: ${{ needs.staging_household_check.result }} + run: .github/scripts/simulation-staging-household-ready.sh + cleanup_entrypoint_revisions: name: Remove superseded Cloud Run revisions if: ${{ !cancelled() && inputs.promote_entrypoint && needs.deployment_ready.result == 'success' }} diff --git a/changelog.d/c1-2-household-calculations.added.md b/changelog.d/c1-2-household-calculations.added.md new file mode 100644 index 000000000..bd3d7ddb1 --- /dev/null +++ b/changelog.d/c1-2-household-calculations.added.md @@ -0,0 +1 @@ +Added `POST /household-calculations` and `GET /household-calculations/{household_job_id}` to the Simulation Entrypoint. A submission names a household, and optionally a policy, stored in the Public API, with optional axes and SPM settings; the service reads them, creates a `household_jobs` row for the household's default year on the v2 version manifest's default worker, and starts its household function's call, answering 202 with the job's `job_id` and status, or 400, 422, 500 or 503 with the reason. A 202 `pending` may also be a job whose call Modal didn't confirm and whose state couldn't then be read: poll it rather than submitting again. A read returns the job's status, result document and error message. A stored policy value with no end date applies to the end of 2200. Every deploy now gives the entrypoint the Modal token and the v2 manifest's Modal environment, and every staging deploy calculates the household that the `beta` variable `SIMULATION_STAGING_HOUSEHOLD_ID` names. diff --git a/docs/engineering/skills/database-access.md b/docs/engineering/skills/database-access.md index 61aa07498..c559f59c7 100644 --- a/docs/engineering/skills/database-access.md +++ b/docs/engineering/skills/database-access.md @@ -40,6 +40,11 @@ and this repository has no migration chain of its own `household_jobs`), modelled in `libs/policyengine-simulation-persistence`. The service connects as its own role, which policyengine-api's privilege manifest grants. +- the policyengine-api tables a calculation is read from (`households`, + `policies`, `parameters`, `parameter_values`, `tax_benefit_models` and + `tax_benefit_model_versions`), which the simulation service only reads. + Their read-only models in that library's `input_tables.py` name only the + columns the service reads, apart from the models of the tables it writes. - the run tables the v2 Modal coordinator, workers, household functions and stuck-run sweep write as the Modal worker role: `report_runs`, `simulation_runs`, the result tables and `household_jobs`. Their mappings in diff --git a/docs/generated/rules.md b/docs/generated/rules.md index e72dad1c2..0b43a4a3f 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -143,7 +143,7 @@ flowchart LR | Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | | --- | --- | --- | --- | --- | --- | --- | --- | | `sim.persistence.missing-privileges-reported` | The simulation service's privilege check, which its readiness check uses, names every table privilege and every column UPDATE privilege its role needs on the Public API's tables and lacks, as PostgreSQL reports it, counts a table missing from the public schema as lacking all of them and a missing column as lacking its own, and names a missing USAGE privilege on that schema. | mechanical | enforced | yes | pytest `libs/policyengine-simulation-persistence/tests/test_access.py` | `pr.yml:simulation-database` | yes | -| `sim.persistence.models-match-public-api` | Sim's models of the Public API's tables match the tables the Public API's v2 migration chain builds at the pinned Public API commit, which is on the Public API's default branch: every column's type, nullability and default, every constraint and index by name and definition, and every enum's values; and the simulation service's role, with the privileges the Public API's manifest declares, writes every modelled table. | mechanical | enforced | yes | pytest `libs/policyengine-simulation-persistence/tests/test_public_api_schema.py`, `libs/policyengine-simulation-persistence/tests/test_service_writes.py` | `pr.yml:public-api-schema`, `public-api-schema-drift.yml:public-api-schema-drift` | yes | +| `sim.persistence.models-match-public-api` | Sim's models of the Public API's tables match the tables the Public API's v2 migration chain builds at the pinned Public API commit, which is on the Public API's default branch: every column's type, nullability and default, every constraint and index by name and definition, and every enum's values; every column its read-only models of the Public API's input tables name exists there with the same type and nullability; and the simulation service's role, with the privileges the Public API's manifest declares, writes every modelled table. | mechanical | enforced | yes | pytest `libs/policyengine-simulation-persistence/tests/test_public_api_schema.py`, `libs/policyengine-simulation-persistence/tests/test_service_writes.py` | `pr.yml:public-api-schema`, `public-api-schema-drift.yml:public-api-schema-drift` | yes | ```mermaid flowchart LR diff --git a/libs/policyengine-simulation-persistence/README.md b/libs/policyengine-simulation-persistence/README.md index 2520772cf..a049a0575 100644 --- a/libs/policyengine-simulation-persistence/README.md +++ b/libs/policyengine-simulation-persistence/README.md @@ -47,6 +47,73 @@ household report with only a baseline). A report with a `budget_window_size` is calculated as a `budget_window_runs` row with one report run per year; each year's report run names that row and its year, and belongs to the same report. +## Input tables + +`input_tables.py` holds read-only models of the Public API's tables a +calculation is read from: `households`, `policies`, `parameters`, +`parameter_values`, `tax_benefit_models` and `tax_benefit_model_versions`, in +`input_metadata`. The service only reads them, with `SELECT`. Each names only +the columns the service reads, so they are not models of the whole tables, +and they are kept apart from the models of the tables the service writes +(`service_metadata`) and from the tables those name by `id` alone +(`referenced_metadata`). + +## Household jobs + +`household_job_records.py` holds the simulation service's reads and writes +for a household calculation, one `household_jobs` row. The service finds the +catalog version by country and .py version (`read_catalog_version_id`) and +the stored household in its country (`read_stored_household`), creates the +job `pending` with no Modal call ID and no dynamic (`insert_household_job`), +starts the household function's call, then records the call's ID +(`record_household_call`), or fails the job at once when Modal refuses the +call (`fail_unstarted_household_job`). Both updates apply only while the job +is still `pending` with no call ID, and set `updated_at` themselves. A job +naming a household, policy or catalog version that does not exist is refused +by its foreign key with SQLSTATE `23503`; `HouseholdJobReferenceError` names +the column. Each function takes a connection, so the caller chooses the +transactions. `HouseholdJobStore` runs them as the Simulation Entrypoint +does, each step in a transaction of its own: `read_inputs` reads the catalog +version, the household and the policy's reform values together, and +`create_job`, `record_call`, `fail_unstarted` and `read` each write or read +the job. `HouseholdJobStore.from_engine` opens each with `bounded_transaction`, +which commits when its block ends and sets `statement_timeout` with +`set_config`, local to the transaction, so the database ends a statement that +runs too long and the limit never stays on a pooled connection. + +## Reading a policy + +`read_policy_reform` (`policy_reform.py`) reads a stored policy as the reform +values a calculation applies, `{parameter name: {period: value}}`: the shape +of the Public API's policy JSON, which the household functions' contract and +the economy requests take. The Public API stores a policy's values against +the parameters of the catalog version it was saved on, so a calculation on +any catalog version applies them by name; a parameter the calculation's +version lacks raises `MissingPolicyParametersError`, naming each one, and a +policy the country doesn't have raises `UnknownPolicyError`. + +Each period is a range, `YYYY-MM-DD.YYYY-MM-DD`, with the stored dates read +in UTC. A value with no end date follows .py's own rule +(`reform_dict_from_parameter_values`): it stops the day before the same +parameter's next later value starts, so once a later bounded value ends +current law applies, and otherwise runs to 31 December 2200, the latest year +a household job or simulation run can be for, where .py stops at 2100. A +start date alone would not do for that last value: policyengine-core reads it +as "from that day on", but policyengine-uk's reform dictionary reads it as +that one day. The executor's +`test_reform_periods_apply_in_2200_only_through_a_stop_in_2200` shows how +both countries' household functions apply each form in 2200, and its +`test_py_reform_dicts.py` pins .py's own periods for the cases the reader's +tests check. The reader gives .py's periods, in .py's order, for the values +in start order, with one difference besides 2200: it leaves out an +open-ended value whose range would end before it starts (one followed the +same day by a later value, or one starting after 2200). .py gives such a +value a range that ends before it starts too. policyengine-core applies that +range on no day, but policyengine-uk refuses it (its `interval_bounds` +raises "ends before it starts"), so a UK calculation given .py's dict fails +where one given the reader's runs. Bounded values starting the same day are +all kept, in start order, as .py keeps them. + ## Checking the models against the Public API Rule `sim.persistence.models-match-public-api`. The PR workflow's @@ -60,8 +127,12 @@ tables their keys name (`referenced_metadata`, by `id` alone), and compares the models' tables with the Public API's through PostgreSQL's catalog (`tests/schema_comparison.py`): every column's type, nullability and default, every constraint and index by name and definition, and every enum's values -must match. `test_service_writes` creates a row in every modelled table as -the service's role. Another test, outside the rule, requires the modelled +must match. `test_the_input_models_match_the_public_api_columns` builds +the read-only input models' tables in another disposable database and +requires every column they name to exist in the Public API's table with the +same type and nullability; their other columns, defaults, constraints and +indexes are the Public API's alone. `test_service_writes` creates a row in every modelled +table as the service's role. Another test, outside the rule, requires the modelled tables' foreign keys to the Public API's users, policies, dynamics, households, datasets, regions and model catalog tables to be exactly the 20 in `tests/public_api_keys.py`, each by its own table, its name and the table diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py index 02d93db83..11f4713b4 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py @@ -7,9 +7,35 @@ read_missing_table_privileges, ) from policyengine_simulation_persistence.engine import ( + bounded_transaction, create_simulation_engine, normalized_database_url, ) +from policyengine_simulation_persistence.household_job_records import ( + HouseholdJobRecord, + HouseholdJobReferenceError, + NewHouseholdJob, + StoredHousehold, + fail_unstarted_household_job, + insert_household_job, + read_catalog_version_id, + read_household_job, + read_stored_household, + record_household_call, +) +from policyengine_simulation_persistence.household_job_store import ( + HouseholdCalculationInputs, + HouseholdJobStore, + UnknownCatalogVersionError, + UnknownHouseholdError, +) +from policyengine_simulation_persistence.input_tables import input_metadata +from policyengine_simulation_persistence.policy_reform import ( + MissingPolicyParametersError, + PolicyReform, + UnknownPolicyError, + read_policy_reform, +) from policyengine_simulation_persistence.tables import ( HouseholdJobStatus, ReportRunStatus, @@ -29,17 +55,37 @@ __all__ = [ "SERVICE_TABLE_PRIVILEGES", "SERVICE_UPDATABLE_COLUMNS", + "HouseholdCalculationInputs", + "HouseholdJobRecord", + "HouseholdJobReferenceError", "HouseholdJobStatus", + "HouseholdJobStore", + "MissingPolicyParametersError", + "NewHouseholdJob", + "PolicyReform", "ReportRunStatus", "ReportRunTrigger", "SimulationStatus", "SimulationType", + "StoredHousehold", + "UnknownCatalogVersionError", + "UnknownHouseholdError", + "UnknownPolicyError", + "bounded_transaction", "budget_window_runs", "create_simulation_engine", + "fail_unstarted_household_job", "household_jobs", + "input_metadata", + "insert_household_job", "missing_table_privileges", "normalized_database_url", + "read_catalog_version_id", + "read_household_job", "read_missing_table_privileges", + "read_policy_reform", + "read_stored_household", + "record_household_call", "referenced_metadata", "report_runs", "reports", diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/engine.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/engine.py index 6d5bd1b15..b97fb9af1 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/engine.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/engine.py @@ -2,14 +2,17 @@ from __future__ import annotations +from contextlib import contextmanager from typing import TYPE_CHECKING -from sqlalchemy import create_engine +from sqlalchemy import create_engine, func, select from sqlalchemy.engine import URL, make_url from sqlalchemy.exc import ArgumentError if TYPE_CHECKING: - from sqlalchemy import Engine + from collections.abc import Iterator + + from sqlalchemy import Connection, Engine CONNECT_TIMEOUT_SECONDS = 5 # Each Cloud Run instance serves up to 80 concurrent requests. Each database @@ -58,3 +61,28 @@ def create_simulation_engine(database_url: str) -> Engine: "prepare_threshold": None, }, ) + + +@contextmanager +def bounded_transaction( + engine: Engine, *, statement_timeout_milliseconds: int +) -> Iterator[Connection]: + """Yield a connection in a transaction whose statements the database bounds. + + The transaction commits when the block ends and rolls back when it + raises. ``statement_timeout`` is set with ``set_config`` local to the + transaction, so it works through Supabase's transaction pooler and never + stays on the pooled connection; a statement that runs longer raises the + database's cancellation as an OperationalError. + """ + with engine.begin() as connection: + connection.execute( + select( + func.set_config( + "statement_timeout", + str(statement_timeout_milliseconds), + True, # noqa: FBT003 # set_config's is_local argument + ) + ) + ) + yield connection diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/household_job_records.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/household_job_records.py new file mode 100644 index 000000000..168f317aa --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/household_job_records.py @@ -0,0 +1,295 @@ +"""The simulation service's reads and writes for a household calculation. + +A household calculation is one ``household_jobs`` row, which the service +creates and a household function on Modal calculates. The service reads the +stored household and the catalog version from the Public API's tables, creates +the job ``pending`` with no Modal call ID, starts the call, then records the +call's ID, or fails the job at once when Modal refuses the call. Both updates +apply only while the job is still ``pending`` with no call ID: the household +function may already have claimed it, and the stuck-run sweep fails a pending +job no call was recorded for. The household function writes the rest: +``running``, then the result and ``succeeded`` or ``failed``. + +Every function takes a connection, so the caller chooses the transactions. +Each update sets only the columns the service's role may update, with +``updated_at``, whose automatic update the Public API's models make on their +own side only. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from sqlalchemy import func, insert, select, update +from sqlalchemy.exc import IntegrityError + +from policyengine_simulation_persistence.input_tables import ( + MODEL_NAME_PREFIX, + households, + tax_benefit_model_versions, + tax_benefit_models, +) +from policyengine_simulation_persistence.tables import ( + HouseholdJobStatus, + household_jobs, +) + +if TYPE_CHECKING: + from datetime import datetime + from uuid import UUID + + from sqlalchemy import ColumnElement, Connection + +# PostgreSQL's SQLSTATE for a row naming one that does not exist. +FOREIGN_KEY_VIOLATION = "23503" +# Each of the job's foreign keys, by name, and the column it constrains. +_KEY_COLUMNS = { + str(constraint.name): constraint.column_keys[0] + for constraint in household_jobs.foreign_key_constraints +} + + +@dataclass(frozen=True) +class StoredHousehold: + """A household as the Public API stores it. + + ``document`` lists the people and each group entity as records with an + ``id`` and ``values``, each person naming its groups, and may hold a saved + SPM selection (``spm``). ``default_year`` is the year it describes, or + None for a household saved without one. + """ + + id: UUID + country: str + default_year: int | None + document: dict[str, Any] + + +@dataclass(frozen=True) +class NewHouseholdJob: + """A household job to create, ``pending`` and with no Modal call yet. + + ``request_data`` holds the request as the service received it. + """ + + id: UUID + country: str + household_id: UUID + policy_id: UUID | None + year: int + request_data: dict[str, Any] + tax_benefit_model_version_id: UUID + country_package_version: str + policyengine_version: str + + +@dataclass(frozen=True) +class HouseholdJobRecord: + """A household job as it stands. + + ``result`` is the household function's outcome document, set once it + has finished: the calculated household, or the failure with its reason. + ``error_message`` summarizes a failure in one line. + """ + + id: UUID + status: HouseholdJobStatus + country: str + household_id: UUID + policy_id: UUID | None + year: int + request_data: dict[str, Any] + tax_benefit_model_version_id: UUID + country_package_version: str + policyengine_version: str + modal_call_id: str | None + result: dict[str, Any] | None + error_message: str | None + created_at: datetime + started_at: datetime | None + completed_at: datetime | None + + +class HouseholdJobReferenceError(LookupError): + """The database refused a job naming a row that does not exist. + + ``column_name`` is the job's column whose foreign key refused it, or None + when the database named a key the job's table doesn't have. + """ + + def __init__(self, column_name: str | None) -> None: + """Keep the column whose key refused the job.""" + self.column_name = column_name + super().__init__( + f"the household job's {column_name or 'reference'} names no row" + ) + + +def read_catalog_version_id( + connection: Connection, *, country: str, policyengine_version: str +) -> UUID | None: + """Return the id of the country's catalog version for a .py version, or None.""" + return connection.execute( + select(tax_benefit_model_versions.c.id) + .join( + tax_benefit_models, + tax_benefit_models.c.id == tax_benefit_model_versions.c.model_id, + ) + .where( + tax_benefit_models.c.name == f"{MODEL_NAME_PREFIX}{country}", + tax_benefit_model_versions.c.version == policyengine_version, + ) + ).scalar_one_or_none() + + +def read_stored_household( + connection: Connection, *, country: str, household_id: UUID +) -> StoredHousehold | None: + """Return the country's stored household, or None when it has no such one.""" + row = connection.execute( + select(households.c.default_year, households.c.household_data).where( + households.c.id == household_id, households.c.country_id == country + ) + ).one_or_none() + if row is None: + return None + return StoredHousehold( + id=household_id, + country=country, + default_year=row.default_year, + document=row.household_data, + ) + + +def insert_household_job(connection: Connection, job: NewHouseholdJob) -> None: + """Create the job ``pending``, with no Modal call ID and no dynamic. + + Raise HouseholdJobReferenceError when the database refuses a row it + names. The refusal ends the caller's transaction, or its savepoint. + """ + try: + connection.execute( + insert(household_jobs).values( + id=job.id, + country=job.country, + household_id=job.household_id, + policy_id=job.policy_id, + dynamic_id=None, + year=job.year, + request_data=job.request_data, + tax_benefit_model_version_id=job.tax_benefit_model_version_id, + country_package_version=job.country_package_version, + policyengine_version=job.policyengine_version, + status=HouseholdJobStatus.PENDING, + modal_call_id=None, + ) + ) + except IntegrityError as error: + driver_error = error.orig + if getattr(driver_error, "sqlstate", None) != FOREIGN_KEY_VIOLATION: + raise + constraint_name = getattr( + getattr(driver_error, "diag", None), "constraint_name", None + ) + raise HouseholdJobReferenceError( + _KEY_COLUMNS.get(str(constraint_name)) + ) from None + + +def _pending_without_call(job_id: UUID) -> tuple[ColumnElement[bool], ...]: + return ( + household_jobs.c.id == job_id, + household_jobs.c.status == HouseholdJobStatus.PENDING, + household_jobs.c.modal_call_id.is_(None), + ) + + +def record_household_call( + connection: Connection, *, job_id: UUID, modal_call_id: str +) -> HouseholdJobStatus | None: + """Record the job's Modal call ID while it is pending with none. + + Return the job's status afterwards, or None when there is no such job. A + job the household function has claimed already keeps its claim. + """ + recorded = connection.execute( + update(household_jobs) + .where(*_pending_without_call(job_id)) + .values(modal_call_id=modal_call_id, updated_at=func.now()) + .returning(household_jobs.c.status) + ).scalar_one_or_none() + if recorded is not None: + return HouseholdJobStatus(recorded) + status = connection.execute( + select(household_jobs.c.status).where(household_jobs.c.id == job_id) + ).scalar_one_or_none() + return None if status is None else HouseholdJobStatus(status) + + +def fail_unstarted_household_job( + connection: Connection, *, job_id: UUID, error_message: str +) -> bool: + """Fail the job at once while it is pending with no Modal call ID. + + Return whether it was failed: a job a call has been recorded for, or one + the household function has claimed, is left as it is. + """ + failed = connection.execute( + update(household_jobs) + .where(*_pending_without_call(job_id)) + .values( + status=HouseholdJobStatus.FAILED, + error_message=error_message, + completed_at=func.now(), + updated_at=func.now(), + ) + .returning(household_jobs.c.id) + ).scalar_one_or_none() + return failed is not None + + +def read_household_job( + connection: Connection, job_id: UUID +) -> HouseholdJobRecord | None: + """Return the household job as it stands, or None when there is no such job.""" + row = connection.execute( + select( + household_jobs.c.id, + household_jobs.c.status, + household_jobs.c.country, + household_jobs.c.household_id, + household_jobs.c.policy_id, + household_jobs.c.year, + household_jobs.c.request_data, + household_jobs.c.tax_benefit_model_version_id, + household_jobs.c.country_package_version, + household_jobs.c.policyengine_version, + household_jobs.c.modal_call_id, + household_jobs.c.result, + household_jobs.c.error_message, + household_jobs.c.created_at, + household_jobs.c.started_at, + household_jobs.c.completed_at, + ).where(household_jobs.c.id == job_id) + ).one_or_none() + if row is None: + return None + return HouseholdJobRecord( + id=row.id, + status=HouseholdJobStatus(row.status), + country=row.country, + household_id=row.household_id, + policy_id=row.policy_id, + year=row.year, + request_data=row.request_data, + tax_benefit_model_version_id=row.tax_benefit_model_version_id, + country_package_version=row.country_package_version, + policyengine_version=row.policyengine_version, + modal_call_id=row.modal_call_id, + result=row.result, + error_message=row.error_message, + created_at=row.created_at, + started_at=row.started_at, + completed_at=row.completed_at, + ) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/household_job_store.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/household_job_store.py new file mode 100644 index 000000000..ab3c1a09c --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/household_job_store.py @@ -0,0 +1,150 @@ +"""The household calculation store: each read and write in a transaction of its own. + +The Simulation Entrypoint creates a household job in three steps, each its +own transaction, with the Modal call started between them: it reads the +inputs, creates the job, then records the call or fails the job. Later reads +return the job as it stands. ``HouseholdJobStore`` runs the reads and writes +of ``household_job_records`` and ``policy_reform`` that way. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from policyengine_simulation_persistence.engine import bounded_transaction +from policyengine_simulation_persistence.household_job_records import ( + fail_unstarted_household_job, + insert_household_job, + read_catalog_version_id, + read_household_job, + read_stored_household, + record_household_call, +) +from policyengine_simulation_persistence.policy_reform import read_policy_reform + +if TYPE_CHECKING: + from collections.abc import Callable + from contextlib import AbstractContextManager + from uuid import UUID + + from sqlalchemy import Connection, Engine + + from policyengine_simulation_persistence.household_job_records import ( + HouseholdJobRecord, + NewHouseholdJob, + StoredHousehold, + ) + from policyengine_simulation_persistence.policy_reform import PolicyReform + from policyengine_simulation_persistence.tables import HouseholdJobStatus + +type TransactionFactory = Callable[[], AbstractContextManager[Connection]] + + +class UnknownCatalogVersionError(LookupError): + """The country has no catalog version for the .py version.""" + + +class UnknownHouseholdError(LookupError): + """The Public API has no such household in the country.""" + + +@dataclass(frozen=True) +class HouseholdCalculationInputs: + """What a household calculation is read from. + + ``catalog_version_id`` is the country's catalog version for the .py + version the calculation runs on; ``reform`` holds the policy's values by + parameter name and period, empty for current law. + """ + + catalog_version_id: UUID + household: StoredHousehold + reform: PolicyReform + + +class HouseholdJobStore: + """Read and write household jobs, each operation in its own transaction.""" + + def __init__(self, transaction: TransactionFactory) -> None: + """Open each transaction with ``transaction``, which commits on success.""" + self._transaction = transaction + + @classmethod + def from_engine( + cls, engine: Engine, *, statement_timeout_milliseconds: int + ) -> HouseholdJobStore: + """Use ``engine``, bounding each statement to the given time.""" + + def transaction() -> AbstractContextManager[Connection]: + return bounded_transaction( + engine, statement_timeout_milliseconds=statement_timeout_milliseconds + ) + + return cls(transaction) + + def read_inputs( + self, + *, + country: str, + household_id: UUID, + policy_id: UUID | None, + policyengine_version: str, + ) -> HouseholdCalculationInputs: + """Read the catalog version, household and policy in one transaction. + + Raise UnknownCatalogVersionError, UnknownHouseholdError, or the + policy reader's UnknownPolicyError or MissingPolicyParametersError. + """ + with self._transaction() as connection: + catalog_version_id = read_catalog_version_id( + connection, country=country, policyengine_version=policyengine_version + ) + if catalog_version_id is None: + msg = f"{country} has no catalog version for .py {policyengine_version}" + raise UnknownCatalogVersionError(msg) + household = read_stored_household( + connection, country=country, household_id=household_id + ) + if household is None: + msg = f"{country} has no household {household_id}" + raise UnknownHouseholdError(msg) + reform = ( + {} + if policy_id is None + else read_policy_reform( + connection, + country=country, + policy_id=policy_id, + catalog_version_id=catalog_version_id, + ) + ) + return HouseholdCalculationInputs( + catalog_version_id=catalog_version_id, household=household, reform=reform + ) + + def create_job(self, job: NewHouseholdJob) -> None: + """Create the job ``pending`` with no call; see ``insert_household_job``.""" + with self._transaction() as connection: + insert_household_job(connection, job) + + def record_call( + self, job_id: UUID, modal_call_id: str + ) -> HouseholdJobStatus | None: + """Record the job's call ID while it is pending with none; return its status.""" + with self._transaction() as connection: + return record_household_call( + connection, job_id=job_id, modal_call_id=modal_call_id + ) + + def fail_unstarted(self, job_id: UUID, error_message: str) -> bool: + """Fail the job while it is pending with no call ID; return whether it was.""" + with self._transaction() as connection: + return fail_unstarted_household_job( + connection, job_id=job_id, error_message=error_message + ) + + def read(self, job_id: UUID) -> HouseholdJobRecord | None: + """Return the job as it stands, or None when there is no such job.""" + with self._transaction() as connection: + return read_household_job(connection, job_id) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/input_tables.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/input_tables.py new file mode 100644 index 000000000..1bde68b74 --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/input_tables.py @@ -0,0 +1,82 @@ +"""Read-only models of the Public API's tables a calculation is read from. + +The Public API writes households, policies and their parameter values, and +its model catalog; the simulation service only reads them, with ``SELECT``. +Each table here names only the columns the service reads, so these are not +models of the whole tables: the Public API's other columns, defaults, +constraints and indexes are its own. The tables live in ``input_metadata``, +apart from the models of the tables the service writes (``service_metadata``) +and from the tables those name by ``id`` alone (``referenced_metadata``). +``tests/test_public_api_schema.py`` compares each column with the Public +API's table. +""" + +from __future__ import annotations + +from typing import Final + +from sqlalchemy import Column, DateTime, Integer, MetaData, String, Table, Uuid +from sqlalchemy.dialects.postgresql import JSONB + +from policyengine_simulation_persistence.tables import TABLE_SCHEMA + +input_metadata = MetaData(schema=TABLE_SCHEMA) + +# A country's tax-benefit model is named "policyengine-". +MODEL_NAME_PREFIX: Final = "policyengine-" + +tax_benefit_models = Table( + "tax_benefit_models", + input_metadata, + Column("id", Uuid, nullable=False), + Column("name", String(32), nullable=False), +) + +# One published catalog version of a model; ``version`` is the .py +# (policyengine package) version it was published from. +tax_benefit_model_versions = Table( + "tax_benefit_model_versions", + input_metadata, + Column("id", Uuid, nullable=False), + Column("model_id", Uuid, nullable=False), + Column("version", String(128), nullable=False), +) + +# ``household_data`` lists the people and each group entity as records, and +# may hold a saved SPM selection; ``default_year`` is the year it describes. +households = Table( + "households", + input_metadata, + Column("id", Uuid, nullable=False), + Column("country_id", String(2), nullable=False), + Column("default_year", Integer, nullable=True), + Column("household_data", JSONB, nullable=False), +) + +policies = Table( + "policies", + input_metadata, + Column("id", Uuid, nullable=False), + Column("country_id", String(2), nullable=False), +) + +# A catalog version's parameters, each named once within it. +parameters = Table( + "parameters", + input_metadata, + Column("id", Uuid, nullable=False), + Column("name", String(512), nullable=False), + Column("tax_benefit_model_version_id", Uuid, nullable=False), +) + +# A parameter's value from ``start_date`` until ``end_date``, or with no end +# date indefinitely; a policy's own values name it in ``policy_id``. +parameter_values = Table( + "parameter_values", + input_metadata, + Column("parameter_id", Uuid, nullable=False), + Column("value_json", JSONB, nullable=False), + Column("start_date", DateTime(timezone=True), nullable=False), + Column("end_date", DateTime(timezone=True), nullable=True), + Column("policy_id", Uuid, nullable=True), +) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/policy_reform.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/policy_reform.py new file mode 100644 index 000000000..c87ebc985 --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/policy_reform.py @@ -0,0 +1,165 @@ +"""Read a stored policy as the reform values a calculation applies. + +The Public API stores a policy's parameter values against the parameters of +the catalog version it was saved on, each from a start date until an end +date, or with no end date indefinitely. A calculation on any catalog version +applies them by parameter name, as ``{parameter name: {period: value}}``, the +shape of the Public API's policy JSON that the household functions' contract +and the economy requests take. + +Each period is a range, ``YYYY-MM-DD.YYYY-MM-DD``, with the dates read in +UTC. A value with no end date follows .py's own rule +(``policyengine.utils.parametric_reforms.reform_dict_from_parameter_values``): +it stops the day before the same parameter's next later value starts, and +otherwise runs to the end of 2200, the latest year a household job or +simulation run can be for, where .py stops at 2100. So once a later bounded +value ends, current law applies. The reader gives the same periods, in the +same order, as .py does for the values in start order, except that it +leaves out an open-ended value whose range would end before it starts: +one followed the same day by a later value, or one starting after 2200. +.py gives such a value a range that ends before it starts too. +policyengine-core applies that range on no day, but policyengine-uk refuses +it (its ``interval_bounds`` raises "ends before it starts"), so a UK +calculation given .py's dict fails where one given the reader's runs. A +range is used even for the last value because the countries' models read a +start date alone differently: policyengine-core reads it as "from that day +on", but policyengine-uk's reform dictionary as that one day. +""" + +from __future__ import annotations + +from datetime import UTC, date, timedelta +from typing import TYPE_CHECKING, Final + +from sqlalchemy import select + +from policyengine_simulation_persistence.input_tables import ( + parameter_values, + parameters, + policies, +) +from policyengine_simulation_persistence.tables import LATEST_YEAR + +if TYPE_CHECKING: + from collections.abc import Sequence + from datetime import datetime + from uuid import UUID + + from sqlalchemy import Connection + +# The last day a value without an end date is applied. +OPEN_ENDED_STOP: Final = date(LATEST_YEAR, 12, 31) + +# A policy's reform values: each parameter's values by period. +type PolicyReform = dict[str, dict[str, object]] + + +class UnknownPolicyError(LookupError): + """The Public API has no such policy in the country.""" + + +class MissingPolicyParametersError(ValueError): + """The policy sets parameters the calculation's catalog version lacks.""" + + def __init__(self, parameter_names: Sequence[str]) -> None: + """Name every parameter the catalog version lacks, in name order.""" + self.parameter_names = tuple(parameter_names) + super().__init__( + "the policy sets parameters the catalog version does not have: " + + ", ".join(self.parameter_names) + ) + + +def _period( + start_date: datetime, + end_date: datetime | None, + next_start_date: datetime | None, +) -> str | None: + """Return a value's period by .py's rule, or None when no day is in it. + + The dates are read in UTC. A value without an end date stops the day + before ``next_start_date``, the start of the same parameter's next later + value, if any, and otherwise at the end of 2200. Its range would end + before it starts when that next value starts the same day, or when it + starts after 2200: then it is None. + """ + start_day = _utc_day(start_date) + if end_date is not None: + return f"{start_day.isoformat()}.{_utc_day(end_date).isoformat()}" + stop_day = ( + OPEN_ENDED_STOP + if next_start_date is None + else _utc_day(next_start_date) - timedelta(days=1) + ) + if stop_day < start_day: + return None + return f"{start_day.isoformat()}.{stop_day.isoformat()}" + + +def read_policy_reform( + connection: Connection, + *, + country: str, + policy_id: UUID, + catalog_version_id: UUID, +) -> PolicyReform: + """Return the values ``country``'s policy sets, by parameter name and period. + + The periods are .py's for the values in start order (see the module + docstring). Raise UnknownPolicyError when the country has no such + policy, and MissingPolicyParametersError when ``catalog_version_id`` + lacks a parameter the policy sets. + """ + found = connection.execute( + select(policies.c.id).where( + policies.c.id == policy_id, policies.c.country_id == country + ) + ).scalar_one_or_none() + if found is None: + msg = f"{country} has no policy {policy_id}" + raise UnknownPolicyError(msg) + values = connection.execute( + select( + parameters.c.name, + parameter_values.c.value_json, + parameter_values.c.start_date, + parameter_values.c.end_date, + ) + .join(parameters, parameters.c.id == parameter_values.c.parameter_id) + .where(parameter_values.c.policy_id == policy_id) + .order_by(parameters.c.name, parameter_values.c.start_date) + ).all() + names = sorted({row.name for row in values}) + available = set( + connection.execute( + select(parameters.c.name).where( + parameters.c.tax_benefit_model_version_id == catalog_version_id, + parameters.c.name.in_(names), + ) + ).scalars() + ) + missing = [name for name in names if name not in available] + if missing: + raise MissingPolicyParametersError(missing) + reform: PolicyReform = {} + # The rows are ordered by parameter name, then start date, the order + # .py takes them in; each value's periods are .py's. + for position, row in enumerate(values): + next_start_date = next( + ( + later.start_date + for later in values[position + 1 :] + if later.name == row.name and later.start_date > row.start_date + ), + None, + ) + period = _period(row.start_date, row.end_date, next_start_date) + if period is None: + # .py gives it a range ending before it starts (module docstring). + continue + reform.setdefault(row.name, {})[period] = row.value_json + return reform + + +def _utc_day(moment: datetime) -> date: + return moment.astimezone(UTC).date() diff --git a/libs/policyengine-simulation-persistence/tests/public_api_rows.py b/libs/policyengine-simulation-persistence/tests/public_api_rows.py new file mode 100644 index 000000000..7bb74ec47 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/public_api_rows.py @@ -0,0 +1,174 @@ +"""Insert the Public API's own rows a calculation is read from, for tests. + +The Public API writes households, policies, their parameter values and its +model catalog; the tests insert them as the superuser, in a transaction they +roll back, before acting as the simulation service's role. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from hashlib import sha256 +from typing import TYPE_CHECKING +from uuid import UUID, uuid4 + +from sqlalchemy import text +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.sql import bindparam + +if TYPE_CHECKING: + from collections.abc import Iterable, Mapping + from datetime import datetime + + from sqlalchemy import Connection + + +@dataclass(frozen=True) +class CatalogVersion: + """A country's tax-benefit model and one published version of its catalog.""" + + country: str + model_id: UUID + version_id: UUID + policyengine_version: str + + +@dataclass(frozen=True) +class StoredValue: + """One parameter value a policy sets, as the Public API stores it.""" + + parameter_name: str + value: object + start_date: datetime + end_date: datetime | None = None + + +def _content_hash(identifier: UUID) -> str: + return sha256(identifier.bytes).hexdigest() + + +def insert_model(connection: Connection, country: str) -> UUID: + """Insert the country's tax-benefit model, named as the Public API names it.""" + model_id = uuid4() + connection.execute( + text("INSERT INTO public.tax_benefit_models (id, name) VALUES (:id, :name)"), + {"id": model_id, "name": f"policyengine-{country}"}, + ) + return model_id + + +def insert_catalog_version( + connection: Connection, + *, + country: str, + model_id: UUID, + policyengine_version: str, + parameter_names: Iterable[str] = (), +) -> CatalogVersion: + """Insert a catalog version of the model and its named parameters.""" + version_id = uuid4() + connection.execute( + text( + "INSERT INTO public.tax_benefit_model_versions " + "(id, model_id, version, current_law_id, metadata_time_periods) " + "VALUES (:id, :model_id, :version, 1, '[]')" + ), + {"id": version_id, "model_id": model_id, "version": policyengine_version}, + ) + for name in parameter_names: + connection.execute( + text( + "INSERT INTO public.parameters " + "(id, name, tax_benefit_model_version_id) " + "VALUES (:id, :name, :version_id)" + ), + {"id": uuid4(), "name": name, "version_id": version_id}, + ) + return CatalogVersion( + country=country, + model_id=model_id, + version_id=version_id, + policyengine_version=policyengine_version, + ) + + +def _parameter_ids(connection: Connection, version_id: UUID) -> dict[str, UUID]: + rows = connection.execute( + text( + "SELECT name, id FROM public.parameters " + "WHERE tax_benefit_model_version_id = :version_id" + ), + {"version_id": version_id}, + ) + return {row.name: row.id for row in rows} + + +def insert_policy( + connection: Connection, + catalog: CatalogVersion, + values: Iterable[StoredValue] = (), +) -> UUID: + """Insert a policy saved on ``catalog`` and the values it sets. + + Each value names a parameter of that catalog version by name. + """ + policy_id = uuid4() + connection.execute( + text( + "INSERT INTO public.policies (id, country_id, tax_benefit_model_id, " + "tax_benefit_model_version_id, canonicalization_version, content_hash) " + "VALUES (:id, :country, :model_id, :version_id, 1, :content_hash)" + ), + { + "id": policy_id, + "country": catalog.country, + "model_id": catalog.model_id, + "version_id": catalog.version_id, + "content_hash": _content_hash(policy_id), + }, + ) + parameter_ids = _parameter_ids(connection, catalog.version_id) + insert_value = text( + "INSERT INTO public.parameter_values " + "(id, parameter_id, value_json, start_date, end_date, policy_id) " + "VALUES (:id, :parameter_id, :value, :start_date, :end_date, :policy_id)" + ).bindparams(bindparam("value", type_=JSONB)) + for stored in values: + connection.execute( + insert_value, + { + "id": uuid4(), + "parameter_id": parameter_ids[stored.parameter_name], + "value": stored.value, + "start_date": stored.start_date, + "end_date": stored.end_date, + "policy_id": policy_id, + }, + ) + return policy_id + + +def insert_household( + connection: Connection, + *, + country: str, + default_year: int | None, + document: Mapping[str, object], +) -> UUID: + """Insert a stored household of ``country`` with this document.""" + household_id = uuid4() + connection.execute( + text( + "INSERT INTO public.households (id, household_data, country_id, " + "default_year, canonicalization_version, content_hash) " + "VALUES (:id, :document, :country, :default_year, 1, :content_hash)" + ).bindparams(bindparam("document", type_=JSONB)), + { + "id": household_id, + "document": dict(document), + "country": country, + "default_year": default_year, + "content_hash": _content_hash(household_id), + }, + ) + return household_id diff --git a/libs/policyengine-simulation-persistence/tests/schema_comparison.py b/libs/policyengine-simulation-persistence/tests/schema_comparison.py index b393297a2..a05115c31 100644 --- a/libs/policyengine-simulation-persistence/tests/schema_comparison.py +++ b/libs/policyengine-simulation-persistence/tests/schema_comparison.py @@ -12,6 +12,8 @@ definition; and the values of each enum its columns use. ``schema_differences`` lists every way the models' tables differ from the real ones. Tables the models leave out are not compared. +``read_column_differences`` compares read-only models, which name only the +columns the service reads: each column's type and whether it allows null. ``foreign_key_targets`` reads which table each foreign key names, and ``foreign_keys_to`` keeps the keys that name the given tables. """ @@ -264,6 +266,40 @@ def schema_differences(models: SchemaShape, real: SchemaShape) -> list[str]: return found +def read_column_differences(models: SchemaShape, real: SchemaShape) -> list[str]: + """Return each way a read-only model's columns differ from the real ones. + + Read-only models name only the columns the service reads, so only those + columns are compared, by type and nullability. Defaults, constraints, + indexes and the real tables' other columns are the Public API's alone. + """ + found = [] + for table_name, modelled in models.tables.items(): + actual = real.tables.get(table_name) + if actual is None: + found.append(f"{table_name}: the Public API has no such table") + continue + for column_name, modelled_column in modelled.columns.items(): + actual_column = actual.columns.get(column_name) + where = f"{table_name}.{column_name}" + if actual_column is None: + found.append(f"{where}: the Public API's table has no such column") + continue + if modelled_column.column_type != actual_column.column_type: + found.append( + f"{where}: the model is {modelled_column.column_type}, the " + f"Public API's is {actual_column.column_type}" + ) + if modelled_column.nullable != actual_column.nullable: + found.append( + f"{where}: the model " + f"{'allows' if modelled_column.nullable else 'refuses'} null, " + "the Public API's " + f"{'allows' if actual_column.nullable else 'refuses'} it" + ) + return found + + def schema_to_json(schema: SchemaShape) -> str: """Write a description as JSON, as the rule's proof fixtures store it.""" return json.dumps(asdict(schema), indent=2, sort_keys=True) + "\n" diff --git a/libs/policyengine-simulation-persistence/tests/test_engine.py b/libs/policyengine-simulation-persistence/tests/test_engine.py index daf2d61d1..6e3121b8a 100644 --- a/libs/policyengine-simulation-persistence/tests/test_engine.py +++ b/libs/policyengine-simulation-persistence/tests/test_engine.py @@ -1,12 +1,17 @@ -"""Connection settings for the simulation service's database, without a database.""" +"""Connection settings for the simulation service's database. + +The tests marked ``postgres`` need a disposable PostgreSQL server (``conftest``). +""" from __future__ import annotations import pytest from policyengine_simulation_persistence import ( + bounded_transaction, create_simulation_engine, normalized_database_url, ) +from sqlalchemy import text def test_database_url_selects_psycopg_without_changing_the_secret_format() -> None: @@ -63,3 +68,61 @@ def capture_engine(url, **options): assert captured_options["connect_args"]["connect_timeout"] > 0 assert captured_options["pool_pre_ping"] is True assert captured_options["pool_size"] >= 1 + + +@pytest.mark.postgres +def test_a_bounded_transaction_limits_its_own_statements_and_commits( + disposable_database, +) -> None: + """The limit is set for the transaction alone, so a pooled connection keeps none.""" + engine = create_simulation_engine(disposable_database.superuser_url) + try: + with engine.begin() as connection: + connection.execute(text("CREATE TABLE written (id integer)")) + with bounded_transaction( + engine, statement_timeout_milliseconds=1234 + ) as connection: + limit = connection.execute( + text("SELECT current_setting('statement_timeout')") + ).scalar_one() + connection.execute(text("INSERT INTO written VALUES (1)")) + with engine.connect() as connection: + later_limit = connection.execute( + text("SELECT current_setting('statement_timeout')") + ).scalar_one() + count = connection.execute( + text("SELECT count(*) FROM written") + ).scalar_one() + finally: + engine.dispose() + + assert limit == "1234ms" + assert later_limit == "0" + assert count == 1 + + +def _insert_then_fail(engine) -> None: + with bounded_transaction(engine, statement_timeout_milliseconds=1000) as connection: + connection.execute(text("INSERT INTO written VALUES (1)")) + msg = "block failed" + raise RuntimeError(msg) + + +@pytest.mark.postgres +def test_a_bounded_transaction_rolls_back_when_its_block_raises( + disposable_database, +) -> None: + engine = create_simulation_engine(disposable_database.superuser_url) + try: + with engine.begin() as connection: + connection.execute(text("CREATE TABLE written (id integer)")) + with pytest.raises(RuntimeError, match="block failed"): + _insert_then_fail(engine) + with engine.connect() as connection: + count = connection.execute( + text("SELECT count(*) FROM written") + ).scalar_one() + finally: + engine.dispose() + + assert count == 0 diff --git a/libs/policyengine-simulation-persistence/tests/test_household_job_records.py b/libs/policyengine-simulation-persistence/tests/test_household_job_records.py new file mode 100644 index 000000000..c1642e2cf --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_household_job_records.py @@ -0,0 +1,313 @@ +"""The simulation service's reads and writes for a household calculation. + +These run against a database the Public API's own v2 migration chain built +(``PUBLIC_API_SCHEMA_DATABASE_URL``), as the simulation service's role, each +in a transaction rolled back afterwards. The Public API's rows a job names are +inserted first, as the superuser. +""" + +from __future__ import annotations + +from dataclasses import replace +from typing import TYPE_CHECKING +from uuid import uuid4 + +import pytest +from conftest import act_as_the_service, rolled_back_transaction +from policyengine_simulation_persistence import ( + HouseholdJobReferenceError, + HouseholdJobStatus, + NewHouseholdJob, + fail_unstarted_household_job, + household_jobs, + insert_household_job, + read_catalog_version_id, + read_household_job, + read_stored_household, + record_household_call, +) +from public_api_rows import ( + CatalogVersion, + insert_catalog_version, + insert_household, + insert_model, + insert_policy, +) +from sqlalchemy import text, update + +if TYPE_CHECKING: + from uuid import UUID + + from sqlalchemy import Connection + +pytestmark = pytest.mark.public_api_schema + +POLICYENGINE_VERSION = "6.17.1" +COUNTRY_PACKAGE_VERSION = "2.2.1" +DOCUMENT = { + "people": [ + { + "id": "you", + "values": {"age": 40}, + "memberships": {"household": "your household"}, + } + ], + "household": [{"id": "your household", "values": {}}], +} + + +def _catalog(connection: Connection, country: str = "us") -> CatalogVersion: + return insert_catalog_version( + connection, + country=country, + model_id=insert_model(connection, country), + policyengine_version=POLICYENGINE_VERSION, + ) + + +def _new_job( + catalog: CatalogVersion, + household_id: UUID, + *, + policy_id: UUID | None = None, +) -> NewHouseholdJob: + return NewHouseholdJob( + id=uuid4(), + country=catalog.country, + household_id=household_id, + policy_id=policy_id, + year=2026, + request_data={"country": catalog.country, "household_id": str(household_id)}, + tax_benefit_model_version_id=catalog.version_id, + country_package_version=COUNTRY_PACKAGE_VERSION, + policyengine_version=POLICYENGINE_VERSION, + ) + + +def _stored_job(connection: Connection) -> NewHouseholdJob: + """Insert, as the superuser, the rows a job names; return a job naming them.""" + catalog = _catalog(connection) + household_id = insert_household( + connection, country="us", default_year=2026, document=DOCUMENT + ) + return _new_job(catalog, household_id, policy_id=insert_policy(connection, catalog)) + + +def test_a_catalog_version_is_found_by_country_and_policyengine_version( + public_api_database_url, +) -> None: + with rolled_back_transaction(public_api_database_url) as connection: + catalog = _catalog(connection) + _catalog(connection, "uk") + act_as_the_service(connection) + + assert ( + read_catalog_version_id( + connection, country="us", policyengine_version=POLICYENGINE_VERSION + ) + == catalog.version_id + ) + assert ( + read_catalog_version_id( + connection, country="us", policyengine_version="6.16.1" + ) + is None + ) + + +def test_a_household_is_found_in_its_own_country_only(public_api_database_url) -> None: + with rolled_back_transaction(public_api_database_url) as connection: + household_id = insert_household( + connection, country="us", default_year=2026, document=DOCUMENT + ) + act_as_the_service(connection) + + stored = read_stored_household( + connection, country="us", household_id=household_id + ) + assert ( + read_stored_household(connection, country="uk", household_id=household_id) + is None + ) + assert ( + read_stored_household(connection, country="us", household_id=uuid4()) + is None + ) + + assert stored is not None + assert (stored.id, stored.country, stored.default_year) == ( + household_id, + "us", + 2026, + ) + assert stored.document == DOCUMENT + + +def test_a_job_is_created_pending_with_no_call(public_api_database_url) -> None: + with rolled_back_transaction(public_api_database_url) as connection: + new_job = _stored_job(connection) + act_as_the_service(connection) + + insert_household_job(connection, new_job) + job = read_household_job(connection, new_job.id) + dynamic_id = connection.execute( + text("SELECT dynamic_id FROM public.household_jobs WHERE id = :id"), + {"id": new_job.id}, + ).scalar_one() + + assert job is not None + assert job.status is HouseholdJobStatus.PENDING + assert job.modal_call_id is None + assert dynamic_id is None + assert ( + job.id, + job.country, + job.household_id, + job.policy_id, + job.year, + job.tax_benefit_model_version_id, + job.country_package_version, + job.policyengine_version, + job.request_data, + ) == ( + new_job.id, + "us", + new_job.household_id, + new_job.policy_id, + 2026, + new_job.tax_benefit_model_version_id, + COUNTRY_PACKAGE_VERSION, + POLICYENGINE_VERSION, + new_job.request_data, + ) + assert (job.result, job.error_message, job.started_at, job.completed_at) == ( + None, + None, + None, + None, + ) + + +def test_an_unknown_job_reads_as_none(public_api_database_url) -> None: + with rolled_back_transaction(public_api_database_url) as connection: + act_as_the_service(connection) + + assert read_household_job(connection, uuid4()) is None + + +def test_a_call_is_recorded_once_while_the_job_is_pending_without_one( + public_api_database_url, +) -> None: + with rolled_back_transaction(public_api_database_url) as connection: + new_job = _stored_job(connection) + act_as_the_service(connection) + insert_household_job(connection, new_job) + + first = record_household_call( + connection, job_id=new_job.id, modal_call_id="fc-first" + ) + second = record_household_call( + connection, job_id=new_job.id, modal_call_id="fc-second" + ) + job = read_household_job(connection, new_job.id) + + assert (first, second) == (HouseholdJobStatus.PENDING, HouseholdJobStatus.PENDING) + assert job is not None + assert job.modal_call_id == "fc-first" + + +def test_a_job_the_function_already_claimed_keeps_its_claim( + public_api_database_url, +) -> None: + """The household function may claim the job before its call ID is recorded.""" + with rolled_back_transaction(public_api_database_url) as connection: + new_job = _stored_job(connection) + act_as_the_service(connection) + insert_household_job(connection, new_job) + connection.execute(text("RESET ROLE")) + connection.execute( + update(household_jobs) + .where(household_jobs.c.id == new_job.id) + .values(status=HouseholdJobStatus.RUNNING, modal_call_id="fc-claimed") + ) + act_as_the_service(connection) + + status = record_household_call( + connection, job_id=new_job.id, modal_call_id="fc-claimed" + ) + failed = fail_unstarted_household_job( + connection, job_id=new_job.id, error_message="Modal refused the call." + ) + job = read_household_job(connection, new_job.id) + + assert status is HouseholdJobStatus.RUNNING + assert failed is False + assert job is not None + assert (job.status, job.modal_call_id, job.error_message) == ( + HouseholdJobStatus.RUNNING, + "fc-claimed", + None, + ) + + +def test_a_job_whose_call_was_refused_fails_at_once(public_api_database_url) -> None: + with rolled_back_transaction(public_api_database_url) as connection: + new_job = _stored_job(connection) + act_as_the_service(connection) + insert_household_job(connection, new_job) + + failed = fail_unstarted_household_job( + connection, job_id=new_job.id, error_message="Modal refused the call." + ) + failed_again = fail_unstarted_household_job( + connection, job_id=new_job.id, error_message="A second refusal." + ) + job = read_household_job(connection, new_job.id) + + assert (failed, failed_again) == (True, False) + assert job is not None + assert job.status is HouseholdJobStatus.FAILED + assert job.error_message == "Modal refused the call." + assert job.completed_at is not None + assert job.modal_call_id is None + + +def test_a_job_whose_call_was_recorded_is_not_failed_as_refused( + public_api_database_url, +) -> None: + with rolled_back_transaction(public_api_database_url) as connection: + new_job = _stored_job(connection) + act_as_the_service(connection) + insert_household_job(connection, new_job) + record_household_call(connection, job_id=new_job.id, modal_call_id="fc-1") + + failed = fail_unstarted_household_job( + connection, job_id=new_job.id, error_message="Modal refused the call." + ) + job = read_household_job(connection, new_job.id) + + assert failed is False + assert job is not None + assert job.status is HouseholdJobStatus.PENDING + + +@pytest.mark.parametrize( + "missing_column", + ["household_id", "policy_id", "tax_benefit_model_version_id"], +) +def test_a_job_naming_a_row_that_does_not_exist_names_its_column( + public_api_database_url, missing_column +) -> None: + """The database refuses the row with SQLSTATE 23503, which names the key.""" + with rolled_back_transaction(public_api_database_url) as connection: + stored = _stored_job(connection) + new_job = replace(stored, **{missing_column: uuid4()}) + act_as_the_service(connection) + + with ( + pytest.raises(HouseholdJobReferenceError) as raised, + connection.begin_nested(), + ): + insert_household_job(connection, new_job) + + assert raised.value.column_name == missing_column diff --git a/libs/policyengine-simulation-persistence/tests/test_household_job_store.py b/libs/policyengine-simulation-persistence/tests/test_household_job_store.py new file mode 100644 index 000000000..ba9ca2d78 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_household_job_store.py @@ -0,0 +1,227 @@ +"""The household calculation store: each read and write in a transaction of its own. + +These run against a database the Public API's own v2 migration chain built +(``PUBLIC_API_SCHEMA_DATABASE_URL``). The store opens each transaction +through a function it is given; here each is a savepoint of one connection, +acting as the simulation service's role, which the test rolls back. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from datetime import UTC, datetime +from typing import TYPE_CHECKING +from uuid import uuid4 + +import pytest +from conftest import act_as_the_service, rolled_back_transaction +from policyengine_simulation_persistence import ( + HouseholdJobStatus, + HouseholdJobStore, + NewHouseholdJob, + UnknownCatalogVersionError, + UnknownHouseholdError, + UnknownPolicyError, +) +from public_api_rows import ( + StoredValue, + insert_catalog_version, + insert_household, + insert_model, + insert_policy, +) + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sqlalchemy import Connection + +pytestmark = pytest.mark.public_api_schema + +POLICYENGINE_VERSION = "6.17.1" +CTC_AMOUNT = "gov.irs.credits.ctc.amount.base[0].amount" +DOCUMENT = { + "people": [ + { + "id": "you", + "values": {"age": 40}, + "memberships": {"household": "your household"}, + } + ], + "household": [{"id": "your household", "values": {}}], +} + + +def _store_on(connection: Connection) -> HouseholdJobStore: + """Return a store whose transactions are savepoints of ``connection``.""" + + @contextmanager + def savepoint() -> Iterator[Connection]: + with connection.begin_nested(): + yield connection + + return HouseholdJobStore(savepoint) + + +def test_the_inputs_are_read_together(public_api_database_url) -> None: + with rolled_back_transaction(public_api_database_url) as connection: + catalog = insert_catalog_version( + connection, + country="us", + model_id=insert_model(connection, "us"), + policyengine_version=POLICYENGINE_VERSION, + parameter_names=[CTC_AMOUNT], + ) + household_id = insert_household( + connection, country="us", default_year=2026, document=DOCUMENT + ) + policy_id = insert_policy( + connection, + catalog, + [StoredValue(CTC_AMOUNT, 3_000, datetime(2026, 1, 1, tzinfo=UTC))], + ) + act_as_the_service(connection) + store = _store_on(connection) + + with_policy = store.read_inputs( + country="us", + household_id=household_id, + policy_id=policy_id, + policyengine_version=POLICYENGINE_VERSION, + ) + current_law = store.read_inputs( + country="us", + household_id=household_id, + policy_id=None, + policyengine_version=POLICYENGINE_VERSION, + ) + + assert with_policy.catalog_version_id == catalog.version_id + assert with_policy.household.id == household_id + assert with_policy.household.default_year == 2026 + assert with_policy.household.document == DOCUMENT + assert with_policy.reform == {CTC_AMOUNT: {"2026-01-01.2200-12-31": 3_000}} + assert current_law.reform == {} + + +def test_each_missing_input_is_named(public_api_database_url) -> None: + with rolled_back_transaction(public_api_database_url) as connection: + insert_catalog_version( + connection, + country="us", + model_id=insert_model(connection, "us"), + policyengine_version=POLICYENGINE_VERSION, + ) + household_id = insert_household( + connection, country="us", default_year=2026, document=DOCUMENT + ) + missing_id = uuid4() + act_as_the_service(connection) + store = _store_on(connection) + + with pytest.raises(UnknownCatalogVersionError) as no_version: + store.read_inputs( + country="us", + household_id=household_id, + policy_id=None, + policyengine_version="6.16.1", + ) + with pytest.raises(UnknownHouseholdError) as no_household: + store.read_inputs( + country="us", + household_id=missing_id, + policy_id=None, + policyengine_version=POLICYENGINE_VERSION, + ) + with pytest.raises(UnknownPolicyError) as no_policy: + store.read_inputs( + country="us", + household_id=household_id, + policy_id=missing_id, + policyengine_version=POLICYENGINE_VERSION, + ) + + assert str(no_version.value) == "us has no catalog version for .py 6.16.1" + assert str(no_household.value) == f"us has no household {missing_id}" + assert str(no_policy.value) == f"us has no policy {missing_id}" + + +def test_a_job_is_created_recorded_and_read_back(public_api_database_url) -> None: + with rolled_back_transaction(public_api_database_url) as connection: + catalog = insert_catalog_version( + connection, + country="uk", + model_id=insert_model(connection, "uk"), + policyengine_version=POLICYENGINE_VERSION, + ) + household_id = insert_household( + connection, country="uk", default_year=2027, document=DOCUMENT + ) + act_as_the_service(connection) + store = _store_on(connection) + job_id = uuid4() + + store.create_job( + NewHouseholdJob( + id=job_id, + country="uk", + household_id=household_id, + policy_id=None, + year=2027, + request_data={"country": "uk"}, + tax_benefit_model_version_id=catalog.version_id, + country_package_version="2.174.1", + policyengine_version=POLICYENGINE_VERSION, + ) + ) + status = store.record_call(job_id, "fc-1") + job = store.read(job_id) + unknown = store.read(uuid4()) + + assert status is HouseholdJobStatus.PENDING + assert job is not None + assert (job.status, job.modal_call_id, job.year) == ( + HouseholdJobStatus.PENDING, + "fc-1", + 2027, + ) + assert unknown is None + + +def test_a_refused_call_fails_its_job(public_api_database_url) -> None: + with rolled_back_transaction(public_api_database_url) as connection: + catalog = insert_catalog_version( + connection, + country="us", + model_id=insert_model(connection, "us"), + policyengine_version=POLICYENGINE_VERSION, + ) + household_id = insert_household( + connection, country="us", default_year=2026, document=DOCUMENT + ) + act_as_the_service(connection) + store = _store_on(connection) + job_id = uuid4() + store.create_job( + NewHouseholdJob( + id=job_id, + country="us", + household_id=household_id, + policy_id=None, + year=2026, + request_data={}, + tax_benefit_model_version_id=catalog.version_id, + country_package_version="2.2.1", + policyengine_version=POLICYENGINE_VERSION, + ) + ) + + failed = store.fail_unstarted(job_id, "Modal refused the call.") + job = store.read(job_id) + + assert failed is True + assert job is not None + assert (job.status, job.error_message) == ( + HouseholdJobStatus.FAILED, + "Modal refused the call.", + ) diff --git a/libs/policyengine-simulation-persistence/tests/test_policy_reform.py b/libs/policyengine-simulation-persistence/tests/test_policy_reform.py new file mode 100644 index 000000000..153171b51 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_policy_reform.py @@ -0,0 +1,459 @@ +"""A stored policy's parameter values, as the reform values a calculation applies. + +The reads run against a database the Public API's own v2 migration chain built +(``PUBLIC_API_SCHEMA_DATABASE_URL``), as the simulation service's role, in a +transaction rolled back afterwards. +""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta, timezone +from uuid import uuid4 + +import pytest +from conftest import act_as_the_service, rolled_back_transaction +from policyengine_simulation_persistence import ( + MissingPolicyParametersError, + UnknownPolicyError, + read_policy_reform, +) +from public_api_rows import ( + StoredValue, + insert_catalog_version, + insert_model, + insert_policy, +) + +START = datetime(2026, 1, 1, tzinfo=UTC) +END = datetime(2027, 12, 31, tzinfo=UTC) +CTC_AMOUNT = "gov.irs.credits.ctc.amount.base[0].amount" +STANDARD_DEDUCTION = "gov.irs.deductions.standard.amount.SINGLE" + + +def _reform_of(public_api_database_url, values: list[StoredValue]) -> dict: + """Store a US policy setting ``values`` and read it as the service's role.""" + with rolled_back_transaction(public_api_database_url) as connection: + catalog = insert_catalog_version( + connection, + country="us", + model_id=insert_model(connection, "us"), + policyengine_version="6.17.1", + parameter_names=sorted({value.parameter_name for value in values}), + ) + policy_id = insert_policy(connection, catalog, values) + act_as_the_service(connection) + return read_policy_reform( + connection, + country="us", + policy_id=policy_id, + catalog_version_id=catalog.version_id, + ) + + +@pytest.mark.public_api_schema +def test_a_value_with_an_end_date_applies_from_its_start_to_its_end( + public_api_database_url, +) -> None: + reform = _reform_of( + public_api_database_url, [StoredValue(CTC_AMOUNT, 3_000, START, END)] + ) + + assert reform == {CTC_AMOUNT: {"2026-01-01.2027-12-31": 3_000}} + + +@pytest.mark.public_api_schema +def test_a_value_without_an_end_date_applies_to_the_end_of_2200( + public_api_database_url, +) -> None: + """A value with no end date, and none after it, runs to the end of 2200. + + Household jobs and simulation runs are for years up to 2200; .py's own + conversion stops there at 2100. A range is read the same way by every + country's model; a start date alone is not (``test_household_calculation.py`` + in the executor shows both). + """ + reform = _reform_of( + public_api_database_url, [StoredValue(CTC_AMOUNT, 3_000, START)] + ) + + assert reform == {CTC_AMOUNT: {"2026-01-01.2200-12-31": 3_000}} + + +@pytest.mark.public_api_schema +def test_a_value_without_an_end_date_stops_before_the_next_value_starts( + public_api_database_url, +) -> None: + """.py's rule: the parameter's next later value takes over.""" + reform = _reform_of( + public_api_database_url, + [ + StoredValue(CTC_AMOUNT, 3_000, START), + StoredValue(CTC_AMOUNT, 4_000, datetime(2028, 7, 1, tzinfo=UTC)), + ], + ) + + assert reform == { + CTC_AMOUNT: { + "2026-01-01.2028-06-30": 3_000, + "2028-07-01.2200-12-31": 4_000, + } + } + + +@pytest.mark.public_api_schema +def test_dates_are_read_in_utc(public_api_database_url) -> None: + eastern = timezone(timedelta(hours=-5)) + reform = _reform_of( + public_api_database_url, + [ + StoredValue( + CTC_AMOUNT, + 3_000, + datetime(2025, 12, 31, 19, tzinfo=eastern), + datetime(2026, 12, 31, 19, tzinfo=eastern), + ) + ], + ) + + assert reform == {CTC_AMOUNT: {"2026-01-01.2027-01-01": 3_000}} + + +@pytest.mark.public_api_schema +def test_a_value_starting_after_2200_without_an_end_is_left_out( + public_api_database_url, +) -> None: + """Its range would end before it starts, so no year is in it. + + .py's rule gives it ``2250-01-01.2100-12-31``, a range policyengine-uk + refuses (see the module docstring of ``policy_reform``). + """ + reform = _reform_of( + public_api_database_url, + [ + StoredValue(CTC_AMOUNT, 3_000, START, END), + StoredValue(STANDARD_DEDUCTION, 9_000, datetime(2250, 1, 1, tzinfo=UTC)), + ], + ) + + assert reform == {CTC_AMOUNT: {"2026-01-01.2027-12-31": 3_000}} + + +@pytest.mark.public_api_schema +def test_a_policy_is_read_as_its_values_by_parameter_name_and_period( + public_api_database_url, +) -> None: + with rolled_back_transaction(public_api_database_url) as connection: + model_id = insert_model(connection, "us") + saved_on = insert_catalog_version( + connection, + country="us", + model_id=model_id, + policyengine_version="6.16.1", + parameter_names=[CTC_AMOUNT, STANDARD_DEDUCTION], + ) + run_on = insert_catalog_version( + connection, + country="us", + model_id=model_id, + policyengine_version="6.17.1", + parameter_names=[CTC_AMOUNT, STANDARD_DEDUCTION], + ) + policy_id = insert_policy( + connection, + saved_on, + [ + StoredValue(CTC_AMOUNT, 3_000, START, END), + StoredValue(CTC_AMOUNT, 4_000, datetime(2028, 1, 1, tzinfo=UTC)), + StoredValue(STANDARD_DEDUCTION, [1, 2], START), + ], + ) + act_as_the_service(connection) + + reform = read_policy_reform( + connection, + country="us", + policy_id=policy_id, + catalog_version_id=run_on.version_id, + ) + + assert reform == { + CTC_AMOUNT: { + "2026-01-01.2027-12-31": 3_000, + "2028-01-01.2200-12-31": 4_000, + }, + STANDARD_DEDUCTION: {"2026-01-01.2200-12-31": [1, 2]}, + } + + +@pytest.mark.public_api_schema +def test_a_value_without_an_end_date_gives_way_to_a_bounded_one_for_good( + public_api_database_url, +) -> None: + """After the bounded value ends, current law applies, as .py has it. + + The executor's ``test_py_reform_dicts.py`` shows .py's + ``reform_dict_from_parameter_values`` giving these periods, with 2100 + where this has 2200. + """ + reform = _reform_of( + public_api_database_url, + [ + StoredValue(CTC_AMOUNT, 3_000, START), + StoredValue( + CTC_AMOUNT, + 5_000, + datetime(2028, 1, 1, tzinfo=UTC), + datetime(2028, 12, 31, tzinfo=UTC), + ), + ], + ) + + assert reform == { + CTC_AMOUNT: { + "2026-01-01.2027-12-31": 3_000, + "2028-01-01.2028-12-31": 5_000, + } + } + + +@pytest.mark.public_api_schema +def test_values_without_end_dates_run_one_after_another( + public_api_database_url, +) -> None: + """Each runs until the next starts; the last runs to the end of 2200.""" + reform = _reform_of( + public_api_database_url, + [ + StoredValue(CTC_AMOUNT, 1_000, START), + StoredValue(CTC_AMOUNT, 2_000, datetime(2027, 7, 1, tzinfo=UTC)), + StoredValue(CTC_AMOUNT, 3_000, datetime(2030, 1, 1, tzinfo=UTC)), + StoredValue(STANDARD_DEDUCTION, 20_000, datetime(2029, 1, 1, tzinfo=UTC)), + ], + ) + + assert reform == { + CTC_AMOUNT: { + "2026-01-01.2027-06-30": 1_000, + "2027-07-01.2029-12-31": 2_000, + "2030-01-01.2200-12-31": 3_000, + }, + STANDARD_DEDUCTION: {"2029-01-01.2200-12-31": 20_000}, + } + + +@pytest.mark.public_api_schema +def test_a_policy_that_changes_nothing_is_read_as_no_values( + public_api_database_url, +) -> None: + with rolled_back_transaction(public_api_database_url) as connection: + catalog = insert_catalog_version( + connection, + country="uk", + model_id=insert_model(connection, "uk"), + policyengine_version="6.17.1", + ) + policy_id = insert_policy(connection, catalog) + act_as_the_service(connection) + + reform = read_policy_reform( + connection, + country="uk", + policy_id=policy_id, + catalog_version_id=catalog.version_id, + ) + + assert reform == {} + + +@pytest.mark.public_api_schema +def test_a_policy_of_another_country_or_none_at_all_is_unknown( + public_api_database_url, +) -> None: + with rolled_back_transaction(public_api_database_url) as connection: + catalog = insert_catalog_version( + connection, + country="us", + model_id=insert_model(connection, "us"), + policyengine_version="6.17.1", + ) + policy_id = insert_policy(connection, catalog) + act_as_the_service(connection) + + for country, unknown_policy_id in (("uk", policy_id), ("us", uuid4())): + with pytest.raises(UnknownPolicyError) as raised: + read_policy_reform( + connection, + country=country, + policy_id=unknown_policy_id, + catalog_version_id=catalog.version_id, + ) + assert str(raised.value) == (f"{country} has no policy {unknown_policy_id}") + + +@pytest.mark.public_api_schema +def test_a_policy_changing_parameters_the_run_lacks_names_each_of_them( + public_api_database_url, +) -> None: + with rolled_back_transaction(public_api_database_url) as connection: + model_id = insert_model(connection, "us") + saved_on = insert_catalog_version( + connection, + country="us", + model_id=model_id, + policyengine_version="6.16.1", + parameter_names=[CTC_AMOUNT, STANDARD_DEDUCTION, "gov.removed.b"], + ) + run_on = insert_catalog_version( + connection, + country="us", + model_id=model_id, + policyengine_version="6.17.1", + parameter_names=[CTC_AMOUNT], + ) + policy_id = insert_policy( + connection, + saved_on, + [ + StoredValue(CTC_AMOUNT, 3_000, START), + StoredValue(STANDARD_DEDUCTION, 1, START), + StoredValue("gov.removed.b", 2, START), + ], + ) + act_as_the_service(connection) + + with pytest.raises(MissingPolicyParametersError) as raised: + read_policy_reform( + connection, + country="us", + policy_id=policy_id, + catalog_version_id=run_on.version_id, + ) + + assert raised.value.parameter_names == (STANDARD_DEDUCTION, "gov.removed.b") + assert str(raised.value) == ( + "the policy sets parameters the catalog version does not have: " + f"{STANDARD_DEDUCTION}, gov.removed.b" + ) + + +@pytest.mark.public_api_schema +def test_of_values_starting_the_same_day_the_latest_takes_over( + public_api_database_url, +) -> None: + """Periods are days, so the later start of one day takes that day over. + + The executor's ``test_py_reform_dicts.py`` shows .py calculating the same: + it ends the earlier value before it starts, so only the later applies. + """ + reform = _reform_of( + public_api_database_url, + [ + StoredValue(CTC_AMOUNT, 3_000, START), + StoredValue(CTC_AMOUNT, 4_000, START + timedelta(hours=12)), + StoredValue(CTC_AMOUNT, 5_000, datetime(2028, 1, 1, tzinfo=UTC)), + ], + ) + + assert reform == { + CTC_AMOUNT: { + "2026-01-01.2027-12-31": 4_000, + "2028-01-01.2200-12-31": 5_000, + } + } + + +# These match .py's reform_dict_from_parameter_values for the same values +# (the executor's test_py_reform_dicts.py), with 2200 where .py has 2100. + + +@pytest.mark.public_api_schema +def test_bounded_values_starting_the_same_day_are_kept_in_start_order( + public_api_database_url, +) -> None: + """Both ranges are kept, as .py keeps them, and applied in this order.""" + reform = _reform_of( + public_api_database_url, + [ + StoredValue(CTC_AMOUNT, 3_000, START, datetime(2026, 12, 31, tzinfo=UTC)), + StoredValue( + CTC_AMOUNT, + 4_000, + START + timedelta(hours=12), + datetime(2026, 6, 30, tzinfo=UTC), + ), + ], + ) + + assert list(reform[CTC_AMOUNT].items()) == [ + ("2026-01-01.2026-12-31", 3_000), + ("2026-01-01.2026-06-30", 4_000), + ] + + +@pytest.mark.public_api_schema +def test_a_bounded_value_then_an_open_ended_one_on_the_same_day( + public_api_database_url, +) -> None: + reform = _reform_of( + public_api_database_url, + [ + StoredValue(CTC_AMOUNT, 3_000, START, datetime(2026, 6, 30, tzinfo=UTC)), + StoredValue(CTC_AMOUNT, 4_000, START + timedelta(hours=12)), + ], + ) + + assert list(reform[CTC_AMOUNT].items()) == [ + ("2026-01-01.2026-06-30", 3_000), + ("2026-01-01.2200-12-31", 4_000), + ] + + +@pytest.mark.public_api_schema +def test_an_offset_start_is_read_on_its_utc_day(public_api_database_url) -> None: + """20:00 on 31 December five hours behind UTC is 1 January in UTC. + + An open-ended value starting after 2200 has no year in its range and is + left out. + """ + eastern = timezone(timedelta(hours=-5)) + reform = _reform_of( + public_api_database_url, + [ + StoredValue(CTC_AMOUNT, 1_000, datetime(2025, 12, 31, 20, tzinfo=eastern)), + StoredValue(CTC_AMOUNT, 2_000, datetime(2027, 1, 1, tzinfo=UTC)), + StoredValue(STANDARD_DEDUCTION, 9_000, datetime(2250, 1, 1, tzinfo=UTC)), + ], + ) + + assert reform == { + CTC_AMOUNT: { + "2026-01-01.2026-12-31": 1_000, + "2027-01-01.2200-12-31": 2_000, + } + } + + +@pytest.mark.public_api_schema +def test_an_open_ended_value_then_a_bounded_one_on_the_same_day( + public_api_database_url, +) -> None: + """The bounded value applies, then current law once it ends. + + The open-ended value would stop the day before the bounded one starts, + the day before its own start, so it is left out. .py gives it that range, + which policyengine-uk refuses (the executor's ``test_py_reform_dicts.py``). + """ + reform = _reform_of( + public_api_database_url, + [ + StoredValue(CTC_AMOUNT, 3_000, START), + StoredValue( + CTC_AMOUNT, + 4_000, + START + timedelta(hours=12), + datetime(2026, 12, 31, tzinfo=UTC), + ), + ], + ) + + assert reform == {CTC_AMOUNT: {"2026-01-01.2026-12-31": 4_000}} diff --git a/libs/policyengine-simulation-persistence/tests/test_public_api_schema.py b/libs/policyengine-simulation-persistence/tests/test_public_api_schema.py index 70215091c..728c54232 100644 --- a/libs/policyengine-simulation-persistence/tests/test_public_api_schema.py +++ b/libs/policyengine-simulation-persistence/tests/test_public_api_schema.py @@ -17,11 +17,16 @@ from pathlib import Path import pytest -from policyengine_simulation_persistence import referenced_metadata, service_metadata +from policyengine_simulation_persistence import ( + input_metadata, + referenced_metadata, + service_metadata, +) from public_api_keys import KEYS_TO_PUBLIC_API_TABLES from schema_comparison import ( describe_tables, foreign_keys_to, + read_column_differences, schema_differences, schema_from_json, ) @@ -71,6 +76,37 @@ def test_the_models_match_the_public_api_tables( ) +@pytest.mark.public_api_schema +def test_the_input_models_match_the_public_api_columns( + disposable_database, public_api_database_url +) -> None: + """Each column the service reads exists, with the type and nullability modelled. + + The read-only models of the Public API's input tables name only the + columns the service reads, so only those are compared: their other + columns, defaults, constraints and indexes are the Public API's alone. + """ + table_names = [table.name for table in input_metadata.tables.values()] + models_engine = _engine(disposable_database.superuser_url) + real_engine = _engine(public_api_database_url) + try: + # Tests may build the models' tables; runtime code never does. + input_metadata.create_all(models_engine) + with models_engine.connect() as connection: + models = describe_tables(connection, table_names) + with real_engine.connect() as connection: + real = describe_tables(connection, table_names) + finally: + models_engine.dispose() + real_engine.dispose() + + assert set(models.tables) == set(table_names) + compared_with = os.environ.get(COMMIT_VARIABLE) or "an unrecorded commit" + assert read_column_differences(models, real) == [], ( + f"compared with the Public API at {compared_with}" + ) + + # Marked so the rule's own CI jobs, which select public_api_schema, collect it. @pytest.mark.public_api_schema def test_rule_proof_models_match_public_api() -> None: diff --git a/libs/policyengine-simulation-persistence/tests/test_schema_comparison.py b/libs/policyengine-simulation-persistence/tests/test_schema_comparison.py index d70b081e4..7af310cf4 100644 --- a/libs/policyengine-simulation-persistence/tests/test_schema_comparison.py +++ b/libs/policyengine-simulation-persistence/tests/test_schema_comparison.py @@ -16,6 +16,7 @@ describe_tables, foreign_key_targets, foreign_keys_to, + read_column_differences, schema_differences, schema_from_json, schema_to_json, @@ -425,3 +426,74 @@ def test_a_description_round_trips_through_json(tmp_path) -> None: path.write_text(schema_to_json(described), encoding="utf-8") assert schema_from_json(path) == described + + +def _households(**column_changes: ColumnShape) -> SchemaShape: + """Return a households table as the Public API builds it, changed as given.""" + columns = { + "id": ColumnShape("uuid", nullable=False, default=None), + "created_at": ColumnShape( + "timestamp with time zone", nullable=False, default="now()" + ), + "household_data": ColumnShape("jsonb", nullable=False, default=None), + "default_year": ColumnShape("integer", nullable=True, default=None), + **column_changes, + } + return SchemaShape( + tables={ + "households": TableShape( + columns=columns, + constraints={"pk_households": "PRIMARY KEY (id)"}, + indexes={}, + ) + }, + enums={}, + ) + + +def test_input_models_may_leave_out_columns_defaults_and_constraints() -> None: + """A read-only model names only the columns the service reads.""" + real = _households() + models = SchemaShape( + tables={ + "households": TableShape( + columns={ + "id": ColumnShape("uuid", nullable=False, default=None), + "default_year": ColumnShape("integer", nullable=True, default=None), + }, + constraints={}, + indexes={}, + ) + }, + enums={}, + ) + + assert read_column_differences(models, real) == [] + + +def test_input_models_differ_where_a_column_they_read_differs() -> None: + real = _households() + models = SchemaShape( + tables={ + **_households( + default_year=ColumnShape("bigint", nullable=True, default=None), + household_data=ColumnShape("jsonb", nullable=True, default=None), + label=ColumnShape( + "character varying(255)", nullable=True, default=None + ), + ).tables, + "parameters": TableShape( + columns={"id": ColumnShape("uuid", nullable=False, default=None)}, + constraints={}, + indexes={}, + ), + }, + enums={}, + ) + + assert read_column_differences(models, real) == [ + "households.household_data: the model allows null, the Public API's refuses it", + "households.default_year: the model is bigint, the Public API's is integer", + "households.label: the Public API's table has no such column", + "parameters: the Public API has no such table", + ] diff --git a/libs/policyengine-simulation-persistence/tests/test_tables.py b/libs/policyengine-simulation-persistence/tests/test_tables.py index 89f8eaa01..b36958a86 100644 --- a/libs/policyengine-simulation-persistence/tests/test_tables.py +++ b/libs/policyengine-simulation-persistence/tests/test_tables.py @@ -6,6 +6,7 @@ SERVICE_TABLE_PRIVILEGES, SimulationStatus, household_jobs, + input_metadata, referenced_metadata, report_runs, reports, @@ -179,3 +180,31 @@ def test_a_budget_window_has_one_report_run_per_year_of_one_report() -> None: "budget_window_run_id", "year", ) + + +def test_the_input_models_are_tables_the_service_reads() -> None: + """Each input table is one the service's role may SELECT and never writes.""" + readable_only = { + table_name + for table_name, privileges in SERVICE_TABLE_PRIVILEGES.items() + if privileges == {"SELECT"} + } + + assert {table.name for table in input_metadata.tables.values()} == { + "households", + "policies", + "parameters", + "parameter_values", + "tax_benefit_models", + "tax_benefit_model_versions", + } + assert {table.name for table in input_metadata.tables.values()} <= readable_only + + +def test_the_input_models_share_no_table_with_the_service_models() -> None: + """The read models are their own metadata, apart from the run models and keys.""" + input_tables = set(input_metadata.tables.values()) + + assert input_metadata.schema == "public" + assert input_tables.isdisjoint(service_metadata.tables.values()) + assert input_tables.isdisjoint(referenced_metadata.tables.values()) diff --git a/projects/policyengine-apis-integ/tests/test_household_generated_client.py b/projects/policyengine-apis-integ/tests/test_household_generated_client.py new file mode 100644 index 000000000..a8dc72a69 --- /dev/null +++ b/projects/policyengine-apis-integ/tests/test_household_generated_client.py @@ -0,0 +1,124 @@ +"""Hermetic wire checks of the household routes' generated client models.""" + +from uuid import UUID + +import httpx +import pytest +from policyengine_api_simulation_client import AuthenticatedClient +from policyengine_api_simulation_client.api.default import ( + get_household_calculation, + submit_household_calculation, +) +from policyengine_api_simulation_client.models import ( + HouseholdCalculationAccepted, + HouseholdCalculationError, + HouseholdCalculationRequest, + HouseholdCalculationState, + HouseholdJobStatus, +) + +HOUSEHOLD_ID = "00000000-0000-4000-8000-0000000000bb" +JOB_ID = "00000000-0000-4000-8000-0000000000aa" + + +def test_omitted_spm_settings_and_axes_stay_omitted(): + """The service uses the saved selection only when the request sends none.""" + omitted = HouseholdCalculationRequest.from_dict( + {"country": "us", "household_id": HOUSEHOLD_ID} + ).to_dict() + given = HouseholdCalculationRequest.from_dict( + { + "country": "us", + "household_id": HOUSEHOLD_ID, + "spm": {"scenario": "zero_real"}, + "axes": [[{"name": "age", "count": 3, "min": 20, "max": 60}]], + } + ).to_dict() + + assert "spm" not in omitted + assert "axes" not in omitted + assert "policy_id" not in omitted + assert given["spm"] == {"scenario": "zero_real"} + assert given["axes"] == [[{"name": "age", "count": 3, "min": 20, "max": 60}]] + + +def test_a_submission_answer_and_an_error_are_read(): + accepted = HouseholdCalculationAccepted.from_dict( + {"job_id": JOB_ID, "status": "pending"} + ) + refused = HouseholdCalculationError.from_dict( + { + "code": "modal_call_refused", + "detail": "Modal did not accept the call.", + "household_job_id": JOB_ID, + } + ) + + assert accepted.status is HouseholdJobStatus.PENDING + assert refused.to_dict()["household_job_id"] == JOB_ID + + +def test_a_job_keeps_its_result_document_whole(): + document = { + "status": "succeeded", + "schema_version": 1, + "calculation_id": JOB_ID, + "output": {"household": {"people": {"you": {"age": {"2026": 40}}}}}, + } + state = { + "job_id": JOB_ID, + "status": "succeeded", + "country": "us", + "household_id": HOUSEHOLD_ID, + "policy_id": None, + "year": 2026, + "policyengine_version": "6.17.1", + "country_package_version": "2.2.1", + "result": document, + "error_message": None, + "created_at": "2026-11-10T12:00:00Z", + "started_at": "2026-11-10T12:00:05Z", + "completed_at": "2026-11-10T12:00:30Z", + } + + parsed = HouseholdCalculationState.from_dict(state) + + assert parsed.status is HouseholdJobStatus.SUCCEEDED + assert parsed.to_dict()["result"] == document + + +# What the service answers to an unexpected failure (test_household_routes.py). +UNEXPECTED_FAILURE_ANSWER = { + "code": "internal_error", + "detail": "The service could not answer the request.", +} + + +def _client_answering(status_code: int, body: dict) -> AuthenticatedClient: + transport = httpx.MockTransport( + lambda request: httpx.Response(status_code, json=body) + ) + client = AuthenticatedClient(base_url="https://simulation.example", token="test") + client.set_httpx_client( + httpx.Client(base_url="https://simulation.example", transport=transport) + ) + return client + + +@pytest.mark.parametrize("route", ["submit", "read"]) +def test_a_500_is_read_as_a_household_calculation_error(route): + """Every 500 of the household routes is the JSON error the document declares.""" + client = _client_answering(500, UNEXPECTED_FAILURE_ANSWER) + + if route == "submit": + response = submit_household_calculation.sync_detailed( + client=client, + body=HouseholdCalculationRequest.from_dict( + {"country": "us", "household_id": HOUSEHOLD_ID} + ), + ) + else: + response = get_household_calculation.sync_detailed(UUID(JOB_ID), client=client) + + assert isinstance(response.parsed, HouseholdCalculationError) + assert response.parsed.to_dict() == UNEXPECTED_FAILURE_ANSWER diff --git a/projects/policyengine-simulation-entry/DEPLOYMENT.md b/projects/policyengine-simulation-entry/DEPLOYMENT.md index 59e00ecd2..e467e87b8 100644 --- a/projects/policyengine-simulation-entry/DEPLOYMENT.md +++ b/projects/policyengine-simulation-entry/DEPLOYMENT.md @@ -76,6 +76,46 @@ privilege missing fails at once. The stable and rollback checks don't require it, because an older image may not use the database. +## Household calculations + +`POST /household-calculations` and `GET /household-calculations/{id}` read +the Public API's household, policy and catalog rows and write +`household_jobs` through the simulation database above, and call the v2 +workers' household functions on Modal. Every deploy, whatever the Stage 12 +setting, gives the entrypoint: + +- `MODAL_TOKEN_ID` and `MODAL_TOKEN_SECRET`, the newest enabled versions of + the Secret Manager secrets the `beta` or `prod` variables + `MODAL_TOKEN_ID_SECRET_NAME` and `MODAL_TOKEN_SECRET_SECRET_NAME` name, + mounted once. The Modal client reads the token from those names, so the + Stage 12 comparison path, in the same process, uses the same mount. A + missing name stops the deploy. +- `SIMULATION_V2_MODAL_ENVIRONMENT`, the deploy's Modal environment + (`staging` or `main`), and `SIMULATION_V2_MANIFEST_NAME`, + `simulation-api-v2-version-manifest`: where the household routes read the + v2 version manifest and find the household functions. These are the + household path's own settings, apart from the `STAGE12_V2_MANIFEST_*` + settings the Stage 12 block sets. + +Without the database URL or the Modal environment the routes answer 503 +(`not_configured`). They use only the manifest's default worker, so a +release's household functions serve calculations once +`publish_stage12_v2_default` has moved the default to its worker. + +### The staging household check + +Every staging deployment runs `staging_household_check` after the v2 +default has moved (or, in a deployment without Stage 12, after the +authenticated tests): `.github/scripts/simulation-staging-household-check.sh` +calculates the stored staging household that the `beta` variable +`SIMULATION_STAGING_HOUSEHOLD_ID` names, through the candidate's tagged URL, +with the authenticated tests' client credentials, and requires its job to +succeed within ten minutes. `deployment_ready` requires the check to pass in +staging, so a failure stops the production deployment. The household must +be a US household stored in the staging Public API with a default year; each +run adds one `household_jobs` row for it. An empty or malformed variable +fails the check. + ## Candidate qualification For each eligible revision, the workflow first deploys `beta` diff --git a/projects/policyengine-simulation-entry/README.md b/projects/policyengine-simulation-entry/README.md index 76ff97afa..2e3a3f6ba 100644 --- a/projects/policyengine-simulation-entry/README.md +++ b/projects/policyengine-simulation-entry/README.md @@ -40,6 +40,8 @@ Auth0 permission: | `GET /budget-window-jobs/{batch_job_id}` | `simulation:read` | | `POST /internal/stage12/reports` | `stage12:operate` | | `GET /internal/stage12/reports/{evaluation_id}` | `stage12:operate` | +| `POST /household-calculations` | `simulation:submit` | +| `GET /household-calculations/{household_job_id}` | `simulation:read` | `GET /health`, `GET /ready`, `GET /versions`, `GET /versions/{kind}` and `POST /ping` stay public. The entrypoint answers `GET /health` and `POST /ping` @@ -66,3 +68,37 @@ Any other value stops the service from starting. When caller authentication is disabled (`SIMULATION_ENTRYPOINT_AUTH_REQUIRED=0`, which production refuses), there is no caller identity and every protected route is allowed. The Auth0 setup required before enforcement is described in `DEPLOYMENT.md`. + +## Household calculations + +`POST /household-calculations` names a stored household, and optionally a +stored policy, of the Public API: `country`, `household_id`, `policy_id`, +`add_missing_variables`, `axes` (within the household contract's limits) +and `spm` (SPM settings as an object; omit it to use the household's saved +selection, since null is refused). The service reads the rows itself, +calculates the household's `default_year` on the v2 version manifest's +default worker, creates a `household_jobs` row and starts the household +function's call with `spawn`, answering 202 with the job's `job_id` and +`status`. Each submission creates a new job. +`GET /household-calculations/{household_job_id}` returns the job's status, +the household function's `result` document once it has one, and its +`error_message`. + +A submission is refused before anything is written: with 400 and a `code` +when the household, policy or catalog version for the default worker's .py +version is missing, or the default worker has no household function for the +country; with 422, one reason per field, when the stored household has no +default year, the policy sets parameters that catalog version lacks, or the +household contract refuses the stored data. A job naming a row deleted +since the reads is refused by the database's foreign keys, also with 400. +The routes answer 503 with `Retry-After` when the manifest or the database +can't be read. When Modal refuses the call, or doesn't confirm it within ten +seconds, the job is failed at once and the 503 (`modal_call_refused`) names +it; a job the household function claimed meanwhile is answered with 202 as +it stands, and a job whose state couldn't then be recorded or read with 202 +`pending`, to be read rather than submitted again (the stuck-run sweep fails +it if no call ever starts). Every 500 of either route, from an unexpected +failure or a request the service built and the household contract refuses, +is the JSON error with code `internal_error`. `DEPLOYMENT.md` describes the deployment settings and the +staging check. + diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py index bf8d4f77f..2d8bbb804 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py @@ -41,6 +41,7 @@ from policyengine_simulation_observability.observability import ( init_simulation_observability, ) +from policyengine_simulation_persistence import HouseholdJobStore from starlette.datastructures import MutableHeaders from policyengine_simulation_entry.auth import ( @@ -64,6 +65,17 @@ route_template, set_observability_header, ) +from policyengine_simulation_entry.household_calculations import ( + HouseholdCalculationService, +) +from policyengine_simulation_entry.household_routes import ( + HouseholdCalculations, + add_household_calculation_routes, +) +from policyengine_simulation_entry.household_workers import ( + ManifestHouseholdWorkers, + ModalHouseholdFunctionCaller, +) from policyengine_simulation_entry.schemas import ( SimulationEntrypointReadinessResponse, TemporaryStage12ReportResponse, @@ -96,6 +108,8 @@ logger = logging.getLogger(__name__) SIMULATION_DATABASE_READINESS_TIMEOUT_SECONDS = 3.0 +# The database ends each statement of a household route that runs longer. +HOUSEHOLD_STATEMENT_TIMEOUT_MILLISECONDS = 5000 def _temporary_submission_payload( @@ -469,6 +483,46 @@ async def get_budget_window_job( ) +def _register_household_calculation_routes( + app: FastAPI, + entry: EntryRuntime, + calculations: HouseholdCalculations | None, + *, + submitter: AuthorizationDependency, + reader: AuthorizationDependency, +) -> None: + """Add the routes that submit household calculations and read their jobs.""" + add_household_calculation_routes( + app, entry, calculations, submitter=submitter, reader=reader + ) + + +def household_calculations_for( + settings: Settings, database: SimulationDatabaseConnection | None +) -> HouseholdCalculations | None: + """Build the household calculation service when the service is configured. + + It needs the simulation database and the v2 Modal environment; without + them the household routes answer 503. Nothing is read from Modal until + the first submission. + """ + if not settings.household_calculations_configured or not isinstance( + database, SimulationDatabase + ): + return None + return HouseholdCalculationService( + store=HouseholdJobStore.from_engine( + database.engine, + statement_timeout_milliseconds=HOUSEHOLD_STATEMENT_TIMEOUT_MILLISECONDS, + ), + workers=ManifestHouseholdWorkers.from_modal( + manifest_name=settings.v2_manifest_name, + environment=settings.v2_modal_environment, + ), + caller=ModalHouseholdFunctionCaller(settings.v2_modal_environment), + ) + + def _register_service_routes(app: FastAPI, entry: EntryRuntime) -> None: """Add the public version, health, readiness and ping routes.""" database_readiness = ( @@ -617,6 +671,13 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: submitter=simulation_submitter, reader=simulation_reader, ) + _register_household_calculation_routes( + app, + entry, + household_calculations_for(runtime_settings, runtime_database), + submitter=simulation_submitter, + reader=simulation_reader, + ) _register_service_routes(app, entry) return app diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/config.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/config.py index e83ead9a9..883b3c077 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/config.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/config.py @@ -97,6 +97,11 @@ class Settings: # API's tables. Empty outside deployed environments; nothing on the # old-gateway or Stage 12 paths reads it. simulation_database_url: str = "" + # The household routes' own v2 manifest settings, apart from the Stage 12 + # ones: the Modal environment that holds the v2 workers and their + # manifest, and the manifest's name. Empty outside deployed environments. + v2_modal_environment: str = "" + v2_manifest_name: str = V2_VERSION_MANIFEST_NAME @classmethod def from_env(cls) -> Settings: @@ -148,6 +153,10 @@ def from_env(cls) -> Settings: stage12_artifact_bucket=os.getenv("STAGE12_ARTIFACT_BUCKET", ""), stage12_retention_days=int(os.getenv("STAGE12_RETENTION_DAYS", "30")), simulation_database_url=os.getenv("SIMULATION_SERVICE_DATABASE_URL", ""), + v2_modal_environment=os.getenv("SIMULATION_V2_MODAL_ENVIRONMENT", ""), + v2_manifest_name=os.getenv( + "SIMULATION_V2_MANIFEST_NAME", V2_VERSION_MANIFEST_NAME + ), ) @property @@ -166,6 +175,11 @@ def stage12_resources_configured(self) -> bool: ) ) + @property + def household_calculations_configured(self) -> bool: + """Whether the database and the v2 Modal environment are both set.""" + return bool(self.simulation_database_url and self.v2_modal_environment) + def validate(self) -> None: """Raise ConfigurationError unless the service can start safely. @@ -177,6 +191,7 @@ def validate(self) -> None: self._validate_old_gateway() self._validate_stage12() self.validate_simulation_database() + self._validate_v2_manifest() def _validate_caller_auth(self) -> None: """Require caller authentication in production, with an HTTPS issuer.""" @@ -275,6 +290,16 @@ def _validate_stage12(self) -> None: msg = "STAGE12_DATABASE_URL must use the canonical postgresql:// scheme." raise ConfigurationError(msg) + def _validate_v2_manifest(self) -> None: + """Keep the household routes' v2 manifest apart from v1 routing state.""" + try: + assert_separate_manifest_names( + v1_name=V1_ROUTING_STATE_NAME, + v2_name=self.v2_manifest_name, + ) + except ValueError as error: + raise ConfigurationError(str(error)) from error + def validate_simulation_database(self) -> None: """Require a complete canonical URL for the service role, when set. diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_api_models.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_api_models.py new file mode 100644 index 000000000..36e4bf49b --- /dev/null +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_api_models.py @@ -0,0 +1,193 @@ +"""Request and response models of the household calculation routes. + +The Public API names a stored household, and optionally a stored policy, by +ID; the Simulation Entrypoint reads them from the Public API's tables itself +and calculates the household's ``default_year`` on the v2 version manifest's +default worker. Each submission creates a ``household_jobs`` row, which a +household function on Modal calculates; the read route returns it as it +stands. +""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any, Literal +from uuid import UUID + +from policyengine_simulation_contract.household_execution import Axes +from policyengine_simulation_contract.stage12_bundle import CountryId +from policyengine_simulation_persistence import HouseholdJobStatus +from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator +from pydantic.json_schema import SkipJsonSchema + +# Why a submission is refused (400): a row it names, or the household +# function it needs, does not exist. +HouseholdRefusalCode = Literal[ + # The country has no catalog version for the default worker's .py version. + "unknown_catalog_version", + "unknown_household", + "unknown_policy", + # The default worker has no household function for the country. + "no_household_function", + # The database refused the job: a row it names was deleted after the reads. + "missing_reference", +] +# Why a route cannot answer now (503); retry after ``Retry-After`` seconds. +HouseholdUnavailableCode = Literal[ + # The service has no database or v2 Modal environment configured. + "not_configured", + "manifest_unavailable", + "database_unavailable", + # Modal refused the household function's call, or did not accept it in + # time; the job was failed at once. + "modal_call_refused", +] +# The service failed unexpectedly, or built a household function request its +# contract refuses. +HouseholdInternalCode = Literal["internal_error"] +# Every code a household route's error answer carries. +HouseholdErrorCode = Literal[ + HouseholdRefusalCode, + HouseholdUnavailableCode, + HouseholdInternalCode, + "unknown_household_job", +] + + +def _omit_spm_default(schema: dict[str, Any]) -> None: + """Leave out the null default: omit ``spm`` to send no settings.""" + schema.pop("default", None) + + +class HouseholdCalculationRequest(BaseModel): + """A stored household to calculate under one stored policy, or current law. + + ``household_id`` and ``policy_id`` (omitted or null for current law) name + the Public API's rows of ``country``. The calculation is for the + household's ``default_year``, on the v2 version manifest's default worker. + ``add_missing_variables`` adds every yearly, monthly and eternal variable + the household lacks, as the Public API's ``/calculate-full`` does. + ``axes`` vary the stored household's inputs across points, within the + household contract's limits. + + ``spm`` holds SPM settings as an object, which the household function + checks with the Public API's SPM error codes. Omit it to use the + household's saved selection, or, without one, the bundle's certified + defaults; an explicit null is refused. + """ + + model_config = ConfigDict(extra="forbid", frozen=True, allow_inf_nan=False) + + country: CountryId + household_id: UUID + policy_id: UUID | None = None + add_missing_variables: bool = False + axes: Axes = None + spm: dict[str, JsonValue] | SkipJsonSchema[None] = Field( + default=None, + description=( + "SPM settings as the household contract takes them, as an object. " + "Omit to use the household's saved selection or the bundle's " + "defaults; null is refused." + ), + json_schema_extra=_omit_spm_default, + ) + + @field_validator("spm", mode="before") + @classmethod + def refuse_null_spm(cls, value: object) -> object: + """Refuse an explicit null; omit ``spm`` to send no settings.""" + if value is None: + msg = "spm must be an object when given; omit it to send no settings" + raise ValueError(msg) + return value + + @property + def spm_requested(self) -> bool: + """Whether the request sent SPM settings, rather than leaving them out.""" + return "spm" in self.model_fields_set + + +class HouseholdCalculationAccepted(BaseModel): + """The household job a submission created, and its status as it stands. + + Poll ``GET /household-calculations/{job_id}`` until the status is + ``succeeded`` or ``failed``. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + job_id: UUID + status: HouseholdJobStatus + + +class HouseholdCalculationState(BaseModel): + """A household job as it stands. + + ``result`` is the household function's outcome document once it has + finished, as the household contract defines it: a + ``HouseholdExecutionResult`` (``status`` ``succeeded``, with the + calculated household in ``output``) or a ``HouseholdExecutionFailure`` + (``status`` ``failed``, with the reason in ``error``). ``error_message`` + summarizes a failure in one line; a job that failed before its function + ran has an ``error_message`` and no ``result``. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + job_id: UUID + status: HouseholdJobStatus + country: str + household_id: UUID + policy_id: UUID | None + year: int + policyengine_version: str + country_package_version: str + result: dict[str, JsonValue] | None + error_message: str | None + created_at: datetime + started_at: datetime | None + completed_at: datetime | None + + +class HouseholdCalculationError(BaseModel): + """Why a household route refused a request or cannot answer now. + + ``household_job_id`` is set only with ``modal_call_refused``: it names + the job the service created and then failed, because Modal refused the + household function's call or didn't accept it in time. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + code: HouseholdErrorCode + detail: str + household_job_id: UUID | None = None + + +class HouseholdRequestReason(BaseModel): + """One reason a request cannot be calculated, as FastAPI writes a field error. + + ``loc`` starts with the request field the reason traces to. FastAPI's own + field errors may also carry ``input`` and ``ctx``. + """ + + model_config = ConfigDict(extra="allow", frozen=True) + + loc: list[str | int] + msg: str + type: str + + +class HouseholdRequestUnprocessable(BaseModel): + """A request that cannot be calculated (422), one reason per refused field. + + Request fields FastAPI refuses, and stored data that cannot be + calculated, share this shape: a household without a default year, a + policy setting parameters the catalog version lacks, or a household or + policy value the household contract refuses. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + detail: list[HouseholdRequestReason] diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_calculations.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_calculations.py new file mode 100644 index 000000000..bc085df3e --- /dev/null +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_calculations.py @@ -0,0 +1,564 @@ +"""Submit a household calculation as a household job, and read the job back. + +A submission names a stored household, and optionally a stored policy. The +service: + +1. finds the v2 version manifest's default worker's household function for + the country, and its bundle; +2. reads, in one transaction, the country's catalog version for that + worker's .py version, the stored household and the policy's reform + values, and builds the household contract's request for the household's + ``default_year``, with the new job's ID as its calculation ID; +3. creates the job ``pending`` with no Modal call ID; +4. starts the household function's call with ``spawn``, passing the job's + ID, then records the call's ID while the job is still pending with none + (the function may have claimed it already). When Modal refuses the call, + or does not confirm it in time, it fails the job at once instead, under + the same condition; a job the function claimed meanwhile, because Modal + accepted the call after all, is answered as it stands. + +Whatever is missing or cannot be calculated is refused in steps 1 and 2, +before anything is written. The household function then claims the job and +records its result or failure; the stuck-run sweep fails a pending job no +call was recorded for. +""" + +from __future__ import annotations + +import asyncio +import logging +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Protocol +from uuid import UUID, uuid4 + +from policyengine_simulation_contract.household_execution import ( + HouseholdExecutionRequest, +) +from policyengine_simulation_persistence import ( + HouseholdJobReferenceError, + HouseholdJobStatus, + MissingPolicyParametersError, + NewHouseholdJob, + UnknownCatalogVersionError, + UnknownHouseholdError, + UnknownPolicyError, +) +from pydantic import ValidationError +from sqlalchemy.exc import SQLAlchemyError + +from policyengine_simulation_entry.household_api_models import ( + HouseholdCalculationAccepted, +) +from policyengine_simulation_entry.household_contract_fields import ( + job_policy_fields, + request_field_of, +) +from policyengine_simulation_entry.household_conversion import ( + UnreadableHouseholdError, + household_situation, + saved_spm_selection, +) +from policyengine_simulation_entry.household_workers import ( + HouseholdFunctionUnavailableError, + VersionManifestUnavailableError, +) + +if TYPE_CHECKING: + from collections.abc import Callable + + from policyengine_simulation_persistence import ( + HouseholdCalculationInputs, + HouseholdJobRecord, + ) + + from policyengine_simulation_entry.household_api_models import ( + HouseholdCalculationRequest, + HouseholdRefusalCode, + HouseholdUnavailableCode, + ) + from policyengine_simulation_entry.household_workers import ( + HouseholdFunctionCaller, + HouseholdWorker, + HouseholdWorkerSource, + ) + +logger = logging.getLogger(__name__) + +# How long a submission waits for Modal to accept the household function's +# call before failing the job. +CALL_START_TIMEOUT_SECONDS = 10.0 +# The job's columns whose foreign keys can refuse it, and the row each names. +_REFERENCED_ROWS = { + "household_id": "household", + "policy_id": "policy", + "tax_benefit_model_version_id": "catalog version", +} + + +class HouseholdCalculationRefusedError(Exception): + """A row the submission names, or the household function it needs, is missing.""" + + def __init__(self, *, code: HouseholdRefusalCode, detail: str) -> None: + """Keep the reason's code and its description.""" + super().__init__(detail) + self.code: HouseholdRefusalCode = code + self.detail = detail + + +@dataclass(frozen=True) +class UnprocessableReason: + """One reason stored data cannot be calculated, as FastAPI writes a field error.""" + + loc: tuple[str | int, ...] + msg: str + type: str + + +class HouseholdCalculationUnprocessableError(Exception): + """The stored household or policy the submission names cannot be calculated.""" + + def __init__(self, reasons: tuple[UnprocessableReason, ...]) -> None: + """Keep every reason, in order.""" + super().__init__("; ".join(reason.msg for reason in reasons)) + self.reasons = reasons + + +class HouseholdCalculationInternalError(Exception): + """The service built a household function request its contract refuses. + + The message names each refused field the service built, never a value. + """ + + +class HouseholdCalculationUnavailableError(Exception): + """A dependency could not be read or refused the call; ask again later. + + ``household_job_id`` names the job that was failed, when one was created. + """ + + def __init__( + self, + *, + code: HouseholdUnavailableCode, + detail: str, + household_job_id: UUID | None = None, + ) -> None: + """Keep the reason's code, its description and any failed job.""" + super().__init__(detail) + self.code: HouseholdUnavailableCode = code + self.detail = detail + self.household_job_id = household_job_id + + +class HouseholdJobs(Protocol): + """The household job reads and writes (``HouseholdJobStore``).""" + + def read_inputs( + self, + *, + country: str, + household_id: UUID, + policy_id: UUID | None, + policyengine_version: str, + ) -> HouseholdCalculationInputs: + """Read the catalog version, household and policy together.""" + ... + + def create_job(self, job: NewHouseholdJob) -> None: + """Create the job pending with no call.""" + ... + + def record_call( + self, job_id: UUID, modal_call_id: str + ) -> HouseholdJobStatus | None: + """Record the call ID while the job is pending with none.""" + ... + + def fail_unstarted(self, job_id: UUID, error_message: str) -> bool: + """Fail the job while it is pending with no call ID.""" + ... + + def read(self, job_id: UUID) -> HouseholdJobRecord | None: + """Return the job as it stands.""" + ... + + +@dataclass(frozen=True) +class PreparedHouseholdJob: + """A household job ready to create, and the call that calculates it.""" + + job: NewHouseholdJob + payload: dict[str, Any] + + +def _database_unavailable() -> HouseholdCalculationUnavailableError: + return HouseholdCalculationUnavailableError( + code="database_unavailable", detail="The simulation database could not be read." + ) + + +def _contract_reasons( + error: ValidationError, request: HouseholdCalculationRequest +) -> tuple[UnprocessableReason, ...]: + """Name each field the contract refuses by the request field it traces to. + + Raise HouseholdCalculationInternalError when the contract refuses a field + the service builds itself, which is no fault of the caller's. + """ + reasons = [] + built_wrong = [] + for item in error.errors(): + location = tuple(item["loc"]) + request_field = request_field_of( + location, spm_from_request=request.spm_requested + ) + if request_field is None: + built_wrong.append(".".join(str(part) for part in location) or "request") + continue + reasons.append( + UnprocessableReason( + loc=("body", request_field, *location), + msg=item["msg"], + type=item["type"], + ) + ) + if built_wrong: + msg = ( + "the household function's request has fields the service built " + f"wrong: {', '.join(built_wrong)}" + ) + raise HouseholdCalculationInternalError(msg) + return tuple(reasons) + + +def _spm_fields( + request: HouseholdCalculationRequest, document: dict[str, Any] +) -> dict[str, object]: + """Return the contract's SPM fields: the request's, else the saved selection.""" + if request.spm_requested: + return {"spm": request.spm, "spm_selection_source": "request"} + saved = saved_spm_selection(document) + if saved is None: + return {} + return {"spm": saved, "spm_selection_source": "saved_household"} + + +def prepare_household_job( + request: HouseholdCalculationRequest, + worker: HouseholdWorker, + inputs: HouseholdCalculationInputs, + *, + job_id: UUID, +) -> PreparedHouseholdJob: + """Build the job and the household function's request from the stored rows. + + Raise HouseholdCalculationUnprocessableError for a household without a + default year or a document that cannot be read, and for anything the + household contract refuses. + """ + household = inputs.household + if household.default_year is None: + raise HouseholdCalculationUnprocessableError( + ( + UnprocessableReason( + loc=("body", "household_id"), + msg="the household has no default year to calculate", + type="household_without_default_year", + ), + ) + ) + try: + situation = household_situation(request.country, household.document) + except UnreadableHouseholdError as error: + raise HouseholdCalculationUnprocessableError( + ( + UnprocessableReason( + loc=("body", "household_id"), + msg=str(error), + type="household_unreadable", + ), + ) + ) from None + try: + contract_request = HouseholdExecutionRequest.model_validate( + { + "calculation_id": job_id, + "country": request.country, + # The household's default year, as a situation's period key. + "year": str(household.default_year), + "household": ( + situation + if request.axes is None + else {**situation, "axes": request.axes} + ), + **job_policy_fields(reform=inputs.reform), + "add_missing_variables": request.add_missing_variables, + "bundle": worker.bundle, + **_spm_fields(request, household.document), + } + ) + except ValidationError as error: + raise HouseholdCalculationUnprocessableError( + _contract_reasons(error, request) + ) from None + return PreparedHouseholdJob( + job=NewHouseholdJob( + id=job_id, + country=request.country, + household_id=request.household_id, + policy_id=request.policy_id, + year=household.default_year, + request_data=request.model_dump(mode="json", exclude_unset=True), + tax_benefit_model_version_id=inputs.catalog_version_id, + country_package_version=worker.bundle.country_package_version, + policyengine_version=worker.bundle.policyengine_version, + ), + payload=contract_request.model_dump(mode="json"), + ) + + +class HouseholdCalculationService: + """Submit household calculations as household jobs, and read the jobs back. + + The database and the manifest are read on threads, so the event loop + never waits on them. + """ + + def __init__( + self, + *, + store: HouseholdJobs, + workers: HouseholdWorkerSource, + caller: HouseholdFunctionCaller, + call_start_timeout_seconds: float = CALL_START_TIMEOUT_SECONDS, + new_job_id: Callable[[], UUID] = uuid4, + ) -> None: + """Use these seams; tests replace each of them.""" + self._store = store + self._workers = workers + self._caller = caller + self._call_start_timeout_seconds = call_start_timeout_seconds + self._new_job_id = new_job_id + + async def submit( + self, + request: HouseholdCalculationRequest, + *, + observability_context: dict[str, Any] | None, + ) -> HouseholdCalculationAccepted: + """Create the household job and start its call; return it as it stands. + + Raise HouseholdCalculationRefusedError (400) or + HouseholdCalculationUnprocessableError (422) before writing anything, + HouseholdCalculationInternalError (500) when the service builds a + request the contract refuses, and HouseholdCalculationUnavailableError + (503) when the manifest or database cannot be read, or Modal doesn't + confirm the call and the job was failed. + """ + worker = await self._default_worker(request.country) + prepared = await self._prepare(request, worker) + await self._create(prepared.job) + job_id = prepared.job.id + try: + call_id = await asyncio.wait_for( + self._caller.spawn( + worker, + prepared.payload, + household_job_id=job_id, + observability_context=observability_context, + ), + timeout=self._call_start_timeout_seconds, + ) + # Modal raises its own errors; any of them, or the timeout, means the + # call was not confirmed, and the job is failed unless it has started. + except Exception as error: # noqa: BLE001 # every unconfirmed call is answered alike + return await self._answer_unconfirmed_call(job_id, error) + status = await self._record_call(job_id, call_id) + return HouseholdCalculationAccepted(job_id=job_id, status=status) + + async def read(self, job_id: UUID) -> HouseholdJobRecord | None: + """Return the household job as it stands, or None when there is none. + + Raise HouseholdCalculationUnavailableError when it cannot be read. + """ + try: + return await asyncio.to_thread(self._store.read, job_id) + except SQLAlchemyError as error: + logger.warning( + "household_job_read_failed", + extra={"error_type": type(error).__name__}, + ) + raise _database_unavailable() from None + + async def _default_worker(self, country: str) -> HouseholdWorker: + try: + return await asyncio.to_thread(self._workers.default_worker, country) + except HouseholdFunctionUnavailableError as error: + raise HouseholdCalculationRefusedError( + code="no_household_function", detail=str(error) + ) from None + except VersionManifestUnavailableError as error: + logger.warning( + "household_manifest_unavailable", + extra={"error_type": type(error.__cause__).__name__}, + ) + raise HouseholdCalculationUnavailableError( + code="manifest_unavailable", + detail="The v2 version manifest could not be read.", + ) from None + + async def _prepare( + self, request: HouseholdCalculationRequest, worker: HouseholdWorker + ) -> PreparedHouseholdJob: + policyengine_version = worker.bundle.policyengine_version + try: + inputs = await asyncio.to_thread( + self._store.read_inputs, + country=request.country, + household_id=request.household_id, + policy_id=request.policy_id, + policyengine_version=policyengine_version, + ) + except UnknownCatalogVersionError as error: + raise HouseholdCalculationRefusedError( + code="unknown_catalog_version", + detail=f"{error}, the default v2 worker's version", + ) from None + except UnknownHouseholdError as error: + raise HouseholdCalculationRefusedError( + code="unknown_household", detail=str(error) + ) from None + except UnknownPolicyError as error: + raise HouseholdCalculationRefusedError( + code="unknown_policy", detail=str(error) + ) from None + except MissingPolicyParametersError as error: + raise HouseholdCalculationUnprocessableError( + tuple( + UnprocessableReason( + loc=("body", "policy_id", name), + msg=( + f"the catalog version for .py {policyengine_version} " + "has no such parameter" + ), + type="missing_parameter", + ) + for name in error.parameter_names + ) + ) from None + except SQLAlchemyError as error: + logger.warning( + "household_inputs_unreadable", + extra={"error_type": type(error).__name__}, + ) + raise _database_unavailable() from None + return prepare_household_job(request, worker, inputs, job_id=self._new_job_id()) + + async def _create(self, job: NewHouseholdJob) -> None: + try: + await asyncio.to_thread(self._store.create_job, job) + except HouseholdJobReferenceError as error: + referenced = { + "household_id": job.household_id, + "policy_id": job.policy_id, + "tax_benefit_model_version_id": job.tax_benefit_model_version_id, + } + column = error.column_name + detail = ( + f"the {_REFERENCED_ROWS[column]} {referenced[column]} no longer exists" + if column in _REFERENCED_ROWS + else "a row the household job names no longer exists" + ) + raise HouseholdCalculationRefusedError( + code="missing_reference", detail=detail + ) from None + except SQLAlchemyError as error: + logger.warning( + "household_job_not_created", + extra={"error_type": type(error).__name__}, + ) + raise _database_unavailable() from None + + async def _answer_unconfirmed_call( + self, job_id: UUID, error: BaseException + ) -> HouseholdCalculationAccepted: + """Fail the job of a call Modal didn't confirm, unless it has started. + + Modal may have accepted the call after the wait ended, and its + household function claimed the job first: the failure write then + changes nothing, and the job is answered as it stands. Raise + HouseholdCalculationUnavailableError ``modal_call_refused`` when the + job was failed. When the failure write, or reading the job back, + itself fails, the job is answered pending: it exists, so the caller + reads it rather than submitting again, and the stuck-run sweep fails + it if no call ever starts. + """ + # The log names the error type only; its text could repeat request data. + logger.error( + "household_call_unconfirmed", + extra={"household_job_id": str(job_id), "error_type": type(error).__name__}, + ) + message = ( + "Modal did not accept the household function's call " + f"({type(error).__name__})." + ) + try: + failed = await asyncio.to_thread( + self._store.fail_unstarted, job_id, message + ) + except SQLAlchemyError as write_error: + # The stuck-run sweep fails the job later if it stays pending. + logger.warning( + "household_job_not_failed", + extra={ + "household_job_id": str(job_id), + "error_type": type(write_error).__name__, + }, + ) + return HouseholdCalculationAccepted( + job_id=job_id, status=HouseholdJobStatus.PENDING + ) + if failed: + raise HouseholdCalculationUnavailableError( + code="modal_call_refused", + detail=( + "Modal did not accept the household function's call; the job " + "was failed." + ), + household_job_id=job_id, + ) + try: + record = await asyncio.to_thread(self._store.read, job_id) + except SQLAlchemyError as read_error: + logger.warning( + "household_job_not_read", + extra={ + "household_job_id": str(job_id), + "error_type": type(read_error).__name__, + }, + ) + else: + if record is not None: + return HouseholdCalculationAccepted(job_id=job_id, status=record.status) + return HouseholdCalculationAccepted( + job_id=job_id, status=HouseholdJobStatus.PENDING + ) + + async def _record_call(self, job_id: UUID, call_id: str) -> HouseholdJobStatus: + """Record the call's ID; when that fails, answer the job as created. + + The household function claims a pending job with no call ID itself, + so the calculation goes ahead either way. + """ + try: + status = await asyncio.to_thread(self._store.record_call, job_id, call_id) + except SQLAlchemyError as error: + logger.warning( + "household_call_not_recorded", + extra={ + "household_job_id": str(job_id), + "error_type": type(error).__name__, + }, + ) + return HouseholdJobStatus.PENDING + return HouseholdJobStatus.PENDING if status is None else status diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_contract_fields.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_contract_fields.py new file mode 100644 index 000000000..e3fe7aa1c --- /dev/null +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_contract_fields.py @@ -0,0 +1,56 @@ +"""How the household path's request maps onto the household contract's request. + +``policyengine_simulation_contract.household_execution`` defines the request +a household function takes. This module holds what the service depends on +about its fields: how the request names the job's one policy +(``job_policy_fields``), and which field of the service's own request each +of its fields comes from (``request_field_of``), so that a refusal names the +caller's field or, for a field the service builds itself, is an internal +error. A change to the contract's fields changes this module alone. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Final + +if TYPE_CHECKING: + from policyengine_simulation_persistence import PolicyReform + +# The request's top-level fields that hold the policy. +POLICY_FIELDS: Final = frozenset({"policy"}) +# The request field each of the contract request's top-level fields is built +# from, apart from the axes and SPM settings; the service builds the rest. +_REQUEST_FIELDS: Final = { + "household": "household_id", + "year": "household_id", + "country": "country", + "add_missing_variables": "add_missing_variables", + **dict.fromkeys(POLICY_FIELDS, "policy_id"), +} + + +def job_policy_fields(*, reform: PolicyReform) -> dict[str, object]: + """Return the request fields naming the job's one policy and its values. + + Current law changes no parameter, so its reform is empty. + """ + return {"policy": {"parameters": reform}} + + +def request_field_of( + location: tuple[str | int, ...], *, spm_from_request: bool +) -> str | None: + """Return the request field a refused field of the contract's request came from. + + The stored household, and its default year, come from ``household_id``, + except the request's own axes; the policy from ``policy_id``; SPM + settings from ``spm`` when the request sent them, else from the stored + household. Return None for a field the service builds itself, such as + the bundle or the calculation ID: its refusal is the service's fault. + """ + top_field = location[0] if location else None + if top_field == "household" and location[1:2] == ("axes",): + return "axes" + if top_field == "spm": + return "spm" if spm_from_request else "household_id" + return _REQUEST_FIELDS.get(str(top_field)) diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_conversion.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_conversion.py new file mode 100644 index 000000000..8e3122983 --- /dev/null +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_conversion.py @@ -0,0 +1,74 @@ +"""Stored households, in the situation shape the household functions take. + +A stored household (the Public API's ``households.household_data``) lists +its people and each group entity as records, each with an ``id`` and its +``values``; each person names, for every group collection, the ``id`` of the +group it belongs to. The household functions take the situation shape: +``people`` maps each person to their values, and each group collection, by +its plural, maps each group to its ``members`` and values. Records keep +their ``id`` as their name, so a calculated value maps back to its record. +The saved SPM selection is the document's ``spm``, kept apart from the +situation. Ported from sim #957. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Final + +if TYPE_CHECKING: + from collections.abc import Mapping + +# Each country's stored group collections and their plural in the situation, +# as the Public API's households service accepts them. +GROUP_COLLECTIONS: Final[dict[str, dict[str, str]]] = { + "us": { + "household": "households", + "family": "families", + "tax_unit": "tax_units", + "spm_unit": "spm_units", + "marital_unit": "marital_units", + }, + "uk": {"household": "households", "benunit": "benunits"}, +} + + +class UnreadableHouseholdError(ValueError): + """A stored household document lacks a field the Public API always stores.""" + + +def household_situation(country: str, document: Mapping[str, Any]) -> dict[str, Any]: + """Return a stored household's people and groups in the situation shape. + + A group no person belongs to keeps an empty member list, which the + household functions' contract refuses. Raise UnreadableHouseholdError + for a document without the records the Public API stores. + """ + try: + people = document["people"] + situation: dict[str, Any] = { + "people": {person["id"]: dict(person["values"]) for person in people} + } + for collection, plural in GROUP_COLLECTIONS[country].items(): + situation[plural] = { + group["id"]: { + "members": [ + person["id"] + for person in people + if person["memberships"].get(collection) == group["id"] + ], + **group["values"], + } + for group in document.get(collection, []) + } + except (KeyError, TypeError, AttributeError) as error: + msg = f"the stored household's people or groups cannot be read: {error!r}" + raise UnreadableHouseholdError(msg) from None + return situation + + +def saved_spm_selection(document: Mapping[str, Any]) -> object: + """Return the household's saved SPM selection, or None when it has none. + + A saved null selection is no choice, as the household contract reads it. + """ + return document.get("spm") diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_routes.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_routes.py new file mode 100644 index 000000000..7709bded3 --- /dev/null +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_routes.py @@ -0,0 +1,453 @@ +"""The household calculation routes: submit a calculation, then read its job. + +- ``POST /household-calculations`` (permission ``simulation:submit``) creates + a household job and starts its household function's call, answering 202 + with the job's ID and status. A 202 ``pending`` may also be a job whose + call Modal didn't confirm and whose state couldn't then be recorded or + read: poll the job rather than submitting again. +- ``GET /household-calculations/{household_job_id}`` (``simulation:read``) + returns the job as it stands: its status, the household function's result + document once it has one, and its error message. + +A refused submission answers 400 (a row it names, or the household function +it needs, is missing) or 422 (its fields, or the stored data it names, cannot +be calculated), naming why. Either route answers 503 with ``Retry-After`` +when household calculations are not configured or a dependency cannot be +read, and a submission does too when Modal refuses the call and its job was +failed. Every 500 of either route, an unexpected failure or a request the +service built and the household contract refuses, is the JSON error with +code ``internal_error``. + +A submission gets its observability ID from the caller's header, or a new +one, and passes it to the household function. ``household_jobs`` keeps no +observability ID, so a read returns none. +""" + +from __future__ import annotations + +import contextlib +import logging +import traceback +from typing import TYPE_CHECKING, Any, Protocol +from uuid import UUID + +from fastapi import APIRouter, Depends, Request, status +from fastapi.responses import JSONResponse + +from policyengine_simulation_entry.household_api_models import ( + HouseholdCalculationAccepted, + HouseholdCalculationError, + HouseholdCalculationRequest, + HouseholdCalculationState, + HouseholdRequestUnprocessable, +) +from policyengine_simulation_entry.household_calculations import ( + HouseholdCalculationInternalError, + HouseholdCalculationRefusedError, + HouseholdCalculationUnavailableError, + HouseholdCalculationUnprocessableError, +) + +if TYPE_CHECKING: + from fastapi import FastAPI + from policyengine_simulation_persistence import HouseholdJobRecord + + from policyengine_simulation_entry.auth import AuthorizationDependency + from policyengine_simulation_entry.entry_runtime import EntryRuntime + +logger = logging.getLogger(__name__) + +SUBMISSION_PATH = "/household-calculations" +JOB_PATH = "/household-calculations/{household_job_id}" +RETRY_AFTER_SECONDS = 10 + +_ERROR = {"model": HouseholdCalculationError} +_RETRY_AFTER = { + "Retry-After": { + "description": "Seconds to wait before asking again", + "schema": {"type": "integer"}, + } +} +_SUBMISSION_UNPROCESSABLE = { + "description": ( + "The request's fields, or the stored household or policy it names, " + "cannot be calculated: one reason per field, its location starting " + "with the request field it traces to" + ), + "model": HouseholdRequestUnprocessable, +} +_SUBMISSION_UNAVAILABLE = { + "description": ( + "Household calculations are not configured; the v2 version manifest " + "or the database cannot be read; or Modal refused the household " + "function's call, or didn't accept it in time, and the job named by " + "household_job_id was failed (modal_call_refused). Retry after " + "Retry-After seconds" + ), + "headers": _RETRY_AFTER, + **_ERROR, +} +_SUBMISSION_INTERNAL_ERROR = { + "description": ( + "The service failed unexpectedly, or built a household function " + "request the household contract refuses (internal_error)" + ), + **_ERROR, +} +_READ_INTERNAL_ERROR = { + "description": "The service failed unexpectedly (internal_error)", + **_ERROR, +} +_READ_UNPROCESSABLE = { + "description": "The household job ID is not a UUID", + "model": HouseholdRequestUnprocessable, +} +_READ_UNAVAILABLE = { + "description": ( + "Household calculations are not configured, or the database cannot " + "be read; retry after Retry-After seconds" + ), + "headers": _RETRY_AFTER, + **_ERROR, +} +_FORBIDDEN = { + "description": "The caller's token is missing, invalid or lacks the permission" +} + + +class HouseholdCalculations(Protocol): + """Submit and read household calculations (``HouseholdCalculationService``).""" + + async def submit( + self, + request: HouseholdCalculationRequest, + *, + observability_context: dict[str, Any] | None, + ) -> HouseholdCalculationAccepted: + """Create the household job and start its call; return it as it stands. + + A ``pending`` job may be one whose call Modal didn't confirm. + """ + ... + + async def read(self, job_id: UUID) -> HouseholdJobRecord | None: + """Return the household job as it stands, or None.""" + ... + + +def _error_response(status_code: int, error: HouseholdCalculationError) -> JSONResponse: + headers = ( + {"Retry-After": str(RETRY_AFTER_SECONDS)} + if status_code == status.HTTP_503_SERVICE_UNAVAILABLE + else None + ) + return JSONResponse( + error.model_dump(mode="json", exclude_none=True), + status_code=status_code, + headers=headers, + ) + + +def _not_configured() -> JSONResponse: + return _error_response( + status.HTTP_503_SERVICE_UNAVAILABLE, + HouseholdCalculationError( + code="not_configured", + detail="Household calculations are not configured.", + ), + ) + + +def _unavailable(error: HouseholdCalculationUnavailableError) -> JSONResponse: + return _error_response( + status.HTTP_503_SERVICE_UNAVAILABLE, + HouseholdCalculationError( + code=error.code, + detail=error.detail, + household_job_id=error.household_job_id, + ), + ) + + +_SUBMISSION_FAILURES = ( + HouseholdCalculationRefusedError, + HouseholdCalculationUnprocessableError, + HouseholdCalculationInternalError, + HouseholdCalculationUnavailableError, +) + + +def _submission_failure_response(error: Exception) -> JSONResponse: + """Answer a submission the service refused or could not serve.""" + if isinstance(error, HouseholdCalculationRefusedError): + return _error_response( + status.HTTP_400_BAD_REQUEST, + HouseholdCalculationError(code=error.code, detail=error.detail), + ) + if isinstance(error, HouseholdCalculationUnprocessableError): + return JSONResponse( + { + "detail": [ + {"loc": list(reason.loc), "msg": reason.msg, "type": reason.type} + for reason in error.reasons + ] + }, + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + ) + if isinstance(error, HouseholdCalculationUnavailableError): + return _unavailable(error) + # The message names the refused fields the service built, never a value. + logger.error("household_request_built_wrong: %s", error) + return _error_response( + status.HTTP_500_INTERNAL_SERVER_ERROR, + HouseholdCalculationError( + code="internal_error", + detail="The service could not build the household function's request.", + ), + ) + + +# Separates the frames in the 500 log's ``stack_frames`` attribute. +STACK_FRAME_SEPARATOR = "; " +# The service's own packages, whose frames the 500 log keeps before others. +SERVICE_PACKAGES = frozenset( + {"policyengine_simulation_entry", "policyengine_simulation_persistence"} +) +# The directories a package's files sit under: an installed package's, then +# a checkout's. A frame's path is shortened to its part after one of them. +_PACKAGE_PARENT_DIRECTORIES = ("/site-packages/", "/src/") + + +def _package_path(file_path: str) -> str: + """Return ``file_path`` from its package's directory on. + + That is its part after the last ``site-packages/``, where the image + installs the service and its libraries, or else after the last ``src/``, + where a checkout keeps the service's packages. Any other path is + returned whole. + """ + for parent_directory in _PACKAGE_PARENT_DIRECTORIES: + _, found, package_path = file_path.rpartition(parent_directory) + if found: + return package_path + return file_path + + +def _stack_frames(error: Exception, max_characters: int) -> tuple[str, int]: + """Return the error's frames that fit ``max_characters``, and how many don't. + + Each frame is ``path:line in function``, with its path from its + package's directory on (``_package_path``). The kept frames are written + innermost first, from where the error was raised out to the route's + handler. When they don't all fit, frames are taken in this order until + the next one would pass ``max_characters``: the frame that raised the + error, then the service's own frames (``SERVICE_PACKAGES``) from the + innermost out, then the others from the innermost out. So other + libraries' outer frames are left out first, and left-out frames may sit + between two kept ones. Staying within ``max_characters`` keeps the + string whole: the observability runtime keeps only the start of a longer + string attribute. + """ + frames: list[str] = [] + from_the_service: list[bool] = [] + for frame in reversed(traceback.extract_tb(error.__traceback__)): + package_path = _package_path(frame.filename) + frames.append(f"{package_path}:{frame.lineno} in {frame.name}") + from_the_service.append(package_path.split("/", 1)[0] in SERVICE_PACKAGES) + positions_by_priority = sorted( + range(len(frames)), + key=lambda position: (position > 0, not from_the_service[position], position), + ) + kept_positions: list[int] = [] + length = 0 + for position in positions_by_priority: + added = len(frames[position]) + ( + len(STACK_FRAME_SEPARATOR) if kept_positions else 0 + ) + if length + added > max_characters: + break + kept_positions.append(position) + length += added + kept = [frames[position] for position in sorted(kept_positions)] + return STACK_FRAME_SEPARATOR.join(kept), len(frames) - len(kept) + + +def _unexpected_failure_response( + route: str, error: Exception, *, max_characters: int +) -> JSONResponse: + """Answer an unexpected failure with the JSON 500 the routes declare. + + The log is the fixed event ``household_route_failed`` with the route, the + error's type, its stack frames (path, line and function; see + ``_stack_frames``) as one string of at most ``max_characters``, the + observability runtime's string limit, and the number of frames left out. + It never carries the error's message or any value, which could repeat + request or stored data. The fields go in ``policyengine_attributes`` too, + the only extra fields the observability runtime's log handler keeps, and + each is a string or an integer: the runtime keeps only strings, numbers + and booleans. + """ + stack_frames, omitted_frame_count = _stack_frames(error, max_characters) + fields = { + "route": route, + "error_type": type(error).__name__, + "stack_frames": stack_frames, + "stack_frames_omitted": omitted_frame_count, + } + logger.error( + "household_route_failed", + extra={**fields, "policyengine_attributes": fields}, + ) + return _error_response( + status.HTTP_500_INTERNAL_SERVER_ERROR, + HouseholdCalculationError( + code="internal_error", detail="The service could not answer the request." + ), + ) + + +def _observability_context( + entry: EntryRuntime, observability_id: str +) -> dict[str, Any]: + """Capture the trace context for the household function, with the ID. + + Telemetry is non-fatal: without a captured context, only the ID is sent. + """ + captured: object = None + with contextlib.suppress(Exception): + captured = entry.runtime.capture_context() + context = dict(captured) if isinstance(captured, dict) else {} + context["observability_id"] = observability_id + return context + + +def _state(record: HouseholdJobRecord) -> HouseholdCalculationState: + return HouseholdCalculationState( + job_id=record.id, + status=record.status, + country=record.country, + household_id=record.household_id, + policy_id=record.policy_id, + year=record.year, + policyengine_version=record.policyengine_version, + country_package_version=record.country_package_version, + result=record.result, + error_message=record.error_message, + created_at=record.created_at, + started_at=record.started_at, + completed_at=record.completed_at, + ) + + +def add_household_calculation_routes( + app: FastAPI, + entry: EntryRuntime, + calculations: HouseholdCalculations | None, + *, + submitter: AuthorizationDependency, + reader: AuthorizationDependency, +) -> None: + """Add the household routes; without a service they answer 503.""" + router = APIRouter() + log_string_limit = entry.runtime.config.limits.max_string_length + + @router.post( + SUBMISSION_PATH, + summary="Submit a household calculation", + description=( + "Create a household job calculating a stored household under one " + "stored policy, or current law, for the household's default year, " + "on the v2 version manifest's default worker, and start it. Poll " + "the returned id until its status is succeeded or failed." + ), + operation_id="submit_household_calculation", + status_code=status.HTTP_202_ACCEPTED, + response_model=HouseholdCalculationAccepted, + responses={ + 202: { + "description": ( + "The job was created and its call started; or Modal didn't " + "confirm the call and the job's state couldn't be recorded " + "or read, and it is answered pending: poll it rather than " + "submitting again" + ) + }, + 400: { + "description": ( + "A row the request names, or the household function it " + "needs, is missing; code says which" + ), + **_ERROR, + }, + 403: _FORBIDDEN, + 422: _SUBMISSION_UNPROCESSABLE, + 500: _SUBMISSION_INTERNAL_ERROR, + 503: _SUBMISSION_UNAVAILABLE, + }, + dependencies=[Depends(submitter)], + ) + async def submit_household_calculation( + body: HouseholdCalculationRequest, request: Request + ) -> JSONResponse: + if calculations is None: + return _not_configured() + observability_id = entry.start_workflow_observability(request) + try: + accepted = await calculations.submit( + body, + observability_context=_observability_context(entry, observability_id), + ) + except _SUBMISSION_FAILURES as error: + return _submission_failure_response(error) + # Every other failure is still answered as the JSON 500 declared. + except Exception as error: # noqa: BLE001 # answered as internal_error + return _unexpected_failure_response( + SUBMISSION_PATH, error, max_characters=log_string_limit + ) + return JSONResponse( + accepted.model_dump(mode="json"), status_code=status.HTTP_202_ACCEPTED + ) + + @router.get( + JOB_PATH, + summary="Get a household calculation", + description=( + "Return the household job as it stands: its status, the household " + "function's result document once it has one, and its error message." + ), + operation_id="get_household_calculation", + response_model=HouseholdCalculationState, + responses={ + 200: {"description": "The job as it stands"}, + 403: _FORBIDDEN, + 404: {"description": "There is no such household job", **_ERROR}, + 422: _READ_UNPROCESSABLE, + 500: _READ_INTERNAL_ERROR, + 503: _READ_UNAVAILABLE, + }, + dependencies=[Depends(reader)], + ) + async def get_household_calculation(household_job_id: UUID) -> JSONResponse: + if calculations is None: + return _not_configured() + try: + record = await calculations.read(household_job_id) + state = None if record is None else _state(record).model_dump(mode="json") + except HouseholdCalculationUnavailableError as error: + return _unavailable(error) + # Every other failure is still answered as the JSON 500 declared. + except Exception as error: # noqa: BLE001 # answered as internal_error + return _unexpected_failure_response( + JOB_PATH, error, max_characters=log_string_limit + ) + if state is None: + return _error_response( + status.HTTP_404_NOT_FOUND, + HouseholdCalculationError( + code="unknown_household_job", + detail=f"There is no household job {household_job_id}.", + ), + ) + return JSONResponse(state) + + app.include_router(router) diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_workers.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_workers.py new file mode 100644 index 000000000..bf08e7668 --- /dev/null +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_workers.py @@ -0,0 +1,173 @@ +"""The default v2 worker's household functions, and starting their calls. + +The household routes use only the v2 version manifest's default worker: the +Modal application deployed for one .py version, which names each country's +household function and the bundle it runs. A calculation's request carries +that bundle, and the household function refuses any other. The manifest is a +Modal Dict, read through the contract's ``V2ManifestLoader``; the function is +called by name with ``spawn``, so the route waits only for Modal to accept +the call. + +This is the household path's own Modal access, apart from the Stage 12 +comparison path's. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Protocol + +import modal +from policyengine_simulation_contract.household_execution import ( + HouseholdBundleProvenance, +) +from policyengine_simulation_contract.stage12_manifest import V2ManifestLoader + +if TYPE_CHECKING: + from uuid import UUID + + from policyengine_simulation_contract.stage12_manifest import ( + V2ManifestStore, + V2VersionManifest, + ) + + +class HouseholdFunctionUnavailableError(LookupError): + """The default v2 worker has no household function for the country.""" + + +class VersionManifestUnavailableError(RuntimeError): + """The v2 version manifest could not be read; ask again later.""" + + +@dataclass(frozen=True) +class HouseholdWorker: + """A country's household function in the default worker, and its bundle.""" + + application_name: str + function_name: str + bundle: HouseholdBundleProvenance + + +class HouseholdWorkerSource(Protocol): + """Find the default worker's household function (``ManifestHouseholdWorkers``).""" + + def default_worker(self, country: str) -> HouseholdWorker: + """Return the country's household function, or raise why there is none.""" + ... + + +class HouseholdFunctionCaller(Protocol): + """Start one household function call (``ModalHouseholdFunctionCaller``).""" + + async def spawn( + self, + worker: HouseholdWorker, + payload: dict[str, Any], + *, + household_job_id: UUID, + observability_context: dict[str, Any] | None, + ) -> str: + """Start the call without waiting for it and return its Modal call ID.""" + ... + + +def default_household_worker( + manifest: V2VersionManifest, country: str +) -> HouseholdWorker: + """Return the manifest's default worker's household function for ``country``. + + Raise HouseholdFunctionUnavailableError when it has none. + """ + version = manifest.default_version + worker = manifest.versions[version] + country_worker = worker.household_worker(country) + if country_worker is None or country_worker.household_callable is None: + msg = ( + f"the default v2 worker, .py {version}, has no household function " + f"for {country}" + ) + raise HouseholdFunctionUnavailableError(msg) + return HouseholdWorker( + application_name=worker.application_name, + function_name=country_worker.household_callable, + bundle=HouseholdBundleProvenance.from_bundle( + worker.bundle, + bundle_manifest_sha256=worker.bundle_manifest_sha256, + country=country, + ), + ) + + +class ManifestHouseholdWorkers: + """Find household functions in the v2 version manifest's default worker. + + Each lookup reads the manifest, which is small, so a moved default is + used at once. The loader keeps its last valid copy for a stored value + that does not validate; a store that cannot be read is unavailable. + """ + + def __init__(self, store: V2ManifestStore) -> None: + """Read the manifest from ``store``; nothing is read until a lookup.""" + self._loader = V2ManifestLoader(store) + + @classmethod + def from_modal( + cls, *, manifest_name: str, environment: str + ) -> ManifestHouseholdWorkers: + """Read the manifest from its Modal Dict, which must already exist.""" + return cls( + modal.Dict.from_name( + manifest_name, environment_name=environment, create_if_missing=False + ) + ) + + def default_worker(self, country: str) -> HouseholdWorker: + """Return the default worker's household function for ``country``. + + Raise HouseholdFunctionUnavailableError when it has none, and + VersionManifestUnavailableError when the manifest cannot be read. + """ + try: + manifest = self._loader.load() + # Modal raises its own errors, and any of them means the same here. + except Exception as error: + msg = "the v2 version manifest could not be read" + raise VersionManifestUnavailableError(msg) from error + return default_household_worker(manifest, country) + + +class ModalHouseholdFunctionCaller: + """Start household function calls on Modal, by application and function name.""" + + def __init__(self, environment: str) -> None: + """Look up household functions in this Modal environment.""" + self._environment = environment + + async def spawn( + self, + worker: HouseholdWorker, + payload: dict[str, Any], + *, + household_job_id: UUID, + observability_context: dict[str, Any] | None, + ) -> str: + """Start the household function's call and return its Modal call ID. + + Raise RuntimeError when Modal returns no call ID. + """ + function = modal.Function.from_name( + worker.application_name, + worker.function_name, + environment_name=self._environment, + ) + call = await function.spawn.aio( + payload, + household_job_id=str(household_job_id), + observability_context=observability_context, + ) + call_id = getattr(call, "object_id", None) + if not isinstance(call_id, str) or not call_id: + msg = "Modal returned no call ID for the household function's call" + raise RuntimeError(msg) + return call_id diff --git a/projects/policyengine-simulation-entry/tests/household_fixtures.py b/projects/policyengine-simulation-entry/tests/household_fixtures.py new file mode 100644 index 000000000..7b6b823fa --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/household_fixtures.py @@ -0,0 +1,127 @@ +"""Test values for the household calculation routes. + +The household path keeps its own fixtures, apart from the Stage 12 +comparison path's, so neither depends on the other's. +""" + +from __future__ import annotations + +from datetime import UTC, datetime + +from policyengine_simulation_contract.stage12_bundle import ( + Stage12BundleManifest, + Stage12CountryBundle, + Stage12Dataset, +) +from policyengine_simulation_contract.stage12_manifest import ( + V2CountryWorker, + V2WorkerValidation, + V2WorkerVersion, + add_v2_worker, + read_v2_manifest, + v2_application_name, + write_v2_manifest, +) + +POLICYENGINE_VERSION = "6.17.1" +CORE_PACKAGE_VERSION = "3.33.3" +COUNTRY_PACKAGE_VERSIONS = {"us": "2.2.1", "uk": "2.174.1"} +BUNDLE_MANIFEST_SHA256 = "b" * 64 + + +def _country_bundle(country: str) -> Stage12CountryBundle: + package = f"policyengine-{country}" + package_version = COUNTRY_PACKAGE_VERSIONS[country] + dataset = f"populace_{country}_2024" + revision = f"{dataset}-revision" + return Stage12CountryBundle( + country=country, + country_package_name=package, + country_package_version=package_version, + country_package_requirement=f"{package}=={package_version}", + data_package_name="populace-data", + data_package_version="0.1.0", + data_release_version=revision, + data_artifact_revision=revision, + default_dataset=dataset, + default_dataset_uri=f"hf://policyengine/data/{dataset}.h5@{revision}", + datasets=( + Stage12Dataset( + identity=dataset, + uri=f"hf://policyengine/data/{dataset}.h5@{revision}", + artifact_revision=revision, + sha256="a" * 64, + repo_type="dataset", + ), + ), + ) + + +def household_worker_version( + policyengine_version: str = POLICYENGINE_VERSION, + *, + household_countries: tuple[str, ...] = ("us", "uk"), +) -> V2WorkerVersion: + """Return a validated v2 worker for the US and the UK. + + Each country runs economy reports; those in ``household_countries`` also + have a household function. + """ + country_workers = tuple( + V2CountryWorker( + country=country, + economy_capable=True, + single_simulation_callable=f"run_single_simulation_{country}", + household_callable=( + f"calculate_household_{country}" + if country in household_countries + else None + ), + ) + for country in ("us", "uk") + ) + application_name = v2_application_name(policyengine_version) + return V2WorkerVersion( + application_name=application_name, + report_coordinator_callable="coordinate_report", + countries=country_workers, + bundle=Stage12BundleManifest( + policyengine_version=policyengine_version, + policyengine_requirement=f"policyengine=={policyengine_version}", + core_package_version=CORE_PACKAGE_VERSION, + core_package_requirement=f"policyengine-core=={CORE_PACKAGE_VERSION}", + countries=(_country_bundle("us"), _country_bundle("uk")), + ), + bundle_manifest_sha256=BUNDLE_MANIFEST_SHA256, + validation=V2WorkerValidation( + validated=True, + application_name=application_name, + bundle_manifest_sha256=BUNDLE_MANIFEST_SHA256, + validated_at=datetime(2026, 11, 10, tzinfo=UTC), + validation_invocation_id="validation-1", + country_validation_invocation_ids={ + country: f"validation-{country}" for country in ("us", "uk") + }, + household_validation_invocation_ids={ + country: f"household-validation-{country}" + for country in household_countries + }, + ), + ) + + +def manifest_store( + default_worker: V2WorkerVersion, *other_workers: V2WorkerVersion +) -> dict[str, object]: + """Return a manifest store whose default is ``default_worker``. + + The other workers are added after it without moving the default, as a + deployment adds its worker before its checks. + """ + store: dict[str, object] = {} + for worker in (default_worker, *other_workers): + write_v2_manifest( + store=store, + manifest=add_v2_worker(current=read_v2_manifest(store), worker=worker), + ) + return store diff --git a/projects/policyengine-simulation-entry/tests/test_auth.py b/projects/policyengine-simulation-entry/tests/test_auth.py index 2122399a0..893ca2648 100644 --- a/projects/policyengine-simulation-entry/tests/test_auth.py +++ b/projects/policyengine-simulation-entry/tests/test_auth.py @@ -202,6 +202,13 @@ def test_malformed_token_returns_403(monkeypatch, signing_keys): "window_size": 1, } STAGE12_REPORT_PATH = "/internal/stage12/reports/00000000-0000-0000-0000-000000000001" +HOUSEHOLD_CALCULATION_PAYLOAD = { + "country": "us", + "household_id": "00000000-0000-4000-8000-0000000000bb", +} +HOUSEHOLD_CALCULATION_PATH = ( + "/household-calculations/00000000-0000-4000-8000-0000000000aa" +) class ProtectedRoute(NamedTuple): @@ -215,8 +222,9 @@ class ProtectedRoute(NamedTuple): # Each protected route, the permission it requires, and the status its handler -# returns once authorized. Stage 12 is disabled and unconfigured in these -# tests, so an authorized Stage 12 request reaches the handler's 503 response. +# returns once authorized. Stage 12 and household calculations are +# unconfigured in these tests, so an authorized request to either reaches the +# handler's 503 response. PROTECTED_ROUTES = [ pytest.param( ProtectedRoute( @@ -278,6 +286,26 @@ class ProtectedRoute(NamedTuple): ), id="stage12-status", ), + pytest.param( + ProtectedRoute( + "post", + "/household-calculations", + HOUSEHOLD_CALCULATION_PAYLOAD, + "simulation:submit", + 503, + ), + id="submit-household-calculation", + ), + pytest.param( + ProtectedRoute( + "get", + HOUSEHOLD_CALCULATION_PATH, + None, + "simulation:read", + 503, + ), + id="read-household-calculation", + ), ] diff --git a/projects/policyengine-simulation-entry/tests/test_config.py b/projects/policyengine-simulation-entry/tests/test_config.py index 17d06d1bc..5546a4668 100644 --- a/projects/policyengine-simulation-entry/tests/test_config.py +++ b/projects/policyengine-simulation-entry/tests/test_config.py @@ -354,3 +354,59 @@ def test_the_simulation_database_url_must_be_canonical(database_url): make_settings(simulation_database_url=database_url).validate() assert "secret" not in str(error.value) + + +DATABASE_URL = "postgresql://simulation_service:secret@db.example:5432/postgres" + + +def test_the_v2_manifest_settings_are_read_from_the_environment(monkeypatch): + from policyengine_simulation_entry.config import Settings + + monkeypatch.setenv("SIMULATION_V2_MODAL_ENVIRONMENT", "staging") + monkeypatch.setenv("SIMULATION_V2_MANIFEST_NAME", "another-v2-manifest") + settings = Settings.from_env() + + assert settings.v2_modal_environment == "staging" + assert settings.v2_manifest_name == "another-v2-manifest" + + +def test_the_v2_manifest_is_the_published_one_unless_named(monkeypatch): + from policyengine_simulation_contract.stage12_manifest import ( + V2_VERSION_MANIFEST_NAME, + ) + from policyengine_simulation_entry.config import Settings + + monkeypatch.delenv("SIMULATION_V2_MODAL_ENVIRONMENT", raising=False) + monkeypatch.delenv("SIMULATION_V2_MANIFEST_NAME", raising=False) + settings = Settings.from_env() + + assert settings.v2_modal_environment == "" + assert settings.v2_manifest_name == V2_VERSION_MANIFEST_NAME + + +def test_household_calculations_need_the_database_and_the_v2_modal_environment(): + assert not make_settings().household_calculations_configured + assert not make_settings( + simulation_database_url=DATABASE_URL + ).household_calculations_configured + assert not make_settings( + v2_modal_environment="staging" + ).household_calculations_configured + assert make_settings( + simulation_database_url=DATABASE_URL, v2_modal_environment="staging" + ).household_calculations_configured + + +def test_the_v2_manifest_must_not_reuse_the_v1_storage_name(): + with pytest.raises(ConfigurationError, match="different storage names"): + make_settings(v2_manifest_name="simulation-api-routing-state").validate() + + +def test_the_v2_manifest_settings_configure_no_stage12_resource(): + """The household path has settings of its own; Stage 12 stays all or nothing.""" + settings = make_settings( + simulation_database_url=DATABASE_URL, v2_modal_environment="staging" + ) + + settings.validate() + assert not settings.stage12_resources_configured diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py index 6e4747e7f..b46faf65d 100644 --- a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py @@ -213,11 +213,12 @@ def _assert_beta_then_prod_release( def _assert_deployment_environment_settings(reusable_workflow: str) -> None: assert ( - # Every job but stage12_v2_ready declares the release environment; - # the job that moves latest after the checks reads the prod - # environment's custom-hostname variable, and the job that moves the - # Stage 12 v2 default after them is the seventeenth. - reusable_workflow.count("environment: ${{ inputs.release_environment }}") == 17 + # Every job but stage12_v2_ready declares the release environment. + # The job that moves latest after the checks reads the prod + # environment's custom-hostname variable, and the staging household + # check reads the beta variable SIMULATION_STAGING_HOUSEHOLD_ID and + # the environment's auth secrets. + reusable_workflow.count("environment: ${{ inputs.release_environment }}") == 18 ) assert "APP_ENVIRONMENT: ${{ inputs.deployment_environment }}" in reusable_workflow assert "STAGE12_ENABLED_VALUE: ${{ vars.STAGE12_ENABLED }}" in reusable_workflow diff --git a/projects/policyengine-simulation-entry/tests/test_household_calculations.py b/projects/policyengine-simulation-entry/tests/test_household_calculations.py new file mode 100644 index 000000000..dd7aa455c --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_household_calculations.py @@ -0,0 +1,804 @@ +"""Submitting and reading household calculations, with the database and Modal faked. + +The store's own reads and writes are tested against PostgreSQL in the +persistence library; here a fake store records what the service asks of it. +""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, field +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any +from uuid import UUID, uuid4 + +import pytest +from household_fixtures import household_worker_version +from policyengine_simulation_contract.household_execution import ( + HouseholdExecutionRequest, +) +from policyengine_simulation_contract.stage12_manifest import V2VersionManifest +from policyengine_simulation_entry.household_api_models import ( + HouseholdCalculationRequest, +) +from policyengine_simulation_entry.household_calculations import ( + HouseholdCalculationInternalError, + HouseholdCalculationRefusedError, + HouseholdCalculationService, + HouseholdCalculationUnavailableError, + HouseholdCalculationUnprocessableError, + prepare_household_job, +) +from policyengine_simulation_entry.household_workers import ( + HouseholdFunctionUnavailableError, + HouseholdWorker, + VersionManifestUnavailableError, + default_household_worker, +) +from policyengine_simulation_persistence import ( + HouseholdCalculationInputs, + HouseholdJobRecord, + HouseholdJobReferenceError, + HouseholdJobStatus, + MissingPolicyParametersError, + NewHouseholdJob, + StoredHousehold, + UnknownCatalogVersionError, + UnknownHouseholdError, + UnknownPolicyError, +) +from pydantic import ValidationError +from sqlalchemy.exc import OperationalError + +if TYPE_CHECKING: + from collections.abc import Callable + +JOB_ID = UUID("00000000-0000-4000-8000-0000000000aa") +HOUSEHOLD_ID = UUID("00000000-0000-4000-8000-0000000000bb") +POLICY_ID = UUID("00000000-0000-4000-8000-0000000000cc") +CATALOG_VERSION_ID = UUID("00000000-0000-4000-8000-0000000000dd") +CTC_AMOUNT = "gov.irs.credits.ctc.amount.base[0].amount" +REFORM = {CTC_AMOUNT: {"2026-01-01.2200-12-31": 3_000}} +STORED_DOCUMENT = { + "people": [ + { + "id": "you", + "values": {"age": 40, "employment_income": {"2026": 30_000}}, + "memberships": { + "household": "your household", + "family": "your family", + "tax_unit": "your tax unit", + "spm_unit": "your spm unit", + "marital_unit": "your marital unit", + }, + } + ], + "household": [{"id": "your household", "values": {"household_net_income": None}}], + "family": [{"id": "your family", "values": {}}], + "tax_unit": [{"id": "your tax unit", "values": {}}], + "spm_unit": [{"id": "your spm unit", "values": {}}], + "marital_unit": [{"id": "your marital unit", "values": {}}], +} +SITUATION = { + "people": {"you": {"age": 40, "employment_income": {"2026": 30_000}}}, + "households": { + "your household": {"members": ["you"], "household_net_income": None} + }, + "families": {"your family": {"members": ["you"]}}, + "tax_units": {"your tax unit": {"members": ["you"]}}, + "spm_units": {"your spm unit": {"members": ["you"]}}, + "marital_units": {"your marital unit": {"members": ["you"]}}, +} +SAVED_SPM = {"scenario": "ce_trend", "geography_kind": "national"} +DATABASE_ERROR = OperationalError("SELECT 1", {}, Exception("server closed")) + + +def _manifest_worker(country: str) -> HouseholdWorker: + """Return the country's household function in a manifest of one worker.""" + version = household_worker_version() + manifest = V2VersionManifest( + generation="test", + default_version=version.bundle.policyengine_version, + versions={version.bundle.policyengine_version: version}, + ) + return default_household_worker(manifest, country) + + +def _expected_policy_fields(reform: dict[str, dict[str, object]]) -> dict[str, object]: + """Return how the household contract's request names the job's one policy.""" + return {"policy": {"parameters": reform}} + + +class FakeWorkers: + """The default worker's household functions, or the lookup's failure.""" + + def __init__(self, failure: Exception | None = None) -> None: + self.failure = failure + self.countries: list[str] = [] + + def default_worker(self, country: str) -> HouseholdWorker: + self.countries.append(country) + if self.failure is not None: + raise self.failure + return _manifest_worker(country) + + +@dataclass +class FakeStore: + """Records what the service asks of the database, and answers as told.""" + + default_year: int | None = 2026 + document: dict[str, Any] = field(default_factory=lambda: dict(STORED_DOCUMENT)) + reform: dict[str, dict[str, object]] = field(default_factory=dict) + read_failure: Exception | None = None + create_failure: Exception | None = None + record_failure: Exception | None = None + recorded_status: HouseholdJobStatus = HouseholdJobStatus.PENDING + stored_job: HouseholdJobRecord | None = None + # Set once the household function has claimed the job: a failure write + # then changes nothing. + claimed_by_function: bool = False + fail_failure: Exception | None = None + job_read_failure: Exception | None = None + reads: list[dict[str, Any]] = field(default_factory=list) + created: list[NewHouseholdJob] = field(default_factory=list) + recorded: list[tuple[UUID, str]] = field(default_factory=list) + failed: list[tuple[UUID, str]] = field(default_factory=list) + + def read_inputs(self, **arguments: Any) -> HouseholdCalculationInputs: + self.reads.append(arguments) + if self.read_failure is not None: + raise self.read_failure + return HouseholdCalculationInputs( + catalog_version_id=CATALOG_VERSION_ID, + household=StoredHousehold( + id=arguments["household_id"], + country=arguments["country"], + default_year=self.default_year, + document=self.document, + ), + reform=self.reform, + ) + + def create_job(self, job: NewHouseholdJob) -> None: + if self.create_failure is not None: + raise self.create_failure + self.created.append(job) + + def record_call(self, job_id: UUID, modal_call_id: str) -> HouseholdJobStatus: + if self.record_failure is not None: + raise self.record_failure + self.recorded.append((job_id, modal_call_id)) + return self.recorded_status + + def fail_unstarted(self, job_id: UUID, error_message: str) -> bool: + self.failed.append((job_id, error_message)) + if self.fail_failure is not None: + raise self.fail_failure + return not self.claimed_by_function + + def claim_as_the_function(self, job_id: UUID) -> None: + """Claim the job as the household function does, once Modal runs it.""" + self.claimed_by_function = True + self.stored_job = _record(job_id, status=HouseholdJobStatus.RUNNING) + + def read(self, job_id: UUID) -> HouseholdJobRecord | None: + if self.read_failure is not None: + raise self.read_failure + if self.job_read_failure is not None: + raise self.job_read_failure + return ( + self.stored_job + if self.stored_job and self.stored_job.id == job_id + else None + ) + + +@dataclass +class FakeCaller: + """Records each household function call, and answers as Modal would.""" + + failure: Exception | None = None + delay_seconds: float = 0.0 + # Called once Modal has accepted the call, before it answers: the + # household function may start then, and its answer may come too late. + on_accepted: Callable[[UUID], None] | None = None + calls: list[dict[str, Any]] = field(default_factory=list) + + async def spawn( + self, + worker: HouseholdWorker, + payload: dict[str, Any], + *, + household_job_id: UUID, + observability_context: dict[str, Any] | None, + ) -> str: + self.calls.append( + { + "worker": worker, + "payload": payload, + "household_job_id": household_job_id, + "observability_context": observability_context, + } + ) + if self.on_accepted is not None: + self.on_accepted(household_job_id) + if self.delay_seconds: + await asyncio.sleep(self.delay_seconds) + if self.failure is not None: + raise self.failure + return "fc-household-1" + + +def _service( + store: FakeStore | None = None, + workers: FakeWorkers | None = None, + caller: FakeCaller | None = None, + **options: Any, +) -> tuple[HouseholdCalculationService, FakeStore, FakeWorkers, FakeCaller]: + store = store or FakeStore() + workers = workers or FakeWorkers() + caller = caller or FakeCaller() + service = HouseholdCalculationService( + store=store, + workers=workers, + caller=caller, + new_job_id=lambda: JOB_ID, + **options, + ) + return service, store, workers, caller + + +def _request(**fields: Any) -> HouseholdCalculationRequest: + return HouseholdCalculationRequest.model_validate( + {"country": "us", "household_id": str(HOUSEHOLD_ID), **fields} + ) + + +@pytest.mark.asyncio +async def test_a_calculation_creates_its_job_then_starts_and_records_its_call(): + service, store, workers, caller = _service(FakeStore(reform=REFORM)) + context = {"observability_id": "00000000-0000-4000-8000-0000000000ee"} + + accepted = await service.submit( + _request(policy_id=str(POLICY_ID), add_missing_variables=True), + observability_context=context, + ) + + worker = _manifest_worker("us") + assert (accepted.job_id, accepted.status) == (JOB_ID, HouseholdJobStatus.PENDING) + assert workers.countries == ["us"] + assert store.reads == [ + { + "country": "us", + "household_id": HOUSEHOLD_ID, + "policy_id": POLICY_ID, + "policyengine_version": "6.17.1", + } + ] + assert store.created == [ + NewHouseholdJob( + id=JOB_ID, + country="us", + household_id=HOUSEHOLD_ID, + policy_id=POLICY_ID, + year=2026, + request_data={ + "country": "us", + "household_id": str(HOUSEHOLD_ID), + "policy_id": str(POLICY_ID), + "add_missing_variables": True, + }, + tax_benefit_model_version_id=CATALOG_VERSION_ID, + country_package_version="2.2.1", + policyengine_version="6.17.1", + ) + ] + (call,) = caller.calls + assert call["worker"] == worker + assert call["household_job_id"] == JOB_ID + assert call["observability_context"] == context + assert call["payload"] == HouseholdExecutionRequest.model_validate( + { + "calculation_id": JOB_ID, + "country": "us", + "year": "2026", + "household": SITUATION, + **_expected_policy_fields(REFORM), + "add_missing_variables": True, + "bundle": worker.bundle, + } + ).model_dump(mode="json") + assert "spm" not in call["payload"] + assert store.recorded == [(JOB_ID, "fc-household-1")] + assert store.failed == [] + + +@pytest.mark.asyncio +async def test_current_law_is_calculated_as_a_policy_without_parameters(): + service, store, _, caller = _service() + + await service.submit(_request(), observability_context=None) + + assert store.reads[0]["policy_id"] is None + assert store.created[0].policy_id is None + payload = caller.calls[0]["payload"] + assert { + name: payload[name] for name in _expected_policy_fields({}) + } == _expected_policy_fields({}) + assert caller.calls[0]["payload"]["add_missing_variables"] is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_fields", "saved", "expected_spm", "expected_source"), + [ + # The request's settings are its choice. + ( + {"spm": {"scenario": "zero_real"}}, + SAVED_SPM, + {"scenario": "zero_real"}, + "request", + ), + # Otherwise the household's saved selection is used, as saved. + ({}, SAVED_SPM, SAVED_SPM, "saved_household"), + ], +) +async def test_the_spm_settings_come_from_the_request_or_the_saved_household( + request_fields, saved, expected_spm, expected_source +): + service, _, _, caller = _service( + FakeStore(document={**STORED_DOCUMENT, "spm": saved}) + ) + + await service.submit(_request(**request_fields), observability_context=None) + + payload = caller.calls[0]["payload"] + assert payload["spm"] == expected_spm + assert payload["spm_selection_source"] == expected_source + + +def test_an_explicit_null_spm_selection_is_refused(): + """Omit ``spm`` to send no settings; null is no selection.""" + with pytest.raises(ValidationError, match="spm"): + _request(spm=None) + + +@pytest.mark.asyncio +async def test_axes_are_calculated_with_the_stored_household(): + """The request's axes vary the stored household's inputs across points.""" + axes = [[{"name": "employment_income", "count": 5, "min": 0, "max": 80_000}]] + service, store, _, caller = _service() + + await service.submit(_request(axes=axes), observability_context=None) + + assert caller.calls[0]["payload"]["household"] == {**SITUATION, "axes": axes} + assert store.created[0].request_data["axes"] == axes + + +@pytest.mark.asyncio +async def test_without_axes_the_household_has_none(): + service, store, _, caller = _service() + + await service.submit(_request(), observability_context=None) + + assert "axes" not in caller.calls[0]["payload"]["household"] + assert "axes" not in store.created[0].request_data + + +@pytest.mark.parametrize( + "axes", + [ + # Parallel axes must share one point count. + [ + [ + {"name": "employment_income", "count": 5, "min": 0, "max": 1}, + {"name": "age", "count": 4, "min": 0, "max": 1}, + ] + ], + # At most 10,000 points in all. + [ + [{"name": "employment_income", "count": 101, "min": 0, "max": 1}], + [{"name": "age", "count": 100, "min": 0, "max": 1}], + ], + # At most eight groups. + [ + [{"name": f"variable_{group}", "count": 1, "min": 0, "max": 1}] + for group in range(9) + ], + ], +) +def test_axes_beyond_the_household_contracts_limits_are_refused(axes): + with pytest.raises(ValidationError): + _request(axes=axes) + + +@pytest.mark.asyncio +async def test_without_spm_settings_or_a_saved_selection_none_is_sent(): + """The household function then inherits the bundle's certified defaults.""" + service, _, _, caller = _service( + FakeStore(document={**STORED_DOCUMENT, "spm": None}) + ) + + await service.submit(_request(), observability_context=None) + + assert "spm" not in caller.calls[0]["payload"] + assert caller.calls[0]["payload"]["spm_selection_source"] == "request" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("store", "workers", "code", "detail"), + [ + ( + FakeStore(), + FakeWorkers( + HouseholdFunctionUnavailableError( + "the default v2 worker, .py 6.17.1, has no household " + "function for us" + ) + ), + "no_household_function", + "the default v2 worker, .py 6.17.1, has no household function for us", + ), + ( + FakeStore( + read_failure=UnknownCatalogVersionError( + "us has no catalog version for .py 6.17.1" + ) + ), + FakeWorkers(), + "unknown_catalog_version", + "us has no catalog version for .py 6.17.1, the default v2 worker's version", + ), + ( + FakeStore(read_failure=UnknownHouseholdError("us has no household x")), + FakeWorkers(), + "unknown_household", + "us has no household x", + ), + ( + FakeStore(read_failure=UnknownPolicyError("us has no policy y")), + FakeWorkers(), + "unknown_policy", + "us has no policy y", + ), + ], +) +async def test_a_missing_row_or_function_is_refused_before_anything_is_written( + store, workers, code, detail +): + service, store, _, caller = _service(store, workers) + + with pytest.raises(HouseholdCalculationRefusedError) as raised: + await service.submit(_request(), observability_context=None) + + assert (raised.value.code, raised.value.detail) == (code, detail) + assert store.created == [] + assert caller.calls == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("store", "reasons"), + [ + ( + FakeStore(default_year=None), + [ + ( + ("body", "household_id"), + "the household has no default year to calculate", + "household_without_default_year", + ) + ], + ), + ( + FakeStore(document={"people": [{"id": "you"}]}), + [ + ( + ("body", "household_id"), + "the stored household's people or groups cannot be read: " + "KeyError('values')", + "household_unreadable", + ) + ], + ), + ( + FakeStore( + read_failure=MissingPolicyParametersError([CTC_AMOUNT, "gov.removed"]) + ), + [ + ( + ("body", "policy_id", name), + "the catalog version for .py 6.17.1 has no such parameter", + "missing_parameter", + ) + for name in (CTC_AMOUNT, "gov.removed") + ], + ), + ], +) +async def test_stored_data_that_cannot_be_calculated_is_unprocessable(store, reasons): + service, store, _, caller = _service(store) + + with pytest.raises(HouseholdCalculationUnprocessableError) as raised: + await service.submit( + _request(policy_id=str(POLICY_ID)), observability_context=None + ) + + assert [ + (reason.loc, reason.msg, reason.type) for reason in raised.value.reasons + ] == reasons + assert store.created == [] + assert caller.calls == [] + + +@pytest.mark.asyncio +async def test_what_the_household_contract_refuses_is_named_by_its_field(): + """A group no one belongs to and a policy value that is an object.""" + document = { + **STORED_DOCUMENT, + "family": [*STORED_DOCUMENT["family"], {"id": "nobody's family", "values": {}}], + } + service, store, _, caller = _service( + FakeStore( + document=document, reform={CTC_AMOUNT: {"2026-01-01.2200-12-31": {"a": 1}}} + ) + ) + + with pytest.raises(HouseholdCalculationUnprocessableError) as raised: + await service.submit( + _request(policy_id=str(POLICY_ID)), observability_context=None + ) + + locations = [reason.loc for reason in raised.value.reasons] + assert ( + "body", + "household_id", + "household", + "families", + "nobody's family", + "members", + ) in locations + assert any( + location[:3] == ("body", "policy_id", "policy") for location in locations + ) + assert store.created == [] + assert caller.calls == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("store", "workers", "code"), + [ + ( + FakeStore(), + FakeWorkers(VersionManifestUnavailableError("x")), + "manifest_unavailable", + ), + (FakeStore(read_failure=DATABASE_ERROR), FakeWorkers(), "database_unavailable"), + ( + FakeStore(create_failure=DATABASE_ERROR), + FakeWorkers(), + "database_unavailable", + ), + ], +) +async def test_an_unreadable_manifest_or_database_starts_nothing(store, workers, code): + service, store, _, caller = _service(store, workers) + + with pytest.raises(HouseholdCalculationUnavailableError) as raised: + await service.submit(_request(), observability_context=None) + + assert raised.value.code == code + assert raised.value.household_job_id is None + assert caller.calls == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("column_name", "detail"), + [ + ("household_id", f"the household {HOUSEHOLD_ID} no longer exists"), + ("policy_id", f"the policy {POLICY_ID} no longer exists"), + ( + "tax_benefit_model_version_id", + f"the catalog version {CATALOG_VERSION_ID} no longer exists", + ), + (None, "a row the household job names no longer exists"), + ], +) +async def test_a_row_deleted_before_the_job_is_created_is_named(column_name, detail): + """The database's foreign keys refuse the job, with SQLSTATE 23503.""" + service, _, _, caller = _service( + FakeStore(create_failure=HouseholdJobReferenceError(column_name)) + ) + + with pytest.raises(HouseholdCalculationRefusedError) as raised: + await service.submit( + _request(policy_id=str(POLICY_ID)), observability_context=None + ) + + assert (raised.value.code, raised.value.detail) == ("missing_reference", detail) + assert caller.calls == [] + + +@pytest.mark.asyncio +async def test_a_call_modal_refuses_fails_its_job_at_once(): + service, store, _, _ = _service( + caller=FakeCaller(failure=ConnectionError("Modal is unreachable")) + ) + + with pytest.raises(HouseholdCalculationUnavailableError) as raised: + await service.submit(_request(), observability_context=None) + + assert raised.value.code == "modal_call_refused" + assert raised.value.household_job_id == JOB_ID + assert store.failed == [ + ( + JOB_ID, + "Modal did not accept the household function's call (ConnectionError).", + ) + ] + assert store.recorded == [] + + +@pytest.mark.asyncio +async def test_a_call_modal_does_not_accept_in_time_fails_its_job(): + service, store, _, _ = _service( + caller=FakeCaller(delay_seconds=1.0), call_start_timeout_seconds=0.01 + ) + + with pytest.raises(HouseholdCalculationUnavailableError) as raised: + await service.submit(_request(), observability_context=None) + + assert raised.value.code == "modal_call_refused" + assert store.failed == [ + ( + JOB_ID, + "Modal did not accept the household function's call (TimeoutError).", + ) + ] + + +@pytest.mark.asyncio +async def test_a_call_accepted_after_the_wait_runs_and_is_answered_as_it_stands(): + """Modal accepted the call late, and the function claimed the job first. + + The failure write then changes nothing, so the job is not failed: the + answer is 202 with the job as it stands, and no second job is needed. + """ + store = FakeStore() + service, store, _, _ = _service( + store, + caller=FakeCaller(delay_seconds=1.0, on_accepted=store.claim_as_the_function), + call_start_timeout_seconds=0.01, + ) + + accepted = await service.submit(_request(), observability_context=None) + + assert (accepted.job_id, accepted.status) == (JOB_ID, HouseholdJobStatus.RUNNING) + assert len(store.failed) == 1 + assert store.recorded == [] + + +@pytest.mark.asyncio +async def test_a_refused_call_whose_failure_cannot_be_written_is_answered_pending(): + """The job exists, so the caller reads it rather than submitting again. + + A job no call started stays pending until the stuck-run sweep fails it. + """ + service, store, _, _ = _service( + FakeStore(fail_failure=DATABASE_ERROR), + caller=FakeCaller(failure=ConnectionError("Modal is unreachable")), + ) + + accepted = await service.submit(_request(), observability_context=None) + + assert (accepted.job_id, accepted.status) == (JOB_ID, HouseholdJobStatus.PENDING) + assert len(store.failed) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "store", + [ + FakeStore(job_read_failure=DATABASE_ERROR), + # The failure write changed nothing, yet the job isn't found. + FakeStore(claimed_by_function=True), + ], + ids=["read-fails", "job-not-found"], +) +async def test_a_job_whose_state_cannot_be_read_back_is_answered_pending(store): + on_accepted = store.claim_as_the_function if not store.claimed_by_function else None + service, store, _, _ = _service( + store, + caller=FakeCaller(delay_seconds=1.0, on_accepted=on_accepted), + call_start_timeout_seconds=0.01, + ) + + accepted = await service.submit(_request(), observability_context=None) + + assert (accepted.job_id, accepted.status) == (JOB_ID, HouseholdJobStatus.PENDING) + + +def test_what_the_service_builds_itself_is_an_internal_error_when_refused(): + """A refused bundle is the service's fault, not the caller's household.""" + worker = _manifest_worker("us") + broken = HouseholdWorker( + application_name=worker.application_name, + function_name=worker.function_name, + bundle={"policyengine_version": "6.17.1"}, # pyright: ignore[reportArgumentType] # a bundle the contract refuses + ) + inputs = FakeStore().read_inputs( + country="us", + household_id=HOUSEHOLD_ID, + policy_id=None, + policyengine_version="6.17.1", + ) + + with pytest.raises(HouseholdCalculationInternalError) as raised: + prepare_household_job(_request(), broken, inputs, job_id=JOB_ID) + + assert "bundle" in str(raised.value) + + +@pytest.mark.asyncio +async def test_a_call_whose_id_cannot_be_recorded_is_still_accepted(): + """The household function claims a pending job with no call ID itself.""" + service, store, _, _ = _service(FakeStore(record_failure=DATABASE_ERROR)) + + accepted = await service.submit(_request(), observability_context=None) + + assert (accepted.job_id, accepted.status) == (JOB_ID, HouseholdJobStatus.PENDING) + assert store.failed == [] + + +@pytest.mark.asyncio +async def test_a_job_the_function_claimed_first_is_answered_as_it_stands(): + service, _, _, _ = _service(FakeStore(recorded_status=HouseholdJobStatus.RUNNING)) + + accepted = await service.submit(_request(), observability_context=None) + + assert accepted.status is HouseholdJobStatus.RUNNING + + +def _record(job_id: UUID, **changes: Any) -> HouseholdJobRecord: + values = { + "id": job_id, + "status": HouseholdJobStatus.SUCCEEDED, + "country": "us", + "household_id": HOUSEHOLD_ID, + "policy_id": None, + "year": 2026, + "request_data": {}, + "tax_benefit_model_version_id": CATALOG_VERSION_ID, + "country_package_version": "2.2.1", + "policyengine_version": "6.17.1", + "modal_call_id": "fc-1", + "result": {"status": "succeeded"}, + "error_message": None, + "created_at": datetime(2026, 11, 10, tzinfo=UTC), + "started_at": datetime(2026, 11, 10, tzinfo=UTC), + "completed_at": datetime(2026, 11, 10, tzinfo=UTC), + **changes, + } + return HouseholdJobRecord(**values) + + +@pytest.mark.asyncio +async def test_a_job_is_read_as_it_stands_and_an_unknown_one_as_none(): + service, _, _, _ = _service(FakeStore(stored_job=_record(JOB_ID))) + + assert await service.read(JOB_ID) == _record(JOB_ID) + assert await service.read(uuid4()) is None + + +@pytest.mark.asyncio +async def test_a_job_that_cannot_be_read_is_unavailable(): + service, _, _, _ = _service(FakeStore(read_failure=DATABASE_ERROR)) + + with pytest.raises(HouseholdCalculationUnavailableError) as raised: + await service.read(JOB_ID) + + assert raised.value.code == "database_unavailable" diff --git a/projects/policyengine-simulation-entry/tests/test_household_contract_fields.py b/projects/policyengine-simulation-entry/tests/test_household_contract_fields.py new file mode 100644 index 000000000..f88847864 --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_household_contract_fields.py @@ -0,0 +1,41 @@ +"""Which request field a refused field of the household contract's request traces to.""" + +from __future__ import annotations + +import pytest +from policyengine_simulation_entry.household_contract_fields import request_field_of + + +@pytest.mark.parametrize( + ("location", "spm_from_request", "request_field"), + [ + # The stored household, and its default year, come from household_id. + (("household", "people", "you", "age"), False, "household_id"), + (("household", "families", "a family", "members"), False, "household_id"), + (("year",), False, "household_id"), + # The request's axes are put into the household. + (("household", "axes", 0, 0, "count"), False, "axes"), + ( + ("policy", "parameters", "gov.x", "2026-01-01.2200-12-31"), + False, + "policy_id", + ), + # SPM settings are the request's, or else the stored household's. + (("spm",), True, "spm"), + (("spm",), False, "household_id"), + (("country",), False, "country"), + (("add_missing_variables",), False, "add_missing_variables"), + # The service builds these itself; no request field is to blame. + (("bundle", "core_package_version"), False, None), + (("calculation_id",), False, None), + (("spm_selection_source",), True, None), + (("schema_version",), False, None), + ((), False, None), + ], +) +def test_each_refused_contract_field_traces_to_the_request_field_that_set_it( + location, spm_from_request, request_field +): + assert request_field_of(location, spm_from_request=spm_from_request) == ( + request_field + ) diff --git a/projects/policyengine-simulation-entry/tests/test_household_conversion.py b/projects/policyengine-simulation-entry/tests/test_household_conversion.py new file mode 100644 index 000000000..e5bb6bac8 --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_household_conversion.py @@ -0,0 +1,135 @@ +"""Stored households, in the situation shape the household functions take. + +Ported from sim #957, whose cases came from the Public API, which converted +households before the Simulation Entrypoint read them itself. +""" + +from __future__ import annotations + +import pytest +from policyengine_simulation_entry.household_conversion import ( + UnreadableHouseholdError, + household_situation, + saved_spm_selection, +) + +US_HOUSEHOLD = { + "people": [ + { + "id": "person-1", + "source_name": "you", + "values": {"age": {"2026": 40}, "employment_income": {"2026": 30000}}, + "memberships": { + "household": "household-1", + "family": "family-1", + "tax_unit": "tax_unit-1", + "spm_unit": "spm_unit-1", + "marital_unit": "marital_unit-1", + }, + }, + { + "id": "person-2", + "values": {"age": {"2026": 8}}, + "memberships": { + "household": "household-1", + "family": "family-1", + "tax_unit": "tax_unit-1", + "spm_unit": "spm_unit-1", + "marital_unit": "marital_unit-2", + }, + }, + ], + "household": [{"id": "household-1", "values": {"state_name": {"2026": "CA"}}}], + "family": [{"id": "family-1", "values": {}}], + "tax_unit": [{"id": "tax_unit-1", "values": {"eitc": {"2026": None}}}], + "spm_unit": [{"id": "spm_unit-1", "values": {}}], + "marital_unit": [ + {"id": "marital_unit-1", "values": {}}, + {"id": "marital_unit-2", "values": {}}, + ], + "spm": {"scenario": "ce_trend", "geography_kind": "national"}, +} +UK_HOUSEHOLD = { + "people": [ + { + "id": "person-1", + "values": {"age": {"2026": 30}}, + "memberships": {"household": "household-1", "benunit": "benunit-1"}, + } + ], + "household": [{"id": "household-1", "values": {"region": {"2026": "WALES"}}}], + "benunit": [{"id": "benunit-1", "values": {"universal_credit": {"2026": None}}}], +} + + +def test_a_us_household_becomes_a_situation_keyed_by_record_id() -> None: + assert household_situation("us", US_HOUSEHOLD) == { + "people": { + "person-1": {"age": {"2026": 40}, "employment_income": {"2026": 30000}}, + "person-2": {"age": {"2026": 8}}, + }, + "households": { + "household-1": { + "members": ["person-1", "person-2"], + "state_name": {"2026": "CA"}, + } + }, + "families": {"family-1": {"members": ["person-1", "person-2"]}}, + "tax_units": { + "tax_unit-1": {"members": ["person-1", "person-2"], "eitc": {"2026": None}} + }, + "spm_units": {"spm_unit-1": {"members": ["person-1", "person-2"]}}, + "marital_units": { + "marital_unit-1": {"members": ["person-1"]}, + "marital_unit-2": {"members": ["person-2"]}, + }, + } + + +def test_a_uk_household_has_households_and_benefit_units() -> None: + assert household_situation("uk", UK_HOUSEHOLD) == { + "people": {"person-1": {"age": {"2026": 30}}}, + "households": { + "household-1": {"members": ["person-1"], "region": {"2026": "WALES"}} + }, + "benunits": { + "benunit-1": { + "members": ["person-1"], + "universal_credit": {"2026": None}, + } + }, + } + + +def test_a_group_without_members_keeps_an_empty_member_list() -> None: + household = {**UK_HOUSEHOLD, "benunit": [*UK_HOUSEHOLD["benunit"]]} + household["benunit"].append({"id": "benunit-2", "values": {}}) + + situation = household_situation("uk", household) + + assert situation["benunits"]["benunit-2"] == {"members": []} + + +def test_the_saved_spm_selection_is_the_documents_selection_or_none() -> None: + """A saved null selection is no choice, as the household contract reads it.""" + assert saved_spm_selection(US_HOUSEHOLD) == { + "scenario": "ce_trend", + "geography_kind": "national", + } + assert saved_spm_selection(UK_HOUSEHOLD) is None + assert saved_spm_selection({**US_HOUSEHOLD, "spm": None}) is None + + +def test_the_saved_spm_selection_is_not_part_of_the_situation() -> None: + assert "spm" not in household_situation("us", US_HOUSEHOLD) + + +def test_a_record_without_values_or_memberships_cannot_be_read() -> None: + """A document the Public API would never store is refused, not guessed at.""" + household = { + **UK_HOUSEHOLD, + "people": [{"id": "person-1", "values": {"age": 30}}], + } + + with pytest.raises(UnreadableHouseholdError, match="people"): + household_situation("uk", household) diff --git a/projects/policyengine-simulation-entry/tests/test_household_deployment.py b/projects/policyengine-simulation-entry/tests/test_household_deployment.py new file mode 100644 index 000000000..bd77c47a4 --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_household_deployment.py @@ -0,0 +1,139 @@ +"""Deployment wiring for the household calculation routes. + +The entrypoint calls the v2 workers' household functions on Modal, so every +deploy, whatever the Stage 12 setting, gives it the Modal token as pinned +Cloud Run secrets and the v2 manifest's Modal environment and name. +""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest +from test_deployment_assets_support import ( + CLOUD_RUN_DEPLOY_SCRIPT, + REUSABLE_DEPLOY_WORKFLOW, + cloud_run_deploy_environment, + workflow_jobs, +) + + +def _run_deploy(tmp_path, **overrides): + environment = cloud_run_deploy_environment(tmp_path) | overrides + result = subprocess.run( + [CLOUD_RUN_DEPLOY_SCRIPT], + check=False, + capture_output=True, + text=True, + env=environment, + ) + return result, environment + + +def _secrets(environment) -> list[str]: + arguments = ( + Path(environment["GCLOUD_ARGUMENTS_CAPTURE"]) + .read_text(encoding="utf-8") + .splitlines() + ) + return arguments[arguments.index("--set-secrets") + 1].split(",") + + +def _runtime_environment(environment) -> dict[str, str]: + return json.loads( + Path(environment["GCLOUD_ENV_FILE_CAPTURE"]).read_text(encoding="utf-8") + ) + + +@pytest.mark.parametrize("deploy_stage12", ["true", "false"]) +def test_every_deploy_mounts_the_modal_token_once(tmp_path, deploy_stage12): + result, environment = _run_deploy(tmp_path, DEPLOY_STAGE12_V2=deploy_stage12) + + assert result.returncode == 0, result.stderr + secrets = _secrets(environment) + assert secrets.count("MODAL_TOKEN_ID=modal-token-id:3") == 1 + assert secrets.count("MODAL_TOKEN_SECRET=modal-token-secret:3") == 1 + assert [secret.partition("=")[0] for secret in secrets].count("MODAL_TOKEN_ID") == 1 + runtime_environment = _runtime_environment(environment) + assert "MODAL_TOKEN_ID" not in runtime_environment + assert "MODAL_TOKEN_SECRET" not in runtime_environment + + +@pytest.mark.parametrize( + ("app_environment", "modal_environment", "deploy_stage12"), + [ + ("staging", "staging", "true"), + ("staging", "staging", "false"), + ("production", "main", "true"), + ("production", "main", "false"), + ], +) +def test_every_deploy_names_the_v2_manifest_and_its_modal_environment( + tmp_path, app_environment, modal_environment, deploy_stage12 +): + result, environment = _run_deploy( + tmp_path, + APP_ENVIRONMENT=app_environment, + MODAL_ENVIRONMENT=modal_environment, + DEPLOY_STAGE12_V2=deploy_stage12, + ) + + assert result.returncode == 0, result.stderr + runtime_environment = _runtime_environment(environment) + assert runtime_environment["SIMULATION_V2_MODAL_ENVIRONMENT"] == modal_environment + assert runtime_environment["SIMULATION_V2_MANIFEST_NAME"] == ( + "simulation-api-v2-version-manifest" + ) + + +def test_without_stage12_no_stage12_setting_is_deployed(tmp_path): + """The household settings are the household path's own.""" + result, environment = _run_deploy(tmp_path, DEPLOY_STAGE12_V2="false") + + assert result.returncode == 0, result.stderr + runtime_environment = _runtime_environment(environment) + assert runtime_environment["STAGE12_ENABLED"] == "0" + assert not { + "STAGE12_V2_MANIFEST_NAME", + "STAGE12_V2_MANIFEST_ENVIRONMENT", + "STAGE12_ARTIFACT_BUCKET", + } & set(runtime_environment) + assert not any( + secret.startswith("STAGE12_DATABASE_URL=") for secret in _secrets(environment) + ) + + +@pytest.mark.parametrize( + "secret_name_variable", + ["MODAL_TOKEN_ID_SECRET_NAME", "MODAL_TOKEN_SECRET_SECRET_NAME"], +) +@pytest.mark.parametrize("deploy_stage12", ["true", "false"]) +def test_a_deploy_without_the_modal_token_secret_names_stops( + tmp_path, secret_name_variable, deploy_stage12 +): + result, environment = _run_deploy( + tmp_path, DEPLOY_STAGE12_V2=deploy_stage12, **{secret_name_variable: ""} + ) + + assert result.returncode != 0 + assert f"{secret_name_variable} must be set" in result.stderr + assert not Path(environment["GCLOUD_ARGUMENTS_CAPTURE"]).exists() + + +def test_the_deploy_step_passes_the_modal_token_secret_names(): + deploy_entrypoint = workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)["deploy_entrypoint"] + deploy_step = next( + step + for step in deploy_entrypoint["steps"] + if step.get("name") == "Deploy Cloud Run entrypoint candidate" + ) + + assert deploy_step["env"]["MODAL_TOKEN_ID_SECRET_NAME"] == ( + "${{ vars.MODAL_TOKEN_ID_SECRET_NAME }}" + ) + assert deploy_step["env"]["MODAL_TOKEN_SECRET_SECRET_NAME"] == ( + "${{ vars.MODAL_TOKEN_SECRET_SECRET_NAME }}" + ) + assert deploy_step["env"]["MODAL_ENVIRONMENT"] == "${{ inputs.modal_environment }}" diff --git a/projects/policyengine-simulation-entry/tests/test_household_routes.py b/projects/policyengine-simulation-entry/tests/test_household_routes.py new file mode 100644 index 000000000..3c1863336 --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_household_routes.py @@ -0,0 +1,603 @@ +"""The household calculation routes, with the service faked.""" + +from __future__ import annotations + +import dataclasses +import json +from dataclasses import dataclass, field +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any, NoReturn +from uuid import UUID + +import pytest +from conftest import FakeBackend, make_settings +from fastapi.testclient import TestClient +from policyengine_observability import TelemetryLimits +from policyengine_simulation_entry import app as app_module +from policyengine_simulation_entry.app import create_app +from policyengine_simulation_entry.household_api_models import ( + HouseholdCalculationAccepted, + HouseholdCalculationRequest, +) +from policyengine_simulation_entry.household_calculations import ( + HouseholdCalculationInternalError, + HouseholdCalculationRefusedError, + HouseholdCalculationService, + HouseholdCalculationUnavailableError, + HouseholdCalculationUnprocessableError, + UnprocessableReason, +) +from policyengine_simulation_entry.simulation_database import SimulationDatabase +from policyengine_simulation_observability.identifiers import OBSERVABILITY_ID_HEADER +from policyengine_simulation_persistence import HouseholdJobRecord, HouseholdJobStatus + +if TYPE_CHECKING: + from collections.abc import Callable + +JOB_ID = UUID("00000000-0000-4000-8000-0000000000aa") +HOUSEHOLD_ID = UUID("00000000-0000-4000-8000-0000000000bb") +CATALOG_VERSION_ID = UUID("00000000-0000-4000-8000-0000000000dd") +OBSERVABILITY_ID = "00000000-0000-4000-8000-000000000001" +SUBMISSION = {"country": "us", "household_id": str(HOUSEHOLD_ID)} +RESULT_DOCUMENT = { + "status": "succeeded", + "schema_version": 1, + "calculation_id": str(JOB_ID), +} +STATE_PATH = f"/household-calculations/{JOB_ID}" +STATE_TEMPLATE = "/household-calculations/{household_job_id}" + + +def _record(**changes: Any) -> HouseholdJobRecord: + values = { + "id": JOB_ID, + "status": HouseholdJobStatus.SUCCEEDED, + "country": "us", + "household_id": HOUSEHOLD_ID, + "policy_id": None, + "year": 2026, + "request_data": SUBMISSION, + "tax_benefit_model_version_id": CATALOG_VERSION_ID, + "country_package_version": "2.2.1", + "policyengine_version": "6.17.1", + "modal_call_id": "fc-1", + "result": RESULT_DOCUMENT, + "error_message": None, + "created_at": datetime(2026, 11, 10, 12, tzinfo=UTC), + "started_at": datetime(2026, 11, 10, 12, 0, 5, tzinfo=UTC), + "completed_at": datetime(2026, 11, 10, 12, 0, 30, tzinfo=UTC), + **changes, + } + return HouseholdJobRecord(**values) + + +def _raise_from_nested_calls(failure: Exception, depth: int) -> NoReturn: + """Raise ``failure`` from ``depth`` calls below this one.""" + if depth == 0: + raise failure + _raise_from_nested_calls(failure, depth - 1) + + +@dataclass +class FakeHouseholdCalculations: + """Records each submission and read, and answers as told. + + ``failure`` is raised by ``raise_failure(failure, failure_depth)``: by + default from ``failure_depth`` nested calls below the first call of + ``_raise_from_nested_calls``. + """ + + failure: Exception | None = None + failure_depth: int = 0 + raise_failure: Callable[[Exception, int], NoReturn] = _raise_from_nested_calls + stored: HouseholdJobRecord | None = None + submissions: list[tuple[HouseholdCalculationRequest, Any]] = field( + default_factory=list + ) + reads: list[UUID] = field(default_factory=list) + + async def submit( + self, + request: HouseholdCalculationRequest, + *, + observability_context: dict[str, Any] | None, + ) -> HouseholdCalculationAccepted: + self.submissions.append((request, observability_context)) + if self.failure is not None: + self.raise_failure(self.failure, self.failure_depth) + return HouseholdCalculationAccepted( + job_id=JOB_ID, status=HouseholdJobStatus.PENDING + ) + + async def read(self, job_id: UUID) -> HouseholdJobRecord | None: + self.reads.append(job_id) + if self.failure is not None: + self.raise_failure(self.failure, self.failure_depth) + return self.stored if self.stored and self.stored.id == job_id else None + + +@pytest.fixture +def client_for(monkeypatch): + """Return a function building a client whose app uses ``calculations``. + + None leaves the app as an unconfigured service builds it. + """ + + def build(calculations: FakeHouseholdCalculations | None) -> TestClient: + if calculations is not None: + monkeypatch.setattr( + app_module, + "household_calculations_for", + lambda settings, database: calculations, + ) + return TestClient( + create_app( + settings=make_settings(), + backend=FakeBackend(), + auth_dependency=lambda: None, + ) + ) + + return build + + +def test_a_submission_answers_202_with_the_job_and_its_status(client_for): + calculations = FakeHouseholdCalculations() + + with client_for(calculations) as client: + result = client.post( + "/household-calculations", + json={ + **SUBMISSION, + "policy_id": None, + "add_missing_variables": True, + "spm": {"scenario": "zero_real"}, + }, + headers={OBSERVABILITY_ID_HEADER: OBSERVABILITY_ID}, + ) + + assert result.status_code == 202 + assert result.json() == {"job_id": str(JOB_ID), "status": "pending"} + assert result.headers[OBSERVABILITY_ID_HEADER] == OBSERVABILITY_ID + ((request, context),) = calculations.submissions + assert request.household_id == HOUSEHOLD_ID + assert request.add_missing_variables is True + assert request.spm_requested is True + assert context["observability_id"] == OBSERVABILITY_ID + + +def test_a_submission_without_an_observability_id_gets_a_new_one(client_for): + calculations = FakeHouseholdCalculations() + + with client_for(calculations) as client: + result = client.post("/household-calculations", json=SUBMISSION) + + observability_id = result.headers[OBSERVABILITY_ID_HEADER] + assert str(UUID(observability_id)) == observability_id + assert calculations.submissions[0][1]["observability_id"] == observability_id + + +@pytest.mark.parametrize( + "body", + [ + {**SUBMISSION, "country": "fr"}, + {**SUBMISSION, "household_id": "not-a-uuid"}, + {**SUBMISSION, "year": 2026}, + {"country": "us"}, + {**SUBMISSION, "spm": None}, + {**SUBMISSION, "spm": "ce_trend"}, + {**SUBMISSION, "axes": [[{"name": "age", "count": 0, "min": 0, "max": 1}]]}, + ], +) +def test_a_malformed_submission_is_refused_with_its_reasons(client_for, body): + calculations = FakeHouseholdCalculations() + + with client_for(calculations) as client: + result = client.post("/household-calculations", json=body) + + assert result.status_code == 422 + assert all( + {"loc", "msg", "type"} <= set(reason) for reason in result.json()["detail"] + ) + assert calculations.submissions == [] + + +@pytest.mark.parametrize( + ("failure", "status_code", "body"), + [ + ( + HouseholdCalculationRefusedError( + code="unknown_household", detail="us has no household x" + ), + 400, + {"code": "unknown_household", "detail": "us has no household x"}, + ), + ( + HouseholdCalculationUnprocessableError( + ( + UnprocessableReason( + loc=("body", "policy_id", "gov.removed"), + msg="the catalog version for .py 6.17.1 has no such parameter", + type="missing_parameter", + ), + ) + ), + 422, + { + "detail": [ + { + "loc": ["body", "policy_id", "gov.removed"], + "msg": ( + "the catalog version for .py 6.17.1 has no such parameter" + ), + "type": "missing_parameter", + } + ] + }, + ), + ( + HouseholdCalculationUnavailableError( + code="database_unavailable", + detail="The simulation database could not be read.", + ), + 503, + { + "code": "database_unavailable", + "detail": "The simulation database could not be read.", + }, + ), + ( + HouseholdCalculationInternalError( + "the household function's request has a field the service " + "built wrong: bundle" + ), + 500, + { + "code": "internal_error", + "detail": ( + "The service could not build the household function's request." + ), + }, + ), + ( + HouseholdCalculationUnavailableError( + code="modal_call_refused", + detail="Modal did not accept the call.", + household_job_id=JOB_ID, + ), + 503, + { + "code": "modal_call_refused", + "detail": "Modal did not accept the call.", + "household_job_id": str(JOB_ID), + }, + ), + ], +) +def test_a_refused_submission_answers_with_its_reason( + client_for, failure, status_code, body +): + with client_for(FakeHouseholdCalculations(failure=failure)) as client: + result = client.post("/household-calculations", json=SUBMISSION) + + assert result.status_code == status_code + assert result.json() == body + assert (result.headers.get("Retry-After") == "10") is (status_code == 503) + + +@pytest.mark.parametrize( + ("method", "path", "body"), + [ + ("post", "/household-calculations", SUBMISSION), + ("get", STATE_PATH, None), + ], +) +def test_without_a_database_or_v2_environment_the_routes_are_unavailable( + client_for, method, path, body +): + with client_for(None) as client: + result = getattr(client, method)(path, **({"json": body} if body else {})) + + assert result.status_code == 503 + assert result.json() == { + "code": "not_configured", + "detail": "Household calculations are not configured.", + } + assert result.headers["Retry-After"] == "10" + + +def test_a_job_is_read_with_its_result_document(client_for): + calculations = FakeHouseholdCalculations(stored=_record()) + + with client_for(calculations) as client: + result = client.get(STATE_PATH) + + assert result.status_code == 200 + assert result.json() == { + "job_id": str(JOB_ID), + "status": "succeeded", + "country": "us", + "household_id": str(HOUSEHOLD_ID), + "policy_id": None, + "year": 2026, + "policyengine_version": "6.17.1", + "country_package_version": "2.2.1", + "result": RESULT_DOCUMENT, + "error_message": None, + "created_at": "2026-11-10T12:00:00Z", + "started_at": "2026-11-10T12:00:05Z", + "completed_at": "2026-11-10T12:00:30Z", + } + assert calculations.reads == [JOB_ID] + + +def test_a_failed_job_is_read_with_its_error_message(client_for): + calculations = FakeHouseholdCalculations( + stored=_record( + status=HouseholdJobStatus.FAILED, + result=None, + error_message="Modal did not accept the household function's call.", + started_at=None, + ) + ) + + with client_for(calculations) as client: + result = client.get(STATE_PATH) + + assert result.status_code == 200 + assert result.json()["status"] == "failed" + assert result.json()["result"] is None + assert result.json()["error_message"] == ( + "Modal did not accept the household function's call." + ) + + +def test_reading_a_job_never_invents_an_observability_id(client_for): + """household_jobs records none, so the read returns none, whatever is sent.""" + with client_for(FakeHouseholdCalculations(stored=_record())) as client: + result = client.get( + STATE_PATH, headers={OBSERVABILITY_ID_HEADER: OBSERVABILITY_ID} + ) + + assert OBSERVABILITY_ID_HEADER not in result.headers + + +def test_an_unknown_job_answers_404(client_for): + with client_for(FakeHouseholdCalculations()) as client: + result = client.get(STATE_PATH) + + assert result.status_code == 404 + assert result.json() == { + "code": "unknown_household_job", + "detail": f"There is no household job {JOB_ID}.", + } + + +def test_a_job_that_cannot_be_read_answers_503(client_for): + failure = HouseholdCalculationUnavailableError( + code="database_unavailable", + detail="The simulation database could not be read.", + ) + + with client_for(FakeHouseholdCalculations(failure=failure)) as client: + result = client.get(STATE_PATH) + + assert result.status_code == 503 + assert result.json()["code"] == "database_unavailable" + assert result.headers["Retry-After"] == "10" + + +def test_a_job_id_that_is_not_a_uuid_is_refused(client_for): + calculations = FakeHouseholdCalculations() + + with client_for(calculations) as client: + result = client.get("/household-calculations/not-a-uuid") + + assert result.status_code == 422 + assert calculations.reads == [] + + +DATABASE_URL = "postgresql://simulation_service:secret@db.example:5432/postgres" + + +def test_a_configured_service_builds_household_calculations_without_connecting(): + """Nothing reaches the database or Modal until the first submission.""" + database = SimulationDatabase.from_url(DATABASE_URL) + try: + calculations = app_module.household_calculations_for( + make_settings( + simulation_database_url=DATABASE_URL, v2_modal_environment="staging" + ), + database, + ) + finally: + database.dispose() + + assert isinstance(calculations, HouseholdCalculationService) + + +@pytest.mark.parametrize( + "settings_changes", + [ + {}, + {"simulation_database_url": DATABASE_URL}, + {"v2_modal_environment": "staging"}, + ], +) +def test_without_its_database_and_v2_environment_none_is_built(settings_changes): + database = SimulationDatabase.from_url(DATABASE_URL) + try: + calculations = app_module.household_calculations_for( + make_settings(**settings_changes), database + ) + finally: + database.dispose() + + assert calculations is None + + +# What an unexpected failure answers, the same in the generated client's test. +UNEXPECTED_FAILURE_ANSWER = { + "code": "internal_error", + "detail": "The service could not answer the request.", +} + + +def _emitted_records(printed: str, message: str) -> list[dict[str, Any]]: + """Return the records with ``message`` the observability runtime printed. + + The service's runtime writes each record to standard output as one JSON + object per line of text. + """ + records = [ + json.loads(printed_line) + for printed_line in printed.splitlines() + if printed_line.startswith("{") + ] + return [record for record in records if record.get("message") == message] + + +# Each route's handler, the outermost frame of an error it answers. +HANDLER_OF_METHOD = { + "post": "submit_household_calculation", + "get": "get_household_calculation", +} + + +@pytest.mark.parametrize( + ("method", "path", "body"), + [ + ("post", "/household-calculations", SUBMISSION), + ("get", STATE_PATH, None), + ], +) +def test_an_unexpected_failure_answers_a_json_500_without_its_values( + client_for, capsys, method, path, body +): + """The record the runtime emits keeps the route, the type and the frames.""" + calculations = FakeHouseholdCalculations( + failure=RuntimeError("a value that must not be logged") + ) + + with client_for(calculations) as client: + result = getattr(client, method)(path, **({"json": body} if body else {})) + + assert result.status_code == 500 + assert result.headers["content-type"] == "application/json" + assert result.json() == UNEXPECTED_FAILURE_ANSWER + printed = capsys.readouterr().out + (record,) = _emitted_records(printed, "household_route_failed") + attributes = record["attributes"] + assert attributes["error_type"] == "RuntimeError" + assert attributes["route"] in {"/household-calculations", STATE_TEMPLATE} + # Every attribute was kept, and every frame, innermost first: from where + # the fake raised the error up to the route's handler. + assert "attributes.omitted_count" not in record + assert attributes["stack_frames_omitted"] == 0 + frames = attributes["stack_frames"].split("; ") + assert "test_household_routes.py" in frames[0] + assert frames[0].endswith(" in _raise_from_nested_calls") + assert frames[1].endswith((" in submit", " in read")) + assert frames[-1].endswith(f" in {HANDLER_OF_METHOD[method]}") + assert not any(key.startswith("error.") for key in record) + assert "a value that must not be logged" not in printed + + +# A string limit that holds a few frames only, and enough calls to pass it. +SMALL_STRING_LIMIT = 600 +NESTED_CALLS = 20 + +# Where the service's image installs packages (its virtual environment is +# /opt/venv), and the files of a library and of the persistence library. +IMAGE_SITE_PACKAGES = "/opt/venv/lib/python3.13/site-packages" +LIBRARY_FILE = "sqlalchemy/engine/base.py" +PERSISTENCE_FILE = "policyengine_simulation_persistence/household_job_store.py" +# The library's connect raises the failure from ``depth`` nested calls of +# itself, on its third line; the persistence library calls it. +LIBRARY_SOURCE = """\ +def connect(failure, depth): + if depth == 0: + raise failure + connect(failure, depth - 1) +""" +LIBRARY_RAISE_LINE_NUMBER = 3 +PERSISTENCE_SOURCE = """\ +def read_household_job(failure, depth): + connect(failure, depth) +""" + + +def _defined_in_file(file_path: str, source: str, **names: Any) -> dict[str, Any]: + """Run ``source`` as code read from ``file_path``; return its globals. + + Each function it defines then has frames with that path, as an installed + package's functions do. ``names`` are its globals before it runs. + """ + namespace = dict(names) + exec(compile(source, file_path, "exec"), namespace) + return namespace + + +def test_an_unexpected_failure_leaves_out_library_frames_before_service_frames( + client_for, capsys, monkeypatch +): + """The service's frames are kept, and the library's outer frames counted. + + The error is raised deep in a library that the persistence library + called, both at the image's paths. Each kept frame's path starts after + ``site-packages/`` or the checkout's ``src/``. The string holds the + frame that raised the error, the service's own frames, and as many of + the library's inner frames as fit the limit, which comes from the + runtime the service logs through. + """ + build_runtime = app_module.init_simulation_observability + + def runtime_with_a_small_limit(*arguments, **keywords): + runtime = build_runtime(*arguments, **keywords) + runtime.config = dataclasses.replace( + runtime.config, + limits=TelemetryLimits(max_string_length=SMALL_STRING_LIMIT), + ) + return runtime + + monkeypatch.setattr( + app_module, "init_simulation_observability", runtime_with_a_small_limit + ) + library = _defined_in_file(f"{IMAGE_SITE_PACKAGES}/{LIBRARY_FILE}", LIBRARY_SOURCE) + persistence = _defined_in_file( + f"{IMAGE_SITE_PACKAGES}/{PERSISTENCE_FILE}", + PERSISTENCE_SOURCE, + connect=library["connect"], + ) + calculations = FakeHouseholdCalculations( + failure=RuntimeError("a value that must not be logged"), + failure_depth=NESTED_CALLS, + raise_failure=persistence["read_household_job"], + ) + + with client_for(calculations) as client: + result = client.post("/household-calculations", json=SUBMISSION) + + assert result.status_code == 500 + (record,) = _emitted_records(capsys.readouterr().out, "household_route_failed") + attributes = record["attributes"] + frames = attributes["stack_frames"].split("; ") + assert len(attributes["stack_frames"]) <= SMALL_STRING_LIMIT + # The service's frames come last, innermost first: the persistence + # library's call, then the route's handler. + assert [frame.rsplit(" in ", 1)[1] for frame in frames[-2:]] == [ + "read_household_job", + "submit_household_calculation", + ] + assert frames[-2].startswith(f"{PERSISTENCE_FILE}:") + assert frames[-1].startswith("policyengine_simulation_entry/household_routes.py:") + # Before them, the library's innermost frames, from where it raised. + library_frames = frames[:-2] + assert library_frames[0] == ( + f"{LIBRARY_FILE}:{LIBRARY_RAISE_LINE_NUMBER} in connect" + ) + assert all(frame.startswith(f"{LIBRARY_FILE}:") for frame in library_frames) + assert 1 < len(library_frames) < NESTED_CALLS + 1 + # Left out: the library's outer frames and the fake's submit, of the + # library's NESTED_CALLS + 1, the persistence call, submit and the handler. + assert len(frames) + attributes["stack_frames_omitted"] == NESTED_CALLS + 4 diff --git a/projects/policyengine-simulation-entry/tests/test_household_workers.py b/projects/policyengine-simulation-entry/tests/test_household_workers.py new file mode 100644 index 000000000..661308082 --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_household_workers.py @@ -0,0 +1,183 @@ +"""Finding the default v2 worker's household function for a country.""" + +from __future__ import annotations + +from types import SimpleNamespace +from uuid import UUID + +import pytest +from household_fixtures import ( + BUNDLE_MANIFEST_SHA256, + POLICYENGINE_VERSION, + household_worker_version, + manifest_store, +) +from policyengine_simulation_contract.stage12_manifest import v2_application_name +from policyengine_simulation_entry import household_workers +from policyengine_simulation_entry.household_workers import ( + HouseholdFunctionUnavailableError, + ManifestHouseholdWorkers, + ModalHouseholdFunctionCaller, + VersionManifestUnavailableError, +) + + +def test_the_default_workers_household_function_and_bundle_are_used() -> None: + workers = ManifestHouseholdWorkers(manifest_store(household_worker_version())) + + worker = workers.default_worker("uk") + + assert worker.application_name == v2_application_name(POLICYENGINE_VERSION) + assert worker.function_name == "calculate_household_uk" + assert worker.bundle.policyengine_version == POLICYENGINE_VERSION + assert worker.bundle.core_package_version == "3.33.3" + assert worker.bundle.country_package_name == "policyengine-uk" + assert worker.bundle.country_package_version == "2.174.1" + assert worker.bundle.bundle_manifest_sha256 == BUNDLE_MANIFEST_SHA256 + + +def test_a_newer_worker_is_not_used_until_it_is_the_default() -> None: + """A deployment adds its worker before its checks and moves the default after.""" + workers = ManifestHouseholdWorkers( + manifest_store( + household_worker_version("6.17.1"), household_worker_version("6.18.0") + ) + ) + + assert workers.default_worker("us").bundle.policyengine_version == "6.17.1" + + +def test_a_default_worker_without_the_countrys_household_function_says_so() -> None: + workers = ManifestHouseholdWorkers( + manifest_store(household_worker_version(household_countries=("uk",))) + ) + + with pytest.raises(HouseholdFunctionUnavailableError) as raised: + workers.default_worker("us") + + assert str(raised.value) == ( + f"the default v2 worker, .py {POLICYENGINE_VERSION}, has no household " + "function for us" + ) + + +def test_an_empty_manifest_store_is_unavailable() -> None: + with pytest.raises(VersionManifestUnavailableError): + ManifestHouseholdWorkers({}).default_worker("us") + + +class UnreachableStore: + """A manifest store whose every read fails, as an unreachable Modal Dict does.""" + + def get(self, key: str, default: object = None) -> object: + """Fail the read.""" + msg = f"Modal is unreachable for {key}" + raise ConnectionError(msg) + + def __setitem__(self, key: str, value: object) -> None: + """Fail the write.""" + raise ConnectionError(key) + + +def test_an_unreachable_manifest_store_is_unavailable() -> None: + with pytest.raises(VersionManifestUnavailableError): + ManifestHouseholdWorkers(UnreachableStore()).default_worker("us") + + +class FlakyStore: + """A manifest store that answers once, then fails every read.""" + + def __init__(self, stored: dict[str, object]) -> None: + self.stored = stored + self.reads_allowed = True + + def get(self, key: str, default: object = None) -> object: + """Answer while reads are allowed.""" + if not self.reads_allowed: + msg = "Modal is unreachable" + raise ConnectionError(msg) + return self.stored.get(key, default) + + def __setitem__(self, key: str, value: object) -> None: + """Never written.""" + raise ConnectionError(key) + + +def test_a_store_that_stops_answering_is_unavailable_even_after_a_read() -> None: + """Only an invalid stored manifest falls back to the last valid one.""" + store = FlakyStore(manifest_store(household_worker_version())) + workers = ManifestHouseholdWorkers(store) + workers.default_worker("us") + store.reads_allowed = False + + with pytest.raises(VersionManifestUnavailableError): + workers.default_worker("us") + + +class FakeModalFunction: + """A deployed Modal function, as ``modal.Function.from_name`` returns it.""" + + def __init__(self, call_id: str | None) -> None: + self.call_id = call_id + self.spawned: list[tuple[tuple, dict]] = [] + self.spawn = SimpleNamespace(aio=self._spawn) + + async def _spawn(self, *arguments, **keywords): + self.spawned.append((arguments, keywords)) + return SimpleNamespace(object_id=self.call_id) + + +JOB_ID = UUID("00000000-0000-4000-8000-0000000000aa") + + +@pytest.mark.asyncio +async def test_a_household_function_is_called_by_name_with_its_job(monkeypatch): + function = FakeModalFunction("fc-household-1") + looked_up: list[tuple] = [] + + def from_name(application_name, function_name, *, environment_name): + looked_up.append((application_name, function_name, environment_name)) + return function + + monkeypatch.setattr(household_workers.modal.Function, "from_name", from_name) + worker = ManifestHouseholdWorkers( + manifest_store(household_worker_version()) + ).default_worker("us") + + call_id = await ModalHouseholdFunctionCaller("staging").spawn( + worker, + {"calculation_id": str(JOB_ID)}, + household_job_id=JOB_ID, + observability_context={"observability_id": "o-1"}, + ) + + assert call_id == "fc-household-1" + assert looked_up == [ + (v2_application_name(POLICYENGINE_VERSION), "calculate_household_us", "staging") + ] + assert function.spawned == [ + ( + ({"calculation_id": str(JOB_ID)},), + { + "household_job_id": str(JOB_ID), + "observability_context": {"observability_id": "o-1"}, + }, + ) + ] + + +@pytest.mark.asyncio +async def test_a_call_modal_gives_no_id_is_refused(monkeypatch): + monkeypatch.setattr( + household_workers.modal.Function, + "from_name", + lambda *names, environment_name: FakeModalFunction(None), + ) + worker = ManifestHouseholdWorkers( + manifest_store(household_worker_version()) + ).default_worker("uk") + + with pytest.raises(RuntimeError, match="no call ID"): + await ModalHouseholdFunctionCaller("staging").spawn( + worker, {}, household_job_id=JOB_ID, observability_context=None + ) diff --git a/projects/policyengine-simulation-entry/tests/test_openapi.py b/projects/policyengine-simulation-entry/tests/test_openapi.py index 83372a245..631db41d6 100644 --- a/projects/policyengine-simulation-entry/tests/test_openapi.py +++ b/projects/policyengine-simulation-entry/tests/test_openapi.py @@ -29,11 +29,13 @@ class OpenAPIDocument(TypedDict): EXPECTED_ROUTES = { ("GET", "/budget-window-jobs/{batch_job_id}"), + ("GET", "/household-calculations/{household_job_id}"), ("GET", "/health"), ("GET", "/jobs/{job_id}"), ("GET", "/ready"), ("GET", "/versions"), ("GET", "/versions/{kind}"), + ("POST", "/household-calculations"), ("POST", "/ping"), ("POST", "/simulate/economy/budget-window"), ("POST", "/simulate/economy/comparison"), @@ -58,10 +60,29 @@ def operation_ids(spec: OpenAPIDocument) -> dict[tuple[str, str], str]: } +# The household calculation routes, which only the Cloud Run entrypoint has. +HOUSEHOLD_PATHS = ( + "/household-calculations", + "/household-calculations/{household_job_id}", +) +HOUSEHOLD_SCHEMAS = ( + "HouseholdAxis", + "HouseholdCalculationAccepted", + "HouseholdCalculationError", + "HouseholdCalculationRequest", + "HouseholdCalculationState", + "HouseholdJobStatus", + "HouseholdRequestReason", + "HouseholdRequestUnprocessable", +) + + def normalized_compatibility_paths(spec: OpenAPIDocument) -> OpenAPIPathMap: """Remove intentional Cloud Run and additive SPM error documentation.""" paths = deepcopy(spec["paths"]) paths.pop("/ready", None) + for path in HOUSEHOLD_PATHS: + paths.pop(path, None) for operations in paths.values(): for method, operation in operations.items(): if method.lower() in {"get", "post", "put", "patch", "delete"}: @@ -191,9 +212,114 @@ def test_normalized_contract_matches_old_gateway(): cloud_run_schemas.pop("SimulationEntrypointReadinessResponse") cloud_run_schemas.pop("SimulationDatabaseReadiness") cloud_run_schemas.pop("SimulationErrorResponse") + for schema_name in HOUSEHOLD_SCHEMAS: + cloud_run_schemas.pop(schema_name) assert cloud_run_schemas == gateway_spec["components"]["schemas"] gateway_operation_ids = operation_ids(gateway_spec) cloud_run_operation_ids = operation_ids(cloud_run_spec) assert { key: cloud_run_operation_ids[key] for key in gateway_operation_ids } == gateway_operation_ids + + +def _responses(spec: dict, method: str, path: str) -> dict[str, dict]: + return spec["paths"][path][method]["responses"] + + +def _response_schema(response: dict) -> dict: + return response["content"]["application/json"]["schema"] + + +def test_household_routes_declare_every_answer(): + spec = create_app().openapi() + error = {"$ref": "#/components/schemas/HouseholdCalculationError"} + unprocessable = {"$ref": "#/components/schemas/HouseholdRequestUnprocessable"} + + submit = _responses(spec, "post", "/household-calculations") + read = _responses(spec, "get", "/household-calculations/{household_job_id}") + + assert set(submit) == {"202", "400", "403", "422", "500", "503"} + # A 202 may also be a job whose call Modal didn't confirm: poll it. + assert "pending" in submit["202"]["description"] + assert _response_schema(submit["202"]) == { + "$ref": "#/components/schemas/HouseholdCalculationAccepted" + } + assert _response_schema(submit["400"]) == error + assert _response_schema(submit["422"]) == unprocessable + assert _response_schema(submit["503"]) == error + assert set(read) == {"200", "403", "404", "422", "500", "503"} + assert _response_schema(read["200"]) == { + "$ref": "#/components/schemas/HouseholdCalculationState" + } + assert _response_schema(read["404"]) == error + assert _response_schema(read["422"]) == unprocessable + assert _response_schema(read["503"]) == error + assert _response_schema(submit["500"]) == error + assert _response_schema(read["500"]) == error + code = spec["components"]["schemas"]["HouseholdCalculationError"]["properties"][ + "code" + ] + assert "job_state_unknown" not in code["enum"] + assert "internal_error" in code["enum"] + + +def test_a_household_job_read_declares_only_the_answers_a_read_gives(): + """A read never refuses stored data or starts a Modal call.""" + read = _responses( + create_app().openapi(), "get", "/household-calculations/{household_job_id}" + ) + + assert "stored" not in read["422"]["description"] + assert "UUID" in read["422"]["description"] + assert "Modal" not in read["503"]["description"] + assert "database" in read["503"]["description"] + + +def test_every_household_503_declares_its_retry_after_header(): + spec = create_app().openapi() + + for method, path in ( + ("post", "/household-calculations"), + ("get", "/household-calculations/{household_job_id}"), + ): + headers = _responses(spec, method, path)["503"]["headers"] + assert headers["Retry-After"]["schema"] == {"type": "integer"} + + +def test_household_routes_name_their_operations(): + spec = create_app().openapi() + + assert operation_ids(spec)[("POST", "/household-calculations")] == ( + "submit_household_calculation" + ) + assert operation_ids(spec)[ + ("GET", "/household-calculations/{household_job_id}") + ] == ("get_household_calculation") + + +def test_a_household_calculation_request_names_its_fields_only(): + schemas = create_app().openapi()["components"]["schemas"] + request = schemas["HouseholdCalculationRequest"] + + assert set(request["properties"]) == { + "country", + "household_id", + "policy_id", + "add_missing_variables", + "axes", + "spm", + } + # SPM settings are an object when given; null is refused. + assert request["properties"]["spm"]["type"] == "object" + assert set(schemas["HouseholdCalculationAccepted"]["properties"]) == { + "job_id", + "status", + } + assert set(request["required"]) == {"country", "household_id"} + assert request["additionalProperties"] is False + assert schemas["HouseholdJobStatus"]["enum"] == [ + "pending", + "running", + "succeeded", + "failed", + ] diff --git a/projects/policyengine-simulation-entry/tests/test_staging_household_check.py b/projects/policyengine-simulation-entry/tests/test_staging_household_check.py new file mode 100644 index 000000000..25c307cec --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_staging_household_check.py @@ -0,0 +1,379 @@ +"""The staging household check, run on every staging deploy. + +``simulation-staging-household-check.sh`` calculates the stored staging +household that the ``beta`` environment variable +``SIMULATION_STAGING_HOUSEHOLD_ID`` names, through the deployed candidate's +``/household-calculations`` routes, and requires the job to succeed. It runs +after the v2 manifest's default has moved to the release's worker, so it +calculates with the release's own household function. +""" + +from __future__ import annotations + +import json + +import pytest +from test_deployment_assets_support import ( + REUSABLE_DEPLOY_WORKFLOW, + evaluate_github_expression, + routing_context, + workflow_jobs, +) +from workflow_script_support import FakeCommands, run_script + +CHECK_SCRIPT = "simulation-staging-household-check.sh" +READY_SCRIPT = "simulation-staging-household-ready.sh" +CHECK_JOB = "staging_household_check" +BASE_URL = "https://policyengine-simulation-entry-staging-s-123.a.run.app" +HOUSEHOLD_ID = "00000000-0000-4000-8000-0000000000bb" +JOB_ID = "00000000-0000-4000-8000-0000000000aa" +ACCESS_TOKEN = "test-access-token" + +# The fake curl answers the token request, the submission and each poll in +# turn, and keeps each request body it was sent on standard input. +FAKE_CURL = r""" +output="" +url="" +method="GET" +data="" +while (($#)); do + case "$1" in + --output) output="$2"; shift 2 ;; + --request) method="$2"; shift 2 ;; + --data) data="$2"; shift 2 ;; + --header | --write-out) shift 2 ;; + *) url="$1"; shift ;; + esac +done +calls="$(($(cat "${FAKE_STATE}/calls" 2>/dev/null || echo 0) + 1))" +echo "${calls}" > "${FAKE_STATE}/calls" +if [[ "${data}" == "@-" ]]; then + cat > "${FAKE_STATE}/body-${calls}.json" +fi +case "${method} ${url}" in + "POST https://issuer.example/oauth/token") + printf '{"access_token": "%s"}' "${FAKE_ACCESS_TOKEN}" > "${output}" + printf '200' + ;; + "POST ${FAKE_BASE_URL}/household-calculations") + printf '%s' "${FAKE_SUBMISSION_BODY}" > "${output}" + printf '%s' "${FAKE_SUBMISSION_STATUS}" + ;; + "GET ${FAKE_BASE_URL}/household-calculations/"*) + polls="$(($(cat "${FAKE_STATE}/polls" 2>/dev/null || echo 0) + 1))" + echo "${polls}" > "${FAKE_STATE}/polls" + answer="$(sed -n "${polls}p" "${FAKE_STATE}/answers")" + printf '%s' "${answer#* }" > "${output}" + printf '%s' "${answer%% *}" + ;; + *) + printf 'unexpected request: %s %s\n' "${method}" "${url}" >&2 + exit 7 + ;; +esac +""" + + +def _job(status: str, **fields) -> str: + return json.dumps( + { + "job_id": JOB_ID, + "status": status, + "country": "us", + "household_id": HOUSEHOLD_ID, + "policyengine_version": "6.17.1", + "country_package_version": "2.2.1", + "result": None, + "error_message": None, + **fields, + } + ) + + +SUCCEEDED = _job("succeeded", result={"status": "succeeded", "output": {}}) + + +@pytest.fixture +def fakes(tmp_path) -> FakeCommands: + commands = FakeCommands.create(tmp_path) + commands.add("curl", FAKE_CURL) + commands.add("sleep") + return commands + + +def _run(fakes, tmp_path, answers: list[str], **overrides): + """Run the check; ``answers`` are the polls' " " in order.""" + state = tmp_path / "state" + state.mkdir() + (state / "answers").write_text("\n".join(answers) + "\n", encoding="utf-8") + environment = { + "STAGING_HOUSEHOLD_ID": HOUSEHOLD_ID, + "SIMULATION_TEST_AUTH_ISSUER": "https://issuer.example/", + "SIMULATION_TEST_AUTH_AUDIENCE": "simulation-entry", + "SIMULATION_TEST_AUTH_CLIENT_ID": "staging-test-client", + "SIMULATION_TEST_AUTH_CLIENT_SECRET": "staging-test-secret", + "FAKE_STATE": str(state), + "FAKE_BASE_URL": BASE_URL, + "FAKE_ACCESS_TOKEN": ACCESS_TOKEN, + "FAKE_SUBMISSION_STATUS": "202", + "FAKE_SUBMISSION_BODY": json.dumps({"job_id": JOB_ID, "status": "pending"}), + **overrides, + } + result = run_script(CHECK_SCRIPT, BASE_URL + "/", env=fakes.env(**environment)) + return result, state + + +def test_the_check_submits_the_staging_household_and_waits_for_success(fakes, tmp_path): + result, state = _run( + fakes, + tmp_path, + [f"200 {_job('pending')}", f"200 {_job('running')}", f"200 {SUCCEEDED}"], + ) + + assert result.returncode == 0, result.stderr + assert JOB_ID in result.stdout + token_request = json.loads((state / "body-1.json").read_text(encoding="utf-8")) + assert token_request == { + "client_id": "staging-test-client", + "client_secret": "staging-test-secret", + "audience": "simulation-entry", + "grant_type": "client_credentials", + } + submission = json.loads((state / "body-2.json").read_text(encoding="utf-8")) + assert submission == {"country": "us", "household_id": HOUSEHOLD_ID} + assert [call[1:] for call in fakes.calls("sleep")] == [["5"], ["5"]] + + +def test_no_secret_reaches_a_command_line(fakes, tmp_path): + """The token is sent from a file, and the client secret on standard input.""" + result, _ = _run(fakes, tmp_path, [f"200 {SUCCEEDED}"]) + + assert result.returncode == 0, result.stderr + arguments = [argument for call in fakes.calls("curl") for argument in call] + assert not any(ACCESS_TOKEN in argument for argument in arguments) + assert not any("staging-test-secret" in argument for argument in arguments) + assert sum(argument.startswith("@") for argument in arguments) >= 3 + + +def test_a_failed_job_fails_the_check_with_its_error_message(fakes, tmp_path): + result, _ = _run( + fakes, + tmp_path, + [ + f"200 {_job('failed', error_message='invalid_inputs: no such variable')}", + ], + ) + + assert result.returncode == 1 + assert "invalid_inputs: no such variable" in result.stderr + assert fakes.calls("sleep") == [] + + +def test_a_succeeded_job_without_a_succeeded_result_fails_the_check(fakes, tmp_path): + result, _ = _run(fakes, tmp_path, [f"200 {_job('succeeded')}"]) + + assert result.returncode == 1 + assert "result" in result.stderr + + +def test_a_refused_submission_fails_the_check_with_its_answer(fakes, tmp_path): + refusal = json.dumps( + {"code": "unknown_household", "detail": f"us has no household {HOUSEHOLD_ID}"} + ) + + result, state = _run( + fakes, + tmp_path, + [], + FAKE_SUBMISSION_STATUS="400", + FAKE_SUBMISSION_BODY=refusal, + ) + + assert result.returncode == 1 + assert "400" in result.stderr + assert "unknown_household" in result.stderr + assert not (state / "polls").exists() + + +def test_an_unavailable_read_is_asked_again(fakes, tmp_path): + unavailable = json.dumps({"code": "database_unavailable", "detail": "x"}) + + result, _ = _run(fakes, tmp_path, [f"503 {unavailable}", f"200 {SUCCEEDED}"]) + + assert result.returncode == 0, result.stderr + + +def test_any_other_read_answer_fails_the_check(fakes, tmp_path): + result, _ = _run(fakes, tmp_path, ['404 {"code": "unknown_household_job"}']) + + assert result.returncode == 1 + assert "404" in result.stderr + + +def test_a_job_that_does_not_finish_in_time_fails_the_check(fakes, tmp_path): + result, _ = _run( + fakes, + tmp_path, + [f"200 {_job('pending')}"], + HOUSEHOLD_CHECK_DEADLINE_SECONDS="0", + ) + + assert result.returncode == 1 + assert "did not finish within 0 seconds" in result.stderr + + +@pytest.mark.parametrize( + ("household_id", "message"), + [ + ("", "SIMULATION_STAGING_HOUSEHOLD_ID"), + ("not-a-uuid", "not a UUID"), + ], +) +def test_the_check_needs_the_staging_household_id( + fakes, tmp_path, household_id, message +): + result, _ = _run(fakes, tmp_path, [], STAGING_HOUSEHOLD_ID=household_id) + + assert result.returncode == 1 + assert message in result.stderr + assert fakes.calls("curl") == [] + + +def test_the_check_needs_the_test_client_credentials(fakes, tmp_path): + result, _ = _run(fakes, tmp_path, [], SIMULATION_TEST_AUTH_CLIENT_SECRET="") + + assert result.returncode == 1 + assert "SIMULATION_TEST_AUTH_CLIENT_SECRET" in result.stderr + assert fakes.calls("curl") == [] + + +# --- The workflow job -------------------------------------------------------- + + +def _jobs() -> dict: + return workflow_jobs(REUSABLE_DEPLOY_WORKFLOW) + + +def _context(**overrides): + context = routing_context() + context["inputs"]["deployment_environment"] = "staging" + context["inputs"]["deploy_stage12_v2"] = True + context["needs"]["deploy_entrypoint"] = {"result": "success"} + context["needs"]["publish_stage12_v2_default"] = {"result": "success"} + for path, value in overrides.items(): + target = context + *parents, last = path.split("__") + for part in parents: + target = target[part] + target[last] = value + return context + + +def test_the_check_job_calculates_through_the_candidate_with_the_test_client(): + job = _jobs()[CHECK_JOB] + (step,) = [step for step in job["steps"] if "run" in step] + + assert set(job["needs"]) == { + "deploy_entrypoint", + "authenticated_test", + "publish_stage12_v2_default", + } + assert job["environment"] == "${{ inputs.release_environment }}" + assert step["run"] == ( + f".github/scripts/{CHECK_SCRIPT} " + '"${{ needs.deploy_entrypoint.outputs.candidate_url }}"' + ) + assert step["env"] == { + "STAGING_HOUSEHOLD_ID": "${{ vars.SIMULATION_STAGING_HOUSEHOLD_ID }}", + "SIMULATION_TEST_AUTH_ISSUER": ( + "${{ secrets.SIMULATION_ENTRYPOINT_AUTH_ISSUER }}" + ), + "SIMULATION_TEST_AUTH_AUDIENCE": ( + "${{ secrets.SIMULATION_ENTRYPOINT_AUTH_AUDIENCE }}" + ), + "SIMULATION_TEST_AUTH_CLIENT_ID": ( + "${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_ID }}" + ), + "SIMULATION_TEST_AUTH_CLIENT_SECRET": ( + "${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_SECRET }}" + ), + } + + +@pytest.mark.parametrize( + ("overrides", "runs"), + [ + ({}, True), + ({"inputs__deployment_environment": "production"}, False), + ({"needs__deploy_entrypoint__result": "failure"}, False), + ({"needs__authenticated_test__result": "failure"}, False), + ({"needs__publish_stage12_v2_default__result": "failure"}, False), + ({"needs__publish_stage12_v2_default__result": "skipped"}, False), + # Without Stage 12 the default stays, and the check uses it. + ( + { + "inputs__deploy_stage12_v2": False, + "needs__publish_stage12_v2_default__result": "skipped", + }, + True, + ), + ({"cancelled": True}, False), + ], + ids=[ + "staging", + "production", + "entrypoint-failed", + "authenticated-tests-failed", + "default-move-failed", + "default-move-skipped", + "no-stage12", + "cancelled", + ], +) +def test_the_check_runs_on_every_staging_deploy_after_the_default_moves( + overrides, runs +): + job = _jobs()[CHECK_JOB] + + assert ( + bool(evaluate_github_expression(job["if"], _context(**overrides), job_if=True)) + is runs + ) + + +def test_deployment_ready_requires_the_staging_household_check(): + ready = _jobs()["deployment_ready"] + (step,) = [ + step + for step in ready["steps"] + if step.get("name") == "Check the staging household check result" + ] + + assert CHECK_JOB in ready["needs"] + assert step["run"] == f".github/scripts/{READY_SCRIPT}" + assert step["env"] == { + "REQUESTED": "${{ inputs.deployment_environment == 'staging' }}", + "CHECK_RESULT": "${{ needs.staging_household_check.result }}", + } + + +@pytest.mark.parametrize( + ("requested", "check_result", "passes"), + [ + ("true", "success", True), + ("true", "failure", False), + ("true", "skipped", False), + ("true", "cancelled", False), + ("false", "skipped", True), + ("false", "success", False), + ], +) +def test_the_ready_check_requires_the_check_exactly_in_staging( + fakes, requested, check_result, passes +): + result = run_script( + READY_SCRIPT, + env=fakes.env(REQUESTED=requested, CHECK_RESULT=check_result), + ) + + assert result.returncode == (0 if passes else 1) diff --git a/projects/policyengine-simulation-executor/tests/test_household_calculation.py b/projects/policyengine-simulation-executor/tests/test_household_calculation.py index c8c91dc3d..8b323e880 100644 --- a/projects/policyengine-simulation-executor/tests/test_household_calculation.py +++ b/projects/policyengine-simulation-executor/tests/test_household_calculation.py @@ -875,6 +875,90 @@ def test_us_an_axis_without_a_period_varies_the_request_year() -> None: } +# The simulation service writes a stored policy value that has no end date as +# a range ending on 31 December 2200, the latest year a household job can be +# calculated for. These show how the household functions apply each form of +# period key in that year, with incomes high enough to be taxed at the +# changed rate after the thresholds' uprating. +SINGLE_ADULT_WITHOUT_PERIODS = { + "us": { + "people": {"you": {"age": 40, "employment_income": 100_000_000}}, + "tax_units": {"your tax unit": {"members": ["you"]}}, + "families": {"your family": {"members": ["you"]}}, + "spm_units": {"your spm unit": {"members": ["you"]}}, + "marital_units": {"your marital unit": {"members": ["you"]}}, + "households": { + "your household": { + "members": ["you"], + "state_name": "TX", + "household_net_income": None, + } + }, + }, + "uk": { + "people": {"you": {"age": 40, "employment_income": 100_000_000}}, + "benunits": {"your benefit unit": {"members": ["you"]}}, + "households": { + "your household": {"members": ["you"], "household_net_income": None} + }, + }, +} +# A parameter each country's reform lowers net income through, and its value. +CHANGED_PARAMETER = { + "us": ("gov.irs.income.bracket.rates.7", 0.5), + "uk": ("gov.hmrc.income_tax.rates.uk[0].rate", 0.3), +} + + +@pytest.mark.parametrize( + ("country", "start_date_alone_applies"), + [ + # policyengine-core reads a start date alone as "from that day on". + ("us", True), + # policyengine-uk's reform dictionary reads it as that one day. + ("uk", False), + ], +) +def test_reform_periods_apply_in_2200_only_through_a_stop_in_2200( + country, start_date_alone_applies +) -> None: + """A range applies through its stop date, in both countries. + + Only a range reaching the end of 2200 changes a 2200 calculation in both + countries: a range ending in 2100 leaves it at current law, and a start + date alone means a different period in each. + """ + parameter, value = CHANGED_PARAMETER[country] + reforms = { + "baseline": {}, + "through_2200": {parameter: {"2026-01-01.2200-12-31": value}}, + "through_2100": {parameter: {"2026-01-01.2100-12-31": value}}, + "start_date_alone": {parameter: {"2026-01-01": value}}, + } + net_incomes = {} + for reform_name, parameters in reforms.items(): + outcome = calculate_household( + _request( + country, + SINGLE_ADULT_WITHOUT_PERIODS[country], + year="2200", + policy={"parameters": parameters}, + ), + worker=_worker(country), + dependencies=HouseholdCalculationDependencies(), + ) + assert isinstance(outcome, HouseholdExecutionResult), outcome + net_incomes[reform_name] = outcome.output.household.to_situation()[ + "households" + ]["your household"]["household_net_income"]["2200"] + + assert net_incomes["through_2200"] < net_incomes["baseline"] + assert net_incomes["through_2100"] == net_incomes["baseline"] + assert (net_incomes["start_date_alone"] < net_incomes["baseline"]) is ( + start_date_alone_applies + ) + + def test_uk_axes_calculate_one_value_per_point() -> None: household = { **UK_SINGLE, diff --git a/projects/policyengine-simulation-executor/tests/test_py_reform_dicts.py b/projects/policyengine-simulation-executor/tests/test_py_reform_dicts.py new file mode 100644 index 000000000..532f5d349 --- /dev/null +++ b/projects/policyengine-simulation-executor/tests/test_py_reform_dicts.py @@ -0,0 +1,245 @@ +"""How .py turns parameter values into the reform values a simulation takes. + +The simulation service's policy reader (``policy_reform`` in the simulation +persistence library, which this environment doesn't install) follows .py's +``reform_dict_from_parameter_values`` for values without an end date. These +tests pin .py's periods for the cases the reader's tests check. The reader +gives the same periods, in the same order, with two differences: + +- its last open-ended value runs to the end of 2200, the latest year a + calculation can be for, where .py's runs to 2100. So the reader gives a + last value starting in 2101 to 2200 a range to the end of 2200, where + .py's range for it ends before it starts; +- it leaves out an open-ended value whose range would end before it starts: + one followed the same day by a later value, or a last one starting after + 2200, in 2201 or later. .py gives that value such a range. + policyengine-core applies it on no day, but policyengine-uk refuses it, + so a UK simulation given .py's dict fails. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from types import SimpleNamespace + +import pytest +from policyengine.core import ParameterValue +from policyengine.utils.parametric_reforms import ( + OPEN_ENDED_REFORM_STOP, + reform_dict_from_parameter_values, +) +from policyengine_core.entities import build_entity +from policyengine_core.parameters import ParameterNode +from policyengine_core.reforms import Reform +from policyengine_core.taxbenefitsystems import TaxBenefitSystem +from policyengine_uk.utils.dated_reforms import read_reform_dict + +CTC_AMOUNT = "gov.irs.credits.ctc.amount.base[0].amount" +STANDARD_DEDUCTION = "gov.irs.deductions.standard.amount.SINGLE" + + +def _value( + parameter_name: str, + value: float, + start_date: datetime, + end_date: datetime | None = None, +) -> ParameterValue: + """Return a parameter value; .py reads only its parameter's name here.""" + return ParameterValue.model_construct( + parameter=SimpleNamespace(name=parameter_name), + value=value, + start_date=start_date, + end_date=end_date, + ) + + +def test_py_gives_way_to_a_later_bounded_value_for_good() -> None: + """After the bounded value ends, current law applies.""" + reform = reform_dict_from_parameter_values( + [ + _value(CTC_AMOUNT, 3_000, datetime(2026, 1, 1, tzinfo=UTC)), + _value( + CTC_AMOUNT, + 5_000, + datetime(2028, 1, 1, tzinfo=UTC), + datetime(2028, 12, 31, tzinfo=UTC), + ), + ] + ) + + assert reform == { + CTC_AMOUNT: { + "2026-01-01.2027-12-31": 3_000, + "2028-01-01.2028-12-31": 5_000, + } + } + + +def test_py_runs_open_ended_values_one_after_another() -> None: + reform = reform_dict_from_parameter_values( + [ + _value(CTC_AMOUNT, 1_000, datetime(2026, 1, 1, tzinfo=UTC)), + _value(CTC_AMOUNT, 2_000, datetime(2027, 7, 1, tzinfo=UTC)), + _value(CTC_AMOUNT, 3_000, datetime(2030, 1, 1, tzinfo=UTC)), + _value(STANDARD_DEDUCTION, 20_000, datetime(2029, 1, 1, tzinfo=UTC)), + ] + ) + + assert OPEN_ENDED_REFORM_STOP == "2100-12-31" + assert reform == { + CTC_AMOUNT: { + "2026-01-01.2027-06-30": 1_000, + "2027-07-01.2029-12-31": 2_000, + "2030-01-01.2100-12-31": 3_000, + }, + STANDARD_DEDUCTION: {"2029-01-01.2100-12-31": 20_000}, + } + + +def _applied(reform: dict): + """Apply ``reform`` with policyengine-core to a system of one parameter. + + ``credit.amount`` is 1,000 from 2020 under current law. + """ + person = build_entity(key="person", plural="people", label="Person", is_person=True) + system = TaxBenefitSystem(entities=[person]) + system.parameters = ParameterNode( + "", data={"credit": {"amount": {"values": {"2020-01-01": 1_000}}}} + ) + return Reform.from_dict(reform)(system).parameters.get_child("credit.amount") + + +def test_py_lets_the_later_of_two_values_starting_the_same_day_take_over() -> None: + """.py ends the earlier value before it starts, so only the later applies. + + The reform is applied with policyengine-core to a one-parameter system. + """ + reform = reform_dict_from_parameter_values( + [ + _value("credit.amount", 3_000, datetime(2026, 1, 1, tzinfo=UTC)), + _value("credit.amount", 4_000, datetime(2026, 1, 1, 12, tzinfo=UTC)), + ] + ) + amount = _applied(reform) + + assert reform == { + "credit.amount": { + "2026-01-01.2025-12-31": 3_000, + "2026-01-01.2100-12-31": 4_000, + } + } + assert [ + amount(instant) for instant in ("2025-12-31", "2026-01-01", "2099-06-01") + ] == [ + 1_000, + 4_000, + 4_000, + ] + + +def test_py_keeps_bounded_values_starting_the_same_day_in_their_order() -> None: + """Both ranges are kept; core applies them in order, so each holds part.""" + reform = reform_dict_from_parameter_values( + [ + _value( + "credit.amount", + 3_000, + datetime(2026, 1, 1, tzinfo=UTC), + datetime(2026, 12, 31, tzinfo=UTC), + ), + _value( + "credit.amount", + 4_000, + datetime(2026, 1, 1, 12, tzinfo=UTC), + datetime(2026, 6, 30, tzinfo=UTC), + ), + ] + ) + + assert list(reform["credit.amount"].items()) == [ + ("2026-01-01.2026-12-31", 3_000), + ("2026-01-01.2026-06-30", 4_000), + ] + amount = _applied(reform) + assert [ + amount(instant) for instant in ("2026-03-01", "2026-09-01", "2027-03-01") + ] == [4_000, 3_000, 1_000] + + +def test_py_lets_an_open_ended_value_follow_a_bounded_one_on_the_same_day() -> None: + reform = reform_dict_from_parameter_values( + [ + _value( + "credit.amount", + 3_000, + datetime(2026, 1, 1, tzinfo=UTC), + datetime(2026, 6, 30, tzinfo=UTC), + ), + _value("credit.amount", 4_000, datetime(2026, 1, 1, 12, tzinfo=UTC)), + ] + ) + + assert list(reform["credit.amount"].items()) == [ + ("2026-01-01.2026-06-30", 3_000), + ("2026-01-01.2100-12-31", 4_000), + ] + amount = _applied(reform) + assert [amount(instant) for instant in ("2026-03-01", "2026-09-01")] == [ + 4_000, + 4_000, + ] + + +def test_py_lets_a_bounded_value_take_over_an_open_ended_one_the_same_day() -> None: + """The bounded value starts the same day; current law applies after it. + + policyengine-core applies the range ending the day before it starts on + no day. policyengine-uk's reform dictionary reader, which a UK + ``Simulation(reform=...)`` uses, refuses it. + """ + reform = reform_dict_from_parameter_values( + [ + _value("credit.amount", 3_000, datetime(2026, 1, 1, tzinfo=UTC)), + _value( + "credit.amount", + 4_000, + datetime(2026, 1, 1, 12, tzinfo=UTC), + datetime(2026, 12, 31, tzinfo=UTC), + ), + ] + ) + + assert list(reform["credit.amount"].items()) == [ + ("2026-01-01.2025-12-31", 3_000), + ("2026-01-01.2026-12-31", 4_000), + ] + amount = _applied(reform) + assert [ + amount(instant) for instant in ("2025-12-31", "2026-06-01", "2027-06-01") + ] == [1_000, 4_000, 1_000] + with pytest.raises(ValueError, match="ends before it starts"): + read_reform_dict(reform, lambda parameter_path: True) + + +def test_py_reads_a_start_the_public_api_stored_in_utc_on_its_utc_day() -> None: + """.py reads a start the Public API normalized to UTC on its UTC day. + + The Public API stores 20:00 on 31 December, five hours behind UTC, as + 1 January 01:00 UTC, which .py reads as 1 January. An open-ended value + starting after its 2100 stop gets a range ending before it starts. + """ + reform = reform_dict_from_parameter_values( + [ + _value(CTC_AMOUNT, 1_000, datetime(2026, 1, 1, 1, tzinfo=UTC)), + _value(CTC_AMOUNT, 2_000, datetime(2027, 1, 1, tzinfo=UTC)), + _value(STANDARD_DEDUCTION, 9_000, datetime(2250, 1, 1, tzinfo=UTC)), + ] + ) + + assert reform == { + CTC_AMOUNT: { + "2026-01-01.2026-12-31": 1_000, + "2027-01-01.2100-12-31": 2_000, + }, + STANDARD_DEDUCTION: {"2250-01-01.2100-12-31": 9_000}, + } diff --git a/rules/registry.toml b/rules/registry.toml index a373fca0e..b0100b932 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -173,10 +173,11 @@ test = "tests/test_access.py::test_rule_proof_missing_privileges_reported" [[rule]] id = "sim.persistence.models-match-public-api" category = "persistence" -statement = "Sim's models of the Public API's tables match the tables the Public API's v2 migration chain builds at the pinned Public API commit, which is on the Public API's default branch: every column's type, nullability and default, every constraint and index by name and definition, and every enum's values; and the simulation service's role, with the privileges the Public API's manifest declares, writes every modelled table." +statement = "Sim's models of the Public API's tables match the tables the Public API's v2 migration chain builds at the pinned Public API commit, which is on the Public API's default branch: every column's type, nullability and default, every constraint and index by name and definition, and every enum's values; every column its read-only models of the Public API's input tables name exists there with the same type and nullability; and the simulation service's role, with the privileges the Public API's manifest declares, writes every modelled table." rationale = "libs/policyengine-simulation-persistence/README.md#checking-the-models-against-the-public-api" scope = [ "libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/tables.py", + "libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/input_tables.py", "libs/policyengine-simulation-persistence/public-api-commit", ".github/scripts/public-api-schema-commit.sh", ".github/scripts/public-api-schema-reference.sh", @@ -190,6 +191,7 @@ kind = "pytest" project = "libs/policyengine-simulation-persistence" nodes = [ "tests/test_public_api_schema.py::test_the_models_match_the_public_api_tables", + "tests/test_public_api_schema.py::test_the_input_models_match_the_public_api_columns", "tests/test_service_writes.py::test_the_service_role_holds_every_privilege_the_service_needs", "tests/test_service_writes.py::test_the_service_role_creates_and_reads_a_row_in_every_table_it_writes", ]