diff --git a/.github/scripts/simulation-staging-preemption-check.sh b/.github/scripts/simulation-staging-preemption-check.sh new file mode 100755 index 000000000..4b59ae36c --- /dev/null +++ b/.github/scripts/simulation-staging-preemption-check.sh @@ -0,0 +1,461 @@ +#!/bin/bash +# Stop the Modal container running a staging calculation, as a pre-emption +# does, and require the calculation to succeed anyway. +# Usage: simulation-staging-preemption-check.sh +# Run from projects/policyengine-simulation-executor, whose environment has +# the Modal client. +# +# household: calculates the stored staging household through the deployed +# Simulation Entrypoint, and stops the household function's container once +# the job is running, that is once the function has claimed it. +# report: creates the staging economy check's report, current law against +# current law for Wales in 2026, whose simulation runs that check has just +# left to reuse, and stops the coordinator's container once the report run +# is running, that is once the coordinator has claimed it. +# +# The container is the one Modal's call graph names for the Modal call the +# record names (modal_call_id), and it must be listed in Modal's staging +# environment. `modal container stop` sends it SIGINT, and Modal gives the +# call's input to another container under the same call ID, as it does +# after a pre-emption. The check passes once the record succeeds under that +# call, completed at least 5 seconds after the stop (both counted in whole +# seconds), and the call graph shows the call's input finished in another +# container. A record completed sooner proves nothing: the stopped container +# may have written the success before the signal reached it, and Modal's +# rerun then finds the record finished and only returns its summary. A +# rerun in a new container can't finish within 5 seconds of the stop. +# A calculation that finishes before its container is stopped proves +# nothing, so a new one is submitted, up to PREEMPTION_CHECK_ATTEMPTS +# times, each a fixed wait after the last attempt ended: on a container +# the last attempt left warm, the calculation would finish too fast to +# stop. Modal stops a container left idle for 60 seconds by default, and +# the household functions and the coordinator don't change that. A +# container that finished the last calculation unstopped went idle when +# the record finished, so the wait covers it. One that Modal ran a stopped +# call in again goes idle only when that rerun returns, which may be after +# the attempt ended, so the wait covers it only when the rerun returns +# within about 30 seconds of the attempt's end. A record that fails, or +# doesn't finish within the deadline, fails the check at once. A stop that +# fails, as Modal's does for a container that has ended, is not tried +# again: the check follows the record to its end, so a record left running +# by a container that ended for another reason, such as running out of +# memory, fails at its deadline rather than being submitted again. Every +# Modal command may take 60 seconds. +# +# Required env vars: the test client's SIMULATION_TEST_AUTH_ISSUER, +# SIMULATION_TEST_AUTH_AUDIENCE, SIMULATION_TEST_AUTH_CLIENT_ID and +# SIMULATION_TEST_AUTH_CLIENT_SECRET; MODAL_TOKEN_ID and MODAL_TOKEN_SECRET, +# which the Modal client reads; MODAL_ENVIRONMENT, which must be staging; +# and, for household, STAGING_HOUSEHOLD_ID (the ID of a US household stored +# in the staging Public API). +# Optional: PREEMPTION_CHECK_ATTEMPTS (3); PREEMPTION_CHECK_DEADLINE_SECONDS, +# each attempt's deadline (600 for household, 900 for report); +# PREEMPTION_CHECK_POLL_SECONDS (2); PREEMPTION_CHECK_RERUN_SECONDS (120), +# how long after the record succeeds the call graph may take to show the +# call's other container; and PREEMPTION_CHECK_SCALEDOWN_SECONDS (90), how +# long to wait before a new attempt. +# The access token is sent from a file, and the client secret on standard +# input through jq's environment, so neither appears on any command line. +# A submission, which isn't asked again, may take 60 seconds, Cloud Run's +# own limit; every other request 30. A read that gets no answer, a 429 or +# any 5xx is asked again until the deadline. + +set -euo pipefail + +fail() { + printf 'Staging preemption check failed: %s\n' "$1" >&2 + exit 1 +} + +kind="${1:-}" +base_url="${2:?Simulation Entrypoint base URL is required}" +base_url="${base_url%/}" +case "${kind}" in + household) default_deadline_seconds=600 ;; + report) default_deadline_seconds=900 ;; + *) fail "the first argument names the calculation, household or report, not '${kind}'" ;; +esac +attempts="${PREEMPTION_CHECK_ATTEMPTS:-3}" +deadline_seconds="${PREEMPTION_CHECK_DEADLINE_SECONDS:-${default_deadline_seconds}}" +poll_seconds="${PREEMPTION_CHECK_POLL_SECONDS:-2}" +rerun_seconds="${PREEMPTION_CHECK_RERUN_SECONDS:-120}" +scaledown_seconds="${PREEMPTION_CHECK_SCALEDOWN_SECONDS:-90}" +request_timeout_seconds=30 +submission_timeout_seconds=60 +modal_command_timeout_seconds=60 +# A record completed sooner after the stop may have been completed by the +# stopped container, not by Modal's rerun in a new one. +rerun_minimum_seconds=5 +uuid_pattern='^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' +call_id_pattern='^fc-[0-9A-Za-z]+$' +container_id_pattern='^ta-[0-9A-Za-z]+$' + +required_variables=(SIMULATION_TEST_AUTH_ISSUER SIMULATION_TEST_AUTH_AUDIENCE + SIMULATION_TEST_AUTH_CLIENT_ID SIMULATION_TEST_AUTH_CLIENT_SECRET + MODAL_TOKEN_ID MODAL_TOKEN_SECRET MODAL_ENVIRONMENT) +if [[ "${kind}" == "household" ]]; then + required_variables+=(STAGING_HOUSEHOLD_ID) +fi +for variable_name in "${required_variables[@]}"; do + [[ -n "${!variable_name:-}" ]] || fail "${variable_name} must be set" +done +[[ "${MODAL_ENVIRONMENT}" == "staging" ]] \ + || fail "MODAL_ENVIRONMENT is ${MODAL_ENVIRONMENT}: this check stops containers, so it runs only in Modal's staging environment" +if [[ "${kind}" == "household" ]]; then + [[ "${STAGING_HOUSEHOLD_ID}" =~ ${uuid_pattern} ]] \ + || fail "STAGING_HOUSEHOLD_ID is not a UUID in lowercase hexadecimal" +fi +[[ "${attempts}" =~ ^[1-9][0-9]*$ ]] \ + || fail "PREEMPTION_CHECK_ATTEMPTS must be a whole number of at least 1" +for setting in "PREEMPTION_CHECK_DEADLINE_SECONDS=${deadline_seconds}" \ + "PREEMPTION_CHECK_POLL_SECONDS=${poll_seconds}" \ + "PREEMPTION_CHECK_RERUN_SECONDS=${rerun_seconds}" \ + "PREEMPTION_CHECK_SCALEDOWN_SECONDS=${scaledown_seconds}"; do + [[ "${setting#*=}" =~ ^[0-9]+$ ]] \ + || fail "${setting%%=*} must be a whole number of seconds" +done + +work_directory="$(mktemp -d)" +trap 'rm -rf "${work_directory}"' EXIT +chmod 700 "${work_directory}" + +# Send one request, allowing it ; write its body to .json and +# print its HTTP status. It fails when no answer arrives. +send() { + local name="$1" + local seconds="$2" + shift 2 + curl --silent --show-error --max-time "${seconds}" \ + --output "${work_directory}/${name}.json" --write-out '%{http_code}' "$@" +} + +token_status="$( + jq -n '{ + client_id: env.SIMULATION_TEST_AUTH_CLIENT_ID, + client_secret: env.SIMULATION_TEST_AUTH_CLIENT_SECRET, + audience: env.SIMULATION_TEST_AUTH_AUDIENCE, + grant_type: "client_credentials" + }' \ + | send token "${request_timeout_seconds}" --request POST --header 'Content-Type: application/json' \ + --data @- "${SIMULATION_TEST_AUTH_ISSUER%/}/oauth/token" +)" || fail "the token request got no answer" +[[ "${token_status}" == "200" ]] || fail "the token request answered ${token_status}" +access_token="$(jq -r '.access_token // empty' "${work_directory}/token.json")" +[[ -n "${access_token}" ]] || fail "the token response has no access_token" +authorization_header="${work_directory}/authorization-header" +printf 'Authorization: Bearer %s\n' "${access_token}" > "${authorization_header}" + +# Run a command in the executor's environment, which has the Modal client, +# allowing it modal_command_timeout_seconds. +modal_command() { + timeout "${modal_command_timeout_seconds}" uv run --frozen "$@" +} + +# Print the inputs of Modal call as JSON; fail when Modal can't be +# asked (src/modal/utils/describe_call_inputs.py). +call_inputs() { + modal_command python -m src.modal.utils.describe_call_inputs "$1" +} + +# Return 0 when Modal lists container as running in +# MODAL_ENVIRONMENT, 1 when it doesn't, and 2 when Modal couldn't list the +# environment's containers as a JSON list of entries that each have a +# "Container ID", which says nothing about the container: a change in +# Modal's output must not read as the container having gone. +container_listed() { + local listing listed + if ! listing="$(modal_command modal container list --env "${MODAL_ENVIRONMENT}" --json)"; then + printf 'Modal could not list the containers in environment %s; asking again\n' \ + "${MODAL_ENVIRONMENT}" >&2 + return 2 + fi + listed="$( + jq -r --arg container_id "$1" \ + 'if type == "array" + and all(.[]; type == "object" and has("Container ID")) + then any(.[]; .["Container ID"] == $container_id) + else error("not a list of containers") end' <<<"${listing}" 2>/dev/null + )" || listed="" + case "${listed}" in + true) return 0 ;; + false) return 1 ;; + *) + printf 'Modal could not list the containers in environment %s readably; asking again\n' \ + "${MODAL_ENVIRONMENT}" >&2 + return 2 + ;; + esac +} + +# Create the attempt's record. Sets record_url, record_noun, record_label +# and, for a report, report_run_id. +submit() { + local submission="${work_directory}/submission.json" + local submission_status + if [[ "${kind}" == "household" ]]; then + submission_status="$( + jq -n --arg household_id "${STAGING_HOUSEHOLD_ID}" \ + '{country: "us", household_id: $household_id}' \ + | send submission "${submission_timeout_seconds}" --request POST \ + --header "@${authorization_header}" --header 'Content-Type: application/json' \ + --data @- "${base_url}/household-calculations" + )" || fail "POST /household-calculations got no answer; it may have created a household job" + [[ "${submission_status}" == "202" ]] \ + || fail "POST /household-calculations answered ${submission_status}: $(cat "${submission}")" + local job_id + job_id="$(jq -r '.job_id // empty' "${submission}")" + [[ "${job_id}" =~ ${uuid_pattern} ]] || fail "the submission named no household job" + record_url="${base_url}/household-calculations/${job_id}" + record_noun="household job" + record_label="${record_noun} ${job_id}" + printf 'Submitted household job %s for staging household %s\n' \ + "${job_id}" "${STAGING_HOUSEHOLD_ID}" + else + submission_status="$( + jq -n '{ + country: "uk", + report_type: "economy", + label: "Staging preemption check", + year: 2026, + region_code: "country/wales", + baseline: {policy_id: null, dynamic_id: null}, + reform: {policy_id: null, dynamic_id: null}, + include_cliffs: false + }' \ + | send submission "${submission_timeout_seconds}" --request POST \ + --header "@${authorization_header}" --header 'Content-Type: application/json' \ + --data @- "${base_url}/reports" + )" || fail "POST /reports got no answer; it may have created a report" + [[ "${submission_status}" == "201" ]] \ + || fail "POST /reports answered ${submission_status}: $(cat "${submission}")" + local report_id + report_id="$(jq -r '.report_id // empty' "${submission}")" + report_run_id="$(jq -r '.report_run_id // empty' "${submission}")" + [[ "${report_id}" =~ ${uuid_pattern} ]] || fail "the submission named no report" + [[ "${report_run_id}" =~ ${uuid_pattern} ]] || fail "the submission named no report run" + record_url="${base_url}/reports/${report_id}" + record_noun="report run" + record_label="${record_noun} ${report_run_id}" + printf 'Created report %s, run %s (baseline reused: %s, reform reused: %s)\n' \ + "${report_id}" "${report_run_id}" \ + "$(jq -r '.baseline_reused' "${submission}")" "$(jq -r '.reform_reused' "${submission}")" + fi +} + +# Read the record once. Sets record_status, record_call_id, record_error, +# record_completed_at and last_read; leaves record_status empty when the +# service can't answer now. +read_record() { + local record="${work_directory}/record.json" + local read_status + record_status="" + if ! read_status="$( + send record "${request_timeout_seconds}" --header "@${authorization_header}" "${record_url}" + )"; then + last_read="it got no answer" + printf 'GET %s got no answer; asking again\n' "${record_url}" >&2 + return 0 + fi + if [[ "${read_status}" == "429" || "${read_status}" == 5?? ]]; then + # The service, or Cloud Run's front end in its place, can't answer now. + last_read="it answered ${read_status}" + printf 'GET %s answered %s; asking again\n' "${record_url}" "${read_status}" >&2 + return 0 + fi + [[ "${read_status}" == "200" ]] \ + || fail "GET ${record_url} answered ${read_status}: $(cat "${record}")" + # The household job's fields, or the report's newest run's. + local record_filter='' + if [[ "${kind}" == "report" ]]; then + local newest_run_id + newest_run_id="$(jq -r '.run.run_id // empty' "${record}")" + [[ "${newest_run_id}" == "${report_run_id}" ]] \ + || fail "the report's newest run is ${newest_run_id:-missing}, not the created run ${report_run_id}" + record_filter='.run' + fi + record_status="$(jq -r "${record_filter}.status" "${record}")" + record_call_id="$(jq -r "${record_filter}.modal_call_id // empty" "${record}")" + record_error="$(jq -r "${record_filter}.error_message // \"no error message\"" "${record}")" + record_completed_at="$(jq -r "${record_filter}.completed_at // empty" "${record}")" + if [[ -n "${record_call_id}" && ! "${record_call_id}" =~ ${call_id_pattern} ]]; then + fail "${record_label} names Modal call '${record_call_id}', which is not a Modal call ID" + fi + if [[ "${kind}" == "household" && "${record_status}" == "succeeded" ]]; then + [[ "$(jq -r '.result.status // empty' "${record}")" == "succeeded" ]] \ + || fail "${record_label} succeeded without a succeeded result document" + fi + last_read="the ${record_noun} was ${record_status}" + return 0 +} + +# Find the container running the record's call. Sets container_id once the +# call graph names a running input's container that Modal lists in +# MODAL_ENVIRONMENT. Until then, listing_note says what the last listing +# found out about the call graph's container: that Modal doesn't list it, +# or that Modal couldn't list the containers. Finding the container clears +# it, so a missed attempt's reason names only the last listing's outcome. +find_container() { + local inputs candidate + if ! inputs="$(call_inputs "${record_call_id}")"; then + printf 'Modal could not say which container runs call %s yet; asking again\n' \ + "${record_call_id}" >&2 + return 0 + fi + candidate="$( + jq -r '[.[] | select(.status == "PENDING" and .container_id != "")] | first | .container_id // empty' \ + <<<"${inputs}" + )" || fail "the call-input helper printed something other than a list of inputs for call ${record_call_id}" + [[ -n "${candidate}" ]] || return 0 + [[ "${candidate}" =~ ${container_id_pattern} ]] \ + || fail "Modal names '${candidate}' as call ${record_call_id}'s container, which is not a Modal container ID" + local listed_status=0 + container_listed "${candidate}" || listed_status=$? + case "${listed_status}" in + 0) + container_id="${candidate}" + listing_note="" + printf 'Call %s of %s runs in container %s\n' "${record_call_id}" "${record_label}" "${container_id}" + ;; + 1) + listing_note="container ${candidate} is not listed in Modal environment ${MODAL_ENVIRONMENT}" + printf '%s; asking again\n' "${listing_note}" >&2 + ;; + # Modal couldn't list: the next read asks again. + *) listing_note="Modal could not list the containers in environment ${MODAL_ENVIRONMENT}" ;; + esac +} + +# Return 0 when the record completed after its container was stopped, so +# its success comes from the rerun; otherwise set attempt_outcome to missed. +completed_after_the_stop() { + local completed_epoch + [[ -n "${record_completed_at}" ]] \ + || fail "${record_label} succeeded with no completion time" + completed_epoch="$( + jq -rn --arg completed_at "${record_completed_at}" \ + '$completed_at | sub("\\.[0-9]+"; "") | sub("\\+00:00$"; "Z") | fromdateiso8601' + )" || fail "${record_label}'s completion time ${record_completed_at} is not a UTC time" + if ((completed_epoch < stopped_at_epoch)); then + attempt_outcome="missed" + miss_reason="${record_label} completed at ${record_completed_at}, before its container ${stopped_container_id} was stopped, so Modal's rerun of call ${stopped_call_id} found it finished" + return 1 + fi + if ((completed_epoch - stopped_at_epoch < rerun_minimum_seconds)); then + attempt_outcome="missed" + miss_reason="${record_label} completed at ${record_completed_at}, $((completed_epoch - stopped_at_epoch)) seconds after its container ${stopped_container_id} was stopped: too soon for Modal to have run call ${stopped_call_id} again in a new container, so the stopped container may have finished it before the signal reached it" + return 1 + fi + return 0 +} + +# After the record succeeded: require the call graph to show the stopped +# call's input finished in another container. Sets attempt_outcome. +confirm_rerun() { + local waiting_since="${SECONDS}" + local inputs rerun_container_id="" + while true; do + if inputs="$(call_inputs "${stopped_call_id}")"; then + rerun_container_id="$( + jq -r --arg stopped "${stopped_container_id}" \ + '[.[] | select(.status == "SUCCESS" and .container_id != "" and .container_id != $stopped)] | first | .container_id // empty' \ + <<<"${inputs}" + )" || fail "the call-input helper printed something other than a list of inputs for call ${stopped_call_id}" + fi + if [[ -n "${rerun_container_id}" ]]; then + printf '%s succeeded after its container %s was stopped: Modal ran call %s again in container %s\n' \ + "${record_label}" "${stopped_container_id}" "${stopped_call_id}" "${rerun_container_id}" + attempt_outcome="proven" + return 0 + fi + if ((SECONDS - waiting_since >= rerun_seconds)); then + attempt_outcome="missed" + miss_reason="${record_label} succeeded, but Modal's call graph never showed call ${stopped_call_id} finishing in a container other than the stopped ${stopped_container_id} within ${rerun_seconds} seconds, so it may have finished before the stop" + return 0 + fi + sleep "${poll_seconds}" + done +} + +# Submit one calculation, stop its container once it's running, and follow +# it to the end. Sets attempt_outcome to proven or missed (with +# miss_reason); fails the check for a failed or late calculation. +attempt_once() { + submit + record_call_id="" + container_id="" + stopped_container_id="" + stopped_call_id="" + stopped_at_epoch=0 + listing_note="" + stop_failure="" + last_read="nothing yet" + local started_at="${SECONDS}" + while true; do + read_record + # A read the service answered (record_status set) must name the stopped + # call: Modal runs it again under the same ID. + if [[ -n "${record_status}" && -n "${stopped_call_id}" \ + && "${record_call_id}" != "${stopped_call_id}" ]]; then + fail "${record_label} names ${record_call_id:+call }${record_call_id:-no call} after its container ${stopped_container_id} was stopped, not call ${stopped_call_id}, which Modal runs again under the same ID" + fi + case "${record_status}" in + "") ;; + succeeded) + if [[ -n "${stop_failure}" ]]; then + attempt_outcome="missed" + miss_reason="${record_label} finished after ${stop_failure}" + elif [[ -z "${stopped_container_id}" ]]; then + attempt_outcome="missed" + miss_reason="${record_label} finished before its container was stopped${listing_note:+ (${listing_note})}" + elif completed_after_the_stop; then + confirm_rerun + fi + return 0 + ;; + failed) + fail "${record_label} failed${stopped_container_id:+ after its container ${stopped_container_id} was stopped}${stop_failure:+ after ${stop_failure}}: ${record_error}" + ;; + pending | running) + if [[ -z "${stopped_container_id}" && -z "${stop_failure}" && -n "${record_call_id}" ]]; then + [[ -n "${container_id}" ]] || find_container + if [[ "${record_status}" == "running" && -n "${container_id}" ]]; then + local stop_status=0 + modal_command modal container stop --yes "${container_id}" || stop_status=$? + if ((stop_status == 0)); then + stopped_at_epoch="$(date -u +%s)" + stopped_container_id="${container_id}" + stopped_call_id="${record_call_id}" + printf 'Stopped container %s while %s was running; waiting for Modal to run call %s again\n' \ + "${stopped_container_id}" "${record_label}" "${stopped_call_id}" + else + # Modal refuses to stop a container that has ended. The + # check doesn't stop it again or submit another calculation + # while this one may still be running. + stop_failure="modal container stop failed for container ${container_id} (exit status ${stop_status}) while ${record_label} was running" + printf '%s; following it to its end\n' "${stop_failure}" >&2 + fi + fi + fi + ;; + *) fail "${record_label} has an unknown status: ${record_status}" ;; + esac + ((SECONDS - started_at < deadline_seconds)) \ + || fail "${record_label} did not finish within ${deadline_seconds} seconds (last read: ${last_read})${stop_failure:+; ${stop_failure}}" + sleep "${poll_seconds}" + done +} + +for ((attempt = 1; attempt <= attempts; attempt++)); do + printf 'Attempt %d of %d\n' "${attempt}" "${attempts}" + attempt_outcome="" + miss_reason="" + attempt_once + [[ "${attempt_outcome}" != "proven" ]] || exit 0 + printf 'Attempt %d of %d proved nothing: %s\n' "${attempt}" "${attempts}" "${miss_reason}" >&2 + if ((attempt < attempts)); then + printf 'Before the next attempt, waiting %d seconds for Modal to stop the containers the last one left idle\n' \ + "${scaledown_seconds}" + sleep "${scaledown_seconds}" + fi +done +fail "no attempt in ${attempts} showed a ${kind} calculation finishing in another container after its container was stopped; the last: ${miss_reason}" diff --git a/.github/scripts/simulation-staging-preemption-ready.sh b/.github/scripts/simulation-staging-preemption-ready.sh new file mode 100755 index 000000000..1bc0ab5a5 --- /dev/null +++ b/.github/scripts/simulation-staging-preemption-ready.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Require the staging preemption check to have succeeded in a staging +# deployment, and to have been skipped in any other. +# Required env vars: REQUESTED ("true" in a staging deployment) and +# CHECK_RESULT (the staging_preemption_check job result). + +set -euo pipefail + +if [[ "${REQUESTED}" == "true" ]]; then + test "${CHECK_RESULT}" = "success" +else + test "${CHECK_RESULT}" = "skipped" +fi diff --git a/.github/workflows/simulation-deploy.reusable.yml b/.github/workflows/simulation-deploy.reusable.yml index b89de5283..2922d9fe6 100644 --- a/.github/workflows/simulation-deploy.reusable.yml +++ b/.github/workflows/simulation-deploy.reusable.yml @@ -1290,6 +1290,78 @@ jobs: SIMULATION_TEST_AUTH_CLIENT_SECRET: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_SECRET }} run: .github/scripts/simulation-staging-economy-check.sh "${{ needs.deploy_entrypoint.outputs.candidate_url }}" + staging_preemption_check: + name: Stop staging calculations' Modal containers and require them to succeed + # Every staging deployment calculates the staging household, then + # creates the staging economy check's report, through the candidate, and + # stops the Modal container running each one's call with `modal + # container stop` once the household function or coordinator has claimed + # it. That sends SIGINT and gives the input to another container under + # the same call ID, as a pre-emption does; each must still succeed. It + # runs after the economy check, so the report reuses that check's + # simulation runs and stopping the coordinator costs no new simulation. + # The script refuses any Modal environment but staging, and stops only a + # container Modal lists there. Each calculation writes rows in the + # staging database, as the other staging checks' do. + if: >- + ${{ + !cancelled() && + inputs.deployment_environment == 'staging' && + needs.deploy_entrypoint.result == 'success' && + needs.authenticated_test.result == 'success' && + needs.staging_economy_check.result == 'success' && + ((inputs.deploy_stage12_v2 && needs.publish_stage12_v2_default.result == 'success') || + (!inputs.deploy_stage12_v2 && needs.publish_stage12_v2_default.result == 'skipped')) + }} + needs: [deploy_entrypoint, authenticated_test, publish_stage12_v2_default, staging_economy_check] + runs-on: ubuntu-latest + # Longer than both steps' worst case together, so the script fails + # first: about 126 minutes, with every request and Modal command taking + # its full time limit (three attempts of up to 17.5 minutes for the + # household and 22.5 for the report, and a 90-second wait before each + # new attempt), and the install. + timeout-minutes: 150 + environment: ${{ inputs.release_environment }} + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - uses: astral-sh/setup-uv@v8.1.0 + with: + version: "0.13.6" + + - name: Install the Modal client + working-directory: projects/policyengine-simulation-executor + run: uv sync --frozen + + - name: Stop a staging household calculation's container and require it to succeed + working-directory: projects/policyengine-simulation-executor + env: + STAGING_HOUSEHOLD_ID: ${{ vars.SIMULATION_STAGING_HOUSEHOLD_ID }} + SIMULATION_TEST_AUTH_ISSUER: ${{ secrets.SIMULATION_ENTRYPOINT_AUTH_ISSUER }} + SIMULATION_TEST_AUTH_AUDIENCE: ${{ secrets.SIMULATION_ENTRYPOINT_AUTH_AUDIENCE }} + SIMULATION_TEST_AUTH_CLIENT_ID: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_ID }} + SIMULATION_TEST_AUTH_CLIENT_SECRET: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_SECRET }} + MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} + MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} + MODAL_ENVIRONMENT: ${{ inputs.modal_environment }} + run: ../../.github/scripts/simulation-staging-preemption-check.sh household "${{ needs.deploy_entrypoint.outputs.candidate_url }}" + + - name: Stop a staging report's coordinator container and require it to succeed + working-directory: projects/policyengine-simulation-executor + env: + SIMULATION_TEST_AUTH_ISSUER: ${{ secrets.SIMULATION_ENTRYPOINT_AUTH_ISSUER }} + SIMULATION_TEST_AUTH_AUDIENCE: ${{ secrets.SIMULATION_ENTRYPOINT_AUTH_AUDIENCE }} + SIMULATION_TEST_AUTH_CLIENT_ID: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_ID }} + SIMULATION_TEST_AUTH_CLIENT_SECRET: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_SECRET }} + MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} + MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} + MODAL_ENVIRONMENT: ${{ inputs.modal_environment }} + run: ../../.github/scripts/simulation-staging-preemption-check.sh report "${{ needs.deploy_entrypoint.outputs.candidate_url }}" + finalize_routing: name: Finalize the Modal routing publication # Finish the durable journal after every gate, including failed uploads @@ -1348,7 +1420,7 @@ jobs: # The reporter only reads job results. Journal completion happens before # it, so a failure or cancellation here cannot strand a publication. if: ${{ !cancelled() }} - needs: [deploy_entrypoint, update_routing, integration, authenticated_test, promote_entrypoint, publish_latest, finalize_routing, publish_stage12_v2_default, staging_household_check, staging_economy_check] + needs: [deploy_entrypoint, update_routing, integration, authenticated_test, promote_entrypoint, publish_latest, finalize_routing, publish_stage12_v2_default, staging_household_check, staging_economy_check, staging_preemption_check] runs-on: ubuntu-latest defaults: run: @@ -1389,6 +1461,12 @@ jobs: CHECK_RESULT: ${{ needs.staging_economy_check.result }} run: .github/scripts/simulation-staging-economy-ready.sh + - name: Check the staging preemption check result + env: + REQUESTED: ${{ inputs.deployment_environment == 'staging' }} + CHECK_RESULT: ${{ needs.staging_preemption_check.result }} + run: .github/scripts/simulation-staging-preemption-ready.sh + cleanup_entrypoint_revisions: name: Remove superseded Cloud Run revisions if: ${{ !cancelled() && inputs.promote_entrypoint && needs.deployment_ready.result == 'success' }} diff --git a/changelog.d/1062.added.md b/changelog.d/1062.added.md new file mode 100644 index 000000000..8e8d823ce --- /dev/null +++ b/changelog.d/1062.added.md @@ -0,0 +1 @@ +Every staging deploy stops the Modal container running a household calculation and a small economy report, as a pre-emption does, and fails unless both still succeed; `GET /household-calculations/{household_job_id}` and the run in `GET /reports/{report_id}` now name the Modal call recorded on them (`modal_call_id`). diff --git a/libs/policyengine-simulation-persistence/README.md b/libs/policyengine-simulation-persistence/README.md index 15b5b51d7..f1609e79a 100644 --- a/libs/policyengine-simulation-persistence/README.md +++ b/libs/policyengine-simulation-persistence/README.md @@ -111,7 +111,8 @@ call's ID, and `fail_unstarted_report_run` and `fail_unstarted_simulation_run` fail a run at once, each only while it is `pending` with no call ID; failing a report run also fails the simulation runs created for it. `economy_run_reads.py` reads a report, or a simulation, -with its newest run's state. +with its newest run's state; a report run's state includes the Modal call +recorded on it. `EconomyRunStore` runs them as the Simulation Entrypoint does, each step in a transaction of its own: `read_report_inputs` reads the catalog version, diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_run_reads.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_run_reads.py index 1cde162ad..6788e801a 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_run_reads.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_run_reads.py @@ -44,7 +44,11 @@ class SimulationRunState: @dataclass(frozen=True) class ReportRunState: - """A report run's state, and the simulation runs it compares.""" + """A report run's state, the simulation runs it compares, and its call. + + ``modal_call_id`` is the coordinator's Modal call recorded on the run, + or None before the service has recorded one. + """ id: UUID status: ReportRunStatus @@ -56,6 +60,7 @@ class ReportRunState: completed_at: datetime | None baseline_simulation_run_id: UUID reform_simulation_run_id: UUID | None + modal_call_id: str | None @dataclass(frozen=True) @@ -104,6 +109,7 @@ def read_economy_report( report_runs.c.completed_at, report_runs.c.baseline_simulation_run_id, report_runs.c.reform_simulation_run_id, + report_runs.c.modal_call_id, ) .where(report_runs.c.report_id == report_id) .order_by(report_runs.c.created_at.desc(), report_runs.c.id.desc()) @@ -178,4 +184,5 @@ def _report_run_state(run: RowMapping) -> ReportRunState: completed_at=run["completed_at"], baseline_simulation_run_id=run["baseline_simulation_run_id"], reform_simulation_run_id=run["reform_simulation_run_id"], + modal_call_id=run["modal_call_id"], ) diff --git a/libs/policyengine-simulation-persistence/tests/test_economy_run_reads.py b/libs/policyengine-simulation-persistence/tests/test_economy_run_reads.py index 836b63c23..ff89477b7 100644 --- a/libs/policyengine-simulation-persistence/tests/test_economy_run_reads.py +++ b/libs/policyengine-simulation-persistence/tests/test_economy_run_reads.py @@ -157,6 +157,52 @@ def test_a_failed_newest_run_is_read_with_its_error(public_api_database_url) -> assert report.run.completed_at is not None +def test_a_report_run_is_read_with_the_modal_call_it_records( + public_api_database_url, +) -> None: + """A run names the Modal call recorded on it, and no call before one is. + + The staging preemption check finds the coordinator's container from it. + """ + with rolled_back_transaction(public_api_database_url) as connection: + economy = insert_economy_catalog(connection) + report_id, run_ids = _report_with_runs(connection, economy) + for run_id in run_ids: + connection.execute( + update(report_runs) + .where(report_runs.c.id == run_id) + .values(modal_call_id=f"fc-{run_id.hex}") + ) + population = economy.population() + uncalled_run_id = insert_report_run_naming( + connection, + economy, + ( + insert_simulation_run(connection, population), + insert_simulation_run(connection, population), + ), + ReportRunOutcome(status=ReportRunStatus.PENDING), + ) + uncalled_report_id = ( + connection.execute( + report_runs.select().where(report_runs.c.id == uncalled_run_id) + ) + .one() + .report_id + ) + act_as_the_service(connection) + + report = read_economy_report(connection, report_id) + uncalled_report = read_economy_report(connection, uncalled_report_id) + + assert report is not None + assert report.run is not None + assert report.run.modal_call_id == f"fc-{report.run.id.hex}" + assert uncalled_report is not None + assert uncalled_report.run is not None + assert uncalled_report.run.modal_call_id is None + + def test_a_simulation_is_read_with_its_newest_run(public_api_database_url) -> None: with rolled_back_transaction(public_api_database_url) as connection: economy = insert_economy_catalog(connection) diff --git a/projects/policyengine-apis-integ/tests/test_economy_generated_client.py b/projects/policyengine-apis-integ/tests/test_economy_generated_client.py index 5f32692db..737786e45 100644 --- a/projects/policyengine-apis-integ/tests/test_economy_generated_client.py +++ b/projects/policyengine-apis-integ/tests/test_economy_generated_client.py @@ -177,6 +177,7 @@ def test_a_report_and_a_simulation_are_read_with_their_newest_runs(): "completed_at": "2026-11-12T15:04:00Z", "baseline_simulation_run_id": SIDE_IDS[1], "reform_simulation_run_id": SIDE_IDS[3], + "modal_call_id": "fc-coordinator", }, } simulation = {"simulation_id": REPORT_ID, "run": None} @@ -192,5 +193,6 @@ def test_a_report_and_a_simulation_are_read_with_their_newest_runs(): run = report_response.parsed.run assert run is not None assert run.status is ReportRunStatus.SUCCEEDED + assert run.modal_call_id == "fc-coordinator" assert isinstance(simulation_response.parsed, EconomySimulationState) assert simulation_response.parsed.to_dict() == simulation diff --git a/projects/policyengine-apis-integ/tests/test_household_generated_client.py b/projects/policyengine-apis-integ/tests/test_household_generated_client.py index a8dc72a69..95519ace0 100644 --- a/projects/policyengine-apis-integ/tests/test_household_generated_client.py +++ b/projects/policyengine-apis-integ/tests/test_household_generated_client.py @@ -74,6 +74,7 @@ def test_a_job_keeps_its_result_document_whole(): "year": 2026, "policyengine_version": "6.17.1", "country_package_version": "2.2.1", + "modal_call_id": "fc-1", "result": document, "error_message": None, "created_at": "2026-11-10T12:00:00Z", @@ -84,6 +85,7 @@ def test_a_job_keeps_its_result_document_whole(): parsed = HouseholdCalculationState.from_dict(state) assert parsed.status is HouseholdJobStatus.SUCCEEDED + assert parsed.modal_call_id == "fc-1" assert parsed.to_dict()["result"] == document diff --git a/projects/policyengine-simulation-entry/DEPLOYMENT.md b/projects/policyengine-simulation-entry/DEPLOYMENT.md index 5972daaff..ee0c89d66 100644 --- a/projects/policyengine-simulation-entry/DEPLOYMENT.md +++ b/projects/policyengine-simulation-entry/DEPLOYMENT.md @@ -142,6 +142,67 @@ check has succeeded, later checks on that version reuse its simulation runs, so they check the coordinator's use of stored outputs rather than the workers', which the v2 deployment's own validation calls. +### The staging preemption check + +Every staging deployment runs `staging_preemption_check` once the economy +check has succeeded. `.github/scripts/simulation-staging-preemption-check.sh` +runs twice, from the executor project, whose environment has the Modal +client: + +- `household` calculates the staging household that + `SIMULATION_STAGING_HOUSEHOLD_ID` names, as the household check does; +- `report` creates the economy check's report again (labelled "Staging + preemption check"), whose simulation runs reuse that check's, so it costs + no new simulation. + +Once the household job or report run is `running`, that is once the +household function or coordinator has claimed it, the check takes the +`modal_call_id` its read answers, asks Modal's call graph which container +runs that call (`src/modal/utils/describe_call_inputs.py`), and stops that +container with `modal container stop`. Modal sends the container SIGINT and +gives the input to another container under the same call ID, as it does +after a pre-emption. The check requires the record to succeed under that +call, completed at least 5 seconds after the stop, and the call graph to +show the call's input finished in another container. A record completed +sooner proves nothing, and that attempt counts as missed: the stopped +container may have written the success before the signal reached it, and +Modal's rerun then finds the record finished and only returns its summary. +A rerun in a new container can't finish within 5 seconds of the stop (both +times are counted in whole seconds). Run again, the household function or +coordinator resumes its claim. The report's simulation runs are reused, so +its coordinator starts no worker, and the check doesn't cover a coordinator +starting its workers again. + +A calculation that finishes before its container is stopped, for example on +a warm container, proves nothing, so the check submits a new one, up to +three attempts. Each new attempt starts 90 seconds after the last one ended. +Modal stops a container left idle for 60 seconds by default, and these +functions don't change that. A container that finished the last calculation +unstopped went idle when the record finished, so the wait covers it, and the +next calculation starts in a new container. A container that Modal ran a +stopped call in again goes idle only when that rerun returns, which may be +after the attempt ended, so the wait covers it only when the rerun returns +within about 30 seconds of the attempt's end. A record that fails, or +doesn't finish within its deadline (10 minutes for the household, 15 for the +report, per attempt), fails the check at once. Modal refuses to stop a +container that has ended; when a stop fails, the check doesn't stop it again +or submit another calculation, but follows the record to its end, so a +record left running by a container that ended for another reason, such as +running out of memory, fails the check at its deadline. Every Modal command +may take 60 seconds. + +It uses the repository's Modal token (the repository secrets +`MODAL_TOKEN_ID` and `MODAL_TOKEN_SECRET`), which must be allowed to read +call graphs and to list and stop containers. Production's deploy jobs use +the same token, and `modal container stop` takes a container ID in any +environment, so the script refuses any `MODAL_ENVIRONMENT` but `staging` and +stops only a container that `modal container list --env staging` lists. It +also uses the authenticated tests' client credentials. `deployment_ready` +requires the check to pass in staging, so a failure stops the production +deployment. +Each attempt adds a household job, or a report and its run, to the staging +database. + ## Candidate qualification For each eligible revision, the workflow first deploys `beta` diff --git a/projects/policyengine-simulation-entry/README.md b/projects/policyengine-simulation-entry/README.md index 0d1408c33..b96c94dae 100644 --- a/projects/policyengine-simulation-entry/README.md +++ b/projects/policyengine-simulation-entry/README.md @@ -85,8 +85,10 @@ default worker, creates a `household_jobs` row and starts the household function's call with `spawn`, answering 202 with the job's `job_id` and `status`. Each submission creates a new job. `GET /household-calculations/{household_job_id}` returns the job's status, -the household function's `result` document once it has one, and its -`error_message`. +the household function's `result` document once it has one, its +`error_message`, and `modal_call_id`, the household function's Modal call +recorded on the job (null until the service records one). Modal runs a +pre-empted call again under the same ID. A submission is refused before anything is written: with 400 and a `code` when the household, policy or catalog version for the default worker's .py @@ -185,8 +187,9 @@ reads. `GET /reports/{report_id}` and `GET /simulations/{simulation_id}` return the record and its newest run (created last, ties to the larger ID): its status, -versions, error message and times, and a report run's simulation runs. The -Public API reads results from the tables itself. +versions, error message and times, and a report run's simulation runs and +`modal_call_id`, the coordinator's Modal call recorded on it (null until the +service records one). The Public API reads results from the tables itself. ## Household reports diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_api_models.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_api_models.py index cd0e71b0a..d52ffafc0 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_api_models.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_api_models.py @@ -226,7 +226,12 @@ class SimulationRunRead(_Answer): class ReportRunRead(_Answer): - """A report run's state, and the simulation runs it compares.""" + """A report run's state, the simulation runs it compares, and its call. + + ``modal_call_id`` is the coordinator's Modal call recorded on the run, + or null before the service has recorded one. Modal runs a pre-empted + call again under the same ID. + """ run_id: UUID status: ReportRunStatus @@ -238,6 +243,7 @@ class ReportRunRead(_Answer): completed_at: datetime | None baseline_simulation_run_id: UUID reform_simulation_run_id: UUID | None + modal_call_id: str | None class EconomyReportState(_Answer): diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_routes.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_routes.py index 129ef37d1..ac89ffa06 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_routes.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_routes.py @@ -17,8 +17,9 @@ function, answering 201 with the simulation's ID, its run's ID and status. - ``GET /reports/{report_id}`` and ``GET /simulations/{simulation_id}`` (``simulation:read``) return the record with its newest run's status, - versions, error message and times. The Public API reads results from the - tables itself; these reads let a deploy check follow a run. + versions, error message and times, and a report run's coordinator call. + The Public API reads results from the tables itself; these reads let a + deploy check follow a run. A refused request answers 400 (a row it names, or a function it needs, is missing; ``code`` says which) or 422 (its fields, or the stored values it @@ -376,6 +377,7 @@ def _report_state(stored: StoredReport) -> EconomyReportState: completed_at=run.completed_at, baseline_simulation_run_id=run.baseline_simulation_run_id, reform_simulation_run_id=run.reform_simulation_run_id, + modal_call_id=run.modal_call_id, ), ) @@ -652,7 +654,8 @@ def _read_routes( summary="Get a report's newest run", description=( "Return the report, its two simulations and its newest run's " - "status, versions, error message and times." + "status, versions, error message, times and coordinator's Modal " + "call (null until one is recorded)." ), operation_id="get_report", response_model=EconomyReportState, diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_api_models.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_api_models.py index 36e4bf49b..d1a577c76 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_api_models.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_api_models.py @@ -130,7 +130,10 @@ class HouseholdCalculationState(BaseModel): calculated household in ``output``) or a ``HouseholdExecutionFailure`` (``status`` ``failed``, with the reason in ``error``). ``error_message`` summarizes a failure in one line; a job that failed before its function - ran has an ``error_message`` and no ``result``. + ran has an ``error_message`` and no ``result``. ``modal_call_id`` is the + household function's Modal call recorded on the job, or null before the + service has recorded one; Modal runs a pre-empted call again under the + same ID. """ model_config = ConfigDict(extra="forbid", frozen=True) @@ -143,6 +146,7 @@ class HouseholdCalculationState(BaseModel): year: int policyengine_version: str country_package_version: str + modal_call_id: str | None result: dict[str, JsonValue] | None error_message: str | None created_at: datetime diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_routes.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_routes.py index 660d4d13b..b71dd15f3 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_routes.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_routes.py @@ -7,7 +7,8 @@ read: poll the job rather than submitting again. - ``GET /household-calculations/{household_job_id}`` (``simulation:read``) returns the job as it stands: its status, the household function's result - document once it has one, and its error message. + document once it has one, its error message, and the household function's + Modal call recorded on it. A refused submission answers 400 (a row it names, or the household function it needs, is missing) or 422 (its fields, or the stored data it names, cannot @@ -242,6 +243,7 @@ def _state(record: HouseholdJobRecord) -> HouseholdCalculationState: year=record.year, policyengine_version=record.policyengine_version, country_package_version=record.country_package_version, + modal_call_id=record.modal_call_id, result=record.result, error_message=record.error_message, created_at=record.created_at, @@ -324,7 +326,9 @@ async def submit_household_calculation( summary="Get a household calculation", description=( "Return the household job as it stands: its status, the household " - "function's result document once it has one, and its error message." + "function's result document once it has one, its error message, " + "and the household function's Modal call recorded on it (null " + "until one is recorded)." ), operation_id="get_household_calculation", response_model=HouseholdCalculationState, diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py index 24e2d1a3e..3b0b61dc8 100644 --- a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py @@ -217,9 +217,10 @@ def _assert_deployment_environment_settings(reusable_workflow: str) -> None: # The job that moves latest after the checks reads the prod # environment's custom-hostname variable, the staging household # check reads the beta variable SIMULATION_STAGING_HOUSEHOLD_ID and - # the environment's auth secrets, and the staging economy check - # reads those auth secrets too. - reusable_workflow.count("environment: ${{ inputs.release_environment }}") == 19 + # the environment's auth secrets, the staging economy check reads + # those auth secrets too, and the staging preemption check reads them + # and that variable. + reusable_workflow.count("environment: ${{ inputs.release_environment }}") == 20 ) assert "APP_ENVIRONMENT: ${{ inputs.deployment_environment }}" in reusable_workflow assert "STAGE12_ENABLED_VALUE: ${{ vars.STAGE12_ENABLED }}" in reusable_workflow diff --git a/projects/policyengine-simulation-entry/tests/test_economy_routes.py b/projects/policyengine-simulation-entry/tests/test_economy_routes.py index 90ca6e108..b348936af 100644 --- a/projects/policyengine-simulation-entry/tests/test_economy_routes.py +++ b/projects/policyengine-simulation-entry/tests/test_economy_routes.py @@ -417,6 +417,7 @@ def test_without_its_database_and_v2_environment_none_is_built(settings_changes) completed_at=datetime.fromisoformat(READ_AT), baseline_simulation_run_id=SIDE_IDS[1], reform_simulation_run_id=SIDE_IDS[3], + modal_call_id="fc-coordinator", ), ) STORED_SIMULATION = EconomySimulationState(id=RECORD_ID, run=None) @@ -446,6 +447,7 @@ def test_a_report_is_read_with_its_newest_runs_status(client_for) -> None: "completed_at": READ_AT, "baseline_simulation_run_id": str(SIDE_IDS[1]), "reform_simulation_run_id": str(SIDE_IDS[3]), + "modal_call_id": "fc-coordinator", }, } assert missing.status_code == 404 diff --git a/projects/policyengine-simulation-entry/tests/test_economy_submissions.py b/projects/policyengine-simulation-entry/tests/test_economy_submissions.py index 062fa9b7c..2712a7a52 100644 --- a/projects/policyengine-simulation-entry/tests/test_economy_submissions.py +++ b/projects/policyengine-simulation-entry/tests/test_economy_submissions.py @@ -487,6 +487,7 @@ def test_the_calls_context_records_when_and_where_it_was_made() -> None: completed_at=CREATED_AT, baseline_simulation_run_id=UUID(int=12), reform_simulation_run_id=UUID(int=13), + modal_call_id="fc-11", ) STORED_REPORT = EconomyReportState( id=UUID(int=10), diff --git a/projects/policyengine-simulation-entry/tests/test_household_routes.py b/projects/policyengine-simulation-entry/tests/test_household_routes.py index 3c1863336..361f1efac 100644 --- a/projects/policyengine-simulation-entry/tests/test_household_routes.py +++ b/projects/policyengine-simulation-entry/tests/test_household_routes.py @@ -322,6 +322,7 @@ def test_a_job_is_read_with_its_result_document(client_for): "year": 2026, "policyengine_version": "6.17.1", "country_package_version": "2.2.1", + "modal_call_id": "fc-1", "result": RESULT_DOCUMENT, "error_message": None, "created_at": "2026-11-10T12:00:00Z", @@ -338,6 +339,7 @@ def test_a_failed_job_is_read_with_its_error_message(client_for): result=None, error_message="Modal did not accept the household function's call.", started_at=None, + modal_call_id=None, ) ) @@ -346,6 +348,7 @@ def test_a_failed_job_is_read_with_its_error_message(client_for): assert result.status_code == 200 assert result.json()["status"] == "failed" + assert result.json()["modal_call_id"] is None assert result.json()["result"] is None assert result.json()["error_message"] == ( "Modal did not accept the household function's call." diff --git a/projects/policyengine-simulation-entry/tests/test_openapi.py b/projects/policyengine-simulation-entry/tests/test_openapi.py index de8226b4d..c52455dc0 100644 --- a/projects/policyengine-simulation-entry/tests/test_openapi.py +++ b/projects/policyengine-simulation-entry/tests/test_openapi.py @@ -545,3 +545,22 @@ def test_economy_reads_declare_every_answer(): names = operation_ids(cast("OpenAPIDocument", spec)) assert names[("GET", "/reports/{report_id}")] == "get_report" assert names[("GET", "/simulations/{simulation_id}")] == "get_simulation" + + +def test_household_job_and_report_run_reads_name_their_modal_call(): + """Each read names the Modal call recorded on it, or null before one is. + + The staging preemption check finds the container to stop from it. + """ + spec = create_app().openapi() + schemas = spec["components"]["schemas"] + + for path in ("/household-calculations/{household_job_id}", "/reports/{report_id}"): + assert "Modal call" in spec["paths"][path]["get"]["description"] + for schema_name in ("HouseholdCalculationState", "ReportRunRead"): + schema = schemas[schema_name] + assert "modal_call_id" in schema["required"] + assert schema["properties"]["modal_call_id"]["anyOf"] == [ + {"type": "string"}, + {"type": "null"}, + ] diff --git a/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py new file mode 100644 index 000000000..a8671b8e6 --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py @@ -0,0 +1,1494 @@ +"""The staging preemption check, run on every staging deploy. + +``simulation-staging-preemption-check.sh`` starts a calculation through the +deployed candidate, a household calculation or the staging economy check's +report, and once the household function or coordinator has claimed it, +stops the Modal container running its call with ``modal container stop``. +That sends the container SIGINT and gives its input to another container +under the same call ID, as a pre-emption does. The check passes once the +calculation succeeds and Modal's call graph shows its call finishing in +another container. A calculation that finishes before its container is +stopped proves nothing and is submitted again, up to three times. + +The fakes stand in for curl (the token issuer and the service) and uv (the +call-input helper and the Modal CLI). Each record's reads, and each call's +inputs, follow one list of answers until the record's container is stopped +and another after. +""" + +from __future__ import annotations + +import json +import shlex +import shutil +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +import pytest +from test_deployment_assets_support import ( + REUSABLE_DEPLOY_WORKFLOW, + evaluate_github_expression, + routing_context, + workflow_jobs, +) +from workflow_script_support import FakeCommands, run_script + +if TYPE_CHECKING: + from pathlib import Path + + +CHECK_SCRIPT = "simulation-staging-preemption-check.sh" +READY_SCRIPT = "simulation-staging-preemption-ready.sh" +CHECK_JOB = "staging_preemption_check" +BASE_URL = "https://policyengine-simulation-entry-staging-s-123.a.run.app" +HOUSEHOLD_ID = "00000000-0000-4000-8000-0000000000bb" +ACCESS_TOKEN = "test-access-token" +CLIENT_SECRET = "staging-test-secret" +JOB_IDS = [f"00000000-0000-4000-8000-00000000a00{digit}" for digit in "123"] +REPORT_IDS = [f"00000000-0000-4000-8000-00000000b00{digit}" for digit in "123"] +REPORT_RUN_IDS = [f"00000000-0000-4000-8000-00000000c00{digit}" for digit in "123"] +CALL_IDS = [f"fc-01CALL{digit}" for digit in "123"] +CONTAINERS = [f"ta-01CONTAINER{digit}" for digit in "123"] +RERUN_CONTAINER = "ta-01RERUN" +# The fake date says every stop returned at 2026-11-13T12:00:00Z. +STOP_EPOCH = 1794571200 +# A succeeded record's completion time, before or after any test's stop. +BEFORE_THE_STOP = "2026-01-01T00:00:00Z" +AFTER_THE_STOP = "2099-01-01T00:00:00Z" +REPORT_REQUEST = { + "country": "uk", + "report_type": "economy", + "label": "Staging preemption check", + "year": 2026, + "region_code": "country/wales", + "baseline": {"policy_id": None, "dynamic_id": None}, + "reform": {"policy_id": None, "dynamic_id": None}, + "include_cliffs": False, +} + +# The fake curl answers the token request, each submission in turn, and each +# record's reads from the spec, keeping each request body it was sent. +FAKE_CURL = r""" +spec="${FAKE_STATE}/spec.json" +output="" +url="" +method="GET" +data="" +while (($#)); do + case "$1" in + --output) output="$2"; shift 2 ;; + --request) method="$2"; shift 2 ;; + --data) data="$2"; shift 2 ;; + --header | --write-out | --max-time) shift 2 ;; + *) url="$1"; shift ;; + esac +done +next() { + local counter="${FAKE_STATE}/$1" + local count="$(($(cat "${counter}" 2>/dev/null || echo 0) + 1))" + echo "${count}" > "${counter}" + echo "${count}" +} +calls="$(next calls)" +if [[ "${data}" == "@-" ]]; then + cat > "${FAKE_STATE}/body-${calls}.json" +fi +case "${method} ${url}" in + "POST https://issuer.example/oauth/token") + printf '{"access_token": "%s"}' "${FAKE_ACCESS_TOKEN}" > "${output}" + printf '200' + ;; + "POST ${FAKE_BASE_URL}/household-calculations" \ + | "POST ${FAKE_BASE_URL}/reports") + submission="$(next submissions)" + cp "${FAKE_STATE}/body-${calls}.json" "${FAKE_STATE}/submission-${submission}.json" + index="$(($(cat "${FAKE_STATE}/submissions") - 1))" + jq -r ".submissions[${index}].body" "${spec}" > "${output}" + jq -r ".submissions[${index}].status" "${spec}" | tr -d '\n' + ;; + "GET ${FAKE_BASE_URL}/household-calculations/"* \ + | "GET ${FAKE_BASE_URL}/reports/"*) + echo "${url}" >> "${FAKE_STATE}/read-urls" + record_id="${url##*/}" + container="$(jq -r --arg id "${record_id}" '.records[$id].container' "${spec}")" + phase="before" + if [[ -e "${FAKE_STATE}/stopped-${container}" ]]; then phase="after"; fi + count="$(next "reads-${record_id}-${phase}")" + answer="$(jq -r --arg id "${record_id}" --arg phase "${phase}" \ + --argjson count "${count}" \ + '.records[$id][$phase] | .[([$count, length] | min) - 1]' "${spec}")" + if [[ "${answer}" == "TRANSPORT" ]]; then + # curl's exit status when the request times out. + exit 28 + fi + printf '%s' "${answer#* }" > "${output}" + printf '%s' "${answer%% *}" + ;; + *) + printf 'unexpected request: %s %s\n' "${method}" "${url}" >&2 + exit 7 + ;; +esac +""" + +# The fake uv runs the call-input helper and the Modal CLI's container +# commands from the spec. +FAKE_UV = r""" +spec="${FAKE_STATE}/spec.json" +last="${*: -1}" +case "$*" in + "run --frozen python -m src.modal.utils.describe_call_inputs "*) + container="$(jq -r --arg id "${last}" '.calls[$id].container // empty' "${spec}")" + phase="before" + if [[ -n "${container}" && -e "${FAKE_STATE}/stopped-${container}" ]]; then + phase="after" + fi + counter="${FAKE_STATE}/inputs-${last}-${phase}" + count="$(($(cat "${counter}" 2>/dev/null || echo 0) + 1))" + echo "${count}" > "${counter}" + answer="$(jq -c --arg id "${last}" --arg phase "${phase}" \ + --argjson count "${count}" \ + '(.calls[$id][$phase] // [[]]) | .[([$count, length] | min) - 1]' "${spec}")" + if [[ "${answer}" == '"ERROR"' ]]; then + printf "Could not read Modal call %s's call graph: NotFoundError\n" "${last}" >&2 + exit 1 + fi + printf '%s\n' "${answer}" + ;; + "run --frozen modal container list --env "*" --json") + counter="${FAKE_STATE}/listings" + listing="$(($(cat "${counter}" 2>/dev/null || echo 0) + 1))" + echo "${listing}" > "${counter}" + if jq -e --argjson listing "${listing}" '.failed_listings | index($listing)' \ + "${spec}" >/dev/null; then + printf 'Error: the Modal server did not answer\n' >&2 + exit 1 + fi + if jq -e --argjson listing "${listing}" '.unreadable_listings | index($listing)' \ + "${spec}" >/dev/null; then + printf 'Active Containers in environment: staging\n' + exit 0 + fi + if jq -e --argjson listing "${listing}" '.empty_listings | index($listing)' \ + "${spec}" >/dev/null; then + exit 0 + fi + if jq -e --arg listing "${listing}" '.listing_outputs | has($listing)' \ + "${spec}" >/dev/null; then + jq -r --arg listing "${listing}" '.listing_outputs[$listing]' "${spec}" + exit 0 + fi + # A stopped container is never listed again. + stopped="$(find "${FAKE_STATE}" -name 'stopped-*' -exec basename {} \; \ + | sed 's/^stopped-//' | jq -R . | jq -s .)" + jq --argjson stopped "${stopped}" \ + '[.listed[] as $id | select(($stopped | index($id)) == null) + | {"Container ID": $id, "App ID": "ap-01V2", "App Name": "policyengine-v2", + "Start Time": "2026-11-13T12:00:00+00:00"}]' "${spec}" + ;; + "run --frozen modal container stop --yes "*) + if jq -e --arg id "${last}" '.refused_stops | index($id)' "${spec}" >/dev/null; then + printf "Container '%s' is already stopped.\n" "${last}" >&2 + exit 1 + fi + if jq -e --arg id "${last}" '.timed_out_stops | index($id)' "${spec}" \ + >/dev/null; then + # GNU timeout's exit status when the command runs out of time. + exit 124 + fi + touch "${FAKE_STATE}/stopped-${last}" + ;; + *) + printf 'unexpected uv command: %s\n' "$*" >&2 + exit 9 + ;; +esac +""" + + +def _household(job_id: str, status: str, call_id: str | None = None, **fields) -> str: + """Return a household job read: " ".""" + body = { + "job_id": job_id, + "status": status, + "country": "us", + "household_id": HOUSEHOLD_ID, + "policy_id": None, + "year": 2026, + "policyengine_version": "6.17.1", + "country_package_version": "1.400.0", + "modal_call_id": call_id, + "result": {"status": "succeeded"} if status == "succeeded" else None, + "error_message": None, + "created_at": "2026-11-13T12:00:00Z", + "started_at": None, + "completed_at": AFTER_THE_STOP if status == "succeeded" else None, + **fields, + } + return f"200 {json.dumps(body)}" + + +def _report(attempt: int, status: str, call_id: str | None = None, **run_fields) -> str: + """Return a report read for the ``attempt``th report: " ".""" + body = { + "report_id": REPORT_IDS[attempt], + "country": "uk", + "baseline_simulation_id": "00000000-0000-4000-8000-0000000000f1", + "reform_simulation_id": "00000000-0000-4000-8000-0000000000f3", + "run": { + "run_id": REPORT_RUN_IDS[attempt], + "status": status, + "country_package_version": "2.174.1", + "policyengine_version": "6.17.1", + "error_message": None, + "created_at": "2026-11-13T12:00:00Z", + "started_at": None, + "completed_at": AFTER_THE_STOP if status == "succeeded" else None, + "baseline_simulation_run_id": "00000000-0000-4000-8000-0000000000f2", + "reform_simulation_run_id": "00000000-0000-4000-8000-0000000000f4", + "modal_call_id": call_id, + **run_fields, + }, + } + return f"200 {json.dumps(body)}" + + +def _inputs(*containers_and_statuses: tuple[str, str]) -> list[dict[str, str]]: + """Return what the call-input helper prints for inputs on these containers.""" + return [ + {"input_id": f"in-{index}", "container_id": container, "status": status} + for index, (container, status) in enumerate(containers_and_statuses) + ] + + +@dataclass +class Attempt: + """One submission's answers: before its container is stopped, and after.""" + + record_id: str + submission: dict + call_id: str + container: str + reads_before_stop: list[str] + reads_after_stop: list[str] = field(default_factory=list) + inputs_before_stop: list = field(default_factory=list) + inputs_after_stop: list = field(default_factory=list) + submission_status: str = "202" + + +def _household_attempt(attempt: int, *, proves: bool = True) -> Attempt: + """Return a household attempt whose container is stopped once it's running. + + Without ``proves``, the job finishes before the check sees it running. + """ + job_id, call_id, container = ( + JOB_IDS[attempt], + CALL_IDS[attempt], + CONTAINERS[attempt], + ) + if not proves: + return Attempt( + record_id=job_id, + submission={"job_id": job_id, "status": "pending"}, + call_id=call_id, + container=container, + reads_before_stop=[ + _household(job_id, "pending", call_id), + _household(job_id, "succeeded", call_id), + ], + inputs_before_stop=[_inputs((container, "PENDING"))], + ) + return Attempt( + record_id=job_id, + submission={"job_id": job_id, "status": "pending"}, + call_id=call_id, + container=container, + reads_before_stop=[ + _household(job_id, "pending"), + _household(job_id, "pending", call_id), + _household(job_id, "running", call_id), + ], + reads_after_stop=[ + _household(job_id, "running", call_id), + _household(job_id, "succeeded", call_id), + ], + inputs_before_stop=[_inputs((container, "PENDING"))], + inputs_after_stop=[ + _inputs((container, "TERMINATED"), (RERUN_CONTAINER, "SUCCESS")) + ], + ) + + +def _report_attempt(attempt: int) -> Attempt: + """Return a report attempt whose coordinator container is stopped once running.""" + call_id, container = CALL_IDS[attempt], CONTAINERS[attempt] + return Attempt( + record_id=REPORT_IDS[attempt], + submission={ + "report_id": REPORT_IDS[attempt], + "report_run_id": REPORT_RUN_IDS[attempt], + "status": "pending", + "baseline_reused": True, + "reform_reused": True, + }, + submission_status="201", + call_id=call_id, + container=container, + reads_before_stop=[ + _report(attempt, "pending", call_id), + _report(attempt, "running", call_id), + ], + reads_after_stop=[ + _report(attempt, "running", call_id), + _report(attempt, "succeeded", call_id), + ], + inputs_before_stop=[_inputs((container, "PENDING"))], + inputs_after_stop=[ + _inputs((container, "TERMINATED"), (RERUN_CONTAINER, "SUCCESS")) + ], + ) + + +def _jq_arguments_log(commands: FakeCommands) -> Path: + return commands.log.with_name("jq-arguments.log") + + +@dataclass(frozen=True) +class ModalContainers: + """The containers Modal lists in staging, and those it refuses to stop.""" + + listed: tuple[str, ...] = (*CONTAINERS, RERUN_CONTAINER) + refused_stops: tuple[str, ...] = () + timed_out_stops: tuple[str, ...] = () + # Which listings, counted from 1, fail as if Modal didn't answer, or + # print something other than JSON. + failed_listings: tuple[int, ...] = () + unreadable_listings: tuple[int, ...] = () + empty_listings: tuple[int, ...] = () + # What some listings print instead, by their number. + listing_outputs: dict[int, str] = field(default_factory=dict) + + +@pytest.fixture +def fakes(tmp_path) -> FakeCommands: + commands = FakeCommands.create(tmp_path) + # jq records its arguments apart from the other calls, separated by NUL, + # since its programs span lines, then runs the real jq. + real_jq = shutil.which("jq") + assert real_jq is not None + fake_jq = commands.directory / "jq" + fake_jq.write_text( + "#!/usr/bin/env bash\n" + f"printf '%s\\0' \"$@\" >> {shlex.quote(str(_jq_arguments_log(commands)))}\n" + f'exec {shlex.quote(real_jq)} "$@"\n', + encoding="utf-8", + ) + fake_jq.chmod(0o755) + commands.add("curl", FAKE_CURL) + commands.add("uv", FAKE_UV) + commands.add("sleep") + # The fake timeout runs its command at once, without a time limit. + commands.add("timeout", 'shift\nexec "$@"') + commands.add( + "date", + """ + if [[ "$*" != "-u +%s" ]]; then + printf 'unexpected date arguments: %s\\n' "$*" >&2 + exit 9 + fi + echo "${FAKE_STOP_EPOCH}" + """, + ) + return commands + + +def _run( + fakes: FakeCommands, + tmp_path: Path, + kind: str, + attempts: list[Attempt], + *, + containers: ModalContainers | None = None, + **overrides: str, +): + containers = containers or ModalContainers() + state = tmp_path / "state" + state.mkdir() + spec = { + "submissions": [ + { + "status": attempt.submission_status, + "body": json.dumps(attempt.submission), + } + for attempt in attempts + ], + "records": { + attempt.record_id: { + "container": attempt.container, + "before": attempt.reads_before_stop, + "after": attempt.reads_after_stop or attempt.reads_before_stop[-1:], + } + for attempt in attempts + }, + "calls": { + attempt.call_id: { + "container": attempt.container, + "before": attempt.inputs_before_stop or [[]], + "after": attempt.inputs_after_stop + or attempt.inputs_before_stop + or [[]], + } + for attempt in attempts + }, + "listed": list(containers.listed), + "refused_stops": list(containers.refused_stops), + "timed_out_stops": list(containers.timed_out_stops), + "failed_listings": list(containers.failed_listings), + "unreadable_listings": list(containers.unreadable_listings), + "empty_listings": list(containers.empty_listings), + "listing_outputs": { + str(listing): output + for listing, output in containers.listing_outputs.items() + }, + } + (state / "spec.json").write_text(json.dumps(spec), encoding="utf-8") + environment = { + "SIMULATION_TEST_AUTH_ISSUER": "https://issuer.example/", + "SIMULATION_TEST_AUTH_AUDIENCE": "simulation-entry", + "SIMULATION_TEST_AUTH_CLIENT_ID": "staging-test-client", + "SIMULATION_TEST_AUTH_CLIENT_SECRET": CLIENT_SECRET, + "MODAL_TOKEN_ID": "ak-test", + "MODAL_TOKEN_SECRET": "as-test", + "MODAL_ENVIRONMENT": "staging", + "STAGING_HOUSEHOLD_ID": HOUSEHOLD_ID, + # The fake sleep returns at once, so a record that never finishes + # fails the test within this many seconds. + "PREEMPTION_CHECK_DEADLINE_SECONDS": "20", + "FAKE_STATE": str(state), + "FAKE_BASE_URL": BASE_URL, + "FAKE_ACCESS_TOKEN": ACCESS_TOKEN, + "FAKE_STOP_EPOCH": str(STOP_EPOCH), + **overrides, + } + result = run_script( + CHECK_SCRIPT, kind, BASE_URL + "/", env=fakes.env(**environment) + ) + return result, state + + +def _stops(fakes: FakeCommands) -> list[str]: + """Return the containers the check asked Modal to stop, in order.""" + return [ + call[-1] + for call in fakes.calls("uv") + if call[1:6] == ["run", "--frozen", "modal", "container", "stop"] + ] + + +def _listings_before_the_first_stop(fakes: FakeCommands) -> int: + listings = 0 + for call in fakes.calls("uv"): + if "stop" in call: + return listings + listings += "list" in call + return listings + + +def _submissions(state: Path) -> list[dict]: + return [ + json.loads(path.read_text(encoding="utf-8")) + for path in sorted(state.glob("submission-*.json")) + ] + + +def test_a_household_job_whose_container_is_stopped_succeeds_in_another( + fakes, tmp_path +): + result, state = _run(fakes, tmp_path, "household", [_household_attempt(0)]) + + assert result.returncode == 0, result.stderr + assert _submissions(state) == [{"country": "us", "household_id": HOUSEHOLD_ID}] + assert _stops(fakes) == [CONTAINERS[0]] + assert (state / "read-urls").read_text(encoding="utf-8").splitlines()[0] == ( + f"{BASE_URL}/household-calculations/{JOB_IDS[0]}" + ) + assert CONTAINERS[0] in result.stdout + assert RERUN_CONTAINER in result.stdout + assert CALL_IDS[0] in result.stdout + + +def test_the_container_is_stopped_only_once_the_job_is_running(fakes, tmp_path): + """The household function claims the job when it marks it running.""" + result, _ = _run(fakes, tmp_path, "household", [_household_attempt(0)]) + + assert result.returncode == 0, result.stderr + calls = fakes.calls() + stop_index = next( + index for index, call in enumerate(calls) if call[0] == "uv" and "stop" in call + ) + reads_before_the_stop = [ + call + for call in calls[:stop_index] + if call[0] == "curl" and "--request" not in call + ] + # Pending without a call, pending with one, then running. + assert len(reads_before_the_stop) == 3 + + +def test_the_container_is_checked_to_run_in_the_staging_environment(fakes, tmp_path): + result, _ = _run(fakes, tmp_path, "household", [_household_attempt(0)]) + + assert result.returncode == 0, result.stderr + listings = [ + call[1:] + for call in fakes.calls("uv") + if call[1:6] == ["run", "--frozen", "modal", "container", "list"] + ] + assert listings == [ + ["run", "--frozen", "modal", "container", "list", "--env", "staging", "--json"] + ] + + +def test_a_container_modal_does_not_list_in_staging_is_never_stopped(fakes, tmp_path): + attempt = _household_attempt(0) + attempt.reads_before_stop = [ + *attempt.reads_before_stop, + _household(JOB_IDS[0], "succeeded", CALL_IDS[0]), + ] + + result, _ = _run( + fakes, + tmp_path, + "household", + [attempt], + containers=ModalContainers(listed=()), + PREEMPTION_CHECK_ATTEMPTS="1", + ) + + assert result.returncode == 1 + assert _stops(fakes) == [] + assert f"{CONTAINERS[0]} is not listed in Modal environment staging" in ( + result.stderr + ) + + +def test_a_report_runs_coordinator_container_is_stopped_and_it_succeeds( + fakes, tmp_path +): + result, state = _run(fakes, tmp_path, "report", [_report_attempt(0)]) + + assert result.returncode == 0, result.stderr + assert _submissions(state) == [REPORT_REQUEST] + assert _stops(fakes) == [CONTAINERS[0]] + assert (state / "read-urls").read_text(encoding="utf-8").splitlines()[0] == ( + f"{BASE_URL}/reports/{REPORT_IDS[0]}" + ) + assert REPORT_RUN_IDS[0] in result.stdout + assert RERUN_CONTAINER in result.stdout + + +def test_a_report_whose_newest_run_is_not_the_created_one_fails(fakes, tmp_path): + attempt = _report_attempt(0) + attempt.reads_before_stop = [ + _report(0, "running", CALL_IDS[0], run_id=REPORT_RUN_IDS[1]) + ] + + result, _ = _run(fakes, tmp_path, "report", [attempt]) + + assert result.returncode == 1 + assert REPORT_RUN_IDS[1] in result.stderr + assert _stops(fakes) == [] + + +def test_a_calculation_that_finishes_before_the_stop_is_submitted_again( + fakes, tmp_path +): + result, state = _run( + fakes, + tmp_path, + "household", + [_household_attempt(0, proves=False), _household_attempt(1)], + ) + + assert result.returncode == 0, result.stderr + assert len(_submissions(state)) == 2 + assert _stops(fakes) == [CONTAINERS[1]] + assert "finished before its container was stopped" in result.stderr + + +def test_a_failed_listing_is_asked_again_rather_than_taken_for_an_absent_container( + fakes, tmp_path +): + result, _ = _run( + fakes, + tmp_path, + "household", + [_household_attempt(0)], + containers=ModalContainers(failed_listings=(1,)), + ) + + assert result.returncode == 0, result.stderr + assert _stops(fakes) == [CONTAINERS[0]] + # Only a listing Modal answered may confirm the container. + assert _listings_before_the_first_stop(fakes) == 2 + assert "Modal could not list" in result.stderr + assert "is not listed" not in result.stderr + + +def test_an_unreadable_listing_is_asked_again(fakes, tmp_path): + result, _ = _run( + fakes, + tmp_path, + "household", + [_household_attempt(0)], + containers=ModalContainers(unreadable_listings=(1,)), + ) + + assert result.returncode == 0, result.stderr + assert _stops(fakes) == [CONTAINERS[0]] + assert _listings_before_the_first_stop(fakes) == 2 + assert "readably" in result.stderr + assert "is not listed" not in result.stderr + + +def test_an_empty_listing_is_asked_again(fakes, tmp_path): + """Modal's listing prints a JSON array, even an empty one.""" + result, _ = _run( + fakes, + tmp_path, + "household", + [_household_attempt(0)], + containers=ModalContainers(empty_listings=(1,)), + ) + + assert result.returncode == 0, result.stderr + assert _stops(fakes) == [CONTAINERS[0]] + assert _listings_before_the_first_stop(fakes) == 2 + assert "readably" in result.stderr + assert "is not listed" not in result.stderr + + +@pytest.mark.parametrize( + "output", + [ + "{}", + "null", + json.dumps([{"ID": CONTAINERS[0]}]), + "[1]", + "[null]", + # The container's own entry is readable; the other entry isn't. + json.dumps([{"Container ID": CONTAINERS[0]}, {"ID": CONTAINERS[1]}]), + ], + ids=[ + "object", + "null", + "renamed-column", + "number-entry", + "null-entry", + "renamed-column-in-one-entry", + ], +) +def test_a_listing_that_is_not_a_list_of_containers_is_asked_again( + fakes, tmp_path, output +): + """A change in Modal's output must not read as the container having gone.""" + result, _ = _run( + fakes, + tmp_path, + "household", + [_household_attempt(0)], + containers=ModalContainers(listing_outputs={1: output}), + ) + + assert result.returncode == 0, result.stderr + assert _stops(fakes) == [CONTAINERS[0]] + assert _listings_before_the_first_stop(fakes) == 2 + assert "readably" in result.stderr + assert "is not listed" not in result.stderr + + +@pytest.mark.parametrize( + ("failed_listings", "last_note", "earlier_note"), + [ + ((1,), "is not listed", "Modal could not list"), + ((2,), "Modal could not list", "is not listed"), + ], + ids=["unlisted-after-a-failure", "failure-after-unlisted"], +) +def test_a_missed_attempts_reason_names_only_the_last_listings_outcome( + fakes, tmp_path, failed_listings, last_note, earlier_note +): + """Modal lists no containers, after or before a listing that fails. + + The reason keeps the last container listing's outcome, even when later + call-graph reads name no container to list. + """ + attempt = _household_attempt(0, proves=False) + attempt.reads_before_stop = [ + _household(JOB_IDS[0], "pending", CALL_IDS[0]), + _household(JOB_IDS[0], "pending", CALL_IDS[0]), + _household(JOB_IDS[0], "succeeded", CALL_IDS[0]), + ] + + result, _ = _run( + fakes, + tmp_path, + "household", + [attempt], + containers=ModalContainers(listed=(), failed_listings=failed_listings), + PREEMPTION_CHECK_ATTEMPTS="1", + ) + + assert result.returncode == 1 + last_reason = result.stderr.split("the last: ")[-1] + assert last_note in last_reason + assert earlier_note not in last_reason + + +def test_a_container_found_after_a_failed_listing_clears_its_note(fakes, tmp_path): + attempt = _household_attempt(0, proves=False) + attempt.reads_before_stop = [ + _household(JOB_IDS[0], "pending", CALL_IDS[0]), + _household(JOB_IDS[0], "pending", CALL_IDS[0]), + _household(JOB_IDS[0], "succeeded", CALL_IDS[0]), + ] + + result, _ = _run( + fakes, + tmp_path, + "household", + [attempt], + containers=ModalContainers(failed_listings=(1,)), + PREEMPTION_CHECK_ATTEMPTS="1", + ) + + assert result.returncode == 1 + last_reason = result.stderr.split("the last: ")[-1] + assert "finished before its container was stopped" in last_reason + assert "Modal could not list" not in last_reason + + +def test_a_job_finished_while_modal_could_not_list_says_so(fakes, tmp_path): + result, _ = _run( + fakes, + tmp_path, + "household", + [_household_attempt(0, proves=False)], + containers=ModalContainers(failed_listings=tuple(range(1, 10))), + PREEMPTION_CHECK_ATTEMPTS="1", + ) + + assert result.returncode == 1 + assert "the last: " in result.stderr + last_reason = result.stderr.split("the last: ")[-1] + assert "Modal could not list" in last_reason + + +def test_a_stop_that_runs_out_of_time_is_followed_like_a_failed_one(fakes, tmp_path): + """Modal may have accepted the stop, so the check doesn't stop it again.""" + first = _household_attempt(0) + first.reads_before_stop = [ + *first.reads_before_stop, + _household(JOB_IDS[0], "succeeded", CALL_IDS[0]), + ] + + result, state = _run( + fakes, + tmp_path, + "household", + [first, _household_attempt(1)], + containers=ModalContainers(timed_out_stops=(CONTAINERS[0],)), + ) + + assert result.returncode == 0, result.stderr + assert len(_submissions(state)) == 2 + assert _stops(fakes) == [CONTAINERS[0], CONTAINERS[1]] + assert "exit status 124" in result.stderr + assert "finished after modal container stop failed" in result.stderr + + +def _indices(fakes: FakeCommands, matches) -> list[int]: + return [index for index, call in enumerate(fakes.calls()) if matches(call)] + + +def _is_submission(call: list[str]) -> bool: + return call[0] == "curl" and call[-1].endswith("/household-calculations") + + +def test_the_next_attempt_waits_out_modals_idle_scale_down_first(fakes, tmp_path): + """Modal stops a container left idle for 60 seconds, by default. + + A new attempt on a container the last one left warm would finish too + fast to stop, so it starts a fixed wait after the last attempt ended. + The wait covers a container Modal ran a stopped call in again only when + that rerun returned within about 30 seconds of the attempt's end. + """ + result, _ = _run( + fakes, + tmp_path, + "household", + [_household_attempt(0, proves=False), _household_attempt(1)], + ) + + assert result.returncode == 0, result.stderr + submissions = _indices(fakes, _is_submission) + waits = _indices(fakes, lambda call: call == ["sleep", "90"]) + assert len(waits) == 1 + assert submissions[0] < waits[0] < submissions[1] + assert "waiting 90 seconds" in result.stdout + + +def test_no_wait_follows_the_last_attempt(fakes, tmp_path): + result, _ = _run( + fakes, + tmp_path, + "household", + [_household_attempt(attempt, proves=False) for attempt in range(3)], + PREEMPTION_CHECK_SCALEDOWN_SECONDS="75", + ) + + assert result.returncode == 1 + submissions = _indices(fakes, _is_submission) + waits = _indices(fakes, lambda call: call == ["sleep", "75"]) + assert len(waits) == 2 + assert submissions[0] < waits[0] < submissions[1] < waits[1] < submissions[2] + + +def test_three_calculations_that_finish_before_the_stop_fail_the_check(fakes, tmp_path): + result, state = _run( + fakes, + tmp_path, + "household", + [_household_attempt(attempt, proves=False) for attempt in range(3)], + ) + + assert result.returncode == 1 + assert len(_submissions(state)) == 3 + assert _stops(fakes) == [] + assert "no attempt in 3 showed" in result.stderr + + +def test_a_refused_stop_is_not_tried_again_and_the_job_is_followed(fakes, tmp_path): + """Modal refuses to stop a container that has ended. + + A job that then succeeds finished just before the stop, so a new one is + submitted. + """ + first = _household_attempt(0) + first.reads_before_stop = [ + *first.reads_before_stop, + _household(JOB_IDS[0], "running", CALL_IDS[0]), + _household(JOB_IDS[0], "succeeded", CALL_IDS[0]), + ] + + result, state = _run( + fakes, + tmp_path, + "household", + [first, _household_attempt(1)], + containers=ModalContainers(refused_stops=(CONTAINERS[0],)), + ) + + assert result.returncode == 0, result.stderr + assert len(_submissions(state)) == 2 + assert _stops(fakes) == [CONTAINERS[0], CONTAINERS[1]] + assert (state / f"reads-{JOB_IDS[0]}-before").read_text().strip() == "5" + assert f"modal container stop failed for container {CONTAINERS[0]}" in ( + result.stderr + ) + + +def test_a_job_left_running_after_a_refused_stop_fails_at_its_deadline(fakes, tmp_path): + """A container that ended for another reason, such as running out of memory. + + Its job stays running, and the check doesn't submit another: it would + usually fail the same way. + """ + result, state = _run( + fakes, + tmp_path, + "household", + [_household_attempt(0), _household_attempt(1)], + containers=ModalContainers(refused_stops=(CONTAINERS[0],)), + # Long enough to reach the stop first. + PREEMPTION_CHECK_DEADLINE_SECONDS="6", + ) + + assert result.returncode == 1 + assert len(_submissions(state)) == 1 + assert _stops(fakes) == [CONTAINERS[0]] + assert "did not finish within 6 seconds" in result.stderr + assert f"modal container stop failed for container {CONTAINERS[0]}" in ( + result.stderr + ) + + +def test_a_call_that_finished_in_the_stopped_container_proves_nothing(fakes, tmp_path): + """The input finished just before the stop reached its container.""" + attempt = _household_attempt(0) + attempt.inputs_after_stop = [_inputs((CONTAINERS[0], "SUCCESS"))] + + result, _ = _run( + fakes, + tmp_path, + "household", + [attempt], + PREEMPTION_CHECK_ATTEMPTS="1", + PREEMPTION_CHECK_RERUN_SECONDS="0", + ) + + assert result.returncode == 1 + assert "never showed" in result.stderr + assert CONTAINERS[0] in result.stderr + + +def test_a_rerun_recorded_on_the_same_input_counts(fakes, tmp_path): + """Modal may record the rerun on the stopped input itself, in place.""" + attempt = _household_attempt(0) + attempt.inputs_after_stop = [_inputs((RERUN_CONTAINER, "SUCCESS"))] + + result, _ = _run(fakes, tmp_path, "household", [attempt]) + + assert result.returncode == 0, result.stderr + assert attempt.inputs_before_stop[0][0]["input_id"] == "in-0" + assert RERUN_CONTAINER in result.stdout + + +def test_a_job_completed_before_its_container_was_stopped_proves_nothing( + fakes, tmp_path +): + """It finished between the running read and the stop. + + Modal runs the call again, but the rerun finds the job finished and + returns its summary, which Modal records as a success. + """ + attempt = _household_attempt(0) + attempt.reads_after_stop = [ + _household(JOB_IDS[0], "succeeded", CALL_IDS[0], completed_at=BEFORE_THE_STOP) + ] + + result, _ = _run( + fakes, tmp_path, "household", [attempt], PREEMPTION_CHECK_ATTEMPTS="1" + ) + + assert result.returncode == 1 + assert f"completed at {BEFORE_THE_STOP}" in result.stderr + assert f"before its container {CONTAINERS[0]} was stopped" in result.stderr + + +@pytest.mark.parametrize( + ("completed_at", "proves"), + [("2026-11-13T12:00:04Z", False), ("2026-11-13T12:00:05Z", True)], + ids=["four-seconds-after", "five-seconds-after"], +) +def test_a_completion_moments_after_the_stop_is_taken_for_the_stopped_containers( + fakes, tmp_path, completed_at, proves +): + """The stopped container may write the success before the signal reaches it. + + A rerun in a new container can't finish within 5 seconds of the stop. + """ + attempt = _household_attempt(0) + attempt.reads_after_stop = [ + _household(JOB_IDS[0], "succeeded", CALL_IDS[0], completed_at=completed_at) + ] + + result, _ = _run( + fakes, tmp_path, "household", [attempt], PREEMPTION_CHECK_ATTEMPTS="1" + ) + + assert result.returncode == (0 if proves else 1), result.stderr + assert ("too soon" in result.stderr) is not proves + + +def test_a_completion_in_the_stops_own_second_is_too_soon(fakes, tmp_path): + """Both times are whole seconds, so the same second may be after the stop.""" + attempt = _household_attempt(0) + attempt.reads_after_stop = [ + _household( + JOB_IDS[0], "succeeded", CALL_IDS[0], completed_at="2026-11-13T12:00:00Z" + ) + ] + + result, _ = _run( + fakes, tmp_path, "household", [attempt], PREEMPTION_CHECK_ATTEMPTS="1" + ) + + assert result.returncode == 1 + assert "0 seconds after" in result.stderr + + +def test_a_record_that_names_no_call_after_the_stop_fails(fakes, tmp_path): + attempt = _household_attempt(0) + attempt.reads_after_stop = [_household(JOB_IDS[0], "succeeded", None)] + + result, _ = _run(fakes, tmp_path, "household", [attempt]) + + assert result.returncode == 1 + assert "names no call" in result.stderr + assert CALL_IDS[0] in result.stderr + + +@pytest.mark.parametrize( + "completed_at", + [AFTER_THE_STOP, "2099-01-01T00:00:00.123456Z", "2099-01-01T00:00:00+00:00"], + ids=["whole-seconds", "fractional-seconds", "utc-offset"], +) +def test_a_completion_after_the_stop_is_read_in_each_form_the_service_writes( + fakes, tmp_path, completed_at +): + attempt = _report_attempt(0) + attempt.reads_after_stop = [ + _report(0, "succeeded", CALL_IDS[0], completed_at=completed_at) + ] + + result, _ = _run(fakes, tmp_path, "report", [attempt]) + + assert result.returncode == 0, result.stderr + + +def test_a_record_that_succeeded_without_a_completion_time_fails(fakes, tmp_path): + attempt = _household_attempt(0) + attempt.reads_after_stop = [ + _household(JOB_IDS[0], "succeeded", CALL_IDS[0], completed_at=None) + ] + + result, _ = _run(fakes, tmp_path, "household", [attempt]) + + assert result.returncode == 1 + assert "no completion time" in result.stderr + + +def test_the_call_graph_is_asked_again_until_it_shows_the_other_container( + fakes, tmp_path +): + """Modal records the call graph best-effort and not in real time.""" + attempt = _household_attempt(0) + attempt.inputs_after_stop = [ + "ERROR", + _inputs((CONTAINERS[0], "TERMINATED")), + _inputs((CONTAINERS[0], "TERMINATED"), (RERUN_CONTAINER, "SUCCESS")), + ] + + result, _ = _run(fakes, tmp_path, "household", [attempt]) + + assert result.returncode == 0, result.stderr + assert RERUN_CONTAINER in result.stdout + + +def test_the_call_graph_is_asked_again_until_it_names_the_running_container( + fakes, tmp_path +): + attempt = _household_attempt(0) + attempt.reads_before_stop = [ + _household(JOB_IDS[0], "running", CALL_IDS[0]), + ] + attempt.inputs_before_stop = [ + "ERROR", + _inputs(("", "PENDING")), + _inputs((CONTAINERS[0], "PENDING")), + ] + + result, _ = _run(fakes, tmp_path, "household", [attempt]) + + assert result.returncode == 0, result.stderr + assert _stops(fakes) == [CONTAINERS[0]] + + +def test_a_failed_calculation_fails_the_check_at_once(fakes, tmp_path): + """A calculation that fails isn't submitted again: it would fail the same way.""" + attempt = _household_attempt(0) + attempt.reads_after_stop = [ + _household( + JOB_IDS[0], + "failed", + CALL_IDS[0], + error_message="household_function_error: the claim was lost", + ) + ] + + result, state = _run(fakes, tmp_path, "household", [attempt, _household_attempt(1)]) + + assert result.returncode == 1 + assert len(_submissions(state)) == 1 + assert "household_function_error: the claim was lost" in result.stderr + assert f"after its container {CONTAINERS[0]} was stopped" in result.stderr + + +def test_a_calculation_under_another_call_after_the_stop_fails(fakes, tmp_path): + """Modal runs a stopped container's input again under the same call ID.""" + attempt = _household_attempt(0) + attempt.reads_after_stop = [_household(JOB_IDS[0], "succeeded", "fc-01OTHER")] + + result, _ = _run(fakes, tmp_path, "household", [attempt]) + + assert result.returncode == 1 + assert "fc-01OTHER" in result.stderr + assert CALL_IDS[0] in result.stderr + + +def test_a_calculation_that_does_not_finish_in_time_fails(fakes, tmp_path): + attempt = _household_attempt(0) + attempt.reads_after_stop = [_household(JOB_IDS[0], "running", CALL_IDS[0])] + + result, _ = _run( + fakes, + tmp_path, + "household", + [attempt], + PREEMPTION_CHECK_DEADLINE_SECONDS="0", + ) + + assert result.returncode == 1 + assert "did not finish within 0 seconds" in result.stderr + assert "last read: the household job was" in result.stderr + + +@pytest.mark.parametrize( + "answer", + ["TRANSPORT", '503 {"code": "x"}', "429 {}"], + ids=["no-answer", "unavailable", "too-many-requests"], +) +def test_a_read_the_service_cannot_answer_now_is_asked_again(fakes, tmp_path, answer): + attempt = _household_attempt(0) + attempt.reads_before_stop = [answer, *attempt.reads_before_stop] + + result, _ = _run(fakes, tmp_path, "household", [attempt]) + + assert result.returncode == 0, result.stderr + + +def test_any_other_read_answer_fails_the_check(fakes, tmp_path): + attempt = _household_attempt(0) + attempt.reads_before_stop = ['404 {"code": "unknown_household_job"}'] + + result, _ = _run(fakes, tmp_path, "household", [attempt]) + + assert result.returncode == 1 + assert "404" in result.stderr + + +@pytest.mark.parametrize( + ("kind", "status", "body"), + [ + ("household", "503", {"code": "modal_call_refused", "detail": "x"}), + ("report", "400", {"code": "unknown_region", "detail": "x"}), + ], +) +def test_a_refused_submission_fails_the_check_with_its_answer( + fakes, tmp_path, kind, status, body +): + attempt = _household_attempt(0) if kind == "household" else _report_attempt(0) + attempt.submission_status = status + attempt.submission = body + + result, state = _run(fakes, tmp_path, kind, [attempt]) + + assert result.returncode == 1 + assert status in result.stderr + assert body["code"] in result.stderr + assert not (state / "read-urls").exists() + + +@pytest.mark.parametrize( + "call_id", + ["fc-01CALL1 --yes", "--help", "ta-01CONTAINER1", "fc-"], + ids=["extra-argument", "option", "container-id", "empty-suffix"], +) +def test_a_recorded_call_that_is_not_a_modal_call_id_fails_the_check( + fakes, tmp_path, call_id +): + attempt = _household_attempt(0) + attempt.reads_before_stop = [_household(JOB_IDS[0], "running", call_id)] + + result, _ = _run(fakes, tmp_path, "household", [attempt]) + + assert result.returncode == 1 + assert "not a Modal call ID" in result.stderr + assert fakes.calls("uv") == [] + + +def test_a_container_that_is_not_a_modal_container_id_is_never_stopped(fakes, tmp_path): + attempt = _household_attempt(0) + attempt.inputs_before_stop = [_inputs(("--all", "PENDING"))] + + result, _ = _run(fakes, tmp_path, "household", [attempt]) + + assert result.returncode == 1 + assert "not a Modal container ID" in result.stderr + assert _stops(fakes) == [] + + +def test_no_secret_reaches_a_command_line(fakes, tmp_path): + """The token is sent from a file, and the client secret on standard input. + + No command, jq included, gets either as an argument. + """ + result, _ = _run(fakes, tmp_path, "household", [_household_attempt(0)]) + + assert result.returncode == 0, result.stderr + jq_arguments = _jq_arguments_log(fakes).read_text(encoding="utf-8").split("\0") + arguments = [ + *(argument for call in fakes.calls() for argument in call), + *jq_arguments, + ] + assert len(jq_arguments) > 1 + assert not any(ACCESS_TOKEN in argument for argument in arguments) + assert not any(CLIENT_SECRET in argument for argument in arguments) + + +def test_every_modal_command_has_a_time_limit(fakes, tmp_path): + """A Modal request that hangs fails its command instead of the job.""" + result, _ = _run( + fakes, + tmp_path, + "household", + [_household_attempt(0, proves=False), _household_attempt(1)], + ) + + assert result.returncode == 0, result.stderr + limited = [call[1:3] for call in fakes.calls("timeout")] + assert len(fakes.calls("uv")) > 3 + assert limited == [["60", "uv"]] * len(fakes.calls("uv")) + + +def test_every_request_has_a_time_limit(fakes, tmp_path): + """The submission, which isn't asked again, may take Cloud Run's full minute.""" + result, _ = _run(fakes, tmp_path, "report", [_report_attempt(0)]) + + assert result.returncode == 0, result.stderr + limits = { + call[-1]: call[call.index("--max-time") + 1] for call in fakes.calls("curl") + } + assert limits == { + "https://issuer.example/oauth/token": "30", + f"{BASE_URL}/reports": "60", + f"{BASE_URL}/reports/{REPORT_IDS[0]}": "30", + } + + +def test_the_check_runs_only_in_modals_staging_environment(fakes, tmp_path): + """It stops containers, so it never runs against production's.""" + result, _ = _run( + fakes, tmp_path, "household", [_household_attempt(0)], MODAL_ENVIRONMENT="main" + ) + + assert result.returncode == 1 + assert "staging" in result.stderr + assert fakes.calls() == [] + + +@pytest.mark.parametrize( + "variable_name", + [ + "SIMULATION_TEST_AUTH_CLIENT_ID", + "MODAL_TOKEN_ID", + "MODAL_TOKEN_SECRET", + "MODAL_ENVIRONMENT", + "STAGING_HOUSEHOLD_ID", + ], +) +def test_the_household_check_needs_its_settings(fakes, tmp_path, variable_name): + result, _ = _run( + fakes, tmp_path, "household", [_household_attempt(0)], **{variable_name: ""} + ) + + assert result.returncode == 1 + assert variable_name in result.stderr + assert fakes.calls() == [] + + +def test_the_report_check_needs_no_household(fakes, tmp_path): + result, _ = _run( + fakes, tmp_path, "report", [_report_attempt(0)], STAGING_HOUSEHOLD_ID="" + ) + + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize( + ("variable_name", "value"), + [ + ("STAGING_HOUSEHOLD_ID", "not-a-uuid"), + ("PREEMPTION_CHECK_ATTEMPTS", "0"), + ("PREEMPTION_CHECK_ATTEMPTS", "three"), + ("PREEMPTION_CHECK_DEADLINE_SECONDS", "ten"), + ("PREEMPTION_CHECK_POLL_SECONDS", "-1"), + ("PREEMPTION_CHECK_RERUN_SECONDS", "2m"), + ("PREEMPTION_CHECK_SCALEDOWN_SECONDS", "1.5"), + ], +) +def test_a_malformed_setting_fails_the_check_before_any_request( + fakes, tmp_path, variable_name, value +): + result, _ = _run( + fakes, tmp_path, "household", [_household_attempt(0)], **{variable_name: value} + ) + + assert result.returncode == 1 + assert variable_name in result.stderr + assert fakes.calls() == [] + + +def test_an_unknown_kind_of_calculation_is_refused(fakes, tmp_path): + result, _ = _run(fakes, tmp_path, "simulation", [_household_attempt(0)]) + + assert result.returncode == 1 + assert "household or report" in result.stderr + assert fakes.calls() == [] + + +# --- The workflow job -------------------------------------------------------- + +CHECK_ENVIRONMENT = { + "SIMULATION_TEST_AUTH_ISSUER": "${{ secrets.SIMULATION_ENTRYPOINT_AUTH_ISSUER }}", + "SIMULATION_TEST_AUTH_AUDIENCE": ( + "${{ secrets.SIMULATION_ENTRYPOINT_AUTH_AUDIENCE }}" + ), + "SIMULATION_TEST_AUTH_CLIENT_ID": ( + "${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_ID }}" + ), + "SIMULATION_TEST_AUTH_CLIENT_SECRET": ( + "${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_SECRET }}" + ), + "MODAL_TOKEN_ID": "${{ secrets.MODAL_TOKEN_ID }}", + "MODAL_TOKEN_SECRET": "${{ secrets.MODAL_TOKEN_SECRET }}", + "MODAL_ENVIRONMENT": "${{ inputs.modal_environment }}", +} + + +def _jobs() -> dict: + return workflow_jobs(REUSABLE_DEPLOY_WORKFLOW) + + +def _context(**overrides): + context = routing_context() + context["inputs"]["deployment_environment"] = "staging" + context["inputs"]["deploy_stage12_v2"] = True + context["needs"]["deploy_entrypoint"] = {"result": "success"} + context["needs"]["publish_stage12_v2_default"] = {"result": "success"} + context["needs"]["staging_economy_check"] = {"result": "success"} + for path, value in overrides.items(): + target = context + *parents, last = path.split("__") + for part in parents: + target = target[part] + target[last] = value + return context + + +def test_the_check_job_stops_a_household_and_a_report_through_the_candidate(): + job = _jobs()[CHECK_JOB] + run_steps = [step for step in job["steps"] if "run" in step] + install, household, report = run_steps + + assert set(job["needs"]) == { + "deploy_entrypoint", + "authenticated_test", + "publish_stage12_v2_default", + "staging_economy_check", + } + assert job["environment"] == "${{ inputs.release_environment }}" + # Longer than both steps' worst case together, with every request and + # Modal command taking its full time limit, so the script fails first. + assert job["timeout-minutes"] == 150 + assert {step.get("uses") for step in job["steps"] if "uses" in step} == { + "actions/checkout@v6", + "actions/setup-python@v6", + "astral-sh/setup-uv@v8.1.0", + } + for step in run_steps: + assert step["working-directory"] == "projects/policyengine-simulation-executor" + assert install["run"] == "uv sync --frozen" + candidate = '"${{ needs.deploy_entrypoint.outputs.candidate_url }}"' + assert household["run"] == ( + f"../../.github/scripts/{CHECK_SCRIPT} household {candidate}" + ) + assert household["env"] == { + **CHECK_ENVIRONMENT, + "STAGING_HOUSEHOLD_ID": "${{ vars.SIMULATION_STAGING_HOUSEHOLD_ID }}", + } + assert report["run"] == f"../../.github/scripts/{CHECK_SCRIPT} report {candidate}" + assert report["env"] == CHECK_ENVIRONMENT + + +@pytest.mark.parametrize( + ("overrides", "runs"), + [ + ({}, True), + ({"inputs__deployment_environment": "production"}, False), + ({"needs__deploy_entrypoint__result": "failure"}, False), + ({"needs__authenticated_test__result": "failure"}, False), + ({"needs__staging_economy_check__result": "failure"}, False), + ({"needs__staging_economy_check__result": "skipped"}, False), + ({"needs__publish_stage12_v2_default__result": "failure"}, False), + ({"needs__publish_stage12_v2_default__result": "skipped"}, False), + ( + { + "inputs__deploy_stage12_v2": False, + "needs__publish_stage12_v2_default__result": "skipped", + }, + True, + ), + ({"cancelled": True}, False), + ], + ids=[ + "staging", + "production", + "entrypoint-failed", + "authenticated-tests-failed", + "economy-check-failed", + "economy-check-skipped", + "default-move-failed", + "default-move-skipped", + "no-stage12", + "cancelled", + ], +) +def test_the_check_runs_on_every_staging_deploy_after_the_economy_check( + overrides, runs +): + job = _jobs()[CHECK_JOB] + + assert ( + bool(evaluate_github_expression(job["if"], _context(**overrides), job_if=True)) + is runs + ) + + +def test_deployment_ready_requires_the_staging_preemption_check(): + ready = _jobs()["deployment_ready"] + (step,) = [ + step + for step in ready["steps"] + if step.get("name") == "Check the staging preemption check result" + ] + + assert CHECK_JOB in ready["needs"] + assert step["run"] == f".github/scripts/{READY_SCRIPT}" + assert step["env"] == { + "REQUESTED": "${{ inputs.deployment_environment == 'staging' }}", + "CHECK_RESULT": "${{ needs.staging_preemption_check.result }}", + } + + +@pytest.mark.parametrize( + ("requested", "check_result", "passes"), + [ + ("true", "success", True), + ("true", "failure", False), + ("true", "skipped", False), + ("true", "cancelled", False), + ("false", "skipped", True), + ("false", "success", False), + ], +) +def test_the_ready_check_requires_the_check_exactly_in_staging( + fakes, requested, check_result, passes +): + result = run_script( + READY_SCRIPT, + env=fakes.env(REQUESTED=requested, CHECK_RESULT=check_result), + ) + + assert result.returncode == (0 if passes else 1) diff --git a/projects/policyengine-simulation-executor/src/modal/utils/describe_call_inputs.py b/projects/policyengine-simulation-executor/src/modal/utils/describe_call_inputs.py new file mode 100644 index 000000000..2a5935b26 --- /dev/null +++ b/projects/policyengine-simulation-executor/src/modal/utils/describe_call_inputs.py @@ -0,0 +1,94 @@ +"""Print a Modal function call's inputs, with the container that runs each. + +Usage, from projects/policyengine-simulation-executor: +``uv run --frozen python -m src.modal.utils.describe_call_inputs ``. +The Modal client reads MODAL_TOKEN_ID and MODAL_TOKEN_SECRET. + +Prints a JSON array with one object per input of the call: +``{"input_id": ..., "container_id": ..., "status": ...}``. ``container_id`` +is Modal's task ID (``ta-...``), which ``modal container stop`` takes, or +empty while Modal hasn't given the input a container. ``status`` names a +``modal.call_graph.InputStatus``: ``PENDING`` until the input finishes, then +``SUCCESS``, ``FAILURE``, ``INIT_FAILURE``, ``TERMINATED`` or ``TIMEOUT``. + +Modal returns the whole call graph a call is part of, where a call spawned +inside another call's input is nested under that input, so the call's own +inputs are found by its ID anywhere in the graph. Modal records the graph +best-effort and not in real time, so a new call may be missing and a caller +asks again. A failure to ask is printed on standard error, with exit status +1. The staging preemption check +(``.github/scripts/simulation-staging-preemption-check.sh``) uses this to +find the container to stop, and the container that ran the call again. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from importlib import import_module +from typing import Any, TypedDict + + +class CallInput(TypedDict): + """One input of a call, as printed: the shell check reads these keys.""" + + input_id: str + container_id: str + status: str + + +def call_inputs( + modal_call_id: str, + *, + modal_module: Any = None, # noqa: ANN401 # the modal module, or a stand-in for it +) -> list[CallInput]: + """Return each input of call ``modal_call_id``, in the graph's order. + + The graph is walked depth first, each input before the calls spawned + inside it. + """ + modal = modal_module if modal_module is not None else import_module("modal") + graph = modal.FunctionCall.from_id(modal_call_id).get_call_graph() + found: list[CallInput] = [] + waiting = list(reversed(graph)) + while waiting: + input_info = waiting.pop() + if input_info.function_call_id == modal_call_id: + found.append( + CallInput( + input_id=input_info.input_id, + container_id=input_info.task_id, + status=input_info.status.name, + ) + ) + waiting.extend(reversed(input_info.children)) + return found + + +def main( + argv: list[str] | None = None, + *, + modal_module: Any = None, # noqa: ANN401 # the modal module, or a stand-in for it +) -> int: + """Print the call's inputs as JSON; return the process's exit status.""" + parser = argparse.ArgumentParser( + description="Print a Modal function call's inputs and their containers." + ) + parser.add_argument("modal_call_id", help="the Modal function call ID (fc-...)") + arguments = parser.parse_args(argv) + try: + inputs = call_inputs(arguments.modal_call_id, modal_module=modal_module) + except Exception as error: # noqa: BLE001 # any failure to ask is the caller's to retry + print( + f"Could not read Modal call {arguments.modal_call_id}'s call graph: " + f"{type(error).__name__}: {error}", + file=sys.stderr, + ) + return 1 + print(json.dumps(inputs)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/projects/policyengine-simulation-executor/tests/test_describe_call_inputs.py b/projects/policyengine-simulation-executor/tests/test_describe_call_inputs.py new file mode 100644 index 000000000..adc56086f --- /dev/null +++ b/projects/policyengine-simulation-executor/tests/test_describe_call_inputs.py @@ -0,0 +1,156 @@ +"""The Modal helper the staging preemption check uses to find a call's container. + +``describe_call_inputs`` asks Modal for the call graph a call is part of and +prints each of the call's inputs with the container (Modal's task ID) that +runs or ran it. The check stops that container with ``modal container +stop``, then waits for the input to finish in another one. +""" + +from __future__ import annotations + +import json +from types import SimpleNamespace + +import pytest +from src.modal.utils import describe_call_inputs as describe + +CALL_ID = "fc-01HOUSEHOLD" +OTHER_CALL_ID = "fc-01COORDINATOR" + + +def _input( + call_id: str, + *, + input_id: str, + container_id: str, + status: str, + children: tuple[SimpleNamespace, ...] = (), +) -> SimpleNamespace: + """Return a stand-in for ``modal.call_graph.InputInfo``, its status named.""" + return SimpleNamespace( + input_id=input_id, + function_call_id=call_id, + task_id=container_id, + status=SimpleNamespace(name=status), + children=list(children), + ) + + +def _modal_answering( + graph: list[SimpleNamespace] | None = None, error: Exception | None = None +) -> tuple[SimpleNamespace, list[str]]: + """Return a stand-in modal module and the call IDs it was asked about.""" + asked: list[str] = [] + + def from_id(call_id: str) -> SimpleNamespace: + asked.append(call_id) + + def get_call_graph() -> list[SimpleNamespace]: + if error is not None: + raise error + return graph or [] + + return SimpleNamespace(get_call_graph=get_call_graph) + + return SimpleNamespace(FunctionCall=SimpleNamespace(from_id=from_id)), asked + + +def test_the_calls_inputs_are_listed_wherever_they_sit_in_the_graph(): + """Modal nests a call spawned inside another call's input under that input.""" + graph = [ + _input( + OTHER_CALL_ID, + input_id="in-parent", + container_id="ta-parent", + status="PENDING", + children=( + _input( + CALL_ID, + input_id="in-stopped", + container_id="ta-stopped", + status="TERMINATED", + ), + _input( + OTHER_CALL_ID, + input_id="in-sibling", + container_id="ta-sibling", + status="SUCCESS", + ), + ), + ), + _input(CALL_ID, input_id="in-again", container_id="ta-again", status="SUCCESS"), + ] + modal, asked = _modal_answering(graph) + + inputs = describe.call_inputs(CALL_ID, modal_module=modal) + + assert asked == [CALL_ID] + assert inputs == [ + { + "input_id": "in-stopped", + "container_id": "ta-stopped", + "status": "TERMINATED", + }, + {"input_id": "in-again", "container_id": "ta-again", "status": "SUCCESS"}, + ] + + +def test_an_input_without_a_container_has_an_empty_container_id(): + """Modal hasn't given a queued input a container yet.""" + modal, _ = _modal_answering( + [_input(CALL_ID, input_id="in-queued", container_id="", status="PENDING")] + ) + + assert describe.call_inputs(CALL_ID, modal_module=modal) == [ + {"input_id": "in-queued", "container_id": "", "status": "PENDING"} + ] + + +def test_a_call_missing_from_its_graph_has_no_inputs(): + """Modal records the call graph best-effort, so a new call may be missing.""" + modal, _ = _modal_answering( + [ + _input( + OTHER_CALL_ID, + input_id="in-other", + container_id="ta-1", + status="SUCCESS", + ) + ] + ) + + assert describe.call_inputs(CALL_ID, modal_module=modal) == [] + + +def test_main_prints_the_inputs_as_json(capsys): + modal, _ = _modal_answering( + [_input(CALL_ID, input_id="in-1", container_id="ta-1", status="PENDING")] + ) + + exit_code = describe.main([CALL_ID], modal_module=modal) + + assert exit_code == 0 + assert json.loads(capsys.readouterr().out) == [ + {"input_id": "in-1", "container_id": "ta-1", "status": "PENDING"} + ] + + +def test_main_reports_a_failure_to_ask_modal_and_exits_non_zero(capsys): + modal, _ = _modal_answering(error=RuntimeError("the call graph is unavailable")) + + exit_code = describe.main([CALL_ID], modal_module=modal) + + captured = capsys.readouterr() + assert exit_code == 1 + assert captured.out == "" + assert CALL_ID in captured.err + assert "RuntimeError: the call graph is unavailable" in captured.err + + +def test_main_needs_one_call_id(): + modal, _ = _modal_answering() + + with pytest.raises(SystemExit) as refused: + describe.main([], modal_module=modal) + + assert refused.value.code == 2