From 144de5ba7e6da713e8654fb35b9bf04dc26a6205 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:24:12 +0400 Subject: [PATCH 01/24] Test that household job and report run reads name their Modal call (failing) The staging preemption check finds the container to stop from the Modal call recorded on a household job or report run, so GET /household-calculations/{id} and the run in GET /reports/{id} name it, or null before one is recorded. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_economy_run_reads.py | 46 +++++++++++++++++++ .../tests/test_economy_generated_client.py | 2 + .../tests/test_household_generated_client.py | 2 + .../tests/test_economy_routes.py | 2 + .../tests/test_economy_submissions.py | 1 + .../tests/test_household_routes.py | 3 ++ .../tests/test_openapi.py | 16 +++++++ 7 files changed, 72 insertions(+) diff --git a/libs/policyengine-simulation-persistence/tests/test_economy_run_reads.py b/libs/policyengine-simulation-persistence/tests/test_economy_run_reads.py index 836b63c23..ff89477b7 100644 --- a/libs/policyengine-simulation-persistence/tests/test_economy_run_reads.py +++ b/libs/policyengine-simulation-persistence/tests/test_economy_run_reads.py @@ -157,6 +157,52 @@ def test_a_failed_newest_run_is_read_with_its_error(public_api_database_url) -> assert report.run.completed_at is not None +def test_a_report_run_is_read_with_the_modal_call_it_records( + public_api_database_url, +) -> None: + """A run names the Modal call recorded on it, and no call before one is. + + The staging preemption check finds the coordinator's container from it. + """ + with rolled_back_transaction(public_api_database_url) as connection: + economy = insert_economy_catalog(connection) + report_id, run_ids = _report_with_runs(connection, economy) + for run_id in run_ids: + connection.execute( + update(report_runs) + .where(report_runs.c.id == run_id) + .values(modal_call_id=f"fc-{run_id.hex}") + ) + population = economy.population() + uncalled_run_id = insert_report_run_naming( + connection, + economy, + ( + insert_simulation_run(connection, population), + insert_simulation_run(connection, population), + ), + ReportRunOutcome(status=ReportRunStatus.PENDING), + ) + uncalled_report_id = ( + connection.execute( + report_runs.select().where(report_runs.c.id == uncalled_run_id) + ) + .one() + .report_id + ) + act_as_the_service(connection) + + report = read_economy_report(connection, report_id) + uncalled_report = read_economy_report(connection, uncalled_report_id) + + assert report is not None + assert report.run is not None + assert report.run.modal_call_id == f"fc-{report.run.id.hex}" + assert uncalled_report is not None + assert uncalled_report.run is not None + assert uncalled_report.run.modal_call_id is None + + def test_a_simulation_is_read_with_its_newest_run(public_api_database_url) -> None: with rolled_back_transaction(public_api_database_url) as connection: economy = insert_economy_catalog(connection) diff --git a/projects/policyengine-apis-integ/tests/test_economy_generated_client.py b/projects/policyengine-apis-integ/tests/test_economy_generated_client.py index 5f32692db..737786e45 100644 --- a/projects/policyengine-apis-integ/tests/test_economy_generated_client.py +++ b/projects/policyengine-apis-integ/tests/test_economy_generated_client.py @@ -177,6 +177,7 @@ def test_a_report_and_a_simulation_are_read_with_their_newest_runs(): "completed_at": "2026-11-12T15:04:00Z", "baseline_simulation_run_id": SIDE_IDS[1], "reform_simulation_run_id": SIDE_IDS[3], + "modal_call_id": "fc-coordinator", }, } simulation = {"simulation_id": REPORT_ID, "run": None} @@ -192,5 +193,6 @@ def test_a_report_and_a_simulation_are_read_with_their_newest_runs(): run = report_response.parsed.run assert run is not None assert run.status is ReportRunStatus.SUCCEEDED + assert run.modal_call_id == "fc-coordinator" assert isinstance(simulation_response.parsed, EconomySimulationState) assert simulation_response.parsed.to_dict() == simulation diff --git a/projects/policyengine-apis-integ/tests/test_household_generated_client.py b/projects/policyengine-apis-integ/tests/test_household_generated_client.py index a8dc72a69..95519ace0 100644 --- a/projects/policyengine-apis-integ/tests/test_household_generated_client.py +++ b/projects/policyengine-apis-integ/tests/test_household_generated_client.py @@ -74,6 +74,7 @@ def test_a_job_keeps_its_result_document_whole(): "year": 2026, "policyengine_version": "6.17.1", "country_package_version": "2.2.1", + "modal_call_id": "fc-1", "result": document, "error_message": None, "created_at": "2026-11-10T12:00:00Z", @@ -84,6 +85,7 @@ def test_a_job_keeps_its_result_document_whole(): parsed = HouseholdCalculationState.from_dict(state) assert parsed.status is HouseholdJobStatus.SUCCEEDED + assert parsed.modal_call_id == "fc-1" assert parsed.to_dict()["result"] == document diff --git a/projects/policyengine-simulation-entry/tests/test_economy_routes.py b/projects/policyengine-simulation-entry/tests/test_economy_routes.py index 90ca6e108..b348936af 100644 --- a/projects/policyengine-simulation-entry/tests/test_economy_routes.py +++ b/projects/policyengine-simulation-entry/tests/test_economy_routes.py @@ -417,6 +417,7 @@ def test_without_its_database_and_v2_environment_none_is_built(settings_changes) completed_at=datetime.fromisoformat(READ_AT), baseline_simulation_run_id=SIDE_IDS[1], reform_simulation_run_id=SIDE_IDS[3], + modal_call_id="fc-coordinator", ), ) STORED_SIMULATION = EconomySimulationState(id=RECORD_ID, run=None) @@ -446,6 +447,7 @@ def test_a_report_is_read_with_its_newest_runs_status(client_for) -> None: "completed_at": READ_AT, "baseline_simulation_run_id": str(SIDE_IDS[1]), "reform_simulation_run_id": str(SIDE_IDS[3]), + "modal_call_id": "fc-coordinator", }, } assert missing.status_code == 404 diff --git a/projects/policyengine-simulation-entry/tests/test_economy_submissions.py b/projects/policyengine-simulation-entry/tests/test_economy_submissions.py index 062fa9b7c..2712a7a52 100644 --- a/projects/policyengine-simulation-entry/tests/test_economy_submissions.py +++ b/projects/policyengine-simulation-entry/tests/test_economy_submissions.py @@ -487,6 +487,7 @@ def test_the_calls_context_records_when_and_where_it_was_made() -> None: completed_at=CREATED_AT, baseline_simulation_run_id=UUID(int=12), reform_simulation_run_id=UUID(int=13), + modal_call_id="fc-11", ) STORED_REPORT = EconomyReportState( id=UUID(int=10), diff --git a/projects/policyengine-simulation-entry/tests/test_household_routes.py b/projects/policyengine-simulation-entry/tests/test_household_routes.py index 3c1863336..361f1efac 100644 --- a/projects/policyengine-simulation-entry/tests/test_household_routes.py +++ b/projects/policyengine-simulation-entry/tests/test_household_routes.py @@ -322,6 +322,7 @@ def test_a_job_is_read_with_its_result_document(client_for): "year": 2026, "policyengine_version": "6.17.1", "country_package_version": "2.2.1", + "modal_call_id": "fc-1", "result": RESULT_DOCUMENT, "error_message": None, "created_at": "2026-11-10T12:00:00Z", @@ -338,6 +339,7 @@ def test_a_failed_job_is_read_with_its_error_message(client_for): result=None, error_message="Modal did not accept the household function's call.", started_at=None, + modal_call_id=None, ) ) @@ -346,6 +348,7 @@ def test_a_failed_job_is_read_with_its_error_message(client_for): assert result.status_code == 200 assert result.json()["status"] == "failed" + assert result.json()["modal_call_id"] is None assert result.json()["result"] is None assert result.json()["error_message"] == ( "Modal did not accept the household function's call." diff --git a/projects/policyengine-simulation-entry/tests/test_openapi.py b/projects/policyengine-simulation-entry/tests/test_openapi.py index 30f8022af..b1f9b7463 100644 --- a/projects/policyengine-simulation-entry/tests/test_openapi.py +++ b/projects/policyengine-simulation-entry/tests/test_openapi.py @@ -456,3 +456,19 @@ def test_economy_reads_declare_every_answer(): names = operation_ids(cast("OpenAPIDocument", spec)) assert names[("GET", "/reports/{report_id}")] == "get_report" assert names[("GET", "/simulations/{simulation_id}")] == "get_simulation" + + +def test_household_job_and_report_run_reads_name_their_modal_call(): + """Each read names the Modal call recorded on it, or null before one is. + + The staging preemption check finds the container to stop from it. + """ + schemas = create_app().openapi()["components"]["schemas"] + + for schema_name in ("HouseholdCalculationState", "ReportRunRead"): + schema = schemas[schema_name] + assert "modal_call_id" in schema["required"] + assert schema["properties"]["modal_call_id"]["anyOf"] == [ + {"type": "string"}, + {"type": "null"}, + ] From af5ec0c61673ee005b8b12d52a1089a9a68dc517 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:25:38 +0400 Subject: [PATCH 02/24] Name the Modal call recorded on a household job or report run in its read GET /household-calculations/{id} and the run in GET /reports/{id} now carry modal_call_id: the household function's or coordinator's call, or null before the service has recorded one. Modal runs a pre-empted call again under the same ID, so the staging preemption check can find the container running it. Co-Authored-By: Claude Opus 5.5 --- .../economy_run_reads.py | 9 ++++++++- .../policyengine_simulation_entry/economy_api_models.py | 8 +++++++- .../src/policyengine_simulation_entry/economy_routes.py | 1 + .../household_api_models.py | 6 +++++- .../policyengine_simulation_entry/household_routes.py | 1 + 5 files changed, 22 insertions(+), 3 deletions(-) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_run_reads.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_run_reads.py index 1cde162ad..6788e801a 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_run_reads.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_run_reads.py @@ -44,7 +44,11 @@ class SimulationRunState: @dataclass(frozen=True) class ReportRunState: - """A report run's state, and the simulation runs it compares.""" + """A report run's state, the simulation runs it compares, and its call. + + ``modal_call_id`` is the coordinator's Modal call recorded on the run, + or None before the service has recorded one. + """ id: UUID status: ReportRunStatus @@ -56,6 +60,7 @@ class ReportRunState: completed_at: datetime | None baseline_simulation_run_id: UUID reform_simulation_run_id: UUID | None + modal_call_id: str | None @dataclass(frozen=True) @@ -104,6 +109,7 @@ def read_economy_report( report_runs.c.completed_at, report_runs.c.baseline_simulation_run_id, report_runs.c.reform_simulation_run_id, + report_runs.c.modal_call_id, ) .where(report_runs.c.report_id == report_id) .order_by(report_runs.c.created_at.desc(), report_runs.c.id.desc()) @@ -178,4 +184,5 @@ def _report_run_state(run: RowMapping) -> ReportRunState: completed_at=run["completed_at"], baseline_simulation_run_id=run["baseline_simulation_run_id"], reform_simulation_run_id=run["reform_simulation_run_id"], + modal_call_id=run["modal_call_id"], ) diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_api_models.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_api_models.py index b161c91e7..8ab06836c 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_api_models.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_api_models.py @@ -215,7 +215,12 @@ class SimulationRunRead(_Answer): class ReportRunRead(_Answer): - """A report run's state, and the simulation runs it compares.""" + """A report run's state, the simulation runs it compares, and its call. + + ``modal_call_id`` is the coordinator's Modal call recorded on the run, + or null before the service has recorded one. Modal runs a pre-empted + call again under the same ID. + """ run_id: UUID status: ReportRunStatus @@ -227,6 +232,7 @@ class ReportRunRead(_Answer): completed_at: datetime | None baseline_simulation_run_id: UUID reform_simulation_run_id: UUID | None + modal_call_id: str | None class EconomyReportState(_Answer): diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_routes.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_routes.py index 9dd6e5285..abc207a79 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_routes.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_routes.py @@ -330,6 +330,7 @@ def _report_state(stored: StoredReport) -> EconomyReportState: completed_at=run.completed_at, baseline_simulation_run_id=run.baseline_simulation_run_id, reform_simulation_run_id=run.reform_simulation_run_id, + modal_call_id=run.modal_call_id, ), ) diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_api_models.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_api_models.py index 36e4bf49b..d1a577c76 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_api_models.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_api_models.py @@ -130,7 +130,10 @@ class HouseholdCalculationState(BaseModel): calculated household in ``output``) or a ``HouseholdExecutionFailure`` (``status`` ``failed``, with the reason in ``error``). ``error_message`` summarizes a failure in one line; a job that failed before its function - ran has an ``error_message`` and no ``result``. + ran has an ``error_message`` and no ``result``. ``modal_call_id`` is the + household function's Modal call recorded on the job, or null before the + service has recorded one; Modal runs a pre-empted call again under the + same ID. """ model_config = ConfigDict(extra="forbid", frozen=True) @@ -143,6 +146,7 @@ class HouseholdCalculationState(BaseModel): year: int policyengine_version: str country_package_version: str + modal_call_id: str | None result: dict[str, JsonValue] | None error_message: str | None created_at: datetime diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_routes.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_routes.py index 660d4d13b..50aa46d84 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_routes.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_routes.py @@ -242,6 +242,7 @@ def _state(record: HouseholdJobRecord) -> HouseholdCalculationState: year=record.year, policyengine_version=record.policyengine_version, country_package_version=record.country_package_version, + modal_call_id=record.modal_call_id, result=record.result, error_message=record.error_message, created_at=record.created_at, From dd9b55cac61763122f09af6aa258715549598c1f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:26:47 +0400 Subject: [PATCH 03/24] Test the helper that lists a Modal call's inputs and their containers (failing) The staging preemption check stops the container running a household job's or report run's call, then requires the call to finish in another container. The helper reads the call graph Modal records for the call and prints each of the call's inputs, wherever it sits in that graph, with the container (Modal's task ID) that runs or ran it, as JSON. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_describe_call_inputs.py | 156 ++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 projects/policyengine-simulation-executor/tests/test_describe_call_inputs.py diff --git a/projects/policyengine-simulation-executor/tests/test_describe_call_inputs.py b/projects/policyengine-simulation-executor/tests/test_describe_call_inputs.py new file mode 100644 index 000000000..adc56086f --- /dev/null +++ b/projects/policyengine-simulation-executor/tests/test_describe_call_inputs.py @@ -0,0 +1,156 @@ +"""The Modal helper the staging preemption check uses to find a call's container. + +``describe_call_inputs`` asks Modal for the call graph a call is part of and +prints each of the call's inputs with the container (Modal's task ID) that +runs or ran it. The check stops that container with ``modal container +stop``, then waits for the input to finish in another one. +""" + +from __future__ import annotations + +import json +from types import SimpleNamespace + +import pytest +from src.modal.utils import describe_call_inputs as describe + +CALL_ID = "fc-01HOUSEHOLD" +OTHER_CALL_ID = "fc-01COORDINATOR" + + +def _input( + call_id: str, + *, + input_id: str, + container_id: str, + status: str, + children: tuple[SimpleNamespace, ...] = (), +) -> SimpleNamespace: + """Return a stand-in for ``modal.call_graph.InputInfo``, its status named.""" + return SimpleNamespace( + input_id=input_id, + function_call_id=call_id, + task_id=container_id, + status=SimpleNamespace(name=status), + children=list(children), + ) + + +def _modal_answering( + graph: list[SimpleNamespace] | None = None, error: Exception | None = None +) -> tuple[SimpleNamespace, list[str]]: + """Return a stand-in modal module and the call IDs it was asked about.""" + asked: list[str] = [] + + def from_id(call_id: str) -> SimpleNamespace: + asked.append(call_id) + + def get_call_graph() -> list[SimpleNamespace]: + if error is not None: + raise error + return graph or [] + + return SimpleNamespace(get_call_graph=get_call_graph) + + return SimpleNamespace(FunctionCall=SimpleNamespace(from_id=from_id)), asked + + +def test_the_calls_inputs_are_listed_wherever_they_sit_in_the_graph(): + """Modal nests a call spawned inside another call's input under that input.""" + graph = [ + _input( + OTHER_CALL_ID, + input_id="in-parent", + container_id="ta-parent", + status="PENDING", + children=( + _input( + CALL_ID, + input_id="in-stopped", + container_id="ta-stopped", + status="TERMINATED", + ), + _input( + OTHER_CALL_ID, + input_id="in-sibling", + container_id="ta-sibling", + status="SUCCESS", + ), + ), + ), + _input(CALL_ID, input_id="in-again", container_id="ta-again", status="SUCCESS"), + ] + modal, asked = _modal_answering(graph) + + inputs = describe.call_inputs(CALL_ID, modal_module=modal) + + assert asked == [CALL_ID] + assert inputs == [ + { + "input_id": "in-stopped", + "container_id": "ta-stopped", + "status": "TERMINATED", + }, + {"input_id": "in-again", "container_id": "ta-again", "status": "SUCCESS"}, + ] + + +def test_an_input_without_a_container_has_an_empty_container_id(): + """Modal hasn't given a queued input a container yet.""" + modal, _ = _modal_answering( + [_input(CALL_ID, input_id="in-queued", container_id="", status="PENDING")] + ) + + assert describe.call_inputs(CALL_ID, modal_module=modal) == [ + {"input_id": "in-queued", "container_id": "", "status": "PENDING"} + ] + + +def test_a_call_missing_from_its_graph_has_no_inputs(): + """Modal records the call graph best-effort, so a new call may be missing.""" + modal, _ = _modal_answering( + [ + _input( + OTHER_CALL_ID, + input_id="in-other", + container_id="ta-1", + status="SUCCESS", + ) + ] + ) + + assert describe.call_inputs(CALL_ID, modal_module=modal) == [] + + +def test_main_prints_the_inputs_as_json(capsys): + modal, _ = _modal_answering( + [_input(CALL_ID, input_id="in-1", container_id="ta-1", status="PENDING")] + ) + + exit_code = describe.main([CALL_ID], modal_module=modal) + + assert exit_code == 0 + assert json.loads(capsys.readouterr().out) == [ + {"input_id": "in-1", "container_id": "ta-1", "status": "PENDING"} + ] + + +def test_main_reports_a_failure_to_ask_modal_and_exits_non_zero(capsys): + modal, _ = _modal_answering(error=RuntimeError("the call graph is unavailable")) + + exit_code = describe.main([CALL_ID], modal_module=modal) + + captured = capsys.readouterr() + assert exit_code == 1 + assert captured.out == "" + assert CALL_ID in captured.err + assert "RuntimeError: the call graph is unavailable" in captured.err + + +def test_main_needs_one_call_id(): + modal, _ = _modal_answering() + + with pytest.raises(SystemExit) as refused: + describe.main([], modal_module=modal) + + assert refused.value.code == 2 From 7a01b0747f357dc19e6f42818e87e6f976eb95f4 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:28:21 +0400 Subject: [PATCH 04/24] List a Modal call's inputs with the container that runs each describe_call_inputs reads the call graph Modal records for a call and prints each of the call's inputs, wherever it sits in that graph, with its container (Modal's task ID, which modal container stop takes) and its input status, as JSON. A failure to ask Modal is printed on standard error with exit status 1, for the caller to ask again. Co-Authored-By: Claude Opus 5.5 --- .../src/modal/utils/describe_call_inputs.py | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 projects/policyengine-simulation-executor/src/modal/utils/describe_call_inputs.py diff --git a/projects/policyengine-simulation-executor/src/modal/utils/describe_call_inputs.py b/projects/policyengine-simulation-executor/src/modal/utils/describe_call_inputs.py new file mode 100644 index 000000000..7292e74ac --- /dev/null +++ b/projects/policyengine-simulation-executor/src/modal/utils/describe_call_inputs.py @@ -0,0 +1,86 @@ +"""Print a Modal function call's inputs, with the container that runs each. + +Usage, from projects/policyengine-simulation-executor: +``uv run --frozen python -m src.modal.utils.describe_call_inputs ``. +The Modal client reads MODAL_TOKEN_ID and MODAL_TOKEN_SECRET. + +Prints a JSON array with one object per input of the call: +``{"input_id": ..., "container_id": ..., "status": ...}``. ``container_id`` +is Modal's task ID (``ta-...``), which ``modal container stop`` takes, or +empty while Modal hasn't given the input a container. ``status`` names a +``modal.call_graph.InputStatus``: ``PENDING`` until the input finishes, then +``SUCCESS``, ``FAILURE``, ``INIT_FAILURE``, ``TERMINATED`` or ``TIMEOUT``. + +Modal returns the whole call graph a call is part of, where a call spawned +inside another call's input is nested under that input, so the call's own +inputs are found by its ID anywhere in the graph. Modal records the graph +best-effort and not in real time, so a new call may be missing and a caller +asks again. A failure to ask is printed on standard error, with exit status +1. The staging preemption check +(``.github/scripts/simulation-staging-preemption-check.sh``) uses this to +find the container to stop, and the container that ran the call again. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from importlib import import_module +from typing import Any + + +def call_inputs( + modal_call_id: str, + *, + modal_module: Any = None, # noqa: ANN401 # the modal module, or a stand-in for it +) -> list[dict[str, str]]: + """Return each input of call ``modal_call_id``, in the graph's order. + + The graph is walked depth first, each input before the calls spawned + inside it. + """ + modal = modal_module if modal_module is not None else import_module("modal") + graph = modal.FunctionCall.from_id(modal_call_id).get_call_graph() + found: list[dict[str, str]] = [] + waiting = list(reversed(graph)) + while waiting: + input_info = waiting.pop() + if input_info.function_call_id == modal_call_id: + found.append( + { + "input_id": input_info.input_id, + "container_id": input_info.task_id, + "status": input_info.status.name, + } + ) + waiting.extend(reversed(input_info.children)) + return found + + +def main( + argv: list[str] | None = None, + *, + modal_module: Any = None, # noqa: ANN401 # the modal module, or a stand-in for it +) -> int: + """Print the call's inputs as JSON; return the process's exit status.""" + parser = argparse.ArgumentParser( + description="Print a Modal function call's inputs and their containers." + ) + parser.add_argument("modal_call_id", help="the Modal function call ID (fc-...)") + arguments = parser.parse_args(argv) + try: + inputs = call_inputs(arguments.modal_call_id, modal_module=modal_module) + except Exception as error: # noqa: BLE001 # any failure to ask is the caller's to retry + print( + f"Could not read Modal call {arguments.modal_call_id}'s call graph: " + f"{type(error).__name__}: {error}", + file=sys.stderr, + ) + return 1 + print(json.dumps(inputs)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 3b776bd4f28ac74cbe9866c073daec95b290222a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:34:46 +0400 Subject: [PATCH 05/24] Test the staging preemption check, its deploy job and its ready check (failing) Every staging deploy starts a household calculation and the staging economy check's report through the candidate, stops the Modal container running each one's call with modal container stop once it is running, and requires it to succeed in another container under the same call. A calculation that finishes before its container is stopped is submitted again, up to three times; one that fails or runs out of time fails the check at once. The check runs only in Modal's staging environment and stops only a container Modal lists there. deployment_ready requires it in staging. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_deployment_assets.py | 7 +- .../tests/test_staging_preemption_check.py | 972 ++++++++++++++++++ 2 files changed, 976 insertions(+), 3 deletions(-) create mode 100644 projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py index 24e2d1a3e..38c994635 100644 --- a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py @@ -217,9 +217,10 @@ def _assert_deployment_environment_settings(reusable_workflow: str) -> None: # The job that moves latest after the checks reads the prod # environment's custom-hostname variable, the staging household # check reads the beta variable SIMULATION_STAGING_HOUSEHOLD_ID and - # the environment's auth secrets, and the staging economy check - # reads those auth secrets too. - reusable_workflow.count("environment: ${{ inputs.release_environment }}") == 19 + # the environment's auth secrets, the staging economy check reads + # those auth secrets too, and the staging preemption check reads them, + # that variable and the environment's Modal token. + reusable_workflow.count("environment: ${{ inputs.release_environment }}") == 20 ) assert "APP_ENVIRONMENT: ${{ inputs.deployment_environment }}" in reusable_workflow assert "STAGE12_ENABLED_VALUE: ${{ vars.STAGE12_ENABLED }}" in reusable_workflow diff --git a/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py new file mode 100644 index 000000000..d5b417d89 --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py @@ -0,0 +1,972 @@ +"""The staging preemption check, run on every staging deploy. + +``simulation-staging-preemption-check.sh`` starts a calculation through the +deployed candidate, a household calculation or the staging economy check's +report, and once the household function or coordinator has claimed it, +stops the Modal container running its call with ``modal container stop``. +That sends the container SIGINT and gives its input to another container +under the same call ID, as a pre-emption does. The check passes once the +calculation succeeds and Modal's call graph shows its call finishing in +another container. A calculation that finishes before its container is +stopped proves nothing and is submitted again, up to three times. + +The fakes stand in for curl (the token issuer and the service) and uv (the +call-input helper and the Modal CLI). Each record's reads, and each call's +inputs, follow one list of answers until the record's container is stopped +and another after. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +import pytest +from test_deployment_assets_support import ( + REUSABLE_DEPLOY_WORKFLOW, + evaluate_github_expression, + routing_context, + workflow_jobs, +) +from workflow_script_support import FakeCommands, run_script + +if TYPE_CHECKING: + from pathlib import Path + +CHECK_SCRIPT = "simulation-staging-preemption-check.sh" +READY_SCRIPT = "simulation-staging-preemption-ready.sh" +CHECK_JOB = "staging_preemption_check" +BASE_URL = "https://policyengine-simulation-entry-staging-s-123.a.run.app" +HOUSEHOLD_ID = "00000000-0000-4000-8000-0000000000bb" +ACCESS_TOKEN = "test-access-token" +CLIENT_SECRET = "staging-test-secret" +JOB_IDS = [f"00000000-0000-4000-8000-00000000a00{digit}" for digit in "123"] +REPORT_IDS = [f"00000000-0000-4000-8000-00000000b00{digit}" for digit in "123"] +REPORT_RUN_IDS = [f"00000000-0000-4000-8000-00000000c00{digit}" for digit in "123"] +CALL_IDS = [f"fc-01CALL{digit}" for digit in "123"] +CONTAINERS = [f"ta-01CONTAINER{digit}" for digit in "123"] +RERUN_CONTAINER = "ta-01RERUN" +REPORT_REQUEST = { + "country": "uk", + "report_type": "economy", + "label": "Staging preemption check", + "year": 2026, + "region_code": "country/wales", + "baseline": {"policy_id": None, "dynamic_id": None}, + "reform": {"policy_id": None, "dynamic_id": None}, + "include_cliffs": False, +} + +# The fake curl answers the token request, each submission in turn, and each +# record's reads from the spec, keeping each request body it was sent. +FAKE_CURL = r""" +spec="${FAKE_STATE}/spec.json" +output="" +url="" +method="GET" +data="" +while (($#)); do + case "$1" in + --output) output="$2"; shift 2 ;; + --request) method="$2"; shift 2 ;; + --data) data="$2"; shift 2 ;; + --header | --write-out | --max-time) shift 2 ;; + *) url="$1"; shift ;; + esac +done +next() { + local counter="${FAKE_STATE}/$1" + local count="$(($(cat "${counter}" 2>/dev/null || echo 0) + 1))" + echo "${count}" > "${counter}" + echo "${count}" +} +calls="$(next calls)" +if [[ "${data}" == "@-" ]]; then + cat > "${FAKE_STATE}/body-${calls}.json" +fi +case "${method} ${url}" in + "POST https://issuer.example/oauth/token") + printf '{"access_token": "%s"}' "${FAKE_ACCESS_TOKEN}" > "${output}" + printf '200' + ;; + "POST ${FAKE_BASE_URL}/household-calculations" \ + | "POST ${FAKE_BASE_URL}/reports") + submission="$(next submissions)" + cp "${FAKE_STATE}/body-${calls}.json" "${FAKE_STATE}/submission-${submission}.json" + index="$(($(cat "${FAKE_STATE}/submissions") - 1))" + jq -r ".submissions[${index}].body" "${spec}" > "${output}" + jq -r ".submissions[${index}].status" "${spec}" | tr -d '\n' + ;; + "GET ${FAKE_BASE_URL}/household-calculations/"* \ + | "GET ${FAKE_BASE_URL}/reports/"*) + echo "${url}" >> "${FAKE_STATE}/read-urls" + record_id="${url##*/}" + container="$(jq -r --arg id "${record_id}" '.records[$id].container' "${spec}")" + phase="before" + if [[ -e "${FAKE_STATE}/stopped-${container}" ]]; then phase="after"; fi + count="$(next "reads-${record_id}-${phase}")" + answer="$(jq -r --arg id "${record_id}" --arg phase "${phase}" \ + --argjson count "${count}" \ + '.records[$id][$phase] | .[([$count, length] | min) - 1]' "${spec}")" + if [[ "${answer}" == "TRANSPORT" ]]; then + # curl's exit status when the request times out. + exit 28 + fi + printf '%s' "${answer#* }" > "${output}" + printf '%s' "${answer%% *}" + ;; + *) + printf 'unexpected request: %s %s\n' "${method}" "${url}" >&2 + exit 7 + ;; +esac +""" + +# The fake uv runs the call-input helper and the Modal CLI's container +# commands from the spec. +FAKE_UV = r""" +spec="${FAKE_STATE}/spec.json" +last="${*: -1}" +case "$*" in + "run --frozen python -m src.modal.utils.describe_call_inputs "*) + container="$(jq -r --arg id "${last}" '.calls[$id].container // empty' "${spec}")" + phase="before" + if [[ -n "${container}" && -e "${FAKE_STATE}/stopped-${container}" ]]; then + phase="after" + fi + counter="${FAKE_STATE}/inputs-${last}-${phase}" + count="$(($(cat "${counter}" 2>/dev/null || echo 0) + 1))" + echo "${count}" > "${counter}" + answer="$(jq -c --arg id "${last}" --arg phase "${phase}" \ + --argjson count "${count}" \ + '(.calls[$id][$phase] // [[]]) | .[([$count, length] | min) - 1]' "${spec}")" + if [[ "${answer}" == '"ERROR"' ]]; then + printf "Could not read Modal call %s's call graph: NotFoundError\n" "${last}" >&2 + exit 1 + fi + printf '%s\n' "${answer}" + ;; + "run --frozen modal container list --env "*" --json") + jq '[.listed[] | {"Container ID": ., "App ID": "ap-01V2", + "App Name": "policyengine-v2", "Start Time": "2026-11-13T12:00:00+00:00"}]' \ + "${spec}" + ;; + "run --frozen modal container stop --yes "*) + if jq -e --arg id "${last}" '.refused_stops | index($id)' "${spec}" >/dev/null; then + printf "Container '%s' is already stopped.\n" "${last}" >&2 + exit 1 + fi + touch "${FAKE_STATE}/stopped-${last}" + ;; + *) + printf 'unexpected uv command: %s\n' "$*" >&2 + exit 9 + ;; +esac +""" + + +def _household(job_id: str, status: str, call_id: str | None = None, **fields) -> str: + """Return a household job read: " ".""" + body = { + "job_id": job_id, + "status": status, + "country": "us", + "household_id": HOUSEHOLD_ID, + "policy_id": None, + "year": 2026, + "policyengine_version": "6.17.1", + "country_package_version": "1.400.0", + "modal_call_id": call_id, + "result": {"status": "succeeded"} if status == "succeeded" else None, + "error_message": None, + "created_at": "2026-11-13T12:00:00Z", + "started_at": None, + "completed_at": None, + **fields, + } + return f"200 {json.dumps(body)}" + + +def _report(attempt: int, status: str, call_id: str | None = None, **run_fields) -> str: + """Return a report read for the ``attempt``th report: " ".""" + body = { + "report_id": REPORT_IDS[attempt], + "country": "uk", + "baseline_simulation_id": "00000000-0000-4000-8000-0000000000f1", + "reform_simulation_id": "00000000-0000-4000-8000-0000000000f3", + "run": { + "run_id": REPORT_RUN_IDS[attempt], + "status": status, + "country_package_version": "2.174.1", + "policyengine_version": "6.17.1", + "error_message": None, + "created_at": "2026-11-13T12:00:00Z", + "started_at": None, + "completed_at": None, + "baseline_simulation_run_id": "00000000-0000-4000-8000-0000000000f2", + "reform_simulation_run_id": "00000000-0000-4000-8000-0000000000f4", + "modal_call_id": call_id, + **run_fields, + }, + } + return f"200 {json.dumps(body)}" + + +def _inputs(*containers_and_statuses: tuple[str, str]) -> list[dict[str, str]]: + """Return what the call-input helper prints for inputs on these containers.""" + return [ + {"input_id": f"in-{index}", "container_id": container, "status": status} + for index, (container, status) in enumerate(containers_and_statuses) + ] + + +@dataclass +class Attempt: + """One submission's answers: before its container is stopped, and after.""" + + record_id: str + submission: dict + call_id: str + container: str + reads_before_stop: list[str] + reads_after_stop: list[str] = field(default_factory=list) + inputs_before_stop: list = field(default_factory=list) + inputs_after_stop: list = field(default_factory=list) + submission_status: str = "202" + + +def _household_attempt(attempt: int, *, proves: bool = True) -> Attempt: + """Return a household attempt whose container is stopped once it's running. + + Without ``proves``, the job finishes before the check sees it running. + """ + job_id, call_id, container = ( + JOB_IDS[attempt], + CALL_IDS[attempt], + CONTAINERS[attempt], + ) + if not proves: + return Attempt( + record_id=job_id, + submission={"job_id": job_id, "status": "pending"}, + call_id=call_id, + container=container, + reads_before_stop=[ + _household(job_id, "pending", call_id), + _household(job_id, "succeeded", call_id), + ], + inputs_before_stop=[_inputs((container, "PENDING"))], + ) + return Attempt( + record_id=job_id, + submission={"job_id": job_id, "status": "pending"}, + call_id=call_id, + container=container, + reads_before_stop=[ + _household(job_id, "pending"), + _household(job_id, "pending", call_id), + _household(job_id, "running", call_id), + ], + reads_after_stop=[ + _household(job_id, "running", call_id), + _household(job_id, "succeeded", call_id), + ], + inputs_before_stop=[_inputs((container, "PENDING"))], + inputs_after_stop=[ + _inputs((container, "TERMINATED"), (RERUN_CONTAINER, "SUCCESS")) + ], + ) + + +def _report_attempt(attempt: int) -> Attempt: + """Return a report attempt whose coordinator container is stopped once running.""" + call_id, container = CALL_IDS[attempt], CONTAINERS[attempt] + return Attempt( + record_id=REPORT_IDS[attempt], + submission={ + "report_id": REPORT_IDS[attempt], + "report_run_id": REPORT_RUN_IDS[attempt], + "status": "pending", + "baseline_reused": True, + "reform_reused": True, + }, + submission_status="201", + call_id=call_id, + container=container, + reads_before_stop=[ + _report(attempt, "pending", call_id), + _report(attempt, "running", call_id), + ], + reads_after_stop=[ + _report(attempt, "running", call_id), + _report(attempt, "succeeded", call_id), + ], + inputs_before_stop=[_inputs((container, "PENDING"))], + inputs_after_stop=[ + _inputs((container, "TERMINATED"), (RERUN_CONTAINER, "SUCCESS")) + ], + ) + + +@dataclass(frozen=True) +class ModalContainers: + """The containers Modal lists in staging, and those it refuses to stop.""" + + listed: tuple[str, ...] = (*CONTAINERS, RERUN_CONTAINER) + refused_stops: tuple[str, ...] = () + + +@pytest.fixture +def fakes(tmp_path) -> FakeCommands: + commands = FakeCommands.create(tmp_path) + commands.add("curl", FAKE_CURL) + commands.add("uv", FAKE_UV) + commands.add("sleep") + return commands + + +def _run( + fakes: FakeCommands, + tmp_path: Path, + kind: str, + attempts: list[Attempt], + *, + containers: ModalContainers | None = None, + **overrides: str, +): + containers = containers or ModalContainers() + state = tmp_path / "state" + state.mkdir() + spec = { + "submissions": [ + { + "status": attempt.submission_status, + "body": json.dumps(attempt.submission), + } + for attempt in attempts + ], + "records": { + attempt.record_id: { + "container": attempt.container, + "before": attempt.reads_before_stop, + "after": attempt.reads_after_stop or attempt.reads_before_stop[-1:], + } + for attempt in attempts + }, + "calls": { + attempt.call_id: { + "container": attempt.container, + "before": attempt.inputs_before_stop or [[]], + "after": attempt.inputs_after_stop + or attempt.inputs_before_stop + or [[]], + } + for attempt in attempts + }, + "listed": list(containers.listed), + "refused_stops": list(containers.refused_stops), + } + (state / "spec.json").write_text(json.dumps(spec), encoding="utf-8") + environment = { + "SIMULATION_TEST_AUTH_ISSUER": "https://issuer.example/", + "SIMULATION_TEST_AUTH_AUDIENCE": "simulation-entry", + "SIMULATION_TEST_AUTH_CLIENT_ID": "staging-test-client", + "SIMULATION_TEST_AUTH_CLIENT_SECRET": CLIENT_SECRET, + "MODAL_TOKEN_ID": "ak-test", + "MODAL_TOKEN_SECRET": "as-test", + "MODAL_ENVIRONMENT": "staging", + "STAGING_HOUSEHOLD_ID": HOUSEHOLD_ID, + "FAKE_STATE": str(state), + "FAKE_BASE_URL": BASE_URL, + "FAKE_ACCESS_TOKEN": ACCESS_TOKEN, + **overrides, + } + result = run_script( + CHECK_SCRIPT, kind, BASE_URL + "/", env=fakes.env(**environment) + ) + return result, state + + +def _stops(fakes: FakeCommands) -> list[str]: + """Return the containers the check asked Modal to stop, in order.""" + return [ + call[-1] + for call in fakes.calls("uv") + if call[1:6] == ["run", "--frozen", "modal", "container", "stop"] + ] + + +def _submissions(state: Path) -> list[dict]: + return [ + json.loads(path.read_text(encoding="utf-8")) + for path in sorted(state.glob("submission-*.json")) + ] + + +def test_a_household_job_whose_container_is_stopped_succeeds_in_another( + fakes, tmp_path +): + result, state = _run(fakes, tmp_path, "household", [_household_attempt(0)]) + + assert result.returncode == 0, result.stderr + assert _submissions(state) == [{"country": "us", "household_id": HOUSEHOLD_ID}] + assert _stops(fakes) == [CONTAINERS[0]] + assert (state / "read-urls").read_text(encoding="utf-8").splitlines()[0] == ( + f"{BASE_URL}/household-calculations/{JOB_IDS[0]}" + ) + assert CONTAINERS[0] in result.stdout + assert RERUN_CONTAINER in result.stdout + assert CALL_IDS[0] in result.stdout + + +def test_the_container_is_stopped_only_once_the_job_is_running(fakes, tmp_path): + """The household function claims the job when it marks it running.""" + result, _ = _run(fakes, tmp_path, "household", [_household_attempt(0)]) + + assert result.returncode == 0, result.stderr + calls = fakes.calls() + stop_index = next( + index for index, call in enumerate(calls) if call[0] == "uv" and "stop" in call + ) + reads_before_the_stop = [ + call + for call in calls[:stop_index] + if call[0] == "curl" and "--request" not in call + ] + # Pending without a call, pending with one, then running. + assert len(reads_before_the_stop) == 3 + + +def test_the_container_is_checked_to_run_in_the_staging_environment(fakes, tmp_path): + result, _ = _run(fakes, tmp_path, "household", [_household_attempt(0)]) + + assert result.returncode == 0, result.stderr + listings = [ + call[1:] + for call in fakes.calls("uv") + if call[1:6] == ["run", "--frozen", "modal", "container", "list"] + ] + assert listings == [ + ["run", "--frozen", "modal", "container", "list", "--env", "staging", "--json"] + ] + + +def test_a_container_modal_does_not_list_in_staging_is_never_stopped(fakes, tmp_path): + result, _ = _run( + fakes, + tmp_path, + "household", + [_household_attempt(0)], + containers=ModalContainers(listed=()), + PREEMPTION_CHECK_ATTEMPTS="1", + ) + + assert result.returncode == 1 + assert _stops(fakes) == [] + assert f"{CONTAINERS[0]} is not listed in Modal environment staging" in ( + result.stderr + ) + + +def test_a_report_runs_coordinator_container_is_stopped_and_it_succeeds( + fakes, tmp_path +): + result, state = _run(fakes, tmp_path, "report", [_report_attempt(0)]) + + assert result.returncode == 0, result.stderr + assert _submissions(state) == [REPORT_REQUEST] + assert _stops(fakes) == [CONTAINERS[0]] + assert (state / "read-urls").read_text(encoding="utf-8").splitlines()[0] == ( + f"{BASE_URL}/reports/{REPORT_IDS[0]}" + ) + assert REPORT_RUN_IDS[0] in result.stdout + assert RERUN_CONTAINER in result.stdout + + +def test_a_report_whose_newest_run_is_not_the_created_one_fails(fakes, tmp_path): + attempt = _report_attempt(0) + attempt.reads_before_stop = [ + _report(0, "running", CALL_IDS[0], run_id=REPORT_RUN_IDS[1]) + ] + + result, _ = _run(fakes, tmp_path, "report", [attempt]) + + assert result.returncode == 1 + assert REPORT_RUN_IDS[1] in result.stderr + assert _stops(fakes) == [] + + +def test_a_calculation_that_finishes_before_the_stop_is_submitted_again( + fakes, tmp_path +): + result, state = _run( + fakes, + tmp_path, + "household", + [_household_attempt(0, proves=False), _household_attempt(1)], + ) + + assert result.returncode == 0, result.stderr + assert len(_submissions(state)) == 2 + assert _stops(fakes) == [CONTAINERS[1]] + assert "finished before its container was stopped" in result.stderr + + +def test_three_calculations_that_finish_before_the_stop_fail_the_check(fakes, tmp_path): + result, state = _run( + fakes, + tmp_path, + "household", + [_household_attempt(attempt, proves=False) for attempt in range(3)], + ) + + assert result.returncode == 1 + assert len(_submissions(state)) == 3 + assert _stops(fakes) == [] + assert "no attempt in 3" in result.stderr + + +def test_a_stop_modal_refuses_moves_on_to_another_attempt(fakes, tmp_path): + result, state = _run( + fakes, + tmp_path, + "household", + [_household_attempt(0), _household_attempt(1)], + containers=ModalContainers(refused_stops=(CONTAINERS[0],)), + ) + + assert result.returncode == 0, result.stderr + assert len(_submissions(state)) == 2 + assert _stops(fakes) == [CONTAINERS[0], CONTAINERS[1]] + assert f"could not stop container {CONTAINERS[0]}" in result.stderr + + +def test_a_call_that_finished_in_the_stopped_container_proves_nothing(fakes, tmp_path): + """The input finished just before the stop reached its container.""" + attempt = _household_attempt(0) + attempt.inputs_after_stop = [_inputs((CONTAINERS[0], "SUCCESS"))] + + result, _ = _run( + fakes, + tmp_path, + "household", + [attempt], + PREEMPTION_CHECK_ATTEMPTS="1", + PREEMPTION_CHECK_RERUN_SECONDS="0", + ) + + assert result.returncode == 1 + assert "never showed" in result.stderr + assert CONTAINERS[0] in result.stderr + + +def test_the_call_graph_is_asked_again_until_it_shows_the_other_container( + fakes, tmp_path +): + """Modal records the call graph best-effort and not in real time.""" + attempt = _household_attempt(0) + attempt.inputs_after_stop = [ + "ERROR", + _inputs((CONTAINERS[0], "TERMINATED")), + _inputs((CONTAINERS[0], "TERMINATED"), (RERUN_CONTAINER, "SUCCESS")), + ] + + result, _ = _run(fakes, tmp_path, "household", [attempt]) + + assert result.returncode == 0, result.stderr + assert RERUN_CONTAINER in result.stdout + + +def test_the_call_graph_is_asked_again_until_it_names_the_running_container( + fakes, tmp_path +): + attempt = _household_attempt(0) + attempt.reads_before_stop = [ + _household(JOB_IDS[0], "running", CALL_IDS[0]), + ] + attempt.inputs_before_stop = [ + "ERROR", + _inputs(("", "PENDING")), + _inputs((CONTAINERS[0], "PENDING")), + ] + + result, _ = _run(fakes, tmp_path, "household", [attempt]) + + assert result.returncode == 0, result.stderr + assert _stops(fakes) == [CONTAINERS[0]] + + +def test_a_failed_calculation_fails_the_check_at_once(fakes, tmp_path): + """A calculation that fails isn't submitted again: it would fail the same way.""" + attempt = _household_attempt(0) + attempt.reads_after_stop = [ + _household( + JOB_IDS[0], + "failed", + CALL_IDS[0], + error_message="household_function_error: the claim was lost", + ) + ] + + result, state = _run(fakes, tmp_path, "household", [attempt, _household_attempt(1)]) + + assert result.returncode == 1 + assert len(_submissions(state)) == 1 + assert "household_function_error: the claim was lost" in result.stderr + assert f"after its container {CONTAINERS[0]} was stopped" in result.stderr + + +def test_a_calculation_under_another_call_after_the_stop_fails(fakes, tmp_path): + """Modal runs a stopped container's input again under the same call ID.""" + attempt = _household_attempt(0) + attempt.reads_after_stop = [_household(JOB_IDS[0], "succeeded", "fc-01OTHER")] + + result, _ = _run(fakes, tmp_path, "household", [attempt]) + + assert result.returncode == 1 + assert "fc-01OTHER" in result.stderr + assert CALL_IDS[0] in result.stderr + + +def test_a_calculation_that_does_not_finish_in_time_fails(fakes, tmp_path): + attempt = _household_attempt(0) + attempt.reads_after_stop = [_household(JOB_IDS[0], "running", CALL_IDS[0])] + + result, _ = _run( + fakes, + tmp_path, + "household", + [attempt], + PREEMPTION_CHECK_DEADLINE_SECONDS="0", + ) + + assert result.returncode == 1 + assert "did not finish within 0 seconds" in result.stderr + assert "last read: the household job was" in result.stderr + + +@pytest.mark.parametrize( + "answer", + ["TRANSPORT", '503 {"code": "x"}', "429 {}"], + ids=["no-answer", "unavailable", "too-many-requests"], +) +def test_a_read_the_service_cannot_answer_now_is_asked_again(fakes, tmp_path, answer): + attempt = _household_attempt(0) + attempt.reads_before_stop = [answer, *attempt.reads_before_stop] + + result, _ = _run(fakes, tmp_path, "household", [attempt]) + + assert result.returncode == 0, result.stderr + + +def test_any_other_read_answer_fails_the_check(fakes, tmp_path): + attempt = _household_attempt(0) + attempt.reads_before_stop = ['404 {"code": "unknown_household_job"}'] + + result, _ = _run(fakes, tmp_path, "household", [attempt]) + + assert result.returncode == 1 + assert "404" in result.stderr + + +@pytest.mark.parametrize( + ("kind", "status", "body"), + [ + ("household", "503", {"code": "modal_call_refused", "detail": "x"}), + ("report", "400", {"code": "unknown_region", "detail": "x"}), + ], +) +def test_a_refused_submission_fails_the_check_with_its_answer( + fakes, tmp_path, kind, status, body +): + attempt = _household_attempt(0) if kind == "household" else _report_attempt(0) + attempt.submission_status = status + attempt.submission = body + + result, state = _run(fakes, tmp_path, kind, [attempt]) + + assert result.returncode == 1 + assert status in result.stderr + assert body["code"] in result.stderr + assert not (state / "read-urls").exists() + + +@pytest.mark.parametrize( + "call_id", + ["fc-01CALL1 --yes", "--help", "ta-01CONTAINER1", "fc-"], + ids=["extra-argument", "option", "container-id", "empty-suffix"], +) +def test_a_recorded_call_that_is_not_a_modal_call_id_fails_the_check( + fakes, tmp_path, call_id +): + attempt = _household_attempt(0) + attempt.reads_before_stop = [_household(JOB_IDS[0], "running", call_id)] + + result, _ = _run(fakes, tmp_path, "household", [attempt]) + + assert result.returncode == 1 + assert "not a Modal call ID" in result.stderr + assert fakes.calls("uv") == [] + + +def test_a_container_that_is_not_a_modal_container_id_is_never_stopped(fakes, tmp_path): + attempt = _household_attempt(0) + attempt.inputs_before_stop = [_inputs(("--all", "PENDING"))] + + result, _ = _run(fakes, tmp_path, "household", [attempt]) + + assert result.returncode == 1 + assert "not a Modal container ID" in result.stderr + assert _stops(fakes) == [] + + +def test_no_secret_reaches_a_command_line(fakes, tmp_path): + """The token is sent from a file, and the client secret on standard input.""" + result, _ = _run(fakes, tmp_path, "household", [_household_attempt(0)]) + + assert result.returncode == 0, result.stderr + arguments = [argument for call in fakes.calls() for argument in call] + assert not any(ACCESS_TOKEN in argument for argument in arguments) + assert not any(CLIENT_SECRET in argument for argument in arguments) + + +def test_every_request_has_a_time_limit(fakes, tmp_path): + """The submission, which isn't asked again, may take Cloud Run's full minute.""" + result, _ = _run(fakes, tmp_path, "report", [_report_attempt(0)]) + + assert result.returncode == 0, result.stderr + limits = { + call[-1]: call[call.index("--max-time") + 1] for call in fakes.calls("curl") + } + assert limits == { + "https://issuer.example/oauth/token": "30", + f"{BASE_URL}/reports": "60", + f"{BASE_URL}/reports/{REPORT_IDS[0]}": "30", + } + + +def test_the_check_runs_only_in_modals_staging_environment(fakes, tmp_path): + """It stops containers, so it never runs against production's.""" + result, _ = _run( + fakes, tmp_path, "household", [_household_attempt(0)], MODAL_ENVIRONMENT="main" + ) + + assert result.returncode == 1 + assert "staging" in result.stderr + assert fakes.calls() == [] + + +@pytest.mark.parametrize( + "variable_name", + [ + "SIMULATION_TEST_AUTH_CLIENT_ID", + "MODAL_TOKEN_ID", + "MODAL_TOKEN_SECRET", + "MODAL_ENVIRONMENT", + "STAGING_HOUSEHOLD_ID", + ], +) +def test_the_household_check_needs_its_settings(fakes, tmp_path, variable_name): + result, _ = _run( + fakes, tmp_path, "household", [_household_attempt(0)], **{variable_name: ""} + ) + + assert result.returncode == 1 + assert variable_name in result.stderr + assert fakes.calls() == [] + + +def test_the_report_check_needs_no_household(fakes, tmp_path): + result, _ = _run( + fakes, tmp_path, "report", [_report_attempt(0)], STAGING_HOUSEHOLD_ID="" + ) + + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize( + ("variable_name", "value"), + [ + ("STAGING_HOUSEHOLD_ID", "not-a-uuid"), + ("PREEMPTION_CHECK_ATTEMPTS", "0"), + ("PREEMPTION_CHECK_ATTEMPTS", "three"), + ("PREEMPTION_CHECK_DEADLINE_SECONDS", "ten"), + ("PREEMPTION_CHECK_POLL_SECONDS", "-1"), + ("PREEMPTION_CHECK_RERUN_SECONDS", "2m"), + ], +) +def test_a_malformed_setting_fails_the_check_before_any_request( + fakes, tmp_path, variable_name, value +): + result, _ = _run( + fakes, tmp_path, "household", [_household_attempt(0)], **{variable_name: value} + ) + + assert result.returncode == 1 + assert variable_name in result.stderr + assert fakes.calls() == [] + + +def test_an_unknown_kind_of_calculation_is_refused(fakes, tmp_path): + result, _ = _run(fakes, tmp_path, "simulation", [_household_attempt(0)]) + + assert result.returncode == 1 + assert "household or report" in result.stderr + assert fakes.calls() == [] + + +# --- The workflow job -------------------------------------------------------- + +CHECK_ENVIRONMENT = { + "SIMULATION_TEST_AUTH_ISSUER": "${{ secrets.SIMULATION_ENTRYPOINT_AUTH_ISSUER }}", + "SIMULATION_TEST_AUTH_AUDIENCE": ( + "${{ secrets.SIMULATION_ENTRYPOINT_AUTH_AUDIENCE }}" + ), + "SIMULATION_TEST_AUTH_CLIENT_ID": ( + "${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_ID }}" + ), + "SIMULATION_TEST_AUTH_CLIENT_SECRET": ( + "${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_SECRET }}" + ), + "MODAL_TOKEN_ID": "${{ secrets.MODAL_TOKEN_ID }}", + "MODAL_TOKEN_SECRET": "${{ secrets.MODAL_TOKEN_SECRET }}", + "MODAL_ENVIRONMENT": "${{ inputs.modal_environment }}", +} + + +def _jobs() -> dict: + return workflow_jobs(REUSABLE_DEPLOY_WORKFLOW) + + +def _context(**overrides): + context = routing_context() + context["inputs"]["deployment_environment"] = "staging" + context["inputs"]["deploy_stage12_v2"] = True + context["needs"]["deploy_entrypoint"] = {"result": "success"} + context["needs"]["publish_stage12_v2_default"] = {"result": "success"} + context["needs"]["staging_economy_check"] = {"result": "success"} + for path, value in overrides.items(): + target = context + *parents, last = path.split("__") + for part in parents: + target = target[part] + target[last] = value + return context + + +def test_the_check_job_stops_a_household_and_a_report_through_the_candidate(): + job = _jobs()[CHECK_JOB] + run_steps = [step for step in job["steps"] if "run" in step] + install, household, report = run_steps + + assert set(job["needs"]) == { + "deploy_entrypoint", + "authenticated_test", + "publish_stage12_v2_default", + "staging_economy_check", + } + assert job["environment"] == "${{ inputs.release_environment }}" + # Longer than both steps' own deadlines together, so the script fails first. + assert job["timeout-minutes"] == 100 + assert {step.get("uses") for step in job["steps"] if "uses" in step} == { + "actions/checkout@v6", + "actions/setup-python@v6", + "astral-sh/setup-uv@v8.1.0", + } + for step in run_steps: + assert step["working-directory"] == "projects/policyengine-simulation-executor" + assert install["run"] == "uv sync --frozen" + candidate = '"${{ needs.deploy_entrypoint.outputs.candidate_url }}"' + assert household["run"] == ( + f"../../.github/scripts/{CHECK_SCRIPT} household {candidate}" + ) + assert household["env"] == { + **CHECK_ENVIRONMENT, + "STAGING_HOUSEHOLD_ID": "${{ vars.SIMULATION_STAGING_HOUSEHOLD_ID }}", + } + assert report["run"] == f"../../.github/scripts/{CHECK_SCRIPT} report {candidate}" + assert report["env"] == CHECK_ENVIRONMENT + + +@pytest.mark.parametrize( + ("overrides", "runs"), + [ + ({}, True), + ({"inputs__deployment_environment": "production"}, False), + ({"needs__deploy_entrypoint__result": "failure"}, False), + ({"needs__authenticated_test__result": "failure"}, False), + ({"needs__staging_economy_check__result": "failure"}, False), + ({"needs__staging_economy_check__result": "skipped"}, False), + ({"needs__publish_stage12_v2_default__result": "failure"}, False), + ({"needs__publish_stage12_v2_default__result": "skipped"}, False), + ( + { + "inputs__deploy_stage12_v2": False, + "needs__publish_stage12_v2_default__result": "skipped", + }, + True, + ), + ({"cancelled": True}, False), + ], + ids=[ + "staging", + "production", + "entrypoint-failed", + "authenticated-tests-failed", + "economy-check-failed", + "economy-check-skipped", + "default-move-failed", + "default-move-skipped", + "no-stage12", + "cancelled", + ], +) +def test_the_check_runs_on_every_staging_deploy_after_the_economy_check( + overrides, runs +): + job = _jobs()[CHECK_JOB] + + assert ( + bool(evaluate_github_expression(job["if"], _context(**overrides), job_if=True)) + is runs + ) + + +def test_deployment_ready_requires_the_staging_preemption_check(): + ready = _jobs()["deployment_ready"] + (step,) = [ + step + for step in ready["steps"] + if step.get("name") == "Check the staging preemption check result" + ] + + assert CHECK_JOB in ready["needs"] + assert step["run"] == f".github/scripts/{READY_SCRIPT}" + assert step["env"] == { + "REQUESTED": "${{ inputs.deployment_environment == 'staging' }}", + "CHECK_RESULT": "${{ needs.staging_preemption_check.result }}", + } + + +@pytest.mark.parametrize( + ("requested", "check_result", "passes"), + [ + ("true", "success", True), + ("true", "failure", False), + ("true", "skipped", False), + ("true", "cancelled", False), + ("false", "skipped", True), + ("false", "success", False), + ], +) +def test_the_ready_check_requires_the_check_exactly_in_staging( + fakes, requested, check_result, passes +): + result = run_script( + READY_SCRIPT, + env=fakes.env(REQUESTED=requested, CHECK_RESULT=check_result), + ) + + assert result.returncode == (0 if passes else 1) From 693de57f2c651b3fc6716a78a6a50934084a91f1 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:49:43 +0400 Subject: [PATCH 06/24] Stop staging calculations' Modal containers on every staging deploy staging_preemption_check runs after the staging economy check. It calculates the staging household, then creates that check's report, whose simulation runs it reuses, through the candidate. Once the household function or coordinator has claimed each one, the check finds the container running its recorded Modal call through Modal's call graph, requires Modal to list it in the staging environment, and stops it with modal container stop, which sends SIGINT and gives the input to another container under the same call ID. Each must then succeed under that call, with the call graph showing it finished in another container. A calculation that finishes before its container is stopped proves nothing, so a new one is submitted, up to three attempts. One that fails or runs past its deadline fails the check at once, and deployment_ready requires the check in staging, so production waits for it. The test fakes now give a record that never finishes a short deadline. Co-Authored-By: Claude Opus 5.5 --- .../simulation-staging-preemption-check.sh | 356 ++++++++++++++++++ .../simulation-staging-preemption-ready.sh | 13 + .../workflows/simulation-deploy.reusable.yml | 78 +++- .../tests/test_staging_preemption_check.py | 11 +- 4 files changed, 456 insertions(+), 2 deletions(-) create mode 100755 .github/scripts/simulation-staging-preemption-check.sh create mode 100755 .github/scripts/simulation-staging-preemption-ready.sh diff --git a/.github/scripts/simulation-staging-preemption-check.sh b/.github/scripts/simulation-staging-preemption-check.sh new file mode 100755 index 000000000..7fd9a8acb --- /dev/null +++ b/.github/scripts/simulation-staging-preemption-check.sh @@ -0,0 +1,356 @@ +#!/bin/bash +# Stop the Modal container running a staging calculation, as a pre-emption +# does, and require the calculation to succeed anyway. +# Usage: simulation-staging-preemption-check.sh +# Run from projects/policyengine-simulation-executor, whose environment has +# the Modal client. +# +# household: calculates the stored staging household through the deployed +# Simulation Entrypoint, and stops the household function's container once +# the job is running, that is once the function has claimed it. +# report: creates the staging economy check's report, current law against +# current law for Wales in 2026, whose simulation runs that check has just +# left to reuse, and stops the coordinator's container once the report run +# is running, that is once the coordinator has claimed it. +# +# The container is the one Modal's call graph names for the Modal call the +# record names (modal_call_id), and it must be listed in Modal's staging +# environment. `modal container stop` sends it SIGINT, and Modal gives the +# call's input to another container under the same call ID, as it does +# after a pre-emption. The check passes once the record succeeds under that +# call and the call graph shows the call's input finished in another +# container. A calculation that finishes before its container is stopped +# proves nothing, so a new one is submitted, up to PREEMPTION_CHECK_ATTEMPTS +# times; a record that fails, or doesn't finish within the deadline, fails +# the check at once. +# +# Required env vars: the test client's SIMULATION_TEST_AUTH_ISSUER, +# SIMULATION_TEST_AUTH_AUDIENCE, SIMULATION_TEST_AUTH_CLIENT_ID and +# SIMULATION_TEST_AUTH_CLIENT_SECRET; MODAL_TOKEN_ID and MODAL_TOKEN_SECRET, +# which the Modal client reads; MODAL_ENVIRONMENT, which must be staging; +# and, for household, STAGING_HOUSEHOLD_ID (the ID of a US household stored +# in the staging Public API). +# Optional: PREEMPTION_CHECK_ATTEMPTS (3); PREEMPTION_CHECK_DEADLINE_SECONDS, +# each attempt's deadline (600 for household, 900 for report); +# PREEMPTION_CHECK_POLL_SECONDS (2); and PREEMPTION_CHECK_RERUN_SECONDS +# (120), how long after the record succeeds the call graph may take to show +# the call's other container. +# The access token is sent from a file and the client secret on standard +# input, so neither appears on a command line. A submission, which isn't +# asked again, may take 60 seconds, Cloud Run's own limit; every other +# request 30. A read that gets no answer, a 429 or any 5xx is asked again +# until the deadline. + +set -euo pipefail + +fail() { + printf 'Staging preemption check failed: %s\n' "$1" >&2 + exit 1 +} + +kind="${1:-}" +base_url="${2:?Simulation Entrypoint base URL is required}" +base_url="${base_url%/}" +case "${kind}" in + household) default_deadline_seconds=600 ;; + report) default_deadline_seconds=900 ;; + *) fail "the first argument names the calculation, household or report, not '${kind}'" ;; +esac +attempts="${PREEMPTION_CHECK_ATTEMPTS:-3}" +deadline_seconds="${PREEMPTION_CHECK_DEADLINE_SECONDS:-${default_deadline_seconds}}" +poll_seconds="${PREEMPTION_CHECK_POLL_SECONDS:-2}" +rerun_seconds="${PREEMPTION_CHECK_RERUN_SECONDS:-120}" +request_timeout_seconds=30 +submission_timeout_seconds=60 +uuid_pattern='^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' +call_id_pattern='^fc-[0-9A-Za-z]+$' +container_id_pattern='^ta-[0-9A-Za-z]+$' + +required_variables=(SIMULATION_TEST_AUTH_ISSUER SIMULATION_TEST_AUTH_AUDIENCE + SIMULATION_TEST_AUTH_CLIENT_ID SIMULATION_TEST_AUTH_CLIENT_SECRET + MODAL_TOKEN_ID MODAL_TOKEN_SECRET MODAL_ENVIRONMENT) +if [[ "${kind}" == "household" ]]; then + required_variables+=(STAGING_HOUSEHOLD_ID) +fi +for variable_name in "${required_variables[@]}"; do + [[ -n "${!variable_name:-}" ]] || fail "${variable_name} must be set" +done +[[ "${MODAL_ENVIRONMENT}" == "staging" ]] \ + || fail "MODAL_ENVIRONMENT is ${MODAL_ENVIRONMENT}: this check stops containers, so it runs only in Modal's staging environment" +if [[ "${kind}" == "household" ]]; then + [[ "${STAGING_HOUSEHOLD_ID}" =~ ${uuid_pattern} ]] \ + || fail "STAGING_HOUSEHOLD_ID is not a UUID in lowercase hexadecimal" +fi +[[ "${attempts}" =~ ^[1-9][0-9]*$ ]] \ + || fail "PREEMPTION_CHECK_ATTEMPTS must be a whole number of at least 1" +for setting in "PREEMPTION_CHECK_DEADLINE_SECONDS=${deadline_seconds}" \ + "PREEMPTION_CHECK_POLL_SECONDS=${poll_seconds}" \ + "PREEMPTION_CHECK_RERUN_SECONDS=${rerun_seconds}"; do + [[ "${setting#*=}" =~ ^[0-9]+$ ]] \ + || fail "${setting%%=*} must be a whole number of seconds" +done + +work_directory="$(mktemp -d)" +trap 'rm -rf "${work_directory}"' EXIT +chmod 700 "${work_directory}" + +# Send one request, allowing it ; write its body to .json and +# print its HTTP status. It fails when no answer arrives. +send() { + local name="$1" + local seconds="$2" + shift 2 + curl --silent --show-error --max-time "${seconds}" \ + --output "${work_directory}/${name}.json" --write-out '%{http_code}' "$@" +} + +token_status="$( + jq -n \ + --arg client_id "${SIMULATION_TEST_AUTH_CLIENT_ID}" \ + --arg client_secret "${SIMULATION_TEST_AUTH_CLIENT_SECRET}" \ + --arg audience "${SIMULATION_TEST_AUTH_AUDIENCE}" \ + '{client_id: $client_id, client_secret: $client_secret, audience: $audience, grant_type: "client_credentials"}' \ + | send token "${request_timeout_seconds}" --request POST --header 'Content-Type: application/json' \ + --data @- "${SIMULATION_TEST_AUTH_ISSUER%/}/oauth/token" +)" || fail "the token request got no answer" +[[ "${token_status}" == "200" ]] || fail "the token request answered ${token_status}" +access_token="$(jq -r '.access_token // empty' "${work_directory}/token.json")" +[[ -n "${access_token}" ]] || fail "the token response has no access_token" +authorization_header="${work_directory}/authorization-header" +printf 'Authorization: Bearer %s\n' "${access_token}" > "${authorization_header}" + +# Print the inputs of Modal call as JSON; fail when Modal can't be +# asked (src/modal/utils/describe_call_inputs.py). +call_inputs() { + uv run --frozen python -m src.modal.utils.describe_call_inputs "$1" +} + +# Succeed when Modal lists container as running in +# MODAL_ENVIRONMENT. +container_listed() { + local listing + listing="$(uv run --frozen modal container list --env "${MODAL_ENVIRONMENT}" --json)" \ + || return 1 + jq -e --arg container_id "$1" 'any(.[]; .["Container ID"] == $container_id)' \ + <<<"${listing}" >/dev/null +} + +# Create the attempt's record. Sets record_url, record_noun, record_label +# and, for a report, report_run_id. +submit() { + local submission="${work_directory}/submission.json" + local submission_status + if [[ "${kind}" == "household" ]]; then + submission_status="$( + jq -n --arg household_id "${STAGING_HOUSEHOLD_ID}" \ + '{country: "us", household_id: $household_id}' \ + | send submission "${submission_timeout_seconds}" --request POST \ + --header "@${authorization_header}" --header 'Content-Type: application/json' \ + --data @- "${base_url}/household-calculations" + )" || fail "POST /household-calculations got no answer; it may have created a household job" + [[ "${submission_status}" == "202" ]] \ + || fail "POST /household-calculations answered ${submission_status}: $(cat "${submission}")" + local job_id + job_id="$(jq -r '.job_id // empty' "${submission}")" + [[ "${job_id}" =~ ${uuid_pattern} ]] || fail "the submission named no household job" + record_url="${base_url}/household-calculations/${job_id}" + record_noun="household job" + record_label="${record_noun} ${job_id}" + printf 'Submitted household job %s for staging household %s\n' \ + "${job_id}" "${STAGING_HOUSEHOLD_ID}" + else + submission_status="$( + jq -n '{ + country: "uk", + report_type: "economy", + label: "Staging preemption check", + year: 2026, + region_code: "country/wales", + baseline: {policy_id: null, dynamic_id: null}, + reform: {policy_id: null, dynamic_id: null}, + include_cliffs: false + }' \ + | send submission "${submission_timeout_seconds}" --request POST \ + --header "@${authorization_header}" --header 'Content-Type: application/json' \ + --data @- "${base_url}/reports" + )" || fail "POST /reports got no answer; it may have created a report" + [[ "${submission_status}" == "201" ]] \ + || fail "POST /reports answered ${submission_status}: $(cat "${submission}")" + local report_id + report_id="$(jq -r '.report_id // empty' "${submission}")" + report_run_id="$(jq -r '.report_run_id // empty' "${submission}")" + [[ "${report_id}" =~ ${uuid_pattern} ]] || fail "the submission named no report" + [[ "${report_run_id}" =~ ${uuid_pattern} ]] || fail "the submission named no report run" + record_url="${base_url}/reports/${report_id}" + record_noun="report run" + record_label="${record_noun} ${report_run_id}" + printf 'Created report %s, run %s (baseline reused: %s, reform reused: %s)\n' \ + "${report_id}" "${report_run_id}" \ + "$(jq -r '.baseline_reused' "${submission}")" "$(jq -r '.reform_reused' "${submission}")" + fi +} + +# Read the record once. Sets record_status, record_call_id, record_error and +# last_read; leaves record_status empty when the service can't answer now. +read_record() { + local record="${work_directory}/record.json" + local read_status + record_status="" + if ! read_status="$( + send record "${request_timeout_seconds}" --header "@${authorization_header}" "${record_url}" + )"; then + last_read="it got no answer" + printf 'GET %s got no answer; asking again\n' "${record_url}" >&2 + return 0 + fi + if [[ "${read_status}" == "429" || "${read_status}" == 5?? ]]; then + # The service, or Cloud Run's front end in its place, can't answer now. + last_read="it answered ${read_status}" + printf 'GET %s answered %s; asking again\n' "${record_url}" "${read_status}" >&2 + return 0 + fi + [[ "${read_status}" == "200" ]] \ + || fail "GET ${record_url} answered ${read_status}: $(cat "${record}")" + # The household job's fields, or the report's newest run's. + local record_filter='' + if [[ "${kind}" == "report" ]]; then + local newest_run_id + newest_run_id="$(jq -r '.run.run_id // empty' "${record}")" + [[ "${newest_run_id}" == "${report_run_id}" ]] \ + || fail "the report's newest run is ${newest_run_id:-missing}, not the created run ${report_run_id}" + record_filter='.run' + fi + record_status="$(jq -r "${record_filter}.status" "${record}")" + record_call_id="$(jq -r "${record_filter}.modal_call_id // empty" "${record}")" + record_error="$(jq -r "${record_filter}.error_message // \"no error message\"" "${record}")" + if [[ -n "${record_call_id}" && ! "${record_call_id}" =~ ${call_id_pattern} ]]; then + fail "${record_label} names Modal call '${record_call_id}', which is not a Modal call ID" + fi + if [[ "${kind}" == "household" && "${record_status}" == "succeeded" ]]; then + [[ "$(jq -r '.result.status // empty' "${record}")" == "succeeded" ]] \ + || fail "${record_label} succeeded without a succeeded result document" + fi + last_read="the ${record_noun} was ${record_status}" + return 0 +} + +# Find the container running the record's call. Sets container_id once the +# call graph names a running input's container that Modal lists in +# MODAL_ENVIRONMENT; until then, unlisted_note may say why not. +find_container() { + local inputs candidate + if ! inputs="$(call_inputs "${record_call_id}")"; then + printf 'Modal could not say which container runs call %s yet; asking again\n' \ + "${record_call_id}" >&2 + return 0 + fi + candidate="$( + jq -r '[.[] | select(.status == "PENDING" and .container_id != "")] | first | .container_id // empty' \ + <<<"${inputs}" + )" || fail "the call-input helper printed something other than a list of inputs for call ${record_call_id}" + [[ -n "${candidate}" ]] || return 0 + [[ "${candidate}" =~ ${container_id_pattern} ]] \ + || fail "Modal names '${candidate}' as call ${record_call_id}'s container, which is not a Modal container ID" + if container_listed "${candidate}"; then + container_id="${candidate}" + printf 'Call %s of %s runs in container %s\n' "${record_call_id}" "${record_label}" "${container_id}" + else + unlisted_note="container ${candidate} is not listed in Modal environment ${MODAL_ENVIRONMENT}" + printf '%s; asking again\n' "${unlisted_note}" >&2 + fi +} + +# After the record succeeded: require the call graph to show the stopped +# call's input finished in another container. Sets attempt_outcome. +confirm_rerun() { + local waiting_since="${SECONDS}" + local inputs rerun_container_id="" + while true; do + if inputs="$(call_inputs "${stopped_call_id}")"; then + rerun_container_id="$( + jq -r --arg stopped "${stopped_container_id}" \ + '[.[] | select(.status == "SUCCESS" and .container_id != "" and .container_id != $stopped)] | first | .container_id // empty' \ + <<<"${inputs}" + )" || fail "the call-input helper printed something other than a list of inputs for call ${stopped_call_id}" + fi + if [[ -n "${rerun_container_id}" ]]; then + printf '%s succeeded after its container %s was stopped: Modal ran call %s again in container %s\n' \ + "${record_label}" "${stopped_container_id}" "${stopped_call_id}" "${rerun_container_id}" + attempt_outcome="proven" + return 0 + fi + if ((SECONDS - waiting_since >= rerun_seconds)); then + attempt_outcome="missed" + miss_reason="${record_label} succeeded, but Modal's call graph never showed call ${stopped_call_id} finishing in a container other than the stopped ${stopped_container_id} within ${rerun_seconds} seconds, so it may have finished before the stop" + return 0 + fi + sleep "${poll_seconds}" + done +} + +# Submit one calculation, stop its container once it's running, and follow +# it to the end. Sets attempt_outcome to proven or missed (with +# miss_reason); fails the check for a failed or late calculation. +attempt_once() { + submit + record_call_id="" + container_id="" + stopped_container_id="" + stopped_call_id="" + unlisted_note="" + last_read="nothing yet" + local started_at="${SECONDS}" + while true; do + read_record + if [[ -n "${stopped_call_id}" && -n "${record_call_id}" \ + && "${record_call_id}" != "${stopped_call_id}" ]]; then + fail "${record_label} names call ${record_call_id} after its container ${stopped_container_id} was stopped, not call ${stopped_call_id}, which Modal runs again under the same ID" + fi + case "${record_status}" in + "") ;; + succeeded) + if [[ -z "${stopped_container_id}" ]]; then + attempt_outcome="missed" + miss_reason="${record_label} finished before its container was stopped${unlisted_note:+ (${unlisted_note})}" + else + confirm_rerun + fi + return 0 + ;; + failed) + fail "${record_label} failed${stopped_container_id:+ after its container ${stopped_container_id} was stopped}: ${record_error}" + ;; + pending | running) + if [[ -z "${stopped_container_id}" && -n "${record_call_id}" ]]; then + [[ -n "${container_id}" ]] || find_container + if [[ "${record_status}" == "running" && -n "${container_id}" ]]; then + if uv run --frozen modal container stop --yes "${container_id}"; then + stopped_container_id="${container_id}" + stopped_call_id="${record_call_id}" + printf 'Stopped container %s while %s was running; waiting for Modal to run call %s again\n' \ + "${stopped_container_id}" "${record_label}" "${stopped_call_id}" + else + attempt_outcome="missed" + miss_reason="modal container stop could not stop container ${container_id} running ${record_label}" + return 0 + fi + fi + fi + ;; + *) fail "${record_label} has an unknown status: ${record_status}" ;; + esac + ((SECONDS - started_at < deadline_seconds)) \ + || fail "${record_label} did not finish within ${deadline_seconds} seconds (last read: ${last_read})" + sleep "${poll_seconds}" + done +} + +for ((attempt = 1; attempt <= attempts; attempt++)); do + printf 'Attempt %d of %d\n' "${attempt}" "${attempts}" + attempt_outcome="" + miss_reason="" + attempt_once + [[ "${attempt_outcome}" != "proven" ]] || exit 0 + printf 'Attempt %d of %d proved nothing: %s\n' "${attempt}" "${attempts}" "${miss_reason}" >&2 +done +fail "no attempt in ${attempts} stopped the container of a running ${kind} calculation; the last: ${miss_reason}" diff --git a/.github/scripts/simulation-staging-preemption-ready.sh b/.github/scripts/simulation-staging-preemption-ready.sh new file mode 100755 index 000000000..1bc0ab5a5 --- /dev/null +++ b/.github/scripts/simulation-staging-preemption-ready.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Require the staging preemption check to have succeeded in a staging +# deployment, and to have been skipped in any other. +# Required env vars: REQUESTED ("true" in a staging deployment) and +# CHECK_RESULT (the staging_preemption_check job result). + +set -euo pipefail + +if [[ "${REQUESTED}" == "true" ]]; then + test "${CHECK_RESULT}" = "success" +else + test "${CHECK_RESULT}" = "skipped" +fi diff --git a/.github/workflows/simulation-deploy.reusable.yml b/.github/workflows/simulation-deploy.reusable.yml index b89de5283..7eae4cac2 100644 --- a/.github/workflows/simulation-deploy.reusable.yml +++ b/.github/workflows/simulation-deploy.reusable.yml @@ -1290,6 +1290,76 @@ jobs: SIMULATION_TEST_AUTH_CLIENT_SECRET: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_SECRET }} run: .github/scripts/simulation-staging-economy-check.sh "${{ needs.deploy_entrypoint.outputs.candidate_url }}" + staging_preemption_check: + name: Stop staging calculations' Modal containers and require them to succeed + # Every staging deployment calculates the staging household, then + # creates the staging economy check's report, through the candidate, and + # stops the Modal container running each one's call with `modal + # container stop` once the household function or coordinator has claimed + # it. That sends SIGINT and gives the input to another container under + # the same call ID, as a pre-emption does; each must still succeed. It + # runs after the economy check, so the report reuses that check's + # simulation runs and stopping the coordinator costs no new simulation. + # The script refuses any Modal environment but staging, and stops only a + # container Modal lists there. Each calculation writes rows in the + # staging database, as the other staging checks' do. + if: >- + ${{ + !cancelled() && + inputs.deployment_environment == 'staging' && + needs.deploy_entrypoint.result == 'success' && + needs.authenticated_test.result == 'success' && + needs.staging_economy_check.result == 'success' && + ((inputs.deploy_stage12_v2 && needs.publish_stage12_v2_default.result == 'success') || + (!inputs.deploy_stage12_v2 && needs.publish_stage12_v2_default.result == 'skipped')) + }} + needs: [deploy_entrypoint, authenticated_test, publish_stage12_v2_default, staging_economy_check] + runs-on: ubuntu-latest + # Longer than both steps' own deadlines together (three attempts of 12 + # minutes for the household and of 17 for the report), so the script + # fails first. + timeout-minutes: 100 + environment: ${{ inputs.release_environment }} + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - uses: astral-sh/setup-uv@v8.1.0 + with: + version: "0.13.6" + + - name: Install the Modal client + working-directory: projects/policyengine-simulation-executor + run: uv sync --frozen + + - name: Stop a staging household calculation's container and require it to succeed + working-directory: projects/policyengine-simulation-executor + env: + STAGING_HOUSEHOLD_ID: ${{ vars.SIMULATION_STAGING_HOUSEHOLD_ID }} + SIMULATION_TEST_AUTH_ISSUER: ${{ secrets.SIMULATION_ENTRYPOINT_AUTH_ISSUER }} + SIMULATION_TEST_AUTH_AUDIENCE: ${{ secrets.SIMULATION_ENTRYPOINT_AUTH_AUDIENCE }} + SIMULATION_TEST_AUTH_CLIENT_ID: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_ID }} + SIMULATION_TEST_AUTH_CLIENT_SECRET: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_SECRET }} + MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} + MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} + MODAL_ENVIRONMENT: ${{ inputs.modal_environment }} + run: ../../.github/scripts/simulation-staging-preemption-check.sh household "${{ needs.deploy_entrypoint.outputs.candidate_url }}" + + - name: Stop a staging report's coordinator container and require it to succeed + working-directory: projects/policyengine-simulation-executor + env: + SIMULATION_TEST_AUTH_ISSUER: ${{ secrets.SIMULATION_ENTRYPOINT_AUTH_ISSUER }} + SIMULATION_TEST_AUTH_AUDIENCE: ${{ secrets.SIMULATION_ENTRYPOINT_AUTH_AUDIENCE }} + SIMULATION_TEST_AUTH_CLIENT_ID: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_ID }} + SIMULATION_TEST_AUTH_CLIENT_SECRET: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_SECRET }} + MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} + MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} + MODAL_ENVIRONMENT: ${{ inputs.modal_environment }} + run: ../../.github/scripts/simulation-staging-preemption-check.sh report "${{ needs.deploy_entrypoint.outputs.candidate_url }}" + finalize_routing: name: Finalize the Modal routing publication # Finish the durable journal after every gate, including failed uploads @@ -1348,7 +1418,7 @@ jobs: # The reporter only reads job results. Journal completion happens before # it, so a failure or cancellation here cannot strand a publication. if: ${{ !cancelled() }} - needs: [deploy_entrypoint, update_routing, integration, authenticated_test, promote_entrypoint, publish_latest, finalize_routing, publish_stage12_v2_default, staging_household_check, staging_economy_check] + needs: [deploy_entrypoint, update_routing, integration, authenticated_test, promote_entrypoint, publish_latest, finalize_routing, publish_stage12_v2_default, staging_household_check, staging_economy_check, staging_preemption_check] runs-on: ubuntu-latest defaults: run: @@ -1389,6 +1459,12 @@ jobs: CHECK_RESULT: ${{ needs.staging_economy_check.result }} run: .github/scripts/simulation-staging-economy-ready.sh + - name: Check the staging preemption check result + env: + REQUESTED: ${{ inputs.deployment_environment == 'staging' }} + CHECK_RESULT: ${{ needs.staging_preemption_check.result }} + run: .github/scripts/simulation-staging-preemption-ready.sh + cleanup_entrypoint_revisions: name: Remove superseded Cloud Run revisions if: ${{ !cancelled() && inputs.promote_entrypoint && needs.deployment_ready.result == 'success' }} diff --git a/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py index d5b417d89..1d1b7b01c 100644 --- a/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py +++ b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py @@ -378,6 +378,9 @@ def _run( "MODAL_TOKEN_SECRET": "as-test", "MODAL_ENVIRONMENT": "staging", "STAGING_HOUSEHOLD_ID": HOUSEHOLD_ID, + # The fake sleep returns at once, so a record that never finishes + # fails the test within this many seconds. + "PREEMPTION_CHECK_DEADLINE_SECONDS": "20", "FAKE_STATE": str(state), "FAKE_BASE_URL": BASE_URL, "FAKE_ACCESS_TOKEN": ACCESS_TOKEN, @@ -454,11 +457,17 @@ def test_the_container_is_checked_to_run_in_the_staging_environment(fakes, tmp_p def test_a_container_modal_does_not_list_in_staging_is_never_stopped(fakes, tmp_path): + attempt = _household_attempt(0) + attempt.reads_before_stop = [ + *attempt.reads_before_stop, + _household(JOB_IDS[0], "succeeded", CALL_IDS[0]), + ] + result, _ = _run( fakes, tmp_path, "household", - [_household_attempt(0)], + [attempt], containers=ModalContainers(listed=()), PREEMPTION_CHECK_ATTEMPTS="1", ) From a429b714cd9dec18b9a235180f8bb0c6ad90c653 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:51:03 +0400 Subject: [PATCH 07/24] Document the staging preemption check and the reads' Modal call DEPLOYMENT.md describes staging_preemption_check: what it starts, when and how it stops each container, what it requires, its attempts and deadlines, and the Modal token it needs. The entry and persistence READMEs name the Modal call the household job and report run reads now return. The changelog fragment is for #1062. Co-Authored-By: Claude Opus 5.5 --- changelog.d/1062.added.md | 1 + .../README.md | 3 +- .../DEPLOYMENT.md | 41 +++++++++++++++++++ .../policyengine-simulation-entry/README.md | 11 +++-- 4 files changed, 51 insertions(+), 5 deletions(-) create mode 100644 changelog.d/1062.added.md diff --git a/changelog.d/1062.added.md b/changelog.d/1062.added.md new file mode 100644 index 000000000..8e8d823ce --- /dev/null +++ b/changelog.d/1062.added.md @@ -0,0 +1 @@ +Every staging deploy stops the Modal container running a household calculation and a small economy report, as a pre-emption does, and fails unless both still succeed; `GET /household-calculations/{household_job_id}` and the run in `GET /reports/{report_id}` now name the Modal call recorded on them (`modal_call_id`). diff --git a/libs/policyengine-simulation-persistence/README.md b/libs/policyengine-simulation-persistence/README.md index 4fea4fd37..7efc391da 100644 --- a/libs/policyengine-simulation-persistence/README.md +++ b/libs/policyengine-simulation-persistence/README.md @@ -111,7 +111,8 @@ call's ID, and `fail_unstarted_report_run` and `fail_unstarted_simulation_run` fail a run at once, each only while it is `pending` with no call ID; failing a report run also fails the simulation runs created for it. `economy_run_reads.py` reads a report, or a simulation, -with its newest run's state. +with its newest run's state; a report run's state includes the Modal call +recorded on it. `EconomyRunStore` runs them as the Simulation Entrypoint does, each step in a transaction of its own: `read_report_inputs` reads the catalog version, diff --git a/projects/policyengine-simulation-entry/DEPLOYMENT.md b/projects/policyengine-simulation-entry/DEPLOYMENT.md index 5972daaff..8076d6202 100644 --- a/projects/policyengine-simulation-entry/DEPLOYMENT.md +++ b/projects/policyengine-simulation-entry/DEPLOYMENT.md @@ -142,6 +142,47 @@ check has succeeded, later checks on that version reuse its simulation runs, so they check the coordinator's use of stored outputs rather than the workers', which the v2 deployment's own validation calls. +### The staging preemption check + +Every staging deployment runs `staging_preemption_check` once the economy +check has succeeded. `.github/scripts/simulation-staging-preemption-check.sh` +runs twice, from the executor project, whose environment has the Modal +client: + +- `household` calculates the staging household that + `SIMULATION_STAGING_HOUSEHOLD_ID` names, as the household check does; +- `report` creates the economy check's report again (labelled "Staging + preemption check"), whose simulation runs reuse that check's, so it costs + no new simulation. + +Once the household job or report run is `running`, that is once the +household function or coordinator has claimed it, the check takes the +`modal_call_id` its read answers, asks Modal's call graph which container +runs that call (`src/modal/utils/describe_call_inputs.py`), and stops that +container with `modal container stop`. Modal sends the container SIGINT and +gives the input to another container under the same call ID, as it does +after a pre-emption. The check requires the record to succeed under that +call, and the call graph to show the call's input finished in another +container. Run again, the household function or coordinator resumes its +claim. The report's simulation runs are reused, so its coordinator +starts no worker, and the check doesn't cover a coordinator starting its +workers again. + +A calculation that finishes before its container is stopped, for example on +a warm container, proves nothing, so the check submits a new one, up to +three attempts. A record that fails, or doesn't finish within its deadline +(10 minutes for the household, 15 for the report, per attempt), fails the +check at once. + +The check refuses any `MODAL_ENVIRONMENT` but `staging`, and stops only a +container that `modal container list --env staging` lists. It uses the +`beta` environment's Modal token (`MODAL_TOKEN_ID`, `MODAL_TOKEN_SECRET`), +which must be allowed to read call graphs and to list and stop containers, +and the authenticated tests' client credentials. `deployment_ready` requires +the check to pass in staging, so a failure stops the production deployment. +Each attempt adds a household job, or a report and its run, to the staging +database. + ## Candidate qualification For each eligible revision, the workflow first deploys `beta` diff --git a/projects/policyengine-simulation-entry/README.md b/projects/policyengine-simulation-entry/README.md index bc32cc1f6..bb7f04f41 100644 --- a/projects/policyengine-simulation-entry/README.md +++ b/projects/policyengine-simulation-entry/README.md @@ -85,8 +85,10 @@ default worker, creates a `household_jobs` row and starts the household function's call with `spawn`, answering 202 with the job's `job_id` and `status`. Each submission creates a new job. `GET /household-calculations/{household_job_id}` returns the job's status, -the household function's `result` document once it has one, and its -`error_message`. +the household function's `result` document once it has one, its +`error_message`, and `modal_call_id`, the household function's Modal call +recorded on the job (null until the service records one). Modal runs a +pre-empted call again under the same ID. A submission is refused before anything is written: with 400 and a `code` when the household, policy or catalog version for the default worker's .py @@ -153,6 +155,7 @@ exception's text. `GET /reports/{report_id}` and `GET /simulations/{simulation_id}` return the record and its newest run (created last, ties to the larger ID): its status, -versions, error message and times, and a report run's simulation runs. The -Public API reads results from the tables itself. +versions, error message and times, and a report run's simulation runs and +`modal_call_id`, the coordinator's Modal call recorded on it (null until the +service records one). The Public API reads results from the tables itself. From 614b3e71e15b1e44a7ec0181ba69274021cf8409 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 02:11:01 +0400 Subject: [PATCH 08/24] Test review round 1's fixes to the staging preemption check (failing) - A stop Modal refuses is not tried again and starts no new submission: the check follows the job to its end, so a job left running by a container that ended for another reason fails at its deadline, and one that succeeded had finished just before the stop. - Before a new attempt, the check waits for the last attempt's container to leave Modal's listing, so the next calculation starts in a new container instead of a warm one too fast to stop. - No command, jq included, gets the client secret as an argument. - A rerun Modal records on the stopped input itself counts (this passes already: the check never compares input IDs). - The read routes' descriptions name the Modal call they return. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_openapi.py | 5 +- .../tests/test_staging_preemption_check.py | 126 +++++++++++++++++- 2 files changed, 123 insertions(+), 8 deletions(-) diff --git a/projects/policyengine-simulation-entry/tests/test_openapi.py b/projects/policyengine-simulation-entry/tests/test_openapi.py index b1f9b7463..26d2ebd94 100644 --- a/projects/policyengine-simulation-entry/tests/test_openapi.py +++ b/projects/policyengine-simulation-entry/tests/test_openapi.py @@ -463,8 +463,11 @@ def test_household_job_and_report_run_reads_name_their_modal_call(): The staging preemption check finds the container to stop from it. """ - schemas = create_app().openapi()["components"]["schemas"] + spec = create_app().openapi() + schemas = spec["components"]["schemas"] + for path in ("/household-calculations/{household_job_id}", "/reports/{report_id}"): + assert "Modal call" in spec["paths"][path]["get"]["description"] for schema_name in ("HouseholdCalculationState", "ReportRunRead"): schema = schemas[schema_name] assert "modal_call_id" in schema["required"] diff --git a/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py index 1d1b7b01c..c7e2ce0d2 100644 --- a/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py +++ b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py @@ -19,6 +19,8 @@ from __future__ import annotations import json +import shlex +import shutil from dataclasses import dataclass, field from typing import TYPE_CHECKING @@ -148,9 +150,14 @@ printf '%s\n' "${answer}" ;; "run --frozen modal container list --env "*" --json") - jq '[.listed[] | {"Container ID": ., "App ID": "ap-01V2", - "App Name": "policyengine-v2", "Start Time": "2026-11-13T12:00:00+00:00"}]' \ - "${spec}" + counter="${FAKE_STATE}/listings" + listing="$(($(cat "${counter}" 2>/dev/null || echo 0) + 1))" + echo "${listing}" > "${counter}" + # A container with a departure count is listed only that many times. + jq --argjson listing "${listing}" \ + '[.listed[] as $id | select((.departures[$id] // $listing) >= $listing) + | {"Container ID": $id, "App ID": "ap-01V2", "App Name": "policyengine-v2", + "Start Time": "2026-11-13T12:00:00+00:00"}]' "${spec}" ;; "run --frozen modal container stop --yes "*) if jq -e --arg id "${last}" '.refused_stops | index($id)' "${spec}" >/dev/null; then @@ -316,11 +323,16 @@ class ModalContainers: listed: tuple[str, ...] = (*CONTAINERS, RERUN_CONTAINER) refused_stops: tuple[str, ...] = () + # How many listings name each container before Modal scales it down. + departures: dict[str, int] = field(default_factory=dict) @pytest.fixture def fakes(tmp_path) -> FakeCommands: commands = FakeCommands.create(tmp_path) + real_jq = shutil.which("jq") + assert real_jq is not None + commands.add("jq", f'exec {shlex.quote(real_jq)} "$@"') commands.add("curl", FAKE_CURL) commands.add("uv", FAKE_UV) commands.add("sleep") @@ -367,6 +379,7 @@ def _run( }, "listed": list(containers.listed), "refused_stops": list(containers.refused_stops), + "departures": containers.departures, } (state / "spec.json").write_text(json.dumps(spec), encoding="utf-8") environment = { @@ -381,6 +394,8 @@ def _run( # The fake sleep returns at once, so a record that never finishes # fails the test within this many seconds. "PREEMPTION_CHECK_DEADLINE_SECONDS": "20", + # Most tests don't wait for an earlier attempt's container to leave. + "PREEMPTION_CHECK_SCALEDOWN_SECONDS": "0", "FAKE_STATE": str(state), "FAKE_BASE_URL": BASE_URL, "FAKE_ACCESS_TOKEN": ACCESS_TOKEN, @@ -523,6 +538,52 @@ def test_a_calculation_that_finishes_before_the_stop_is_submitted_again( assert "finished before its container was stopped" in result.stderr +def test_the_next_attempt_waits_for_the_last_attempts_container_to_leave( + fakes, tmp_path +): + """A warm container calculates too fast to stop; Modal scales it down idle.""" + result, state = _run( + fakes, + tmp_path, + "household", + [_household_attempt(0, proves=False), _household_attempt(1)], + containers=ModalContainers(departures={CONTAINERS[0]: 3}), + PREEMPTION_CHECK_SCALEDOWN_SECONDS="120", + ) + + assert result.returncode == 0, result.stderr + calls = fakes.calls() + listings = [ + index for index, call in enumerate(calls) if call[0] == "uv" and "list" in call + ] + submissions = [ + index + for index, call in enumerate(calls) + if call[0] == "curl" and call[-1].endswith("/household-calculations") + ] + # The first attempt's lookup, then two listings naming its container and + # one without it, then the second attempt's lookup. + assert len(listings) == 5 + assert listings[3] < submissions[1] < listings[4] + assert len(_submissions(state)) == 2 + assert f"waiting for container {CONTAINERS[0]}" in result.stdout + + +def test_a_container_still_listed_after_the_wait_does_not_hold_the_check( + fakes, tmp_path +): + result, state = _run( + fakes, + tmp_path, + "household", + [_household_attempt(0, proves=False), _household_attempt(1)], + ) + + assert result.returncode == 0, result.stderr + assert len(_submissions(state)) == 2 + assert f"container {CONTAINERS[0]} is still listed" in result.stderr + + def test_three_calculations_that_finish_before_the_stop_fail_the_check(fakes, tmp_path): result, state = _run( fakes, @@ -537,19 +598,54 @@ def test_three_calculations_that_finish_before_the_stop_fail_the_check(fakes, tm assert "no attempt in 3" in result.stderr -def test_a_stop_modal_refuses_moves_on_to_another_attempt(fakes, tmp_path): +def test_a_refused_stop_is_not_tried_again_and_the_job_is_followed(fakes, tmp_path): + """Modal refuses to stop a container that has ended. + + A job that then succeeds finished just before the stop, so a new one is + submitted. + """ + first = _household_attempt(0) + first.reads_before_stop = [ + *first.reads_before_stop, + _household(JOB_IDS[0], "running", CALL_IDS[0]), + _household(JOB_IDS[0], "succeeded", CALL_IDS[0]), + ] + result, state = _run( fakes, tmp_path, "household", - [_household_attempt(0), _household_attempt(1)], + [first, _household_attempt(1)], containers=ModalContainers(refused_stops=(CONTAINERS[0],)), ) assert result.returncode == 0, result.stderr assert len(_submissions(state)) == 2 assert _stops(fakes) == [CONTAINERS[0], CONTAINERS[1]] - assert f"could not stop container {CONTAINERS[0]}" in result.stderr + assert (state / f"reads-{JOB_IDS[0]}-before").read_text().strip() == "5" + assert f"refused to stop container {CONTAINERS[0]}" in result.stderr + + +def test_a_job_left_running_after_a_refused_stop_fails_at_its_deadline(fakes, tmp_path): + """A container that ended for another reason, such as running out of memory. + + Its job stays running, and the check doesn't submit another: it would + usually fail the same way. + """ + result, state = _run( + fakes, + tmp_path, + "household", + [_household_attempt(0), _household_attempt(1)], + containers=ModalContainers(refused_stops=(CONTAINERS[0],)), + PREEMPTION_CHECK_DEADLINE_SECONDS="2", + ) + + assert result.returncode == 1 + assert len(_submissions(state)) == 1 + assert _stops(fakes) == [CONTAINERS[0]] + assert "did not finish within 2 seconds" in result.stderr + assert f"refused to stop container {CONTAINERS[0]}" in result.stderr def test_a_call_that_finished_in_the_stopped_container_proves_nothing(fakes, tmp_path): @@ -571,6 +667,18 @@ def test_a_call_that_finished_in_the_stopped_container_proves_nothing(fakes, tmp assert CONTAINERS[0] in result.stderr +def test_a_rerun_recorded_on_the_same_input_counts(fakes, tmp_path): + """Modal may record the rerun on the stopped input itself, in place.""" + attempt = _household_attempt(0) + attempt.inputs_after_stop = [_inputs((RERUN_CONTAINER, "SUCCESS"))] + + result, _ = _run(fakes, tmp_path, "household", [attempt]) + + assert result.returncode == 0, result.stderr + assert attempt.inputs_before_stop[0][0]["input_id"] == "in-0" + assert RERUN_CONTAINER in result.stdout + + def test_the_call_graph_is_asked_again_until_it_shows_the_other_container( fakes, tmp_path ): @@ -732,7 +840,10 @@ def test_a_container_that_is_not_a_modal_container_id_is_never_stopped(fakes, tm def test_no_secret_reaches_a_command_line(fakes, tmp_path): - """The token is sent from a file, and the client secret on standard input.""" + """The token is sent from a file, and the client secret on standard input. + + No command, jq included, gets either as an argument. + """ result, _ = _run(fakes, tmp_path, "household", [_household_attempt(0)]) assert result.returncode == 0, result.stderr @@ -804,6 +915,7 @@ def test_the_report_check_needs_no_household(fakes, tmp_path): ("PREEMPTION_CHECK_DEADLINE_SECONDS", "ten"), ("PREEMPTION_CHECK_POLL_SECONDS", "-1"), ("PREEMPTION_CHECK_RERUN_SECONDS", "2m"), + ("PREEMPTION_CHECK_SCALEDOWN_SECONDS", "1.5"), ], ) def test_a_malformed_setting_fails_the_check_before_any_request( From 1a3387a1a11cc9641a827971d091d18a061e93bb Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 02:14:31 +0400 Subject: [PATCH 09/24] Fix review round 1: follow a job after a refused stop, wait out a warm container - A stop Modal refuses, as it does for a container that has ended, is not tried again and starts no new submission. The check follows that job to its end: one that succeeds had finished just before the stop and counts as a missed attempt; one left running by a container that ended for another reason, such as running out of memory, fails the check at its deadline, naming the refusal. - Before a new attempt, the check waits up to PREEMPTION_CHECK_SCALEDOWN_SECONDS (180) for Modal to stop listing the last attempt's container, so the next calculation starts in a new container instead of the warm one, where it would finish too fast to stop. - The token request takes the client's ID, secret and audience from jq's environment rather than its arguments, so no command line holds the secret. - The read routes' descriptions and docstrings name the Modal call they return, and the deployment notes say the check uses the repository's Modal token, which production's jobs share. Co-Authored-By: Claude Opus 5.5 --- .../simulation-staging-preemption-check.sh | 76 ++++++++++++++----- .../DEPLOYMENT.md | 26 ++++--- .../economy_routes.py | 8 +- .../household_routes.py | 7 +- .../tests/test_deployment_assets.py | 4 +- .../tests/test_staging_preemption_check.py | 23 +++++- 6 files changed, 106 insertions(+), 38 deletions(-) diff --git a/.github/scripts/simulation-staging-preemption-check.sh b/.github/scripts/simulation-staging-preemption-check.sh index 7fd9a8acb..8751cee36 100755 --- a/.github/scripts/simulation-staging-preemption-check.sh +++ b/.github/scripts/simulation-staging-preemption-check.sh @@ -21,8 +21,14 @@ # call and the call graph shows the call's input finished in another # container. A calculation that finishes before its container is stopped # proves nothing, so a new one is submitted, up to PREEMPTION_CHECK_ATTEMPTS -# times; a record that fails, or doesn't finish within the deadline, fails -# the check at once. +# times, once Modal no longer lists the last attempt's container: on that +# warm container the calculation would finish too fast to stop. A record +# that fails, or doesn't finish within the deadline, fails the check at +# once. A stop Modal refuses, as it does for a container that has ended, is +# not tried again: the check follows the record to its end, so a record +# left running by a container that ended for another reason, such as +# running out of memory, fails at its deadline rather than being submitted +# again. # # Required env vars: the test client's SIMULATION_TEST_AUTH_ISSUER, # SIMULATION_TEST_AUTH_AUDIENCE, SIMULATION_TEST_AUTH_CLIENT_ID and @@ -32,11 +38,13 @@ # in the staging Public API). # Optional: PREEMPTION_CHECK_ATTEMPTS (3); PREEMPTION_CHECK_DEADLINE_SECONDS, # each attempt's deadline (600 for household, 900 for report); -# PREEMPTION_CHECK_POLL_SECONDS (2); and PREEMPTION_CHECK_RERUN_SECONDS -# (120), how long after the record succeeds the call graph may take to show -# the call's other container. -# The access token is sent from a file and the client secret on standard -# input, so neither appears on a command line. A submission, which isn't +# PREEMPTION_CHECK_POLL_SECONDS (2); PREEMPTION_CHECK_RERUN_SECONDS (120), +# how long after the record succeeds the call graph may take to show the +# call's other container; and PREEMPTION_CHECK_SCALEDOWN_SECONDS (180), how +# long to wait for the last attempt's container to leave before a new +# attempt (Modal stops an idle container after 60 seconds by default). +# The access token is sent from a file, and the client secret on standard +# input through jq's environment, so neither appears on any command line. A submission, which isn't # asked again, may take 60 seconds, Cloud Run's own limit; every other # request 30. A read that gets no answer, a 429 or any 5xx is asked again # until the deadline. @@ -60,6 +68,7 @@ attempts="${PREEMPTION_CHECK_ATTEMPTS:-3}" deadline_seconds="${PREEMPTION_CHECK_DEADLINE_SECONDS:-${default_deadline_seconds}}" poll_seconds="${PREEMPTION_CHECK_POLL_SECONDS:-2}" rerun_seconds="${PREEMPTION_CHECK_RERUN_SECONDS:-120}" +scaledown_seconds="${PREEMPTION_CHECK_SCALEDOWN_SECONDS:-180}" request_timeout_seconds=30 submission_timeout_seconds=60 uuid_pattern='^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' @@ -85,7 +94,8 @@ fi || fail "PREEMPTION_CHECK_ATTEMPTS must be a whole number of at least 1" for setting in "PREEMPTION_CHECK_DEADLINE_SECONDS=${deadline_seconds}" \ "PREEMPTION_CHECK_POLL_SECONDS=${poll_seconds}" \ - "PREEMPTION_CHECK_RERUN_SECONDS=${rerun_seconds}"; do + "PREEMPTION_CHECK_RERUN_SECONDS=${rerun_seconds}" \ + "PREEMPTION_CHECK_SCALEDOWN_SECONDS=${scaledown_seconds}"; do [[ "${setting#*=}" =~ ^[0-9]+$ ]] \ || fail "${setting%%=*} must be a whole number of seconds" done @@ -105,11 +115,12 @@ send() { } token_status="$( - jq -n \ - --arg client_id "${SIMULATION_TEST_AUTH_CLIENT_ID}" \ - --arg client_secret "${SIMULATION_TEST_AUTH_CLIENT_SECRET}" \ - --arg audience "${SIMULATION_TEST_AUTH_AUDIENCE}" \ - '{client_id: $client_id, client_secret: $client_secret, audience: $audience, grant_type: "client_credentials"}' \ + jq -n '{ + client_id: env.SIMULATION_TEST_AUTH_CLIENT_ID, + client_secret: env.SIMULATION_TEST_AUTH_CLIENT_SECRET, + audience: env.SIMULATION_TEST_AUTH_AUDIENCE, + grant_type: "client_credentials" + }' \ | send token "${request_timeout_seconds}" --request POST --header 'Content-Type: application/json' \ --data @- "${SIMULATION_TEST_AUTH_ISSUER%/}/oauth/token" )" || fail "the token request got no answer" @@ -260,6 +271,25 @@ find_container() { fi } +# Wait until Modal no longer lists container , at most +# scaledown_seconds, so the next attempt's calculation starts in a new +# container rather than the warm one the last attempt left. +wait_for_container_to_leave() { + local leaving="$1" + local waiting_since="${SECONDS}" + [[ -n "${leaving}" ]] || return 0 + printf 'Before the next attempt, waiting for container %s to leave Modal environment %s\n' \ + "${leaving}" "${MODAL_ENVIRONMENT}" + while container_listed "${leaving}"; do + if ((SECONDS - waiting_since >= scaledown_seconds)); then + printf 'container %s is still listed after %d seconds; submitting again anyway\n' \ + "${leaving}" "${scaledown_seconds}" >&2 + return 0 + fi + sleep "${poll_seconds}" + done +} + # After the record succeeded: require the call graph to show the stopped # call's input finished in another container. Sets attempt_outcome. confirm_rerun() { @@ -298,6 +328,7 @@ attempt_once() { stopped_container_id="" stopped_call_id="" unlisted_note="" + stop_refusal="" last_read="nothing yet" local started_at="${SECONDS}" while true; do @@ -311,17 +342,17 @@ attempt_once() { succeeded) if [[ -z "${stopped_container_id}" ]]; then attempt_outcome="missed" - miss_reason="${record_label} finished before its container was stopped${unlisted_note:+ (${unlisted_note})}" + miss_reason="${record_label} finished before its container was stopped${unlisted_note:+ (${unlisted_note})}${stop_refusal:+ (${stop_refusal})}" else confirm_rerun fi return 0 ;; failed) - fail "${record_label} failed${stopped_container_id:+ after its container ${stopped_container_id} was stopped}: ${record_error}" + fail "${record_label} failed${stopped_container_id:+ after its container ${stopped_container_id} was stopped}${stop_refusal:+ after ${stop_refusal}}: ${record_error}" ;; pending | running) - if [[ -z "${stopped_container_id}" && -n "${record_call_id}" ]]; then + if [[ -z "${stopped_container_id}" && -z "${stop_refusal}" && -n "${record_call_id}" ]]; then [[ -n "${container_id}" ]] || find_container if [[ "${record_status}" == "running" && -n "${container_id}" ]]; then if uv run --frozen modal container stop --yes "${container_id}"; then @@ -330,9 +361,11 @@ attempt_once() { printf 'Stopped container %s while %s was running; waiting for Modal to run call %s again\n' \ "${stopped_container_id}" "${record_label}" "${stopped_call_id}" else - attempt_outcome="missed" - miss_reason="modal container stop could not stop container ${container_id} running ${record_label}" - return 0 + # Modal refuses to stop a container that has ended. The + # check doesn't stop it again or submit another calculation + # while this one may still be running. + stop_refusal="modal container stop refused to stop container ${container_id} while ${record_label} was running" + printf '%s; following it to its end\n' "${stop_refusal}" >&2 fi fi fi @@ -340,7 +373,7 @@ attempt_once() { *) fail "${record_label} has an unknown status: ${record_status}" ;; esac ((SECONDS - started_at < deadline_seconds)) \ - || fail "${record_label} did not finish within ${deadline_seconds} seconds (last read: ${last_read})" + || fail "${record_label} did not finish within ${deadline_seconds} seconds (last read: ${last_read})${stop_refusal:+; ${stop_refusal}}" sleep "${poll_seconds}" done } @@ -352,5 +385,8 @@ for ((attempt = 1; attempt <= attempts; attempt++)); do attempt_once [[ "${attempt_outcome}" != "proven" ]] || exit 0 printf 'Attempt %d of %d proved nothing: %s\n' "${attempt}" "${attempts}" "${miss_reason}" >&2 + if ((attempt < attempts)); then + wait_for_container_to_leave "${container_id}" + fi done fail "no attempt in ${attempts} stopped the container of a running ${kind} calculation; the last: ${miss_reason}" diff --git a/projects/policyengine-simulation-entry/DEPLOYMENT.md b/projects/policyengine-simulation-entry/DEPLOYMENT.md index 8076d6202..6c5ec63c1 100644 --- a/projects/policyengine-simulation-entry/DEPLOYMENT.md +++ b/projects/policyengine-simulation-entry/DEPLOYMENT.md @@ -170,15 +170,23 @@ workers again. A calculation that finishes before its container is stopped, for example on a warm container, proves nothing, so the check submits a new one, up to -three attempts. A record that fails, or doesn't finish within its deadline -(10 minutes for the household, 15 for the report, per attempt), fails the -check at once. - -The check refuses any `MODAL_ENVIRONMENT` but `staging`, and stops only a -container that `modal container list --env staging` lists. It uses the -`beta` environment's Modal token (`MODAL_TOKEN_ID`, `MODAL_TOKEN_SECRET`), -which must be allowed to read call graphs and to list and stop containers, -and the authenticated tests' client credentials. `deployment_ready` requires +three attempts. Before each new attempt it waits, up to three minutes, for +Modal to stop listing the last attempt's container (Modal stops an idle +container after 60 seconds), so the next calculation starts in a new +container. A record that fails, or doesn't finish within its deadline (10 +minutes for the household, 15 for the report, per attempt), fails the check +at once. Modal refuses to stop a container that has ended; the check then +doesn't stop it again or submit another calculation, but follows the record +to its end, so a record left running by a container that ended for another +reason, such as running out of memory, fails the check at its deadline. + +It uses the repository's Modal token (the repository secrets +`MODAL_TOKEN_ID` and `MODAL_TOKEN_SECRET`), which must be allowed to read +call graphs and to list and stop containers. Production's deploy jobs use +the same token, and `modal container stop` takes a container ID in any +environment, so the script refuses any `MODAL_ENVIRONMENT` but `staging` and +stops only a container that `modal container list --env staging` lists. It +also uses the authenticated tests' client credentials. `deployment_ready` requires the check to pass in staging, so a failure stops the production deployment. Each attempt adds a household job, or a report and its run, to the staging database. diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_routes.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_routes.py index abc207a79..c8a4c5d50 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_routes.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_routes.py @@ -9,8 +9,9 @@ function, answering 201 with the simulation's ID, its run's ID and status. - ``GET /reports/{report_id}`` and ``GET /simulations/{simulation_id}`` (``simulation:read``) return the record with its newest run's status, - versions, error message and times. The Public API reads results from the - tables itself; these reads let a deploy check follow a run. + versions, error message and times, and a report run's coordinator call. + The Public API reads results from the tables itself; these reads let a + deploy check follow a run. A refused request answers 400 (a row it names, or a function it needs, is missing; ``code`` says which) or 422 (its fields, or the stored values it @@ -528,7 +529,8 @@ def _read_routes( summary="Get a report's newest run", description=( "Return the report, its two simulations and its newest run's " - "status, versions, error message and times." + "status, versions, error message, times and coordinator's Modal " + "call (null until one is recorded)." ), operation_id="get_report", response_model=EconomyReportState, diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_routes.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_routes.py index 50aa46d84..b71dd15f3 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_routes.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_routes.py @@ -7,7 +7,8 @@ read: poll the job rather than submitting again. - ``GET /household-calculations/{household_job_id}`` (``simulation:read``) returns the job as it stands: its status, the household function's result - document once it has one, and its error message. + document once it has one, its error message, and the household function's + Modal call recorded on it. A refused submission answers 400 (a row it names, or the household function it needs, is missing) or 422 (its fields, or the stored data it names, cannot @@ -325,7 +326,9 @@ async def submit_household_calculation( summary="Get a household calculation", description=( "Return the household job as it stands: its status, the household " - "function's result document once it has one, and its error message." + "function's result document once it has one, its error message, " + "and the household function's Modal call recorded on it (null " + "until one is recorded)." ), operation_id="get_household_calculation", response_model=HouseholdCalculationState, diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py index 38c994635..3b0b61dc8 100644 --- a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py @@ -218,8 +218,8 @@ def _assert_deployment_environment_settings(reusable_workflow: str) -> None: # environment's custom-hostname variable, the staging household # check reads the beta variable SIMULATION_STAGING_HOUSEHOLD_ID and # the environment's auth secrets, the staging economy check reads - # those auth secrets too, and the staging preemption check reads them, - # that variable and the environment's Modal token. + # those auth secrets too, and the staging preemption check reads them + # and that variable. reusable_workflow.count("environment: ${{ inputs.release_environment }}") == 20 ) assert "APP_ENVIRONMENT: ${{ inputs.deployment_environment }}" in reusable_workflow diff --git a/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py index c7e2ce0d2..a0b2bf4d9 100644 --- a/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py +++ b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py @@ -36,6 +36,7 @@ if TYPE_CHECKING: from pathlib import Path + CHECK_SCRIPT = "simulation-staging-preemption-check.sh" READY_SCRIPT = "simulation-staging-preemption-ready.sh" CHECK_JOB = "staging_preemption_check" @@ -317,6 +318,10 @@ def _report_attempt(attempt: int) -> Attempt: ) +def _jq_arguments_log(commands: FakeCommands) -> Path: + return commands.log.with_name("jq-arguments.log") + + @dataclass(frozen=True) class ModalContainers: """The containers Modal lists in staging, and those it refuses to stop.""" @@ -330,9 +335,18 @@ class ModalContainers: @pytest.fixture def fakes(tmp_path) -> FakeCommands: commands = FakeCommands.create(tmp_path) + # jq records its arguments apart from the other calls, separated by NUL, + # since its programs span lines, then runs the real jq. real_jq = shutil.which("jq") assert real_jq is not None - commands.add("jq", f'exec {shlex.quote(real_jq)} "$@"') + fake_jq = commands.directory / "jq" + fake_jq.write_text( + "#!/usr/bin/env bash\n" + f"printf '%s\\0' \"$@\" >> {shlex.quote(str(_jq_arguments_log(commands)))}\n" + f'exec {shlex.quote(real_jq)} "$@"\n', + encoding="utf-8", + ) + fake_jq.chmod(0o755) commands.add("curl", FAKE_CURL) commands.add("uv", FAKE_UV) commands.add("sleep") @@ -847,7 +861,12 @@ def test_no_secret_reaches_a_command_line(fakes, tmp_path): result, _ = _run(fakes, tmp_path, "household", [_household_attempt(0)]) assert result.returncode == 0, result.stderr - arguments = [argument for call in fakes.calls() for argument in call] + jq_arguments = _jq_arguments_log(fakes).read_text(encoding="utf-8").split("\0") + arguments = [ + *(argument for call in fakes.calls() for argument in call), + *jq_arguments, + ] + assert len(jq_arguments) > 1 assert not any(ACCESS_TOKEN in argument for argument in arguments) assert not any(CLIENT_SECRET in argument for argument in arguments) From b85b914cf38c84adc3469bd15ec37a06e801b6a7 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 02:14:50 +0400 Subject: [PATCH 10/24] Test that the preemption job's time limit covers the waits between attempts (failing) Co-Authored-By: Claude Opus 5.5 --- .../tests/test_staging_preemption_check.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py index a0b2bf4d9..d3bdb0d2e 100644 --- a/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py +++ b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py @@ -1008,8 +1008,9 @@ def test_the_check_job_stops_a_household_and_a_report_through_the_candidate(): "staging_economy_check", } assert job["environment"] == "${{ inputs.release_environment }}" - # Longer than both steps' own deadlines together, so the script fails first. - assert job["timeout-minutes"] == 100 + # Longer than both steps' own deadlines together, with the waits before + # each new attempt, so the script fails first. + assert job["timeout-minutes"] == 120 assert {step.get("uses") for step in job["steps"] if "uses" in step} == { "actions/checkout@v6", "actions/setup-python@v6", From 68ae26b95f1c46fbe1c29397a23fde2906651d6b Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 02:15:05 +0400 Subject: [PATCH 11/24] Give the preemption job 120 minutes, past its waits between attempts Its two steps can take up to 99 minutes together: three attempts of 12 minutes for the household and of 17 for the report, each new attempt after a wait of up to 3 minutes for the last attempt's container to leave. The job's limit stays past that and the install, so the script fails first. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/simulation-deploy.reusable.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/simulation-deploy.reusable.yml b/.github/workflows/simulation-deploy.reusable.yml index 7eae4cac2..e2a69a467 100644 --- a/.github/workflows/simulation-deploy.reusable.yml +++ b/.github/workflows/simulation-deploy.reusable.yml @@ -1315,10 +1315,11 @@ jobs: }} needs: [deploy_entrypoint, authenticated_test, publish_stage12_v2_default, staging_economy_check] runs-on: ubuntu-latest - # Longer than both steps' own deadlines together (three attempts of 12 - # minutes for the household and of 17 for the report), so the script - # fails first. - timeout-minutes: 100 + # Longer than both steps' own deadlines together, so the script fails + # first: up to three attempts of 12 minutes for the household and of 17 + # for the report, each new attempt after a wait of up to 3 minutes (99 + # minutes), and the install. + timeout-minutes: 120 environment: ${{ inputs.release_environment }} steps: - uses: actions/checkout@v6 From d6e8e173cf28a2336272bf66c909304e5baa1e40 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 02:30:33 +0400 Subject: [PATCH 12/24] Test review round 2's fixes to the staging preemption check (failing) - A record that completed before its container was stopped proves nothing, even though Modal's rerun of the call then succeeds in another container: the rerun finds the record finished and returns its summary. The check compares the record's completion time with the stop, in each form the service writes, and refuses a succeeded record with no completion time. - Before a new attempt, the check also waits for a container it first saw only after it had finished the calculation. - Every Modal command runs with a 60-second time limit. - The final failure says no attempt showed the calculation finishing in another container, and a failed stop says it failed, whatever the cause. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_staging_preemption_check.py | 118 +++++++++++++++++- 1 file changed, 113 insertions(+), 5 deletions(-) diff --git a/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py index d3bdb0d2e..be18ab8d0 100644 --- a/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py +++ b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py @@ -50,6 +50,9 @@ CALL_IDS = [f"fc-01CALL{digit}" for digit in "123"] CONTAINERS = [f"ta-01CONTAINER{digit}" for digit in "123"] RERUN_CONTAINER = "ta-01RERUN" +# A succeeded record's completion time, before or after any test's stop. +BEFORE_THE_STOP = "2026-01-01T00:00:00Z" +AFTER_THE_STOP = "2099-01-01T00:00:00Z" REPORT_REQUEST = { "country": "uk", "report_type": "economy", @@ -191,7 +194,7 @@ def _household(job_id: str, status: str, call_id: str | None = None, **fields) - "error_message": None, "created_at": "2026-11-13T12:00:00Z", "started_at": None, - "completed_at": None, + "completed_at": AFTER_THE_STOP if status == "succeeded" else None, **fields, } return f"200 {json.dumps(body)}" @@ -212,7 +215,7 @@ def _report(attempt: int, status: str, call_id: str | None = None, **run_fields) "error_message": None, "created_at": "2026-11-13T12:00:00Z", "started_at": None, - "completed_at": None, + "completed_at": AFTER_THE_STOP if status == "succeeded" else None, "baseline_simulation_run_id": "00000000-0000-4000-8000-0000000000f2", "reform_simulation_run_id": "00000000-0000-4000-8000-0000000000f4", "modal_call_id": call_id, @@ -350,6 +353,8 @@ def fakes(tmp_path) -> FakeCommands: commands.add("curl", FAKE_CURL) commands.add("uv", FAKE_UV) commands.add("sleep") + # The fake timeout runs its command at once, without a time limit. + commands.add("timeout", 'shift\nexec "$@"') return commands @@ -583,6 +588,38 @@ def test_the_next_attempt_waits_for_the_last_attempts_container_to_leave( assert f"waiting for container {CONTAINERS[0]}" in result.stdout +def test_the_next_attempt_waits_for_a_container_first_seen_finished(fakes, tmp_path): + """A warm container finished the job before the check saw it running.""" + first = _household_attempt(0, proves=False) + first.reads_before_stop = [_household(JOB_IDS[0], "succeeded", CALL_IDS[0])] + first.inputs_before_stop = [_inputs((CONTAINERS[0], "SUCCESS"))] + + result, _ = _run( + fakes, + tmp_path, + "household", + [first, _household_attempt(1)], + containers=ModalContainers(departures={CONTAINERS[0]: 2}), + PREEMPTION_CHECK_SCALEDOWN_SECONDS="120", + ) + + assert result.returncode == 0, result.stderr + calls = fakes.calls() + listings = [ + index for index, call in enumerate(calls) if call[0] == "uv" and "list" in call + ] + submissions = [ + index + for index, call in enumerate(calls) + if call[0] == "curl" and call[-1].endswith("/household-calculations") + ] + # Two listings naming the first attempt's container and one without it, + # then the second attempt's lookup. + assert len(listings) == 4 + assert listings[2] < submissions[1] < listings[3] + assert f"waiting for container {CONTAINERS[0]}" in result.stdout + + def test_a_container_still_listed_after_the_wait_does_not_hold_the_check( fakes, tmp_path ): @@ -609,7 +646,7 @@ def test_three_calculations_that_finish_before_the_stop_fail_the_check(fakes, tm assert result.returncode == 1 assert len(_submissions(state)) == 3 assert _stops(fakes) == [] - assert "no attempt in 3" in result.stderr + assert "no attempt in 3 showed" in result.stderr def test_a_refused_stop_is_not_tried_again_and_the_job_is_followed(fakes, tmp_path): @@ -637,7 +674,9 @@ def test_a_refused_stop_is_not_tried_again_and_the_job_is_followed(fakes, tmp_pa assert len(_submissions(state)) == 2 assert _stops(fakes) == [CONTAINERS[0], CONTAINERS[1]] assert (state / f"reads-{JOB_IDS[0]}-before").read_text().strip() == "5" - assert f"refused to stop container {CONTAINERS[0]}" in result.stderr + assert f"modal container stop failed for container {CONTAINERS[0]}" in ( + result.stderr + ) def test_a_job_left_running_after_a_refused_stop_fails_at_its_deadline(fakes, tmp_path): @@ -659,7 +698,9 @@ def test_a_job_left_running_after_a_refused_stop_fails_at_its_deadline(fakes, tm assert len(_submissions(state)) == 1 assert _stops(fakes) == [CONTAINERS[0]] assert "did not finish within 2 seconds" in result.stderr - assert f"refused to stop container {CONTAINERS[0]}" in result.stderr + assert f"modal container stop failed for container {CONTAINERS[0]}" in ( + result.stderr + ) def test_a_call_that_finished_in_the_stopped_container_proves_nothing(fakes, tmp_path): @@ -693,6 +734,58 @@ def test_a_rerun_recorded_on_the_same_input_counts(fakes, tmp_path): assert RERUN_CONTAINER in result.stdout +def test_a_job_completed_before_its_container_was_stopped_proves_nothing( + fakes, tmp_path +): + """It finished between the running read and the stop. + + Modal runs the call again, but the rerun finds the job finished and + returns its summary, which Modal records as a success. + """ + attempt = _household_attempt(0) + attempt.reads_after_stop = [ + _household(JOB_IDS[0], "succeeded", CALL_IDS[0], completed_at=BEFORE_THE_STOP) + ] + + result, _ = _run( + fakes, tmp_path, "household", [attempt], PREEMPTION_CHECK_ATTEMPTS="1" + ) + + assert result.returncode == 1 + assert f"completed at {BEFORE_THE_STOP}" in result.stderr + assert f"before its container {CONTAINERS[0]} was stopped" in result.stderr + + +@pytest.mark.parametrize( + "completed_at", + [AFTER_THE_STOP, "2099-01-01T00:00:00.123456Z", "2099-01-01T00:00:00+00:00"], + ids=["whole-seconds", "fractional-seconds", "utc-offset"], +) +def test_a_completion_after_the_stop_is_read_in_each_form_the_service_writes( + fakes, tmp_path, completed_at +): + attempt = _report_attempt(0) + attempt.reads_after_stop = [ + _report(0, "succeeded", CALL_IDS[0], completed_at=completed_at) + ] + + result, _ = _run(fakes, tmp_path, "report", [attempt]) + + assert result.returncode == 0, result.stderr + + +def test_a_record_that_succeeded_without_a_completion_time_fails(fakes, tmp_path): + attempt = _household_attempt(0) + attempt.reads_after_stop = [ + _household(JOB_IDS[0], "succeeded", CALL_IDS[0], completed_at=None) + ] + + result, _ = _run(fakes, tmp_path, "household", [attempt]) + + assert result.returncode == 1 + assert "no completion time" in result.stderr + + def test_the_call_graph_is_asked_again_until_it_shows_the_other_container( fakes, tmp_path ): @@ -871,6 +964,21 @@ def test_no_secret_reaches_a_command_line(fakes, tmp_path): assert not any(CLIENT_SECRET in argument for argument in arguments) +def test_every_modal_command_has_a_time_limit(fakes, tmp_path): + """A Modal request that hangs fails its command instead of the job.""" + result, _ = _run( + fakes, + tmp_path, + "household", + [_household_attempt(0, proves=False), _household_attempt(1)], + ) + + assert result.returncode == 0, result.stderr + limited = [call[1:3] for call in fakes.calls("timeout")] + assert len(fakes.calls("uv")) > 3 + assert limited == [["60", "uv"]] * len(fakes.calls("uv")) + + def test_every_request_has_a_time_limit(fakes, tmp_path): """The submission, which isn't asked again, may take Cloud Run's full minute.""" result, _ = _run(fakes, tmp_path, "report", [_report_attempt(0)]) From 50724bc32fcd3311bbab6f862b126d6ceb8bf1a8 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 02:31:10 +0400 Subject: [PATCH 13/24] Test that the preemption job's time limit covers its Modal commands' limits (failing) Co-Authored-By: Claude Opus 5.5 --- .../tests/test_staging_preemption_check.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py index be18ab8d0..e981a9189 100644 --- a/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py +++ b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py @@ -1116,9 +1116,9 @@ def test_the_check_job_stops_a_household_and_a_report_through_the_candidate(): "staging_economy_check", } assert job["environment"] == "${{ inputs.release_environment }}" - # Longer than both steps' own deadlines together, with the waits before - # each new attempt, so the script fails first. - assert job["timeout-minutes"] == 120 + # Longer than both steps' worst case together, with every request and + # Modal command taking its full time limit, so the script fails first. + assert job["timeout-minutes"] == 150 assert {step.get("uses") for step in job["steps"] if "uses" in step} == { "actions/checkout@v6", "actions/setup-python@v6", From 165db2c7bd53f5acf6243830e05c30f43f38cd87 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 02:33:25 +0400 Subject: [PATCH 14/24] Fix review round 2: require completion after the stop, wait out every warm container - A stopped record must have completed after the stop: one completed before it proves nothing, because Modal's rerun of the call then finds it finished and only returns its summary, which Modal records as a success in another container. The check compares the record's completed_at (with or without fractional seconds, Z or +00:00) with the time the stop returned, and refuses a succeeded record with no completion time. - When an attempt is missed before the check found the call's container, it asks the call graph which container ran the call, so the next attempt waits for that warm container to leave too. - Every Modal command (the call-input helper, the container listing and the stop) runs with a 60-second limit, and the job's time limit is 150 minutes, past the steps' worst case of about 136. - A failed stop is reported as failed with its exit status, whatever the cause, and the final failure says no attempt showed the calculation finishing in another container. - The call-input helper's records are a TypedDict naming the keys the check reads. Co-Authored-By: Claude Opus 5.5 --- .../simulation-staging-preemption-check.sh | 90 +++++++++++++++---- .../workflows/simulation-deploy.reusable.yml | 11 +-- .../DEPLOYMENT.md | 20 +++-- .../src/modal/utils/describe_call_inputs.py | 24 +++-- 4 files changed, 105 insertions(+), 40 deletions(-) diff --git a/.github/scripts/simulation-staging-preemption-check.sh b/.github/scripts/simulation-staging-preemption-check.sh index 8751cee36..dd3eb3c24 100755 --- a/.github/scripts/simulation-staging-preemption-check.sh +++ b/.github/scripts/simulation-staging-preemption-check.sh @@ -18,17 +18,19 @@ # environment. `modal container stop` sends it SIGINT, and Modal gives the # call's input to another container under the same call ID, as it does # after a pre-emption. The check passes once the record succeeds under that -# call and the call graph shows the call's input finished in another -# container. A calculation that finishes before its container is stopped +# call, completed after the stop, and the call graph shows the call's input +# finished in another container. A record completed before the stop proves +# nothing: Modal's rerun then finds it finished and only returns its +# summary. A calculation that finishes before its container is stopped # proves nothing, so a new one is submitted, up to PREEMPTION_CHECK_ATTEMPTS # times, once Modal no longer lists the last attempt's container: on that # warm container the calculation would finish too fast to stop. A record # that fails, or doesn't finish within the deadline, fails the check at -# once. A stop Modal refuses, as it does for a container that has ended, is -# not tried again: the check follows the record to its end, so a record +# once. A stop that fails, as Modal's does for a container that has ended, +# is not tried again: the check follows the record to its end, so a record # left running by a container that ended for another reason, such as # running out of memory, fails at its deadline rather than being submitted -# again. +# again. Every Modal command may take 60 seconds. # # Required env vars: the test client's SIMULATION_TEST_AUTH_ISSUER, # SIMULATION_TEST_AUTH_AUDIENCE, SIMULATION_TEST_AUTH_CLIENT_ID and @@ -71,6 +73,7 @@ rerun_seconds="${PREEMPTION_CHECK_RERUN_SECONDS:-120}" scaledown_seconds="${PREEMPTION_CHECK_SCALEDOWN_SECONDS:-180}" request_timeout_seconds=30 submission_timeout_seconds=60 +modal_command_timeout_seconds=60 uuid_pattern='^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' call_id_pattern='^fc-[0-9A-Za-z]+$' container_id_pattern='^ta-[0-9A-Za-z]+$' @@ -130,17 +133,23 @@ access_token="$(jq -r '.access_token // empty' "${work_directory}/token.json")" authorization_header="${work_directory}/authorization-header" printf 'Authorization: Bearer %s\n' "${access_token}" > "${authorization_header}" +# Run a command in the executor's environment, which has the Modal client, +# allowing it modal_command_timeout_seconds. +modal_command() { + timeout "${modal_command_timeout_seconds}" uv run --frozen "$@" +} + # Print the inputs of Modal call as JSON; fail when Modal can't be # asked (src/modal/utils/describe_call_inputs.py). call_inputs() { - uv run --frozen python -m src.modal.utils.describe_call_inputs "$1" + modal_command python -m src.modal.utils.describe_call_inputs "$1" } # Succeed when Modal lists container as running in # MODAL_ENVIRONMENT. container_listed() { local listing - listing="$(uv run --frozen modal container list --env "${MODAL_ENVIRONMENT}" --json)" \ + listing="$(modal_command modal container list --env "${MODAL_ENVIRONMENT}" --json)" \ || return 1 jq -e --arg container_id "$1" 'any(.[]; .["Container ID"] == $container_id)' \ <<<"${listing}" >/dev/null @@ -201,8 +210,9 @@ submit() { fi } -# Read the record once. Sets record_status, record_call_id, record_error and -# last_read; leaves record_status empty when the service can't answer now. +# Read the record once. Sets record_status, record_call_id, record_error, +# record_completed_at and last_read; leaves record_status empty when the +# service can't answer now. read_record() { local record="${work_directory}/record.json" local read_status @@ -234,6 +244,7 @@ read_record() { record_status="$(jq -r "${record_filter}.status" "${record}")" record_call_id="$(jq -r "${record_filter}.modal_call_id // empty" "${record}")" record_error="$(jq -r "${record_filter}.error_message // \"no error message\"" "${record}")" + record_completed_at="$(jq -r "${record_filter}.completed_at // empty" "${record}")" if [[ -n "${record_call_id}" && ! "${record_call_id}" =~ ${call_id_pattern} ]]; then fail "${record_label} names Modal call '${record_call_id}', which is not a Modal call ID" fi @@ -271,6 +282,42 @@ find_container() { fi } +# When container_id is empty, set it to a container the call graph says ran +# an input of the record's call, so the next attempt can wait for it to +# leave: a warm container may finish the calculation before the check sees +# it running. +remember_finished_container() { + local inputs candidate + if [[ -n "${container_id}" || -z "${record_call_id}" ]]; then + return 0 + fi + inputs="$(call_inputs "${record_call_id}")" || return 0 + candidate="$( + jq -r '[.[] | select(.container_id != "")] | first | .container_id // empty' <<<"${inputs}" + )" || return 0 + if [[ "${candidate}" =~ ${container_id_pattern} ]]; then + container_id="${candidate}" + fi +} + +# Return 0 when the record completed after its container was stopped, so +# its success comes from the rerun; otherwise set attempt_outcome to missed. +completed_after_the_stop() { + local completed_epoch + [[ -n "${record_completed_at}" ]] \ + || fail "${record_label} succeeded with no completion time" + completed_epoch="$( + jq -rn --arg completed_at "${record_completed_at}" \ + '$completed_at | sub("\\.[0-9]+"; "") | sub("\\+00:00$"; "Z") | fromdateiso8601' + )" || fail "${record_label}'s completion time ${record_completed_at} is not a UTC time" + if ((completed_epoch > stopped_at_epoch)); then + return 0 + fi + attempt_outcome="missed" + miss_reason="${record_label} completed at ${record_completed_at}, before its container ${stopped_container_id} was stopped, so Modal's rerun of call ${stopped_call_id} found it finished" + return 1 +} + # Wait until Modal no longer lists container , at most # scaledown_seconds, so the next attempt's calculation starts in a new # container rather than the warm one the last attempt left. @@ -327,8 +374,9 @@ attempt_once() { container_id="" stopped_container_id="" stopped_call_id="" + stopped_at_epoch=0 unlisted_note="" - stop_refusal="" + stop_failure="" last_read="nothing yet" local started_at="${SECONDS}" while true; do @@ -342,20 +390,24 @@ attempt_once() { succeeded) if [[ -z "${stopped_container_id}" ]]; then attempt_outcome="missed" - miss_reason="${record_label} finished before its container was stopped${unlisted_note:+ (${unlisted_note})}${stop_refusal:+ (${stop_refusal})}" - else + miss_reason="${record_label} finished before its container was stopped${unlisted_note:+ (${unlisted_note})}${stop_failure:+ (${stop_failure})}" + remember_finished_container + elif completed_after_the_stop; then confirm_rerun fi return 0 ;; failed) - fail "${record_label} failed${stopped_container_id:+ after its container ${stopped_container_id} was stopped}${stop_refusal:+ after ${stop_refusal}}: ${record_error}" + fail "${record_label} failed${stopped_container_id:+ after its container ${stopped_container_id} was stopped}${stop_failure:+ after ${stop_failure}}: ${record_error}" ;; pending | running) - if [[ -z "${stopped_container_id}" && -z "${stop_refusal}" && -n "${record_call_id}" ]]; then + if [[ -z "${stopped_container_id}" && -z "${stop_failure}" && -n "${record_call_id}" ]]; then [[ -n "${container_id}" ]] || find_container if [[ "${record_status}" == "running" && -n "${container_id}" ]]; then - if uv run --frozen modal container stop --yes "${container_id}"; then + local stop_status=0 + modal_command modal container stop --yes "${container_id}" || stop_status=$? + if ((stop_status == 0)); then + stopped_at_epoch="$(date -u +%s)" stopped_container_id="${container_id}" stopped_call_id="${record_call_id}" printf 'Stopped container %s while %s was running; waiting for Modal to run call %s again\n' \ @@ -364,8 +416,8 @@ attempt_once() { # Modal refuses to stop a container that has ended. The # check doesn't stop it again or submit another calculation # while this one may still be running. - stop_refusal="modal container stop refused to stop container ${container_id} while ${record_label} was running" - printf '%s; following it to its end\n' "${stop_refusal}" >&2 + stop_failure="modal container stop failed for container ${container_id} (exit status ${stop_status}) while ${record_label} was running" + printf '%s; following it to its end\n' "${stop_failure}" >&2 fi fi fi @@ -373,7 +425,7 @@ attempt_once() { *) fail "${record_label} has an unknown status: ${record_status}" ;; esac ((SECONDS - started_at < deadline_seconds)) \ - || fail "${record_label} did not finish within ${deadline_seconds} seconds (last read: ${last_read})${stop_refusal:+; ${stop_refusal}}" + || fail "${record_label} did not finish within ${deadline_seconds} seconds (last read: ${last_read})${stop_failure:+; ${stop_failure}}" sleep "${poll_seconds}" done } @@ -389,4 +441,4 @@ for ((attempt = 1; attempt <= attempts; attempt++)); do wait_for_container_to_leave "${container_id}" fi done -fail "no attempt in ${attempts} stopped the container of a running ${kind} calculation; the last: ${miss_reason}" +fail "no attempt in ${attempts} showed a ${kind} calculation finishing in another container after its container was stopped; the last: ${miss_reason}" diff --git a/.github/workflows/simulation-deploy.reusable.yml b/.github/workflows/simulation-deploy.reusable.yml index e2a69a467..318929cef 100644 --- a/.github/workflows/simulation-deploy.reusable.yml +++ b/.github/workflows/simulation-deploy.reusable.yml @@ -1315,11 +1315,12 @@ jobs: }} needs: [deploy_entrypoint, authenticated_test, publish_stage12_v2_default, staging_economy_check] runs-on: ubuntu-latest - # Longer than both steps' own deadlines together, so the script fails - # first: up to three attempts of 12 minutes for the household and of 17 - # for the report, each new attempt after a wait of up to 3 minutes (99 - # minutes), and the install. - timeout-minutes: 120 + # Longer than both steps' worst case together, so the script fails + # first: about 136 minutes, with every request and Modal command taking + # its full time limit (three attempts of up to 17.5 minutes for the + # household and 22.5 for the report, and up to 4 minutes' wait before + # each new attempt), and the install. + timeout-minutes: 150 environment: ${{ inputs.release_environment }} steps: - uses: actions/checkout@v6 diff --git a/projects/policyengine-simulation-entry/DEPLOYMENT.md b/projects/policyengine-simulation-entry/DEPLOYMENT.md index 6c5ec63c1..2ff92cb64 100644 --- a/projects/policyengine-simulation-entry/DEPLOYMENT.md +++ b/projects/policyengine-simulation-entry/DEPLOYMENT.md @@ -162,8 +162,10 @@ runs that call (`src/modal/utils/describe_call_inputs.py`), and stops that container with `modal container stop`. Modal sends the container SIGINT and gives the input to another container under the same call ID, as it does after a pre-emption. The check requires the record to succeed under that -call, and the call graph to show the call's input finished in another -container. Run again, the household function or coordinator resumes its +call, with a completion time after the stop, and the call graph to show the +call's input finished in another container. A record that completed before +the stop proves nothing: Modal's rerun finds it finished and only returns +its summary, so that attempt counts as missed. Run again, the household function or coordinator resumes its claim. The report's simulation runs are reused, so its coordinator starts no worker, and the check doesn't cover a coordinator starting its workers again. @@ -173,12 +175,14 @@ a warm container, proves nothing, so the check submits a new one, up to three attempts. Before each new attempt it waits, up to three minutes, for Modal to stop listing the last attempt's container (Modal stops an idle container after 60 seconds), so the next calculation starts in a new -container. A record that fails, or doesn't finish within its deadline (10 -minutes for the household, 15 for the report, per attempt), fails the check -at once. Modal refuses to stop a container that has ended; the check then -doesn't stop it again or submit another calculation, but follows the record -to its end, so a record left running by a container that ended for another -reason, such as running out of memory, fails the check at its deadline. +container, including a container it first saw only after it had finished +the calculation. A record that fails, or doesn't finish within its deadline +(10 minutes for the household, 15 for the report, per attempt), fails the +check at once. Modal refuses to stop a container that has ended; when a +stop fails, the check doesn't stop it again or submit another calculation, +but follows the record to its end, so a record left running by a container +that ended for another reason, such as running out of memory, fails the +check at its deadline. Every Modal command may take 60 seconds. It uses the repository's Modal token (the repository secrets `MODAL_TOKEN_ID` and `MODAL_TOKEN_SECRET`), which must be allowed to read diff --git a/projects/policyengine-simulation-executor/src/modal/utils/describe_call_inputs.py b/projects/policyengine-simulation-executor/src/modal/utils/describe_call_inputs.py index 7292e74ac..2a5935b26 100644 --- a/projects/policyengine-simulation-executor/src/modal/utils/describe_call_inputs.py +++ b/projects/policyengine-simulation-executor/src/modal/utils/describe_call_inputs.py @@ -27,14 +27,22 @@ import json import sys from importlib import import_module -from typing import Any +from typing import Any, TypedDict + + +class CallInput(TypedDict): + """One input of a call, as printed: the shell check reads these keys.""" + + input_id: str + container_id: str + status: str def call_inputs( modal_call_id: str, *, modal_module: Any = None, # noqa: ANN401 # the modal module, or a stand-in for it -) -> list[dict[str, str]]: +) -> list[CallInput]: """Return each input of call ``modal_call_id``, in the graph's order. The graph is walked depth first, each input before the calls spawned @@ -42,17 +50,17 @@ def call_inputs( """ modal = modal_module if modal_module is not None else import_module("modal") graph = modal.FunctionCall.from_id(modal_call_id).get_call_graph() - found: list[dict[str, str]] = [] + found: list[CallInput] = [] waiting = list(reversed(graph)) while waiting: input_info = waiting.pop() if input_info.function_call_id == modal_call_id: found.append( - { - "input_id": input_info.input_id, - "container_id": input_info.task_id, - "status": input_info.status.name, - } + CallInput( + input_id=input_info.input_id, + container_id=input_info.task_id, + status=input_info.status.name, + ) ) waiting.extend(reversed(input_info.children)) return found From af18a1a88dccbc9823ee387a7144392c3ab3d81d Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 02:57:06 +0400 Subject: [PATCH 15/24] Test review round 3's quality fixes to the staging preemption check (failing) - A record completed within 5 seconds of the stop is taken for the stopped container's own work, since that container may write the success before the signal reaches it; a rerun in a new container can't finish that soon. A fake date fixes the stop's time. - A succeeded record that names no call after the stop fails. - A container listing Modal doesn't answer is asked again: it neither marks the container as unlisted nor ends the wait for it to leave. - A stop that runs out of time is followed like a failed one (this passes already: both are a non-zero exit). - The refused-stop deadline test gets 6 seconds, enough to reach the stop with the extra fakes. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_staging_preemption_check.py | 136 +++++++++++++++++- 1 file changed, 134 insertions(+), 2 deletions(-) diff --git a/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py index e981a9189..a50b7a05a 100644 --- a/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py +++ b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py @@ -50,6 +50,8 @@ CALL_IDS = [f"fc-01CALL{digit}" for digit in "123"] CONTAINERS = [f"ta-01CONTAINER{digit}" for digit in "123"] RERUN_CONTAINER = "ta-01RERUN" +# The fake date says every stop returned at 2026-11-13T12:00:00Z. +STOP_EPOCH = 1794571200 # A succeeded record's completion time, before or after any test's stop. BEFORE_THE_STOP = "2026-01-01T00:00:00Z" AFTER_THE_STOP = "2099-01-01T00:00:00Z" @@ -157,6 +159,11 @@ counter="${FAKE_STATE}/listings" listing="$(($(cat "${counter}" 2>/dev/null || echo 0) + 1))" echo "${listing}" > "${counter}" + if jq -e --argjson listing "${listing}" '.failed_listings | index($listing)' \ + "${spec}" >/dev/null; then + printf 'Error: the Modal server did not answer\n' >&2 + exit 1 + fi # A container with a departure count is listed only that many times. jq --argjson listing "${listing}" \ '[.listed[] as $id | select((.departures[$id] // $listing) >= $listing) @@ -168,6 +175,11 @@ printf "Container '%s' is already stopped.\n" "${last}" >&2 exit 1 fi + if jq -e --arg id "${last}" '.timed_out_stops | index($id)' "${spec}" \ + >/dev/null; then + # GNU timeout's exit status when the command runs out of time. + exit 124 + fi touch "${FAKE_STATE}/stopped-${last}" ;; *) @@ -331,6 +343,9 @@ class ModalContainers: listed: tuple[str, ...] = (*CONTAINERS, RERUN_CONTAINER) refused_stops: tuple[str, ...] = () + timed_out_stops: tuple[str, ...] = () + # Which listings, counted from 1, fail as if Modal didn't answer. + failed_listings: tuple[int, ...] = () # How many listings name each container before Modal scales it down. departures: dict[str, int] = field(default_factory=dict) @@ -355,6 +370,16 @@ def fakes(tmp_path) -> FakeCommands: commands.add("sleep") # The fake timeout runs its command at once, without a time limit. commands.add("timeout", 'shift\nexec "$@"') + commands.add( + "date", + """ + if [[ "$*" != "-u +%s" ]]; then + printf 'unexpected date arguments: %s\\n' "$*" >&2 + exit 9 + fi + echo "${FAKE_STOP_EPOCH}" + """, + ) return commands @@ -398,6 +423,8 @@ def _run( }, "listed": list(containers.listed), "refused_stops": list(containers.refused_stops), + "timed_out_stops": list(containers.timed_out_stops), + "failed_listings": list(containers.failed_listings), "departures": containers.departures, } (state / "spec.json").write_text(json.dumps(spec), encoding="utf-8") @@ -418,6 +445,7 @@ def _run( "FAKE_STATE": str(state), "FAKE_BASE_URL": BASE_URL, "FAKE_ACCESS_TOKEN": ACCESS_TOKEN, + "FAKE_STOP_EPOCH": str(STOP_EPOCH), **overrides, } result = run_script( @@ -620,6 +648,73 @@ def test_the_next_attempt_waits_for_a_container_first_seen_finished(fakes, tmp_p assert f"waiting for container {CONTAINERS[0]}" in result.stdout +def test_a_failed_listing_is_asked_again_rather_than_taken_for_an_absent_container( + fakes, tmp_path +): + result, _ = _run( + fakes, + tmp_path, + "household", + [_household_attempt(0)], + containers=ModalContainers(failed_listings=(1,)), + ) + + assert result.returncode == 0, result.stderr + assert _stops(fakes) == [CONTAINERS[0]] + assert "Modal could not list" in result.stderr + assert "is not listed" not in result.stderr + + +def test_the_wait_asks_again_after_a_failed_listing(fakes, tmp_path): + """A listing Modal doesn't answer doesn't mean the container has left.""" + result, _ = _run( + fakes, + tmp_path, + "household", + [_household_attempt(0, proves=False), _household_attempt(1)], + containers=ModalContainers(departures={CONTAINERS[0]: 3}, failed_listings=(2,)), + PREEMPTION_CHECK_SCALEDOWN_SECONDS="120", + ) + + assert result.returncode == 0, result.stderr + calls = fakes.calls() + listings = [ + index for index, call in enumerate(calls) if call[0] == "uv" and "list" in call + ] + submissions = [ + index + for index, call in enumerate(calls) + if call[0] == "curl" and call[-1].endswith("/household-calculations") + ] + # The first attempt's lookup; then, waiting, a failed listing, one naming + # the container and one without it; then the second attempt's lookup. + assert len(listings) == 5 + assert listings[3] < submissions[1] < listings[4] + assert "Modal could not list" in result.stderr + + +def test_a_stop_that_runs_out_of_time_is_followed_like_a_failed_one(fakes, tmp_path): + """Modal may have accepted the stop, so the check doesn't stop it again.""" + first = _household_attempt(0) + first.reads_before_stop = [ + *first.reads_before_stop, + _household(JOB_IDS[0], "succeeded", CALL_IDS[0]), + ] + + result, state = _run( + fakes, + tmp_path, + "household", + [first, _household_attempt(1)], + containers=ModalContainers(timed_out_stops=(CONTAINERS[0],)), + ) + + assert result.returncode == 0, result.stderr + assert len(_submissions(state)) == 2 + assert _stops(fakes) == [CONTAINERS[0], CONTAINERS[1]] + assert "exit status 124" in result.stderr + + def test_a_container_still_listed_after_the_wait_does_not_hold_the_check( fakes, tmp_path ): @@ -691,13 +786,14 @@ def test_a_job_left_running_after_a_refused_stop_fails_at_its_deadline(fakes, tm "household", [_household_attempt(0), _household_attempt(1)], containers=ModalContainers(refused_stops=(CONTAINERS[0],)), - PREEMPTION_CHECK_DEADLINE_SECONDS="2", + # Long enough to reach the stop first. + PREEMPTION_CHECK_DEADLINE_SECONDS="6", ) assert result.returncode == 1 assert len(_submissions(state)) == 1 assert _stops(fakes) == [CONTAINERS[0]] - assert "did not finish within 2 seconds" in result.stderr + assert "did not finish within 6 seconds" in result.stderr assert f"modal container stop failed for container {CONTAINERS[0]}" in ( result.stderr ) @@ -756,6 +852,42 @@ def test_a_job_completed_before_its_container_was_stopped_proves_nothing( assert f"before its container {CONTAINERS[0]} was stopped" in result.stderr +@pytest.mark.parametrize( + ("completed_at", "proves"), + [("2026-11-13T12:00:04Z", False), ("2026-11-13T12:00:05Z", True)], + ids=["four-seconds-after", "five-seconds-after"], +) +def test_a_completion_moments_after_the_stop_is_taken_for_the_stopped_containers( + fakes, tmp_path, completed_at, proves +): + """The stopped container may write the success before the signal reaches it. + + A rerun in a new container can't finish within 5 seconds of the stop. + """ + attempt = _household_attempt(0) + attempt.reads_after_stop = [ + _household(JOB_IDS[0], "succeeded", CALL_IDS[0], completed_at=completed_at) + ] + + result, _ = _run( + fakes, tmp_path, "household", [attempt], PREEMPTION_CHECK_ATTEMPTS="1" + ) + + assert result.returncode == (0 if proves else 1), result.stderr + assert ("too soon" in result.stderr) is not proves + + +def test_a_record_that_names_no_call_after_the_stop_fails(fakes, tmp_path): + attempt = _household_attempt(0) + attempt.reads_after_stop = [_household(JOB_IDS[0], "succeeded", None)] + + result, _ = _run(fakes, tmp_path, "household", [attempt]) + + assert result.returncode == 1 + assert "names no call" in result.stderr + assert CALL_IDS[0] in result.stderr + + @pytest.mark.parametrize( "completed_at", [AFTER_THE_STOP, "2099-01-01T00:00:00.123456Z", "2099-01-01T00:00:00+00:00"], From e53b3be23f003fda60a42498f967ce17707c0bc3 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 02:59:56 +0400 Subject: [PATCH 16/24] Fix review round 3's findings: a completion margin, unanswered listings, a lost call - A stopped record must complete at least 5 seconds after the stop. The stopped container may write the success before the signal reaches it, and Modal's rerun then only returns the record's summary; a rerun in a new container can't finish that soon, so a sooner completion counts as a missed attempt. - A container listing Modal doesn't answer, or answers unreadably, is asked again: it no longer marks the container as unlisted, and it no longer ends the wait for the last attempt's container to leave. - After the stop, a read the service answered must name the stopped call; one naming no call fails the check like one naming another call. Co-Authored-By: Claude Opus 5.5 --- .../simulation-staging-preemption-check.sh | 93 +++++++++++++------ .../DEPLOYMENT.md | 10 +- 2 files changed, 73 insertions(+), 30 deletions(-) diff --git a/.github/scripts/simulation-staging-preemption-check.sh b/.github/scripts/simulation-staging-preemption-check.sh index dd3eb3c24..41786a0ca 100755 --- a/.github/scripts/simulation-staging-preemption-check.sh +++ b/.github/scripts/simulation-staging-preemption-check.sh @@ -18,10 +18,12 @@ # environment. `modal container stop` sends it SIGINT, and Modal gives the # call's input to another container under the same call ID, as it does # after a pre-emption. The check passes once the record succeeds under that -# call, completed after the stop, and the call graph shows the call's input -# finished in another container. A record completed before the stop proves -# nothing: Modal's rerun then finds it finished and only returns its -# summary. A calculation that finishes before its container is stopped +# call, completed at least 5 seconds after the stop, and the call graph +# shows the call's input finished in another container. A record completed +# before then proves nothing: the stopped container may have written the +# success before the signal reached it, and Modal's rerun then finds the +# record finished and only returns its summary. A rerun in a new container +# can't finish within 5 seconds of the stop. A calculation that finishes before its container is stopped # proves nothing, so a new one is submitted, up to PREEMPTION_CHECK_ATTEMPTS # times, once Modal no longer lists the last attempt's container: on that # warm container the calculation would finish too fast to stop. A record @@ -74,6 +76,9 @@ scaledown_seconds="${PREEMPTION_CHECK_SCALEDOWN_SECONDS:-180}" request_timeout_seconds=30 submission_timeout_seconds=60 modal_command_timeout_seconds=60 +# A record completed sooner after the stop was completed by the stopped +# container, not by Modal's rerun in a new one. +rerun_minimum_seconds=5 uuid_pattern='^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' call_id_pattern='^fc-[0-9A-Za-z]+$' container_id_pattern='^ta-[0-9A-Za-z]+$' @@ -145,14 +150,27 @@ call_inputs() { modal_command python -m src.modal.utils.describe_call_inputs "$1" } -# Succeed when Modal lists container as running in -# MODAL_ENVIRONMENT. +# Return 0 when Modal lists container as running in +# MODAL_ENVIRONMENT, 1 when it doesn't, and 2 when Modal couldn't list the +# environment's containers, which says nothing about the container. container_listed() { local listing - listing="$(modal_command modal container list --env "${MODAL_ENVIRONMENT}" --json)" \ - || return 1 + local search_status=0 + if ! listing="$(modal_command modal container list --env "${MODAL_ENVIRONMENT}" --json)"; then + printf 'Modal could not list the containers in environment %s; asking again\n' \ + "${MODAL_ENVIRONMENT}" >&2 + return 2 + fi jq -e --arg container_id "$1" 'any(.[]; .["Container ID"] == $container_id)' \ - <<<"${listing}" >/dev/null + <<<"${listing}" >/dev/null || search_status=$? + case "${search_status}" in + 0 | 1) return "${search_status}" ;; + *) + printf 'Modal could not list the containers in environment %s readably; asking again\n' \ + "${MODAL_ENVIRONMENT}" >&2 + return 2 + ;; + esac } # Create the attempt's record. Sets record_url, record_noun, record_label @@ -273,13 +291,20 @@ find_container() { [[ -n "${candidate}" ]] || return 0 [[ "${candidate}" =~ ${container_id_pattern} ]] \ || fail "Modal names '${candidate}' as call ${record_call_id}'s container, which is not a Modal container ID" - if container_listed "${candidate}"; then - container_id="${candidate}" - printf 'Call %s of %s runs in container %s\n' "${record_call_id}" "${record_label}" "${container_id}" - else - unlisted_note="container ${candidate} is not listed in Modal environment ${MODAL_ENVIRONMENT}" - printf '%s; asking again\n' "${unlisted_note}" >&2 - fi + local listed_status=0 + container_listed "${candidate}" || listed_status=$? + case "${listed_status}" in + 0) + container_id="${candidate}" + printf 'Call %s of %s runs in container %s\n' "${record_call_id}" "${record_label}" "${container_id}" + ;; + 1) + unlisted_note="container ${candidate} is not listed in Modal environment ${MODAL_ENVIRONMENT}" + printf '%s; asking again\n' "${unlisted_note}" >&2 + ;; + # Modal couldn't list: the next read asks again. + *) ;; + esac } # When container_id is empty, set it to a container the call graph says ran @@ -310,12 +335,17 @@ completed_after_the_stop() { jq -rn --arg completed_at "${record_completed_at}" \ '$completed_at | sub("\\.[0-9]+"; "") | sub("\\+00:00$"; "Z") | fromdateiso8601' )" || fail "${record_label}'s completion time ${record_completed_at} is not a UTC time" - if ((completed_epoch > stopped_at_epoch)); then - return 0 + if ((completed_epoch <= stopped_at_epoch)); then + attempt_outcome="missed" + miss_reason="${record_label} completed at ${record_completed_at}, before its container ${stopped_container_id} was stopped, so Modal's rerun of call ${stopped_call_id} found it finished" + return 1 + fi + if ((completed_epoch - stopped_at_epoch < rerun_minimum_seconds)); then + attempt_outcome="missed" + miss_reason="${record_label} completed at ${record_completed_at}, $((completed_epoch - stopped_at_epoch)) seconds after its container ${stopped_container_id} was stopped: too soon for Modal to have run call ${stopped_call_id} again in a new container, so the stopped container may have finished it before the signal reached it" + return 1 fi - attempt_outcome="missed" - miss_reason="${record_label} completed at ${record_completed_at}, before its container ${stopped_container_id} was stopped, so Modal's rerun of call ${stopped_call_id} found it finished" - return 1 + return 0 } # Wait until Modal no longer lists container , at most @@ -327,10 +357,19 @@ wait_for_container_to_leave() { [[ -n "${leaving}" ]] || return 0 printf 'Before the next attempt, waiting for container %s to leave Modal environment %s\n' \ "${leaving}" "${MODAL_ENVIRONMENT}" - while container_listed "${leaving}"; do + local listed_status + while true; do + listed_status=0 + container_listed "${leaving}" || listed_status=$? + ((listed_status != 1)) || return 0 if ((SECONDS - waiting_since >= scaledown_seconds)); then - printf 'container %s is still listed after %d seconds; submitting again anyway\n' \ - "${leaving}" "${scaledown_seconds}" >&2 + if ((listed_status == 0)); then + printf 'container %s is still listed after %d seconds; submitting again anyway\n' \ + "${leaving}" "${scaledown_seconds}" >&2 + else + printf 'Modal could not say within %d seconds whether container %s has left; submitting again anyway\n' \ + "${scaledown_seconds}" "${leaving}" >&2 + fi return 0 fi sleep "${poll_seconds}" @@ -381,9 +420,11 @@ attempt_once() { local started_at="${SECONDS}" while true; do read_record - if [[ -n "${stopped_call_id}" && -n "${record_call_id}" \ + # A read the service answered (record_status set) must name the stopped + # call: Modal runs it again under the same ID. + if [[ -n "${record_status}" && -n "${stopped_call_id}" \ && "${record_call_id}" != "${stopped_call_id}" ]]; then - fail "${record_label} names call ${record_call_id} after its container ${stopped_container_id} was stopped, not call ${stopped_call_id}, which Modal runs again under the same ID" + fail "${record_label} names ${record_call_id:+call }${record_call_id:-no call} after its container ${stopped_container_id} was stopped, not call ${stopped_call_id}, which Modal runs again under the same ID" fi case "${record_status}" in "") ;; diff --git a/projects/policyengine-simulation-entry/DEPLOYMENT.md b/projects/policyengine-simulation-entry/DEPLOYMENT.md index 2ff92cb64..edbc18fdf 100644 --- a/projects/policyengine-simulation-entry/DEPLOYMENT.md +++ b/projects/policyengine-simulation-entry/DEPLOYMENT.md @@ -162,10 +162,12 @@ runs that call (`src/modal/utils/describe_call_inputs.py`), and stops that container with `modal container stop`. Modal sends the container SIGINT and gives the input to another container under the same call ID, as it does after a pre-emption. The check requires the record to succeed under that -call, with a completion time after the stop, and the call graph to show the -call's input finished in another container. A record that completed before -the stop proves nothing: Modal's rerun finds it finished and only returns -its summary, so that attempt counts as missed. Run again, the household function or coordinator resumes its +call, completed at least 5 seconds after the stop, and the call graph to +show the call's input finished in another container. A record completed +sooner proves nothing, and that attempt counts as missed: the stopped +container may have written the success before the signal reached it, and +Modal's rerun then finds the record finished and only returns its summary. +A rerun in a new container can't finish within 5 seconds of the stop. Run again, the household function or coordinator resumes its claim. The report's simulation runs are reused, so its coordinator starts no worker, and the check doesn't cover a coordinator starting its workers again. From d0d041d77f121e3dc3ccdd31d6c23a4dcbf20506 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:14:50 +0400 Subject: [PATCH 17/24] Test review round 4's fixes to the staging preemption check (failing) - After an attempt whose record completed too soon after the stop, the next attempt waits for the container Modal's rerun ran in, which is now idle and warm, not only the stopped one. The fake listing drops a stopped container, as Modal does. - A missed attempt's reason says when Modal could not list containers, and after a failed stop says the record finished after it. - Tests for paths round 3's code already handles: a listing Modal prints unreadably is asked again; a wait Modal can't answer ends at its time limit; only a listing Modal answered may confirm the container to stop. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_staging_preemption_check.py | 116 +++++++++++++++++- 1 file changed, 112 insertions(+), 4 deletions(-) diff --git a/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py index a50b7a05a..ae76c74f6 100644 --- a/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py +++ b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py @@ -164,9 +164,18 @@ printf 'Error: the Modal server did not answer\n' >&2 exit 1 fi - # A container with a departure count is listed only that many times. - jq --argjson listing "${listing}" \ - '[.listed[] as $id | select((.departures[$id] // $listing) >= $listing) + if jq -e --argjson listing "${listing}" '.unreadable_listings | index($listing)' \ + "${spec}" >/dev/null; then + printf 'Active Containers in environment: staging\n' + exit 0 + fi + # A stopped container is never listed again, and one with a departure + # count only that many times. + stopped="$(find "${FAKE_STATE}" -name 'stopped-*' -exec basename {} \; \ + | sed 's/^stopped-//' | jq -R . | jq -s .)" + jq --argjson listing "${listing}" --argjson stopped "${stopped}" \ + '[.listed[] as $id | select(($stopped | index($id)) == null) + | select((.departures[$id] // $listing) >= $listing) | {"Container ID": $id, "App ID": "ap-01V2", "App Name": "policyengine-v2", "Start Time": "2026-11-13T12:00:00+00:00"}]' "${spec}" ;; @@ -344,8 +353,10 @@ class ModalContainers: listed: tuple[str, ...] = (*CONTAINERS, RERUN_CONTAINER) refused_stops: tuple[str, ...] = () timed_out_stops: tuple[str, ...] = () - # Which listings, counted from 1, fail as if Modal didn't answer. + # Which listings, counted from 1, fail as if Modal didn't answer, or + # print something other than JSON. failed_listings: tuple[int, ...] = () + unreadable_listings: tuple[int, ...] = () # How many listings name each container before Modal scales it down. departures: dict[str, int] = field(default_factory=dict) @@ -425,6 +436,7 @@ def _run( "refused_stops": list(containers.refused_stops), "timed_out_stops": list(containers.timed_out_stops), "failed_listings": list(containers.failed_listings), + "unreadable_listings": list(containers.unreadable_listings), "departures": containers.departures, } (state / "spec.json").write_text(json.dumps(spec), encoding="utf-8") @@ -463,6 +475,15 @@ def _stops(fakes: FakeCommands) -> list[str]: ] +def _listings_before_the_first_stop(fakes: FakeCommands) -> int: + listings = 0 + for call in fakes.calls("uv"): + if "stop" in call: + return listings + listings += "list" in call + return listings + + def _submissions(state: Path) -> list[dict]: return [ json.loads(path.read_text(encoding="utf-8")) @@ -661,10 +682,58 @@ def test_a_failed_listing_is_asked_again_rather_than_taken_for_an_absent_contain assert result.returncode == 0, result.stderr assert _stops(fakes) == [CONTAINERS[0]] + # Only a listing Modal answered may confirm the container. + assert _listings_before_the_first_stop(fakes) == 2 assert "Modal could not list" in result.stderr assert "is not listed" not in result.stderr +def test_an_unreadable_listing_is_asked_again(fakes, tmp_path): + result, _ = _run( + fakes, + tmp_path, + "household", + [_household_attempt(0)], + containers=ModalContainers(unreadable_listings=(1,)), + ) + + assert result.returncode == 0, result.stderr + assert _stops(fakes) == [CONTAINERS[0]] + assert _listings_before_the_first_stop(fakes) == 2 + assert "readably" in result.stderr + assert "is not listed" not in result.stderr + + +def test_a_job_finished_while_modal_could_not_list_says_so(fakes, tmp_path): + result, _ = _run( + fakes, + tmp_path, + "household", + [_household_attempt(0, proves=False)], + containers=ModalContainers(failed_listings=tuple(range(1, 10))), + PREEMPTION_CHECK_ATTEMPTS="1", + ) + + assert result.returncode == 1 + assert "the last: " in result.stderr + last_reason = result.stderr.split("the last: ")[-1] + assert "Modal could not list" in last_reason + + +def test_a_wait_modal_cannot_answer_ends_at_its_time_limit(fakes, tmp_path): + result, state = _run( + fakes, + tmp_path, + "household", + [_household_attempt(0, proves=False), _household_attempt(1)], + containers=ModalContainers(failed_listings=(2,)), + ) + + assert result.returncode == 0, result.stderr + assert len(_submissions(state)) == 2 + assert "could not say within 0 seconds" in result.stderr + + def test_the_wait_asks_again_after_a_failed_listing(fakes, tmp_path): """A listing Modal doesn't answer doesn't mean the container has left.""" result, _ = _run( @@ -713,6 +782,7 @@ def test_a_stop_that_runs_out_of_time_is_followed_like_a_failed_one(fakes, tmp_p assert len(_submissions(state)) == 2 assert _stops(fakes) == [CONTAINERS[0], CONTAINERS[1]] assert "exit status 124" in result.stderr + assert "finished after modal container stop failed" in result.stderr def test_a_container_still_listed_after_the_wait_does_not_hold_the_check( @@ -877,6 +947,44 @@ def test_a_completion_moments_after_the_stop_is_taken_for_the_stopped_containers assert ("too soon" in result.stderr) is not proves +def test_after_a_too_soon_completion_the_next_attempt_waits_for_the_rerun_container( + fakes, tmp_path +): + """Modal's rerun ran in a container that is now idle and warm.""" + first = _household_attempt(0) + first.reads_after_stop = [ + _household( + JOB_IDS[0], "succeeded", CALL_IDS[0], completed_at="2026-11-13T12:00:02Z" + ) + ] + + result, _ = _run( + fakes, + tmp_path, + "household", + [first, _household_attempt(1)], + containers=ModalContainers(departures={RERUN_CONTAINER: 2}), + PREEMPTION_CHECK_SCALEDOWN_SECONDS="120", + ) + + assert result.returncode == 0, result.stderr + calls = fakes.calls() + listings = [ + index for index, call in enumerate(calls) if call[0] == "uv" and "list" in call + ] + submissions = [ + index + for index, call in enumerate(calls) + if call[0] == "curl" and call[-1].endswith("/household-calculations") + ] + # The first attempt's lookup; then, waiting, one listing naming the + # rerun's container and one without it; then the second attempt's lookup. + assert len(listings) == 4 + assert listings[2] < submissions[1] < listings[3] + assert f"waiting for container {RERUN_CONTAINER}" in result.stdout + assert "too soon" in result.stderr + + def test_a_record_that_names_no_call_after_the_stop_fails(fakes, tmp_path): attempt = _household_attempt(0) attempt.reads_after_stop = [_household(JOB_IDS[0], "succeeded", None)] From 921743a254982936e2ed7990b1b9bf8fdbc36cd6 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:18:37 +0400 Subject: [PATCH 18/24] Fix review round 4: wait for the rerun's warm container too - Before a new attempt, the check waits for every container the call graph says the last attempt's call ran in, with the one it found or stopped: after a record that completed too soon after the stop, the container Modal's rerun ran in is idle and warm, and the next calculation would land there and finish too fast to stop. One listing per wait checks them all. - A missed attempt's reason says when Modal could not list containers, and after a failed stop says the record finished after it. - The 5-second rule is described as counted in whole seconds, a completion in the stop's own second reports how many seconds after the stop it came, and the edited paragraphs are re-wrapped. Co-Authored-By: Claude Opus 5.5 --- .../simulation-staging-preemption-check.sh | 136 ++++++++++-------- .../DEPLOYMENT.md | 34 ++--- 2 files changed, 93 insertions(+), 77 deletions(-) diff --git a/.github/scripts/simulation-staging-preemption-check.sh b/.github/scripts/simulation-staging-preemption-check.sh index 41786a0ca..6f204c4d4 100755 --- a/.github/scripts/simulation-staging-preemption-check.sh +++ b/.github/scripts/simulation-staging-preemption-check.sh @@ -18,15 +18,17 @@ # environment. `modal container stop` sends it SIGINT, and Modal gives the # call's input to another container under the same call ID, as it does # after a pre-emption. The check passes once the record succeeds under that -# call, completed at least 5 seconds after the stop, and the call graph -# shows the call's input finished in another container. A record completed -# before then proves nothing: the stopped container may have written the -# success before the signal reached it, and Modal's rerun then finds the -# record finished and only returns its summary. A rerun in a new container -# can't finish within 5 seconds of the stop. A calculation that finishes before its container is stopped -# proves nothing, so a new one is submitted, up to PREEMPTION_CHECK_ATTEMPTS -# times, once Modal no longer lists the last attempt's container: on that -# warm container the calculation would finish too fast to stop. A record +# call, completed at least 5 seconds after the stop (both counted in whole +# seconds), and the call graph shows the call's input finished in another +# container. A record completed sooner proves nothing: the stopped container +# may have written the success before the signal reached it, and Modal's +# rerun then finds the record finished and only returns its summary. A +# rerun in a new container can't finish within 5 seconds of the stop. +# A calculation that finishes before its container is stopped proves +# nothing, so a new one is submitted, up to PREEMPTION_CHECK_ATTEMPTS +# times, once Modal no longer lists any container the last attempt's call +# ran in: on such a warm container the calculation would finish too fast +# to stop. A record # that fails, or doesn't finish within the deadline, fails the check at # once. A stop that fails, as Modal's does for a container that has ended, # is not tried again: the check follows the record to its end, so a record @@ -76,8 +78,8 @@ scaledown_seconds="${PREEMPTION_CHECK_SCALEDOWN_SECONDS:-180}" request_timeout_seconds=30 submission_timeout_seconds=60 modal_command_timeout_seconds=60 -# A record completed sooner after the stop was completed by the stopped -# container, not by Modal's rerun in a new one. +# A record completed sooner after the stop may have been completed by the +# stopped container, not by Modal's rerun in a new one. rerun_minimum_seconds=5 uuid_pattern='^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' call_id_pattern='^fc-[0-9A-Za-z]+$' @@ -150,27 +152,28 @@ call_inputs() { modal_command python -m src.modal.utils.describe_call_inputs "$1" } -# Return 0 when Modal lists container as running in -# MODAL_ENVIRONMENT, 1 when it doesn't, and 2 when Modal couldn't list the -# environment's containers, which says nothing about the container. -container_listed() { - local listing - local search_status=0 +# Of the containers ..., print those Modal lists as running +# in MODAL_ENVIRONMENT, one per line. Return 0 when it lists any, 1 when it +# lists none, and 2 when Modal couldn't list the environment's containers +# readably, which says nothing about them. +containers_listed() { + local listing listed if ! listing="$(modal_command modal container list --env "${MODAL_ENVIRONMENT}" --json)"; then printf 'Modal could not list the containers in environment %s; asking again\n' \ "${MODAL_ENVIRONMENT}" >&2 return 2 fi - jq -e --arg container_id "$1" 'any(.[]; .["Container ID"] == $container_id)' \ - <<<"${listing}" >/dev/null || search_status=$? - case "${search_status}" in - 0 | 1) return "${search_status}" ;; - *) - printf 'Modal could not list the containers in environment %s readably; asking again\n' \ - "${MODAL_ENVIRONMENT}" >&2 - return 2 - ;; - esac + if ! listed="$( + jq -r '[.[] | .["Container ID"]] as $running + | $ARGS.positional[] | select(. as $id | $running | index($id))' \ + --args "$@" <<<"${listing}" 2>/dev/null + )"; then + printf 'Modal could not list the containers in environment %s readably; asking again\n' \ + "${MODAL_ENVIRONMENT}" >&2 + return 2 + fi + [[ -n "${listed}" ]] || return 1 + printf '%s\n' "${listed}" } # Create the attempt's record. Sets record_url, record_noun, record_label @@ -292,7 +295,7 @@ find_container() { [[ "${candidate}" =~ ${container_id_pattern} ]] \ || fail "Modal names '${candidate}' as call ${record_call_id}'s container, which is not a Modal container ID" local listed_status=0 - container_listed "${candidate}" || listed_status=$? + containers_listed "${candidate}" >/dev/null || listed_status=$? case "${listed_status}" in 0) container_id="${candidate}" @@ -303,26 +306,30 @@ find_container() { printf '%s; asking again\n' "${unlisted_note}" >&2 ;; # Modal couldn't list: the next read asks again. - *) ;; + *) listing_note="Modal could not list the containers in environment ${MODAL_ENVIRONMENT}" ;; esac } -# When container_id is empty, set it to a container the call graph says ran -# an input of the record's call, so the next attempt can wait for it to -# leave: a warm container may finish the calculation before the check sees -# it running. -remember_finished_container() { +# Set leaving_containers to the container the check found or stopped and +# every container the call graph says ran an input of this attempt's call, +# such as the one Modal's rerun ran in, so the next attempt can wait for +# them to leave: a warm container may finish a calculation before the check +# sees it running. +remember_call_containers() { + local call="${stopped_call_id:-${record_call_id}}" local inputs candidate - if [[ -n "${container_id}" || -z "${record_call_id}" ]]; then - return 0 - fi - inputs="$(call_inputs "${record_call_id}")" || return 0 - candidate="$( - jq -r '[.[] | select(.container_id != "")] | first | .container_id // empty' <<<"${inputs}" - )" || return 0 - if [[ "${candidate}" =~ ${container_id_pattern} ]]; then - container_id="${candidate}" + leaving_containers=() + if [[ -n "${container_id}" ]]; then + leaving_containers+=("${container_id}") fi + [[ -n "${call}" ]] || return 0 + inputs="$(call_inputs "${call}")" || return 0 + while IFS= read -r candidate; do + if [[ "${candidate}" =~ ${container_id_pattern} \ + && " ${leaving_containers[*]+${leaving_containers[*]}} " != *" ${candidate} "* ]]; then + leaving_containers+=("${candidate}") + fi + done < <(jq -r '.[] | select(.container_id != "") | .container_id' <<<"${inputs}" 2>/dev/null || true) } # Return 0 when the record completed after its container was stopped, so @@ -335,7 +342,7 @@ completed_after_the_stop() { jq -rn --arg completed_at "${record_completed_at}" \ '$completed_at | sub("\\.[0-9]+"; "") | sub("\\+00:00$"; "Z") | fromdateiso8601' )" || fail "${record_label}'s completion time ${record_completed_at} is not a UTC time" - if ((completed_epoch <= stopped_at_epoch)); then + if ((completed_epoch < stopped_at_epoch)); then attempt_outcome="missed" miss_reason="${record_label} completed at ${record_completed_at}, before its container ${stopped_container_id} was stopped, so Modal's rerun of call ${stopped_call_id} found it finished" return 1 @@ -348,28 +355,31 @@ completed_after_the_stop() { return 0 } -# Wait until Modal no longer lists container , at most +# Wait until Modal lists none of leaving_containers, at most # scaledown_seconds, so the next attempt's calculation starts in a new -# container rather than the warm one the last attempt left. -wait_for_container_to_leave() { - local leaving="$1" +# container rather than a warm one the last attempt left. +wait_for_containers_to_leave() { local waiting_since="${SECONDS}" - [[ -n "${leaving}" ]] || return 0 - printf 'Before the next attempt, waiting for container %s to leave Modal environment %s\n' \ - "${leaving}" "${MODAL_ENVIRONMENT}" - local listed_status + local leaving listed_status still_listed + ((${#leaving_containers[@]} > 0)) || return 0 + for leaving in "${leaving_containers[@]}"; do + printf 'Before the next attempt, waiting for container %s to leave Modal environment %s\n' \ + "${leaving}" "${MODAL_ENVIRONMENT}" + done while true; do listed_status=0 - container_listed "${leaving}" || listed_status=$? + still_listed="$(containers_listed "${leaving_containers[@]}")" || listed_status=$? ((listed_status != 1)) || return 0 if ((SECONDS - waiting_since >= scaledown_seconds)); then if ((listed_status == 0)); then - printf 'container %s is still listed after %d seconds; submitting again anyway\n' \ - "${leaving}" "${scaledown_seconds}" >&2 + while IFS= read -r leaving; do + printf 'container %s is still listed after %d seconds\n' "${leaving}" "${scaledown_seconds}" >&2 + done <<<"${still_listed}" else - printf 'Modal could not say within %d seconds whether container %s has left; submitting again anyway\n' \ - "${scaledown_seconds}" "${leaving}" >&2 + printf 'Modal could not say within %d seconds whether the last attempt'"'"'s containers have left\n' \ + "${scaledown_seconds}" >&2 fi + printf 'Submitting again anyway\n' >&2 return 0 fi sleep "${poll_seconds}" @@ -415,6 +425,7 @@ attempt_once() { stopped_call_id="" stopped_at_epoch=0 unlisted_note="" + listing_note="" stop_failure="" last_read="nothing yet" local started_at="${SECONDS}" @@ -429,10 +440,12 @@ attempt_once() { case "${record_status}" in "") ;; succeeded) - if [[ -z "${stopped_container_id}" ]]; then + if [[ -n "${stop_failure}" ]]; then + attempt_outcome="missed" + miss_reason="${record_label} finished after ${stop_failure}" + elif [[ -z "${stopped_container_id}" ]]; then attempt_outcome="missed" - miss_reason="${record_label} finished before its container was stopped${unlisted_note:+ (${unlisted_note})}${stop_failure:+ (${stop_failure})}" - remember_finished_container + miss_reason="${record_label} finished before its container was stopped${unlisted_note:+ (${unlisted_note})}${listing_note:+ (${listing_note})}" elif completed_after_the_stop; then confirm_rerun fi @@ -479,7 +492,8 @@ for ((attempt = 1; attempt <= attempts; attempt++)); do [[ "${attempt_outcome}" != "proven" ]] || exit 0 printf 'Attempt %d of %d proved nothing: %s\n' "${attempt}" "${attempts}" "${miss_reason}" >&2 if ((attempt < attempts)); then - wait_for_container_to_leave "${container_id}" + remember_call_containers + wait_for_containers_to_leave fi done fail "no attempt in ${attempts} showed a ${kind} calculation finishing in another container after its container was stopped; the last: ${miss_reason}" diff --git a/projects/policyengine-simulation-entry/DEPLOYMENT.md b/projects/policyengine-simulation-entry/DEPLOYMENT.md index edbc18fdf..a6401fabe 100644 --- a/projects/policyengine-simulation-entry/DEPLOYMENT.md +++ b/projects/policyengine-simulation-entry/DEPLOYMENT.md @@ -167,24 +167,25 @@ show the call's input finished in another container. A record completed sooner proves nothing, and that attempt counts as missed: the stopped container may have written the success before the signal reached it, and Modal's rerun then finds the record finished and only returns its summary. -A rerun in a new container can't finish within 5 seconds of the stop. Run again, the household function or coordinator resumes its -claim. The report's simulation runs are reused, so its coordinator -starts no worker, and the check doesn't cover a coordinator starting its -workers again. +A rerun in a new container can't finish within 5 seconds of the stop (both +times are counted in whole seconds). Run again, the household function or +coordinator resumes its claim. The report's simulation runs are reused, so +its coordinator starts no worker, and the check doesn't cover a coordinator +starting its workers again. A calculation that finishes before its container is stopped, for example on a warm container, proves nothing, so the check submits a new one, up to three attempts. Before each new attempt it waits, up to three minutes, for -Modal to stop listing the last attempt's container (Modal stops an idle -container after 60 seconds), so the next calculation starts in a new -container, including a container it first saw only after it had finished -the calculation. A record that fails, or doesn't finish within its deadline -(10 minutes for the household, 15 for the report, per attempt), fails the -check at once. Modal refuses to stop a container that has ended; when a -stop fails, the check doesn't stop it again or submit another calculation, -but follows the record to its end, so a record left running by a container -that ended for another reason, such as running out of memory, fails the -check at its deadline. Every Modal command may take 60 seconds. +Modal to stop listing every container the last attempt's call ran in, as +the call graph names them, including the one Modal's rerun ran in (Modal +stops an idle container after 60 seconds), so the next calculation starts +in a new container. A record that fails, or doesn't finish within its +deadline (10 minutes for the household, 15 for the report, per attempt), +fails the check at once. Modal refuses to stop a container that has +ended; when a stop fails, the check doesn't stop it again or submit another +calculation, but follows the record to its end, so a record left running by +a container that ended for another reason, such as running out of memory, +fails the check at its deadline. Every Modal command may take 60 seconds. It uses the repository's Modal token (the repository secrets `MODAL_TOKEN_ID` and `MODAL_TOKEN_SECRET`), which must be allowed to read @@ -192,8 +193,9 @@ call graphs and to list and stop containers. Production's deploy jobs use the same token, and `modal container stop` takes a container ID in any environment, so the script refuses any `MODAL_ENVIRONMENT` but `staging` and stops only a container that `modal container list --env staging` lists. It -also uses the authenticated tests' client credentials. `deployment_ready` requires -the check to pass in staging, so a failure stops the production deployment. +also uses the authenticated tests' client credentials. `deployment_ready` +requires the check to pass in staging, so a failure stops the production +deployment. Each attempt adds a household job, or a report and its run, to the staging database. From 6aa9ed08c01712e8eb70c5c955b81728d179776c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:43:11 +0400 Subject: [PATCH 19/24] Test review round 5's fixes to the staging preemption check (failing) - Before each new attempt, and after none of the last, the check waits a fixed PREEMPTION_CHECK_SCALEDOWN_SECONDS (90) for Modal to stop the containers the last attempt left idle, instead of waiting for the containers one call-graph read names, which may not name the one Modal's rerun ran in yet. The tests of the old wait go. - An empty container listing is unreadable, not a listing without the container. - A container found after a failed listing clears the failure from the attempt's reason. - A completion in the stop's own second reports "0 seconds after" (this passes already since round 4's fix). Co-Authored-By: Claude Opus 5.5 --- .../tests/test_staging_preemption_check.py | 227 +++++++----------- 1 file changed, 85 insertions(+), 142 deletions(-) diff --git a/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py index ae76c74f6..fb1690a83 100644 --- a/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py +++ b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py @@ -169,13 +169,15 @@ printf 'Active Containers in environment: staging\n' exit 0 fi - # A stopped container is never listed again, and one with a departure - # count only that many times. + if jq -e --argjson listing "${listing}" '.empty_listings | index($listing)' \ + "${spec}" >/dev/null; then + exit 0 + fi + # A stopped container is never listed again. stopped="$(find "${FAKE_STATE}" -name 'stopped-*' -exec basename {} \; \ | sed 's/^stopped-//' | jq -R . | jq -s .)" - jq --argjson listing "${listing}" --argjson stopped "${stopped}" \ + jq --argjson stopped "${stopped}" \ '[.listed[] as $id | select(($stopped | index($id)) == null) - | select((.departures[$id] // $listing) >= $listing) | {"Container ID": $id, "App ID": "ap-01V2", "App Name": "policyengine-v2", "Start Time": "2026-11-13T12:00:00+00:00"}]' "${spec}" ;; @@ -357,8 +359,7 @@ class ModalContainers: # print something other than JSON. failed_listings: tuple[int, ...] = () unreadable_listings: tuple[int, ...] = () - # How many listings name each container before Modal scales it down. - departures: dict[str, int] = field(default_factory=dict) + empty_listings: tuple[int, ...] = () @pytest.fixture @@ -437,7 +438,7 @@ def _run( "timed_out_stops": list(containers.timed_out_stops), "failed_listings": list(containers.failed_listings), "unreadable_listings": list(containers.unreadable_listings), - "departures": containers.departures, + "empty_listings": list(containers.empty_listings), } (state / "spec.json").write_text(json.dumps(spec), encoding="utf-8") environment = { @@ -452,8 +453,6 @@ def _run( # The fake sleep returns at once, so a record that never finishes # fails the test within this many seconds. "PREEMPTION_CHECK_DEADLINE_SECONDS": "20", - # Most tests don't wait for an earlier attempt's container to leave. - "PREEMPTION_CHECK_SCALEDOWN_SECONDS": "0", "FAKE_STATE": str(state), "FAKE_BASE_URL": BASE_URL, "FAKE_ACCESS_TOKEN": ACCESS_TOKEN, @@ -606,69 +605,6 @@ def test_a_calculation_that_finishes_before_the_stop_is_submitted_again( assert "finished before its container was stopped" in result.stderr -def test_the_next_attempt_waits_for_the_last_attempts_container_to_leave( - fakes, tmp_path -): - """A warm container calculates too fast to stop; Modal scales it down idle.""" - result, state = _run( - fakes, - tmp_path, - "household", - [_household_attempt(0, proves=False), _household_attempt(1)], - containers=ModalContainers(departures={CONTAINERS[0]: 3}), - PREEMPTION_CHECK_SCALEDOWN_SECONDS="120", - ) - - assert result.returncode == 0, result.stderr - calls = fakes.calls() - listings = [ - index for index, call in enumerate(calls) if call[0] == "uv" and "list" in call - ] - submissions = [ - index - for index, call in enumerate(calls) - if call[0] == "curl" and call[-1].endswith("/household-calculations") - ] - # The first attempt's lookup, then two listings naming its container and - # one without it, then the second attempt's lookup. - assert len(listings) == 5 - assert listings[3] < submissions[1] < listings[4] - assert len(_submissions(state)) == 2 - assert f"waiting for container {CONTAINERS[0]}" in result.stdout - - -def test_the_next_attempt_waits_for_a_container_first_seen_finished(fakes, tmp_path): - """A warm container finished the job before the check saw it running.""" - first = _household_attempt(0, proves=False) - first.reads_before_stop = [_household(JOB_IDS[0], "succeeded", CALL_IDS[0])] - first.inputs_before_stop = [_inputs((CONTAINERS[0], "SUCCESS"))] - - result, _ = _run( - fakes, - tmp_path, - "household", - [first, _household_attempt(1)], - containers=ModalContainers(departures={CONTAINERS[0]: 2}), - PREEMPTION_CHECK_SCALEDOWN_SECONDS="120", - ) - - assert result.returncode == 0, result.stderr - calls = fakes.calls() - listings = [ - index for index, call in enumerate(calls) if call[0] == "uv" and "list" in call - ] - submissions = [ - index - for index, call in enumerate(calls) - if call[0] == "curl" and call[-1].endswith("/household-calculations") - ] - # Two listings naming the first attempt's container and one without it, - # then the second attempt's lookup. - assert len(listings) == 4 - assert listings[2] < submissions[1] < listings[3] - assert f"waiting for container {CONTAINERS[0]}" in result.stdout - - def test_a_failed_listing_is_asked_again_rather_than_taken_for_an_absent_container( fakes, tmp_path ): @@ -704,62 +640,60 @@ def test_an_unreadable_listing_is_asked_again(fakes, tmp_path): assert "is not listed" not in result.stderr -def test_a_job_finished_while_modal_could_not_list_says_so(fakes, tmp_path): +def test_an_empty_listing_is_asked_again(fakes, tmp_path): + """Modal's listing prints a JSON array, even an empty one.""" result, _ = _run( fakes, tmp_path, "household", - [_household_attempt(0, proves=False)], - containers=ModalContainers(failed_listings=tuple(range(1, 10))), - PREEMPTION_CHECK_ATTEMPTS="1", + [_household_attempt(0)], + containers=ModalContainers(empty_listings=(1,)), ) - assert result.returncode == 1 - assert "the last: " in result.stderr - last_reason = result.stderr.split("the last: ")[-1] - assert "Modal could not list" in last_reason + assert result.returncode == 0, result.stderr + assert _stops(fakes) == [CONTAINERS[0]] + assert _listings_before_the_first_stop(fakes) == 2 + assert "readably" in result.stderr + assert "is not listed" not in result.stderr -def test_a_wait_modal_cannot_answer_ends_at_its_time_limit(fakes, tmp_path): - result, state = _run( +def test_a_container_found_after_a_failed_listing_clears_its_note(fakes, tmp_path): + attempt = _household_attempt(0, proves=False) + attempt.reads_before_stop = [ + _household(JOB_IDS[0], "pending", CALL_IDS[0]), + _household(JOB_IDS[0], "pending", CALL_IDS[0]), + _household(JOB_IDS[0], "succeeded", CALL_IDS[0]), + ] + + result, _ = _run( fakes, tmp_path, "household", - [_household_attempt(0, proves=False), _household_attempt(1)], - containers=ModalContainers(failed_listings=(2,)), + [attempt], + containers=ModalContainers(failed_listings=(1,)), + PREEMPTION_CHECK_ATTEMPTS="1", ) - assert result.returncode == 0, result.stderr - assert len(_submissions(state)) == 2 - assert "could not say within 0 seconds" in result.stderr + assert result.returncode == 1 + last_reason = result.stderr.split("the last: ")[-1] + assert "finished before its container was stopped" in last_reason + assert "Modal could not list" not in last_reason -def test_the_wait_asks_again_after_a_failed_listing(fakes, tmp_path): - """A listing Modal doesn't answer doesn't mean the container has left.""" +def test_a_job_finished_while_modal_could_not_list_says_so(fakes, tmp_path): result, _ = _run( fakes, tmp_path, "household", - [_household_attempt(0, proves=False), _household_attempt(1)], - containers=ModalContainers(departures={CONTAINERS[0]: 3}, failed_listings=(2,)), - PREEMPTION_CHECK_SCALEDOWN_SECONDS="120", + [_household_attempt(0, proves=False)], + containers=ModalContainers(failed_listings=tuple(range(1, 10))), + PREEMPTION_CHECK_ATTEMPTS="1", ) - assert result.returncode == 0, result.stderr - calls = fakes.calls() - listings = [ - index for index, call in enumerate(calls) if call[0] == "uv" and "list" in call - ] - submissions = [ - index - for index, call in enumerate(calls) - if call[0] == "curl" and call[-1].endswith("/household-calculations") - ] - # The first attempt's lookup; then, waiting, a failed listing, one naming - # the container and one without it; then the second attempt's lookup. - assert len(listings) == 5 - assert listings[3] < submissions[1] < listings[4] - assert "Modal could not list" in result.stderr + assert result.returncode == 1 + assert "the last: " in result.stderr + last_reason = result.stderr.split("the last: ")[-1] + assert "Modal could not list" in last_reason def test_a_stop_that_runs_out_of_time_is_followed_like_a_failed_one(fakes, tmp_path): @@ -785,10 +719,21 @@ def test_a_stop_that_runs_out_of_time_is_followed_like_a_failed_one(fakes, tmp_p assert "finished after modal container stop failed" in result.stderr -def test_a_container_still_listed_after_the_wait_does_not_hold_the_check( - fakes, tmp_path -): - result, state = _run( +def _indices(fakes: FakeCommands, matches) -> list[int]: + return [index for index, call in enumerate(fakes.calls()) if matches(call)] + + +def _is_submission(call: list[str]) -> bool: + return call[0] == "curl" and call[-1].endswith("/household-calculations") + + +def test_the_next_attempt_waits_out_modals_idle_scale_down_first(fakes, tmp_path): + """Modal stops a container left idle for 60 seconds, by default. + + A new attempt on a warm container the last one left, the one the check + found or the one Modal's rerun ran in, would finish too fast to stop. + """ + result, _ = _run( fakes, tmp_path, "household", @@ -796,8 +741,27 @@ def test_a_container_still_listed_after_the_wait_does_not_hold_the_check( ) assert result.returncode == 0, result.stderr - assert len(_submissions(state)) == 2 - assert f"container {CONTAINERS[0]} is still listed" in result.stderr + submissions = _indices(fakes, _is_submission) + waits = _indices(fakes, lambda call: call == ["sleep", "90"]) + assert len(waits) == 1 + assert submissions[0] < waits[0] < submissions[1] + assert "waiting 90 seconds" in result.stdout + + +def test_no_wait_follows_the_last_attempt(fakes, tmp_path): + result, _ = _run( + fakes, + tmp_path, + "household", + [_household_attempt(attempt, proves=False) for attempt in range(3)], + PREEMPTION_CHECK_SCALEDOWN_SECONDS="75", + ) + + assert result.returncode == 1 + submissions = _indices(fakes, _is_submission) + waits = _indices(fakes, lambda call: call == ["sleep", "75"]) + assert len(waits) == 2 + assert submissions[0] < waits[0] < submissions[1] < waits[1] < submissions[2] def test_three_calculations_that_finish_before_the_stop_fail_the_check(fakes, tmp_path): @@ -947,42 +911,21 @@ def test_a_completion_moments_after_the_stop_is_taken_for_the_stopped_containers assert ("too soon" in result.stderr) is not proves -def test_after_a_too_soon_completion_the_next_attempt_waits_for_the_rerun_container( - fakes, tmp_path -): - """Modal's rerun ran in a container that is now idle and warm.""" - first = _household_attempt(0) - first.reads_after_stop = [ +def test_a_completion_in_the_stops_own_second_is_too_soon(fakes, tmp_path): + """Both times are whole seconds, so the same second may be after the stop.""" + attempt = _household_attempt(0) + attempt.reads_after_stop = [ _household( - JOB_IDS[0], "succeeded", CALL_IDS[0], completed_at="2026-11-13T12:00:02Z" + JOB_IDS[0], "succeeded", CALL_IDS[0], completed_at="2026-11-13T12:00:00Z" ) ] result, _ = _run( - fakes, - tmp_path, - "household", - [first, _household_attempt(1)], - containers=ModalContainers(departures={RERUN_CONTAINER: 2}), - PREEMPTION_CHECK_SCALEDOWN_SECONDS="120", + fakes, tmp_path, "household", [attempt], PREEMPTION_CHECK_ATTEMPTS="1" ) - assert result.returncode == 0, result.stderr - calls = fakes.calls() - listings = [ - index for index, call in enumerate(calls) if call[0] == "uv" and "list" in call - ] - submissions = [ - index - for index, call in enumerate(calls) - if call[0] == "curl" and call[-1].endswith("/household-calculations") - ] - # The first attempt's lookup; then, waiting, one listing naming the - # rerun's container and one without it; then the second attempt's lookup. - assert len(listings) == 4 - assert listings[2] < submissions[1] < listings[3] - assert f"waiting for container {RERUN_CONTAINER}" in result.stdout - assert "too soon" in result.stderr + assert result.returncode == 1 + assert "0 seconds after" in result.stderr def test_a_record_that_names_no_call_after_the_stop_fails(fakes, tmp_path): From 7d71756bceee69e26d2e05f379cf97911c51ae32 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:45:05 +0400 Subject: [PATCH 20/24] Fix review round 5: wait a fixed time before a new attempt, refuse empty listings - Before each new attempt the check now waits a fixed PREEMPTION_CHECK_SCALEDOWN_SECONDS (90), past Modal's default 60-second idle scale-down, which the household functions and the coordinator don't change. That stops both the container the check found and the one Modal's rerun ran in, without depending on one call-graph read naming the rerun's container, which Modal records best-effort and not in real time. The listing-based wait and its call-graph read go. - container_listed answers 2 (couldn't list) for a listing that isn't a JSON list, an empty one included, rather than taking it as a listing without the container. - A container found after a failed or unlisted lookup clears those notes, so an attempt's reason names only what last happened. - The deployment notes and the job's time-limit comment describe the fixed wait. Co-Authored-By: Claude Opus 5.5 --- .../simulation-staging-preemption-check.sh | 112 ++++++------------ .../workflows/simulation-deploy.reusable.yml | 6 +- .../DEPLOYMENT.md | 14 +-- 3 files changed, 43 insertions(+), 89 deletions(-) diff --git a/.github/scripts/simulation-staging-preemption-check.sh b/.github/scripts/simulation-staging-preemption-check.sh index 6f204c4d4..235650258 100755 --- a/.github/scripts/simulation-staging-preemption-check.sh +++ b/.github/scripts/simulation-staging-preemption-check.sh @@ -26,9 +26,11 @@ # rerun in a new container can't finish within 5 seconds of the stop. # A calculation that finishes before its container is stopped proves # nothing, so a new one is submitted, up to PREEMPTION_CHECK_ATTEMPTS -# times, once Modal no longer lists any container the last attempt's call -# ran in: on such a warm container the calculation would finish too fast -# to stop. A record +# times, each after a wait for Modal to stop the containers the last +# attempt left idle, the one the check found and the one Modal's rerun ran +# in: on such a warm container the calculation would finish too fast to +# stop. Modal stops a container left idle for 60 seconds by default, and +# the household functions and the coordinator don't change that. A record # that fails, or doesn't finish within the deadline, fails the check at # once. A stop that fails, as Modal's does for a container that has ended, # is not tried again: the check follows the record to its end, so a record @@ -46,9 +48,8 @@ # each attempt's deadline (600 for household, 900 for report); # PREEMPTION_CHECK_POLL_SECONDS (2); PREEMPTION_CHECK_RERUN_SECONDS (120), # how long after the record succeeds the call graph may take to show the -# call's other container; and PREEMPTION_CHECK_SCALEDOWN_SECONDS (180), how -# long to wait for the last attempt's container to leave before a new -# attempt (Modal stops an idle container after 60 seconds by default). +# call's other container; and PREEMPTION_CHECK_SCALEDOWN_SECONDS (90), how +# long to wait before a new attempt. # The access token is sent from a file, and the client secret on standard # input through jq's environment, so neither appears on any command line. A submission, which isn't # asked again, may take 60 seconds, Cloud Run's own limit; every other @@ -74,7 +75,7 @@ attempts="${PREEMPTION_CHECK_ATTEMPTS:-3}" deadline_seconds="${PREEMPTION_CHECK_DEADLINE_SECONDS:-${default_deadline_seconds}}" poll_seconds="${PREEMPTION_CHECK_POLL_SECONDS:-2}" rerun_seconds="${PREEMPTION_CHECK_RERUN_SECONDS:-120}" -scaledown_seconds="${PREEMPTION_CHECK_SCALEDOWN_SECONDS:-180}" +scaledown_seconds="${PREEMPTION_CHECK_SCALEDOWN_SECONDS:-90}" request_timeout_seconds=30 submission_timeout_seconds=60 modal_command_timeout_seconds=60 @@ -152,28 +153,31 @@ call_inputs() { modal_command python -m src.modal.utils.describe_call_inputs "$1" } -# Of the containers ..., print those Modal lists as running -# in MODAL_ENVIRONMENT, one per line. Return 0 when it lists any, 1 when it -# lists none, and 2 when Modal couldn't list the environment's containers -# readably, which says nothing about them. -containers_listed() { +# Return 0 when Modal lists container as running in +# MODAL_ENVIRONMENT, 1 when it doesn't, and 2 when Modal couldn't list the +# environment's containers as a JSON list, which says nothing about the +# container. +container_listed() { local listing listed if ! listing="$(modal_command modal container list --env "${MODAL_ENVIRONMENT}" --json)"; then printf 'Modal could not list the containers in environment %s; asking again\n' \ "${MODAL_ENVIRONMENT}" >&2 return 2 fi - if ! listed="$( - jq -r '[.[] | .["Container ID"]] as $running - | $ARGS.positional[] | select(. as $id | $running | index($id))' \ - --args "$@" <<<"${listing}" 2>/dev/null - )"; then - printf 'Modal could not list the containers in environment %s readably; asking again\n' \ - "${MODAL_ENVIRONMENT}" >&2 - return 2 - fi - [[ -n "${listed}" ]] || return 1 - printf '%s\n' "${listed}" + listed="$( + jq -r --arg container_id "$1" \ + 'if type == "array" then any(.[]; .["Container ID"] == $container_id) + else error("not a list") end' <<<"${listing}" 2>/dev/null + )" || listed="" + case "${listed}" in + true) return 0 ;; + false) return 1 ;; + *) + printf 'Modal could not list the containers in environment %s readably; asking again\n' \ + "${MODAL_ENVIRONMENT}" >&2 + return 2 + ;; + esac } # Create the attempt's record. Sets record_url, record_noun, record_label @@ -295,10 +299,12 @@ find_container() { [[ "${candidate}" =~ ${container_id_pattern} ]] \ || fail "Modal names '${candidate}' as call ${record_call_id}'s container, which is not a Modal container ID" local listed_status=0 - containers_listed "${candidate}" >/dev/null || listed_status=$? + container_listed "${candidate}" || listed_status=$? case "${listed_status}" in 0) container_id="${candidate}" + unlisted_note="" + listing_note="" printf 'Call %s of %s runs in container %s\n' "${record_call_id}" "${record_label}" "${container_id}" ;; 1) @@ -310,28 +316,6 @@ find_container() { esac } -# Set leaving_containers to the container the check found or stopped and -# every container the call graph says ran an input of this attempt's call, -# such as the one Modal's rerun ran in, so the next attempt can wait for -# them to leave: a warm container may finish a calculation before the check -# sees it running. -remember_call_containers() { - local call="${stopped_call_id:-${record_call_id}}" - local inputs candidate - leaving_containers=() - if [[ -n "${container_id}" ]]; then - leaving_containers+=("${container_id}") - fi - [[ -n "${call}" ]] || return 0 - inputs="$(call_inputs "${call}")" || return 0 - while IFS= read -r candidate; do - if [[ "${candidate}" =~ ${container_id_pattern} \ - && " ${leaving_containers[*]+${leaving_containers[*]}} " != *" ${candidate} "* ]]; then - leaving_containers+=("${candidate}") - fi - done < <(jq -r '.[] | select(.container_id != "") | .container_id' <<<"${inputs}" 2>/dev/null || true) -} - # Return 0 when the record completed after its container was stopped, so # its success comes from the rerun; otherwise set attempt_outcome to missed. completed_after_the_stop() { @@ -355,37 +339,6 @@ completed_after_the_stop() { return 0 } -# Wait until Modal lists none of leaving_containers, at most -# scaledown_seconds, so the next attempt's calculation starts in a new -# container rather than a warm one the last attempt left. -wait_for_containers_to_leave() { - local waiting_since="${SECONDS}" - local leaving listed_status still_listed - ((${#leaving_containers[@]} > 0)) || return 0 - for leaving in "${leaving_containers[@]}"; do - printf 'Before the next attempt, waiting for container %s to leave Modal environment %s\n' \ - "${leaving}" "${MODAL_ENVIRONMENT}" - done - while true; do - listed_status=0 - still_listed="$(containers_listed "${leaving_containers[@]}")" || listed_status=$? - ((listed_status != 1)) || return 0 - if ((SECONDS - waiting_since >= scaledown_seconds)); then - if ((listed_status == 0)); then - while IFS= read -r leaving; do - printf 'container %s is still listed after %d seconds\n' "${leaving}" "${scaledown_seconds}" >&2 - done <<<"${still_listed}" - else - printf 'Modal could not say within %d seconds whether the last attempt'"'"'s containers have left\n' \ - "${scaledown_seconds}" >&2 - fi - printf 'Submitting again anyway\n' >&2 - return 0 - fi - sleep "${poll_seconds}" - done -} - # After the record succeeded: require the call graph to show the stopped # call's input finished in another container. Sets attempt_outcome. confirm_rerun() { @@ -492,8 +445,9 @@ for ((attempt = 1; attempt <= attempts; attempt++)); do [[ "${attempt_outcome}" != "proven" ]] || exit 0 printf 'Attempt %d of %d proved nothing: %s\n' "${attempt}" "${attempts}" "${miss_reason}" >&2 if ((attempt < attempts)); then - remember_call_containers - wait_for_containers_to_leave + printf 'Before the next attempt, waiting %d seconds for Modal to stop the containers the last one left idle\n' \ + "${scaledown_seconds}" + sleep "${scaledown_seconds}" fi done fail "no attempt in ${attempts} showed a ${kind} calculation finishing in another container after its container was stopped; the last: ${miss_reason}" diff --git a/.github/workflows/simulation-deploy.reusable.yml b/.github/workflows/simulation-deploy.reusable.yml index 318929cef..2922d9fe6 100644 --- a/.github/workflows/simulation-deploy.reusable.yml +++ b/.github/workflows/simulation-deploy.reusable.yml @@ -1316,10 +1316,10 @@ jobs: needs: [deploy_entrypoint, authenticated_test, publish_stage12_v2_default, staging_economy_check] runs-on: ubuntu-latest # Longer than both steps' worst case together, so the script fails - # first: about 136 minutes, with every request and Modal command taking + # first: about 126 minutes, with every request and Modal command taking # its full time limit (three attempts of up to 17.5 minutes for the - # household and 22.5 for the report, and up to 4 minutes' wait before - # each new attempt), and the install. + # household and 22.5 for the report, and a 90-second wait before each + # new attempt), and the install. timeout-minutes: 150 environment: ${{ inputs.release_environment }} steps: diff --git a/projects/policyengine-simulation-entry/DEPLOYMENT.md b/projects/policyengine-simulation-entry/DEPLOYMENT.md index a6401fabe..1f700ffbe 100644 --- a/projects/policyengine-simulation-entry/DEPLOYMENT.md +++ b/projects/policyengine-simulation-entry/DEPLOYMENT.md @@ -175,13 +175,13 @@ starting its workers again. A calculation that finishes before its container is stopped, for example on a warm container, proves nothing, so the check submits a new one, up to -three attempts. Before each new attempt it waits, up to three minutes, for -Modal to stop listing every container the last attempt's call ran in, as -the call graph names them, including the one Modal's rerun ran in (Modal -stops an idle container after 60 seconds), so the next calculation starts -in a new container. A record that fails, or doesn't finish within its -deadline (10 minutes for the household, 15 for the report, per attempt), -fails the check at once. Modal refuses to stop a container that has +three attempts. Before each new attempt it waits 90 seconds, so Modal +stops the containers the last attempt left idle, the one the check found +and the one Modal's rerun ran in (Modal stops a container left idle for 60 +seconds by default, and these functions don't change that), and the next +calculation starts in a new container. A record that fails, or doesn't +finish within its deadline (10 minutes for the household, 15 for the +report, per attempt), fails the check at once. Modal refuses to stop a container that has ended; when a stop fails, the check doesn't stop it again or submit another calculation, but follows the record to its end, so a record left running by a container that ended for another reason, such as running out of memory, From b6a68df555084febd1e87f8050912435697a9cf9 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:45:15 +0400 Subject: [PATCH 21/24] Re-wrap the preemption check's paragraph on missed attempts Co-Authored-By: Claude Opus 5.5 --- .../policyengine-simulation-entry/DEPLOYMENT.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/projects/policyengine-simulation-entry/DEPLOYMENT.md b/projects/policyengine-simulation-entry/DEPLOYMENT.md index 1f700ffbe..27486b34d 100644 --- a/projects/policyengine-simulation-entry/DEPLOYMENT.md +++ b/projects/policyengine-simulation-entry/DEPLOYMENT.md @@ -175,14 +175,14 @@ starting its workers again. A calculation that finishes before its container is stopped, for example on a warm container, proves nothing, so the check submits a new one, up to -three attempts. Before each new attempt it waits 90 seconds, so Modal -stops the containers the last attempt left idle, the one the check found -and the one Modal's rerun ran in (Modal stops a container left idle for 60 -seconds by default, and these functions don't change that), and the next -calculation starts in a new container. A record that fails, or doesn't -finish within its deadline (10 minutes for the household, 15 for the -report, per attempt), fails the check at once. Modal refuses to stop a container that has -ended; when a stop fails, the check doesn't stop it again or submit another +three attempts. Before each new attempt it waits 90 seconds, so Modal stops +the containers the last attempt left idle, the one the check found and the +one Modal's rerun ran in (Modal stops a container left idle for 60 seconds +by default, and these functions don't change that), and the next calculation +starts in a new container. A record that fails, or doesn't finish within its +deadline (10 minutes for the household, 15 for the report, per attempt), +fails the check at once. Modal refuses to stop a container that has ended; +when a stop fails, the check doesn't stop it again or submit another calculation, but follows the record to its end, so a record left running by a container that ended for another reason, such as running out of memory, fails the check at its deadline. Every Modal command may take 60 seconds. From d17ab4e90dcf15d3074afbcd015fec4889599a7b Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:55:35 +0400 Subject: [PATCH 22/24] Test review round 6's quality fixes to the staging preemption check (failing) - A listing whose entries lack Modal's "Container ID" column is unreadable, not a listing without the container; so are a JSON object and null (those pass already). - A missed attempt's reason names only the last container lookup's outcome: a listing without the container after a failed one, or a failed listing after one without the container, not both. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_staging_preemption_check.py | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py index fb1690a83..fd22105db 100644 --- a/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py +++ b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py @@ -173,6 +173,11 @@ "${spec}" >/dev/null; then exit 0 fi + if jq -e --arg listing "${listing}" '.listing_outputs | has($listing)' \ + "${spec}" >/dev/null; then + jq -r --arg listing "${listing}" '.listing_outputs[$listing]' "${spec}" + exit 0 + fi # A stopped container is never listed again. stopped="$(find "${FAKE_STATE}" -name 'stopped-*' -exec basename {} \; \ | sed 's/^stopped-//' | jq -R . | jq -s .)" @@ -360,6 +365,8 @@ class ModalContainers: failed_listings: tuple[int, ...] = () unreadable_listings: tuple[int, ...] = () empty_listings: tuple[int, ...] = () + # What some listings print instead, by their number. + listing_outputs: dict[int, str] = field(default_factory=dict) @pytest.fixture @@ -439,6 +446,10 @@ def _run( "failed_listings": list(containers.failed_listings), "unreadable_listings": list(containers.unreadable_listings), "empty_listings": list(containers.empty_listings), + "listing_outputs": { + str(listing): output + for listing, output in containers.listing_outputs.items() + }, } (state / "spec.json").write_text(json.dumps(spec), encoding="utf-8") environment = { @@ -657,6 +668,64 @@ def test_an_empty_listing_is_asked_again(fakes, tmp_path): assert "is not listed" not in result.stderr +@pytest.mark.parametrize( + "output", + ["{}", "null", json.dumps([{"ID": CONTAINERS[0]}])], + ids=["object", "null", "renamed-column"], +) +def test_a_listing_that_is_not_a_list_of_containers_is_asked_again( + fakes, tmp_path, output +): + """A change in Modal's output must not read as the container having gone.""" + result, _ = _run( + fakes, + tmp_path, + "household", + [_household_attempt(0)], + containers=ModalContainers(listing_outputs={1: output}), + ) + + assert result.returncode == 0, result.stderr + assert _stops(fakes) == [CONTAINERS[0]] + assert _listings_before_the_first_stop(fakes) == 2 + assert "readably" in result.stderr + assert "is not listed" not in result.stderr + + +@pytest.mark.parametrize( + ("failed_listings", "last_note", "earlier_note"), + [ + ((1,), "is not listed", "Modal could not list"), + ((2,), "Modal could not list", "is not listed"), + ], + ids=["unlisted-after-a-failure", "failure-after-unlisted"], +) +def test_a_missed_attempts_reason_names_only_the_last_lookups_outcome( + fakes, tmp_path, failed_listings, last_note, earlier_note +): + """Modal lists no containers, after or before a listing that fails.""" + attempt = _household_attempt(0, proves=False) + attempt.reads_before_stop = [ + _household(JOB_IDS[0], "pending", CALL_IDS[0]), + _household(JOB_IDS[0], "pending", CALL_IDS[0]), + _household(JOB_IDS[0], "succeeded", CALL_IDS[0]), + ] + + result, _ = _run( + fakes, + tmp_path, + "household", + [attempt], + containers=ModalContainers(listed=(), failed_listings=failed_listings), + PREEMPTION_CHECK_ATTEMPTS="1", + ) + + assert result.returncode == 1 + last_reason = result.stderr.split("the last: ")[-1] + assert last_note in last_reason + assert earlier_note not in last_reason + + def test_a_container_found_after_a_failed_listing_clears_its_note(fakes, tmp_path): attempt = _household_attempt(0, proves=False) attempt.reads_before_stop = [ From 3a2dc5fef107b06a8c2bdbcb11ef54e3071ff9ca Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:26:26 +0400 Subject: [PATCH 23/24] Fix review round 6's quality findings in the staging preemption check - A container listing counts as readable only when it is a JSON list whose entries each have Modal's "Container ID" field, so a change in Modal's output reads as an unreadable listing that is asked again, not as the container having gone. - One note holds the last container listing's outcome, so a missed attempt's reason names only that outcome, not an earlier one. - The script's header and DEPLOYMENT.md no longer claim the 90-second wait always covers the container Modal ran a stopped call in again: that container goes idle only when the rerun returns, so the wait covers it only when the rerun returns within about 30 seconds of the attempt's end. Co-Authored-By: Claude Opus 5.5 --- .../simulation-staging-preemption-check.sh | 56 +++++++++++-------- .../DEPLOYMENT.md | 26 +++++---- 2 files changed, 47 insertions(+), 35 deletions(-) diff --git a/.github/scripts/simulation-staging-preemption-check.sh b/.github/scripts/simulation-staging-preemption-check.sh index 235650258..4b59ae36c 100755 --- a/.github/scripts/simulation-staging-preemption-check.sh +++ b/.github/scripts/simulation-staging-preemption-check.sh @@ -26,17 +26,21 @@ # rerun in a new container can't finish within 5 seconds of the stop. # A calculation that finishes before its container is stopped proves # nothing, so a new one is submitted, up to PREEMPTION_CHECK_ATTEMPTS -# times, each after a wait for Modal to stop the containers the last -# attempt left idle, the one the check found and the one Modal's rerun ran -# in: on such a warm container the calculation would finish too fast to +# times, each a fixed wait after the last attempt ended: on a container +# the last attempt left warm, the calculation would finish too fast to # stop. Modal stops a container left idle for 60 seconds by default, and -# the household functions and the coordinator don't change that. A record -# that fails, or doesn't finish within the deadline, fails the check at -# once. A stop that fails, as Modal's does for a container that has ended, -# is not tried again: the check follows the record to its end, so a record -# left running by a container that ended for another reason, such as -# running out of memory, fails at its deadline rather than being submitted -# again. Every Modal command may take 60 seconds. +# the household functions and the coordinator don't change that. A +# container that finished the last calculation unstopped went idle when +# the record finished, so the wait covers it. One that Modal ran a stopped +# call in again goes idle only when that rerun returns, which may be after +# the attempt ended, so the wait covers it only when the rerun returns +# within about 30 seconds of the attempt's end. A record that fails, or +# doesn't finish within the deadline, fails the check at once. A stop that +# fails, as Modal's does for a container that has ended, is not tried +# again: the check follows the record to its end, so a record left running +# by a container that ended for another reason, such as running out of +# memory, fails at its deadline rather than being submitted again. Every +# Modal command may take 60 seconds. # # Required env vars: the test client's SIMULATION_TEST_AUTH_ISSUER, # SIMULATION_TEST_AUTH_AUDIENCE, SIMULATION_TEST_AUTH_CLIENT_ID and @@ -51,10 +55,10 @@ # call's other container; and PREEMPTION_CHECK_SCALEDOWN_SECONDS (90), how # long to wait before a new attempt. # The access token is sent from a file, and the client secret on standard -# input through jq's environment, so neither appears on any command line. A submission, which isn't -# asked again, may take 60 seconds, Cloud Run's own limit; every other -# request 30. A read that gets no answer, a 429 or any 5xx is asked again -# until the deadline. +# input through jq's environment, so neither appears on any command line. +# A submission, which isn't asked again, may take 60 seconds, Cloud Run's +# own limit; every other request 30. A read that gets no answer, a 429 or +# any 5xx is asked again until the deadline. set -euo pipefail @@ -155,8 +159,9 @@ call_inputs() { # Return 0 when Modal lists container as running in # MODAL_ENVIRONMENT, 1 when it doesn't, and 2 when Modal couldn't list the -# environment's containers as a JSON list, which says nothing about the -# container. +# environment's containers as a JSON list of entries that each have a +# "Container ID", which says nothing about the container: a change in +# Modal's output must not read as the container having gone. container_listed() { local listing listed if ! listing="$(modal_command modal container list --env "${MODAL_ENVIRONMENT}" --json)"; then @@ -166,8 +171,10 @@ container_listed() { fi listed="$( jq -r --arg container_id "$1" \ - 'if type == "array" then any(.[]; .["Container ID"] == $container_id) - else error("not a list") end' <<<"${listing}" 2>/dev/null + 'if type == "array" + and all(.[]; type == "object" and has("Container ID")) + then any(.[]; .["Container ID"] == $container_id) + else error("not a list of containers") end' <<<"${listing}" 2>/dev/null )" || listed="" case "${listed}" in true) return 0 ;; @@ -283,7 +290,10 @@ read_record() { # Find the container running the record's call. Sets container_id once the # call graph names a running input's container that Modal lists in -# MODAL_ENVIRONMENT; until then, unlisted_note may say why not. +# MODAL_ENVIRONMENT. Until then, listing_note says what the last listing +# found out about the call graph's container: that Modal doesn't list it, +# or that Modal couldn't list the containers. Finding the container clears +# it, so a missed attempt's reason names only the last listing's outcome. find_container() { local inputs candidate if ! inputs="$(call_inputs "${record_call_id}")"; then @@ -303,13 +313,12 @@ find_container() { case "${listed_status}" in 0) container_id="${candidate}" - unlisted_note="" listing_note="" printf 'Call %s of %s runs in container %s\n' "${record_call_id}" "${record_label}" "${container_id}" ;; 1) - unlisted_note="container ${candidate} is not listed in Modal environment ${MODAL_ENVIRONMENT}" - printf '%s; asking again\n' "${unlisted_note}" >&2 + listing_note="container ${candidate} is not listed in Modal environment ${MODAL_ENVIRONMENT}" + printf '%s; asking again\n' "${listing_note}" >&2 ;; # Modal couldn't list: the next read asks again. *) listing_note="Modal could not list the containers in environment ${MODAL_ENVIRONMENT}" ;; @@ -377,7 +386,6 @@ attempt_once() { stopped_container_id="" stopped_call_id="" stopped_at_epoch=0 - unlisted_note="" listing_note="" stop_failure="" last_read="nothing yet" @@ -398,7 +406,7 @@ attempt_once() { miss_reason="${record_label} finished after ${stop_failure}" elif [[ -z "${stopped_container_id}" ]]; then attempt_outcome="missed" - miss_reason="${record_label} finished before its container was stopped${unlisted_note:+ (${unlisted_note})}${listing_note:+ (${listing_note})}" + miss_reason="${record_label} finished before its container was stopped${listing_note:+ (${listing_note})}" elif completed_after_the_stop; then confirm_rerun fi diff --git a/projects/policyengine-simulation-entry/DEPLOYMENT.md b/projects/policyengine-simulation-entry/DEPLOYMENT.md index 27486b34d..ee0c89d66 100644 --- a/projects/policyengine-simulation-entry/DEPLOYMENT.md +++ b/projects/policyengine-simulation-entry/DEPLOYMENT.md @@ -175,17 +175,21 @@ starting its workers again. A calculation that finishes before its container is stopped, for example on a warm container, proves nothing, so the check submits a new one, up to -three attempts. Before each new attempt it waits 90 seconds, so Modal stops -the containers the last attempt left idle, the one the check found and the -one Modal's rerun ran in (Modal stops a container left idle for 60 seconds -by default, and these functions don't change that), and the next calculation -starts in a new container. A record that fails, or doesn't finish within its -deadline (10 minutes for the household, 15 for the report, per attempt), -fails the check at once. Modal refuses to stop a container that has ended; -when a stop fails, the check doesn't stop it again or submit another -calculation, but follows the record to its end, so a record left running by -a container that ended for another reason, such as running out of memory, -fails the check at its deadline. Every Modal command may take 60 seconds. +three attempts. Each new attempt starts 90 seconds after the last one ended. +Modal stops a container left idle for 60 seconds by default, and these +functions don't change that. A container that finished the last calculation +unstopped went idle when the record finished, so the wait covers it, and the +next calculation starts in a new container. A container that Modal ran a +stopped call in again goes idle only when that rerun returns, which may be +after the attempt ended, so the wait covers it only when the rerun returns +within about 30 seconds of the attempt's end. A record that fails, or +doesn't finish within its deadline (10 minutes for the household, 15 for the +report, per attempt), fails the check at once. Modal refuses to stop a +container that has ended; when a stop fails, the check doesn't stop it again +or submit another calculation, but follows the record to its end, so a +record left running by a container that ended for another reason, such as +running out of memory, fails the check at its deadline. Every Modal command +may take 60 seconds. It uses the repository's Modal token (the repository secrets `MODAL_TOKEN_ID` and `MODAL_TOKEN_SECRET`), which must be allowed to read From d574a1c2fd274c27d1385127543c363e8f2f22a3 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:41:01 +0400 Subject: [PATCH 24/24] Fix review round 7's quality findings in the preemption check's tests - Test the listing's entry check with non-object entries ([1], [null]) and a list in which only the container's own entry has "Container ID". The [null] and one-renamed-entry cases fail against the script before round 6's fix (b6a68df5); [1] already returned "couldn't list" there. - Rename the miss-reason test to say it keeps the last container listing's outcome, which is what the script promises, and say so in its docstring. - The fixed-wait test's docstring no longer says the wait always covers the container Modal ran a stopped call in again. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_staging_preemption_check.py | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py index fd22105db..a8671b8e6 100644 --- a/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py +++ b/projects/policyengine-simulation-entry/tests/test_staging_preemption_check.py @@ -670,8 +670,23 @@ def test_an_empty_listing_is_asked_again(fakes, tmp_path): @pytest.mark.parametrize( "output", - ["{}", "null", json.dumps([{"ID": CONTAINERS[0]}])], - ids=["object", "null", "renamed-column"], + [ + "{}", + "null", + json.dumps([{"ID": CONTAINERS[0]}]), + "[1]", + "[null]", + # The container's own entry is readable; the other entry isn't. + json.dumps([{"Container ID": CONTAINERS[0]}, {"ID": CONTAINERS[1]}]), + ], + ids=[ + "object", + "null", + "renamed-column", + "number-entry", + "null-entry", + "renamed-column-in-one-entry", + ], ) def test_a_listing_that_is_not_a_list_of_containers_is_asked_again( fakes, tmp_path, output @@ -700,10 +715,14 @@ def test_a_listing_that_is_not_a_list_of_containers_is_asked_again( ], ids=["unlisted-after-a-failure", "failure-after-unlisted"], ) -def test_a_missed_attempts_reason_names_only_the_last_lookups_outcome( +def test_a_missed_attempts_reason_names_only_the_last_listings_outcome( fakes, tmp_path, failed_listings, last_note, earlier_note ): - """Modal lists no containers, after or before a listing that fails.""" + """Modal lists no containers, after or before a listing that fails. + + The reason keeps the last container listing's outcome, even when later + call-graph reads name no container to list. + """ attempt = _household_attempt(0, proves=False) attempt.reads_before_stop = [ _household(JOB_IDS[0], "pending", CALL_IDS[0]), @@ -799,8 +818,10 @@ def _is_submission(call: list[str]) -> bool: def test_the_next_attempt_waits_out_modals_idle_scale_down_first(fakes, tmp_path): """Modal stops a container left idle for 60 seconds, by default. - A new attempt on a warm container the last one left, the one the check - found or the one Modal's rerun ran in, would finish too fast to stop. + A new attempt on a container the last one left warm would finish too + fast to stop, so it starts a fixed wait after the last attempt ended. + The wait covers a container Modal ran a stopped call in again only when + that rerun returned within about 30 seconds of the attempt's end. """ result, _ = _run( fakes,