From 257e9be11df8a20496c3024e843a83a5c1a722c1 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:17:27 +0400 Subject: [PATCH 01/29] Test the restart check's reads and writes for unstarted runs (failing) List the pending runs and household jobs with no Modal call recorded, mark one restart of each, and read a report run's or standalone simulation run's stored inputs back, as the Modal worker role. Co-Authored-By: Claude Opus 5.5 --- .../tests/conftest.py | 12 + .../tests/test_run_restart_records.py | 540 ++++++++++++++++++ 2 files changed, 552 insertions(+) create mode 100644 libs/policyengine-simulation-persistence/tests/test_run_restart_records.py diff --git a/libs/policyengine-simulation-persistence/tests/conftest.py b/libs/policyengine-simulation-persistence/tests/conftest.py index 328c731b3..da9a52c38 100644 --- a/libs/policyengine-simulation-persistence/tests/conftest.py +++ b/libs/policyengine-simulation-persistence/tests/conftest.py @@ -43,6 +43,9 @@ # The role the Public API's privilege manifest manages for the simulation # service; the deployed SIMULATION_SERVICE_DATABASE_URL logs in as it. SERVICE_ROLE = "policyengine_v2_simulation_service" +# The role the Public API's privilege manifest manages for the v2 Modal +# functions; the Modal secret policyengine-modal-worker-database logs in as it. +MODAL_WORKER_ROLE = "policyengine_v2_modal_worker" @dataclass(frozen=True) @@ -140,6 +143,15 @@ def act_as_the_service(connection: Connection) -> None: connection.execute(text(f"SET LOCAL ROLE {SERVICE_ROLE}")) +def act_as_the_modal_worker(connection: Connection) -> None: + """Take the Modal worker's role for the rest of the connection's transaction. + + The restart check connects as this role, from the Modal secret the v2 + functions and the stuck-run sweep use. + """ + connection.execute(text(f"SET LOCAL ROLE {MODAL_WORKER_ROLE}")) + + @contextmanager def connection_as_the_service(database_url: str) -> Iterator[Connection]: """Yield a connection acting as the service's role, rolled back afterwards.""" diff --git a/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py b/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py new file mode 100644 index 000000000..fddb135a8 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py @@ -0,0 +1,540 @@ +"""The restart check's reads and writes for runs whose Modal call never started. + +These run against a database the Public API's own v2 migration chain built +(``PUBLIC_API_SCHEMA_DATABASE_URL``), as the Modal worker role the restart +check connects as, each in a transaction rolled back afterwards. The rows are +inserted first, as the superuser, with the simulation service's own writes. +Every row a test inserts shares the transaction's start time as its +``created_at``, so each listing names its window around that time. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING +from uuid import UUID, uuid4 + +import pytest +from conftest import act_as_the_modal_worker, rolled_back_transaction +from economy_rows import ( + CURRENT_LAW, + VERSIONS, + EconomyCatalog, + insert_economy_catalog, + insert_reusable_run, +) +from policyengine_simulation_persistence import ( + NewEconomyReport, + NewEconomySimulation, + NewHouseholdJob, + ReportRunTrigger, + ReportSimulation, + RestartKind, + RunRestartStore, + SimulationChoice, + SimulationType, + StoredRunUnreadableError, + UnstartedRun, + budget_window_runs, + insert_economy_report, + insert_economy_simulation, + insert_household_job, + list_unstarted_runs, + mark_restart, + read_unstarted_report_run, + read_unstarted_simulation_run, + record_household_call, + record_report_run_call, + report_runs, + simulation_runs, + simulations, +) +from public_api_rows import ( + StoredValue, + insert_dynamic, + insert_household, + insert_policy, +) +from sqlalchemy import func, insert, select, text, update + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sqlalchemy import Connection + +pytestmark = pytest.mark.public_api_schema + +START = datetime(2026, 1, 1, tzinfo=UTC) +CTC_AMOUNT = "gov.irs.credits.ctc.amount.base[0].amount" +ELASTICITY = "gov.simulation.labor_supply_responses.elasticities.income" +REPORT_OPTIONS = {"include_cliffs": True, "spm": {"geography_kind": "national"}} +HOUSEHOLD_DOCUMENT = { + "people": [ + { + "id": "you", + "values": {"age": 40}, + "memberships": {"household": "your household"}, + } + ], + "household": [{"id": "your household", "values": {}}], +} +LISTING_LIMIT = 100 + + +def _transaction_start(connection: Connection) -> datetime: + """Return the time every row the test inserts records as its creation.""" + return connection.execute(select(func.now())).scalar_one() + + +def _listed( + connection: Connection, *, around: datetime, limit: int = LISTING_LIMIT +) -> tuple[UnstartedRun, ...]: + """List the unstarted runs created within a second of ``around``.""" + return list_unstarted_runs( + connection, + created_after=around - timedelta(seconds=1), + created_before=around + timedelta(seconds=1), + limit=limit, + ) + + +def _new_side(choice: SimulationChoice = CURRENT_LAW) -> ReportSimulation: + return ReportSimulation( + simulation_id=uuid4(), simulation_run_id=uuid4(), choice=choice, reused=False + ) + + +def _new_report( + economy: EconomyCatalog, + *, + baseline: ReportSimulation | None = None, + reform: ReportSimulation | None = None, +) -> NewEconomyReport: + return NewEconomyReport( + id=uuid4(), + run_id=uuid4(), + country=economy.catalog.country, + label="Child tax credit", + description=None, + tax_benefit_model_id=economy.catalog.model_id, + population=economy.population(), + versions=VERSIONS, + inputs=REPORT_OPTIONS, + baseline=baseline or _new_side(), + reform=reform or _new_side(), + ) + + +def _new_simulation( + economy: EconomyCatalog, choice: SimulationChoice = CURRENT_LAW +) -> NewEconomySimulation: + return NewEconomySimulation( + id=uuid4(), + run_id=uuid4(), + choice=choice, + population=economy.population(), + versions=VERSIONS, + ) + + +def _insert_household_job(connection: Connection, economy: EconomyCatalog) -> UUID: + household_id = insert_household( + connection, country="us", default_year=2026, document=HOUSEHOLD_DOCUMENT + ) + job_id = uuid4() + insert_household_job( + connection, + NewHouseholdJob( + id=job_id, + country="us", + household_id=household_id, + policy_id=None, + year=2026, + request_data={"country": "us", "household_id": str(household_id)}, + tax_benefit_model_version_id=economy.catalog.version_id, + country_package_version=VERSIONS.country_package_version, + policyengine_version=VERSIONS.policyengine_version, + ), + ) + return job_id + + +@pytest.fixture +def connection(public_api_database_url) -> Iterator[Connection]: + """Yield the superuser's connection in a transaction rolled back afterwards.""" + with rolled_back_transaction(public_api_database_url) as superuser_connection: + yield superuser_connection + + +def test_an_unstarted_run_of_each_kind_is_listed_and_a_reports_sides_are_not( + connection, +) -> None: + economy = insert_economy_catalog(connection) + report = _new_report(economy) + insert_economy_report(connection, report) + simulation = _new_simulation(economy) + insert_economy_simulation(connection, simulation) + job_id = _insert_household_job(connection, economy) + created_at = _transaction_start(connection) + act_as_the_modal_worker(connection) + + listed = _listed(connection, around=created_at) + + assert {(run.kind, run.run_id) for run in listed} == { + (RestartKind.REPORT_RUN, report.run_id), + (RestartKind.SIMULATION_RUN, simulation.run_id), + (RestartKind.HOUSEHOLD_JOB, job_id), + } + assert {run.created_at for run in listed} == {created_at} + + +def test_a_run_created_outside_the_window_is_not_listed(connection) -> None: + economy = insert_economy_catalog(connection) + insert_economy_simulation(connection, _new_simulation(economy)) + created_at = _transaction_start(connection) + act_as_the_modal_worker(connection) + + too_new = list_unstarted_runs( + connection, + created_after=created_at - timedelta(minutes=12), + created_before=created_at - timedelta(minutes=3), + limit=LISTING_LIMIT, + ) + too_old = list_unstarted_runs( + connection, + created_after=created_at + timedelta(minutes=3), + created_before=created_at + timedelta(minutes=12), + limit=LISTING_LIMIT, + ) + + assert too_new == () + assert too_old == () + + +def test_the_listing_takes_the_oldest_runs_up_to_its_limit(connection) -> None: + economy = insert_economy_catalog(connection) + for _ in range(3): + insert_economy_simulation(connection, _new_simulation(economy)) + created_at = _transaction_start(connection) + act_as_the_modal_worker(connection) + + assert len(_listed(connection, around=created_at, limit=2)) == 2 + + +def test_a_run_with_a_call_or_no_longer_pending_is_not_listed(connection) -> None: + economy = insert_economy_catalog(connection) + called = _new_report(economy) + insert_economy_report(connection, called) + record_report_run_call( + connection, report_run_id=called.run_id, modal_call_id="fc-original" + ) + claimed = _new_simulation(economy) + insert_economy_simulation(connection, claimed) + connection.execute( + update(simulation_runs) + .where(simulation_runs.c.id == claimed.run_id) + .values(status="running", started_at=func.now()) + ) + job_id = _insert_household_job(connection, economy) + record_household_call(connection, job_id=job_id, modal_call_id="fc-household") + created_at = _transaction_start(connection) + act_as_the_modal_worker(connection) + + listed = _listed(connection, around=created_at) + + # The called report run's sides stay named by a pending report run. + assert listed == () + + +def test_a_marked_run_is_marked_once_and_never_listed_again(connection) -> None: + economy = insert_economy_catalog(connection) + simulation = _new_simulation(economy) + insert_economy_simulation(connection, simulation) + created_at = _transaction_start(connection) + act_as_the_modal_worker(connection) + (unstarted,) = _listed(connection, around=created_at) + + first = mark_restart(connection, unstarted) + second = mark_restart(connection, unstarted) + + assert (first, second) == (True, False) + assert _listed(connection, around=created_at) == () + row = connection.execute( + select(simulation_runs.c.status, simulation_runs.c.modal_call_id).where( + simulation_runs.c.id == simulation.run_id + ) + ).one() + assert (row.status, row.modal_call_id) == ("pending", None) + + +def test_a_run_given_a_call_after_it_was_listed_is_not_marked(connection) -> None: + economy = insert_economy_catalog(connection) + report = _new_report(economy) + insert_economy_report(connection, report) + created_at = _transaction_start(connection) + act_as_the_modal_worker(connection) + (unstarted,) = _listed(connection, around=created_at) + record_report_run_call( + connection, report_run_id=report.run_id, modal_call_id="fc-late" + ) + + assert mark_restart(connection, unstarted) is False + + +def test_each_kind_is_marked_in_its_own_table(connection) -> None: + economy = insert_economy_catalog(connection) + report = _new_report(economy) + insert_economy_report(connection, report) + _insert_household_job(connection, economy) + created_at = _transaction_start(connection) + act_as_the_modal_worker(connection) + + marked = { + run.kind: mark_restart(connection, run) + for run in _listed(connection, around=created_at) + } + + assert marked == {RestartKind.REPORT_RUN: True, RestartKind.HOUSEHOLD_JOB: True} + assert _listed(connection, around=created_at) == () + + +def test_a_budget_window_year_is_not_listed(connection) -> None: + economy = insert_economy_catalog(connection) + report = _new_report(economy) + insert_economy_report(connection, report) + window_run_id = uuid4() + connection.execute( + insert(budget_window_runs).values( + id=window_run_id, + report_id=report.id, + tax_benefit_model_version_id=economy.catalog.version_id, + trigger=ReportRunTrigger.INITIAL, + recalculate=False, + status="pending", + ) + ) + connection.execute( + update(report_runs) + .where(report_runs.c.id == report.run_id) + .values(budget_window_run_id=window_run_id, year=2026) + ) + # The update above changes the year's updated_at; put it back to its + # creation, as a year the service creates with its window stays. + connection.execute( + update(report_runs) + .where(report_runs.c.id == report.run_id) + .values(updated_at=report_runs.c.created_at) + ) + created_at = _transaction_start(connection) + act_as_the_modal_worker(connection) + + assert _listed(connection, around=created_at) == () + + +def test_a_household_simulations_run_and_report_run_are_not_listed( + connection, +) -> None: + economy = insert_economy_catalog(connection) + report = _new_report(economy) + insert_economy_report(connection, report) + simulation = _new_simulation(economy) + insert_economy_simulation(connection, simulation) + household_id = insert_household( + connection, country="us", default_year=2026, document=HOUSEHOLD_DOCUMENT + ) + # Make every simulation a household simulation, as C1 ยท 15's household + # reports and household simulations will be. + for simulation_id in ( + report.baseline.simulation_id, + report.reform.simulation_id, + simulation.id, + ): + connection.execute( + update(simulations) + .where(simulations.c.id == simulation_id) + .values( + simulation_type=SimulationType.HOUSEHOLD, + dataset_id=None, + household_id=household_id, + ) + ) + created_at = _transaction_start(connection) + act_as_the_modal_worker(connection) + + assert _listed(connection, around=created_at) == () + + +def test_a_report_runs_stored_inputs_are_read_back(connection) -> None: + economy = insert_economy_catalog( + connection, parameter_names=(CTC_AMOUNT, ELASTICITY) + ) + policy_id = insert_policy( + connection, economy.catalog, [StoredValue(CTC_AMOUNT, 3_000, START)] + ) + dynamic_id = insert_dynamic( + connection, economy.catalog, [StoredValue(ELASTICITY, -0.05, START)] + ) + reform_choice = SimulationChoice(policy_id=policy_id, dynamic_id=dynamic_id) + reused = insert_reusable_run(connection, economy) + report = _new_report( + economy, + baseline=ReportSimulation( + simulation_id=reused.simulation_id, + simulation_run_id=reused.simulation_run_id, + choice=CURRENT_LAW, + reused=True, + ), + reform=_new_side(reform_choice), + ) + insert_economy_report(connection, report) + act_as_the_modal_worker(connection) + + stored = read_unstarted_report_run(connection, report.run_id) + + assert stored is not None + assert stored.report_run_id == report.run_id + assert stored.country == "us" + assert stored.versions == VERSIONS + assert stored.options == REPORT_OPTIONS + assert stored.scope.catalog_version_id == economy.catalog.version_id + assert stored.scope.region_code == "us" + assert stored.scope.dataset_name == "populace_us_2024" + assert stored.scope.year == 2026 + assert stored.baseline.simulation_run_id == reused.simulation_run_id + assert stored.baseline.choice == CURRENT_LAW + assert stored.baseline.reform == {} + assert stored.reform.simulation_run_id == report.reform.simulation_run_id + assert stored.reform.choice == reform_choice + assert stored.reform.reform == { + CTC_AMOUNT: {"2026-01-01.2200-12-31": 3_000}, + ELASTICITY: {"2026-01-01.2200-12-31": -0.05}, + } + assert stored.created_at == _transaction_start(connection) + + +def test_a_standalone_runs_stored_inputs_are_read_back(connection) -> None: + economy = insert_economy_catalog(connection, parameter_names=(CTC_AMOUNT,)) + policy_id = insert_policy( + connection, economy.catalog, [StoredValue(CTC_AMOUNT, 2_500, START)] + ) + choice = SimulationChoice(policy_id=policy_id, dynamic_id=None) + simulation = _new_simulation(economy, choice) + insert_economy_simulation(connection, simulation) + act_as_the_modal_worker(connection) + + stored = read_unstarted_simulation_run(connection, simulation.run_id) + + assert stored is not None + assert stored.country == "us" + assert stored.versions == VERSIONS + assert stored.scope.catalog_version_id == economy.catalog.version_id + assert (stored.scope.region_code, stored.scope.dataset_name) == ( + "us", + "populace_us_2024", + ) + assert stored.scope.year == 2026 + assert stored.run.simulation_run_id == simulation.run_id + assert stored.run.choice == choice + assert stored.run.reform == {CTC_AMOUNT: {"2026-01-01.2200-12-31": 2_500}} + assert stored.created_at == _transaction_start(connection) + + +def test_a_missing_run_reads_as_none(connection) -> None: + act_as_the_modal_worker(connection) + + assert read_unstarted_report_run(connection, uuid4()) is None + assert read_unstarted_simulation_run(connection, uuid4()) is None + + +def test_a_run_whose_region_is_gone_cannot_be_read(connection) -> None: + economy = insert_economy_catalog(connection) + simulation = _new_simulation(economy) + insert_economy_simulation(connection, simulation) + connection.execute( + update(simulation_runs) + .where(simulation_runs.c.id == simulation.run_id) + .values(region_id=None) + ) + act_as_the_modal_worker(connection) + + with pytest.raises(StoredRunUnreadableError, match="region"): + read_unstarted_simulation_run(connection, simulation.run_id) + + +def test_a_report_run_whose_sides_differ_cannot_be_read(connection) -> None: + economy = insert_economy_catalog(connection) + report = _new_report(economy) + insert_economy_report(connection, report) + connection.execute( + update(simulation_runs) + .where(simulation_runs.c.id == report.reform.simulation_run_id) + .values(year=2027) + ) + act_as_the_modal_worker(connection) + + with pytest.raises(StoredRunUnreadableError, match="sides"): + read_unstarted_report_run(connection, report.run_id) + + +@pytest.fixture +def savepoint_store(connection) -> RunRestartStore: + """Return a store whose every transaction is a savepoint of ``connection``.""" + + @contextmanager + def transaction() -> Iterator[Connection]: + with connection.begin_nested(): + yield connection + + return RunRestartStore(transaction) + + +def test_the_store_lists_marks_and_reads_in_transactions_of_its_own( + connection, savepoint_store +) -> None: + economy = insert_economy_catalog(connection) + report = _new_report(economy) + insert_economy_report(connection, report) + simulation = _new_simulation(economy) + insert_economy_simulation(connection, simulation) + created_at = _transaction_start(connection) + act_as_the_modal_worker(connection) + + listed = savepoint_store.list_unstarted( + created_after=created_at - timedelta(seconds=1), + created_before=created_at + timedelta(seconds=1), + limit=LISTING_LIMIT, + ) + by_kind = {run.kind: run for run in listed} + report_run = savepoint_store.read_report_run(report.run_id) + simulation_run = savepoint_store.read_simulation_run(simulation.run_id) + marked = savepoint_store.mark_restart(by_kind[RestartKind.REPORT_RUN]) + + assert set(by_kind) == {RestartKind.REPORT_RUN, RestartKind.SIMULATION_RUN} + assert report_run is not None + assert report_run.report_run_id == report.run_id + assert simulation_run is not None + assert simulation_run.run.simulation_run_id == simulation.run_id + assert marked is True + + +def test_the_restart_writes_need_only_the_modal_worker_roles_privileges( + connection, +) -> None: + economy = insert_economy_catalog(connection) + report = _new_report(economy) + insert_economy_report(connection, report) + created_at = _transaction_start(connection) + act_as_the_modal_worker(connection) + (unstarted,) = _listed(connection, around=created_at) + + assert mark_restart(connection, unstarted) is True + assert ( + record_report_run_call( + connection, report_run_id=report.run_id, modal_call_id="fc-restarted" + ) + == "pending" + ) + assert connection.execute(text("SELECT current_user")).scalar_one() == ( + "policyengine_v2_modal_worker" + ) From 8ad78f6358ae7add849abe39a9975e3f5a99ad32 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:19:33 +0400 Subject: [PATCH 02/29] List, mark and read back runs whose Modal call never started The restart check lists the household jobs, report runs and standalone simulation runs still pending with no Modal call that nothing has changed since their insert, marks each one before restarting it so it is restarted at most once, and reads back the stored rows its call is built from, against the run's own catalog version. Co-Authored-By: Claude Opus 5.5 --- .../__init__.py | 28 ++ .../run_restart_records.py | 419 ++++++++++++++++++ .../run_restart_store.py | 85 ++++ 3 files changed, 532 insertions(+) create mode 100644 libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py create mode 100644 libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_store.py diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py index 1741cf7b0..e1c2048a0 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py @@ -86,6 +86,21 @@ read_dynamic_reform, read_policy_reform, ) +from policyengine_simulation_persistence.run_restart_records import ( + RESTART_TABLES, + EconomyScope, + RestartKind, + StoredEconomySide, + StoredReportRun, + StoredRunUnreadableError, + StoredSimulationRun, + UnstartedRun, + list_unstarted_runs, + mark_restart, + read_unstarted_report_run, + read_unstarted_simulation_run, +) +from policyengine_simulation_persistence.run_restart_store import RunRestartStore from policyengine_simulation_persistence.tables import ( HouseholdJobStatus, ReportRunStatus, @@ -103,6 +118,7 @@ ) __all__ = [ + "RESTART_TABLES", "SERVICE_TABLE_PRIVILEGES", "SERVICE_UPDATABLE_COLUMNS", "CatalogDataset", @@ -115,6 +131,7 @@ "EconomyReportState", "EconomyRequest", "EconomyRunStore", + "EconomyScope", "EconomySimulationInputs", "EconomySimulationState", "HouseholdCalculationInputs", @@ -134,7 +151,9 @@ "ReportSide", "ReportSideError", "ReportSimulation", + "RestartKind", "ReusableRun", + "RunRestartStore", "RunReuse", "RunVersions", "SideRefusal", @@ -143,13 +162,18 @@ "SimulationRunState", "SimulationStatus", "SimulationType", + "StoredEconomySide", "StoredHousehold", + "StoredReportRun", + "StoredRunUnreadableError", + "StoredSimulationRun", "UnknownCatalogVersionError", "UnknownDatasetError", "UnknownDynamicError", "UnknownHouseholdError", "UnknownPolicyError", "UnknownRegionError", + "UnstartedRun", "bounded_transaction", "budget_window_runs", "combine_reforms", @@ -163,6 +187,8 @@ "insert_economy_report", "insert_economy_simulation", "insert_household_job", + "list_unstarted_runs", + "mark_restart", "missing_table_privileges", "normalized_database_url", "read_catalog_version_id", @@ -177,6 +203,8 @@ "read_policy_reform", "read_region", "read_stored_household", + "read_unstarted_report_run", + "read_unstarted_simulation_run", "record_household_call", "record_report_run_call", "record_simulation_run_call", diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py new file mode 100644 index 000000000..98f5f36bb --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py @@ -0,0 +1,419 @@ +"""The restart check's reads and writes, for runs whose Modal call never started. + +The simulation service creates a run or household job ``pending`` with no +Modal call ID, starts its call, then records the call's ID. When the service +stops between the insert and the start, nothing ever records a call, and the +stuck-run sweep would fail the run. The restart check, a scheduled Modal +function, starts such a run's call once more: + +1. ``list_unstarted_runs`` lists the rows still pending with no call that were + created within a window and that nothing has changed since their insert + (``updated_at`` still equals ``created_at``): household jobs, report runs, + and standalone simulation runs, which no report run names. A report run's + own simulation runs are its coordinator's to start. Budget-window years + and household simulations are left out: their own tasks restart them. +2. ``mark_restart`` sets the row's ``updated_at`` before the check starts its + call, so the row is never listed again and is restarted at most once. It + uses the statement's clock rather than the transaction's, so the mark + differs from a creation time in the same transaction too. +3. ``read_unstarted_report_run`` and ``read_unstarted_simulation_run`` read + back what the service built the call from: each side's simulation and + values, read against the run's own catalog version, the region, dataset + and year the run records, and the report's options. + +The check then records its call's ID, or fails the row, with the service's +own conditional writes (``record_report_run_call`` and the rest). Every +function here takes a connection, so the caller chooses the transactions, +and every update sets only columns both the service's and the Modal worker's +roles may update. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum +from typing import TYPE_CHECKING, Any, Final + +from sqlalchemy import exists, func, or_, select, update + +from policyengine_simulation_persistence.economy_run_records import ( + RunVersions, + SimulationChoice, +) +from policyengine_simulation_persistence.input_tables import ( + MODEL_NAME_PREFIX, + datasets, + regions, + tax_benefit_model_versions, + tax_benefit_models, +) +from policyengine_simulation_persistence.policy_reform import ( + combine_reforms, + read_dynamic_reform, + read_policy_reform, +) +from policyengine_simulation_persistence.tables import ( + SimulationStatus, + SimulationType, + household_jobs, + report_runs, + reports, + simulation_runs, + simulations, +) + +if TYPE_CHECKING: + from datetime import datetime + from uuid import UUID + + from sqlalchemy import ColumnElement, Connection, Row, Select, Table + + from policyengine_simulation_persistence.policy_reform import PolicyReform + +# Every restartable table's status uses this value for a row not yet claimed. +_PENDING: Final = SimulationStatus.PENDING.value + + +class RestartKind(StrEnum): + """What kind of row the restart check starts again.""" + + HOUSEHOLD_JOB = "household_job" + REPORT_RUN = "report_run" + SIMULATION_RUN = "simulation_run" + + +RESTART_TABLES: Final[dict[RestartKind, Table]] = { + RestartKind.HOUSEHOLD_JOB: household_jobs, + RestartKind.REPORT_RUN: report_runs, + RestartKind.SIMULATION_RUN: simulation_runs, +} + + +@dataclass(frozen=True) +class UnstartedRun: + """A row still pending with no Modal call, and when it was created.""" + + kind: RestartKind + run_id: UUID + created_at: datetime + + +class StoredRunUnreadableError(LookupError): + """A run's stored rows no longer describe one calculation to start.""" + + +@dataclass(frozen=True) +class EconomyScope: + """What every simulation of an economy run calculates on, as stored. + + ``catalog_version_id`` is the catalog version the run was created on, and + its region and dataset are named by code and name, as the worker reads + them. + """ + + catalog_version_id: UUID + region_code: str + dataset_name: str + year: int + + +@dataclass(frozen=True) +class StoredEconomySide: + """One simulation run of an economy run, with its simulation's values. + + ``reform`` holds the simulation's policy and dynamic values together, by + parameter name and period, read against the run's catalog version. + """ + + simulation_run_id: UUID + choice: SimulationChoice + reform: PolicyReform + + +@dataclass(frozen=True) +class StoredReportRun: + """A report run as its coordinator's call was built from it. + + ``options`` are the report's (``reports.inputs``), and ``versions`` those + the report run records, which choose the worker that calculates it. + """ + + report_run_id: UUID + country: str + versions: RunVersions + options: dict[str, Any] + scope: EconomyScope + baseline: StoredEconomySide + reform: StoredEconomySide + created_at: datetime + + +@dataclass(frozen=True) +class StoredSimulationRun: + """A standalone simulation run as its simulation function's call was built.""" + + country: str + versions: RunVersions + scope: EconomyScope + run: StoredEconomySide + created_at: datetime + + +def _unstarted(table: Table) -> tuple[ColumnElement[bool], ...]: + """Return the conditions that a row is pending, with no call, never changed.""" + return ( + table.c.status == _PENDING, + table.c.modal_call_id.is_(None), + table.c.updated_at == table.c.created_at, + ) + + +def _economy_simulation_run( + simulation_run_id: ColumnElement[Any], +) -> ColumnElement[bool]: + """Return the condition that the simulation run is an economy simulation's.""" + return exists().where( + simulations.c.id == simulation_runs.c.simulation_id, + simulation_runs.c.id == simulation_run_id, + simulations.c.simulation_type == SimulationType.ECONOMY, + ) + + +def _restart_candidates(kind: RestartKind) -> Select[Any]: + """Return the query for one kind's unstarted rows, before the window.""" + table = RESTART_TABLES[kind] + statement = select(table.c.id, table.c.created_at).where(*_unstarted(table)) + if kind is RestartKind.REPORT_RUN: + return statement.where( + report_runs.c.budget_window_run_id.is_(None), + _economy_simulation_run(report_runs.c.baseline_simulation_run_id), + ) + if kind is RestartKind.SIMULATION_RUN: + naming_report_run = report_runs.alias("naming_report_run") + return statement.where( + exists().where( + simulations.c.id == simulation_runs.c.simulation_id, + simulations.c.simulation_type == SimulationType.ECONOMY, + ), + ~exists().where( + or_( + naming_report_run.c.baseline_simulation_run_id + == simulation_runs.c.id, + naming_report_run.c.reform_simulation_run_id + == simulation_runs.c.id, + ) + ), + ) + return statement + + +def list_unstarted_runs( + connection: Connection, + *, + created_after: datetime, + created_before: datetime, + limit: int, +) -> tuple[UnstartedRun, ...]: + """Return the oldest restartable rows created in the window, at most ``limit``. + + A row is restartable while it is pending with no Modal call recorded and + nothing has changed it since its insert. Report runs and standalone + simulation runs are listed only for economy simulations, and report runs + only outside a budget window. + """ + listed: list[UnstartedRun] = [] + for kind, table in RESTART_TABLES.items(): + statement = ( + _restart_candidates(kind) + .where( + table.c.created_at > created_after, + table.c.created_at <= created_before, + ) + .order_by(table.c.created_at, table.c.id) + .limit(limit) + ) + listed.extend( + UnstartedRun(kind=kind, run_id=row.id, created_at=row.created_at) + for row in connection.execute(statement) + ) + listed.sort(key=lambda run: (run.created_at, run.run_id)) + return tuple(listed[:limit]) + + +def mark_restart(connection: Connection, run: UnstartedRun) -> bool: + """Mark the row as restarted while it is still unstarted; return whether it was. + + A row given a call, claimed, finished or already marked since it was + listed is left as it is, so two checks at once restart it only once. + """ + table = RESTART_TABLES[run.kind] + marked = connection.execute( + update(table) + .where(table.c.id == run.run_id, *_unstarted(table)) + .values(updated_at=func.clock_timestamp()) + .returning(table.c.id) + ).scalar_one_or_none() + return marked is not None + + +def _side_rows(connection: Connection, simulation_run_id: UUID) -> Row[Any] | None: + """Return a simulation run's scope and its simulation's policy and dynamic.""" + return connection.execute( + select( + simulation_runs.c.id, + simulation_runs.c.created_at, + simulation_runs.c.year, + simulation_runs.c.tax_benefit_model_version_id, + simulation_runs.c.country_package_version, + simulation_runs.c.policyengine_version, + simulations.c.policy_id, + simulations.c.dynamic_id, + regions.c.code.label("region_code"), + datasets.c.name.label("dataset_name"), + tax_benefit_models.c.name.label("model_name"), + ) + .join(simulations, simulations.c.id == simulation_runs.c.simulation_id) + .join( + tax_benefit_model_versions, + tax_benefit_model_versions.c.id + == simulation_runs.c.tax_benefit_model_version_id, + ) + .join( + tax_benefit_models, + tax_benefit_models.c.id == tax_benefit_model_versions.c.model_id, + ) + .outerjoin(regions, regions.c.id == simulation_runs.c.region_id) + .outerjoin(datasets, datasets.c.id == simulation_runs.c.dataset_id) + .where(simulation_runs.c.id == simulation_run_id) + ).one_or_none() + + +def _scope(side: Row[Any]) -> EconomyScope: + """Return what the side calculates on; raise when its region or dataset is gone.""" + if side.region_code is None: + msg = f"simulation run {side.id} names no region" + raise StoredRunUnreadableError(msg) + if side.dataset_name is None: + msg = f"simulation run {side.id} names no dataset" + raise StoredRunUnreadableError(msg) + return EconomyScope( + catalog_version_id=side.tax_benefit_model_version_id, + region_code=side.region_code, + dataset_name=side.dataset_name, + year=side.year, + ) + + +def _side( + connection: Connection, country: str, scope: EconomyScope, side: Row[Any] +) -> StoredEconomySide: + """Read the side's policy and dynamic values against the scope's catalog version. + + Raise the policy reader's errors, or OverlappingReformError. + """ + choice = SimulationChoice(policy_id=side.policy_id, dynamic_id=side.dynamic_id) + policy_reform = ( + {} + if choice.policy_id is None + else read_policy_reform( + connection, + country=country, + policy_id=choice.policy_id, + catalog_version_id=scope.catalog_version_id, + ) + ) + dynamic_reform = ( + {} + if choice.dynamic_id is None + else read_dynamic_reform( + connection, + dynamic_id=choice.dynamic_id, + catalog_version_id=scope.catalog_version_id, + ) + ) + return StoredEconomySide( + simulation_run_id=side.id, + choice=choice, + reform=combine_reforms(policy_reform, dynamic_reform), + ) + + +def read_unstarted_report_run( + connection: Connection, report_run_id: UUID +) -> StoredReportRun | None: + """Return what the report run's coordinator call is built from, or None. + + Raise StoredRunUnreadableError when the report run names no reform, a + side's run is missing or names no region or dataset, or its sides differ + in catalog version, region, dataset or year; and the policy reader's + errors when a side's values can't be read. + """ + run = connection.execute( + select( + report_runs.c.created_at, + report_runs.c.baseline_simulation_run_id, + report_runs.c.reform_simulation_run_id, + report_runs.c.country_package_version, + report_runs.c.policyengine_version, + reports.c.country, + reports.c.inputs, + ) + .join(reports, reports.c.id == report_runs.c.report_id) + .where(report_runs.c.id == report_run_id) + ).one_or_none() + if run is None: + return None + if run.reform_simulation_run_id is None: + msg = f"report run {report_run_id} names no reform simulation run" + raise StoredRunUnreadableError(msg) + baseline_rows = _side_rows(connection, run.baseline_simulation_run_id) + reform_rows = _side_rows(connection, run.reform_simulation_run_id) + if baseline_rows is None or reform_rows is None: + msg = f"report run {report_run_id} names a missing simulation run" + raise StoredRunUnreadableError(msg) + scope = _scope(baseline_rows) + if _scope(reform_rows) != scope: + msg = ( + f"report run {report_run_id}'s sides calculate on different catalog " + "versions, regions, datasets or years" + ) + raise StoredRunUnreadableError(msg) + return StoredReportRun( + report_run_id=report_run_id, + country=run.country, + versions=RunVersions( + country_package_version=run.country_package_version, + policyengine_version=run.policyengine_version, + ), + options=dict(run.inputs), + scope=scope, + baseline=_side(connection, run.country, scope, baseline_rows), + reform=_side(connection, run.country, scope, reform_rows), + created_at=run.created_at, + ) + + +def read_unstarted_simulation_run( + connection: Connection, simulation_run_id: UUID +) -> StoredSimulationRun | None: + """Return what a standalone run's simulation function call is built from. + + Return None when there is no such run. Raise StoredRunUnreadableError + when it names no region or dataset, and the policy reader's errors when + its values can't be read. + """ + side_rows = _side_rows(connection, simulation_run_id) + if side_rows is None: + return None + country = str(side_rows.model_name).removeprefix(MODEL_NAME_PREFIX) + scope = _scope(side_rows) + return StoredSimulationRun( + country=country, + versions=RunVersions( + country_package_version=side_rows.country_package_version, + policyengine_version=side_rows.policyengine_version, + ), + scope=scope, + run=_side(connection, country, scope, side_rows), + created_at=side_rows.created_at, + ) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_store.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_store.py new file mode 100644 index 000000000..f7005c662 --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_store.py @@ -0,0 +1,85 @@ +"""The restart check's store: each read and write in a transaction of its own. + +The restart check lists the runs and household jobs whose Modal call never +started, marks each one it restarts, and reads back what its call is built +from, each in its own transaction, with Modal calls started between them. +``RunRestartStore`` runs ``run_restart_records``' reads and writes that way. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from policyengine_simulation_persistence.engine import bounded_transaction +from policyengine_simulation_persistence.run_restart_records import ( + list_unstarted_runs, + mark_restart, + read_unstarted_report_run, + read_unstarted_simulation_run, +) + +if TYPE_CHECKING: + from collections.abc import Callable + from contextlib import AbstractContextManager + from datetime import datetime + from uuid import UUID + + from sqlalchemy import Connection, Engine + + from policyengine_simulation_persistence.run_restart_records import ( + StoredReportRun, + StoredSimulationRun, + UnstartedRun, + ) + +type TransactionFactory = Callable[[], AbstractContextManager[Connection]] + + +class RunRestartStore: + """List, mark and read unstarted runs, each operation in its own transaction.""" + + def __init__(self, transaction: TransactionFactory) -> None: + """Open each transaction with ``transaction``, which commits on success.""" + self._transaction = transaction + + @classmethod + def from_engine( + cls, engine: Engine, *, statement_timeout_milliseconds: int + ) -> RunRestartStore: + """Use ``engine``, bounding each statement to the given time.""" + + def transaction() -> AbstractContextManager[Connection]: + return bounded_transaction( + engine, statement_timeout_milliseconds=statement_timeout_milliseconds + ) + + return cls(transaction) + + def list_unstarted( + self, *, created_after: datetime, created_before: datetime, limit: int + ) -> tuple[UnstartedRun, ...]: + """Return the oldest restartable rows created in the window; see the reader.""" + with self._transaction() as connection: + return list_unstarted_runs( + connection, + created_after=created_after, + created_before=created_before, + limit=limit, + ) + + def mark_restart(self, run: UnstartedRun) -> bool: + """Mark the row restarted while it is still unstarted; return whether it was.""" + with self._transaction() as connection: + return mark_restart(connection, run) + + def read_report_run(self, report_run_id: UUID) -> StoredReportRun | None: + """Return what the report run's call is built from, or None.""" + with self._transaction() as connection: + return read_unstarted_report_run(connection, report_run_id) + + def read_simulation_run( + self, simulation_run_id: UUID + ) -> StoredSimulationRun | None: + """Return what the standalone simulation run's call is built from, or None.""" + with self._transaction() as connection: + return read_unstarted_simulation_run(connection, simulation_run_id) From a95c14b8ff72369544dac84b69d3649065a2edda Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:24:13 +0400 Subject: [PATCH 03/29] Test restarting runs and household jobs whose call never started (failing) The restart rebuilds each run's call from its stored rows and starts the same call its submission started, on the worker of the run's own .py version; a run that can't be rebuilt, or whose restart Modal refuses, is failed with run_not_started, and each listed run is one telemetry scope with its outcome. Co-Authored-By: Claude Opus 5.5 --- .../tests/economy_service_fakes.py | 3 +- .../tests/test_economy_workers.py | 30 + .../tests/test_household_workers.py | 29 + .../tests/test_run_restarts.py | 783 ++++++++++++++++++ 4 files changed, 844 insertions(+), 1 deletion(-) create mode 100644 projects/policyengine-simulation-entry/tests/test_run_restarts.py diff --git a/projects/policyengine-simulation-entry/tests/economy_service_fakes.py b/projects/policyengine-simulation-entry/tests/economy_service_fakes.py index 535873f0f..8becd4e9e 100644 --- a/projects/policyengine-simulation-entry/tests/economy_service_fakes.py +++ b/projects/policyengine-simulation-entry/tests/economy_service_fakes.py @@ -43,6 +43,7 @@ if TYPE_CHECKING: from collections.abc import Sequence + from policyengine_simulation_entry.economy_workers import EconomyWorkerSource from policyengine_simulation_persistence import ( EconomyRequest, NewEconomyReport, @@ -296,7 +297,7 @@ def numbered_ids(): def service( store: FakeStore, *, - workers: FakeWorkers | None = None, + workers: EconomyWorkerSource | None = None, caller: FakeCaller | None = None, output_bucket: str | None = OUTPUT_BUCKET, ) -> EconomySubmissionService: diff --git a/projects/policyengine-simulation-entry/tests/test_economy_workers.py b/projects/policyengine-simulation-entry/tests/test_economy_workers.py index 9192e4f27..38632d558 100644 --- a/projects/policyengine-simulation-entry/tests/test_economy_workers.py +++ b/projects/policyengine-simulation-entry/tests/test_economy_workers.py @@ -194,3 +194,33 @@ def test_a_call_modal_gives_no_id_is_an_error(modal_functions, object_id) -> Non worker, {}, {}, observability_context=None ) ) + + +def test_the_worker_for_a_runs_version_is_found_when_it_is_not_the_default() -> None: + """A run keeps the .py version it was created on after the default moves.""" + workers = ManifestEconomyWorkers( + manifest_store(household_worker_version(), household_worker_version("6.18.0")) + ) + + worker = workers.worker_for_version("us", "6.18.0") + + assert worker.version == "6.18.0" + assert worker.policyengine_version == "6.18.0" + assert worker.application_name == v2_application_name("6.18.0") + assert worker.simulation_callable == "run_single_simulation_us" + + +def test_a_version_the_manifest_no_longer_holds_has_no_economy_worker() -> None: + workers = ManifestEconomyWorkers(manifest_store(household_worker_version())) + + with pytest.raises(EconomyWorkerUnavailableError) as raised: + workers.worker_for_version("us", "6.0.0") + + assert str(raised.value) == "the v2 version manifest has no worker for .py 6.0.0" + + +def test_a_workers_version_lookup_reads_an_unreadable_manifest_as_unavailable() -> None: + with pytest.raises(VersionManifestUnavailableError): + ManifestEconomyWorkers(UnreadableStore()).worker_for_version( + "us", POLICYENGINE_VERSION + ) diff --git a/projects/policyengine-simulation-entry/tests/test_household_workers.py b/projects/policyengine-simulation-entry/tests/test_household_workers.py index 661308082..cbc2786eb 100644 --- a/projects/policyengine-simulation-entry/tests/test_household_workers.py +++ b/projects/policyengine-simulation-entry/tests/test_household_workers.py @@ -181,3 +181,32 @@ async def test_a_call_modal_gives_no_id_is_refused(monkeypatch): await ModalHouseholdFunctionCaller("staging").spawn( worker, {}, household_job_id=JOB_ID, observability_context=None ) + + +def test_the_household_function_for_a_jobs_version_is_found_when_not_the_default() -> ( + None +): + """A job keeps the .py version it was created on after the default moves.""" + workers = ManifestHouseholdWorkers( + manifest_store(household_worker_version(), household_worker_version("6.18.0")) + ) + + worker = workers.worker_for_version("us", "6.18.0") + + assert worker.bundle.policyengine_version == "6.18.0" + assert worker.application_name == v2_application_name("6.18.0") + assert worker.function_name == "calculate_household_us" + + +def test_a_version_the_manifest_no_longer_holds_has_no_household_function() -> None: + workers = ManifestHouseholdWorkers(manifest_store(household_worker_version())) + + with pytest.raises(HouseholdFunctionUnavailableError) as raised: + workers.worker_for_version("us", "6.0.0") + + assert str(raised.value) == "the v2 version manifest has no worker for .py 6.0.0" + + +def test_a_household_version_lookup_reads_an_unreadable_store_as_unavailable() -> None: + with pytest.raises(VersionManifestUnavailableError): + ManifestHouseholdWorkers({}).worker_for_version("us", POLICYENGINE_VERSION) diff --git a/projects/policyengine-simulation-entry/tests/test_run_restarts.py b/projects/policyengine-simulation-entry/tests/test_run_restarts.py new file mode 100644 index 000000000..726194210 --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_run_restarts.py @@ -0,0 +1,783 @@ +"""Starting a run or household job again when its Modal call never started. + +The restart check rebuilds a run's call from its stored rows. Each test of a +restart first submits the run through the service that creates it, captures +the call that submission started, and requires the restart to start the same +call: the same payload, and the same context apart from the request ID. The +store's reads are tested against PostgreSQL in the persistence library; here +fakes answer as the stored rows would. +""" + +from __future__ import annotations + +import asyncio +from contextlib import contextmanager +from dataclasses import dataclass, field, replace +from datetime import timedelta +from typing import TYPE_CHECKING, Any +from uuid import UUID + +import pytest +from economy_fixtures import economy_worker_version +from economy_service_fakes import ( + CATALOG_VERSION_ID, + CREATED_AT, + DYNAMIC_ID, + POLICY_ID, + REFORM, + FakeCaller, + FakeStore, + report_request, + service, + simulation_request, +) +from household_fixtures import manifest_store +from policyengine_simulation_entry.economy_workers import ManifestEconomyWorkers +from policyengine_simulation_entry.household_api_models import ( + HouseholdCalculationRequest, +) +from policyengine_simulation_entry.household_calculations import ( + HouseholdCalculationService, +) +from policyengine_simulation_entry.household_workers import ( + ManifestHouseholdWorkers, +) +from policyengine_simulation_entry.run_restarts import ( + RESTART_AFTER, + RESTART_UNTIL, + RESTARTS_PER_CHECK, + RestartCallers, + RestartOutcome, + RestartSettings, + RestartStores, + RestartSummary, + RestartWorkers, + RunRestartService, +) +from policyengine_simulation_persistence import ( + EconomyScope, + HouseholdCalculationInputs, + HouseholdJobRecord, + HouseholdJobStatus, + RestartKind, + SimulationChoice, + StoredEconomySide, + StoredHousehold, + StoredReportRun, + StoredRunUnreadableError, + StoredSimulationRun, + UnstartedRun, +) +from sqlalchemy.exc import OperationalError + +if TYPE_CHECKING: + from collections.abc import Iterator + from datetime import datetime + + from policyengine_simulation_entry.household_workers import HouseholdWorker + from policyengine_simulation_persistence import NewHouseholdJob + +NOW = CREATED_AT + timedelta(minutes=4) +DATABASE_ERROR = OperationalError("SELECT 1", {}, Exception("server closed")) +HOUSEHOLD_ID = UUID("00000000-0000-4000-8000-0000000000bb") +HOUSEHOLD_JOB_ID = UUID("00000000-0000-4000-8000-0000000000aa") +HOUSEHOLD_DOCUMENT = { + "people": [ + { + "id": "you", + "values": {"age": 40, "employment_income": 30_000}, + "memberships": { + "household": "your household", + "family": "your family", + "tax_unit": "your tax unit", + "spm_unit": "your spm unit", + "marital_unit": "your marital unit", + }, + } + ], + "household": [{"id": "your household", "values": {"state_name": "TX"}}], + "family": [{"id": "your family", "values": {}}], + "tax_unit": [{"id": "your tax unit", "values": {}}], + "spm_unit": [{"id": "your spm unit", "values": {}}], + "marital_unit": [{"id": "your marital unit", "values": {}}], +} +SETTINGS = RestartSettings(environment="staging", modal_environment="staging") +# The manifest both a submission and its restart read: the call's context +# records the digest of the manifest its worker was read from. +MANIFEST_STORE = manifest_store(economy_worker_version()) + + +@dataclass +class FakeRestartStore: + """Lists the unstarted runs it is given and reads back their stored rows. + + ``marked_elsewhere`` makes every mark change nothing, as when another + check marked the run first. + """ + + unstarted: tuple[UnstartedRun, ...] = () + report_runs: dict[UUID, StoredReportRun] = field(default_factory=dict) + simulation_runs: dict[UUID, StoredSimulationRun] = field(default_factory=dict) + read_failure: Exception | None = None + marked_elsewhere: bool = False + windows: list[tuple[datetime, datetime, int]] = field(default_factory=list) + marked: list[UnstartedRun] = field(default_factory=list) + + def list_unstarted( + self, *, created_after: datetime, created_before: datetime, limit: int + ) -> tuple[UnstartedRun, ...]: + self.windows.append((created_after, created_before, limit)) + return self.unstarted + + def mark_restart(self, run: UnstartedRun) -> bool: + if self.marked_elsewhere: + return False + self.marked.append(run) + return True + + def read_report_run(self, report_run_id: UUID) -> StoredReportRun | None: + if self.read_failure is not None: + raise self.read_failure + return self.report_runs.get(report_run_id) + + def read_simulation_run( + self, simulation_run_id: UUID + ) -> StoredSimulationRun | None: + if self.read_failure is not None: + raise self.read_failure + return self.simulation_runs.get(simulation_run_id) + + +@dataclass +class FakeHouseholdJobs: + """Holds the household jobs created, and answers the restart's reads of them.""" + + default_year: int | None = 2026 + reform: dict[str, dict[str, object]] = field(default_factory=dict) + claimed: bool = False + created: list[NewHouseholdJob] = field(default_factory=list) + recorded: list[tuple[UUID, str]] = field(default_factory=list) + failed: list[tuple[UUID, str]] = field(default_factory=list) + + def read_inputs(self, **arguments: Any) -> HouseholdCalculationInputs: + return HouseholdCalculationInputs( + catalog_version_id=CATALOG_VERSION_ID, + household=StoredHousehold( + id=arguments["household_id"], + country=arguments["country"], + default_year=self.default_year, + document=HOUSEHOLD_DOCUMENT, + ), + reform=self.reform, + ) + + def create_job(self, job: NewHouseholdJob) -> None: + self.created.append(job) + + def record_call(self, job_id: UUID, modal_call_id: str) -> HouseholdJobStatus: + self.recorded.append((job_id, modal_call_id)) + return ( + HouseholdJobStatus.RUNNING if self.claimed else HouseholdJobStatus.PENDING + ) + + def fail_unstarted(self, job_id: UUID, error_message: str) -> bool: + if self.claimed: + return False + self.failed.append((job_id, error_message)) + return True + + def read(self, job_id: UUID) -> HouseholdJobRecord | None: + job = next((job for job in self.created if job.id == job_id), None) + if job is None: + return None + return HouseholdJobRecord( + id=job.id, + status=HouseholdJobStatus.PENDING, + country=job.country, + household_id=job.household_id, + policy_id=job.policy_id, + year=job.year, + request_data=job.request_data, + tax_benefit_model_version_id=job.tax_benefit_model_version_id, + country_package_version=job.country_package_version, + policyengine_version=job.policyengine_version, + modal_call_id=None, + result=None, + error_message=None, + created_at=CREATED_AT, + started_at=None, + completed_at=None, + ) + + +@dataclass +class FakeHouseholdCaller: + """Records each household function call; ``failure`` is raised instead.""" + + failure: Exception | None = None + calls: list[dict[str, Any]] = field(default_factory=list) + + async def spawn( + self, + worker: HouseholdWorker, + payload: dict[str, Any], + *, + household_job_id: UUID, + observability_context: dict[str, Any] | None, + ) -> str: + if self.failure is not None: + raise self.failure + self.calls.append( + { + "worker": worker, + "payload": payload, + "household_job_id": household_job_id, + "observability_context": observability_context, + } + ) + return "fc-household-restarted" + + +@dataclass +class RecordingTelemetry: + """Records each restart scope and outcome the check reports.""" + + scopes: list[tuple[RestartKind, UUID, float]] = field(default_factory=list) + outcomes: list[tuple[RestartKind, UUID, RestartOutcome, str | None]] = field( + default_factory=list + ) + + @contextmanager + def restart_scope(self, run: UnstartedRun, *, age_seconds: float) -> Iterator[None]: + self.scopes.append((run.kind, run.run_id, age_seconds)) + yield + + def record_outcome( + self, + run: UnstartedRun, + outcome: RestartOutcome, + *, + error_type: str | None = None, + ) -> None: + self.outcomes.append((run.kind, run.run_id, outcome, error_type)) + + def capture_context(self) -> dict[str, str]: + return {"traceparent": "00-restart-check"} + + +@dataclass +class RestartRig: + """The restart service and every fake it was built with.""" + + restarts: FakeRestartStore + economy: FakeStore + households: FakeHouseholdJobs + economy_caller: FakeCaller + household_caller: FakeHouseholdCaller + telemetry: RecordingTelemetry + service: RunRestartService + + def restart(self) -> RestartSummary: + return asyncio.run(self.service.restart_unstarted_runs()) + + +def _rig( # noqa: PLR0913 # each fake a test may replace + *, + restarts: FakeRestartStore | None = None, + economy: FakeStore | None = None, + households: FakeHouseholdJobs | None = None, + economy_caller: FakeCaller | None = None, + household_caller: FakeHouseholdCaller | None = None, + manifest: dict[str, object] | None = None, +) -> RestartRig: + restarts = restarts or FakeRestartStore() + economy = economy or FakeStore() + households = households or FakeHouseholdJobs() + economy_caller = economy_caller or FakeCaller() + household_caller = household_caller or FakeHouseholdCaller() + telemetry = RecordingTelemetry() + store = MANIFEST_STORE if manifest is None else manifest + restart_service = RunRestartService( + stores=RestartStores(restarts=restarts, economy=economy, households=households), + workers=RestartWorkers( + economy=ManifestEconomyWorkers(store), + households=ManifestHouseholdWorkers(store), + ), + callers=RestartCallers(economy=economy_caller, households=household_caller), + settings=SETTINGS, + telemetry=telemetry, + clock=lambda: NOW, + call_start_timeout_seconds=0.05, + new_request_id=lambda: "restart-request", + ) + return RestartRig( + restarts=restarts, + economy=economy, + households=households, + economy_caller=economy_caller, + household_caller=household_caller, + telemetry=telemetry, + service=restart_service, + ) + + +def _submitted_report() -> tuple[StoredReportRun, tuple[dict, dict]]: + """Submit a report as the service does; return its stored run and its call.""" + caller = FakeCaller() + created = asyncio.run( + service( + FakeStore(), workers=ManifestEconomyWorkers(MANIFEST_STORE), caller=caller + ).submit_report( + report_request(), request_id="original-request", observability_context=None + ) + ) + (worker, payload, context, _observability) = caller.report_runs[0] + stored = StoredReportRun( + report_run_id=created.report_run_id, + country="us", + versions=worker.versions, + options=report_request().options(), + scope=EconomyScope( + catalog_version_id=CATALOG_VERSION_ID, + region_code="us", + dataset_name="populace_us_2024", + year=2026, + ), + baseline=StoredEconomySide( + simulation_run_id=created.baseline_simulation_run_id, + choice=SimulationChoice(policy_id=None, dynamic_id=None), + reform={}, + ), + reform=StoredEconomySide( + simulation_run_id=created.reform_simulation_run_id, + choice=SimulationChoice(policy_id=POLICY_ID, dynamic_id=DYNAMIC_ID), + reform=REFORM, + ), + created_at=CREATED_AT, + ) + return stored, (payload, context) + + +def _submitted_simulation() -> tuple[StoredSimulationRun, tuple[dict, dict]]: + """Submit a standalone simulation; return its stored run and its call.""" + caller = FakeCaller() + created = asyncio.run( + service( + FakeStore(), workers=ManifestEconomyWorkers(MANIFEST_STORE), caller=caller + ).submit_simulation( + simulation_request(), + request_id="original-request", + observability_context=None, + ) + ) + (worker, payload, context, _observability) = caller.simulation_runs[0] + stored = StoredSimulationRun( + country="us", + versions=worker.versions, + scope=EconomyScope( + catalog_version_id=CATALOG_VERSION_ID, + region_code="us", + dataset_name="populace_us_2024", + year=2026, + ), + run=StoredEconomySide( + simulation_run_id=created.simulation_run_id, + choice=SimulationChoice(policy_id=POLICY_ID, dynamic_id=None), + reform=REFORM, + ), + created_at=CREATED_AT, + ) + return stored, (payload, context) + + +def _submitted_household_job( + households: FakeHouseholdJobs, **request_fields: Any +) -> dict[str, Any]: + """Submit a household calculation into ``households``; return its call's payload.""" + caller = FakeHouseholdCaller() + household_service = HouseholdCalculationService( + store=households, + workers=ManifestHouseholdWorkers(MANIFEST_STORE), + caller=caller, + new_job_id=lambda: HOUSEHOLD_JOB_ID, + ) + asyncio.run( + household_service.submit( + HouseholdCalculationRequest.model_validate( + {"country": "us", "household_id": str(HOUSEHOLD_ID), **request_fields} + ), + observability_context=None, + ) + ) + households.recorded.clear() + return caller.calls[0]["payload"] + + +def _unstarted(kind: RestartKind, run_id: UUID) -> UnstartedRun: + return UnstartedRun(kind=kind, run_id=run_id, created_at=CREATED_AT) + + +def _without_request_id(context: dict[str, Any]) -> dict[str, Any]: + return {key: value for key, value in context.items() if key != "request_id"} + + +def test_the_check_lists_runs_created_three_to_twelve_minutes_ago() -> None: + rig = _rig() + + summary = rig.restart() + + assert rig.restarts.windows == [ + (NOW - RESTART_UNTIL, NOW - RESTART_AFTER, RESTARTS_PER_CHECK) + ] + assert ( + timedelta(minutes=3), + timedelta(minutes=12), + ) == (RESTART_AFTER, RESTART_UNTIL) + assert summary == RestartSummary(listed=0) + + +def test_a_report_run_is_started_again_with_the_call_its_submission_started() -> None: + stored, (payload, context) = _submitted_report() + run = _unstarted(RestartKind.REPORT_RUN, stored.report_run_id) + rig = _rig( + restarts=FakeRestartStore( + unstarted=(run,), report_runs={stored.report_run_id: stored} + ) + ) + + summary = rig.restart() + + ((_worker, restarted_payload, restarted_context, observability),) = ( + rig.economy_caller.report_runs + ) + assert restarted_payload == payload + assert _without_request_id(restarted_context) == _without_request_id(context) + assert restarted_context["request_id"] == "restart-request" + assert observability == {"traceparent": "00-restart-check"} + assert rig.restarts.marked == [run] + assert rig.economy.recorded_calls == [(stored.report_run_id, "fc-coordinator")] + assert summary == RestartSummary(listed=1, restarted=1) + + +def test_a_standalone_simulation_run_is_started_again_with_its_submissions_call() -> ( + None +): + stored, (payload, context) = _submitted_simulation() + run_id = stored.run.simulation_run_id + rig = _rig( + restarts=FakeRestartStore( + unstarted=(_unstarted(RestartKind.SIMULATION_RUN, run_id),), + simulation_runs={run_id: stored}, + ) + ) + + summary = rig.restart() + + ((_worker, restarted_payload, restarted_context, _observability),) = ( + rig.economy_caller.simulation_runs + ) + assert restarted_payload == payload + assert _without_request_id(restarted_context) == _without_request_id(context) + assert rig.economy.recorded_calls == [(run_id, "fc-worker")] + assert summary == RestartSummary(listed=1, restarted=1) + + +def test_a_household_job_is_started_again_with_its_submissions_call() -> None: + households = FakeHouseholdJobs(reform=REFORM) + payload = _submitted_household_job( + households, policy_id=str(POLICY_ID), add_missing_variables=True + ) + rig = _rig( + restarts=FakeRestartStore( + unstarted=(_unstarted(RestartKind.HOUSEHOLD_JOB, HOUSEHOLD_JOB_ID),) + ), + households=households, + ) + + summary = rig.restart() + + (call,) = rig.household_caller.calls + assert call["payload"] == payload + assert call["household_job_id"] == HOUSEHOLD_JOB_ID + assert call["observability_context"] == {"traceparent": "00-restart-check"} + assert households.recorded == [(HOUSEHOLD_JOB_ID, "fc-household-restarted")] + assert summary == RestartSummary(listed=1, restarted=1) + + +def test_a_run_another_check_marked_first_is_left_without_a_call() -> None: + stored, _call = _submitted_report() + rig = _rig( + restarts=FakeRestartStore( + unstarted=(_unstarted(RestartKind.REPORT_RUN, stored.report_run_id),), + report_runs={stored.report_run_id: stored}, + marked_elsewhere=True, + ) + ) + + summary = rig.restart() + + assert rig.economy_caller.report_runs == [] + assert rig.economy.failed_runs == [] + assert summary == RestartSummary(listed=1, left=1) + + +def test_a_run_gone_before_it_is_read_is_left_alone() -> None: + rig = _rig( + restarts=FakeRestartStore( + unstarted=( + _unstarted(RestartKind.REPORT_RUN, UUID(int=1)), + _unstarted(RestartKind.SIMULATION_RUN, UUID(int=2)), + _unstarted(RestartKind.HOUSEHOLD_JOB, UUID(int=3)), + ) + ) + ) + + summary = rig.restart() + + assert rig.restarts.marked == [] + assert summary == RestartSummary(listed=3, left=3) + + +def test_a_run_whose_rows_cannot_be_read_is_left_for_the_next_check() -> None: + stored, _call = _submitted_report() + rig = _rig( + restarts=FakeRestartStore( + unstarted=(_unstarted(RestartKind.REPORT_RUN, stored.report_run_id),), + read_failure=DATABASE_ERROR, + ) + ) + + summary = rig.restart() + + assert rig.restarts.marked == [] + assert rig.economy.failed_runs == [] + assert summary == RestartSummary(listed=1, deferred=1) + + +def test_an_unreadable_manifest_leaves_the_run_for_the_next_check() -> None: + stored, _call = _submitted_report() + rig = _rig( + restarts=FakeRestartStore( + unstarted=(_unstarted(RestartKind.REPORT_RUN, stored.report_run_id),), + report_runs={stored.report_run_id: stored}, + ), + manifest={}, + ) + + summary = rig.restart() + + assert rig.restarts.marked == [] + assert summary == RestartSummary(listed=1, deferred=1) + + +def test_a_run_whose_worker_left_the_manifest_is_failed_as_not_started() -> None: + stored, _call = _submitted_report() + rig = _rig( + restarts=FakeRestartStore( + unstarted=(_unstarted(RestartKind.REPORT_RUN, stored.report_run_id),), + report_runs={stored.report_run_id: stored}, + ), + manifest=manifest_store(economy_worker_version("6.18.0")), + ) + + summary = rig.restart() + + ((run_id, simulation_run_ids, message),) = rig.economy.failed_runs + assert run_id == stored.report_run_id + assert simulation_run_ids == ( + stored.baseline.simulation_run_id, + stored.reform.simulation_run_id, + ) + assert message.startswith("run_not_started: ") + assert f".py {stored.versions.policyengine_version}" in message + assert rig.economy_caller.report_runs == [] + assert summary == RestartSummary(listed=1, failed=1) + + +def test_a_run_whose_stored_rows_no_longer_describe_it_is_failed() -> None: + run_id = UUID(int=7) + rig = _rig( + restarts=FakeRestartStore( + unstarted=(_unstarted(RestartKind.SIMULATION_RUN, run_id),), + read_failure=StoredRunUnreadableError("simulation run 7 names no region"), + ) + ) + + summary = rig.restart() + + ((failed_run_id, _sides, message),) = rig.economy.failed_runs + assert failed_run_id == run_id + assert message.startswith("run_not_started: ") + assert "stored rows" in message + assert summary == RestartSummary(listed=1, failed=1) + + +def test_a_household_job_whose_household_changed_is_failed_as_not_started() -> None: + households = FakeHouseholdJobs() + _submitted_household_job(households) + households.default_year = 2027 + rig = _rig( + restarts=FakeRestartStore( + unstarted=(_unstarted(RestartKind.HOUSEHOLD_JOB, HOUSEHOLD_JOB_ID),) + ), + households=households, + ) + + summary = rig.restart() + + ((job_id, message),) = households.failed + assert job_id == HOUSEHOLD_JOB_ID + assert message.startswith("run_not_started: ") + assert rig.household_caller.calls == [] + assert summary == RestartSummary(listed=1, failed=1) + + +def test_a_restart_modal_refuses_fails_the_run_as_not_started() -> None: + stored, _call = _submitted_report() + rig = _rig( + restarts=FakeRestartStore( + unstarted=(_unstarted(RestartKind.REPORT_RUN, stored.report_run_id),), + report_runs={stored.report_run_id: stored}, + ), + economy_caller=FakeCaller(failure=ConnectionError("Modal is down")), + ) + + summary = rig.restart() + + ((run_id, simulation_run_ids, message),) = rig.economy.failed_runs + assert run_id == stored.report_run_id + assert len(simulation_run_ids) == 2 + assert message == ( + "run_not_started: The run stayed pending with no Modal call recorded for " + "it, and Modal did not accept the call that started it again " + "(ConnectionError)." + ) + assert summary == RestartSummary(listed=1, failed=1) + assert rig.telemetry.outcomes == [ + ( + RestartKind.REPORT_RUN, + stored.report_run_id, + RestartOutcome.FAILED, + "ConnectionError", + ) + ] + + +def test_a_restart_modal_does_not_confirm_in_time_fails_the_run() -> None: + stored, _call = _submitted_simulation() + run_id = stored.run.simulation_run_id + rig = _rig( + restarts=FakeRestartStore( + unstarted=(_unstarted(RestartKind.SIMULATION_RUN, run_id),), + simulation_runs={run_id: stored}, + ), + economy_caller=FakeCaller(hang=True), + ) + + summary = rig.restart() + + ((failed_run_id, _sides, message),) = rig.economy.failed_runs + assert failed_run_id == run_id + assert "(TimeoutError)" in message + assert summary == RestartSummary(listed=1, failed=1) + + +def test_a_refused_restart_of_a_run_another_call_claimed_leaves_it() -> None: + stored, _call = _submitted_report() + rig = _rig( + restarts=FakeRestartStore( + unstarted=(_unstarted(RestartKind.REPORT_RUN, stored.report_run_id),), + report_runs={stored.report_run_id: stored}, + ), + economy=FakeStore(claimed=True), + economy_caller=FakeCaller(failure=ConnectionError("Modal is down")), + ) + + summary = rig.restart() + + assert rig.economy.failed_runs == [] + assert summary == RestartSummary(listed=1, started_meanwhile=1) + + +def test_a_restarted_call_another_call_claimed_first_counts_as_started_meanwhile() -> ( + None +): + households = FakeHouseholdJobs() + _submitted_household_job(households) + households.claimed = True + rig = _rig( + restarts=FakeRestartStore( + unstarted=(_unstarted(RestartKind.HOUSEHOLD_JOB, HOUSEHOLD_JOB_ID),) + ), + households=households, + ) + + summary = rig.restart() + + assert len(rig.household_caller.calls) == 1 + assert summary == RestartSummary(listed=1, started_meanwhile=1) + + +def test_a_restart_whose_call_cannot_be_recorded_still_counts_its_call() -> None: + stored, _call = _submitted_report() + rig = _rig( + restarts=FakeRestartStore( + unstarted=(_unstarted(RestartKind.REPORT_RUN, stored.report_run_id),), + report_runs={stored.report_run_id: stored}, + ), + economy=FakeStore(record_failure=DATABASE_ERROR), + ) + + summary = rig.restart() + + assert len(rig.economy_caller.report_runs) == 1 + assert rig.economy.failed_runs == [] + assert summary == RestartSummary(listed=1, call_not_recorded=1) + + +def test_each_listed_run_is_one_telemetry_scope_with_its_age_and_outcome() -> None: + report, _report_call = _submitted_report() + simulation, _simulation_call = _submitted_simulation() + simulation_run_id = simulation.run.simulation_run_id + report_run = replace( + _unstarted(RestartKind.REPORT_RUN, report.report_run_id), + created_at=CREATED_AT + timedelta(seconds=30), + ) + rig = _rig( + restarts=FakeRestartStore( + unstarted=( + report_run, + _unstarted(RestartKind.SIMULATION_RUN, simulation_run_id), + ), + report_runs={report.report_run_id: report}, + ) + ) + + summary = rig.restart() + + assert rig.telemetry.scopes == [ + (RestartKind.REPORT_RUN, report.report_run_id, 210.0), + (RestartKind.SIMULATION_RUN, simulation_run_id, 240.0), + ] + assert rig.telemetry.outcomes == [ + (RestartKind.REPORT_RUN, report.report_run_id, RestartOutcome.RESTARTED, None), + (RestartKind.SIMULATION_RUN, simulation_run_id, RestartOutcome.LEFT, None), + ] + assert summary == RestartSummary(listed=2, restarted=1, left=1) + + +@pytest.mark.parametrize( + ("modal_environment", "environment"), + [("staging", "staging"), ("main", "production")], +) +def test_the_deployment_label_follows_the_modal_environment( + modal_environment, environment +) -> None: + assert RestartSettings.for_modal_environment(modal_environment) == ( + RestartSettings(environment=environment, modal_environment=modal_environment) + ) + + +def test_a_modal_environment_no_deployment_uses_is_refused() -> None: + with pytest.raises(ValueError, match="dev"): + RestartSettings.for_modal_environment("dev") From 7ede4eb542344da2fe762ce5ef55252b11e285ef Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:28:04 +0400 Subject: [PATCH 04/29] Start a run or household job again when its Modal call never started The restart service lists the rows created 3 to 12 minutes ago that are still pending with no call, rebuilds each call from its stored rows on the worker of the row's own .py version, marks the row, starts the call and records its ID. A row that can't be rebuilt, or whose restart Modal refuses, is failed with run_not_started; one whose rows or manifest can't be read now is left for the next check. Co-Authored-By: Claude Opus 5.5 --- .../economy_workers.py | 60 +- .../household_workers.py | 59 +- .../run_restarts.py | 789 ++++++++++++++++++ 3 files changed, 896 insertions(+), 12 deletions(-) create mode 100644 projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_workers.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_workers.py index b07a6e693..8e123beb4 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_workers.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_workers.py @@ -75,6 +75,43 @@ def default_economy_worker(manifest: V2VersionManifest, country: str) -> Economy country's economy runs. """ version = manifest.default_version + return _economy_worker( + manifest, + country, + version, + described_as=f"the default v2 worker, .py {version},", + ) + + +def economy_worker_for_version( + manifest: V2VersionManifest, country: str, policyengine_version: str +) -> EconomyWorker: + """Return the economy functions of the manifest's worker for a .py version. + + A run is calculated by the worker of the .py version it records, which + may no longer be the default. Raise EconomyWorkerUnavailableError when the + manifest no longer holds that worker, or it cannot calculate the + country's economy runs. + """ + if policyengine_version not in manifest.versions: + msg = f"the v2 version manifest has no worker for .py {policyengine_version}" + raise EconomyWorkerUnavailableError(msg) + return _economy_worker( + manifest, + country, + policyengine_version, + described_as=f"the v2 worker for .py {policyengine_version}", + ) + + +def _economy_worker( + manifest: V2VersionManifest, country: str, version: str, *, described_as: str +) -> EconomyWorker: + """Return the economy functions of the manifest's worker ``version``. + + ``described_as`` names the worker in the error raised when it cannot + calculate the country's economy runs. + """ worker = manifest.versions[version] country_worker = worker.economy_worker(country) country_bundle = next( @@ -85,10 +122,7 @@ def default_economy_worker(manifest: V2VersionManifest, country: str) -> Economy or country_worker.single_simulation_callable is None or country_bundle is None ): - msg = ( - f"the default v2 worker, .py {version}, cannot calculate economy " - f"runs for {country}" - ) + msg = f"{described_as} cannot calculate economy runs for {country}" raise EconomyWorkerUnavailableError(msg) return EconomyWorker( version=version, @@ -139,13 +173,27 @@ def default_worker(self, country: str) -> EconomyWorker: Raise EconomyWorkerUnavailableError when it has none, and VersionManifestUnavailableError when the manifest cannot be read. """ + return default_economy_worker(self._manifest(), country) + + def worker_for_version( + self, country: str, policyengine_version: str + ) -> EconomyWorker: + """Return the economy functions of the worker for a .py version. + + Raise EconomyWorkerUnavailableError when the manifest has none, and + VersionManifestUnavailableError when it cannot be read. + """ + return economy_worker_for_version( + self._manifest(), country, policyengine_version + ) + + def _manifest(self) -> V2VersionManifest: try: - manifest = self._loader.load() + return self._loader.load() # Modal raises its own errors, and any of them means the same here. except Exception as error: msg = "the v2 version manifest could not be read" raise VersionManifestUnavailableError(msg) from error - return default_economy_worker(manifest, country) class EconomyRunCaller(Protocol): diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_workers.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_workers.py index bf08e7668..dd8a0646e 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_workers.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_workers.py @@ -80,13 +80,46 @@ def default_household_worker( Raise HouseholdFunctionUnavailableError when it has none. """ version = manifest.default_version + return _household_worker( + manifest, + country, + version, + described_as=f"the default v2 worker, .py {version},", + ) + + +def household_worker_for_version( + manifest: V2VersionManifest, country: str, policyengine_version: str +) -> HouseholdWorker: + """Return the household function of the manifest's worker for a .py version. + + A household job is calculated by the worker of the .py version it + records, which may no longer be the default. Raise + HouseholdFunctionUnavailableError when the manifest no longer holds that + worker, or it has no household function for the country. + """ + if policyengine_version not in manifest.versions: + msg = f"the v2 version manifest has no worker for .py {policyengine_version}" + raise HouseholdFunctionUnavailableError(msg) + return _household_worker( + manifest, + country, + policyengine_version, + described_as=f"the v2 worker for .py {policyengine_version}", + ) + + +def _household_worker( + manifest: V2VersionManifest, country: str, version: str, *, described_as: str +) -> HouseholdWorker: + """Return the household function of the manifest's worker ``version``. + + ``described_as`` names the worker in the error raised when it has none. + """ worker = manifest.versions[version] country_worker = worker.household_worker(country) if country_worker is None or country_worker.household_callable is None: - msg = ( - f"the default v2 worker, .py {version}, has no household function " - f"for {country}" - ) + msg = f"{described_as} has no household function for {country}" raise HouseholdFunctionUnavailableError(msg) return HouseholdWorker( application_name=worker.application_name, @@ -128,13 +161,27 @@ def default_worker(self, country: str) -> HouseholdWorker: Raise HouseholdFunctionUnavailableError when it has none, and VersionManifestUnavailableError when the manifest cannot be read. """ + return default_household_worker(self._manifest(), country) + + def worker_for_version( + self, country: str, policyengine_version: str + ) -> HouseholdWorker: + """Return the household function of the worker for a .py version. + + Raise HouseholdFunctionUnavailableError when the manifest has none, + and VersionManifestUnavailableError when it cannot be read. + """ + return household_worker_for_version( + self._manifest(), country, policyengine_version + ) + + def _manifest(self) -> V2VersionManifest: try: - manifest = self._loader.load() + return self._loader.load() # Modal raises its own errors, and any of them means the same here. except Exception as error: msg = "the v2 version manifest could not be read" raise VersionManifestUnavailableError(msg) from error - return default_household_worker(manifest, country) class ModalHouseholdFunctionCaller: diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py new file mode 100644 index 000000000..14b46f204 --- /dev/null +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py @@ -0,0 +1,789 @@ +"""Start a run or household job again when its Modal call never started. + +The simulation service creates a household job, a report run or a standalone +simulation run ``pending`` with no Modal call ID, starts the call that +calculates it, then records the call's ID. When the service stops between +the insert and the start (a Cloud Run instance shut down, for example), +nothing records a call, and the stuck-run sweep would fail the run 15 +minutes after its creation. The restart check, a scheduled Modal function +beside the sweep, starts such a row's call once more: + +1. it lists the rows created 3 to 12 minutes ago that are still pending with + no call and that nothing has changed since their insert; +2. for each, it reads back what the service built the call from and builds + the same call, on the worker of the .py version the row records; +3. it marks the row, so it is never restarted twice, starts the call, and + records the call's ID while the row is still pending with none. + +A row whose call can't be built again (its worker left the version manifest, +or its stored rows changed), or whose new call Modal refuses, is failed at +once with ``run_not_started``, as the sweep would fail it later. A row whose +rows or manifest can't be read now is left unmarked, for the next check. A +call the service started after all claims the row only while the row records +no call or that call's own ID, so once the restart records its call, the +earlier call's claim is refused; if the earlier call claimed it first, the +restart's call is refused instead. + +Each listed row is one telemetry scope, ``restart_scope``, with its age, and +its outcome is reported through ``record_outcome``, so how often runs need a +restart, and how each restart ended, can be counted. +""" + +from __future__ import annotations + +import asyncio +import uuid +from collections import Counter +from contextlib import nullcontext +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from enum import StrEnum +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Final, Protocol, get_args + +from policyengine_simulation_contract.stage12_bundle import CountryId +from policyengine_simulation_contract.stage12_execution import ReportRunIds +from policyengine_simulation_persistence import ( + MissingPolicyParametersError, + OverlappingReformError, + RestartKind, + StoredRunUnreadableError, + UnknownCatalogVersionError, + UnknownDynamicError, + UnknownHouseholdError, + UnknownPolicyError, +) +from pydantic import ValidationError +from sqlalchemy.exc import SQLAlchemyError + +from policyengine_simulation_entry.economy_calculation_inputs import ( + CalculationScope, + CallOrigin, + DatasetNotInBundleError, + bundle_dataset, + report_run_calculation, + simulation_options, + simulation_run_calculation, +) +from policyengine_simulation_entry.economy_workers import ( + EconomyWorkerUnavailableError, +) +from policyengine_simulation_entry.household_api_models import ( + HouseholdCalculationRequest, +) +from policyengine_simulation_entry.household_calculations import ( + HouseholdCalculationInternalError, + HouseholdCalculationUnprocessableError, + prepare_household_job, +) +from policyengine_simulation_entry.household_workers import ( + HouseholdFunctionUnavailableError, + VersionManifestUnavailableError, +) + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable, Sequence + from contextlib import AbstractContextManager + from uuid import UUID + + from policyengine_simulation_persistence import ( + EconomyScope, + HouseholdCalculationInputs, + HouseholdJobRecord, + HouseholdJobStatus, + NewHouseholdJob, + ReportRunStatus, + RunVersions, + SimulationStatus, + StoredReportRun, + StoredSimulationRun, + UnstartedRun, + ) + + from policyengine_simulation_entry.economy_workers import ( + EconomyRunCaller, + EconomyWorker, + ) + from policyengine_simulation_entry.household_workers import ( + HouseholdFunctionCaller, + HouseholdWorker, + ) + +# A row is restarted once it has stayed pending with no call this long, which +# is far longer than a submission takes to start and record its call. +RESTART_AFTER: Final = timedelta(minutes=3) +# A row older than this is left to the stuck-run sweep, which fails it at 15 +# minutes, so a restart has time to record its call before the sweep judges it. +RESTART_UNTIL: Final = timedelta(minutes=12) +# At most this many rows are restarted by one check; the next takes the rest. +RESTARTS_PER_CHECK: Final = 50 +# How long the check waits for Modal to accept a restarted call. +CALL_START_TIMEOUT_SECONDS: Final = 10.0 +# The deployment each Modal environment holding the v2 workers belongs to, as +# the Simulation Entrypoint's deploy pairs them (APP_ENVIRONMENT). A call's +# context and artifact prefix name the deployment. +DEPLOYMENT_ENVIRONMENTS: Final = MappingProxyType( + {"staging": "staging", "main": "production"} +) +_NOT_STARTED: Final = ( + "run_not_started: The run stayed pending with no Modal call recorded for it" +) +_PENDING: Final = "pending" + + +class RestartOutcome(StrEnum): + """How the check ended for one listed row. + + ``restarted``: its new call started and the call's ID was recorded. + ``call_not_recorded``: its new call started, but the ID couldn't be + recorded; the call claims the row itself. ``started_meanwhile``: another + call claimed the row after the check started its own, which that call + then can't claim. ``left``: the row was gone, or changed before the check + started a call; also a row whose failure couldn't be written, which the + sweep fails later. ``deferred``: its rows or the manifest couldn't be + read, so it is left unmarked for the next check. ``failed``: it couldn't + be started again and was failed with ``run_not_started``. + """ + + RESTARTED = "restarted" + CALL_NOT_RECORDED = "call_not_recorded" + STARTED_MEANWHILE = "started_meanwhile" + LEFT = "left" + DEFERRED = "deferred" + FAILED = "failed" + + +@dataclass(frozen=True) +class RestartSettings: + """The deployment the check restarts runs for. + + ``environment`` labels each call's context and artifacts, and + ``modal_environment`` holds the v2 workers. + """ + + environment: str + modal_environment: str + + @classmethod + def for_modal_environment(cls, modal_environment: str) -> RestartSettings: + """Return the settings of the deployment whose workers are in the environment. + + Raise ValueError for an environment no deployment uses. + """ + environment = DEPLOYMENT_ENVIRONMENTS.get(modal_environment) + if environment is None: + msg = ( + "no deployment runs the v2 workers in Modal environment " + f"{modal_environment!r}" + ) + raise ValueError(msg) + return cls(environment=environment, modal_environment=modal_environment) + + +@dataclass(frozen=True) +class RestartSummary: + """How many rows one check listed, and how each ended (``RestartOutcome``).""" + + listed: int + restarted: int = 0 + call_not_recorded: int = 0 + started_meanwhile: int = 0 + left: int = 0 + deferred: int = 0 + failed: int = 0 + + +class UnstartedRunSource(Protocol): + """The restart check's own reads and writes (``RunRestartStore``).""" + + def list_unstarted( + self, *, created_after: datetime, created_before: datetime, limit: int + ) -> tuple[UnstartedRun, ...]: + """Return the oldest restartable rows created in the window.""" + ... + + def mark_restart(self, run: UnstartedRun) -> bool: + """Mark the row restarted while it is still unstarted.""" + ... + + def read_report_run(self, report_run_id: UUID) -> StoredReportRun | None: + """Return what the report run's call is built from.""" + ... + + def read_simulation_run( + self, simulation_run_id: UUID + ) -> StoredSimulationRun | None: + """Return what the standalone simulation run's call is built from.""" + ... + + +class EconomyRunWrites(Protocol): + """The submission's own call writes on economy runs (``EconomyRunStore``).""" + + def record_report_call( + self, report_run_id: UUID, modal_call_id: str + ) -> ReportRunStatus | None: + """Record the report run's call ID while it is pending with none.""" + ... + + def fail_unstarted_report( + self, + report_run_id: UUID, + simulation_run_ids: Sequence[UUID], + error_message: str, + ) -> bool: + """Fail the report run, and its simulation runs, while unstarted.""" + ... + + def record_simulation_call( + self, simulation_run_id: UUID, modal_call_id: str + ) -> SimulationStatus | None: + """Record the simulation run's call ID while it is pending with none.""" + ... + + def fail_unstarted_simulation( + self, simulation_run_id: UUID, error_message: str + ) -> bool: + """Fail the simulation run while it is pending with no call ID.""" + ... + + +class HouseholdJobRestarts(Protocol): + """The household job reads and writes a restart uses (``HouseholdJobStore``).""" + + def read(self, job_id: UUID) -> HouseholdJobRecord | None: + """Return the job as it stands.""" + ... + + def read_inputs( + self, + *, + country: str, + household_id: UUID, + policy_id: UUID | None, + policyengine_version: str, + ) -> HouseholdCalculationInputs: + """Read the catalog version, household and policy together.""" + ... + + def record_call( + self, job_id: UUID, modal_call_id: str + ) -> HouseholdJobStatus | None: + """Record the call ID while the job is pending with none.""" + ... + + def fail_unstarted(self, job_id: UUID, error_message: str) -> bool: + """Fail the job while it is pending with no call ID.""" + ... + + +class VersionedEconomyWorkers(Protocol): + """Find a .py version's economy functions (``ManifestEconomyWorkers``).""" + + def worker_for_version( + self, country: str, policyengine_version: str + ) -> EconomyWorker: + """Return the economy functions of the worker for the .py version.""" + ... + + +class VersionedHouseholdWorkers(Protocol): + """Find a .py version's household function (``ManifestHouseholdWorkers``).""" + + def worker_for_version( + self, country: str, policyengine_version: str + ) -> HouseholdWorker: + """Return the household function of the worker for the .py version.""" + ... + + +class RestartTelemetry(Protocol): + """Where the check reports each listed row (``ObservabilityRestartTelemetry``).""" + + def restart_scope( + self, run: UnstartedRun, *, age_seconds: float + ) -> AbstractContextManager[None]: + """Return the scope one row's restart runs in.""" + ... + + def record_outcome( + self, + run: UnstartedRun, + outcome: RestartOutcome, + *, + error_type: str | None = None, + ) -> None: + """Report how the row's restart ended, and the error type that ended it.""" + ... + + def capture_context(self) -> dict[str, str] | None: + """Return the context a restarted call continues the check's trace from.""" + ... + + +class NoRestartTelemetry: + """Report nothing.""" + + def restart_scope( + self, + run: UnstartedRun, # noqa: ARG002 # the protocol's argument + *, + age_seconds: float, # noqa: ARG002 # the protocol's argument + ) -> AbstractContextManager[None]: + """Return a scope that does nothing.""" + return nullcontext() + + def record_outcome( + self, + run: UnstartedRun, + outcome: RestartOutcome, + *, + error_type: str | None = None, + ) -> None: + """Report nothing.""" + + def capture_context(self) -> dict[str, str] | None: + """Return no context.""" + return None + + +@dataclass(frozen=True) +class RestartStores: + """The stores a restart reads and writes.""" + + restarts: UnstartedRunSource + economy: EconomyRunWrites + households: HouseholdJobRestarts + + +@dataclass(frozen=True) +class RestartWorkers: + """Where a restart finds the worker of a row's .py version.""" + + economy: VersionedEconomyWorkers + households: VersionedHouseholdWorkers + + +@dataclass(frozen=True) +class RestartCallers: + """What starts a restarted call on Modal.""" + + economy: EconomyRunCaller + households: HouseholdFunctionCaller + + +type CallStart = Callable[[dict[str, Any] | None], Awaitable[str]] + + +class _DeferredError(Exception): + """The row's rows or the manifest couldn't be read now.""" + + +class _CannotRebuildError(Exception): + """The row's call can't be built again; ``reason`` says why, in a sentence part. + + ``simulation_run_ids`` are a report run's simulation runs, failed with it. + """ + + def __init__( + self, + reason: str, + *, + error_type: str, + simulation_run_ids: tuple[UUID, ...] = (), + ) -> None: + super().__init__(reason) + self.reason = reason + self.error_type = error_type + self.simulation_run_ids = simulation_run_ids + + +class _ChangedRunError(LookupError): + """What the call would be built from now differs from what the row records.""" + + +# The errors that mean a row's call can't be built again, and why, in words a +# failed run's error message can hold. A worker's absence is described by the +# error itself, which names only the .py version and the country. +_REBUILD_REASONS: Final[dict[type[Exception], str | None]] = { + EconomyWorkerUnavailableError: None, + HouseholdFunctionUnavailableError: None, + _ChangedRunError: None, + StoredRunUnreadableError: "its stored rows no longer describe one calculation", + UnknownCatalogVersionError: "its catalog version is gone", + UnknownHouseholdError: "its household is gone", + UnknownPolicyError: "its policy is gone", + UnknownDynamicError: "its dynamic is gone", + MissingPolicyParametersError: "its policy names parameters its catalog lacks", + OverlappingReformError: "its policy and dynamic set the same parameters", + DatasetNotInBundleError: "its worker's bundle has no dataset it names", + HouseholdCalculationUnprocessableError: "its household can no longer be read", + HouseholdCalculationInternalError: "its household request no longer validates", + ValidationError: "its stored request no longer validates", +} +_REBUILD_ERRORS: Final = tuple(_REBUILD_REASONS) + + +def _rebuild_reason(error: Exception) -> str: + reason = next( + ( + reason + for error_class, reason in _REBUILD_REASONS.items() + if isinstance(error, error_class) + ), + None, + ) + return str(error) if reason is None else reason + + +def _same_job(rebuilt: NewHouseholdJob, stored: HouseholdJobRecord) -> bool: + """Return whether the rebuilt job records what the stored job records.""" + return ( + rebuilt.id, + rebuilt.country, + rebuilt.household_id, + rebuilt.policy_id, + rebuilt.year, + rebuilt.request_data, + rebuilt.tax_benefit_model_version_id, + rebuilt.country_package_version, + rebuilt.policyengine_version, + ) == ( + stored.id, + stored.country, + stored.household_id, + stored.policy_id, + stored.year, + stored.request_data, + stored.tax_benefit_model_version_id, + stored.country_package_version, + stored.policyengine_version, + ) + + +def _country(country: str) -> CountryId: + """Return the stored country as the contract names it.""" + for known in get_args(CountryId): + if country == known: + return known + msg = f"no v2 worker calculates economy runs for country {country}" + raise StoredRunUnreadableError(msg) + + +class RunRestartService: + """Start the calls of the runs and household jobs whose call never started. + + The store reads and writes are synchronous: the check is a batch job of + its own, so nothing else waits on its event loop. + """ + + def __init__( # noqa: PLR0913 # each seam tests replace + self, + *, + stores: RestartStores, + workers: RestartWorkers, + callers: RestartCallers, + settings: RestartSettings, + telemetry: RestartTelemetry | None = None, + clock: Callable[[], datetime] = lambda: datetime.now(UTC), + call_start_timeout_seconds: float = CALL_START_TIMEOUT_SECONDS, + new_request_id: Callable[[], str] = lambda: f"run-restart-{uuid.uuid4()}", + ) -> None: + """Use these seams; tests replace each of them.""" + self._stores = stores + self._workers = workers + self._callers = callers + self._settings = settings + self._telemetry = telemetry or NoRestartTelemetry() + self._clock = clock + self._call_start_timeout_seconds = call_start_timeout_seconds + self._new_request_id = new_request_id + + async def restart_unstarted_runs(self) -> RestartSummary: + """Start once more each row created 3 to 12 minutes ago with no call. + + Raise the store's error when the rows can't be listed. + """ + now = self._clock() + runs = self._stores.restarts.list_unstarted( + created_after=now - RESTART_UNTIL, + created_before=now - RESTART_AFTER, + limit=RESTARTS_PER_CHECK, + ) + outcomes: Counter[RestartOutcome] = Counter() + for run in runs: + age_seconds = (now - run.created_at).total_seconds() + with self._telemetry.restart_scope(run, age_seconds=age_seconds): + outcome, error_type = await self._restart(run) + self._telemetry.record_outcome(run, outcome, error_type=error_type) + outcomes[outcome] += 1 + return RestartSummary( + listed=len(runs), + **{outcome.value: count for outcome, count in outcomes.items()}, + ) + + async def _restart(self, run: UnstartedRun) -> tuple[RestartOutcome, str | None]: + """Start the row's call once more; return how it ended and any error type.""" + try: + rebuilt = self._rebuilt_call(run) + except _DeferredError as deferred: + return RestartOutcome.DEFERRED, str(deferred) + except _CannotRebuildError as cannot: + message = ( + f"{_NOT_STARTED}, and its call could not be built again: " + f"{cannot.reason}." + ) + failed = self._fail(run, message, cannot.simulation_run_ids) + # An unmarked row whose failure couldn't be written is listed again. + outcome = { + True: RestartOutcome.FAILED, + False: RestartOutcome.LEFT, + None: RestartOutcome.DEFERRED, + }[failed] + return outcome, cannot.error_type + if rebuilt is None: + return RestartOutcome.LEFT, None + start_call, simulation_run_ids = rebuilt + return await self._start_marked(run, start_call, simulation_run_ids) + + async def _start_marked( + self, + run: UnstartedRun, + start_call: CallStart, + simulation_run_ids: tuple[UUID, ...], + ) -> tuple[RestartOutcome, str | None]: + """Mark the row, then start its call and record the call's ID.""" + try: + if not self._stores.restarts.mark_restart(run): + return RestartOutcome.LEFT, None + except SQLAlchemyError as error: + return RestartOutcome.DEFERRED, type(error).__name__ + try: + call_id = await asyncio.wait_for( + start_call(self._telemetry.capture_context()), + timeout=self._call_start_timeout_seconds, + ) + # Modal raises its own errors; any of them, or the timeout, means the + # call was not confirmed, and the row is failed unless it has started. + except Exception as error: # noqa: BLE001 # every unconfirmed call is answered alike + message = ( + f"{_NOT_STARTED}, and Modal did not accept the call that started it " + f"again ({type(error).__name__})." + ) + failed = self._fail(run, message, simulation_run_ids) + # A failure that changed nothing means another call claimed the + # row; one that couldn't be written leaves the row to the sweep. + outcome = { + True: RestartOutcome.FAILED, + False: RestartOutcome.STARTED_MEANWHILE, + None: RestartOutcome.LEFT, + }[failed] + return outcome, type(error).__name__ + return self._recorded(run, call_id) + + def _recorded( + self, run: UnstartedRun, call_id: str + ) -> tuple[RestartOutcome, str | None]: + """Record the started call's ID; return how the restart ended.""" + try: + status = self._record(run, call_id) + except SQLAlchemyError as error: + return RestartOutcome.CALL_NOT_RECORDED, type(error).__name__ + if status is None: + return RestartOutcome.LEFT, None + if status == _PENDING: + return RestartOutcome.RESTARTED, None + return RestartOutcome.STARTED_MEANWHILE, None + + def _rebuilt_call( + self, run: UnstartedRun + ) -> tuple[CallStart, tuple[UUID, ...]] | None: + """Return what starts the row's call again and its report's simulation runs. + + Return None when the row is gone. Raise _DeferredError when its rows or + the manifest can't be read now, and _CannotRebuildError when its call + can't be built again. + """ + simulation_run_ids: tuple[UUID, ...] = () + try: + if run.kind is RestartKind.REPORT_RUN: + stored_report = self._stores.restarts.read_report_run(run.run_id) + if stored_report is None: + return None + simulation_run_ids = ( + stored_report.baseline.simulation_run_id, + stored_report.reform.simulation_run_id, + ) + return self._report_call(stored_report), simulation_run_ids + if run.kind is RestartKind.SIMULATION_RUN: + stored_simulation = self._stores.restarts.read_simulation_run( + run.run_id + ) + if stored_simulation is None: + return None + return self._simulation_call(stored_simulation), () + job = self._stores.households.read(run.run_id) + if job is None: + return None + return self._household_call(job), () + except (SQLAlchemyError, VersionManifestUnavailableError) as error: + raise _DeferredError(type(error).__name__) from error + except _REBUILD_ERRORS as error: + raise _CannotRebuildError( + _rebuild_reason(error), + error_type=type(error).__name__, + simulation_run_ids=simulation_run_ids, + ) from error + + def _economy_worker(self, country: str, versions: RunVersions) -> EconomyWorker: + """Return the worker of the run's .py version, which must match its versions.""" + worker = self._workers.economy.worker_for_version( + country, versions.policyengine_version + ) + if worker.versions != versions: + msg = ( + f"the v2 worker for .py {versions.policyengine_version} calculates " + "on other versions than the run records" + ) + raise _ChangedRunError(msg) + return worker + + def _economy_scope( + self, + worker: EconomyWorker, + country: str, + scope: EconomyScope, + options: dict[str, Any], + ) -> CalculationScope: + return CalculationScope( + country=_country(country), + region_code=scope.region_code, + dataset=bundle_dataset(worker, scope.dataset_name), + year=scope.year, + options=simulation_options( + include_cliffs=bool(options.get("include_cliffs", False)), + spm=options.get("spm"), + ), + ) + + def _origin(self, created_at: datetime) -> CallOrigin: + """Return where the restarted call comes from, dated as its row was created. + + The row's creation time places the call's artifacts where its first + call's would have been. + """ + return CallOrigin( + environment=self._settings.environment, + modal_environment=self._settings.modal_environment, + request_id=self._new_request_id(), + created_at=created_at, + ) + + def _report_call(self, stored: StoredReportRun) -> CallStart: + worker = self._economy_worker(stored.country, stored.versions) + calculation = report_run_calculation( + worker, + self._economy_scope(worker, stored.country, stored.scope, stored.options), + self._origin(stored.created_at), + run_ids=ReportRunIds( + report_run_id=stored.report_run_id, + baseline_simulation_run_id=stored.baseline.simulation_run_id, + reform_simulation_run_id=stored.reform.simulation_run_id, + ), + baseline_policy=stored.baseline.reform, + reform_policy=stored.reform.reform, + ) + caller = self._callers.economy + + def start(observability_context: dict[str, Any] | None) -> Awaitable[str]: + return caller.start_report_run( + worker, + calculation.payload, + calculation.context, + observability_context=observability_context, + ) + + return start + + def _simulation_call(self, stored: StoredSimulationRun) -> CallStart: + worker = self._economy_worker(stored.country, stored.versions) + calculation = simulation_run_calculation( + worker, + # A standalone simulation takes no report options. + self._economy_scope(worker, stored.country, stored.scope, {}), + self._origin(stored.created_at), + simulation_run_id=stored.run.simulation_run_id, + policy=stored.run.reform, + ) + caller = self._callers.economy + + def start(observability_context: dict[str, Any] | None) -> Awaitable[str]: + return caller.start_simulation_run( + worker, + calculation.payload, + calculation.context, + observability_context=observability_context, + ) + + return start + + def _household_call(self, job: HouseholdJobRecord) -> CallStart: + request = HouseholdCalculationRequest.model_validate(job.request_data) + worker = self._workers.households.worker_for_version( + job.country, job.policyengine_version + ) + inputs = self._stores.households.read_inputs( + country=request.country, + household_id=request.household_id, + policy_id=request.policy_id, + policyengine_version=job.policyengine_version, + ) + prepared = prepare_household_job(request, worker, inputs, job_id=job.id) + if not _same_job(prepared.job, job): + msg = ( + "its household, policy or catalog version changed since it was created" + ) + raise _ChangedRunError(msg) + caller = self._callers.households + + def start(observability_context: dict[str, Any] | None) -> Awaitable[str]: + return caller.spawn( + worker, + prepared.payload, + household_job_id=job.id, + observability_context=observability_context, + ) + + return start + + def _fail( + self, run: UnstartedRun, message: str, simulation_run_ids: tuple[UUID, ...] + ) -> bool | None: + """Fail the row while it is unstarted; return whether it was failed. + + Return None when the failure couldn't be written. + """ + try: + if run.kind is RestartKind.REPORT_RUN: + failed = self._stores.economy.fail_unstarted_report( + run.run_id, simulation_run_ids, message + ) + elif run.kind is RestartKind.SIMULATION_RUN: + failed = self._stores.economy.fail_unstarted_simulation( + run.run_id, message + ) + else: + failed = self._stores.households.fail_unstarted(run.run_id, message) + except SQLAlchemyError: + return None + return failed + + def _record(self, run: UnstartedRun, call_id: str) -> str | None: + """Record the call's ID while the row is unstarted; return its status after.""" + if run.kind is RestartKind.REPORT_RUN: + status = self._stores.economy.record_report_call(run.run_id, call_id) + elif run.kind is RestartKind.SIMULATION_RUN: + status = self._stores.economy.record_simulation_call(run.run_id, call_id) + else: + status = self._stores.households.record_call(run.run_id, call_id) + return None if status is None else str(status) From fa2236a188471c190947b90d38c7321e2840eab8 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:32:54 +0400 Subject: [PATCH 05/29] Build a restarted call in a module of its own The restart service keeps the check's loop and its writes; the call builder, its stores' protocols and the errors that defer or fail a restart move to run_restart_calls. Co-Authored-By: Claude Opus 5.5 --- .../run_restart_calls.py | 495 ++++++++++++++++++ .../run_restarts.py | 489 ++--------------- 2 files changed, 538 insertions(+), 446 deletions(-) create mode 100644 projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py new file mode 100644 index 000000000..8c7d12b9d --- /dev/null +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py @@ -0,0 +1,495 @@ +"""Rebuild the Modal call of a run or household job whose call never started. + +The restart check (``run_restarts``) starts such a row's call once more. The +call must be the one the row's submission started: ``RestartCallBuilder`` +reads back what the submission built it from and builds it again with the +submission's own builders (``report_run_calculation``, +``simulation_run_calculation`` and ``prepare_household_job``), on the worker +of the .py version the row records. A household job whose rebuilt job would +record anything else (another year, catalog version or policy) can't be +rebuilt, so the restart never calculates something its row doesn't record. +""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Final, Protocol, get_args + +from policyengine_simulation_contract.stage12_bundle import CountryId +from policyengine_simulation_contract.stage12_execution import ReportRunIds +from policyengine_simulation_persistence import ( + MissingPolicyParametersError, + OverlappingReformError, + RestartKind, + StoredRunUnreadableError, + UnknownCatalogVersionError, + UnknownDynamicError, + UnknownHouseholdError, + UnknownPolicyError, +) +from pydantic import ValidationError +from sqlalchemy.exc import SQLAlchemyError + +from policyengine_simulation_entry.economy_calculation_inputs import ( + CalculationScope, + CallOrigin, + DatasetNotInBundleError, + bundle_dataset, + report_run_calculation, + simulation_options, + simulation_run_calculation, +) +from policyengine_simulation_entry.economy_workers import ( + EconomyWorkerUnavailableError, +) +from policyengine_simulation_entry.household_api_models import ( + HouseholdCalculationRequest, +) +from policyengine_simulation_entry.household_calculations import ( + HouseholdCalculationInternalError, + HouseholdCalculationUnprocessableError, + prepare_household_job, +) +from policyengine_simulation_entry.household_workers import ( + HouseholdFunctionUnavailableError, + VersionManifestUnavailableError, +) + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable + from datetime import datetime + from uuid import UUID + + from policyengine_simulation_persistence import ( + EconomyScope, + HouseholdCalculationInputs, + HouseholdJobRecord, + HouseholdJobStatus, + NewHouseholdJob, + RunVersions, + StoredReportRun, + StoredSimulationRun, + UnstartedRun, + ) + + from policyengine_simulation_entry.economy_workers import ( + EconomyRunCaller, + EconomyWorker, + ) + from policyengine_simulation_entry.household_workers import ( + HouseholdFunctionCaller, + HouseholdWorker, + ) + +# The deployment each Modal environment holding the v2 workers belongs to, as +# the Simulation Entrypoint's deploy pairs them (APP_ENVIRONMENT). A call's +# context and artifact prefix name the deployment. +DEPLOYMENT_ENVIRONMENTS: Final = MappingProxyType( + {"staging": "staging", "main": "production"} +) + + +@dataclass(frozen=True) +class RestartSettings: + """The deployment the check restarts runs for. + + ``environment`` labels each call's context and artifacts, and + ``modal_environment`` holds the v2 workers. + """ + + environment: str + modal_environment: str + + @classmethod + def for_modal_environment(cls, modal_environment: str) -> RestartSettings: + """Return the settings of the deployment whose workers are in the environment. + + Raise ValueError for an environment no deployment uses. + """ + environment = DEPLOYMENT_ENVIRONMENTS.get(modal_environment) + if environment is None: + msg = ( + "no deployment runs the v2 workers in Modal environment " + f"{modal_environment!r}" + ) + raise ValueError(msg) + return cls(environment=environment, modal_environment=modal_environment) + + +class UnstartedRunSource(Protocol): + """The restart check's own reads and writes (``RunRestartStore``).""" + + def list_unstarted( + self, *, created_after: datetime, created_before: datetime, limit: int + ) -> tuple[UnstartedRun, ...]: + """Return the oldest restartable rows created in the window.""" + ... + + def mark_restart(self, run: UnstartedRun) -> bool: + """Mark the row restarted while it is still unstarted.""" + ... + + def read_report_run(self, report_run_id: UUID) -> StoredReportRun | None: + """Return what the report run's call is built from.""" + ... + + def read_simulation_run( + self, simulation_run_id: UUID + ) -> StoredSimulationRun | None: + """Return what the standalone simulation run's call is built from.""" + ... + + +class HouseholdJobRestarts(Protocol): + """The household job reads and writes a restart uses (``HouseholdJobStore``).""" + + def read(self, job_id: UUID) -> HouseholdJobRecord | None: + """Return the job as it stands.""" + ... + + def read_inputs( + self, + *, + country: str, + household_id: UUID, + policy_id: UUID | None, + policyengine_version: str, + ) -> HouseholdCalculationInputs: + """Read the catalog version, household and policy together.""" + ... + + def record_call( + self, job_id: UUID, modal_call_id: str + ) -> HouseholdJobStatus | None: + """Record the call ID while the job is pending with none.""" + ... + + def fail_unstarted(self, job_id: UUID, error_message: str) -> bool: + """Fail the job while it is pending with no call ID.""" + ... + + +class VersionedEconomyWorkers(Protocol): + """Find a .py version's economy functions (``ManifestEconomyWorkers``).""" + + def worker_for_version( + self, country: str, policyengine_version: str + ) -> EconomyWorker: + """Return the economy functions of the worker for the .py version.""" + ... + + +class VersionedHouseholdWorkers(Protocol): + """Find a .py version's household function (``ManifestHouseholdWorkers``).""" + + def worker_for_version( + self, country: str, policyengine_version: str + ) -> HouseholdWorker: + """Return the household function of the worker for the .py version.""" + ... + + +@dataclass(frozen=True) +class RestartWorkers: + """Where a restart finds the worker of a row's .py version.""" + + economy: VersionedEconomyWorkers + households: VersionedHouseholdWorkers + + +@dataclass(frozen=True) +class RestartCallers: + """What starts a restarted call on Modal.""" + + economy: EconomyRunCaller + households: HouseholdFunctionCaller + + +type CallStart = Callable[[dict[str, Any] | None], Awaitable[str]] + + +class DeferredRestartError(Exception): + """The row's rows or the manifest couldn't be read now; the message is its type.""" + + +class CallNotRebuiltError(Exception): + """The row's call can't be built again; ``reason`` says why, as a clause. + + ``simulation_run_ids`` are a report run's simulation runs, failed with it. + """ + + def __init__( + self, + reason: str, + *, + error_type: str, + simulation_run_ids: tuple[UUID, ...] = (), + ) -> None: + """Keep why, the error type that said so, and the report's simulation runs.""" + super().__init__(reason) + self.reason = reason + self.error_type = error_type + self.simulation_run_ids = simulation_run_ids + + +class _ChangedRunError(LookupError): + """What the call would be built from now differs from what the row records.""" + + +# The errors that mean a row's call can't be built again, and why, in words a +# failed run's error message can hold. A worker's absence is described by the +# error itself, which names only the .py version and the country. +_REBUILD_REASONS: Final[dict[type[Exception], str | None]] = { + EconomyWorkerUnavailableError: None, + HouseholdFunctionUnavailableError: None, + _ChangedRunError: None, + StoredRunUnreadableError: "its stored rows no longer describe one calculation", + UnknownCatalogVersionError: "its catalog version is gone", + UnknownHouseholdError: "its household is gone", + UnknownPolicyError: "its policy is gone", + UnknownDynamicError: "its dynamic is gone", + MissingPolicyParametersError: "its policy names parameters its catalog lacks", + OverlappingReformError: "its policy and dynamic set the same parameters", + DatasetNotInBundleError: "its worker's bundle has no dataset it names", + HouseholdCalculationUnprocessableError: "its household can no longer be read", + HouseholdCalculationInternalError: "its household request no longer validates", + ValidationError: "its stored request no longer validates", +} +_REBUILD_ERRORS: Final = tuple(_REBUILD_REASONS) + + +def _rebuild_reason(error: Exception) -> str: + reason = next( + ( + reason + for error_class, reason in _REBUILD_REASONS.items() + if isinstance(error, error_class) + ), + None, + ) + return str(error) if reason is None else reason + + +def _same_job(rebuilt: NewHouseholdJob, stored: HouseholdJobRecord) -> bool: + """Return whether the rebuilt job records what the stored job records.""" + return ( + rebuilt.id, + rebuilt.country, + rebuilt.household_id, + rebuilt.policy_id, + rebuilt.year, + rebuilt.request_data, + rebuilt.tax_benefit_model_version_id, + rebuilt.country_package_version, + rebuilt.policyengine_version, + ) == ( + stored.id, + stored.country, + stored.household_id, + stored.policy_id, + stored.year, + stored.request_data, + stored.tax_benefit_model_version_id, + stored.country_package_version, + stored.policyengine_version, + ) + + +def _country(country: str) -> CountryId: + """Return the stored country as the contract names it.""" + for known in get_args(CountryId): + if country == known: + return known + msg = f"no v2 worker calculates economy runs for country {country}" + raise StoredRunUnreadableError(msg) + + +@dataclass(frozen=True) +class RebuiltCall: + """What starts a row's call again, and a report run's simulation runs. + + ``start`` takes the observability context the call continues from. + """ + + start: CallStart + simulation_run_ids: tuple[UUID, ...] = () + + +class RestartCallBuilder: + """Build a row's call again from its stored rows.""" + + def __init__( # noqa: PLR0913 # each seam tests replace + self, + *, + restarts: UnstartedRunSource, + households: HouseholdJobRestarts, + workers: RestartWorkers, + callers: RestartCallers, + settings: RestartSettings, + new_request_id: Callable[[], str] = lambda: f"run-restart-{uuid.uuid4()}", + ) -> None: + """Read rows from the two stores; start calls with ``callers``.""" + self._restarts = restarts + self._households = households + self._workers = workers + self._callers = callers + self._settings = settings + self._new_request_id = new_request_id + + def build(self, run: UnstartedRun) -> RebuiltCall | None: + """Return what starts the row's call again, or None when the row is gone. + + Raise DeferredRestartError when its rows or the manifest can't be read + now, and CallNotRebuiltError when its call can't be built again. + """ + simulation_run_ids: tuple[UUID, ...] = () + try: + if run.kind is RestartKind.REPORT_RUN: + stored_report = self._restarts.read_report_run(run.run_id) + if stored_report is None: + return None + simulation_run_ids = ( + stored_report.baseline.simulation_run_id, + stored_report.reform.simulation_run_id, + ) + return RebuiltCall(self._report_call(stored_report), simulation_run_ids) + if run.kind is RestartKind.SIMULATION_RUN: + stored_simulation = self._restarts.read_simulation_run(run.run_id) + if stored_simulation is None: + return None + return RebuiltCall(self._simulation_call(stored_simulation)) + job = self._households.read(run.run_id) + if job is None: + return None + return RebuiltCall(self._household_call(job)) + except (SQLAlchemyError, VersionManifestUnavailableError) as error: + raise DeferredRestartError(type(error).__name__) from error + except _REBUILD_ERRORS as error: + raise CallNotRebuiltError( + _rebuild_reason(error), + error_type=type(error).__name__, + simulation_run_ids=simulation_run_ids, + ) from error + + def _economy_worker(self, country: str, versions: RunVersions) -> EconomyWorker: + """Return the worker of the run's .py version, which must match its versions.""" + worker = self._workers.economy.worker_for_version( + country, versions.policyengine_version + ) + if worker.versions != versions: + msg = ( + f"the v2 worker for .py {versions.policyengine_version} calculates " + "on other versions than the run records" + ) + raise _ChangedRunError(msg) + return worker + + def _economy_scope( + self, + worker: EconomyWorker, + country: str, + scope: EconomyScope, + options: dict[str, Any], + ) -> CalculationScope: + return CalculationScope( + country=_country(country), + region_code=scope.region_code, + dataset=bundle_dataset(worker, scope.dataset_name), + year=scope.year, + options=simulation_options( + include_cliffs=bool(options.get("include_cliffs", False)), + spm=options.get("spm"), + ), + ) + + def _origin(self, created_at: datetime) -> CallOrigin: + """Return where the restarted call comes from, dated as its row was created. + + The row's creation time places the call's artifacts where its first + call's would have been. + """ + return CallOrigin( + environment=self._settings.environment, + modal_environment=self._settings.modal_environment, + request_id=self._new_request_id(), + created_at=created_at, + ) + + def _report_call(self, stored: StoredReportRun) -> CallStart: + worker = self._economy_worker(stored.country, stored.versions) + calculation = report_run_calculation( + worker, + self._economy_scope(worker, stored.country, stored.scope, stored.options), + self._origin(stored.created_at), + run_ids=ReportRunIds( + report_run_id=stored.report_run_id, + baseline_simulation_run_id=stored.baseline.simulation_run_id, + reform_simulation_run_id=stored.reform.simulation_run_id, + ), + baseline_policy=stored.baseline.reform, + reform_policy=stored.reform.reform, + ) + caller = self._callers.economy + + def start(observability_context: dict[str, Any] | None) -> Awaitable[str]: + return caller.start_report_run( + worker, + calculation.payload, + calculation.context, + observability_context=observability_context, + ) + + return start + + def _simulation_call(self, stored: StoredSimulationRun) -> CallStart: + worker = self._economy_worker(stored.country, stored.versions) + calculation = simulation_run_calculation( + worker, + # A standalone simulation takes no report options. + self._economy_scope(worker, stored.country, stored.scope, {}), + self._origin(stored.created_at), + simulation_run_id=stored.run.simulation_run_id, + policy=stored.run.reform, + ) + caller = self._callers.economy + + def start(observability_context: dict[str, Any] | None) -> Awaitable[str]: + return caller.start_simulation_run( + worker, + calculation.payload, + calculation.context, + observability_context=observability_context, + ) + + return start + + def _household_call(self, job: HouseholdJobRecord) -> CallStart: + request = HouseholdCalculationRequest.model_validate(job.request_data) + worker = self._workers.households.worker_for_version( + job.country, job.policyengine_version + ) + inputs = self._households.read_inputs( + country=request.country, + household_id=request.household_id, + policy_id=request.policy_id, + policyengine_version=job.policyengine_version, + ) + prepared = prepare_household_job(request, worker, inputs, job_id=job.id) + if not _same_job(prepared.job, job): + msg = ( + "its household, policy or catalog version changed since it was created" + ) + raise _ChangedRunError(msg) + caller = self._callers.households + + def start(observability_context: dict[str, Any] | None) -> Awaitable[str]: + return caller.spawn( + worker, + prepared.payload, + household_job_id=job.id, + observability_context=observability_context, + ) + + return start diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py index 14b46f204..0dfe483d7 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py @@ -38,76 +38,51 @@ from dataclasses import dataclass from datetime import UTC, datetime, timedelta from enum import StrEnum -from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Protocol, get_args - -from policyengine_simulation_contract.stage12_bundle import CountryId -from policyengine_simulation_contract.stage12_execution import ReportRunIds -from policyengine_simulation_persistence import ( - MissingPolicyParametersError, - OverlappingReformError, - RestartKind, - StoredRunUnreadableError, - UnknownCatalogVersionError, - UnknownDynamicError, - UnknownHouseholdError, - UnknownPolicyError, -) -from pydantic import ValidationError +from typing import TYPE_CHECKING, Final, Protocol + +from policyengine_simulation_persistence import RestartKind from sqlalchemy.exc import SQLAlchemyError -from policyengine_simulation_entry.economy_calculation_inputs import ( - CalculationScope, - CallOrigin, - DatasetNotInBundleError, - bundle_dataset, - report_run_calculation, - simulation_options, - simulation_run_calculation, -) -from policyengine_simulation_entry.economy_workers import ( - EconomyWorkerUnavailableError, -) -from policyengine_simulation_entry.household_api_models import ( - HouseholdCalculationRequest, -) -from policyengine_simulation_entry.household_calculations import ( - HouseholdCalculationInternalError, - HouseholdCalculationUnprocessableError, - prepare_household_job, -) -from policyengine_simulation_entry.household_workers import ( - HouseholdFunctionUnavailableError, - VersionManifestUnavailableError, +from policyengine_simulation_entry.run_restart_calls import ( + CallNotRebuiltError, + DeferredRestartError, + HouseholdJobRestarts, + RestartCallBuilder, + RestartCallers, + RestartSettings, + RestartWorkers, + UnstartedRunSource, ) if TYPE_CHECKING: - from collections.abc import Awaitable, Callable, Sequence + from collections.abc import Callable, Sequence from contextlib import AbstractContextManager from uuid import UUID from policyengine_simulation_persistence import ( - EconomyScope, - HouseholdCalculationInputs, - HouseholdJobRecord, - HouseholdJobStatus, - NewHouseholdJob, ReportRunStatus, - RunVersions, SimulationStatus, - StoredReportRun, - StoredSimulationRun, UnstartedRun, ) - from policyengine_simulation_entry.economy_workers import ( - EconomyRunCaller, - EconomyWorker, - ) - from policyengine_simulation_entry.household_workers import ( - HouseholdFunctionCaller, - HouseholdWorker, - ) + from policyengine_simulation_entry.run_restart_calls import CallStart + +__all__ = [ + "CALL_START_TIMEOUT_SECONDS", + "RESTARTS_PER_CHECK", + "RESTART_AFTER", + "RESTART_UNTIL", + "EconomyRunWrites", + "NoRestartTelemetry", + "RestartCallers", + "RestartOutcome", + "RestartSettings", + "RestartStores", + "RestartSummary", + "RestartTelemetry", + "RestartWorkers", + "RunRestartService", +] # A row is restarted once it has stayed pending with no call this long, which # is far longer than a submission takes to start and record its call. @@ -119,12 +94,6 @@ RESTARTS_PER_CHECK: Final = 50 # How long the check waits for Modal to accept a restarted call. CALL_START_TIMEOUT_SECONDS: Final = 10.0 -# The deployment each Modal environment holding the v2 workers belongs to, as -# the Simulation Entrypoint's deploy pairs them (APP_ENVIRONMENT). A call's -# context and artifact prefix name the deployment. -DEPLOYMENT_ENVIRONMENTS: Final = MappingProxyType( - {"staging": "staging", "main": "production"} -) _NOT_STARTED: Final = ( "run_not_started: The run stayed pending with no Modal call recorded for it" ) @@ -153,33 +122,6 @@ class RestartOutcome(StrEnum): FAILED = "failed" -@dataclass(frozen=True) -class RestartSettings: - """The deployment the check restarts runs for. - - ``environment`` labels each call's context and artifacts, and - ``modal_environment`` holds the v2 workers. - """ - - environment: str - modal_environment: str - - @classmethod - def for_modal_environment(cls, modal_environment: str) -> RestartSettings: - """Return the settings of the deployment whose workers are in the environment. - - Raise ValueError for an environment no deployment uses. - """ - environment = DEPLOYMENT_ENVIRONMENTS.get(modal_environment) - if environment is None: - msg = ( - "no deployment runs the v2 workers in Modal environment " - f"{modal_environment!r}" - ) - raise ValueError(msg) - return cls(environment=environment, modal_environment=modal_environment) - - @dataclass(frozen=True) class RestartSummary: """How many rows one check listed, and how each ended (``RestartOutcome``).""" @@ -193,30 +135,6 @@ class RestartSummary: failed: int = 0 -class UnstartedRunSource(Protocol): - """The restart check's own reads and writes (``RunRestartStore``).""" - - def list_unstarted( - self, *, created_after: datetime, created_before: datetime, limit: int - ) -> tuple[UnstartedRun, ...]: - """Return the oldest restartable rows created in the window.""" - ... - - def mark_restart(self, run: UnstartedRun) -> bool: - """Mark the row restarted while it is still unstarted.""" - ... - - def read_report_run(self, report_run_id: UUID) -> StoredReportRun | None: - """Return what the report run's call is built from.""" - ... - - def read_simulation_run( - self, simulation_run_id: UUID - ) -> StoredSimulationRun | None: - """Return what the standalone simulation run's call is built from.""" - ... - - class EconomyRunWrites(Protocol): """The submission's own call writes on economy runs (``EconomyRunStore``).""" @@ -248,55 +166,6 @@ def fail_unstarted_simulation( ... -class HouseholdJobRestarts(Protocol): - """The household job reads and writes a restart uses (``HouseholdJobStore``).""" - - def read(self, job_id: UUID) -> HouseholdJobRecord | None: - """Return the job as it stands.""" - ... - - def read_inputs( - self, - *, - country: str, - household_id: UUID, - policy_id: UUID | None, - policyengine_version: str, - ) -> HouseholdCalculationInputs: - """Read the catalog version, household and policy together.""" - ... - - def record_call( - self, job_id: UUID, modal_call_id: str - ) -> HouseholdJobStatus | None: - """Record the call ID while the job is pending with none.""" - ... - - def fail_unstarted(self, job_id: UUID, error_message: str) -> bool: - """Fail the job while it is pending with no call ID.""" - ... - - -class VersionedEconomyWorkers(Protocol): - """Find a .py version's economy functions (``ManifestEconomyWorkers``).""" - - def worker_for_version( - self, country: str, policyengine_version: str - ) -> EconomyWorker: - """Return the economy functions of the worker for the .py version.""" - ... - - -class VersionedHouseholdWorkers(Protocol): - """Find a .py version's household function (``ManifestHouseholdWorkers``).""" - - def worker_for_version( - self, country: str, policyengine_version: str - ) -> HouseholdWorker: - """Return the household function of the worker for the .py version.""" - ... - - class RestartTelemetry(Protocol): """Where the check reports each listed row (``ObservabilityRestartTelemetry``).""" @@ -356,120 +225,6 @@ class RestartStores: households: HouseholdJobRestarts -@dataclass(frozen=True) -class RestartWorkers: - """Where a restart finds the worker of a row's .py version.""" - - economy: VersionedEconomyWorkers - households: VersionedHouseholdWorkers - - -@dataclass(frozen=True) -class RestartCallers: - """What starts a restarted call on Modal.""" - - economy: EconomyRunCaller - households: HouseholdFunctionCaller - - -type CallStart = Callable[[dict[str, Any] | None], Awaitable[str]] - - -class _DeferredError(Exception): - """The row's rows or the manifest couldn't be read now.""" - - -class _CannotRebuildError(Exception): - """The row's call can't be built again; ``reason`` says why, in a sentence part. - - ``simulation_run_ids`` are a report run's simulation runs, failed with it. - """ - - def __init__( - self, - reason: str, - *, - error_type: str, - simulation_run_ids: tuple[UUID, ...] = (), - ) -> None: - super().__init__(reason) - self.reason = reason - self.error_type = error_type - self.simulation_run_ids = simulation_run_ids - - -class _ChangedRunError(LookupError): - """What the call would be built from now differs from what the row records.""" - - -# The errors that mean a row's call can't be built again, and why, in words a -# failed run's error message can hold. A worker's absence is described by the -# error itself, which names only the .py version and the country. -_REBUILD_REASONS: Final[dict[type[Exception], str | None]] = { - EconomyWorkerUnavailableError: None, - HouseholdFunctionUnavailableError: None, - _ChangedRunError: None, - StoredRunUnreadableError: "its stored rows no longer describe one calculation", - UnknownCatalogVersionError: "its catalog version is gone", - UnknownHouseholdError: "its household is gone", - UnknownPolicyError: "its policy is gone", - UnknownDynamicError: "its dynamic is gone", - MissingPolicyParametersError: "its policy names parameters its catalog lacks", - OverlappingReformError: "its policy and dynamic set the same parameters", - DatasetNotInBundleError: "its worker's bundle has no dataset it names", - HouseholdCalculationUnprocessableError: "its household can no longer be read", - HouseholdCalculationInternalError: "its household request no longer validates", - ValidationError: "its stored request no longer validates", -} -_REBUILD_ERRORS: Final = tuple(_REBUILD_REASONS) - - -def _rebuild_reason(error: Exception) -> str: - reason = next( - ( - reason - for error_class, reason in _REBUILD_REASONS.items() - if isinstance(error, error_class) - ), - None, - ) - return str(error) if reason is None else reason - - -def _same_job(rebuilt: NewHouseholdJob, stored: HouseholdJobRecord) -> bool: - """Return whether the rebuilt job records what the stored job records.""" - return ( - rebuilt.id, - rebuilt.country, - rebuilt.household_id, - rebuilt.policy_id, - rebuilt.year, - rebuilt.request_data, - rebuilt.tax_benefit_model_version_id, - rebuilt.country_package_version, - rebuilt.policyengine_version, - ) == ( - stored.id, - stored.country, - stored.household_id, - stored.policy_id, - stored.year, - stored.request_data, - stored.tax_benefit_model_version_id, - stored.country_package_version, - stored.policyengine_version, - ) - - -def _country(country: str) -> CountryId: - """Return the stored country as the contract names it.""" - for known in get_args(CountryId): - if country == known: - return known - msg = f"no v2 worker calculates economy runs for country {country}" - raise StoredRunUnreadableError(msg) - - class RunRestartService: """Start the calls of the runs and household jobs whose call never started. @@ -491,13 +246,17 @@ def __init__( # noqa: PLR0913 # each seam tests replace ) -> None: """Use these seams; tests replace each of them.""" self._stores = stores - self._workers = workers - self._callers = callers - self._settings = settings + self._calls = RestartCallBuilder( + restarts=stores.restarts, + households=stores.households, + workers=workers, + callers=callers, + settings=settings, + new_request_id=new_request_id, + ) self._telemetry = telemetry or NoRestartTelemetry() self._clock = clock self._call_start_timeout_seconds = call_start_timeout_seconds - self._new_request_id = new_request_id async def restart_unstarted_runs(self) -> RestartSummary: """Start once more each row created 3 to 12 minutes ago with no call. @@ -525,10 +284,10 @@ async def restart_unstarted_runs(self) -> RestartSummary: async def _restart(self, run: UnstartedRun) -> tuple[RestartOutcome, str | None]: """Start the row's call once more; return how it ended and any error type.""" try: - rebuilt = self._rebuilt_call(run) - except _DeferredError as deferred: + rebuilt = self._calls.build(run) + except DeferredRestartError as deferred: return RestartOutcome.DEFERRED, str(deferred) - except _CannotRebuildError as cannot: + except CallNotRebuiltError as cannot: message = ( f"{_NOT_STARTED}, and its call could not be built again: " f"{cannot.reason}." @@ -543,8 +302,7 @@ async def _restart(self, run: UnstartedRun) -> tuple[RestartOutcome, str | None] return outcome, cannot.error_type if rebuilt is None: return RestartOutcome.LEFT, None - start_call, simulation_run_ids = rebuilt - return await self._start_marked(run, start_call, simulation_run_ids) + return await self._start_marked(run, rebuilt.start, rebuilt.simulation_run_ids) async def _start_marked( self, @@ -595,167 +353,6 @@ def _recorded( return RestartOutcome.RESTARTED, None return RestartOutcome.STARTED_MEANWHILE, None - def _rebuilt_call( - self, run: UnstartedRun - ) -> tuple[CallStart, tuple[UUID, ...]] | None: - """Return what starts the row's call again and its report's simulation runs. - - Return None when the row is gone. Raise _DeferredError when its rows or - the manifest can't be read now, and _CannotRebuildError when its call - can't be built again. - """ - simulation_run_ids: tuple[UUID, ...] = () - try: - if run.kind is RestartKind.REPORT_RUN: - stored_report = self._stores.restarts.read_report_run(run.run_id) - if stored_report is None: - return None - simulation_run_ids = ( - stored_report.baseline.simulation_run_id, - stored_report.reform.simulation_run_id, - ) - return self._report_call(stored_report), simulation_run_ids - if run.kind is RestartKind.SIMULATION_RUN: - stored_simulation = self._stores.restarts.read_simulation_run( - run.run_id - ) - if stored_simulation is None: - return None - return self._simulation_call(stored_simulation), () - job = self._stores.households.read(run.run_id) - if job is None: - return None - return self._household_call(job), () - except (SQLAlchemyError, VersionManifestUnavailableError) as error: - raise _DeferredError(type(error).__name__) from error - except _REBUILD_ERRORS as error: - raise _CannotRebuildError( - _rebuild_reason(error), - error_type=type(error).__name__, - simulation_run_ids=simulation_run_ids, - ) from error - - def _economy_worker(self, country: str, versions: RunVersions) -> EconomyWorker: - """Return the worker of the run's .py version, which must match its versions.""" - worker = self._workers.economy.worker_for_version( - country, versions.policyengine_version - ) - if worker.versions != versions: - msg = ( - f"the v2 worker for .py {versions.policyengine_version} calculates " - "on other versions than the run records" - ) - raise _ChangedRunError(msg) - return worker - - def _economy_scope( - self, - worker: EconomyWorker, - country: str, - scope: EconomyScope, - options: dict[str, Any], - ) -> CalculationScope: - return CalculationScope( - country=_country(country), - region_code=scope.region_code, - dataset=bundle_dataset(worker, scope.dataset_name), - year=scope.year, - options=simulation_options( - include_cliffs=bool(options.get("include_cliffs", False)), - spm=options.get("spm"), - ), - ) - - def _origin(self, created_at: datetime) -> CallOrigin: - """Return where the restarted call comes from, dated as its row was created. - - The row's creation time places the call's artifacts where its first - call's would have been. - """ - return CallOrigin( - environment=self._settings.environment, - modal_environment=self._settings.modal_environment, - request_id=self._new_request_id(), - created_at=created_at, - ) - - def _report_call(self, stored: StoredReportRun) -> CallStart: - worker = self._economy_worker(stored.country, stored.versions) - calculation = report_run_calculation( - worker, - self._economy_scope(worker, stored.country, stored.scope, stored.options), - self._origin(stored.created_at), - run_ids=ReportRunIds( - report_run_id=stored.report_run_id, - baseline_simulation_run_id=stored.baseline.simulation_run_id, - reform_simulation_run_id=stored.reform.simulation_run_id, - ), - baseline_policy=stored.baseline.reform, - reform_policy=stored.reform.reform, - ) - caller = self._callers.economy - - def start(observability_context: dict[str, Any] | None) -> Awaitable[str]: - return caller.start_report_run( - worker, - calculation.payload, - calculation.context, - observability_context=observability_context, - ) - - return start - - def _simulation_call(self, stored: StoredSimulationRun) -> CallStart: - worker = self._economy_worker(stored.country, stored.versions) - calculation = simulation_run_calculation( - worker, - # A standalone simulation takes no report options. - self._economy_scope(worker, stored.country, stored.scope, {}), - self._origin(stored.created_at), - simulation_run_id=stored.run.simulation_run_id, - policy=stored.run.reform, - ) - caller = self._callers.economy - - def start(observability_context: dict[str, Any] | None) -> Awaitable[str]: - return caller.start_simulation_run( - worker, - calculation.payload, - calculation.context, - observability_context=observability_context, - ) - - return start - - def _household_call(self, job: HouseholdJobRecord) -> CallStart: - request = HouseholdCalculationRequest.model_validate(job.request_data) - worker = self._workers.households.worker_for_version( - job.country, job.policyengine_version - ) - inputs = self._stores.households.read_inputs( - country=request.country, - household_id=request.household_id, - policy_id=request.policy_id, - policyengine_version=job.policyengine_version, - ) - prepared = prepare_household_job(request, worker, inputs, job_id=job.id) - if not _same_job(prepared.job, job): - msg = ( - "its household, policy or catalog version changed since it was created" - ) - raise _ChangedRunError(msg) - caller = self._callers.households - - def start(observability_context: dict[str, Any] | None) -> Awaitable[str]: - return caller.spawn( - worker, - prepared.payload, - household_job_id=job.id, - observability_context=observability_context, - ) - - return start - def _fail( self, run: UnstartedRun, message: str, simulation_run_ids: tuple[UUID, ...] ) -> bool | None: From 116780d76b1b4c5e9acfc55ad1f7e2a093975851 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:36:39 +0400 Subject: [PATCH 06/29] Test the scheduled restart check, its telemetry and the sweep's age (failing) The restart check runs every two minutes in the sweep's Modal app on the Modal worker's database secret, with the Modal environment and the v2 workers' telemetry settings baked into its image; each check and each restart is an operation named by a registered stage, with the restart's outcome logged as an event. The sweep now waits 15 minutes before failing a pending run with no call, after the restart window. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_stages.py | 11 + .../tests/test_run_restart_check.py | 221 ++++++++++++++++++ .../tests/test_stage12_run_sweep.py | 16 ++ .../tests/test_stage12_run_sweep_app.py | 127 +++++++++- 4 files changed, 370 insertions(+), 5 deletions(-) create mode 100644 projects/policyengine-simulation-entry/tests/test_run_restart_check.py diff --git a/libs/policyengine-simulation-observability/tests/test_stages.py b/libs/policyengine-simulation-observability/tests/test_stages.py index 0861363b9..bfa157a42 100644 --- a/libs/policyengine-simulation-observability/tests/test_stages.py +++ b/libs/policyengine-simulation-observability/tests/test_stages.py @@ -1,6 +1,7 @@ import pytest from policyengine_simulation_observability.stages import ( RUN_STAGE_REGISTRY, + V2_RUN_RESTART_STAGES, RunConfiguration, Stage, ) @@ -66,3 +67,13 @@ def test_v2_household_plan_measures_model_load_inputs_and_each_policy() -> None: Stage.V2_HOUSEHOLD_CALCULATION, ) assert household.name(Stage.V2_HOUSEHOLD_CALCULATION) == "v2_household_calculation" + + +def test_v2_run_restart_plan_measures_each_check_and_each_restart() -> None: + """The scheduled check is one operation; each row it lists is one more.""" + restart = RUN_STAGE_REGISTRY[RunConfiguration.V2_RUN_RESTART] + + assert restart is V2_RUN_RESTART_STAGES + assert restart.stages == (Stage.V2_RUN_RESTART_CHECK, Stage.V2_RUN_RESTART) + assert restart.name(Stage.V2_RUN_RESTART_CHECK) == "v2_run_restart_check" + assert restart.name(Stage.V2_RUN_RESTART) == "v2_run_restart" diff --git a/projects/policyengine-simulation-entry/tests/test_run_restart_check.py b/projects/policyengine-simulation-entry/tests/test_run_restart_check.py new file mode 100644 index 000000000..4b778a3e1 --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_run_restart_check.py @@ -0,0 +1,221 @@ +"""The scheduled restart check's telemetry, and the service it runs.""" + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass, field +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any +from uuid import UUID + +import pytest +from policyengine_simulation_entry.run_restart_check import ( + ObservabilityRestartTelemetry, + run_scheduled_check, + scheduled_restart_service, +) +from policyengine_simulation_entry.run_restarts import ( + RestartOutcome, + RestartSettings, + RestartSummary, + RunRestartService, +) +from policyengine_simulation_persistence import RestartKind, UnstartedRun + +if TYPE_CHECKING: + from collections.abc import Iterator + +RUN = UnstartedRun( + kind=RestartKind.REPORT_RUN, + run_id=UUID("00000000-0000-4000-8000-0000000000e1"), + created_at=datetime(2026, 11, 13, 9, tzinfo=UTC), +) +DATABASE_URL = "postgresql://worker:secret@database.example/postgres" + + +@dataclass +class RecordingRuntime: + """Records what the telemetry asks of the observability runtime.""" + + operations: list[tuple[str, dict[str, Any]]] = field(default_factory=list) + contexts: list[dict[str, Any]] = field(default_factory=list) + events: list[tuple[str, str, dict[str, Any]]] = field(default_factory=list) + open_operations: list[str] = field(default_factory=list) + + @contextmanager + def operation( + self, name: str, *, attributes: dict[str, Any] | None = None + ) -> Iterator[None]: + self.operations.append((name, dict(attributes or {}))) + self.open_operations.append(name) + try: + yield + finally: + self.open_operations.pop() + + def set_context(self, **attributes: Any) -> None: + self.contexts.append({"within": list(self.open_operations), **attributes}) + + def event( + self, + name: str, + *, + severity: str = "INFO", + attributes: dict[str, Any] | None = None, + ) -> None: + self.events.append((name, severity, dict(attributes or {}))) + + def capture_context(self) -> dict[str, str]: + return {"traceparent": "00-restart-check"} + + +def test_each_restart_is_an_operation_named_by_its_registered_stage() -> None: + runtime = RecordingRuntime() + telemetry = ObservabilityRestartTelemetry(runtime) + + with telemetry.restart_scope(RUN, age_seconds=210.5): + within = list(runtime.open_operations) + + assert within == ["v2_run_restart"] + assert runtime.operations == [ + ( + "v2_run_restart", + { + "run_kind": "report_run", + "run_id": str(RUN.run_id), + "run_age_seconds": 210.5, + }, + ) + ] + + +@pytest.mark.parametrize( + ("outcome", "error_type", "severity"), + [ + (RestartOutcome.RESTARTED, None, "INFO"), + (RestartOutcome.STARTED_MEANWHILE, None, "INFO"), + (RestartOutcome.LEFT, None, "INFO"), + (RestartOutcome.DEFERRED, "OperationalError", "WARNING"), + (RestartOutcome.CALL_NOT_RECORDED, "OperationalError", "WARNING"), + (RestartOutcome.FAILED, "ConnectionError", "WARNING"), + ], +) +def test_an_outcome_is_set_on_the_restart_and_logged_as_an_event( + outcome, error_type, severity +) -> None: + runtime = RecordingRuntime() + telemetry = ObservabilityRestartTelemetry(runtime) + + with telemetry.restart_scope(RUN, age_seconds=200.0): + telemetry.record_outcome(RUN, outcome, error_type=error_type) + + expected = {"restart_outcome": outcome.value} + if error_type is not None: + expected["restart_error_type"] = error_type + assert runtime.contexts == [{"within": ["v2_run_restart"], **expected}] + assert runtime.events == [ + ( + "v2_run_restart_outcome", + severity, + {"run_kind": "report_run", "run_id": str(RUN.run_id), **expected}, + ) + ] + + +def test_a_restarted_call_continues_the_checks_trace() -> None: + telemetry = ObservabilityRestartTelemetry(RecordingRuntime()) + + assert telemetry.capture_context() == {"traceparent": "00-restart-check"} + + +@dataclass +class FakeRestartService: + """Answers one check with a summary, noting the operation it ran in.""" + + runtime: RecordingRuntime + summary: RestartSummary + ran_within: list[list[str]] = field(default_factory=list) + + async def restart_unstarted_runs(self) -> RestartSummary: + self.ran_within.append(list(self.runtime.open_operations)) + return self.summary + + +def test_a_check_runs_in_its_own_operation_and_answers_its_counts() -> None: + runtime = RecordingRuntime() + summary = RestartSummary(listed=3, restarted=1, failed=1, left=1) + restart_service = FakeRestartService(runtime, summary) + + counts = run_scheduled_check(restart_service, runtime) + + assert counts == { + "listed": 3, + "restarted": 1, + "call_not_recorded": 0, + "started_meanwhile": 0, + "left": 1, + "deferred": 0, + "failed": 1, + } + assert restart_service.ran_within == [["v2_run_restart_check"]] + assert runtime.operations == [("v2_run_restart_check", {})] + assert runtime.contexts == [ + {"within": ["v2_run_restart_check"]} + | {f"restart_{name}": count for name, count in counts.items()} + ] + + +def test_the_scheduled_service_restarts_in_its_modal_environments_deployment() -> None: + scheduled = scheduled_restart_service( + database_url=DATABASE_URL, + modal_environment="main", + runtime=RecordingRuntime(), + ) + try: + assert isinstance(scheduled.service, RunRestartService) + assert scheduled.settings == RestartSettings( + environment="production", modal_environment="main" + ) + finally: + scheduled.engine.dispose() + + +def test_the_scheduled_service_refuses_a_modal_environment_no_deployment_uses() -> None: + with pytest.raises(ValueError, match="local"): + scheduled_restart_service( + database_url=DATABASE_URL, + modal_environment="local", + runtime=RecordingRuntime(), + ) + + +def test_the_restart_module_imports_without_the_web_service(tmp_path) -> None: + """The restart check runs in the sweep's image, which has no web packages. + + Importing it must not import the FastAPI app, whose authentication needs + PyJWT and cryptography, which that image does not install. + """ + import subprocess # this test alone starts a process + import sys # this test alone starts a process + + script = tmp_path / "import_restart_check.py" + script.write_text( + "import sys\n" + "class Refuse:\n" + " def find_spec(self, name, path=None, target=None):\n" + " if name.split('.')[0] in {'jwt', 'cryptography', 'uvicorn'}:\n" + " raise ImportError(name)\n" + "sys.meta_path.insert(0, Refuse())\n" + "import policyengine_simulation_entry.run_restart_check\n" + "assert 'policyengine_simulation_entry.app' not in sys.modules\n", + encoding="utf-8", + ) + + completed = subprocess.run( # the test's own interpreter and script + [sys.executable, str(script)], + capture_output=True, + text=True, + check=False, + ) + + assert completed.returncode == 0, completed.stderr diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep.py b/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep.py index f72b380b8..0ad0be027 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep.py @@ -8,8 +8,10 @@ from uuid import uuid4 import pytest +from policyengine_simulation_entry.run_restarts import RESTART_UNTIL from policyengine_simulation_executor.stage12_runtime.run_sweep import ( DEFAULT_MAXIMUM_AGE, + DEFAULT_MINIMUM_AGE, CallState, SweepSummary, modal_call_state, @@ -144,6 +146,20 @@ def test_the_sweep_lists_runs_older_than_its_minimum_age() -> None: assert store.cutoffs == [NOW - timedelta(minutes=15)] * 3 +def test_by_default_unstarted_runs_wait_until_the_restart_check_has_passed() -> None: + """The restart check starts a pending run with no call 3 to 12 minutes old. + + The sweep fails such a run only once it is 15 minutes old, so a run whose + restart started is judged by its call, and only a second miss is failed. + """ + store = FakeSweepStore([]) + + sweep_stuck_runs(store, call_state=lambda _: CallState.ENDED, now=NOW) + + assert store.cutoffs == [NOW - timedelta(minutes=15)] * 3 + assert DEFAULT_MINIMUM_AGE > RESTART_UNTIL + + def test_report_runs_are_swept_before_simulation_runs_are_listed() -> None: # A simulation run is listed only once no report run naming it is # running, so failing a dead coordinator's report run first lets the same diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep_app.py b/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep_app.py index ffccee8ab..3266f683d 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep_app.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep_app.py @@ -8,10 +8,12 @@ import yaml from fixtures.fake_modal import install_fake_modal +from policyengine_simulation_entry import run_restart_check from policyengine_simulation_executor.stage12_runtime import ( report_run_access as report_run_access_module, ) from policyengine_simulation_executor.stage12_runtime import run_sweep +from policyengine_simulation_observability import observability REUSABLE_DEPLOY_WORKFLOW = ( Path(__file__).resolve().parents[3] @@ -27,6 +29,23 @@ def _load(monkeypatch): return importlib.import_module("src.modal.run_sweep_app") +def _deploy_steps(job: dict, app_file: str) -> list[dict]: + return [step for step in job.get("steps", []) if app_file in step.get("run", "")] + + +def _telemetry_settings(environment: dict[str, str]) -> dict[str, str]: + """Return the deploy-time settings ``modal_image_environment`` bakes in.""" + return { + name: value + for name, value in environment.items() + if name.startswith(("OBSERVABILITY_", "OTEL_", "POLICYENGINE_OTEL_")) + } + + +def _functions(module) -> dict[str, dict]: + return dict(module.app.function_calls) + + def _sweep_deploy_steps(job: dict) -> list[dict]: return [ step @@ -65,11 +84,18 @@ def test_the_sweep_deploys_with_the_stage12_default_after_every_check() -> None: assert ( sweep_step["working-directory"] == "projects/policyengine-simulation-executor" ) - # Only Modal's tokens: the database URL reaches the scheduled function from - # its Modal secret, which must exist before this step can deploy it. + # Modal's tokens, the Modal environment the restart check starts calls in, + # and the v2 workers' own telemetry settings, which the image bakes in. + # The database URL reaches the scheduled functions from their Modal + # secret, which must exist before this step can deploy them. + (v2_workers_step,) = _deploy_steps(jobs["deploy_stage12_v2"], "src/modal/v2_app.py") + telemetry = _telemetry_settings(v2_workers_step["env"]) + assert len(telemetry) == 8 assert sweep_step["env"] == { "MODAL_TOKEN_ID": "${{ secrets.MODAL_TOKEN_ID }}", "MODAL_TOKEN_SECRET": "${{ secrets.MODAL_TOKEN_SECRET }}", + "SIMULATION_V2_MODAL_ENVIRONMENT": "${{ inputs.modal_environment }}", + **telemetry, } # Only from main. With no status function in it, the condition also # requires every earlier step to have passed (an implicit success()). @@ -85,10 +111,9 @@ def test_the_sweep_runs_every_ten_minutes_with_the_worker_database_secret( monkeypatch, ) -> None: module = _load(monkeypatch) - ((name, options),) = module.app.function_calls + options = _functions(module)["sweep_stuck_runs"] assert module.app.name == "policyengine-simulation-run-sweep" - assert name == "sweep_stuck_runs" assert options["schedule"] == {"schedule": "*/10 * * * *"} assert options["secrets"] == [ { @@ -110,13 +135,16 @@ def test_the_sweep_image_installs_the_locked_group_and_the_runtime_sources( assert calls["uv_sync"][1]["frozen"] is True assert calls["uv_sync"][1]["extra_options"] == "--only-group modal-simulation-image" - # The app module imports src.modal at module level, in the container too. + # The app module imports src.modal at module level, in the container too; + # the restart check runs the simulation service's own restart code. assert calls["add_local_python_source"][0] == ( "src.modal", "policyengine_simulation_executor", "policyengine_simulation_observability", "policyengine_simulation_contract", "policyengine_stage12_persistence", + "policyengine_simulation_persistence", + "policyengine_simulation_entry", ) @@ -150,3 +178,92 @@ def recording_sweep(received_store): "failed_without_call": 4, } assert swept == [store] + + +def test_the_restart_check_runs_every_two_minutes_with_the_worker_database_secret( + monkeypatch, +) -> None: + """It restarts runs 3 to 12 minutes old, so each is seen several times.""" + module = _load(monkeypatch) + options = _functions(module)["restart_unstarted_runs"] + + assert options["schedule"] == {"schedule": "*/2 * * * *"} + assert options["secrets"] == [ + { + "args": ("policyengine-modal-worker-database",), + "kwargs": {"required_keys": ["MODAL_WORKER_DATABASE_URL"]}, + } + ] + # One check at a time; two at once would still restart each run once. + assert options["max_containers"] == 1 + assert options["retries"] == 0 + assert options["timeout"] <= 600 + assert options["image"] is module.sweep_image + + +def test_the_image_bakes_the_modal_environment_and_telemetry_settings_in( + monkeypatch, +) -> None: + monkeypatch.setenv("SIMULATION_V2_MODAL_ENVIRONMENT", "staging") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "https://collector.example") + monkeypatch.setenv("OBSERVABILITY_LOG_NAME", "policyengine-api-v1-modal") + module = _load(monkeypatch) + baked: dict[str, str] = {} + for call in module.sweep_image.calls: + if call[0] == "env": + baked.update(call[1]) + + assert baked["SIMULATION_V2_MODAL_ENVIRONMENT"] == "staging" + assert baked["OTEL_EXPORTER_OTLP_ENDPOINT"] == "https://collector.example" + assert baked["OBSERVABILITY_LOG_NAME"] == "policyengine-api-v1-modal" + assert baked["OTEL_METRICS_EXPORTER"] == "otlp" + + +class RecordingRuntime: + """Stands in for the observability runtime; records its shutdown.""" + + def __init__(self) -> None: + self.shut_down = False + + def shutdown(self) -> None: + self.shut_down = True + + +def test_the_scheduled_restart_runs_the_check_on_the_worker_database( + monkeypatch, +) -> None: + module = _load(monkeypatch) + monkeypatch.setenv("MODAL_WORKER_DATABASE_URL", "postgresql://worker@db/postgres") + monkeypatch.setenv("SIMULATION_V2_MODAL_ENVIRONMENT", "main") + runtime = RecordingRuntime() + built = [] + checked = [] + + def recording_runtime(**identity): + built.append(identity) + return runtime + + def recording_check(**arguments): + checked.append(arguments) + return {"listed": 2, "restarted": 2} + + monkeypatch.setattr(observability, "init_process_observability", recording_runtime) + monkeypatch.setattr(run_restart_check, "run_restart_check", recording_check) + + assert module.restart_unstarted_runs() == {"listed": 2, "restarted": 2} + assert checked == [ + { + "database_url": "postgresql://worker@db/postgres", + "modal_environment": "main", + "runtime": runtime, + } + ] + assert built == [ + { + "service_name": "policyengine-simulation-run-restart", + "service_role": "v2_run_restart_check", + "platform": "modal", + "environment": "main", + } + ] + assert runtime.shut_down From ace6c6427914805ff3d71ace0776713fd809d616 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:40:30 +0400 Subject: [PATCH 07/29] Run the restart check every two minutes beside the stuck-run sweep The sweep's Modal app gains a scheduled function that runs the simulation service's restart check as the Modal worker role, with the Modal environment and the v2 workers' telemetry settings baked into its image at deployment. Each check and each restarted run is an operation named by a registered stage, and each restart's outcome is logged as an event. The sweep now fails a pending run with no call at 15 minutes, after the restart window; the entry package loads its web app lazily so the restart code imports in the sweep's image. Co-Authored-By: Claude Opus 5.5 --- .../workflows/simulation-deploy.reusable.yml | 28 +- .../stages.py | 10 + .../policyengine_simulation_entry/__init__.py | 23 +- .../run_restart_check.py | 242 ++++++++++++++++++ .../pyproject.toml | 6 + .../src/modal/run_sweep_app.py | 92 ++++++- .../stage12_runtime/run_sweep.py | 24 +- .../policyengine-simulation-executor/uv.lock | 67 +++++ 8 files changed, 458 insertions(+), 34 deletions(-) create mode 100644 projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py diff --git a/.github/workflows/simulation-deploy.reusable.yml b/.github/workflows/simulation-deploy.reusable.yml index b89de5283..5df51c15c 100644 --- a/.github/workflows/simulation-deploy.reusable.yml +++ b/.github/workflows/simulation-deploy.reusable.yml @@ -1189,22 +1189,34 @@ jobs: run: ../../.github/scripts/stage12-v2-default-verify.sh # The scheduled sweep that fails report runs, simulation runs and - # household jobs nothing will finish. Its app is unversioned and its - # schedule starts when it deploys, so it deploys here, after every - # check, with the release's Stage 12 default; if it fails, the next - # step puts the default back. + # household jobs nothing will finish, and the restart check that starts + # again those whose Modal call never started. Their app is unversioned + # and its schedules start when it deploys, so it deploys here, after + # every check, with the release's Stage 12 default; if it fails, the + # next step puts the default back. # Only from main: a dispatch from another branch, which may be behind - # main, leaves the environment's sweep as it is. With no status + # main, leaves the environment's app as it is. With no status # function, the condition also requires every earlier step to pass. - # Its database URL comes from the Modal secret + # Their database URL comes from the Modal secret # policyengine-modal-worker-database, which the v2 workers' deploy in - # deploy_stage12_v2 already requires. - - name: Deploy the stuck-run sweep + # deploy_stage12_v2 already requires. The image bakes in the Modal + # environment the restart check starts calls in, and the v2 workers' + # telemetry settings, from the repository variables they use. + - name: Deploy the stuck-run sweep and the restart check if: ${{ github.ref == 'refs/heads/main' }} working-directory: projects/policyengine-simulation-executor env: MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} + SIMULATION_V2_MODAL_ENVIRONMENT: ${{ inputs.modal_environment }} + OBSERVABILITY_SERVICE_NAMESPACE: ${{ vars.OBSERVABILITY_SERVICE_NAMESPACE }} + OBSERVABILITY_TRACE_PROJECT_ID: ${{ vars.OBSERVABILITY_TRACE_PROJECT_ID }} + OBSERVABILITY_LOGGING_PROJECT_ID: ${{ vars.OBSERVABILITY_LOGGING_PROJECT_ID }} + OBSERVABILITY_LOG_NAME: ${{ vars.OBSERVABILITY_LOG_NAME }} + OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} + OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL: ${{ vars.OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL }} + OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OBSERVABILITY_OTLP_ENDPOINT }} + POLICYENGINE_OTEL_GOOGLE_AUDIENCE: ${{ vars.OBSERVABILITY_OTLP_GOOGLE_AUDIENCE }} run: uv run --frozen modal deploy --env="${{ inputs.modal_environment }}" src/modal/run_sweep_app.py # Writes back the manifest the move replaced, only while the stored diff --git a/libs/policyengine-simulation-observability/src/policyengine_simulation_observability/stages.py b/libs/policyengine-simulation-observability/src/policyengine_simulation_observability/stages.py index d2c72e5f5..5f72afe1a 100644 --- a/libs/policyengine-simulation-observability/src/policyengine_simulation_observability/stages.py +++ b/libs/policyengine-simulation-observability/src/policyengine_simulation_observability/stages.py @@ -27,6 +27,7 @@ class RunConfiguration(StrEnum): STAGE12_SIMULATION = "stage12_simulation" STAGE12_SEGMENT = "stage12_segment" V2_HOUSEHOLD = "v2_household" + V2_RUN_RESTART = "v2_run_restart" class Stage(StrEnum): @@ -116,6 +117,8 @@ class Stage(StrEnum): V2_HOUSEHOLD_COUNTRY_MODEL_LOAD = "v2_household_country_model_load" V2_HOUSEHOLD_INPUT_NORMALIZATION = "v2_household_input_normalization" V2_HOUSEHOLD_CALCULATION = "v2_household_calculation" + V2_RUN_RESTART_CHECK = "v2_run_restart_check" + V2_RUN_RESTART = "v2_run_restart" @dataclass(frozen=True) @@ -287,6 +290,12 @@ def name(self, stage: Stage) -> str: Stage.V2_HOUSEHOLD_CALCULATION, ), ), + # One scheduled restart check, then each run or household job it + # lists because its Modal call never started. + RunConfiguration.V2_RUN_RESTART: StagePlan( + RunConfiguration.V2_RUN_RESTART, + (Stage.V2_RUN_RESTART_CHECK, Stage.V2_RUN_RESTART), + ), } ) @@ -302,3 +311,4 @@ def name(self, stage: Stage) -> str: STAGE12_SIMULATION_STAGES = RUN_STAGE_REGISTRY[RunConfiguration.STAGE12_SIMULATION] STAGE12_SEGMENT_STAGES = RUN_STAGE_REGISTRY[RunConfiguration.STAGE12_SEGMENT] V2_HOUSEHOLD_STAGES = RUN_STAGE_REGISTRY[RunConfiguration.V2_HOUSEHOLD] +V2_RUN_RESTART_STAGES = RUN_STAGE_REGISTRY[RunConfiguration.V2_RUN_RESTART] diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/__init__.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/__init__.py index 722f82b54..b34ddc052 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/__init__.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/__init__.py @@ -1,5 +1,24 @@ -"""PolicyEngine Cloud Run Simulation Entrypoint.""" +"""PolicyEngine Cloud Run Simulation Entrypoint. -from policyengine_simulation_entry.app import create_app +``create_app`` is loaded on first use, so importing one of the package's +modules, such as the restart check that runs in the stuck-run sweep's Modal +image, never imports the web service and the packages only it needs. +""" + +from __future__ import annotations + +from importlib import import_module +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from policyengine_simulation_entry.app import create_app __all__ = ["create_app"] + + +def __getattr__(name: str) -> Any: # noqa: ANN401 # the attribute the app module defines + """Load ``create_app`` from the app module when it is first asked for.""" + if name == "create_app": + return import_module("policyengine_simulation_entry.app").create_app + msg = f"module {__name__!r} has no attribute {name!r}" + raise AttributeError(msg) diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py new file mode 100644 index 000000000..0030b49ac --- /dev/null +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py @@ -0,0 +1,242 @@ +"""The scheduled restart check: its Modal wiring and its telemetry. + +The stuck-run sweep's Modal application runs ``run_restart_check`` every two +minutes, as the Modal worker role, from the sweep's Modal secret. It builds +the restart service on the run tables (``scheduled_restart_service``) and +runs one check (``run_scheduled_check``) in its own telemetry operation. + +How often runs need a restart is the check's telemetry: + +- each check is one ``v2_run_restart_check`` operation, with the number of + rows it listed and how each ended (``restart_listed``, ``restart_failed`` + and the rest) set on it; +- each row it lists is one ``v2_run_restart`` operation, with the row's kind, + ID and age, so the operation count metric counts the runs whose call never + started; +- each row's outcome is set on its operation (``restart_outcome``, with + ``restart_error_type`` when an error ended it) and logged as a + ``v2_run_restart_outcome`` event, at WARNING when the row could not be + started again or its rows could not be read. + +This module imports none of the web service, so the sweep's image, which +has none of its packages, can run it. +""" + +from __future__ import annotations + +import asyncio +from dataclasses import asdict, dataclass +from typing import TYPE_CHECKING, Any, Final, Protocol + +from policyengine_simulation_contract.stage12_manifest import V2_VERSION_MANIFEST_NAME +from policyengine_simulation_observability.stages import ( + V2_RUN_RESTART_STAGES, + Stage, +) +from policyengine_simulation_persistence import ( + EconomyRunStore, + HouseholdJobStore, + RunRestartStore, + create_simulation_engine, +) + +from policyengine_simulation_entry.economy_workers import ( + ManifestEconomyWorkers, + ModalEconomyRunCaller, +) +from policyengine_simulation_entry.household_workers import ( + ManifestHouseholdWorkers, + ModalHouseholdFunctionCaller, +) +from policyengine_simulation_entry.run_restarts import ( + RestartCallers, + RestartOutcome, + RestartSettings, + RestartStores, + RestartWorkers, + RunRestartService, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + from contextlib import AbstractContextManager + + from policyengine_simulation_persistence import UnstartedRun + from sqlalchemy import Engine + + from policyengine_simulation_entry.run_restarts import RestartSummary + +# The longest any one statement of a check may take. +RESTART_STATEMENT_TIMEOUT_MILLISECONDS: Final = 5_000 +OUTCOME_EVENT: Final = "v2_run_restart_outcome" +# The outcomes worth a warning: the row could not be started again, or the +# check could not tell, so a person may need to look. +WARNING_OUTCOMES: Final = frozenset( + { + RestartOutcome.FAILED, + RestartOutcome.DEFERRED, + RestartOutcome.CALL_NOT_RECORDED, + } +) + + +class RestartRuntime(Protocol): + """What the check's telemetry uses of ``ObservabilityRuntime``.""" + + def operation( + self, name: str, *, attributes: Mapping[str, Any] | None = None + ) -> AbstractContextManager[Any]: + """Return the scope of one operation.""" + ... + + def set_context(self, **attributes: Any) -> None: # noqa: ANN401 # the runtime's own signature + """Set attributes on the innermost open operation.""" + ... + + def event( + self, + name: str, + *, + severity: str = "INFO", + attributes: Mapping[str, Any] | None = None, + ) -> None: + """Log one structured event.""" + ... + + def capture_context(self) -> dict[str, str]: + """Return the context a call started now continues from.""" + ... + + +class ObservabilityRestartTelemetry: + """Report each listed row as an operation, and its outcome as an event.""" + + def __init__(self, runtime: RestartRuntime) -> None: + """Report through ``runtime``, which never raises.""" + self._runtime = runtime + + def restart_scope( + self, run: UnstartedRun, *, age_seconds: float + ) -> AbstractContextManager[Any]: + """Return the operation one row's restart runs in.""" + return self._runtime.operation( + V2_RUN_RESTART_STAGES.name(Stage.V2_RUN_RESTART), + attributes={ + "run_kind": run.kind.value, + "run_id": str(run.run_id), + "run_age_seconds": age_seconds, + }, + ) + + def record_outcome( + self, + run: UnstartedRun, + outcome: RestartOutcome, + *, + error_type: str | None = None, + ) -> None: + """Set the outcome on the row's operation and log it as an event.""" + outcome_attributes: dict[str, str] = {"restart_outcome": outcome.value} + if error_type is not None: + outcome_attributes["restart_error_type"] = error_type + self._runtime.set_context(**outcome_attributes) + self._runtime.event( + OUTCOME_EVENT, + severity="WARNING" if outcome in WARNING_OUTCOMES else "INFO", + attributes={ + "run_kind": run.kind.value, + "run_id": str(run.run_id), + **outcome_attributes, + }, + ) + + def capture_context(self) -> dict[str, str] | None: + """Return the context a restarted call continues the check's trace from.""" + return self._runtime.capture_context() + + +class RestartCheck(Protocol): + """One restart check (``RunRestartService``).""" + + async def restart_unstarted_runs(self) -> RestartSummary: + """Restart the rows whose call never started; count how each ended.""" + ... + + +def run_scheduled_check( + service: RestartCheck, runtime: RestartRuntime +) -> dict[str, int]: + """Run one check in its own operation; return how many rows ended each way.""" + with runtime.operation(V2_RUN_RESTART_STAGES.name(Stage.V2_RUN_RESTART_CHECK)): + summary = asyncio.run(service.restart_unstarted_runs()) + counts = asdict(summary) + runtime.set_context( + **{f"restart_{name}": count for name, count in counts.items()} + ) + return counts + + +@dataclass(frozen=True) +class ScheduledRestart: + """The restart service on the run tables, and the engine it connects through.""" + + service: RunRestartService + settings: RestartSettings + engine: Engine + + +def scheduled_restart_service( + *, database_url: str, modal_environment: str, runtime: RestartRuntime +) -> ScheduledRestart: + """Build the restart service for the Modal environment's deployment. + + ``database_url`` is the Modal worker role's ``postgresql://`` URL. Nothing + connects to the database or reads Modal until the check runs. Raise + ValueError for a Modal environment no deployment uses, or a malformed URL. + """ + settings = RestartSettings.for_modal_environment(modal_environment) + engine = create_simulation_engine(database_url) + timeout = RESTART_STATEMENT_TIMEOUT_MILLISECONDS + service = RunRestartService( + stores=RestartStores( + restarts=RunRestartStore.from_engine( + engine, statement_timeout_milliseconds=timeout + ), + economy=EconomyRunStore.from_engine( + engine, statement_timeout_milliseconds=timeout + ), + households=HouseholdJobStore.from_engine( + engine, statement_timeout_milliseconds=timeout + ), + ), + workers=RestartWorkers( + economy=ManifestEconomyWorkers.from_modal( + manifest_name=V2_VERSION_MANIFEST_NAME, environment=modal_environment + ), + households=ManifestHouseholdWorkers.from_modal( + manifest_name=V2_VERSION_MANIFEST_NAME, environment=modal_environment + ), + ), + callers=RestartCallers( + economy=ModalEconomyRunCaller(modal_environment), + households=ModalHouseholdFunctionCaller(modal_environment), + ), + settings=settings, + telemetry=ObservabilityRestartTelemetry(runtime), + ) + return ScheduledRestart(service=service, settings=settings, engine=engine) + + +def run_restart_check( + *, database_url: str, modal_environment: str, runtime: RestartRuntime +) -> dict[str, int]: + """Run one restart check on the run tables; return how each row ended.""" + scheduled = scheduled_restart_service( + database_url=database_url, + modal_environment=modal_environment, + runtime=runtime, + ) + try: + return run_scheduled_check(scheduled.service, runtime) + finally: + scheduled.engine.dispose() diff --git a/projects/policyengine-simulation-executor/pyproject.toml b/projects/policyengine-simulation-executor/pyproject.toml index 2b4f42ac6..f566c5f93 100644 --- a/projects/policyengine-simulation-executor/pyproject.toml +++ b/projects/policyengine-simulation-executor/pyproject.toml @@ -19,6 +19,10 @@ dependencies = [ "policyengine-simulation-observability", "policyengine-simulation-contract[modal]", "policyengine-stage12-persistence", + # The restart check in the stuck-run sweep's Modal app runs the + # simulation service's own restart code on its persistence library. + "policyengine-simulation-entry", + "policyengine-simulation-persistence", # PolicyEngine.py's models extra is the only selector for core, country # models, and the SPM calculator. Do not duplicate those versions here. "policyengine[models]==6.17.1", @@ -80,6 +84,8 @@ policyengine-fastapi = { path = "../../libs/policyengine-fastapi", editable = tr policyengine-simulation-observability = { path = "../../libs/policyengine-simulation-observability", editable = true } policyengine-simulation-contract = { path = "../../libs/policyengine-simulation-contract", editable = true } policyengine-stage12-persistence = { path = "../../libs/policyengine-stage12-persistence", editable = true } +policyengine-simulation-entry = { path = "../policyengine-simulation-entry", editable = true } +policyengine-simulation-persistence = { path = "../../libs/policyengine-simulation-persistence", editable = true } policyengine-simulation-gateway = { path = "../policyengine-simulation-gateway", editable = true } [project.optional-dependencies] diff --git a/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py b/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py index 722c6d8b1..1054a9d12 100644 --- a/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py +++ b/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py @@ -1,17 +1,29 @@ -"""A scheduled Modal function that fails runs and household jobs nothing will finish. - -Every ten minutes it lists the Public API's report runs, simulation runs and -household jobs that have been pending or running for at least ten minutes, -counted from a run's first claim or, while it is pending, from its creation, -and from a household job's creation. It asks Modal for the status of each -recorded call and marks a run or job failed once its call has ended, and -fails a pending one that has no call recorded, because nothing will run it. -It also fails any still unfinished 24 hours after it was created or started, -longer than any Modal call of it can run, whatever Modal reports. Its result -counts runs and household jobs together. It connects as the Modal worker -role, from the same Modal secret the v2 coordinator, workers and household -functions use. This application is not versioned with the .py bundle: one -copy runs per Modal environment. +"""Scheduled Modal functions that fail or restart runs and household jobs. + +The stuck-run sweep runs every ten minutes. It lists the Public API's report +runs, simulation runs and household jobs that have been pending or running +for at least 15 minutes, counted from a run's first claim or, while it is +pending, from its creation, and from a household job's creation. It asks +Modal for the status of each recorded call and marks a run or job failed +once its call has ended, and fails a pending one that has no call recorded, +because nothing will run it. It also fails any still unfinished 24 hours +after it was created or started, longer than any Modal call of it can run, +whatever Modal reports. Its result counts runs and household jobs together. + +The restart check runs every two minutes. It starts once more the call of +each run or household job created 3 to 12 minutes ago that is still pending +with no call recorded, as when the simulation service stopped between +creating it and starting its call; the simulation service's own restart code +(``policyengine_simulation_entry.run_restart_check``) rebuilds the call from +the stored rows and records it. A run it can't start again is failed with +``run_not_started``, as the sweep would fail it at 15 minutes. Its telemetry +counts each restart and how it ended. + +Both connect as the Modal worker role, from the same Modal secret the v2 +coordinator, workers and household functions use. This application is not +versioned with the .py bundle: one copy runs per Modal environment. Its +image bakes in, at deployment, the Modal environment the restart check starts +calls in and the v2 workers' telemetry settings. Each deployment from ``main`` deploys it to its Modal environment after every check, in ``publish_stage12_v2_default``, once that job has moved the @@ -23,6 +35,7 @@ from __future__ import annotations +import os from dataclasses import asdict from pathlib import Path @@ -30,11 +43,17 @@ MODAL_WORKER_DATABASE_SECRET_NAME, MODAL_WORKER_DATABASE_URL_ENV, ) +from policyengine_simulation_observability.observability import ( + modal_image_environment, +) from src.modal.ca_certificates import GRPC_CA_BUNDLE_ENV, OPERATING_SYSTEM_CA_PACKAGE import modal APP_NAME = "policyengine-simulation-run-sweep" +# The Modal environment the restart check reads the v2 manifest from and +# starts calls in, baked into the image from the deploy step's environment. +V2_MODAL_ENVIRONMENT_ENV = "SIMULATION_V2_MODAL_ENVIRONMENT" _UV_PROJECT_DIR = str(Path(__file__).resolve().parents[2]) if modal.is_local() else "." app = modal.App(APP_NAME) @@ -48,6 +67,12 @@ # bundle's data, which the sweep never reads. .apt_install(OPERATING_SYSTEM_CA_PACKAGE) .env(GRPC_CA_BUNDLE_ENV) + .env( + { + **modal_image_environment(), + V2_MODAL_ENVIRONMENT_ENV: os.getenv(V2_MODAL_ENVIRONMENT_ENV, ""), + } + ) .uv_sync( uv_project_dir=_UV_PROJECT_DIR, frozen=True, @@ -62,6 +87,9 @@ "policyengine_simulation_observability", "policyengine_simulation_contract", "policyengine_stage12_persistence", + # The restart check runs the simulation service's restart code. + "policyengine_simulation_persistence", + "policyengine_simulation_entry", copy=True, ) ) @@ -86,3 +114,39 @@ def sweep_stuck_runs() -> dict: summary = run_sweep.sweep_stuck_runs(report_run_access.report_run_store()) return asdict(summary) + + +@app.function( + image=sweep_image, + schedule=modal.Cron("*/2 * * * *"), + cpu=0.25, + memory=512, + timeout=300, + retries=0, + max_containers=1, + secrets=[modal_worker_database_secret], +) +def restart_unstarted_runs() -> dict: + """Start again the runs and household jobs whose call never started; count them.""" + from policyengine_simulation_entry import ( # noqa: PLC0415 # Modal function body: loaded in the container, not at deploy time + run_restart_check, + ) + from policyengine_simulation_observability import ( # noqa: PLC0415 # Modal function body: loaded in the container, not at deploy time + observability, + ) + + modal_environment = os.environ[V2_MODAL_ENVIRONMENT_ENV] + runtime = observability.init_process_observability( + service_name="policyengine-simulation-run-restart", + service_role="v2_run_restart_check", + platform="modal", + environment=modal_environment, + ) + try: + return run_restart_check.run_restart_check( + database_url=os.environ[MODAL_WORKER_DATABASE_URL_ENV], + modal_environment=modal_environment, + runtime=runtime, + ) + finally: + runtime.shutdown() diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/run_sweep.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/run_sweep.py index 14841c4bd..708118ece 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/run_sweep.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/run_sweep.py @@ -6,14 +6,16 @@ call ends without recording the run's outcome, for example because its container ran out of memory, the run stays pending or running. The sweep asks Modal for the status of each recorded call and marks the run failed -once the call has ended. A pending run with no call recorded soon after its -creation is failed without asking Modal: usually the simulation service -stopped before starting its call. If it started the call but failed to -record it, that call's claim is refused once the run has failed, so it -finishes nothing. The sweep also fails a run still unfinished long after it -was created or started, longer than any Modal call of it can run, whatever -Modal reports and with or without a recorded call, so a run Modal can't -decide doesn't stay unfinished for good. +once the call has ended. A pending run with no call recorded 15 minutes +after its creation is failed without asking Modal: usually the simulation +service stopped before starting its call, and the restart check, which +starts such a run's call once more 3 to 12 minutes after its creation, could +not start it either. If a call was started but not recorded, its claim is +refused once the run has failed, so it finishes nothing. The sweep also +fails a run still unfinished long after it was created or started, longer +than any Modal call of it can run, whatever Modal reports and with or +without a recorded call, so a run Modal can't decide doesn't stay +unfinished for good. Modal restarts a pre-empted call under the same call ID and reports it pending meanwhile, so a pre-empted coordinator's or worker's run is never @@ -49,8 +51,10 @@ # A run must have been unfinished at least this long, counted from its first # claim or, while it is pending, from its creation (a household job's from its # creation), before the sweep judges it; a pending run with no call recorded -# by then is failed. -DEFAULT_MINIMUM_AGE = timedelta(minutes=10) +# by then is failed. It is longer than the restart check's window (3 to 12 +# minutes after creation), so a run that check started again is judged by +# its call. +DEFAULT_MINIMUM_AGE = timedelta(minutes=15) ENDED_CALL_MESSAGE = ( "modal_call_ended: The Modal call recorded for this run ended without " "recording the run's outcome." diff --git a/projects/policyengine-simulation-executor/uv.lock b/projects/policyengine-simulation-executor/uv.lock index b46aa9520..8ddd1c0d5 100644 --- a/projects/policyengine-simulation-executor/uv.lock +++ b/projects/policyengine-simulation-executor/uv.lock @@ -1855,6 +1855,9 @@ google = [ { name = "google-auth" }, { name = "google-cloud-logging" }, ] +httpx = [ + { name = "httpx" }, +] otlp-grpc = [ { name = "opentelemetry-api" }, { name = "opentelemetry-exporter-otlp-proto-grpc" }, @@ -1891,6 +1894,46 @@ requires-dist = [ ] provides-extras = ["modal", "test", "build"] +[[package]] +name = "policyengine-simulation-entry" +version = "0.1.0" +source = { editable = "../policyengine-simulation-entry" } +dependencies = [ + { name = "cryptography" }, + { name = "fastapi" }, + { name = "httpx" }, + { name = "modal" }, + { name = "policyengine-observability", extra = ["fastapi", "google", "httpx", "otlp-grpc"] }, + { name = "policyengine-simulation-contract" }, + { name = "policyengine-simulation-observability" }, + { name = "policyengine-simulation-persistence" }, + { name = "policyengine-stage12-persistence" }, + { name = "pyjwt" }, + { name = "uvicorn" }, +] + +[package.metadata] +requires-dist = [ + { name = "cryptography", specifier = ">=41.0.0" }, + { name = "fastapi", specifier = ">=0.115.0,<1" }, + { name = "httpx", specifier = ">=0.28.0,<1" }, + { name = "modal", specifier = ">=1.4,<2" }, + { name = "openapi-python-client", marker = "extra == 'build'", specifier = ">=0.21.6" }, + { name = "policyengine-observability", extras = ["fastapi", "google", "httpx", "otlp-grpc"], specifier = ">=3.0.7,<4" }, + { name = "policyengine-simulation-contract", editable = "../../libs/policyengine-simulation-contract" }, + { name = "policyengine-simulation-observability", editable = "../../libs/policyengine-simulation-observability" }, + { name = "policyengine-simulation-persistence", editable = "../../libs/policyengine-simulation-persistence" }, + { name = "policyengine-stage12-persistence", editable = "../../libs/policyengine-stage12-persistence" }, + { name = "pyjwt", specifier = ">=2.10.1,<3" }, + { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, + { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, + { name = "pytest-asyncio", marker = "extra == 'test'", specifier = ">=0.25.3" }, + { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=6.1.1" }, + { name = "pyyaml", marker = "extra == 'test'", specifier = ">=6.0.2" }, + { name = "uvicorn", specifier = ">=0.35.0,<1" }, +] +provides-extras = ["test", "build"] + [[package]] name = "policyengine-simulation-executor" version = "0.1.0" @@ -1905,7 +1948,9 @@ dependencies = [ { name = "policyengine-fastapi" }, { name = "policyengine-observability", extra = ["fastapi", "google", "otlp-grpc"] }, { name = "policyengine-simulation-contract", extra = ["modal"] }, + { name = "policyengine-simulation-entry" }, { name = "policyengine-simulation-observability" }, + { name = "policyengine-simulation-persistence" }, { name = "policyengine-stage12-persistence" }, { name = "pyarrow" }, { name = "pydantic-settings" }, @@ -1957,7 +2002,9 @@ requires-dist = [ { name = "policyengine-fastapi", editable = "../../libs/policyengine-fastapi" }, { name = "policyengine-observability", extras = ["fastapi", "google", "otlp-grpc"], specifier = ">=3.0.7,<4" }, { name = "policyengine-simulation-contract", extras = ["modal"], editable = "../../libs/policyengine-simulation-contract" }, + { name = "policyengine-simulation-entry", editable = "../policyengine-simulation-entry" }, { name = "policyengine-simulation-observability", editable = "../../libs/policyengine-simulation-observability" }, + { name = "policyengine-simulation-persistence", editable = "../../libs/policyengine-simulation-persistence" }, { name = "policyengine-stage12-persistence", editable = "../../libs/policyengine-stage12-persistence" }, { name = "pyarrow", specifier = ">=20,<24" }, { name = "pydantic-settings", specifier = ">=2.7.1,<3.0.0" }, @@ -2049,6 +2096,26 @@ requires-dist = [ ] provides-extras = ["test", "build"] +[[package]] +name = "policyengine-simulation-persistence" +version = "0.1.0" +source = { editable = "../../libs/policyengine-simulation-persistence" } +dependencies = [ + { name = "psycopg", extra = ["binary"] }, + { name = "sqlalchemy" }, +] + +[package.metadata] +requires-dist = [ + { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" }, + { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, + { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, + { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=6.1.1" }, + { name = "pyyaml", marker = "extra == 'test'", specifier = ">=6.0.2" }, + { name = "sqlalchemy", specifier = ">=2,<2.1" }, +] +provides-extras = ["test", "build"] + [[package]] name = "policyengine-stage12-persistence" version = "0.1.0" From 83d0b80989ba2bf30efc8f3f34e514a9f224aa1d Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:41:46 +0400 Subject: [PATCH 08/29] Describe the restart check and the sweep's 15 minutes Co-Authored-By: Claude Opus 5.5 --- .../c1-19-restart-unstarted-runs.added.md | 1 + .../stage-12-modal-worker-foundation.md | 30 ++++++++++++++-- .../README.md | 26 ++++++++++++++ .../policyengine-simulation-entry/README.md | 36 +++++++++++++++++-- .../README.md | 5 +-- 5 files changed, 92 insertions(+), 6 deletions(-) create mode 100644 changelog.d/c1-19-restart-unstarted-runs.added.md diff --git a/changelog.d/c1-19-restart-unstarted-runs.added.md b/changelog.d/c1-19-restart-unstarted-runs.added.md new file mode 100644 index 000000000..f81d27024 --- /dev/null +++ b/changelog.d/c1-19-restart-unstarted-runs.added.md @@ -0,0 +1 @@ +A restart check in the stuck-run sweep's Modal application starts the Modal call of a household job, economy report run or standalone economy simulation run again when the run stayed pending with no call recorded 3 to 12 minutes after its creation, as when the Simulation Entrypoint stopped before starting its call; its telemetry counts every restart and how it ended. The sweep now fails a run still pending with no call at 15 minutes instead of 10. diff --git a/docs/migration/stage-12-modal-worker-foundation.md b/docs/migration/stage-12-modal-worker-foundation.md index ccb02b6f8..2865fb8b5 100644 --- a/docs/migration/stage-12-modal-worker-foundation.md +++ b/docs/migration/stage-12-modal-worker-foundation.md @@ -356,7 +356,7 @@ run's maximum age of 24 hours. anything: their `coordinate_report` requires an evaluation record, and their context model refuses `simulation_run_id`. Modal accepts the call, which fails at once, and the run stays pending with its call recorded until the sweep fails -it (`modal_call_ended`), at least ten minutes after its creation. A call +it (`modal_call_ended`), at least 15 minutes after its creation. A call reaches such an application only while the manifest's default, or `STAGE12_V2_WORKER_VERSION`, names one: during the deployment of this change, if it adds a newer .py version, until it moves the default to it, or while a @@ -709,10 +709,36 @@ call that names no job, as deployment validation makes, touches no database. The stuck-run sweep (`src/modal/run_sweep_app.py`) fails household jobs nothing will finish by the rule it applies to report runs and simulation runs, counting a job's age from its creation: a pending job with no call -recorded ten minutes after its creation, a pending or running job once +recorded 15 minutes after its creation, a pending or running job once Modal reports its recorded call ended, and any job still unfinished 24 hours after its creation, whatever Modal reports. +**Restarting a run whose call never started.** The same Modal application +runs a restart check every two minutes, as the Modal worker role. It lists +the household jobs, report runs and standalone simulation runs created 3 to +12 minutes ago that are still pending with no call recorded and that +nothing has changed since their insert, as when the Simulation Entrypoint +stopped between creating a run and starting its call. For each, it reads +back what the entrypoint built the call from, builds the same call with the +entrypoint's own builders on the worker of the .py version the row records, +sets the row's `updated_at` so it is never restarted twice, starts the call +and records its ID while the row is still pending with none. A row it can't +start again (its worker left the manifest, its stored rows changed, or Modal +refuses the call) is failed with `run_not_started`; one whose rows or +manifest it can't read now is left for the next check. The sweep's 15 +minutes come after that window, so a second miss is still failed with +`run_not_started`. The restart code is the simulation service's +(`policyengine_simulation_entry.run_restarts`, `run_restart_calls` and +`run_restart_check`, on `policyengine_simulation_persistence`), which the +image adds as source. + +It leaves alone the report runs of a budget window and the runs of +household simulations, whose own calls start them. Each check is a +`v2_run_restart_check` operation and each listed row a `v2_run_restart` +operation, with the row's kind, ID, age and outcome, and each outcome is +also logged as a `v2_run_restart_outcome` event, so the operation count +metric and the events say how often runs need a restart and how each ended. + - Image: the CA store, the frozen `modal-simulation-image` dependency group and the source packages, without the bundle's datasets or the Stage 12 cache. Building it needs no secret. diff --git a/libs/policyengine-simulation-persistence/README.md b/libs/policyengine-simulation-persistence/README.md index 4fea4fd37..183845032 100644 --- a/libs/policyengine-simulation-persistence/README.md +++ b/libs/policyengine-simulation-persistence/README.md @@ -125,6 +125,32 @@ commit fails after its rows were inserted raises `EconomyRecordOutcomeUnknownError`: PostgreSQL may have applied the commit, so the rows may exist. +## Restarting runs whose call never started + +`run_restart_records.py` holds the restart check's reads and writes. The +check runs as the Modal worker role, whose grants cover them: `SELECT` on +the run tables and their inputs, and `UPDATE` of `updated_at`, +`modal_call_id`, `status`, `error_message` and `completed_at`. + +- `list_unstarted_runs` returns the oldest rows, up to a limit, created in a + window that are still `pending` with no Modal call and whose `updated_at` + still equals `created_at`: household jobs, report runs outside a budget + window whose baseline is an economy simulation, and runs of economy + simulations that no report run names (standalone runs). +- `mark_restart` sets a listed row's `updated_at` to the statement's clock + while it is still unstarted, so it is listed and marked only once. +- `read_unstarted_report_run` and `read_unstarted_simulation_run` read back + what the Simulation Entrypoint built a call from: each side's simulation + run, policy and dynamic, with the values read against the run's own + catalog version, the region, dataset and year the run records, the + report's options and the run's versions. `StoredRunUnreadableError` says + the rows no longer describe one calculation: a side's region or dataset + is gone, or the two sides differ. + +`RunRestartStore` runs each in a transaction of its own. The check records +a call, or fails a row, with `EconomyRunStore`'s and `HouseholdJobStore`'s +own conditional writes. + ## Reading a policy `read_policy_reform` (`policy_reform.py`) reads a stored policy as the reform diff --git a/projects/policyengine-simulation-entry/README.md b/projects/policyengine-simulation-entry/README.md index bc32cc1f6..9e00f34be 100644 --- a/projects/policyengine-simulation-entry/README.md +++ b/projects/policyengine-simulation-entry/README.md @@ -100,12 +100,44 @@ can't be read. When Modal refuses the call, or doesn't confirm it within ten seconds, the job is failed at once and the 503 (`modal_call_refused`) names it; a job the household function claimed meanwhile is answered with 202 as it stands, and a job whose state couldn't then be recorded or read with 202 -`pending`, to be read rather than submitted again (the stuck-run sweep fails -it if no call ever starts). Every 500 of either route, from an unexpected +`pending`, to be read rather than submitted again (the restart check starts +its call once more 3 to 12 minutes after its creation, and the stuck-run +sweep fails it if no call ever starts). Every 500 of either route, from an unexpected failure or a request the service built and the household contract refuses, is the JSON error with code `internal_error`. `DEPLOYMENT.md` describes the deployment settings and the staging check. +## Restarting a run whose call never started + +When the service stops between creating a household job, a report run or a +standalone simulation run and starting its Modal call, nothing records a +call for it. The restart check (`run_restarts`, with `run_restart_calls` and +`run_restart_check`) starts such a row's call once more. It doesn't run in +this service: the stuck-run sweep's Modal application +(`projects/policyengine-simulation-executor/src/modal/run_sweep_app.py`) +runs it every two minutes as the Modal worker role. + +- It takes the rows created 3 to 12 minutes ago that are still `pending` + with no call and that nothing has changed since their insert, at most 50 + a check, oldest first. A budget window's report runs and household + simulations' runs are left out. +- It rebuilds each call with this service's own builders + (`report_run_calculation`, `simulation_run_calculation`, + `prepare_household_job`) from the stored rows, on the manifest's worker + for the .py version the row records, even when that worker is no longer + the default. The context's request ID is the check's own + (`run-restart-`), and its creation time is the row's. +- It sets the row's `updated_at` before starting the call, so a row is + restarted at most once, then records the call's ID with the submission's + own conditional write. +- A row whose call can't be rebuilt, or whose restart Modal refuses or + doesn't confirm within ten seconds, is failed with + `run_not_started: The run stayed pending with no Modal call recorded for + it, and โ€ฆ`. A row whose rows or manifest can't be read now is left + unmarked for the next check. +- Each check, each row and each outcome are telemetry; see + `run_restart_check`'s docstring for the operations and event. + ## Economy reports and simulations `POST /reports` creates an economy report: a baseline and a reform diff --git a/projects/policyengine-simulation-executor/README.md b/projects/policyengine-simulation-executor/README.md index acf16e000..fe41ed136 100644 --- a/projects/policyengine-simulation-executor/README.md +++ b/projects/policyengine-simulation-executor/README.md @@ -154,8 +154,9 @@ with its own mechanism, separate from the v1 routing jobs above: while the stored manifest is still what it wrote (`--restore-plan`). In a deployment from `main`, the same job then deploys the unversioned stuck-run sweep (`src/modal/run_sweep_app.py`), which fails the report - runs, simulation runs and household jobs nothing will finish, and whose - schedule starts as soon as it deploys; a failed sweep deploy, including a failed build of its + runs, simulation runs and household jobs nothing will finish, with the + restart check, which starts again those whose Modal call never started, + and whose schedules start as soon as it deploys; a failed sweep deploy, including a failed build of its image, fails the job and so puts the previous default back. A deployment from another branch leaves the sweep as it is. `deployment_ready` requires this job to succeed whenever Stage 12 was From 6aacd1af79f2e0b9927489c5d50c2ba73a1789e0 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:42:44 +0400 Subject: [PATCH 09/29] Test that a restart check waiting on every call ends within its timeout (failing) Co-Authored-By: Claude Opus 5.5 --- .../tests/test_stage12_run_sweep_app.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep_app.py b/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep_app.py index 3266f683d..553d2d214 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep_app.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep_app.py @@ -9,6 +9,10 @@ import yaml from fixtures.fake_modal import install_fake_modal from policyengine_simulation_entry import run_restart_check +from policyengine_simulation_entry.run_restarts import ( + CALL_START_TIMEOUT_SECONDS, + RESTARTS_PER_CHECK, +) from policyengine_simulation_executor.stage12_runtime import ( report_run_access as report_run_access_module, ) @@ -197,10 +201,21 @@ def test_the_restart_check_runs_every_two_minutes_with_the_worker_database_secre # One check at a time; two at once would still restart each run once. assert options["max_containers"] == 1 assert options["retries"] == 0 - assert options["timeout"] <= 600 assert options["image"] is module.sweep_image +def test_a_check_that_waits_on_every_call_still_ends_within_its_timeout( + monkeypatch, +) -> None: + """A row marked just before the timeout stops the check is never restarted.""" + module = _load(monkeypatch) + options = _functions(module)["restart_unstarted_runs"] + longest_call_waits = RESTARTS_PER_CHECK * CALL_START_TIMEOUT_SECONDS + + # A minute for the database and the manifest. + assert longest_call_waits + 60 <= options["timeout"] + + def test_the_image_bakes_the_modal_environment_and_telemetry_settings_in( monkeypatch, ) -> None: From 5a8f7e9f162493611a7b65268ff906016973ca98 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:43:01 +0400 Subject: [PATCH 10/29] Restart at most 20 runs a check, so a check ends within its timeout Co-Authored-By: Claude Opus 5.5 --- projects/policyengine-simulation-entry/README.md | 2 +- .../src/policyengine_simulation_entry/run_restarts.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/projects/policyengine-simulation-entry/README.md b/projects/policyengine-simulation-entry/README.md index 9e00f34be..240b68e8f 100644 --- a/projects/policyengine-simulation-entry/README.md +++ b/projects/policyengine-simulation-entry/README.md @@ -118,7 +118,7 @@ this service: the stuck-run sweep's Modal application runs it every two minutes as the Modal worker role. - It takes the rows created 3 to 12 minutes ago that are still `pending` - with no call and that nothing has changed since their insert, at most 50 + with no call and that nothing has changed since their insert, at most 20 a check, oldest first. A budget window's report runs and household simulations' runs are left out. - It rebuilds each call with this service's own builders diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py index 0dfe483d7..631de8d18 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py @@ -91,7 +91,10 @@ # minutes, so a restart has time to record its call before the sweep judges it. RESTART_UNTIL: Final = timedelta(minutes=12) # At most this many rows are restarted by one check; the next takes the rest. -RESTARTS_PER_CHECK: Final = 50 +# With every call waited on for its full time, a check still ends within the +# scheduled function's five-minute timeout, so no row it marks is left +# without its call. +RESTARTS_PER_CHECK: Final = 20 # How long the check waits for Modal to accept a restarted call. CALL_START_TIMEOUT_SECONDS: Final = 10.0 _NOT_STARTED: Final = ( From d5ee476e41b93cf292cdf974fe277188658a947c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 02:10:48 +0400 Subject: [PATCH 11/29] Test review round 1's restart findings (failing) The executor's image builds only from the directories its Dockerfile copies; an unreadable report run is failed with its simulation runs; a restart is counted by whose call holds the row afterwards; a check marks no row once its time limit has passed; and a call is rebuilt off the event loop. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_run_restart_records.py | 60 ++++++ .../tests/test_run_restarts.py | 175 +++++++++++++++++- .../tests/test_executor_image_dependencies.py | 51 +++++ .../tests/test_stage12_run_sweep_app.py | 22 ++- 4 files changed, 301 insertions(+), 7 deletions(-) create mode 100644 projects/policyengine-simulation-executor/tests/test_executor_image_dependencies.py diff --git a/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py b/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py index fddb135a8..0ceac82e5 100644 --- a/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py +++ b/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py @@ -31,6 +31,7 @@ ReportRunTrigger, ReportSimulation, RestartKind, + RunCall, RunRestartStore, SimulationChoice, SimulationType, @@ -42,6 +43,8 @@ insert_household_job, list_unstarted_runs, mark_restart, + read_report_run_simulation_run_ids, + read_run_call, read_unstarted_report_run, read_unstarted_simulation_run, record_household_call, @@ -477,6 +480,56 @@ def test_a_report_run_whose_sides_differ_cannot_be_read(connection) -> None: read_unstarted_report_run(connection, report.run_id) +def test_a_report_runs_simulation_runs_are_read_even_when_its_rows_are_not( + connection, +) -> None: + """A report run that can't be rebuilt is failed with its new simulation runs.""" + economy = insert_economy_catalog(connection) + report = _new_report(economy) + insert_economy_report(connection, report) + connection.execute( + update(simulation_runs) + .where(simulation_runs.c.id == report.reform.simulation_run_id) + .values(region_id=None) + ) + act_as_the_modal_worker(connection) + + with pytest.raises(StoredRunUnreadableError): + read_unstarted_report_run(connection, report.run_id) + assert read_report_run_simulation_run_ids(connection, report.run_id) == ( + report.baseline.simulation_run_id, + report.reform.simulation_run_id, + ) + assert read_report_run_simulation_run_ids(connection, uuid4()) == () + + +def test_a_rows_status_and_recorded_call_are_read_back(connection) -> None: + economy = insert_economy_catalog(connection) + report = _new_report(economy) + insert_economy_report(connection, report) + record_report_run_call( + connection, report_run_id=report.run_id, modal_call_id="fc-restarted" + ) + job_id = _insert_household_job(connection, economy) + act_as_the_modal_worker(connection) + + report_run = UnstartedRun( + kind=RestartKind.REPORT_RUN, run_id=report.run_id, created_at=START + ) + job = UnstartedRun(kind=RestartKind.HOUSEHOLD_JOB, run_id=job_id, created_at=START) + missing = UnstartedRun( + kind=RestartKind.SIMULATION_RUN, run_id=uuid4(), created_at=START + ) + + assert read_run_call(connection, report_run) == RunCall( + status="pending", modal_call_id="fc-restarted" + ) + assert read_run_call(connection, job) == RunCall( + status="pending", modal_call_id=None + ) + assert read_run_call(connection, missing) is None + + @pytest.fixture def savepoint_store(connection) -> RunRestartStore: """Return a store whose every transaction is a savepoint of ``connection``.""" @@ -509,8 +562,15 @@ def test_the_store_lists_marks_and_reads_in_transactions_of_its_own( report_run = savepoint_store.read_report_run(report.run_id) simulation_run = savepoint_store.read_simulation_run(simulation.run_id) marked = savepoint_store.mark_restart(by_kind[RestartKind.REPORT_RUN]) + call = savepoint_store.read_call(by_kind[RestartKind.SIMULATION_RUN]) + sides = savepoint_store.read_report_run_simulation_run_ids(report.run_id) assert set(by_kind) == {RestartKind.REPORT_RUN, RestartKind.SIMULATION_RUN} + assert call == RunCall(status="pending", modal_call_id=None) + assert sides == ( + report.baseline.simulation_run_id, + report.reform.simulation_run_id, + ) assert report_run is not None assert report_run.report_run_id == report.run_id assert simulation_run is not None diff --git a/projects/policyengine-simulation-entry/tests/test_run_restarts.py b/projects/policyengine-simulation-entry/tests/test_run_restarts.py index 726194210..42c8cce0c 100644 --- a/projects/policyengine-simulation-entry/tests/test_run_restarts.py +++ b/projects/policyengine-simulation-entry/tests/test_run_restarts.py @@ -43,6 +43,7 @@ ManifestHouseholdWorkers, ) from policyengine_simulation_entry.run_restarts import ( + CHECK_TIME_LIMIT, RESTART_AFTER, RESTART_UNTIL, RESTARTS_PER_CHECK, @@ -60,6 +61,7 @@ HouseholdJobRecord, HouseholdJobStatus, RestartKind, + RunCall, SimulationChoice, StoredEconomySide, StoredHousehold, @@ -71,7 +73,7 @@ from sqlalchemy.exc import OperationalError if TYPE_CHECKING: - from collections.abc import Iterator + from collections.abc import Callable, Iterator from datetime import datetime from policyengine_simulation_entry.household_workers import HouseholdWorker @@ -118,11 +120,28 @@ class FakeRestartStore: unstarted: tuple[UnstartedRun, ...] = () report_runs: dict[UUID, StoredReportRun] = field(default_factory=dict) simulation_runs: dict[UUID, StoredSimulationRun] = field(default_factory=dict) + report_simulation_run_ids: dict[UUID, tuple[UUID, ...]] = field( + default_factory=dict + ) read_failure: Exception | None = None marked_elsewhere: bool = False + # Each row's status and call as stored, by run ID; a row not named here is + # as the rig's other fakes left it (``holder``). + held: dict[UUID, RunCall | None] = field(default_factory=dict) + holder: Callable[[UnstartedRun], RunCall | None] | None = None windows: list[tuple[datetime, datetime, int]] = field(default_factory=list) marked: list[UnstartedRun] = field(default_factory=list) + def read_call(self, run: UnstartedRun) -> RunCall | None: + if run.run_id in self.held: + return self.held[run.run_id] + return None if self.holder is None else self.holder(run) + + def read_report_run_simulation_run_ids( + self, report_run_id: UUID + ) -> tuple[UUID, ...]: + return self.report_simulation_run_ids.get(report_run_id, ()) + def list_unstarted( self, *, created_after: datetime, created_before: datetime, limit: int ) -> tuple[UnstartedRun, ...]: @@ -288,7 +307,8 @@ def _rig( # noqa: PLR0913 # each fake a test may replace households: FakeHouseholdJobs | None = None, economy_caller: FakeCaller | None = None, household_caller: FakeHouseholdCaller | None = None, - manifest: dict[str, object] | None = None, + manifest: object | None = None, + timer: Callable[[], float] | None = None, ) -> RestartRig: restarts = restarts or FakeRestartStore() economy = economy or FakeStore() @@ -297,6 +317,16 @@ def _rig( # noqa: PLR0913 # each fake a test may replace household_caller = household_caller or FakeHouseholdCaller() telemetry = RecordingTelemetry() store = MANIFEST_STORE if manifest is None else manifest + + def holder(run: UnstartedRun) -> RunCall: + """Return the row as the fakes left it: claimed, given a call, or not.""" + if economy.claimed or households.claimed: + return RunCall(status="running", modal_call_id="fc-original") + recorded = dict([*economy.recorded_calls, *households.recorded]) + return RunCall(status="pending", modal_call_id=recorded.get(run.run_id)) + + if restarts.holder is None: + restarts.holder = holder restart_service = RunRestartService( stores=RestartStores(restarts=restarts, economy=economy, households=households), workers=RestartWorkers( @@ -307,6 +337,7 @@ def _rig( # noqa: PLR0913 # each fake a test may replace settings=SETTINGS, telemetry=telemetry, clock=lambda: NOW, + timer=timer or (lambda: 0.0), call_start_timeout_seconds=0.05, new_request_id=lambda: "restart-request", ) @@ -781,3 +812,143 @@ def test_the_deployment_label_follows_the_modal_environment( def test_a_modal_environment_no_deployment_uses_is_refused() -> None: with pytest.raises(ValueError, match="dev"): RestartSettings.for_modal_environment("dev") + + +def test_an_unreadable_report_run_is_failed_with_its_simulation_runs() -> None: + """As a submission fails a report run, with the simulation runs made for it.""" + run_id = UUID(int=8) + sides = (UUID(int=81), UUID(int=82)) + rig = _rig( + restarts=FakeRestartStore( + unstarted=(_unstarted(RestartKind.REPORT_RUN, run_id),), + read_failure=StoredRunUnreadableError("report run 8 names no region"), + report_simulation_run_ids={run_id: sides}, + ) + ) + + summary = rig.restart() + + ((failed_run_id, simulation_run_ids, message),) = rig.economy.failed_runs + assert (failed_run_id, simulation_run_ids) == (run_id, sides) + assert message.startswith("run_not_started: ") + assert summary == RestartSummary(listed=1, failed=1) + + +def test_a_restarted_call_claiming_its_run_before_the_record_is_restarted() -> None: + stored, _call = _submitted_report() + rig = _rig( + restarts=FakeRestartStore( + unstarted=(_unstarted(RestartKind.REPORT_RUN, stored.report_run_id),), + report_runs={stored.report_run_id: stored}, + held={ + stored.report_run_id: RunCall( + status="running", modal_call_id="fc-coordinator" + ) + }, + ), + economy=FakeStore(claimed=True), + ) + + assert rig.restart() == RestartSummary(listed=1, restarted=1) + + +def test_a_run_failed_before_its_restart_was_recorded_is_left() -> None: + stored, _call = _submitted_report() + rig = _rig( + restarts=FakeRestartStore( + unstarted=(_unstarted(RestartKind.REPORT_RUN, stored.report_run_id),), + report_runs={stored.report_run_id: stored}, + held={stored.report_run_id: RunCall(status="failed", modal_call_id=None)}, + ), + economy=FakeStore(claimed=True), + ) + + assert rig.restart() == RestartSummary(listed=1, left=1) + + +def test_a_refused_restart_of_a_run_the_sweep_failed_meanwhile_is_left() -> None: + stored, _call = _submitted_report() + rig = _rig( + restarts=FakeRestartStore( + unstarted=(_unstarted(RestartKind.REPORT_RUN, stored.report_run_id),), + report_runs={stored.report_run_id: stored}, + held={stored.report_run_id: RunCall(status="failed", modal_call_id=None)}, + ), + economy=FakeStore(claimed=True), + economy_caller=FakeCaller(failure=ConnectionError("Modal is down")), + ) + + assert rig.restart() == RestartSummary(listed=1, left=1) + + +@dataclass +class AdvancingCaller(FakeCaller): + """Starts calls as ``FakeCaller`` does, moving a timer on by each start.""" + + elapsed: list[float] = field(default_factory=lambda: [0.0]) + step_seconds: float = 0.0 + + async def start_report_run(self, *arguments: Any, **keywords: Any) -> str: + self.elapsed[0] += self.step_seconds + return await super().start_report_run(*arguments, **keywords) + + +def test_a_check_past_its_time_limit_leaves_the_other_runs_for_the_next() -> None: + """No row is marked once the check's time is up, so none waits for the sweep.""" + first, _first_call = _submitted_report() + second, _second_call = _submitted_report() + caller = AdvancingCaller(step_seconds=CHECK_TIME_LIMIT.total_seconds()) + rig = _rig( + restarts=FakeRestartStore( + unstarted=( + _unstarted(RestartKind.REPORT_RUN, first.report_run_id), + _unstarted(RestartKind.REPORT_RUN, second.report_run_id), + ), + report_runs={first.report_run_id: first, second.report_run_id: second}, + ), + economy_caller=caller, + timer=lambda: caller.elapsed[0], + ) + + summary = rig.restart() + + assert [run.run_id for run in rig.restarts.marked] == [first.report_run_id] + assert [scope[1] for scope in rig.telemetry.scopes] == [first.report_run_id] + assert summary == RestartSummary(listed=2, restarted=1, deferred=1) + + +class LoopWatchingManifestStore: + """A manifest store that notes whether each read ran on an event loop.""" + + def __init__(self, store: dict[str, object]) -> None: + self.store = store + self.read_on_event_loop: list[bool] = [] + + def get(self, key: str, default: object = None) -> object: + try: + asyncio.get_running_loop() + except RuntimeError: + self.read_on_event_loop.append(False) + else: + self.read_on_event_loop.append(True) + return self.store.get(key, default) + + def __setitem__(self, key: str, value: object) -> None: + raise AssertionError(key) + + +def test_a_runs_call_is_rebuilt_off_the_event_loop() -> None: + """Modal's blocking reads, such as the manifest's, never hold up the loop.""" + stored, _call = _submitted_report() + manifest = LoopWatchingManifestStore(MANIFEST_STORE) + rig = _rig( + restarts=FakeRestartStore( + unstarted=(_unstarted(RestartKind.REPORT_RUN, stored.report_run_id),), + report_runs={stored.report_run_id: stored}, + ), + manifest=manifest, + ) + + assert rig.restart() == RestartSummary(listed=1, restarted=1) + assert manifest.read_on_event_loop + assert not any(manifest.read_on_event_loop) diff --git a/projects/policyengine-simulation-executor/tests/test_executor_image_dependencies.py b/projects/policyengine-simulation-executor/tests/test_executor_image_dependencies.py new file mode 100644 index 000000000..64bbc3e18 --- /dev/null +++ b/projects/policyengine-simulation-executor/tests/test_executor_image_dependencies.py @@ -0,0 +1,51 @@ +"""The executor's Docker image can install every dependency it declares. + +The image's build context is the repository root, and its Dockerfile copies +only some directories before ``uv sync --frozen --no-dev``. A main dependency +installed from a local path outside those directories can't be found, and +the image's build fails. A development group's local packages, such as the +gateway or the Simulation Entrypoint, are not installed there. +""" + +from __future__ import annotations + +import re +import tomllib +from pathlib import Path + +PROJECT = Path(__file__).resolve().parents[1] +REPOSITORY = PROJECT.parents[1] +DEPENDENCY_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*") + + +def _copied_directories() -> list[Path]: + """Return each repository directory the Dockerfile copies into the image.""" + copied = [] + for dockerfile_line in (PROJECT / "Dockerfile").read_text().splitlines(): + words = dockerfile_line.split() + if len(words) == 3 and words[0] == "COPY" and not words[1].startswith("--"): + copied.append((REPOSITORY / words[1]).resolve()) + return copied + + +def test_every_main_dependency_from_a_local_path_is_in_the_images_build() -> None: + pyproject = tomllib.loads((PROJECT / "pyproject.toml").read_text()) + sources = pyproject["tool"]["uv"]["sources"] + main_dependencies = { + DEPENDENCY_NAME.match(requirement).group(0) + for requirement in pyproject["project"]["dependencies"] + } + local_main_dependencies = { + name: (PROJECT / source["path"]).resolve() + for name, source in sources.items() + if "path" in source and name in main_dependencies + } + copied = _copied_directories() + + assert local_main_dependencies + outside_the_build = sorted( + name + for name, path in local_main_dependencies.items() + if not any(path.is_relative_to(directory) for directory in copied) + ) + assert outside_the_build == [] diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep_app.py b/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep_app.py index 553d2d214..2410e7f8b 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep_app.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep_app.py @@ -9,9 +9,12 @@ import yaml from fixtures.fake_modal import install_fake_modal from policyengine_simulation_entry import run_restart_check +from policyengine_simulation_entry.run_restart_check import ( + RESTART_STATEMENT_TIMEOUT_MILLISECONDS, +) from policyengine_simulation_entry.run_restarts import ( CALL_START_TIMEOUT_SECONDS, - RESTARTS_PER_CHECK, + CHECK_TIME_LIMIT, ) from policyengine_simulation_executor.stage12_runtime import ( report_run_access as report_run_access_module, @@ -207,13 +210,22 @@ def test_the_restart_check_runs_every_two_minutes_with_the_worker_database_secre def test_a_check_that_waits_on_every_call_still_ends_within_its_timeout( monkeypatch, ) -> None: - """A row marked just before the timeout stops the check is never restarted.""" + """A row marked just before the timeout stops the check is never restarted. + + The check marks no row once its time limit has passed; after a row is + marked, it waits at most for the call's start and for the mark, record or + failure statements, each bounded by the statement timeout. + """ module = _load(monkeypatch) options = _functions(module)["restart_unstarted_runs"] - longest_call_waits = RESTARTS_PER_CHECK * CALL_START_TIMEOUT_SECONDS + statement_seconds = RESTART_STATEMENT_TIMEOUT_MILLISECONDS / 1000 + longest_after_the_limit = CALL_START_TIMEOUT_SECONDS + 3 * statement_seconds - # A minute for the database and the manifest. - assert longest_call_waits + 60 <= options["timeout"] + # Half a minute more for connecting and Modal's own start. + assert ( + CHECK_TIME_LIMIT.total_seconds() + longest_after_the_limit + 30 + <= options["timeout"] + ) def test_the_image_bakes_the_modal_environment_and_telemetry_settings_in( From 5e158d399ac6538366973fe6f5ba289902674351 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 02:14:35 +0400 Subject: [PATCH 12/29] Fix review round 1's restart findings The executor's Docker image builds again: the Simulation Entrypoint and its persistence library move to the executor's dev group, which the deploy job installs for modal deploy and the image does not. A report run whose rows can't be rebuilt is failed with its simulation runs. A restart is counted by whose call the row records afterwards, so a restarted call that claims its run before the record counts as restarted. A check marks no row after three minutes, rebuilds each call on a thread so Modal's blocking reads don't hold its event loop, and a restart reads its values with the submission's own reader. Co-Authored-By: Claude Opus 5.5 --- .../stage-12-modal-worker-foundation.md | 55 ++++---- .../README.md | 7 +- .../__init__.py | 8 ++ .../economy_run_store.py | 27 +++- .../run_restart_records.py | 79 ++++++++---- .../run_restart_store.py | 15 +++ .../policyengine-simulation-entry/README.md | 10 +- .../run_restart_calls.py | 26 ++++ .../run_restarts.py | 119 +++++++++++------- .../pyproject.toml | 11 +- .../policyengine-simulation-executor/uv.lock | 12 +- 11 files changed, 258 insertions(+), 111 deletions(-) diff --git a/docs/migration/stage-12-modal-worker-foundation.md b/docs/migration/stage-12-modal-worker-foundation.md index 2865fb8b5..47c930f1a 100644 --- a/docs/migration/stage-12-modal-worker-foundation.md +++ b/docs/migration/stage-12-modal-worker-foundation.md @@ -713,32 +713,6 @@ recorded 15 minutes after its creation, a pending or running job once Modal reports its recorded call ended, and any job still unfinished 24 hours after its creation, whatever Modal reports. -**Restarting a run whose call never started.** The same Modal application -runs a restart check every two minutes, as the Modal worker role. It lists -the household jobs, report runs and standalone simulation runs created 3 to -12 minutes ago that are still pending with no call recorded and that -nothing has changed since their insert, as when the Simulation Entrypoint -stopped between creating a run and starting its call. For each, it reads -back what the entrypoint built the call from, builds the same call with the -entrypoint's own builders on the worker of the .py version the row records, -sets the row's `updated_at` so it is never restarted twice, starts the call -and records its ID while the row is still pending with none. A row it can't -start again (its worker left the manifest, its stored rows changed, or Modal -refuses the call) is failed with `run_not_started`; one whose rows or -manifest it can't read now is left for the next check. The sweep's 15 -minutes come after that window, so a second miss is still failed with -`run_not_started`. The restart code is the simulation service's -(`policyengine_simulation_entry.run_restarts`, `run_restart_calls` and -`run_restart_check`, on `policyengine_simulation_persistence`), which the -image adds as source. - -It leaves alone the report runs of a budget window and the runs of -household simulations, whose own calls start them. Each check is a -`v2_run_restart_check` operation and each listed row a `v2_run_restart` -operation, with the row's kind, ID, age and outcome, and each outcome is -also logged as a `v2_run_restart_outcome` event, so the operation count -metric and the events say how often runs need a restart and how each ended. - - Image: the CA store, the frozen `modal-simulation-image` dependency group and the source packages, without the bundle's datasets or the Stage 12 cache. Building it needs no secret. @@ -765,6 +739,35 @@ validation like any other validation failure: the manifest is not published, and, as described under "Release and deployment constraints", the deployment does not promote the new Simulation Entrypoint revision. +**Restarting a run whose call never started.** The stuck-run sweep's Modal +application also runs a restart check every two minutes, as the Modal worker +role. It lists, at most 20 a check, the household jobs, report runs and +standalone simulation runs created 3 to 12 minutes ago that are still pending +with no call recorded and that nothing has changed since their insert, as +when the Simulation Entrypoint stopped between creating a run and starting +its call. For each, on a thread, it reads back what the entrypoint built the +call from and builds the same call with the entrypoint's own builders, on the +worker of the .py version the row records. It then sets the row's +`updated_at`, so the row is never restarted twice, starts the call and +records its ID while the row is still pending with none. A row it can't start +again (its worker left the manifest, its stored rows changed, or Modal +refuses the call) is failed with `run_not_started`, a report run with its +pending simulation runs; one whose rows or manifest it can't read now is left +for the next check. After three minutes a check marks no more rows, so it +ends within its five-minute timeout without leaving a marked row with no +call. The sweep's 15 minutes come after the restart window, so a second miss +is still failed with `run_not_started`. The restart code is the simulation +service's (`policyengine_simulation_entry.run_restarts`, `run_restart_calls` +and `run_restart_check`, on `policyengine_simulation_persistence`), which the +image adds as source. + +It leaves alone a budget window's report runs and household simulations' +runs: it can't rebuild their calls yet. Each check is a +`v2_run_restart_check` operation and each listed row a `v2_run_restart` +operation, with the row's kind, ID, age and outcome, and each outcome is +also logged as a `v2_run_restart_outcome` event, so the operation count +metric and the events say how often runs need a restart and how each ended. + ## Temporary direct runner endpoint > **Temporary Stage 12 interface:** The authenticated routes in this section diff --git a/libs/policyengine-simulation-persistence/README.md b/libs/policyengine-simulation-persistence/README.md index 183845032..4c4b8cd9b 100644 --- a/libs/policyengine-simulation-persistence/README.md +++ b/libs/policyengine-simulation-persistence/README.md @@ -145,7 +145,12 @@ the run tables and their inputs, and `UPDATE` of `updated_at`, catalog version, the region, dataset and year the run records, the report's options and the run's versions. `StoredRunUnreadableError` says the rows no longer describe one calculation: a side's region or dataset - is gone, or the two sides differ. + is gone, or the two sides differ. The values are read with + `read_simulation_reform`, the reader a submission uses. +- `read_report_run_simulation_run_ids` returns the simulation runs a report + run names, so one that can't be rebuilt is failed with them, and + `read_run_call` returns a row's status and recorded call, which says whose + call holds it after a restart. `RunRestartStore` runs each in a transaction of its own. The check records a call, or fails a row, with `EconomyRunStore`'s and `HouseholdJobStore`'s diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py index e1c2048a0..f7236c1f0 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py @@ -51,6 +51,7 @@ SimulationInputs, UnknownDatasetError, UnknownRegionError, + read_simulation_reform, ) from policyengine_simulation_persistence.engine import ( bounded_transaction, @@ -90,6 +91,7 @@ RESTART_TABLES, EconomyScope, RestartKind, + RunCall, StoredEconomySide, StoredReportRun, StoredRunUnreadableError, @@ -97,6 +99,8 @@ UnstartedRun, list_unstarted_runs, mark_restart, + read_report_run_simulation_run_ids, + read_run_call, read_unstarted_report_run, read_unstarted_simulation_run, ) @@ -153,6 +157,7 @@ "ReportSimulation", "RestartKind", "ReusableRun", + "RunCall", "RunRestartStore", "RunReuse", "RunVersions", @@ -202,6 +207,9 @@ "read_model_id", "read_policy_reform", "read_region", + "read_report_run_simulation_run_ids", + "read_run_call", + "read_simulation_reform", "read_stored_household", "read_unstarted_report_run", "read_unstarted_simulation_run", diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_run_store.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_run_store.py index 6e66b2a50..ca05c3b3a 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_run_store.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/economy_run_store.py @@ -443,6 +443,29 @@ def _simulation_inputs( choice: SimulationChoice, ) -> SimulationInputs: """Read one simulation's policy and dynamic values, combined.""" + return SimulationInputs( + choice=choice, + reform=read_simulation_reform( + connection, + country=country, + catalog_version_id=catalog_version_id, + choice=choice, + ), + ) + + +def read_simulation_reform( + connection: Connection, + *, + country: str, + catalog_version_id: UUID, + choice: SimulationChoice, +) -> PolicyReform: + """Read a simulation's policy and dynamic values on a catalog version, combined. + + A submission and a restart of its run read them this way. Raise the + policy reader's errors, or OverlappingReformError. + """ policy_reform = ( {} if choice.policy_id is None @@ -462,6 +485,4 @@ def _simulation_inputs( catalog_version_id=catalog_version_id, ) ) - return SimulationInputs( - choice=choice, reform=combine_reforms(policy_reform, dynamic_reform) - ) + return combine_reforms(policy_reform, dynamic_reform) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py index 98f5f36bb..0613d1f5c 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py @@ -11,7 +11,8 @@ (``updated_at`` still equals ``created_at``): household jobs, report runs, and standalone simulation runs, which no report run names. A report run's own simulation runs are its coordinator's to start. Budget-window years - and household simulations are left out: their own tasks restart them. + and household simulations are left out: the check can't rebuild their + calls yet. 2. ``mark_restart`` sets the row's ``updated_at`` before the check starts its call, so the row is never listed again and is restarted at most once. It uses the statement's clock rather than the transaction's, so the mark @@ -40,6 +41,9 @@ RunVersions, SimulationChoice, ) +from policyengine_simulation_persistence.economy_run_store import ( + read_simulation_reform, +) from policyengine_simulation_persistence.input_tables import ( MODEL_NAME_PREFIX, datasets, @@ -47,11 +51,6 @@ tax_benefit_model_versions, tax_benefit_models, ) -from policyengine_simulation_persistence.policy_reform import ( - combine_reforms, - read_dynamic_reform, - read_policy_reform, -) from policyengine_simulation_persistence.tables import ( SimulationStatus, SimulationType, @@ -102,6 +101,14 @@ class StoredRunUnreadableError(LookupError): """A run's stored rows no longer describe one calculation to start.""" +@dataclass(frozen=True) +class RunCall: + """A row's status and the Modal call it records, if any.""" + + status: str + modal_call_id: str | None + + @dataclass(frozen=True) class EconomyScope: """What every simulation of an economy run calculates on, as stored. @@ -256,6 +263,40 @@ def mark_restart(connection: Connection, run: UnstartedRun) -> bool: return marked is not None +def read_run_call(connection: Connection, run: UnstartedRun) -> RunCall | None: + """Return the row's status and recorded call, or None when it is gone.""" + table = RESTART_TABLES[run.kind] + row = connection.execute( + select(table.c.status, table.c.modal_call_id).where(table.c.id == run.run_id) + ).one_or_none() + if row is None: + return None + return RunCall(status=str(row.status), modal_call_id=row.modal_call_id) + + +def read_report_run_simulation_run_ids( + connection: Connection, report_run_id: UUID +) -> tuple[UUID, ...]: + """Return the simulation runs the report run names, baseline first. + + Return no IDs when there is no such run. A report run the check can't + rebuild is failed with them, as its submission would fail it. + """ + row = connection.execute( + select( + report_runs.c.baseline_simulation_run_id, + report_runs.c.reform_simulation_run_id, + ).where(report_runs.c.id == report_run_id) + ).one_or_none() + if row is None: + return () + return tuple( + run_id + for run_id in (row.baseline_simulation_run_id, row.reform_simulation_run_id) + if run_id is not None + ) + + def _side_rows(connection: Connection, simulation_run_id: UUID) -> Row[Any] | None: """Return a simulation run's scope and its simulation's policy and dynamic.""" return connection.execute( @@ -312,29 +353,15 @@ def _side( Raise the policy reader's errors, or OverlappingReformError. """ choice = SimulationChoice(policy_id=side.policy_id, dynamic_id=side.dynamic_id) - policy_reform = ( - {} - if choice.policy_id is None - else read_policy_reform( - connection, - country=country, - policy_id=choice.policy_id, - catalog_version_id=scope.catalog_version_id, - ) - ) - dynamic_reform = ( - {} - if choice.dynamic_id is None - else read_dynamic_reform( - connection, - dynamic_id=choice.dynamic_id, - catalog_version_id=scope.catalog_version_id, - ) - ) return StoredEconomySide( simulation_run_id=side.id, choice=choice, - reform=combine_reforms(policy_reform, dynamic_reform), + reform=read_simulation_reform( + connection, + country=country, + catalog_version_id=scope.catalog_version_id, + choice=choice, + ), ) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_store.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_store.py index f7005c662..ce1733723 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_store.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_store.py @@ -14,6 +14,8 @@ from policyengine_simulation_persistence.run_restart_records import ( list_unstarted_runs, mark_restart, + read_report_run_simulation_run_ids, + read_run_call, read_unstarted_report_run, read_unstarted_simulation_run, ) @@ -27,6 +29,7 @@ from sqlalchemy import Connection, Engine from policyengine_simulation_persistence.run_restart_records import ( + RunCall, StoredReportRun, StoredSimulationRun, UnstartedRun, @@ -83,3 +86,15 @@ def read_simulation_run( """Return what the standalone simulation run's call is built from, or None.""" with self._transaction() as connection: return read_unstarted_simulation_run(connection, simulation_run_id) + + def read_call(self, run: UnstartedRun) -> RunCall | None: + """Return the row's status and recorded call, or None when it is gone.""" + with self._transaction() as connection: + return read_run_call(connection, run) + + def read_report_run_simulation_run_ids( + self, report_run_id: UUID + ) -> tuple[UUID, ...]: + """Return the simulation runs the report run names, baseline first.""" + with self._transaction() as connection: + return read_report_run_simulation_run_ids(connection, report_run_id) diff --git a/projects/policyengine-simulation-entry/README.md b/projects/policyengine-simulation-entry/README.md index 240b68e8f..fee081036 100644 --- a/projects/policyengine-simulation-entry/README.md +++ b/projects/policyengine-simulation-entry/README.md @@ -120,7 +120,8 @@ runs it every two minutes as the Modal worker role. - It takes the rows created 3 to 12 minutes ago that are still `pending` with no call and that nothing has changed since their insert, at most 20 a check, oldest first. A budget window's report runs and household - simulations' runs are left out. + simulations' runs are left out. After three minutes a check marks no more + rows and leaves the rest for the next. - It rebuilds each call with this service's own builders (`report_run_calculation`, `simulation_run_calculation`, `prepare_household_job`) from the stored rows, on the manifest's worker @@ -129,11 +130,14 @@ runs it every two minutes as the Modal worker role. (`run-restart-`), and its creation time is the row's. - It sets the row's `updated_at` before starting the call, so a row is restarted at most once, then records the call's ID with the submission's - own conditional write. + own conditional write. The restart counts as done when the row then + records its call; a row recording another call was claimed by that call + first. - A row whose call can't be rebuilt, or whose restart Modal refuses or doesn't confirm within ten seconds, is failed with `run_not_started: The run stayed pending with no Modal call recorded for - it, and โ€ฆ`. A row whose rows or manifest can't be read now is left + it, and โ€ฆ`; a report run is failed with its simulation runs still pending + with no call. A row whose rows or manifest can't be read now is left unmarked for the next check. - Each check, each row and each outcome are telemetry; see `run_restart_check`'s docstring for the operations and event. diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py index 8c7d12b9d..40d089da9 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py @@ -68,6 +68,7 @@ HouseholdJobRecord, HouseholdJobStatus, NewHouseholdJob, + RunCall, RunVersions, StoredReportRun, StoredSimulationRun, @@ -141,6 +142,16 @@ def read_simulation_run( """Return what the standalone simulation run's call is built from.""" ... + def read_report_run_simulation_run_ids( + self, report_run_id: UUID + ) -> tuple[UUID, ...]: + """Return the simulation runs the report run names, baseline first.""" + ... + + def read_call(self, run: UnstartedRun) -> RunCall | None: + """Return the row's status and recorded call, or None when it is gone.""" + ... + class HouseholdJobRestarts(Protocol): """The household job reads and writes a restart uses (``HouseholdJobStore``).""" @@ -367,12 +378,27 @@ def build(self, run: UnstartedRun) -> RebuiltCall | None: except (SQLAlchemyError, VersionManifestUnavailableError) as error: raise DeferredRestartError(type(error).__name__) from error except _REBUILD_ERRORS as error: + if run.kind is RestartKind.REPORT_RUN and not simulation_run_ids: + simulation_run_ids = self._report_simulation_run_ids(run.run_id) raise CallNotRebuiltError( _rebuild_reason(error), error_type=type(error).__name__, simulation_run_ids=simulation_run_ids, ) from error + def _report_simulation_run_ids(self, report_run_id: UUID) -> tuple[UUID, ...]: + """Return the simulation runs a report run its rows can't rebuild names. + + It is failed with them, as its submission fails it with the simulation + runs created for it; only those still pending with no call are failed. + When even they can't be read, it is failed alone, and the sweep fails + them later. + """ + try: + return self._restarts.read_report_run_simulation_run_ids(report_run_id) + except SQLAlchemyError: + return () + def _economy_worker(self, country: str, versions: RunVersions) -> EconomyWorker: """Return the worker of the run's .py version, which must match its versions.""" worker = self._workers.economy.worker_for_version( diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py index 631de8d18..95a7ba665 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py @@ -32,6 +32,7 @@ from __future__ import annotations import asyncio +import time import uuid from collections import Counter from contextlib import nullcontext @@ -69,6 +70,7 @@ __all__ = [ "CALL_START_TIMEOUT_SECONDS", + "CHECK_TIME_LIMIT", "RESTARTS_PER_CHECK", "RESTART_AFTER", "RESTART_UNTIL", @@ -90,31 +92,35 @@ # A row older than this is left to the stuck-run sweep, which fails it at 15 # minutes, so a restart has time to record its call before the sweep judges it. RESTART_UNTIL: Final = timedelta(minutes=12) -# At most this many rows are restarted by one check; the next takes the rest. -# With every call waited on for its full time, a check still ends within the -# scheduled function's five-minute timeout, so no row it marks is left -# without its call. +# At most this many rows are listed by one check; the next takes the rest. RESTARTS_PER_CHECK: Final = 20 # How long the check waits for Modal to accept a restarted call. CALL_START_TIMEOUT_SECONDS: Final = 10.0 +# No row is marked once a check has run this long: the rest are left unmarked +# for the next check. A marked row then needs only its call's start and a few +# bounded statements, so the check ends within the scheduled function's +# timeout and no row it marks is left without its call. +CHECK_TIME_LIMIT: Final = timedelta(minutes=3) _NOT_STARTED: Final = ( "run_not_started: The run stayed pending with no Modal call recorded for it" ) -_PENDING: Final = "pending" class RestartOutcome(StrEnum): """How the check ended for one listed row. - ``restarted``: its new call started and the call's ID was recorded. - ``call_not_recorded``: its new call started, but the ID couldn't be - recorded; the call claims the row itself. ``started_meanwhile``: another - call claimed the row after the check started its own, which that call - then can't claim. ``left``: the row was gone, or changed before the check - started a call; also a row whose failure couldn't be written, which the + ``restarted``: its new call started, and the row records that call. + ``call_not_recorded``: its new call started, but the check couldn't + record or read the call's ID; the call claims the row itself. + ``started_meanwhile``: the row records another call, which claimed it + after the check marked it (or the check's own call, accepted after the + check stopped waiting for it); the other call then can't claim it. + ``left``: the row was gone, changed before the check started a call, or + failed meanwhile; also a row whose failure couldn't be written, which the sweep fails later. ``deferred``: its rows or the manifest couldn't be - read, so it is left unmarked for the next check. ``failed``: it couldn't - be started again and was failed with ``run_not_started``. + read, or the check's time ran out, so it is left unmarked for the next + check. ``failed``: it couldn't be started again and was failed with + ``run_not_started``. """ RESTARTED = "restarted" @@ -231,8 +237,9 @@ class RestartStores: class RunRestartService: """Start the calls of the runs and household jobs whose call never started. - The store reads and writes are synchronous: the check is a batch job of - its own, so nothing else waits on its event loop. + A call is rebuilt on a thread: the reads, Modal's manifest read included, + are synchronous. The check's other database writes are short and run on + its own event loop, which only the check uses. """ def __init__( # noqa: PLR0913 # each seam tests replace @@ -244,6 +251,7 @@ def __init__( # noqa: PLR0913 # each seam tests replace settings: RestartSettings, telemetry: RestartTelemetry | None = None, clock: Callable[[], datetime] = lambda: datetime.now(UTC), + timer: Callable[[], float] = time.monotonic, call_start_timeout_seconds: float = CALL_START_TIMEOUT_SECONDS, new_request_id: Callable[[], str] = lambda: f"run-restart-{uuid.uuid4()}", ) -> None: @@ -259,13 +267,18 @@ def __init__( # noqa: PLR0913 # each seam tests replace ) self._telemetry = telemetry or NoRestartTelemetry() self._clock = clock + self._timer = timer self._call_start_timeout_seconds = call_start_timeout_seconds + self._deadline = 0.0 async def restart_unstarted_runs(self) -> RestartSummary: """Start once more each row created 3 to 12 minutes ago with no call. - Raise the store's error when the rows can't be listed. + Rows the check has no time left for are counted ``deferred``, with no + telemetry scope, since the next check lists them again. Raise the + store's error when the rows can't be listed. """ + self._deadline = self._timer() + CHECK_TIME_LIMIT.total_seconds() now = self._clock() runs = self._stores.restarts.list_unstarted( created_after=now - RESTART_UNTIL, @@ -273,7 +286,10 @@ async def restart_unstarted_runs(self) -> RestartSummary: limit=RESTARTS_PER_CHECK, ) outcomes: Counter[RestartOutcome] = Counter() - for run in runs: + for position, run in enumerate(runs): + if self._out_of_time(): + outcomes[RestartOutcome.DEFERRED] += len(runs) - position + break age_seconds = (now - run.created_at).total_seconds() with self._telemetry.restart_scope(run, age_seconds=age_seconds): outcome, error_type = await self._restart(run) @@ -284,10 +300,13 @@ async def restart_unstarted_runs(self) -> RestartSummary: **{outcome.value: count for outcome, count in outcomes.items()}, ) + def _out_of_time(self) -> bool: + return self._timer() >= self._deadline + async def _restart(self, run: UnstartedRun) -> tuple[RestartOutcome, str | None]: """Start the row's call once more; return how it ended and any error type.""" try: - rebuilt = self._calls.build(run) + rebuilt = await asyncio.to_thread(self._calls.build, run) except DeferredRestartError as deferred: return RestartOutcome.DEFERRED, str(deferred) except CallNotRebuiltError as cannot: @@ -314,6 +333,8 @@ async def _start_marked( simulation_run_ids: tuple[UUID, ...], ) -> tuple[RestartOutcome, str | None]: """Mark the row, then start its call and record the call's ID.""" + if self._out_of_time(): + return RestartOutcome.DEFERRED, None try: if not self._stores.restarts.mark_restart(run): return RestartOutcome.LEFT, None @@ -332,29 +353,42 @@ async def _start_marked( f"again ({type(error).__name__})." ) failed = self._fail(run, message, simulation_run_ids) - # A failure that changed nothing means another call claimed the - # row; one that couldn't be written leaves the row to the sweep. - outcome = { - True: RestartOutcome.FAILED, - False: RestartOutcome.STARTED_MEANWHILE, - None: RestartOutcome.LEFT, - }[failed] + # A failure that changed nothing means a call holds the row now, + # or something failed it; one that couldn't be written leaves the + # row to the sweep. + if failed is None: + outcome = RestartOutcome.LEFT + elif failed: + outcome = RestartOutcome.FAILED + else: + outcome = self._held_since(run, call_id=None) return outcome, type(error).__name__ - return self._recorded(run, call_id) - - def _recorded( - self, run: UnstartedRun, call_id: str - ) -> tuple[RestartOutcome, str | None]: - """Record the started call's ID; return how the restart ended.""" try: - status = self._record(run, call_id) + self._record(run, call_id) except SQLAlchemyError as error: return RestartOutcome.CALL_NOT_RECORDED, type(error).__name__ - if status is None: - return RestartOutcome.LEFT, None - if status == _PENDING: - return RestartOutcome.RESTARTED, None - return RestartOutcome.STARTED_MEANWHILE, None + return self._held_since(run, call_id=call_id), None + + def _held_since(self, run: UnstartedRun, *, call_id: str | None) -> RestartOutcome: + """Return how the restart ended, from the call the row records now. + + ``call_id`` is the check's own call, or None when Modal didn't + confirm one. The record write keeps a call recorded or claimed + before it, so whose call the row records says whose start counts. + """ + try: + held = self._stores.restarts.read_call(run) + except SQLAlchemyError: + return ( + RestartOutcome.LEFT + if call_id is None + else RestartOutcome.CALL_NOT_RECORDED + ) + if held is None or held.modal_call_id is None: + return RestartOutcome.LEFT + if held.modal_call_id == call_id: + return RestartOutcome.RESTARTED + return RestartOutcome.STARTED_MEANWHILE def _fail( self, run: UnstartedRun, message: str, simulation_run_ids: tuple[UUID, ...] @@ -378,12 +412,11 @@ def _fail( return None return failed - def _record(self, run: UnstartedRun, call_id: str) -> str | None: - """Record the call's ID while the row is unstarted; return its status after.""" + def _record(self, run: UnstartedRun, call_id: str) -> None: + """Record the call's ID while the row is pending with no call.""" if run.kind is RestartKind.REPORT_RUN: - status = self._stores.economy.record_report_call(run.run_id, call_id) + self._stores.economy.record_report_call(run.run_id, call_id) elif run.kind is RestartKind.SIMULATION_RUN: - status = self._stores.economy.record_simulation_call(run.run_id, call_id) + self._stores.economy.record_simulation_call(run.run_id, call_id) else: - status = self._stores.households.record_call(run.run_id, call_id) - return None if status is None else str(status) + self._stores.households.record_call(run.run_id, call_id) diff --git a/projects/policyengine-simulation-executor/pyproject.toml b/projects/policyengine-simulation-executor/pyproject.toml index f566c5f93..cdb6f32e3 100644 --- a/projects/policyengine-simulation-executor/pyproject.toml +++ b/projects/policyengine-simulation-executor/pyproject.toml @@ -19,10 +19,6 @@ dependencies = [ "policyengine-simulation-observability", "policyengine-simulation-contract[modal]", "policyengine-stage12-persistence", - # The restart check in the stuck-run sweep's Modal app runs the - # simulation service's own restart code on its persistence library. - "policyengine-simulation-entry", - "policyengine-simulation-persistence", # PolicyEngine.py's models extra is the only selector for core, country # models, and the SPM calculator. Do not duplicate those versions here. "policyengine[models]==6.17.1", @@ -75,8 +71,15 @@ modal-simulation-image = [ ] # The executor's budget-window scheduler tests exercise the gateway app # end-to-end; the gateway project rides in as a dev-only path dependency. +# The restart check in the stuck-run sweep's Modal app runs the simulation +# service's own restart code: ``modal deploy`` adds the Simulation Entrypoint +# and its persistence library to that app's image as local sources, so the +# deploy job's ``uv sync`` installs them here. Nothing the executor's own +# Docker image runs imports them, and it is built without this group. dev = [ "policyengine-simulation-gateway", + "policyengine-simulation-entry", + "policyengine-simulation-persistence", ] [tool.uv.sources] diff --git a/projects/policyengine-simulation-executor/uv.lock b/projects/policyengine-simulation-executor/uv.lock index 8ddd1c0d5..2fab1f614 100644 --- a/projects/policyengine-simulation-executor/uv.lock +++ b/projects/policyengine-simulation-executor/uv.lock @@ -1948,9 +1948,7 @@ dependencies = [ { name = "policyengine-fastapi" }, { name = "policyengine-observability", extra = ["fastapi", "google", "otlp-grpc"] }, { name = "policyengine-simulation-contract", extra = ["modal"] }, - { name = "policyengine-simulation-entry" }, { name = "policyengine-simulation-observability" }, - { name = "policyengine-simulation-persistence" }, { name = "policyengine-stage12-persistence" }, { name = "pyarrow" }, { name = "pydantic-settings" }, @@ -1973,7 +1971,9 @@ test = [ [package.dev-dependencies] dev = [ + { name = "policyengine-simulation-entry" }, { name = "policyengine-simulation-gateway" }, + { name = "policyengine-simulation-persistence" }, ] modal-simulation-image = [ { name = "fastapi" }, @@ -2002,9 +2002,7 @@ requires-dist = [ { name = "policyengine-fastapi", editable = "../../libs/policyengine-fastapi" }, { name = "policyengine-observability", extras = ["fastapi", "google", "otlp-grpc"], specifier = ">=3.0.7,<4" }, { name = "policyengine-simulation-contract", extras = ["modal"], editable = "../../libs/policyengine-simulation-contract" }, - { name = "policyengine-simulation-entry", editable = "../policyengine-simulation-entry" }, { name = "policyengine-simulation-observability", editable = "../../libs/policyengine-simulation-observability" }, - { name = "policyengine-simulation-persistence", editable = "../../libs/policyengine-simulation-persistence" }, { name = "policyengine-stage12-persistence", editable = "../../libs/policyengine-stage12-persistence" }, { name = "pyarrow", specifier = ">=20,<24" }, { name = "pydantic-settings", specifier = ">=2.7.1,<3.0.0" }, @@ -2018,7 +2016,11 @@ requires-dist = [ provides-extras = ["test", "build"] [package.metadata.requires-dev] -dev = [{ name = "policyengine-simulation-gateway", editable = "../policyengine-simulation-gateway" }] +dev = [ + { name = "policyengine-simulation-entry", editable = "../policyengine-simulation-entry" }, + { name = "policyengine-simulation-gateway", editable = "../policyengine-simulation-gateway" }, + { name = "policyengine-simulation-persistence", editable = "../../libs/policyengine-simulation-persistence" }, +] modal-simulation-image = [ { name = "fastapi", specifier = ">=0.115.0" }, { name = "google-cloud-storage", specifier = ">=2" }, From 1768f26f4f91d752e3cbe594b1b0034e2b216d6f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 02:41:38 +0400 Subject: [PATCH 13/29] Test review round 2's restart findings (failing) A restart is recorded with a minute to spare before the sweep judges its run, an unexpected rebuild error defers only its own run, and the restart check imports with only the sweep image's packages (this last test passed when first run: it guards what already holds). Co-Authored-By: Claude Opus 5.5 --- .../tests/test_run_restarts.py | 35 ++++- .../tests/test_run_restart_image.py | 132 ++++++++++++++++++ .../tests/test_stage12_run_sweep.py | 16 ++- 3 files changed, 178 insertions(+), 5 deletions(-) create mode 100644 projects/policyengine-simulation-executor/tests/test_run_restart_image.py diff --git a/projects/policyengine-simulation-entry/tests/test_run_restarts.py b/projects/policyengine-simulation-entry/tests/test_run_restarts.py index 42c8cce0c..68ecd2eda 100644 --- a/projects/policyengine-simulation-entry/tests/test_run_restarts.py +++ b/projects/policyengine-simulation-entry/tests/test_run_restarts.py @@ -124,6 +124,8 @@ class FakeRestartStore: default_factory=dict ) read_failure: Exception | None = None + # A read of one run that raises, by run ID. + read_failures: dict[UUID, Exception] = field(default_factory=dict) marked_elsewhere: bool = False # Each row's status and call as stored, by run ID; a row not named here is # as the rig's other fakes left it (``holder``). @@ -157,6 +159,8 @@ def mark_restart(self, run: UnstartedRun) -> bool: def read_report_run(self, report_run_id: UUID) -> StoredReportRun | None: if self.read_failure is not None: raise self.read_failure + if report_run_id in self.read_failures: + raise self.read_failures[report_run_id] return self.report_runs.get(report_run_id) def read_simulation_run( @@ -452,7 +456,7 @@ def _without_request_id(context: dict[str, Any]) -> dict[str, Any]: return {key: value for key, value in context.items() if key != "request_id"} -def test_the_check_lists_runs_created_three_to_twelve_minutes_ago() -> None: +def test_the_check_lists_runs_created_three_to_ten_minutes_ago() -> None: rig = _rig() summary = rig.restart() @@ -462,7 +466,7 @@ def test_the_check_lists_runs_created_three_to_twelve_minutes_ago() -> None: ] assert ( timedelta(minutes=3), - timedelta(minutes=12), + timedelta(minutes=10), ) == (RESTART_AFTER, RESTART_UNTIL) assert summary == RestartSummary(listed=0) @@ -952,3 +956,30 @@ def test_a_runs_call_is_rebuilt_off_the_event_loop() -> None: assert rig.restart() == RestartSummary(listed=1, restarted=1) assert manifest.read_on_event_loop assert not any(manifest.read_on_event_loop) + + +def test_an_unexpected_rebuild_error_defers_its_run_and_the_check_goes_on() -> None: + """One row the check can't rebuild for an unforeseen reason holds up no other.""" + broken_run_id = UUID(int=9) + stored, _call = _submitted_report() + rig = _rig( + restarts=FakeRestartStore( + unstarted=( + _unstarted(RestartKind.REPORT_RUN, broken_run_id), + _unstarted(RestartKind.REPORT_RUN, stored.report_run_id), + ), + report_runs={stored.report_run_id: stored}, + read_failures={broken_run_id: RuntimeError("unforeseen")}, + ) + ) + + summary = rig.restart() + + assert rig.telemetry.outcomes[0] == ( + RestartKind.REPORT_RUN, + broken_run_id, + RestartOutcome.DEFERRED, + "RuntimeError", + ) + assert [run.run_id for run in rig.restarts.marked] == [stored.report_run_id] + assert summary == RestartSummary(listed=2, restarted=1, deferred=1) diff --git a/projects/policyengine-simulation-executor/tests/test_run_restart_image.py b/projects/policyengine-simulation-executor/tests/test_run_restart_image.py new file mode 100644 index 000000000..6f36bd130 --- /dev/null +++ b/projects/policyengine-simulation-executor/tests/test_run_restart_image.py @@ -0,0 +1,132 @@ +"""The restart check imports with only the packages the sweep's image installs. + +The sweep's Modal image installs the executor's locked +``modal-simulation-image`` group alone, and Modal adds its own client; the +restart check's code comes in as local sources. The scheduled function +imports that code only when it runs, so ``modal deploy`` would not notice a +package the image lacks. Here the import runs in a process that refuses +every other installed distribution's modules. +""" + +from __future__ import annotations + +import re +import subprocess +import sys +import tomllib +from importlib import metadata +from pathlib import Path + +PROJECT = Path(__file__).resolve().parents[1] +IMAGE_GROUP = "modal-simulation-image" +# Modal installs its client into every image it builds. +MODAL_CLIENT = "modal" +# The local sources the sweep's image adds, which the lock doesn't install. +LOCAL_SOURCES = frozenset( + { + "policyengine-simulation-executor", + "policyengine-simulation-observability", + "policyengine-simulation-contract", + "policyengine-stage12-persistence", + "policyengine-simulation-persistence", + "policyengine-simulation-entry", + } +) +# What the scheduled function's body imports. +RESTART_MODULES = ( + "policyengine_simulation_entry.run_restart_check", + "policyengine_simulation_observability.observability", +) + + +def _normalized(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() + + +def _image_distributions() -> set[str]: + """Return the locked image group's distributions, with their dependencies.""" + lock = tomllib.loads((PROJECT / "uv.lock").read_text()) + packages = {_normalized(package["name"]): package for package in lock["package"]} + project = packages["policyengine-simulation-executor"] + wanted = [ + (_normalized(dependency["name"]), tuple(dependency.get("extra", ()))) + for dependency in project["dev-dependencies"][IMAGE_GROUP] + ] + wanted.append((MODAL_CLIENT, ())) + installed: set[str] = set() + seen: set[tuple[str, tuple[str, ...]]] = set() + while wanted: + name, extras = wanted.pop() + if (name, extras) in seen or name not in packages: + continue + seen.add((name, extras)) + installed.add(name) + package = packages[name] + dependencies = list(package.get("dependencies", ())) + for extra in extras: + dependencies.extend(package.get("optional-dependencies", {}).get(extra, ())) + wanted.extend( + (_normalized(dependency["name"]), tuple(dependency.get("extra", ()))) + for dependency in dependencies + ) + return installed + + +def _top_level_modules(distribution: metadata.Distribution) -> set[str]: + """Return the importable top-level names a distribution installs.""" + listed = distribution.read_text("top_level.txt") + if listed: + return {name.strip() for name in listed.splitlines() if name.strip()} + modules = set() + for installed_file in distribution.files or (): + first = installed_file.parts[0] + if first.endswith((".dist-info", ".data")) or first in {"..", "__pycache__"}: + continue + if len(installed_file.parts) == 1 and installed_file.suffix != ".py": + continue + modules.add(first.removesuffix(".py")) + return modules + + +def _refused_modules() -> set[str]: + """Return the modules only distributions outside the image install.""" + allowed = _image_distributions() | LOCAL_SOURCES + allowed_modules: set[str] = set() + other_modules: set[str] = set() + for distribution in metadata.distributions(): + modules = _top_level_modules(distribution) + if _normalized(distribution.metadata["Name"]) in allowed: + allowed_modules |= modules + else: + other_modules |= modules + return other_modules - allowed_modules + + +def test_the_restart_check_imports_with_only_the_sweep_images_packages( + tmp_path, +) -> None: + refused = _refused_modules() + # The executor's own environment holds packages the image doesn't. + assert {"jwt", "uvicorn"} <= refused + script = tmp_path / "import_restart_check.py" + script.write_text( + "import importlib, sys\n" + f"refused = {sorted(refused)!r}\n" + "class Refuse:\n" + " def find_spec(self, name, path=None, target=None):\n" + " if name.split('.')[0] in refused:\n" + " raise ImportError(f'not in the sweep image: {name}')\n" + "sys.meta_path.insert(0, Refuse())\n" + f"for module in {list(RESTART_MODULES)!r}:\n" + " importlib.import_module(module)\n", + encoding="utf-8", + ) + + completed = subprocess.run( # the test's own interpreter and script + [sys.executable, str(script)], + capture_output=True, + text=True, + check=False, + ) + + assert completed.returncode == 0, completed.stderr diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep.py b/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep.py index 0ad0be027..8ec46da1f 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep.py @@ -8,7 +8,11 @@ from uuid import uuid4 import pytest -from policyengine_simulation_entry.run_restarts import RESTART_UNTIL +from policyengine_simulation_entry.run_restarts import ( + CALL_START_TIMEOUT_SECONDS, + CHECK_TIME_LIMIT, + RESTART_UNTIL, +) from policyengine_simulation_executor.stage12_runtime.run_sweep import ( DEFAULT_MAXIMUM_AGE, DEFAULT_MINIMUM_AGE, @@ -147,7 +151,7 @@ def test_the_sweep_lists_runs_older_than_its_minimum_age() -> None: def test_by_default_unstarted_runs_wait_until_the_restart_check_has_passed() -> None: - """The restart check starts a pending run with no call 3 to 12 minutes old. + """The restart check starts a pending run with no call 3 to 10 minutes old. The sweep fails such a run only once it is 15 minutes old, so a run whose restart started is judged by its call, and only a second miss is failed. @@ -157,7 +161,13 @@ def test_by_default_unstarted_runs_wait_until_the_restart_check_has_passed() -> sweep_stuck_runs(store, call_state=lambda _: CallState.ENDED, now=NOW) assert store.cutoffs == [NOW - timedelta(minutes=15)] * 3 - assert DEFAULT_MINIMUM_AGE > RESTART_UNTIL + # A row listed at the end of the window may be marked at the end of the + # check's time limit; its call starts and is recorded with a minute to + # spare before the sweep judges it. + latest_restart = ( + RESTART_UNTIL + CHECK_TIME_LIMIT + timedelta(seconds=CALL_START_TIMEOUT_SECONDS) + ) + assert latest_restart + timedelta(minutes=1) <= DEFAULT_MINIMUM_AGE def test_report_runs_are_swept_before_simulation_runs_are_listed() -> None: From b0db539efedd1bc426e9a90569f647447bd9f59b Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 02:42:55 +0400 Subject: [PATCH 14/29] Fix review round 2's restart findings The restart window is now 3 to 10 minutes after creation, so a row marked at the end of a check's time limit has its call recorded minutes before the sweep judges it; an unexpected error rebuilding one row's call defers that row and the check goes on; and the telemetry docs say how to count runs from restart attempts. Co-Authored-By: Claude Opus 5.5 --- changelog.d/c1-19-restart-unstarted-runs.added.md | 2 +- docs/migration/stage-12-modal-worker-foundation.md | 12 +++++++----- projects/policyengine-simulation-entry/README.md | 4 ++-- .../run_restart_check.py | 8 +++++--- .../policyengine_simulation_entry/run_restarts.py | 14 ++++++++++---- .../src/modal/run_sweep_app.py | 2 +- .../stage12_runtime/run_sweep.py | 4 ++-- .../tests/test_stage12_run_sweep_app.py | 2 +- 8 files changed, 29 insertions(+), 19 deletions(-) diff --git a/changelog.d/c1-19-restart-unstarted-runs.added.md b/changelog.d/c1-19-restart-unstarted-runs.added.md index f81d27024..2ad18f24b 100644 --- a/changelog.d/c1-19-restart-unstarted-runs.added.md +++ b/changelog.d/c1-19-restart-unstarted-runs.added.md @@ -1 +1 @@ -A restart check in the stuck-run sweep's Modal application starts the Modal call of a household job, economy report run or standalone economy simulation run again when the run stayed pending with no call recorded 3 to 12 minutes after its creation, as when the Simulation Entrypoint stopped before starting its call; its telemetry counts every restart and how it ended. The sweep now fails a run still pending with no call at 15 minutes instead of 10. +A restart check in the stuck-run sweep's Modal application starts the Modal call of a household job, economy report run or standalone economy simulation run again when the run stayed pending with no call recorded 3 to 10 minutes after its creation, as when the Simulation Entrypoint stopped before starting its call; its telemetry counts every restart and how it ended. The sweep now fails a run still pending with no call at 15 minutes instead of 10. diff --git a/docs/migration/stage-12-modal-worker-foundation.md b/docs/migration/stage-12-modal-worker-foundation.md index 47c930f1a..84311afae 100644 --- a/docs/migration/stage-12-modal-worker-foundation.md +++ b/docs/migration/stage-12-modal-worker-foundation.md @@ -742,7 +742,7 @@ does not promote the new Simulation Entrypoint revision. **Restarting a run whose call never started.** The stuck-run sweep's Modal application also runs a restart check every two minutes, as the Modal worker role. It lists, at most 20 a check, the household jobs, report runs and -standalone simulation runs created 3 to 12 minutes ago that are still pending +standalone simulation runs created 3 to 10 minutes ago that are still pending with no call recorded and that nothing has changed since their insert, as when the Simulation Entrypoint stopped between creating a run and starting its call. For each, on a thread, it reads back what the entrypoint built the @@ -763,10 +763,12 @@ image adds as source. It leaves alone a budget window's report runs and household simulations' runs: it can't rebuild their calls yet. Each check is a -`v2_run_restart_check` operation and each listed row a `v2_run_restart` -operation, with the row's kind, ID, age and outcome, and each outcome is -also logged as a `v2_run_restart_outcome` event, so the operation count -metric and the events say how often runs need a restart and how each ended. +`v2_run_restart_check` operation and each row it takes up a +`v2_run_restart` operation, with the row's kind, ID, age and outcome, and +each outcome is also logged as a `v2_run_restart_outcome` event. A deferred +row is taken up again by the next check, so the runs that needed a restart +are the distinct run IDs, or the outcomes other than `deferred`; the events +say how each ended. ## Temporary direct runner endpoint diff --git a/projects/policyengine-simulation-entry/README.md b/projects/policyengine-simulation-entry/README.md index fee081036..42dd96279 100644 --- a/projects/policyengine-simulation-entry/README.md +++ b/projects/policyengine-simulation-entry/README.md @@ -101,7 +101,7 @@ seconds, the job is failed at once and the 503 (`modal_call_refused`) names it; a job the household function claimed meanwhile is answered with 202 as it stands, and a job whose state couldn't then be recorded or read with 202 `pending`, to be read rather than submitted again (the restart check starts -its call once more 3 to 12 minutes after its creation, and the stuck-run +its call once more 3 to 10 minutes after its creation, and the stuck-run sweep fails it if no call ever starts). Every 500 of either route, from an unexpected failure or a request the service built and the household contract refuses, is the JSON error with code `internal_error`. `DEPLOYMENT.md` describes the deployment settings and the @@ -117,7 +117,7 @@ this service: the stuck-run sweep's Modal application (`projects/policyengine-simulation-executor/src/modal/run_sweep_app.py`) runs it every two minutes as the Modal worker role. -- It takes the rows created 3 to 12 minutes ago that are still `pending` +- It takes the rows created 3 to 10 minutes ago that are still `pending` with no call and that nothing has changed since their insert, at most 20 a check, oldest first. A budget window's report runs and household simulations' runs are left out. After three minutes a check marks no more diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py index 0030b49ac..dbdae0e00 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py @@ -10,9 +10,11 @@ - each check is one ``v2_run_restart_check`` operation, with the number of rows it listed and how each ended (``restart_listed``, ``restart_failed`` and the rest) set on it; -- each row it lists is one ``v2_run_restart`` operation, with the row's kind, - ID and age, so the operation count metric counts the runs whose call never - started; +- each row it takes up is one ``v2_run_restart`` operation, with the row's + kind, ID and age. A row left ``deferred`` is taken up again by the next + check, so the operation count metric counts restart attempts; the runs + whose call never started are the distinct ``run_id`` values, or the + operations with any outcome but ``deferred``; - each row's outcome is set on its operation (``restart_outcome``, with ``restart_error_type`` when an error ended it) and logged as a ``v2_run_restart_outcome`` event, at WARNING when the row could not be diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py index 95a7ba665..be7b230b4 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py @@ -8,7 +8,7 @@ minutes after its creation. The restart check, a scheduled Modal function beside the sweep, starts such a row's call once more: -1. it lists the rows created 3 to 12 minutes ago that are still pending with +1. it lists the rows created 3 to 10 minutes ago that are still pending with no call and that nothing has changed since their insert; 2. for each, it reads back what the service built the call from and builds the same call, on the worker of the .py version the row records; @@ -90,8 +90,10 @@ # is far longer than a submission takes to start and record its call. RESTART_AFTER: Final = timedelta(minutes=3) # A row older than this is left to the stuck-run sweep, which fails it at 15 -# minutes, so a restart has time to record its call before the sweep judges it. -RESTART_UNTIL: Final = timedelta(minutes=12) +# minutes. A row listed just under this age may be marked as late as the +# check's time limit allows, and its call still starts and is recorded +# minutes before the sweep judges it. +RESTART_UNTIL: Final = timedelta(minutes=10) # At most this many rows are listed by one check; the next takes the rest. RESTARTS_PER_CHECK: Final = 20 # How long the check waits for Modal to accept a restarted call. @@ -272,7 +274,7 @@ def __init__( # noqa: PLR0913 # each seam tests replace self._deadline = 0.0 async def restart_unstarted_runs(self) -> RestartSummary: - """Start once more each row created 3 to 12 minutes ago with no call. + """Start once more each row created 3 to 10 minutes ago with no call. Rows the check has no time left for are counted ``deferred``, with no telemetry scope, since the next check lists them again. Raise the @@ -322,6 +324,10 @@ async def _restart(self, run: UnstartedRun) -> tuple[RestartOutcome, str | None] None: RestartOutcome.DEFERRED, }[failed] return outcome, cannot.error_type + # Any other error leaves this row unmarked for the next check, and + # the check goes on to the next row. + except Exception as error: # noqa: BLE001 # one row's unforeseen error holds up no other + return RestartOutcome.DEFERRED, type(error).__name__ if rebuilt is None: return RestartOutcome.LEFT, None return await self._start_marked(run, rebuilt.start, rebuilt.simulation_run_ids) diff --git a/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py b/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py index 1054a9d12..1ed512e20 100644 --- a/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py +++ b/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py @@ -11,7 +11,7 @@ whatever Modal reports. Its result counts runs and household jobs together. The restart check runs every two minutes. It starts once more the call of -each run or household job created 3 to 12 minutes ago that is still pending +each run or household job created 3 to 10 minutes ago that is still pending with no call recorded, as when the simulation service stopped between creating it and starting its call; the simulation service's own restart code (``policyengine_simulation_entry.run_restart_check``) rebuilds the call from diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/run_sweep.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/run_sweep.py index 708118ece..4e6722e04 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/run_sweep.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/run_sweep.py @@ -9,7 +9,7 @@ once the call has ended. A pending run with no call recorded 15 minutes after its creation is failed without asking Modal: usually the simulation service stopped before starting its call, and the restart check, which -starts such a run's call once more 3 to 12 minutes after its creation, could +starts such a run's call once more 3 to 10 minutes after its creation, could not start it either. If a call was started but not recorded, its claim is refused once the run has failed, so it finishes nothing. The sweep also fails a run still unfinished long after it was created or started, longer @@ -51,7 +51,7 @@ # A run must have been unfinished at least this long, counted from its first # claim or, while it is pending, from its creation (a household job's from its # creation), before the sweep judges it; a pending run with no call recorded -# by then is failed. It is longer than the restart check's window (3 to 12 +# by then is failed. It is longer than the restart check's window (3 to 10 # minutes after creation), so a run that check started again is judged by # its call. DEFAULT_MINIMUM_AGE = timedelta(minutes=15) diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep_app.py b/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep_app.py index 2410e7f8b..b482069a0 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep_app.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep_app.py @@ -190,7 +190,7 @@ def recording_sweep(received_store): def test_the_restart_check_runs_every_two_minutes_with_the_worker_database_secret( monkeypatch, ) -> None: - """It restarts runs 3 to 12 minutes old, so each is seen several times.""" + """It restarts runs 3 to 10 minutes old, so each is seen several times.""" module = _load(monkeypatch) options = _functions(module)["restart_unstarted_runs"] From a9fb34e66556b144f79f5c395d47903398d4be2f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 02:59:06 +0400 Subject: [PATCH 15/29] Test review round 3's restart findings (failing) An unexpected rebuild error is logged with its traceback, the image test also requires every module the restart check loads to come from the standard library, a local source or an image distribution, and the persistence window test uses the 3-to-10-minute window. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_run_restart_records.py | 4 +- .../tests/test_run_restarts.py | 17 ++++- .../tests/test_run_restart_image.py | 76 ++++++++++++++++++- 3 files changed, 90 insertions(+), 7 deletions(-) diff --git a/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py b/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py index 0ceac82e5..7573c4f69 100644 --- a/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py +++ b/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py @@ -200,14 +200,14 @@ def test_a_run_created_outside_the_window_is_not_listed(connection) -> None: too_new = list_unstarted_runs( connection, - created_after=created_at - timedelta(minutes=12), + created_after=created_at - timedelta(minutes=10), created_before=created_at - timedelta(minutes=3), limit=LISTING_LIMIT, ) too_old = list_unstarted_runs( connection, created_after=created_at + timedelta(minutes=3), - created_before=created_at + timedelta(minutes=12), + created_before=created_at + timedelta(minutes=10), limit=LISTING_LIMIT, ) diff --git a/projects/policyengine-simulation-entry/tests/test_run_restarts.py b/projects/policyengine-simulation-entry/tests/test_run_restarts.py index 68ecd2eda..bf78b53b8 100644 --- a/projects/policyengine-simulation-entry/tests/test_run_restarts.py +++ b/projects/policyengine-simulation-entry/tests/test_run_restarts.py @@ -958,8 +958,13 @@ def test_a_runs_call_is_rebuilt_off_the_event_loop() -> None: assert not any(manifest.read_on_event_loop) -def test_an_unexpected_rebuild_error_defers_its_run_and_the_check_goes_on() -> None: - """One row the check can't rebuild for an unforeseen reason holds up no other.""" +def test_an_unexpected_rebuild_error_defers_its_run_and_the_check_goes_on( + caplog, +) -> None: + """One row the check can't rebuild for an unforeseen reason holds up no other. + + The error is logged with its traceback, so it can be diagnosed. + """ broken_run_id = UUID(int=9) stored, _call = _submitted_report() rig = _rig( @@ -983,3 +988,11 @@ def test_an_unexpected_rebuild_error_defers_its_run_and_the_check_goes_on() -> N ) assert [run.run_id for run in rig.restarts.marked] == [stored.report_run_id] assert summary == RestartSummary(listed=2, restarted=1, deferred=1) + (logged,) = [ + record + for record in caplog.records + if record.getMessage() == "v2_run_restart_unexpected_error" + ] + assert logged.exc_info is not None + assert isinstance(logged.exc_info[1], RuntimeError) + assert (logged.run_kind, logged.run_id) == ("report_run", str(broken_run_id)) diff --git a/projects/policyengine-simulation-executor/tests/test_run_restart_image.py b/projects/policyengine-simulation-executor/tests/test_run_restart_image.py index 6f36bd130..b078d2b3f 100644 --- a/projects/policyengine-simulation-executor/tests/test_run_restart_image.py +++ b/projects/policyengine-simulation-executor/tests/test_run_restart_image.py @@ -4,20 +4,27 @@ ``modal-simulation-image`` group alone, and Modal adds its own client; the restart check's code comes in as local sources. The scheduled function imports that code only when it runs, so ``modal deploy`` would not notice a -package the image lacks. Here the import runs in a process that refuses -every other installed distribution's modules. +package the image lacks. Here the import runs in a process that refuses the +top-level modules only other installed distributions provide, then every +module it loaded must come from the standard library, a local source, or a +distribution the image installs: a module under a namespace that an image +distribution shares, or from an editable install, is caught there. """ from __future__ import annotations +import json import re +import site import subprocess import sys +import sysconfig import tomllib from importlib import metadata from pathlib import Path PROJECT = Path(__file__).resolve().parents[1] +REPOSITORY = PROJECT.parents[1] IMAGE_GROUP = "modal-simulation-image" # Modal installs its client into every image it builds. MODAL_CLIENT = "modal" @@ -32,6 +39,25 @@ "policyengine-simulation-entry", } ) +# Where each local source's package lies in the repository. +LOCAL_SOURCE_DIRECTORIES = ( + PROJECT / "src" / "policyengine_simulation_executor", + REPOSITORY + / "projects/policyengine-simulation-entry/src/policyengine_simulation_entry", + *( + REPOSITORY / "libs" / library / "src" / library.replace("-", "_") + for library in ( + "policyengine-simulation-observability", + "policyengine-simulation-contract", + "policyengine-stage12-persistence", + "policyengine-simulation-persistence", + ) + ), +) +# The modules the test's own process loads that are not the restart check's: +# its script (also named ``__mp_main__`` once multiprocessing is imported) and +# the virtual environment's start-up module. +NOT_THE_RESTART_CHECK = ("__main__", "__mp_main__", "_virtualenv") # What the scheduled function's body imports. RESTART_MODULES = ( "policyengine_simulation_entry.run_restart_check", @@ -88,6 +114,32 @@ def _top_level_modules(distribution: metadata.Distribution) -> set[str]: return modules +def _image_files() -> set[Path]: + """Return every file the image's distributions install here.""" + allowed = _image_distributions() + return { + Path(str(distribution.locate_file(installed_file))).resolve() + for distribution in metadata.distributions() + if _normalized(distribution.metadata["Name"]) in allowed + for installed_file in distribution.files or () + } + + +def _from_the_image(module_file: Path, image_files: set[Path]) -> bool: + """Return whether a loaded module's file is one the sweep's image has.""" + site_packages = [Path(path).resolve() for path in site.getsitepackages()] + standard_library = [ + Path(sysconfig.get_paths()[name]).resolve() for name in ("stdlib", "platstdlib") + ] + if module_file in image_files: + return True + if any(module_file.is_relative_to(source) for source in LOCAL_SOURCE_DIRECTORIES): + return True + return any( + module_file.is_relative_to(library) for library in standard_library + ) and not any(module_file.is_relative_to(packages) for packages in site_packages) + + def _refused_modules() -> set[str]: """Return the modules only distributions outside the image install.""" allowed = _image_distributions() | LOCAL_SOURCES @@ -109,6 +161,7 @@ def test_the_restart_check_imports_with_only_the_sweep_images_packages( # The executor's own environment holds packages the image doesn't. assert {"jwt", "uvicorn"} <= refused script = tmp_path / "import_restart_check.py" + loaded_list = tmp_path / "loaded_module_files.json" script.write_text( "import importlib, sys\n" f"refused = {sorted(refused)!r}\n" @@ -118,7 +171,12 @@ def test_the_restart_check_imports_with_only_the_sweep_images_packages( " raise ImportError(f'not in the sweep image: {name}')\n" "sys.meta_path.insert(0, Refuse())\n" f"for module in {list(RESTART_MODULES)!r}:\n" - " importlib.import_module(module)\n", + " importlib.import_module(module)\n" + "import json\n" + "loaded = sorted({getattr(module, '__file__', None) or ''\n" + " for name, module in list(sys.modules.items())\n" + f" if name not in {list(NOT_THE_RESTART_CHECK)!r}}} - {{''}})\n" + f"open({str(loaded_list)!r}, 'w').write(json.dumps(loaded))\n", encoding="utf-8", ) @@ -130,3 +188,15 @@ def test_the_restart_check_imports_with_only_the_sweep_images_packages( ) assert completed.returncode == 0, completed.stderr + image_files = _image_files() + loaded = [Path(name).resolve() for name in json.loads(loaded_list.read_text())] + assert any( + module_file.is_relative_to(LOCAL_SOURCE_DIRECTORIES[1]) + for module_file in loaded + ) + outside_the_image = [ + str(module_file) + for module_file in loaded + if not _from_the_image(module_file, image_files) + ] + assert outside_the_image == [] From 80a2e56d03f0a7984455861dcb63eb3a77abd3c3 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 02:59:39 +0400 Subject: [PATCH 16/29] Fix review round 3's restart findings An unexpected error rebuilding a row's call is logged with its traceback, the descriptions of a deferred row name that case, and the telemetry docs count the runs that needed a restart by distinct run ID. Co-Authored-By: Claude Opus 5.5 --- .../stage-12-modal-worker-foundation.md | 8 +++++--- .../policyengine-simulation-entry/README.md | 3 ++- .../run_restart_check.py | 11 +++++++---- .../run_restarts.py | 19 ++++++++++++++----- 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/docs/migration/stage-12-modal-worker-foundation.md b/docs/migration/stage-12-modal-worker-foundation.md index 84311afae..80441db51 100644 --- a/docs/migration/stage-12-modal-worker-foundation.md +++ b/docs/migration/stage-12-modal-worker-foundation.md @@ -752,7 +752,8 @@ worker of the .py version the row records. It then sets the row's records its ID while the row is still pending with none. A row it can't start again (its worker left the manifest, its stored rows changed, or Modal refuses the call) is failed with `run_not_started`, a report run with its -pending simulation runs; one whose rows or manifest it can't read now is left +pending simulation runs; one whose rows or manifest it can't read now, or +whose rebuild raises an unexpected error (logged with its traceback), is left for the next check. After three minutes a check marks no more rows, so it ends within its five-minute timeout without leaving a marked row with no call. The sweep's 15 minutes come after the restart window, so a second miss @@ -767,8 +768,9 @@ runs: it can't rebuild their calls yet. Each check is a `v2_run_restart` operation, with the row's kind, ID, age and outcome, and each outcome is also logged as a `v2_run_restart_outcome` event. A deferred row is taken up again by the next check, so the runs that needed a restart -are the distinct run IDs, or the outcomes other than `deferred`; the events -say how each ended. +are the distinct run IDs (the outcomes other than `deferred` leave out a row +deferred in every check, which the sweep then fails); the events say how +each ended. ## Temporary direct runner endpoint diff --git a/projects/policyengine-simulation-entry/README.md b/projects/policyengine-simulation-entry/README.md index 42dd96279..1beaae9f3 100644 --- a/projects/policyengine-simulation-entry/README.md +++ b/projects/policyengine-simulation-entry/README.md @@ -137,7 +137,8 @@ runs it every two minutes as the Modal worker role. doesn't confirm within ten seconds, is failed with `run_not_started: The run stayed pending with no Modal call recorded for it, and โ€ฆ`; a report run is failed with its simulation runs still pending - with no call. A row whose rows or manifest can't be read now is left + with no call. A row whose rows or manifest can't be read now, or whose + rebuild raises an unexpected error (logged with its traceback), is left unmarked for the next check. - Each check, each row and each outcome are telemetry; see `run_restart_check`'s docstring for the operations and event. diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py index dbdae0e00..93c2b40a6 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py @@ -12,13 +12,16 @@ and the rest) set on it; - each row it takes up is one ``v2_run_restart`` operation, with the row's kind, ID and age. A row left ``deferred`` is taken up again by the next - check, so the operation count metric counts restart attempts; the runs - whose call never started are the distinct ``run_id`` values, or the - operations with any outcome but ``deferred``; + check, so the operation count metric counts restart attempts, and the + runs whose call never started are the distinct ``run_id`` values. (A + count of the outcomes other than ``deferred`` leaves out the rows deferred + in every check, which the sweep then fails.) - each row's outcome is set on its operation (``restart_outcome``, with ``restart_error_type`` when an error ended it) and logged as a ``v2_run_restart_outcome`` event, at WARNING when the row could not be - started again or its rows could not be read. + started again or its rows could not be read; an unexpected error + rebuilding a row's call is also logged with its traceback + (``v2_run_restart_unexpected_error``). This module imports none of the web service, so the sweep's image, which has none of its packages, can run it. diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py index be7b230b4..9f11e8d44 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py @@ -18,7 +18,8 @@ A row whose call can't be built again (its worker left the version manifest, or its stored rows changed), or whose new call Modal refuses, is failed at once with ``run_not_started``, as the sweep would fail it later. A row whose -rows or manifest can't be read now is left unmarked, for the next check. A +rows or manifest can't be read now, or whose rebuild raises an unexpected +error, is left unmarked, for the next check. A call the service started after all claims the row only while the row records no call or that call's own ID, so once the restart records its call, the earlier call's claim is refused; if the earlier call claimed it first, the @@ -32,6 +33,7 @@ from __future__ import annotations import asyncio +import logging import time import uuid from collections import Counter @@ -86,6 +88,8 @@ "RunRestartService", ] +logger = logging.getLogger(__name__) + # A row is restarted once it has stayed pending with no call this long, which # is far longer than a submission takes to start and record its call. RESTART_AFTER: Final = timedelta(minutes=3) @@ -120,8 +124,9 @@ class RestartOutcome(StrEnum): ``left``: the row was gone, changed before the check started a call, or failed meanwhile; also a row whose failure couldn't be written, which the sweep fails later. ``deferred``: its rows or the manifest couldn't be - read, or the check's time ran out, so it is left unmarked for the next - check. ``failed``: it couldn't be started again and was failed with + read, rebuilding its call raised an unexpected error (logged with its + traceback), or the check's time ran out, so it is left unmarked for the + next check. ``failed``: it couldn't be started again and was failed with ``run_not_started``. """ @@ -325,8 +330,12 @@ async def _restart(self, run: UnstartedRun) -> tuple[RestartOutcome, str | None] }[failed] return outcome, cannot.error_type # Any other error leaves this row unmarked for the next check, and - # the check goes on to the next row. - except Exception as error: # noqa: BLE001 # one row's unforeseen error holds up no other + # the check goes on to the next row; its traceback is logged. + except Exception as error: + logger.exception( + "v2_run_restart_unexpected_error", + extra={"run_kind": run.kind.value, "run_id": str(run.run_id)}, + ) return RestartOutcome.DEFERRED, type(error).__name__ if rebuilt is None: return RestartOutcome.LEFT, None From a78cf822ec98618cab1c2e157408bcb6a18f9712 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:13:37 +0400 Subject: [PATCH 17/29] Test that the unexpected-error log carries its run to the runtime (failing) Co-Authored-By: Claude Opus 5.5 --- .../tests/test_run_restarts.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/projects/policyengine-simulation-entry/tests/test_run_restarts.py b/projects/policyengine-simulation-entry/tests/test_run_restarts.py index bf78b53b8..8eaabe323 100644 --- a/projects/policyengine-simulation-entry/tests/test_run_restarts.py +++ b/projects/policyengine-simulation-entry/tests/test_run_restarts.py @@ -995,4 +995,8 @@ def test_an_unexpected_rebuild_error_defers_its_run_and_the_check_goes_on( ] assert logged.exc_info is not None assert isinstance(logged.exc_info[1], RuntimeError) - assert (logged.run_kind, logged.run_id) == ("report_run", str(broken_run_id)) + # The observability runtime's log handler writes only these attributes. + assert logged.policyengine_attributes == { + "run_kind": "report_run", + "run_id": str(broken_run_id), + } From 37980447a4dd26bb9309958aac0f48b0feb94708 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:14:53 +0400 Subject: [PATCH 18/29] Fix review round 4's restart findings The unexpected-error log passes its run kind and ID as policyengine_attributes, which the observability runtime's log handler writes; the image test checks only the modules the import adds; and the descriptions of a deferred row and of the restart counts name every case. Co-Authored-By: Claude Opus 5.5 --- .../stage-12-modal-worker-foundation.md | 8 ++++---- .../run_restart_check.py | 8 +++++--- .../run_restarts.py | 20 ++++++++++++------- .../tests/test_run_restart_image.py | 13 +++++++----- 4 files changed, 30 insertions(+), 19 deletions(-) diff --git a/docs/migration/stage-12-modal-worker-foundation.md b/docs/migration/stage-12-modal-worker-foundation.md index 80441db51..ae02b2132 100644 --- a/docs/migration/stage-12-modal-worker-foundation.md +++ b/docs/migration/stage-12-modal-worker-foundation.md @@ -767,10 +767,10 @@ runs: it can't rebuild their calls yet. Each check is a `v2_run_restart_check` operation and each row it takes up a `v2_run_restart` operation, with the row's kind, ID, age and outcome, and each outcome is also logged as a `v2_run_restart_outcome` event. A deferred -row is taken up again by the next check, so the runs that needed a restart -are the distinct run IDs (the outcomes other than `deferred` leave out a row -deferred in every check, which the sweep then fails); the events say how -each ended. +row is taken up again by the next check, so the distinct run IDs are the +runs that needed a restart that a check took up; a row no check took up, in +a burst larger than the checks in its window had time for, is failed by the +sweep with `run_not_started` instead. The events say how each ended. ## Temporary direct runner endpoint diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py index 93c2b40a6..f4c254d42 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py @@ -13,9 +13,11 @@ - each row it takes up is one ``v2_run_restart`` operation, with the row's kind, ID and age. A row left ``deferred`` is taken up again by the next check, so the operation count metric counts restart attempts, and the - runs whose call never started are the distinct ``run_id`` values. (A - count of the outcomes other than ``deferred`` leaves out the rows deferred - in every check, which the sweep then fails.) + distinct ``run_id`` values are the runs whose call never started that a + check took up. A row no check took up (more rows at once than the checks + in its window had time for) is failed by the sweep with + ``run_not_started`` instead; a count of the outcomes other than + ``deferred`` also leaves out the rows deferred in every check. - each row's outcome is set on its operation (``restart_outcome``, with ``restart_error_type`` when an error ended it) and logged as a ``v2_run_restart_outcome`` event, at WARNING when the row could not be diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py index 9f11e8d44..ac7ad6bc1 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py @@ -19,7 +19,8 @@ or its stored rows changed), or whose new call Modal refuses, is failed at once with ``run_not_started``, as the sweep would fail it later. A row whose rows or manifest can't be read now, or whose rebuild raises an unexpected -error, is left unmarked, for the next check. A +error, is left unmarked, for the next check, as is one the check has no +time left for or whose mark or failure can't be written. A call the service started after all claims the row only while the row records no call or that call's own ID, so once the restart records its call, the earlier call's claim is refused; if the earlier call claimed it first, the @@ -122,11 +123,13 @@ class RestartOutcome(StrEnum): after the check marked it (or the check's own call, accepted after the check stopped waiting for it); the other call then can't claim it. ``left``: the row was gone, changed before the check started a call, or - failed meanwhile; also a row whose failure couldn't be written, which the - sweep fails later. ``deferred``: its rows or the manifest couldn't be - read, rebuilding its call raised an unexpected error (logged with its - traceback), or the check's time ran out, so it is left unmarked for the - next check. ``failed``: it couldn't be started again and was failed with + failed meanwhile; also a marked row whose failure couldn't be written, + which the sweep fails later. ``deferred``: the row is left unmarked for + the next check, because its rows or the manifest couldn't be read, + rebuilding its call raised an unexpected error (logged with its + traceback), the failure of a row that can't be rebuilt couldn't be + written, the mark couldn't be written, or the check's time ran out. + ``failed``: it couldn't be started again and was failed with ``run_not_started``. """ @@ -332,9 +335,12 @@ async def _restart(self, run: UnstartedRun) -> tuple[RestartOutcome, str | None] # Any other error leaves this row unmarked for the next check, and # the check goes on to the next row; its traceback is logged. except Exception as error: + fields = {"run_kind": run.kind.value, "run_id": str(run.run_id)} + # The observability runtime's log handler writes only + # ``policyengine_attributes``, as route_support's own log does. logger.exception( "v2_run_restart_unexpected_error", - extra={"run_kind": run.kind.value, "run_id": str(run.run_id)}, + extra={**fields, "policyengine_attributes": fields}, ) return RestartOutcome.DEFERRED, type(error).__name__ if rebuilt is None: diff --git a/projects/policyengine-simulation-executor/tests/test_run_restart_image.py b/projects/policyengine-simulation-executor/tests/test_run_restart_image.py index b078d2b3f..5273226f3 100644 --- a/projects/policyengine-simulation-executor/tests/test_run_restart_image.py +++ b/projects/policyengine-simulation-executor/tests/test_run_restart_image.py @@ -54,10 +54,9 @@ ) ), ) -# The modules the test's own process loads that are not the restart check's: -# its script (also named ``__mp_main__`` once multiprocessing is imported) and -# the virtual environment's start-up module. -NOT_THE_RESTART_CHECK = ("__main__", "__mp_main__", "_virtualenv") +# The script's own module, which multiprocessing, once imported, also names +# ``__mp_main__``; it is not the restart check's. +NOT_THE_RESTART_CHECK = ("__mp_main__",) # What the scheduled function's body imports. RESTART_MODULES = ( "policyengine_simulation_entry.run_restart_check", @@ -164,6 +163,8 @@ def test_the_restart_check_imports_with_only_the_sweep_images_packages( loaded_list = tmp_path / "loaded_module_files.json" script.write_text( "import importlib, sys\n" + # Only what the import adds is checked, not what started the process. + "loaded_before = set(sys.modules)\n" f"refused = {sorted(refused)!r}\n" "class Refuse:\n" " def find_spec(self, name, path=None, target=None):\n" @@ -175,7 +176,9 @@ def test_the_restart_check_imports_with_only_the_sweep_images_packages( "import json\n" "loaded = sorted({getattr(module, '__file__', None) or ''\n" " for name, module in list(sys.modules.items())\n" - f" if name not in {list(NOT_THE_RESTART_CHECK)!r}}} - {{''}})\n" + " if name not in loaded_before\n" + f" and name not in {list(NOT_THE_RESTART_CHECK)!r}}}\n" + " - {''})\n" f"open({str(loaded_list)!r}, 'w').write(json.dumps(loaded))\n", encoding="utf-8", ) From b00679187db0fce7de2473c8cf22887871f15d6e Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:25:38 +0400 Subject: [PATCH 19/29] Describe every way a restart is left or deferred Wording fixes from review round 5; no behaviour changes. Co-Authored-By: Claude Opus 5.5 --- .../migration/stage-12-modal-worker-foundation.md | 11 ++++++----- projects/policyengine-simulation-entry/README.md | 7 ++++--- .../policyengine_simulation_entry/run_restarts.py | 15 +++++++++------ 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/docs/migration/stage-12-modal-worker-foundation.md b/docs/migration/stage-12-modal-worker-foundation.md index ae02b2132..185e8f44c 100644 --- a/docs/migration/stage-12-modal-worker-foundation.md +++ b/docs/migration/stage-12-modal-worker-foundation.md @@ -752,11 +752,12 @@ worker of the .py version the row records. It then sets the row's records its ID while the row is still pending with none. A row it can't start again (its worker left the manifest, its stored rows changed, or Modal refuses the call) is failed with `run_not_started`, a report run with its -pending simulation runs; one whose rows or manifest it can't read now, or -whose rebuild raises an unexpected error (logged with its traceback), is left -for the next check. After three minutes a check marks no more rows, so it -ends within its five-minute timeout without leaving a marked row with no -call. The sweep's 15 minutes come after the restart window, so a second miss +pending simulation runs. A row is left unmarked for the next check when its +rows or manifest can't be read now, its rebuild raises an unexpected error +(logged with its traceback), its mark can't be written, its call can't be +rebuilt and its failure can't be written, or the check's time is up. After +three minutes a check marks no more rows, so it ends within its five-minute +timeout without leaving a marked row with no call. The sweep's 15 minutes come after the restart window, so a second miss is still failed with `run_not_started`. The restart code is the simulation service's (`policyengine_simulation_entry.run_restarts`, `run_restart_calls` and `run_restart_check`, on `policyengine_simulation_persistence`), which the diff --git a/projects/policyengine-simulation-entry/README.md b/projects/policyengine-simulation-entry/README.md index 1beaae9f3..ab0890cc3 100644 --- a/projects/policyengine-simulation-entry/README.md +++ b/projects/policyengine-simulation-entry/README.md @@ -137,9 +137,10 @@ runs it every two minutes as the Modal worker role. doesn't confirm within ten seconds, is failed with `run_not_started: The run stayed pending with no Modal call recorded for it, and โ€ฆ`; a report run is failed with its simulation runs still pending - with no call. A row whose rows or manifest can't be read now, or whose - rebuild raises an unexpected error (logged with its traceback), is left - unmarked for the next check. + with no call. A row is left unmarked for the next check when its rows or + manifest can't be read now, its rebuild raises an unexpected error + (logged with its traceback), its mark can't be written, or its call can't + be rebuilt and its failure can't be written. - Each check, each row and each outcome are telemetry; see `run_restart_check`'s docstring for the operations and event. diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py index ac7ad6bc1..5c7afb78f 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py @@ -20,7 +20,9 @@ once with ``run_not_started``, as the sweep would fail it later. A row whose rows or manifest can't be read now, or whose rebuild raises an unexpected error, is left unmarked, for the next check, as is one the check has no -time left for or whose mark or failure can't be written. A +time left for, whose mark can't be written, or whose call can't be rebuilt +and whose failure can't be written. A marked row whose failure can't be +written is left to the sweep. A call the service started after all claims the row only while the row records no call or that call's own ID, so once the restart records its call, the earlier call's claim is refused; if the earlier call claimed it first, the @@ -124,11 +126,12 @@ class RestartOutcome(StrEnum): check stopped waiting for it); the other call then can't claim it. ``left``: the row was gone, changed before the check started a call, or failed meanwhile; also a marked row whose failure couldn't be written, - which the sweep fails later. ``deferred``: the row is left unmarked for - the next check, because its rows or the manifest couldn't be read, - rebuilding its call raised an unexpected error (logged with its - traceback), the failure of a row that can't be rebuilt couldn't be - written, the mark couldn't be written, or the check's time ran out. + which the sweep fails later, and a row whose call couldn't be read + after Modal didn't confirm the check's call. ``deferred``: the row is + left unmarked for the next check, because its rows or the manifest + couldn't be read, rebuilding its call raised an unexpected error (logged + with its traceback), the failure of a row that can't be rebuilt couldn't + be written, the mark couldn't be written, or the check's time ran out. ``failed``: it couldn't be started again and was failed with ``run_not_started``. """ From 7f8882fac49d6ae47dd76490eea91e13c1879312 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:35:42 +0400 Subject: [PATCH 20/29] Test restarting a budget window whose years never started (failing) An idle window, with no year under way and its waiting years untouched, is listed for a restart; its earliest waiting year's call is rebuilt from its stored rows, that year is marked, and the window is advanced as the service starts its first years. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_run_restart_records.py | 158 ++++++++++++++ .../tests/test_run_restarts.py | 203 +++++++++++++++++- 2 files changed, 355 insertions(+), 6 deletions(-) diff --git a/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py b/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py index 7573c4f69..e206f86ef 100644 --- a/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py +++ b/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py @@ -25,6 +25,8 @@ insert_reusable_run, ) from policyengine_simulation_persistence import ( + BudgetWindowYear, + NewBudgetWindowReport, NewEconomyReport, NewEconomySimulation, NewHouseholdJob, @@ -37,12 +39,16 @@ SimulationType, StoredRunUnreadableError, UnstartedRun, + WaitingYear, budget_window_runs, + insert_budget_window_report, insert_economy_report, insert_economy_simulation, insert_household_job, + list_idle_budget_windows, list_unstarted_runs, mark_restart, + read_earliest_waiting_year, read_report_run_simulation_run_ids, read_run_call, read_unstarted_report_run, @@ -530,6 +536,146 @@ def test_a_rows_status_and_recorded_call_are_read_back(connection) -> None: assert read_run_call(connection, missing) is None +def _new_window(economy: EconomyCatalog, years: int = 3) -> NewBudgetWindowReport: + return NewBudgetWindowReport( + id=uuid4(), + window_run_id=uuid4(), + country="us", + label="Three-year budget window", + description=None, + tax_benefit_model_id=economy.catalog.model_id, + population=economy.population(year=2026), + versions=VERSIONS, + inputs=REPORT_OPTIONS, + years=tuple( + BudgetWindowYear( + year=2026 + offset, + report_run_id=uuid4(), + baseline=_new_side(), + reform=_new_side(), + ) + for offset in range(years) + ), + ) + + +def _idle_windows( + connection: Connection, *, around: datetime +) -> tuple[UnstartedRun, ...]: + """List the idle windows whose last change is within a second of ``around``.""" + return list_idle_budget_windows( + connection, + idle_after=around - timedelta(seconds=1), + idle_before=around + timedelta(seconds=1), + limit=LISTING_LIMIT, + ) + + +def test_an_idle_window_with_years_waiting_is_listed_for_a_restart( + connection, +) -> None: + """A window whose first years the service never started waits for nothing.""" + economy = insert_economy_catalog(connection) + window = _new_window(economy) + insert_budget_window_report(connection, window) + created_at = _transaction_start(connection) + act_as_the_modal_worker(connection) + + assert _idle_windows(connection, around=created_at) == ( + UnstartedRun( + kind=RestartKind.BUDGET_WINDOW, + run_id=window.window_run_id, + created_at=created_at, + ), + ) + # Its years are never listed as report runs of their own. + assert _listed(connection, around=created_at) == () + + +def test_a_window_with_a_year_under_way_or_finished_is_not_listed( + connection, +) -> None: + economy = insert_economy_catalog(connection) + started = _new_window(economy) + insert_budget_window_report(connection, started) + record_report_run_call( + connection, + report_run_id=started.years[0].report_run_id, + modal_call_id="fc-year", + ) + finished = _new_window(economy) + insert_budget_window_report(connection, finished) + connection.execute( + update(budget_window_runs) + .where(budget_window_runs.c.id == finished.window_run_id) + .values(status="failed", completed_at=func.now()) + ) + created_at = _transaction_start(connection) + act_as_the_modal_worker(connection) + + assert _idle_windows(connection, around=created_at) == () + + +def test_a_window_whose_earliest_waiting_year_was_marked_is_not_listed( + connection, +) -> None: + """A window is restarted at most once for each time it stops.""" + economy = insert_economy_catalog(connection) + window = _new_window(economy) + insert_budget_window_report(connection, window) + created_at = _transaction_start(connection) + act_as_the_modal_worker(connection) + first_year = UnstartedRun( + kind=RestartKind.REPORT_RUN, + run_id=window.years[0].report_run_id, + created_at=created_at, + ) + + assert mark_restart(connection, first_year) is True + marked_at = connection.execute( + select(report_runs.c.updated_at).where(report_runs.c.id == first_year.run_id) + ).scalar_one() + assert _idle_windows(connection, around=created_at) == () + assert _idle_windows(connection, around=marked_at) == () + + +def test_a_window_idle_outside_the_listings_bounds_is_not_listed( + connection, +) -> None: + economy = insert_economy_catalog(connection) + insert_budget_window_report(connection, _new_window(economy)) + created_at = _transaction_start(connection) + act_as_the_modal_worker(connection) + + too_recent = list_idle_budget_windows( + connection, + idle_after=created_at - timedelta(minutes=10), + idle_before=created_at - timedelta(minutes=3), + limit=LISTING_LIMIT, + ) + + assert too_recent == () + + +def test_the_earliest_unmarked_waiting_year_of_a_window_is_read_back( + connection, +) -> None: + economy = insert_economy_catalog(connection) + window = _new_window(economy) + insert_budget_window_report(connection, window) + record_report_run_call( + connection, + report_run_id=window.years[0].report_run_id, + modal_call_id="fc-first-year", + ) + act_as_the_modal_worker(connection) + + assert read_earliest_waiting_year(connection, window.window_run_id) == ( + WaitingYear(report_run_id=window.years[1].report_run_id, year=2027) + ) + assert read_earliest_waiting_year(connection, uuid4()) is None + + @pytest.fixture def savepoint_store(connection) -> RunRestartStore: """Return a store whose every transaction is a savepoint of ``connection``.""" @@ -550,6 +696,8 @@ def test_the_store_lists_marks_and_reads_in_transactions_of_its_own( insert_economy_report(connection, report) simulation = _new_simulation(economy) insert_economy_simulation(connection, simulation) + window = _new_window(economy) + insert_budget_window_report(connection, window) created_at = _transaction_start(connection) act_as_the_modal_worker(connection) @@ -563,10 +711,20 @@ def test_the_store_lists_marks_and_reads_in_transactions_of_its_own( simulation_run = savepoint_store.read_simulation_run(simulation.run_id) marked = savepoint_store.mark_restart(by_kind[RestartKind.REPORT_RUN]) call = savepoint_store.read_call(by_kind[RestartKind.SIMULATION_RUN]) + windows = savepoint_store.list_idle_windows( + idle_after=created_at - timedelta(seconds=1), + idle_before=created_at + timedelta(seconds=1), + limit=LISTING_LIMIT, + ) + waiting = savepoint_store.read_earliest_waiting_year(window.window_run_id) sides = savepoint_store.read_report_run_simulation_run_ids(report.run_id) assert set(by_kind) == {RestartKind.REPORT_RUN, RestartKind.SIMULATION_RUN} assert call == RunCall(status="pending", modal_call_id=None) + assert [run.run_id for run in windows] == [window.window_run_id] + assert waiting == WaitingYear( + report_run_id=window.years[0].report_run_id, year=2026 + ) assert sides == ( report.baseline.simulation_run_id, report.reform.simulation_run_id, diff --git a/projects/policyengine-simulation-entry/tests/test_run_restarts.py b/projects/policyengine-simulation-entry/tests/test_run_restarts.py index 8eaabe323..fd9ca778d 100644 --- a/projects/policyengine-simulation-entry/tests/test_run_restarts.py +++ b/projects/policyengine-simulation-entry/tests/test_run_restarts.py @@ -32,6 +32,7 @@ simulation_request, ) from household_fixtures import manifest_store +from policyengine_simulation_contract.stage12_execution import ReportRunIds from policyengine_simulation_entry.economy_workers import ManifestEconomyWorkers from policyengine_simulation_entry.household_api_models import ( HouseholdCalculationRequest, @@ -69,6 +70,12 @@ StoredRunUnreadableError, StoredSimulationRun, UnstartedRun, + WaitingYear, +) +from policyengine_stage12_persistence import ( + RunStatus, + WindowAdvance, + YearToStart, ) from sqlalchemy.exc import OperationalError @@ -130,10 +137,21 @@ class FakeRestartStore: # Each row's status and call as stored, by run ID; a row not named here is # as the rig's other fakes left it (``holder``). held: dict[UUID, RunCall | None] = field(default_factory=dict) + idle_windows: tuple[UnstartedRun, ...] = () + waiting_years: dict[UUID, WaitingYear] = field(default_factory=dict) holder: Callable[[UnstartedRun], RunCall | None] | None = None - windows: list[tuple[datetime, datetime, int]] = field(default_factory=list) + listings: list[tuple[datetime, datetime, int]] = field(default_factory=list) marked: list[UnstartedRun] = field(default_factory=list) + def list_idle_windows( + self, *, idle_after: datetime, idle_before: datetime, limit: int + ) -> tuple[UnstartedRun, ...]: + self.listings.append((idle_after, idle_before, limit)) + return self.idle_windows + + def read_earliest_waiting_year(self, window_run_id: UUID) -> WaitingYear | None: + return self.waiting_years.get(window_run_id) + def read_call(self, run: UnstartedRun) -> RunCall | None: if run.run_id in self.held: return self.held[run.run_id] @@ -147,7 +165,7 @@ def read_report_run_simulation_run_ids( def list_unstarted( self, *, created_after: datetime, created_before: datetime, limit: int ) -> tuple[UnstartedRun, ...]: - self.windows.append((created_after, created_before, limit)) + self.listings.append((created_after, created_before, limit)) return self.unstarted def mark_restart(self, run: UnstartedRun) -> bool: @@ -261,6 +279,37 @@ async def spawn( return "fc-household-restarted" +@dataclass +class FakeWindows: + """Advances a window as ``BudgetWindowStore`` does: its waiting years in order. + + Each waiting year is offered to the starter; the advance ends with + ``status``, or failed when a start raises. + """ + + waiting: tuple[YearToStart, ...] = () + status: RunStatus = RunStatus.PENDING + advanced: list[UUID] = field(default_factory=list) + started_calls: list[str] = field(default_factory=list) + + def advance_window( + self, window_run_id: UUID, *, now: datetime, start_year: Any = None + ) -> WindowAdvance: + self.advanced.append(window_run_id) + started = [] + for year in self.waiting: + try: + self.started_calls.append(start_year(year)) + except Exception: # noqa: BLE001 # as the store fails a refused year + return WindowAdvance( + status=RunStatus.FAILED, changed=True, started_years=tuple(started) + ) + started.append(year.year) + return WindowAdvance( + status=self.status, changed=False, started_years=tuple(started) + ) + + @dataclass class RecordingTelemetry: """Records each restart scope and outcome the check reports.""" @@ -293,6 +342,7 @@ class RestartRig: """The restart service and every fake it was built with.""" restarts: FakeRestartStore + windows: FakeWindows economy: FakeStore households: FakeHouseholdJobs economy_caller: FakeCaller @@ -313,8 +363,10 @@ def _rig( # noqa: PLR0913 # each fake a test may replace household_caller: FakeHouseholdCaller | None = None, manifest: object | None = None, timer: Callable[[], float] | None = None, + windows: FakeWindows | None = None, ) -> RestartRig: restarts = restarts or FakeRestartStore() + windows = windows or FakeWindows() economy = economy or FakeStore() households = households or FakeHouseholdJobs() economy_caller = economy_caller or FakeCaller() @@ -332,7 +384,12 @@ def holder(run: UnstartedRun) -> RunCall: if restarts.holder is None: restarts.holder = holder restart_service = RunRestartService( - stores=RestartStores(restarts=restarts, economy=economy, households=households), + stores=RestartStores( + restarts=restarts, + economy=economy, + households=households, + windows=windows, + ), workers=RestartWorkers( economy=ManifestEconomyWorkers(store), households=ManifestHouseholdWorkers(store), @@ -347,6 +404,7 @@ def holder(run: UnstartedRun) -> RunCall: ) return RestartRig( restarts=restarts, + windows=windows, economy=economy, households=households, economy_caller=economy_caller, @@ -461,9 +519,11 @@ def test_the_check_lists_runs_created_three_to_ten_minutes_ago() -> None: summary = rig.restart() - assert rig.restarts.windows == [ - (NOW - RESTART_UNTIL, NOW - RESTART_AFTER, RESTARTS_PER_CHECK) - ] + # Rows, then idle budget windows. + assert ( + rig.restarts.listings + == [(NOW - RESTART_UNTIL, NOW - RESTART_AFTER, RESTARTS_PER_CHECK)] * 2 + ) assert ( timedelta(minutes=3), timedelta(minutes=10), @@ -1000,3 +1060,134 @@ def test_an_unexpected_rebuild_error_defers_its_run_and_the_check_goes_on( "run_kind": "report_run", "run_id": str(broken_run_id), } + + +WINDOW_RUN_ID = UUID("00000000-0000-4000-8000-0000000000f1") + + +def _idle_window_rig(**options: Any) -> tuple[RestartRig, StoredReportRun, dict]: + """Return a rig with one idle window whose earliest waiting year was submitted. + + Its second waiting year is the following year, on runs of its own. + """ + stored, (payload, _context) = _submitted_report() + first_year = YearToStart( + year=2026, + run_ids=ReportRunIds( + report_run_id=stored.report_run_id, + baseline_simulation_run_id=stored.baseline.simulation_run_id, + reform_simulation_run_id=stored.reform.simulation_run_id, + ), + ) + second_year = YearToStart( + year=2027, + run_ids=ReportRunIds( + report_run_id=UUID(int=2027), + baseline_simulation_run_id=UUID(int=20271), + reform_simulation_run_id=UUID(int=20272), + ), + ) + restarts = options.pop("restarts", None) or FakeRestartStore( + idle_windows=( + UnstartedRun( + kind=RestartKind.BUDGET_WINDOW, + run_id=WINDOW_RUN_ID, + created_at=CREATED_AT, + ), + ), + report_runs={stored.report_run_id: stored}, + waiting_years={ + WINDOW_RUN_ID: WaitingYear(report_run_id=stored.report_run_id, year=2026) + }, + ) + windows = options.pop("windows", None) or FakeWindows( + waiting=(first_year, second_year) + ) + return _rig(restarts=restarts, windows=windows, **options), stored, payload + + +def test_an_idle_window_is_restarted_from_its_earliest_waiting_year() -> None: + """Its first years start as the service starts them, from its first year's call.""" + rig, stored, payload = _idle_window_rig() + + summary = rig.restart() + + assert rig.windows.advanced == [WINDOW_RUN_ID] + assert [run.run_id for run in rig.restarts.marked] == [stored.report_run_id] + first_call, second_call = rig.economy_caller.report_runs + assert first_call[1] == payload + assert second_call[1]["evaluation_id"] == str(UUID(int=2027)) + assert second_call[1]["baseline"]["year"] == 2027 + assert summary == RestartSummary(listed=1, restarted=1) + assert rig.telemetry.outcomes == [ + (RestartKind.BUDGET_WINDOW, WINDOW_RUN_ID, RestartOutcome.RESTARTED, None) + ] + + +def test_the_check_lists_idle_windows_in_the_same_window_of_time() -> None: + rig, _stored, _payload = _idle_window_rig() + + rig.restart() + + assert ( + rig.restarts.listings + == [(NOW - RESTART_UNTIL, NOW - RESTART_AFTER, RESTARTS_PER_CHECK)] * 2 + ) + + +def test_a_window_whose_restarted_year_modal_refuses_counts_as_failed() -> None: + rig, _stored, _payload = _idle_window_rig( + economy_caller=FakeCaller(failure=ConnectionError("Modal is down")) + ) + + summary = rig.restart() + + assert summary == RestartSummary(listed=1, failed=1) + + +def test_a_window_no_longer_waiting_is_left_unmarked() -> None: + rig, _stored, _payload = _idle_window_rig( + restarts=FakeRestartStore( + idle_windows=( + UnstartedRun( + kind=RestartKind.BUDGET_WINDOW, + run_id=WINDOW_RUN_ID, + created_at=CREATED_AT, + ), + ) + ) + ) + + summary = rig.restart() + + assert rig.restarts.marked == [] + assert rig.windows.advanced == [] + assert summary == RestartSummary(listed=1, left=1) + + +def test_a_window_another_check_marked_first_is_left_without_an_advance() -> None: + rig, stored, _payload = _idle_window_rig() + rig.restarts.marked_elsewhere = True + + summary = rig.restart() + + assert rig.windows.advanced == [] + assert summary == RestartSummary(listed=1, left=1) + assert stored.report_run_id not in [run.run_id for run in rig.restarts.marked] + + +def test_a_window_whose_first_years_call_cannot_be_rebuilt_is_left_to_the_sweep() -> ( + None +): + """It is marked, so no later check tries again; the sweep fails it as stalled.""" + rig, stored, _payload = _idle_window_rig() + rig.restarts.read_failures[stored.report_run_id] = StoredRunUnreadableError( + "report run names no region" + ) + + summary = rig.restart() + + assert [run.run_id for run in rig.restarts.marked] == [stored.report_run_id] + assert rig.windows.advanced == [] + assert summary == RestartSummary(listed=1, left=1) + assert rig.telemetry.outcomes[0][3] == "StoredRunUnreadableError" From e923c881e733bd315f6b7c543042b6a6cdad8772 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:39:42 +0400 Subject: [PATCH 21/29] Restart a budget window whose waiting years never started The restart check also lists the budget windows that stopped 3 to 10 minutes ago, with no year under way and waiting years untouched. It rebuilds the earliest waiting year's call from its stored rows, marks that year so the window restarts once each time it stops, and advances the window with the service's own year starter, as the service starts a new window's first years. A window whose first call can't be rebuilt is marked and left for the sweep to fail as stalled. Co-Authored-By: Claude Opus 5.5 --- .../c1-19-restart-unstarted-runs.added.md | 2 +- .../stage-12-modal-worker-foundation.md | 13 +- .../README.md | 4 + .../__init__.py | 6 + .../run_restart_records.py | 121 ++++++++++- .../run_restart_store.py | 20 ++ .../policyengine-simulation-entry/README.md | 9 +- .../budget_window_submissions.py | 4 +- .../run_restart_calls.py | 43 +++- .../run_restart_check.py | 2 + .../run_restart_outcomes.py | 34 +++ .../run_restart_windows.py | 201 ++++++++++++++++++ .../run_restarts.py | 75 ++++--- .../src/modal/run_sweep_app.py | 4 +- 14 files changed, 489 insertions(+), 49 deletions(-) create mode 100644 projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_outcomes.py create mode 100644 projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_windows.py diff --git a/changelog.d/c1-19-restart-unstarted-runs.added.md b/changelog.d/c1-19-restart-unstarted-runs.added.md index 2ad18f24b..a08af1223 100644 --- a/changelog.d/c1-19-restart-unstarted-runs.added.md +++ b/changelog.d/c1-19-restart-unstarted-runs.added.md @@ -1 +1 @@ -A restart check in the stuck-run sweep's Modal application starts the Modal call of a household job, economy report run or standalone economy simulation run again when the run stayed pending with no call recorded 3 to 10 minutes after its creation, as when the Simulation Entrypoint stopped before starting its call; its telemetry counts every restart and how it ended. The sweep now fails a run still pending with no call at 15 minutes instead of 10. +A restart check in the stuck-run sweep's Modal application starts the Modal call of a household job, economy report run or standalone economy simulation run again when the run stayed pending with no call recorded 3 to 10 minutes after its creation, as when the Simulation Entrypoint stopped before starting its call. It also starts a budget window's waiting years again when the window stopped with none under way, as the Simulation Entrypoint starts a new window's first years. Its telemetry counts every restart and how it ended. The sweep now fails a run still pending with no call at 15 minutes instead of 10. diff --git a/docs/migration/stage-12-modal-worker-foundation.md b/docs/migration/stage-12-modal-worker-foundation.md index b74e7e47e..e4134511e 100644 --- a/docs/migration/stage-12-modal-worker-foundation.md +++ b/docs/migration/stage-12-modal-worker-foundation.md @@ -816,8 +816,17 @@ service's (`policyengine_simulation_entry.run_restarts`, `run_restart_calls` and `run_restart_check`, on `policyengine_simulation_persistence`), which the image adds as source. -It leaves alone a budget window's report runs and household simulations' -runs: it can't rebuild their calls yet. Each check is a +A budget window's years are never restarted one by one. A window stops when +no year is under way and years still wait: the entrypoint stopped before +starting its first years, or a year's coordinator before starting the next. +The check restarts a window that stopped 3 to 10 minutes ago, with its +waiting years untouched, as the entrypoint starts a new window: it rebuilds +the earliest waiting year's call from its stored rows, marks that year, and +advances the window (`BudgetWindowStore.advance_window`) with the +entrypoint's year starter (`YearStarts`), which derives each later year's +call from the first's. A window whose first call can't be rebuilt is marked +and left, and the sweep fails it as stalled. Household simulations' runs +are left alone: the check can't rebuild their calls yet. Each check is a `v2_run_restart_check` operation and each row it takes up a `v2_run_restart` operation, with the row's kind, ID, age and outcome, and each outcome is also logged as a `v2_run_restart_outcome` event. A deferred diff --git a/libs/policyengine-simulation-persistence/README.md b/libs/policyengine-simulation-persistence/README.md index bdfd3794b..bec513aec 100644 --- a/libs/policyengine-simulation-persistence/README.md +++ b/libs/policyengine-simulation-persistence/README.md @@ -137,6 +137,10 @@ the run tables and their inputs, and `UPDATE` of `updated_at`, still equals `created_at`: household jobs, report runs outside a budget window whose baseline is an economy simulation, and runs of economy simulations that no report run names (standalone runs). +- `list_idle_budget_windows` returns the budget windows that stopped: pending + or running, with no year under way and years still waiting untouched, + last changed in a window. `read_earliest_waiting_year` names the year a + restart starts the window from, and marks. - `mark_restart` sets a listed row's `updated_at` to the statement's clock while it is still unstarted, so it is listed and marked only once. - `read_unstarted_report_run` and `read_unstarted_simulation_run` read back diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py index 3192a61e1..13dfcc1c3 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py @@ -103,8 +103,11 @@ StoredRunUnreadableError, StoredSimulationRun, UnstartedRun, + WaitingYear, + list_idle_budget_windows, list_unstarted_runs, mark_restart, + read_earliest_waiting_year, read_report_run_simulation_run_ids, read_run_call, read_unstarted_report_run, @@ -192,6 +195,7 @@ "UnknownPolicyError", "UnknownRegionError", "UnstartedRun", + "WaitingYear", "bounded_transaction", "budget_window_runs", "combine_reforms", @@ -206,6 +210,7 @@ "insert_economy_report", "insert_economy_simulation", "insert_household_job", + "list_idle_budget_windows", "list_unstarted_runs", "mark_restart", "missing_table_privileges", @@ -214,6 +219,7 @@ "read_dataset", "read_dataset_by_id", "read_dynamic_reform", + "read_earliest_waiting_year", "read_economy_report", "read_economy_simulation", "read_household_job", diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py index 0613d1f5c..f939b5aac 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py @@ -11,8 +11,12 @@ (``updated_at`` still equals ``created_at``): household jobs, report runs, and standalone simulation runs, which no report run names. A report run's own simulation runs are its coordinator's to start. Budget-window years - and household simulations are left out: the check can't rebuild their - calls yet. + are left out, and household simulations too: the check can't rebuild + their calls yet. + ``list_idle_budget_windows`` lists, apart, the budget windows that + stopped: no year is under way and years are still waiting, untouched, + since a time in the window. ``read_earliest_waiting_year`` names the year + a restart starts the window from, and marks. 2. ``mark_restart`` sets the row's ``updated_at`` before the check starts its call, so the row is never listed again and is restarted at most once. It uses the statement's clock rather than the transaction's, so the mark @@ -35,7 +39,7 @@ from enum import StrEnum from typing import TYPE_CHECKING, Any, Final -from sqlalchemy import exists, func, or_, select, update +from sqlalchemy import and_, exists, func, or_, select, update from policyengine_simulation_persistence.economy_run_records import ( RunVersions, @@ -52,8 +56,10 @@ tax_benefit_models, ) from policyengine_simulation_persistence.tables import ( + ReportRunStatus, SimulationStatus, SimulationType, + budget_window_runs, household_jobs, report_runs, reports, @@ -79,8 +85,12 @@ class RestartKind(StrEnum): HOUSEHOLD_JOB = "household_job" REPORT_RUN = "report_run" SIMULATION_RUN = "simulation_run" + # A budget window none of whose years is under way: a restart starts its + # earliest waiting years, as the service starts a new window's. + BUDGET_WINDOW = "budget_window" +# The table of each kind restarted one row at a time. RESTART_TABLES: Final[dict[RestartKind, Table]] = { RestartKind.HOUSEHOLD_JOB: household_jobs, RestartKind.REPORT_RUN: report_runs, @@ -90,7 +100,11 @@ class RestartKind(StrEnum): @dataclass(frozen=True) class UnstartedRun: - """A row still pending with no Modal call, and when it was created.""" + """A row still pending with no Modal call, and since when it has waited. + + ``created_at`` is a row's creation, and a budget window's last change: + the latest change of the window or any of its years. + """ kind: RestartKind run_id: UUID @@ -101,6 +115,14 @@ class StoredRunUnreadableError(LookupError): """A run's stored rows no longer describe one calculation to start.""" +@dataclass(frozen=True) +class WaitingYear: + """A budget window's year still pending with no call, never changed since.""" + + report_run_id: UUID + year: int + + @dataclass(frozen=True) class RunCall: """A row's status and the Modal call it records, if any.""" @@ -247,6 +269,97 @@ def list_unstarted_runs( return tuple(listed[:limit]) +_ACTIVE_WINDOW_STATUSES: Final = ( + ReportRunStatus.PENDING.value, + ReportRunStatus.RUNNING.value, +) + + +def _year_of_window() -> ColumnElement[bool]: + return report_runs.c.budget_window_run_id == budget_window_runs.c.id + + +def _year_waiting() -> ColumnElement[bool]: + return and_(report_runs.c.status == _PENDING, report_runs.c.modal_call_id.is_(None)) + + +def _year_under_way() -> ColumnElement[bool]: + return or_( + report_runs.c.status == ReportRunStatus.RUNNING.value, + and_( + report_runs.c.status == _PENDING, + report_runs.c.modal_call_id.is_not(None), + ), + ) + + +def list_idle_budget_windows( + connection: Connection, + *, + idle_after: datetime, + idle_before: datetime, + limit: int, +) -> tuple[UnstartedRun, ...]: + """Return the stopped budget windows last changed in the window, oldest first. + + A window has stopped when it is pending or running, none of its years is + under way (running, or pending with a call), and years are still waiting + (pending with no call) that nothing has changed since their insert: the + service stopped before starting its first years, or a year's coordinator + before starting the next. Its last change is the latest ``updated_at`` of + the window and its years. A window whose waiting year was marked by a + restart is not listed again until that year has started. + """ + last_year_change = ( + select(func.max(report_runs.c.updated_at)) + .where(_year_of_window()) + .scalar_subquery() + ) + last_change = func.greatest( + budget_window_runs.c.updated_at, + func.coalesce(last_year_change, budget_window_runs.c.updated_at), + ) + statement = ( + select(budget_window_runs.c.id, last_change.label("last_change")) + .where( + budget_window_runs.c.status.in_(_ACTIVE_WINDOW_STATUSES), + ~exists().where(_year_of_window(), _year_under_way()), + exists().where(_year_of_window(), _year_waiting()), + ~exists().where( + _year_of_window(), + _year_waiting(), + report_runs.c.updated_at != report_runs.c.created_at, + ), + last_change > idle_after, + last_change <= idle_before, + ) + .order_by(last_change, budget_window_runs.c.id) + .limit(limit) + ) + return tuple( + UnstartedRun( + kind=RestartKind.BUDGET_WINDOW, run_id=row.id, created_at=row.last_change + ) + for row in connection.execute(statement) + ) + + +def read_earliest_waiting_year( + connection: Connection, window_run_id: UUID +) -> WaitingYear | None: + """Return the window's earliest year still waiting, never changed, or None.""" + row = connection.execute( + select(report_runs.c.id, report_runs.c.year) + .where( + report_runs.c.budget_window_run_id == window_run_id, + *_unstarted(report_runs), + ) + .order_by(report_runs.c.year) + .limit(1) + ).one_or_none() + return None if row is None else WaitingYear(report_run_id=row.id, year=row.year) + + def mark_restart(connection: Connection, run: UnstartedRun) -> bool: """Mark the row as restarted while it is still unstarted; return whether it was. diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_store.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_store.py index ce1733723..094dd116e 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_store.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_store.py @@ -12,8 +12,10 @@ from policyengine_simulation_persistence.engine import bounded_transaction from policyengine_simulation_persistence.run_restart_records import ( + list_idle_budget_windows, list_unstarted_runs, mark_restart, + read_earliest_waiting_year, read_report_run_simulation_run_ids, read_run_call, read_unstarted_report_run, @@ -33,6 +35,7 @@ StoredReportRun, StoredSimulationRun, UnstartedRun, + WaitingYear, ) type TransactionFactory = Callable[[], AbstractContextManager[Connection]] @@ -98,3 +101,20 @@ def read_report_run_simulation_run_ids( """Return the simulation runs the report run names, baseline first.""" with self._transaction() as connection: return read_report_run_simulation_run_ids(connection, report_run_id) + + def list_idle_windows( + self, *, idle_after: datetime, idle_before: datetime, limit: int + ) -> tuple[UnstartedRun, ...]: + """Return the stopped budget windows last changed in the window.""" + with self._transaction() as connection: + return list_idle_budget_windows( + connection, + idle_after=idle_after, + idle_before=idle_before, + limit=limit, + ) + + def read_earliest_waiting_year(self, window_run_id: UUID) -> WaitingYear | None: + """Return the window's earliest year still waiting, never changed, or None.""" + with self._transaction() as connection: + return read_earliest_waiting_year(connection, window_run_id) diff --git a/projects/policyengine-simulation-entry/README.md b/projects/policyengine-simulation-entry/README.md index 7426e83f7..0b4058349 100644 --- a/projects/policyengine-simulation-entry/README.md +++ b/projects/policyengine-simulation-entry/README.md @@ -119,9 +119,12 @@ runs it every two minutes as the Modal worker role. - It takes the rows created 3 to 10 minutes ago that are still `pending` with no call and that nothing has changed since their insert, at most 20 - a check, oldest first. A budget window's report runs and household - simulations' runs are left out. After three minutes a check marks no more - rows and leaves the rest for the next. + a check, oldest first, with the budget windows that stopped 3 to 10 + minutes ago (`run_restart_windows`). A window is restarted as + `budget_window_submissions` starts a new one: from its earliest waiting + year's rebuilt call, with `YearStarts`. Household simulations' runs are + left out. After three minutes a check marks no more rows and leaves the + rest for the next. - It rebuilds each call with this service's own builders (`report_run_calculation`, `simulation_run_calculation`, `prepare_household_job`) from the stored rows, on the manifest's worker diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/budget_window_submissions.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/budget_window_submissions.py index 748905f3c..df927df32 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/budget_window_submissions.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/budget_window_submissions.py @@ -253,7 +253,7 @@ async def _start_first_years( started claims itself, while the stuck-run sweep fails a window no year advances. """ - start_year = _YearStarts( + start_year = YearStarts( worker=worker, caller=self._caller, first_year=report.first_year.year, @@ -295,7 +295,7 @@ async def _start_first_years( return ReportRunStatus(advance.status.value) -class _YearStarts: +class YearStarts: """Start a window year's coordinator from the advance's thread; return its call. The first year's call takes the input built for it; each later year's is diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py index 40d089da9..068cceee3 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py @@ -73,8 +73,12 @@ StoredReportRun, StoredSimulationRun, UnstartedRun, + WaitingYear, ) + from policyengine_simulation_entry.economy_calculation_inputs import ( + RunCalculation, + ) from policyengine_simulation_entry.economy_workers import ( EconomyRunCaller, EconomyWorker, @@ -152,6 +156,16 @@ def read_call(self, run: UnstartedRun) -> RunCall | None: """Return the row's status and recorded call, or None when it is gone.""" ... + def list_idle_windows( + self, *, idle_after: datetime, idle_before: datetime, limit: int + ) -> tuple[UnstartedRun, ...]: + """Return the stopped budget windows last changed in the window.""" + ... + + def read_earliest_waiting_year(self, window_run_id: UUID) -> WaitingYear | None: + """Return the window's earliest year still waiting, never changed.""" + ... + class HouseholdJobRestarts(Protocol): """The household job reads and writes a restart uses (``HouseholdJobStore``).""" @@ -443,7 +457,30 @@ def _origin(self, created_at: datetime) -> CallOrigin: created_at=created_at, ) - def _report_call(self, stored: StoredReportRun) -> CallStart: + def report_calculation( + self, report_run_id: UUID + ) -> tuple[EconomyWorker, RunCalculation] | None: + """Return a report run's worker and coordinator call, or None when gone. + + A budget window is restarted from its earliest waiting year's call, + and the service's year starter derives each later year's from it. + Raise DeferredRestartError or CallNotRebuiltError as ``build`` does, + the latter naming no simulation runs: a window's years are failed by + the window, never by the restart. + """ + try: + stored = self._restarts.read_report_run(report_run_id) + return None if stored is None else self._report_calculation(stored) + except (SQLAlchemyError, VersionManifestUnavailableError) as error: + raise DeferredRestartError(type(error).__name__) from error + except _REBUILD_ERRORS as error: + raise CallNotRebuiltError( + _rebuild_reason(error), error_type=type(error).__name__ + ) from error + + def _report_calculation( + self, stored: StoredReportRun + ) -> tuple[EconomyWorker, RunCalculation]: worker = self._economy_worker(stored.country, stored.versions) calculation = report_run_calculation( worker, @@ -457,6 +494,10 @@ def _report_call(self, stored: StoredReportRun) -> CallStart: baseline_policy=stored.baseline.reform, reform_policy=stored.reform.reform, ) + return worker, calculation + + def _report_call(self, stored: StoredReportRun) -> CallStart: + worker, calculation = self._report_calculation(stored) caller = self._callers.economy def start(observability_context: dict[str, Any] | None) -> Awaitable[str]: diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py index f4c254d42..981c46e02 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py @@ -46,6 +46,7 @@ RunRestartStore, create_simulation_engine, ) +from policyengine_stage12_persistence import BudgetWindowStore from policyengine_simulation_entry.economy_workers import ( ManifestEconomyWorkers, @@ -215,6 +216,7 @@ def scheduled_restart_service( households=HouseholdJobStore.from_engine( engine, statement_timeout_milliseconds=timeout ), + windows=BudgetWindowStore(engine=engine), ), workers=RestartWorkers( economy=ManifestEconomyWorkers.from_modal( diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_outcomes.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_outcomes.py new file mode 100644 index 000000000..461eb6240 --- /dev/null +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_outcomes.py @@ -0,0 +1,34 @@ +"""How the restart check ended for each row it took up.""" + +from __future__ import annotations + +from enum import StrEnum + + +class RestartOutcome(StrEnum): + """How the check ended for one listed row. + + ``restarted``: its new call started, and the row records that call. + ``call_not_recorded``: its new call started, but the check couldn't + record or read the call's ID; the call claims the row itself. + ``started_meanwhile``: the row records another call, which claimed it + after the check marked it (or the check's own call, accepted after the + check stopped waiting for it); the other call then can't claim it. + ``left``: the row was gone, changed before the check started a call, or + failed meanwhile; also a marked row whose failure couldn't be written, + which the sweep fails later, and a row whose call couldn't be read + after Modal didn't confirm the check's call. ``deferred``: the row is + left unmarked for the next check, because its rows or the manifest + couldn't be read, rebuilding its call raised an unexpected error (logged + with its traceback), the failure of a row that can't be rebuilt couldn't + be written, the mark couldn't be written, or the check's time ran out. + ``failed``: it couldn't be started again and was failed with + ``run_not_started``. + """ + + RESTARTED = "restarted" + CALL_NOT_RECORDED = "call_not_recorded" + STARTED_MEANWHILE = "started_meanwhile" + LEFT = "left" + DEFERRED = "deferred" + FAILED = "failed" diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_windows.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_windows.py new file mode 100644 index 000000000..438371045 --- /dev/null +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_windows.py @@ -0,0 +1,201 @@ +"""Start a budget window again when its waiting years never started. + +The simulation service creates a budget window with every year pending with +no Modal call, then starts its first years; each year's coordinator starts +the next when its year ends. When the service stops before starting the +first years, or a coordinator before starting the next, the window stops: +no year is under way, and years wait for a start nothing will make. The +stuck-run sweep fails such a window as stalled once nothing has changed for +its minimum age. + +The restart check starts a stopped window again, as the service starts a +new window's first years. ``WindowRestarter`` rebuilds the earliest waiting +year's coordinator call from its stored rows, marks that year, so the window +is restarted once each time it stops, and advances the window +(``BudgetWindowStore.advance_window``) with the service's own year starter +(``YearStarts``). That starter derives each later year's call from the +first's, and shares one wait for Modal among them; under the window's row +lock, the earliest years are started, at most five under way, and their +calls recorded. A year Modal refuses fails the window, as it would at the +service's start. A window whose first call can't be rebuilt is marked and +left, and the sweep fails it as stalled. +""" + +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING + +from policyengine_simulation_persistence import RestartKind, UnstartedRun +from policyengine_stage12_persistence import RunStatus +from sqlalchemy.exc import SQLAlchemyError + +from policyengine_simulation_entry.budget_window_submissions import YearStarts +from policyengine_simulation_entry.run_restart_calls import ( + CallNotRebuiltError, + DeferredRestartError, +) +from policyengine_simulation_entry.run_restart_outcomes import RestartOutcome + +if TYPE_CHECKING: + from collections.abc import Callable + from datetime import datetime + + from policyengine_simulation_persistence import WaitingYear + from policyengine_stage12_persistence import WindowAdvance + + from policyengine_simulation_entry.budget_window_submissions import ( + BudgetWindowAdvancer, + ) + from policyengine_simulation_entry.economy_calculation_inputs import ( + RunCalculation, + ) + from policyengine_simulation_entry.economy_workers import ( + EconomyRunCaller, + EconomyWorker, + ) + from policyengine_simulation_entry.run_restart_calls import ( + RestartCallBuilder, + UnstartedRunSource, + ) + from policyengine_simulation_entry.run_restarts import RestartTelemetry + + +class _RestartEnded(Exception): # noqa: N818 # it names an outcome, not a fault + """The window's restart ended early, with this outcome.""" + + def __init__(self, outcome: RestartOutcome, error_type: str | None = None) -> None: + """Keep the outcome and the error type that ended it, if any.""" + super().__init__(outcome.value) + self.outcome = outcome + self.error_type = error_type + + +class WindowRestarter: + """Start a stopped budget window's earliest waiting years again.""" + + def __init__( # noqa: PLR0913 # the service's seams this restart shares + self, + *, + restarts: UnstartedRunSource, + windows: BudgetWindowAdvancer, + calls: RestartCallBuilder, + caller: EconomyRunCaller, + telemetry: RestartTelemetry, + clock: Callable[[], datetime], + out_of_time: Callable[[], bool], + call_start_timeout_seconds: float, + ) -> None: + """Use the restart service's own stores, builder, caller and clocks.""" + self._restarts = restarts + self._windows = windows + self._calls = calls + self._caller = caller + self._telemetry = telemetry + self._clock = clock + self._out_of_time = out_of_time + self._call_start_timeout_seconds = call_start_timeout_seconds + + async def restart(self, window: UnstartedRun) -> tuple[RestartOutcome, str | None]: + """Start the window's earliest waiting years; return how it ended.""" + try: + waiting, built = await self._first_year_call(window) + worker, calculation = await self._mark(window, waiting, built) + advance = await self._advance(window, waiting, worker, calculation) + except _RestartEnded as ended: + return ended.outcome, ended.error_type + if advance.status is RunStatus.FAILED: + return RestartOutcome.FAILED, None + if advance.started_years: + return RestartOutcome.RESTARTED, None + return RestartOutcome.LEFT, None + + async def _first_year_call( + self, window: UnstartedRun + ) -> tuple[WaitingYear, tuple[EconomyWorker, RunCalculation] | CallNotRebuiltError]: + """Return the earliest waiting year and its rebuilt call, or why there's none. + + A call that can't be rebuilt is returned, not raised, so the year is + still marked. + """ + try: + waiting = await asyncio.to_thread( + self._restarts.read_earliest_waiting_year, window.run_id + ) + except SQLAlchemyError as error: + raise _RestartEnded(RestartOutcome.DEFERRED, type(error).__name__) from None + if waiting is None: + raise _RestartEnded(RestartOutcome.LEFT) + try: + built = await asyncio.to_thread( + self._calls.report_calculation, waiting.report_run_id + ) + except DeferredRestartError as deferred: + raise _RestartEnded(RestartOutcome.DEFERRED, str(deferred)) from None + except CallNotRebuiltError as cannot: + return waiting, cannot + if built is None: + raise _RestartEnded(RestartOutcome.LEFT) + return waiting, built + + async def _mark( + self, + window: UnstartedRun, + waiting: WaitingYear, + built: tuple[EconomyWorker, RunCalculation] | CallNotRebuiltError, + ) -> tuple[EconomyWorker, RunCalculation]: + """Mark the earliest waiting year, so the window restarts once per stop. + + Return the year's rebuilt call. Raise _RestartEnded when time is up, + the year changed meanwhile, the mark can't be written, or the year's + call can't be rebuilt (once marked). + """ + if self._out_of_time(): + raise _RestartEnded(RestartOutcome.DEFERRED) + first_year = UnstartedRun( + kind=RestartKind.REPORT_RUN, + run_id=waiting.report_run_id, + created_at=window.created_at, + ) + try: + marked = await asyncio.to_thread(self._restarts.mark_restart, first_year) + except SQLAlchemyError as error: + raise _RestartEnded(RestartOutcome.DEFERRED, type(error).__name__) from None + if not marked: + raise _RestartEnded(RestartOutcome.LEFT) + if isinstance(built, CallNotRebuiltError): + raise _RestartEnded(RestartOutcome.LEFT, built.error_type) + return built + + async def _advance( + self, + window: UnstartedRun, + waiting: WaitingYear, + worker: EconomyWorker, + calculation: RunCalculation, + ) -> WindowAdvance: + """Advance the window with the service's year starter, from the first year.""" + start_year = YearStarts( + worker=worker, + caller=self._caller, + first_year=waiting.year, + first_calculation=calculation, + observability_context=self._telemetry.capture_context(), + loop=asyncio.get_running_loop(), + timeout_seconds=self._call_start_timeout_seconds, + clock=self._clock, + ) + try: + return await asyncio.to_thread( + self._windows.advance_window, + window.run_id, + now=self._clock(), + start_year=start_year, + ) + except SQLAlchemyError as error: + # A year whose call Modal accepted claims itself. + raise _RestartEnded( + RestartOutcome.CALL_NOT_RECORDED, type(error).__name__ + ) from None + except LookupError: + raise _RestartEnded(RestartOutcome.LEFT) from None diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py index 5c7afb78f..6b051e393 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py @@ -9,7 +9,8 @@ beside the sweep, starts such a row's call once more: 1. it lists the rows created 3 to 10 minutes ago that are still pending with - no call and that nothing has changed since their insert; + no call and that nothing has changed since their insert, and the budget + windows that stopped 3 to 10 minutes ago (``run_restart_windows``); 2. for each, it reads back what the service built the call from and builds the same call, on the worker of the .py version the row records; 3. it marks the row, so it is never restarted twice, starts the call, and @@ -43,7 +44,8 @@ from contextlib import nullcontext from dataclasses import dataclass from datetime import UTC, datetime, timedelta -from enum import StrEnum +from heapq import merge +from itertools import islice from typing import TYPE_CHECKING, Final, Protocol from policyengine_simulation_persistence import RestartKind @@ -59,6 +61,8 @@ RestartWorkers, UnstartedRunSource, ) +from policyengine_simulation_entry.run_restart_outcomes import RestartOutcome +from policyengine_simulation_entry.run_restart_windows import WindowRestarter if TYPE_CHECKING: from collections.abc import Callable, Sequence @@ -71,6 +75,9 @@ UnstartedRun, ) + from policyengine_simulation_entry.budget_window_submissions import ( + BudgetWindowAdvancer, + ) from policyengine_simulation_entry.run_restart_calls import CallStart __all__ = [ @@ -115,35 +122,6 @@ ) -class RestartOutcome(StrEnum): - """How the check ended for one listed row. - - ``restarted``: its new call started, and the row records that call. - ``call_not_recorded``: its new call started, but the check couldn't - record or read the call's ID; the call claims the row itself. - ``started_meanwhile``: the row records another call, which claimed it - after the check marked it (or the check's own call, accepted after the - check stopped waiting for it); the other call then can't claim it. - ``left``: the row was gone, changed before the check started a call, or - failed meanwhile; also a marked row whose failure couldn't be written, - which the sweep fails later, and a row whose call couldn't be read - after Modal didn't confirm the check's call. ``deferred``: the row is - left unmarked for the next check, because its rows or the manifest - couldn't be read, rebuilding its call raised an unexpected error (logged - with its traceback), the failure of a row that can't be rebuilt couldn't - be written, the mark couldn't be written, or the check's time ran out. - ``failed``: it couldn't be started again and was failed with - ``run_not_started``. - """ - - RESTARTED = "restarted" - CALL_NOT_RECORDED = "call_not_recorded" - STARTED_MEANWHILE = "started_meanwhile" - LEFT = "left" - DEFERRED = "deferred" - FAILED = "failed" - - @dataclass(frozen=True) class RestartSummary: """How many rows one check listed, and how each ended (``RestartOutcome``).""" @@ -245,6 +223,7 @@ class RestartStores: restarts: UnstartedRunSource economy: EconomyRunWrites households: HouseholdJobRestarts + windows: BudgetWindowAdvancer class RunRestartService: @@ -283,21 +262,45 @@ def __init__( # noqa: PLR0913 # each seam tests replace self._timer = timer self._call_start_timeout_seconds = call_start_timeout_seconds self._deadline = 0.0 + self._windows = WindowRestarter( + restarts=stores.restarts, + windows=stores.windows, + calls=self._calls, + caller=callers.economy, + telemetry=self._telemetry, + clock=clock, + out_of_time=self._out_of_time, + call_start_timeout_seconds=call_start_timeout_seconds, + ) async def restart_unstarted_runs(self) -> RestartSummary: """Start once more each row created 3 to 10 minutes ago with no call. - Rows the check has no time left for are counted ``deferred``, with no - telemetry scope, since the next check lists them again. Raise the - store's error when the rows can't be listed. + So too each budget window that stopped 3 to 10 minutes ago with years + still waiting. Rows the check has no time left for are counted + ``deferred``, with no telemetry scope, since the next check lists + them again. Raise the store's error when the rows can't be listed. """ self._deadline = self._timer() + CHECK_TIME_LIMIT.total_seconds() now = self._clock() - runs = self._stores.restarts.list_unstarted( + restarts = self._stores.restarts + rows = restarts.list_unstarted( created_after=now - RESTART_UNTIL, created_before=now - RESTART_AFTER, limit=RESTARTS_PER_CHECK, ) + windows = restarts.list_idle_windows( + idle_after=now - RESTART_UNTIL, + idle_before=now - RESTART_AFTER, + limit=RESTARTS_PER_CHECK, + ) + # Each listing is oldest first; together, oldest first too. + runs = list( + islice( + merge(rows, windows, key=lambda run: run.created_at), + RESTARTS_PER_CHECK, + ) + ) outcomes: Counter[RestartOutcome] = Counter() for position, run in enumerate(runs): if self._out_of_time(): @@ -318,6 +321,8 @@ def _out_of_time(self) -> bool: async def _restart(self, run: UnstartedRun) -> tuple[RestartOutcome, str | None]: """Start the row's call once more; return how it ended and any error type.""" + if run.kind is RestartKind.BUDGET_WINDOW: + return await self._windows.restart(run) try: rebuilt = await asyncio.to_thread(self._calls.build, run) except DeferredRestartError as deferred: diff --git a/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py b/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py index b0e346f63..963584567 100644 --- a/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py +++ b/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py @@ -20,7 +20,9 @@ creating it and starting its call; the simulation service's own restart code (``policyengine_simulation_entry.run_restart_check``) rebuilds the call from the stored rows and records it. A run it can't start again is failed with -``run_not_started``, as the sweep would fail it at 15 minutes. Its telemetry +``run_not_started``, as the sweep would fail it at 15 minutes. A budget +window that stopped 3 to 10 minutes ago, with no year under way and years +still waiting, has its earliest waiting years started again. Its telemetry counts each restart and how it ended. Both connect as the Modal worker role, from the same Modal secret the v2 From 3d2c6f351e04c0b5d9aafe74238ef213bfaf4150 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:34:19 +0400 Subject: [PATCH 22/29] Test review round 6's budget-window restart findings (failing) A window is marked only just before its first start, so a database error before any start leaves it for the next check; an unexpected error defers it and the check goes on; a window whose first call can't be built again fails with the reason. The window tests move to a module of their own, and idle windows become a type of their own. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_run_restart_records.py | 9 +- .../tests/test_budget_window_store.py | 70 +++ .../tests/test_run_restart_check.py | 21 + .../tests/test_run_restart_windows.py | 542 ++++++++++++++++++ .../tests/test_run_restarts.py | 279 +++------ 5 files changed, 702 insertions(+), 219 deletions(-) create mode 100644 projects/policyengine-simulation-entry/tests/test_run_restart_windows.py diff --git a/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py b/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py index e206f86ef..a43c3192d 100644 --- a/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py +++ b/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py @@ -26,6 +26,7 @@ ) from policyengine_simulation_persistence import ( BudgetWindowYear, + IdleBudgetWindow, NewBudgetWindowReport, NewEconomyReport, NewEconomySimulation, @@ -561,7 +562,7 @@ def _new_window(economy: EconomyCatalog, years: int = 3) -> NewBudgetWindowRepor def _idle_windows( connection: Connection, *, around: datetime -) -> tuple[UnstartedRun, ...]: +) -> tuple[IdleBudgetWindow, ...]: """List the idle windows whose last change is within a second of ``around``.""" return list_idle_budget_windows( connection, @@ -582,11 +583,7 @@ def test_an_idle_window_with_years_waiting_is_listed_for_a_restart( act_as_the_modal_worker(connection) assert _idle_windows(connection, around=created_at) == ( - UnstartedRun( - kind=RestartKind.BUDGET_WINDOW, - run_id=window.window_run_id, - created_at=created_at, - ), + IdleBudgetWindow(run_id=window.window_run_id, last_changed_at=created_at), ) # Its years are never listed as report runs of their own. assert _listed(connection, around=created_at) == () diff --git a/libs/policyengine-stage12-persistence/tests/test_budget_window_store.py b/libs/policyengine-stage12-persistence/tests/test_budget_window_store.py index d7b6b8bb7..5f80c543b 100644 --- a/libs/policyengine-stage12-persistence/tests/test_budget_window_store.py +++ b/libs/policyengine-stage12-persistence/tests/test_budget_window_store.py @@ -32,11 +32,13 @@ RunStatus, WindowAdvance, YearStartDeferred, + YearStartRefused, YearToStart, budget_window_store, ) from report_run_database import NOW, SEED_URL_ENV, worker_database_url from sqlalchemy import Engine, create_engine, event +from sqlalchemy.exc import OperationalError # The simulation service's role, which the Public API's manifest grants. SERVICE_ROLE = "policyengine_v2_simulation_service" @@ -324,6 +326,74 @@ def test_a_refused_start_names_its_cause_on_its_own_simulations( assert "Modal refused the call" not in caplog.text +def test_a_start_refused_with_its_own_message_records_that_message( + store, windows, window +) -> None: + """A starter that can't build a year's call says why, rather than Modal.""" + message = ( + "run_not_started: The run stayed pending with no Modal call recorded " + "for it, and its call could not be built again: the region is gone." + ) + + def refuse(year_to_start: YearToStart) -> str: + raise YearStartRefused(message) + + advance = store.advance_window(window.window_run_id, now=LATER, start_year=refuse) + + assert advance == WindowAdvance(status=RunStatus.FAILED, changed=True) + rows = windows.year_rows(window) + assert rows[2026]["status"] == "failed" + assert rows[2026]["error_message"] == message + for simulation_run_id in ( + window.year(2026).baseline_simulation_run_id, + window.year(2026).reform_simulation_run_id, + ): + row = windows.seeder.row("simulation_runs", simulation_run_id) + assert row["error_message"] == message + assert rows[2027]["error_message"].startswith("budget_window_failed: ") + window_row = windows.window_row(window) + assert window_row["status"] == "failed" + assert window_row["error_message"] == ( + "budget_window_year_failed: The budget window's 2026 report run failed." + ) + + +def test_a_statement_timeout_stops_an_advance_waiting_for_the_windows_lock( + store, window +) -> None: + """The restart check's store gives up on a window another advance holds.""" + holding = threading.Event() + release = threading.Event() + + def hold_the_lock(year_to_start: YearToStart) -> str: + holding.set() + release.wait(timeout=10) + raise YearStartDeferred + + bounded = BudgetWindowStore( + worker_database_url(), statement_timeout_milliseconds=200 + ) + with ThreadPoolExecutor(max_workers=1) as pool: + holder = pool.submit( + store.advance_window, + window.window_run_id, + now=LATER, + start_year=hold_the_lock, + ) + try: + assert holding.wait(timeout=10) + waited_from = time.monotonic() + with pytest.raises(OperationalError, match="statement timeout"): + bounded.advance_window( + window.window_run_id, now=LATER, start_year=RecordingStarter() + ) + assert time.monotonic() - waited_from < 5 + finally: + release.set() + holder.result() + bounded.engine.dispose() + + def test_the_service_role_starts_and_fails_a_window(windows, window) -> None: # The simulation service starts a window's first years with the same # store, as its own role, and fails the window when Modal refuses one. diff --git a/projects/policyengine-simulation-entry/tests/test_run_restart_check.py b/projects/policyengine-simulation-entry/tests/test_run_restart_check.py index 4b778a3e1..5ac130102 100644 --- a/projects/policyengine-simulation-entry/tests/test_run_restart_check.py +++ b/projects/policyengine-simulation-entry/tests/test_run_restart_check.py @@ -10,6 +10,7 @@ import pytest from policyengine_simulation_entry.run_restart_check import ( + RESTART_STATEMENT_TIMEOUT_MILLISECONDS, ObservabilityRestartTelemetry, run_scheduled_check, scheduled_restart_service, @@ -21,6 +22,7 @@ RunRestartService, ) from policyengine_simulation_persistence import RestartKind, UnstartedRun +from policyengine_stage12_persistence import BudgetWindowStore if TYPE_CHECKING: from collections.abc import Iterator @@ -180,6 +182,25 @@ def test_the_scheduled_service_restarts_in_its_modal_environments_deployment() - scheduled.engine.dispose() +def test_the_scheduled_window_advance_bounds_each_statement_as_the_check_does() -> None: + """A window's lock or reads can't hold the check past its Modal timeout.""" + scheduled = scheduled_restart_service( + database_url=DATABASE_URL, + modal_environment="main", + runtime=RecordingRuntime(), + ) + try: + windows = scheduled.service._stores.windows + assert isinstance(windows, BudgetWindowStore) + assert windows.engine is scheduled.engine + assert ( + windows.statement_timeout_milliseconds + == RESTART_STATEMENT_TIMEOUT_MILLISECONDS + ) + finally: + scheduled.engine.dispose() + + def test_the_scheduled_service_refuses_a_modal_environment_no_deployment_uses() -> None: with pytest.raises(ValueError, match="local"): scheduled_restart_service( diff --git a/projects/policyengine-simulation-entry/tests/test_run_restart_windows.py b/projects/policyengine-simulation-entry/tests/test_run_restart_windows.py new file mode 100644 index 000000000..eb79354f5 --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_run_restart_windows.py @@ -0,0 +1,542 @@ +"""Starting a stopped budget window again, from its earliest waiting year. + +A budget window stops when none of its years is under way and years still +wait for a start: the service stopped before starting its first years, or a +year's coordinator before starting the next. The restart check rebuilds the +earliest waiting year's call and advances the window with the service's own +year starter, marking that year just before the window's first start. The +window advance is faked here as ``BudgetWindowStore`` advances a window; the +store is tested against PostgreSQL in its own library. +""" + +from __future__ import annotations + +import asyncio +import itertools +from dataclasses import dataclass, field, replace +from datetime import timedelta +from typing import TYPE_CHECKING, Any +from uuid import UUID + +import pytest +from economy_service_fakes import ( + CATALOG_VERSION_ID, + CREATED_AT, + DYNAMIC_ID, + OUTPUT_BUCKET, + POLICY_ID, + REFORM, + FakeCaller, + numbered_ids, +) +from policyengine_simulation_contract.stage12_execution import ReportRunIds +from policyengine_simulation_entry.budget_window_api_models import ( + BudgetWindowReportRequest, +) +from policyengine_simulation_entry.budget_window_submissions import ( + ReportSubmissionService, +) +from policyengine_simulation_entry.economy_submissions import ServiceSettings +from policyengine_simulation_entry.economy_workers import ManifestEconomyWorkers +from policyengine_simulation_entry.run_restarts import ( + CHECK_TIME_LIMIT, + RESTART_AFTER, + RESTART_UNTIL, + RESTARTS_PER_CHECK, + RestartOutcome, + RestartSummary, +) +from policyengine_simulation_persistence import ( + EconomyScope, + IdleBudgetWindow, + RestartKind, + SimulationChoice, + StoredEconomySide, + StoredReportRun, + StoredRunUnreadableError, + UnstartedRun, + WaitingYear, +) +from policyengine_stage12_persistence import ( + RunStatus, + WindowAdvance, + YearStartDeferred, + YearStartRefused, + YearToStart, +) +from test_budget_window_requests import WINDOW_BODY +from test_budget_window_submissions import FakeAdvancer, FakeWindowStore +from test_run_restarts import ( + DATABASE_ERROR, + MANIFEST_STORE, + NOW, + FakeRestartStore, + restart_rig, + submitted_report, +) + +if TYPE_CHECKING: + from datetime import datetime + + from test_run_restarts import RestartRig + +WINDOW_RUN_ID = UUID("00000000-0000-4000-8000-0000000000f1") +NOT_REBUILT = ( + "run_not_started: The run stayed pending with no Modal call recorded for it, " + "and its call could not be built again: its stored rows no longer describe " + "one calculation." +) + + +@dataclass +class FakeWindows: + """Advances a window as ``BudgetWindowStore`` does: its waiting years in order. + + Each waiting year is offered to the starter until it defers one. A start + that raises fails the window, and the year records the refusal's own + message, or the error's type. ``ended`` is returned with no year offered, + as for a window that finished, or that a failed year fails now. + ``failure`` is raised before any year is offered, as when the window's + lock or reads fail, and ``failure_after_starts`` after every start, as + when the commit fails. + """ + + waiting: tuple[YearToStart, ...] = () + status: RunStatus = RunStatus.PENDING + ended: WindowAdvance | None = None + failure: BaseException | None = None + failure_after_starts: BaseException | None = None + advanced: list[UUID] = field(default_factory=list) + started_calls: list[str] = field(default_factory=list) + refusals: list[str] = field(default_factory=list) + + def advance_window( + self, window_run_id: UUID, *, now: datetime, start_year: Any = None + ) -> WindowAdvance: + self.advanced.append(window_run_id) + if self.failure is not None: + raise self.failure + if self.ended is not None: + return self.ended + started = [] + for year in self.waiting: + try: + self.started_calls.append(start_year(year)) + except YearStartDeferred: + break + except Exception as error: # noqa: BLE001 # as the store fails a refused year + self.refusals.append( + error.error_message + if isinstance(error, YearStartRefused) + else type(error).__name__ + ) + return WindowAdvance( + status=RunStatus.FAILED, changed=True, started_years=tuple(started) + ) + started.append(year.year) + if self.failure_after_starts is not None: + raise self.failure_after_starts + return WindowAdvance( + status=self.status, changed=False, started_years=tuple(started) + ) + + +def _idle_window( + run_id: UUID = WINDOW_RUN_ID, last_changed_at: datetime = CREATED_AT +) -> IdleBudgetWindow: + return IdleBudgetWindow(run_id=run_id, last_changed_at=last_changed_at) + + +def _year_to_start(stored: StoredReportRun, year: int) -> YearToStart: + return YearToStart( + year=year, + run_ids=ReportRunIds( + report_run_id=stored.report_run_id, + baseline_simulation_run_id=stored.baseline.simulation_run_id, + reform_simulation_run_id=stored.reform.simulation_run_id, + ), + ) + + +def _idle_window_rig(**options: Any) -> tuple[RestartRig, StoredReportRun, dict]: + """Return a rig with one idle window whose earliest waiting year was submitted. + + Its second waiting year is the following year, on runs of its own. + """ + stored, (payload, _context) = submitted_report() + second_year = YearToStart( + year=2027, + run_ids=ReportRunIds( + report_run_id=UUID(int=2027), + baseline_simulation_run_id=UUID(int=20271), + reform_simulation_run_id=UUID(int=20272), + ), + ) + restarts = options.pop("restarts", None) or FakeRestartStore( + idle_windows=(_idle_window(),), + report_runs={stored.report_run_id: stored}, + waiting_years={ + WINDOW_RUN_ID: WaitingYear(report_run_id=stored.report_run_id, year=2026) + }, + ) + windows = options.pop("windows", None) or FakeWindows( + waiting=(_year_to_start(stored, 2026), second_year) + ) + return restart_rig(restarts=restarts, windows=windows, **options), stored, payload + + +def _marked_run_ids(rig: RestartRig) -> list[UUID]: + return [run.run_id for run in rig.restarts.marked] + + +def test_an_idle_window_is_restarted_from_its_earliest_waiting_year() -> None: + """Its first years start as the service starts them, from its first year's call.""" + rig, stored, payload = _idle_window_rig() + + summary = rig.restart() + + assert rig.windows.advanced == [WINDOW_RUN_ID] + assert _marked_run_ids(rig) == [stored.report_run_id] + first_call, second_call = rig.economy_caller.report_runs + assert first_call[1] == payload + assert second_call[1]["evaluation_id"] == str(UUID(int=2027)) + assert second_call[1]["baseline"]["year"] == 2027 + assert summary == RestartSummary(listed=1, restarted=1) + assert rig.telemetry.outcomes == [ + (RestartKind.BUDGET_WINDOW, WINDOW_RUN_ID, RestartOutcome.RESTARTED, None) + ] + + +def test_the_check_lists_idle_windows_in_the_same_window_of_time() -> None: + rig, _stored, _payload = _idle_window_rig() + + rig.restart() + + assert ( + rig.restarts.listings + == [(NOW - RESTART_UNTIL, NOW - RESTART_AFTER, RESTARTS_PER_CHECK)] * 2 + ) + + +def test_rows_and_windows_are_taken_together_oldest_first_up_to_the_limit() -> None: + """The check takes at most its limit, the longest waiting first, of either kind.""" + rows = tuple( + UnstartedRun( + kind=RestartKind.REPORT_RUN, + run_id=UUID(int=100 + position), + created_at=CREATED_AT + timedelta(seconds=2 * position), + ) + for position in range(15) + ) + windows = tuple( + _idle_window( + run_id=UUID(int=200 + position), + last_changed_at=CREATED_AT + timedelta(seconds=2 * position + 1), + ) + for position in range(15) + ) + rig = restart_rig( + restarts=FakeRestartStore(unstarted=rows, idle_windows=windows), + windows=FakeWindows(), + ) + + summary = rig.restart() + + taken = [run_id for _kind, run_id, _age in rig.telemetry.scopes] + interleaved = [ + run.run_id for pair in zip(rows, windows, strict=True) for run in pair + ] + assert taken == interleaved[:RESTARTS_PER_CHECK] + assert summary.listed == RESTARTS_PER_CHECK + + +def test_a_window_whose_restarted_year_modal_refuses_counts_as_failed() -> None: + rig, stored, _payload = _idle_window_rig( + economy_caller=FakeCaller(failure=ConnectionError("Modal is down")) + ) + + summary = rig.restart() + + assert _marked_run_ids(rig) == [stored.report_run_id] + assert rig.windows.refusals == ["ConnectionError"] + assert summary == RestartSummary(listed=1, failed=1) + + +def test_a_window_no_longer_waiting_is_left_unmarked() -> None: + rig, _stored, _payload = _idle_window_rig( + restarts=FakeRestartStore(idle_windows=(_idle_window(),)) + ) + + summary = rig.restart() + + assert rig.restarts.marked == [] + assert rig.windows.advanced == [] + assert summary == RestartSummary(listed=1, left=1) + + +def test_a_window_another_check_marked_first_starts_no_call() -> None: + rig, _stored, _payload = _idle_window_rig() + rig.restarts.marked_elsewhere = True + + summary = rig.restart() + + assert rig.economy_caller.report_runs == [] + assert summary == RestartSummary(listed=1, left=1) + + +def test_a_window_whose_first_years_call_cannot_be_rebuilt_fails_with_the_reason() -> ( + None +): + """The year records why, as a row the check can't rebuild is failed at once.""" + rig, stored, _payload = _idle_window_rig() + rig.restarts.read_failures[stored.report_run_id] = StoredRunUnreadableError( + "report run names no region" + ) + + summary = rig.restart() + + assert rig.windows.refusals == [NOT_REBUILT] + assert rig.economy_caller.report_runs == [] + assert rig.restarts.marked == [] + assert summary == RestartSummary(listed=1, failed=1) + assert rig.telemetry.outcomes == [ + ( + RestartKind.BUDGET_WINDOW, + WINDOW_RUN_ID, + RestartOutcome.FAILED, + "StoredRunUnreadableError", + ) + ] + + +def test_a_database_error_before_any_start_leaves_the_window_for_the_next_check() -> ( + None +): + """Nothing started and nothing is marked, so the next check restarts it.""" + rig, _stored, _payload = _idle_window_rig( + windows=FakeWindows(failure=DATABASE_ERROR) + ) + + summary = rig.restart() + + assert rig.restarts.marked == [] + assert rig.economy_caller.report_runs == [] + assert summary == RestartSummary(listed=1, deferred=1) + assert rig.telemetry.outcomes[0][2:] == ( + RestartOutcome.DEFERRED, + "OperationalError", + ) + + +@pytest.mark.parametrize( + "failure", + [DATABASE_ERROR, RuntimeError("unforeseen")], + ids=["database-error", "unexpected-error"], +) +def test_an_advance_that_fails_after_its_starts_counts_the_calls_not_recorded( + failure, +) -> None: + """The calls Modal accepted claim their years themselves.""" + rig, stored, _payload = _idle_window_rig() + rig.windows.failure_after_starts = failure + + summary = rig.restart() + + assert _marked_run_ids(rig) == [stored.report_run_id] + assert len(rig.economy_caller.report_runs) == 2 + assert summary == RestartSummary(listed=1, call_not_recorded=1) + assert rig.telemetry.outcomes[0][3] == type(failure).__name__ + + +def test_a_mark_that_cannot_be_written_starts_no_call() -> None: + rig, _stored, _payload = _idle_window_rig() + rig.restarts.mark_failure = DATABASE_ERROR + + summary = rig.restart() + + assert rig.economy_caller.report_runs == [] + assert summary == RestartSummary(listed=1, deferred=1) + assert rig.telemetry.outcomes[0][3] == "OperationalError" + + +def test_an_unexpected_error_in_a_windows_restart_defers_it_and_the_check_goes_on( + caplog, +) -> None: + """One window the check can't restart for an unforeseen reason holds up no row. + + The error is logged with its traceback, so it can be diagnosed. + """ + stored, _call = submitted_report() + window_year = replace(stored, report_run_id=UUID(int=2026)) + later_row = UnstartedRun( + kind=RestartKind.REPORT_RUN, + run_id=stored.report_run_id, + created_at=CREATED_AT + timedelta(seconds=1), + ) + rig = restart_rig( + restarts=FakeRestartStore( + unstarted=(later_row,), + idle_windows=(_idle_window(),), + report_runs={ + stored.report_run_id: stored, + window_year.report_run_id: window_year, + }, + waiting_years={ + WINDOW_RUN_ID: WaitingYear( + report_run_id=window_year.report_run_id, year=2026 + ) + }, + ), + windows=FakeWindows(failure=RuntimeError("unforeseen")), + ) + + summary = rig.restart() + + assert rig.telemetry.outcomes[0] == ( + RestartKind.BUDGET_WINDOW, + WINDOW_RUN_ID, + RestartOutcome.DEFERRED, + "RuntimeError", + ) + assert _marked_run_ids(rig) == [stored.report_run_id] + assert summary == RestartSummary(listed=2, restarted=1, deferred=1) + (logged,) = [ + record + for record in caplog.records + if record.getMessage() == "v2_run_restart_unexpected_error" + ] + assert isinstance(logged.exc_info[1], RuntimeError) + assert logged.policyengine_attributes == { + "run_kind": "budget_window", + "run_id": str(WINDOW_RUN_ID), + } + + +def test_a_window_gone_before_its_advance_is_left() -> None: + rig, _stored, _payload = _idle_window_rig( + windows=FakeWindows(failure=LookupError("budget-window run does not exist")) + ) + + summary = rig.restart() + + assert rig.restarts.marked == [] + assert summary == RestartSummary(listed=1, left=1) + + +@pytest.mark.parametrize( + "ended", + [ + # A year failed meanwhile, so this advance fails the window. + WindowAdvance(status=RunStatus.FAILED, changed=True), + # The window had already finished. + WindowAdvance(status=RunStatus.SUCCEEDED, changed=False), + ], + ids=["failed-now", "finished"], +) +def test_a_window_that_ends_without_a_start_is_left_unmarked(ended) -> None: + """The restart started nothing, so it failed nothing either.""" + rig, _stored, _payload = _idle_window_rig(windows=FakeWindows(ended=ended)) + + summary = rig.restart() + + assert rig.restarts.marked == [] + assert summary == RestartSummary(listed=1, left=1) + + +def test_a_window_the_check_has_no_time_left_for_is_left_for_the_next_check() -> None: + # The check starts, takes the window up, then runs out of time once its + # first year's call is rebuilt. + timer_readings = itertools.chain( + [0.0, 0.0], itertools.repeat(CHECK_TIME_LIMIT.total_seconds()) + ) + rig, _stored, _payload = _idle_window_rig(timer=lambda: next(timer_readings)) + + summary = rig.restart() + + assert rig.windows.advanced == [] + assert rig.restarts.marked == [] + assert summary == RestartSummary(listed=1, deferred=1) + + +def test_a_window_whose_manifest_cannot_be_read_is_left_for_the_next_check() -> None: + rig, _stored, _payload = _idle_window_rig(manifest={}) + + summary = rig.restart() + + assert rig.windows.advanced == [] + assert summary == RestartSummary(listed=1, deferred=1) + + +def test_a_windows_first_year_restarts_with_the_call_its_window_started() -> None: + """The restarted first year's call is the one the window's submission started.""" + window_store = FakeWindowStore() + caller = FakeCaller() + submissions = ReportSubmissionService( + windows=FakeAdvancer(window_store), + store=window_store, + workers=ManifestEconomyWorkers(MANIFEST_STORE), + caller=caller, + settings=ServiceSettings( + environment="staging", + modal_environment="staging", + output_bucket=OUTPUT_BUCKET, + ), + call_start_timeout_seconds=0.05, + new_id=numbered_ids(), + clock=lambda: CREATED_AT, + ) + request = BudgetWindowReportRequest.model_validate(WINDOW_BODY) + asyncio.run( + submissions.submit_budget_window( + request, request_id="original-request", observability_context=None + ) + ) + (report,) = window_store.windows + first_year = report.first_year + (worker, payload, context, _observability) = caller.report_runs[0] + stored = StoredReportRun( + report_run_id=first_year.report_run_id, + country="us", + versions=worker.versions, + options=report.inputs, + scope=EconomyScope( + catalog_version_id=CATALOG_VERSION_ID, + region_code="us", + dataset_name="populace_us_2024", + year=first_year.year, + ), + baseline=StoredEconomySide( + simulation_run_id=first_year.baseline.simulation_run_id, + choice=SimulationChoice(policy_id=None, dynamic_id=None), + reform={}, + ), + reform=StoredEconomySide( + simulation_run_id=first_year.reform.simulation_run_id, + choice=SimulationChoice(policy_id=POLICY_ID, dynamic_id=DYNAMIC_ID), + reform=REFORM, + ), + created_at=CREATED_AT, + ) + rig = restart_rig( + restarts=FakeRestartStore( + idle_windows=(_idle_window(run_id=report.window_run_id),), + report_runs={stored.report_run_id: stored}, + waiting_years={ + report.window_run_id: WaitingYear( + report_run_id=stored.report_run_id, year=first_year.year + ) + }, + ), + windows=FakeWindows(waiting=(_year_to_start(stored, first_year.year),)), + ) + + summary = rig.restart() + + (restarted,) = rig.economy_caller.report_runs + assert restarted[1] == payload + assert _without_request_id(restarted[2]) == _without_request_id(context) + assert summary == RestartSummary(listed=1, restarted=1) + + +def _without_request_id(context: dict[str, Any]) -> dict[str, Any]: + return {key: value for key, value in context.items() if key != "request_id"} diff --git a/projects/policyengine-simulation-entry/tests/test_run_restarts.py b/projects/policyengine-simulation-entry/tests/test_run_restarts.py index fd9ca778d..b40f72f0c 100644 --- a/projects/policyengine-simulation-entry/tests/test_run_restarts.py +++ b/projects/policyengine-simulation-entry/tests/test_run_restarts.py @@ -32,7 +32,6 @@ simulation_request, ) from household_fixtures import manifest_store -from policyengine_simulation_contract.stage12_execution import ReportRunIds from policyengine_simulation_entry.economy_workers import ManifestEconomyWorkers from policyengine_simulation_entry.household_api_models import ( HouseholdCalculationRequest, @@ -61,6 +60,7 @@ HouseholdCalculationInputs, HouseholdJobRecord, HouseholdJobStatus, + IdleBudgetWindow, RestartKind, RunCall, SimulationChoice, @@ -70,12 +70,6 @@ StoredRunUnreadableError, StoredSimulationRun, UnstartedRun, - WaitingYear, -) -from policyengine_stage12_persistence import ( - RunStatus, - WindowAdvance, - YearToStart, ) from sqlalchemy.exc import OperationalError @@ -83,8 +77,13 @@ from collections.abc import Callable, Iterator from datetime import datetime + from policyengine_simulation_entry.budget_window_submissions import ( + BudgetWindowAdvancer, + ) from policyengine_simulation_entry.household_workers import HouseholdWorker - from policyengine_simulation_persistence import NewHouseholdJob + from policyengine_simulation_entry.run_restarts import RestartTarget + from policyengine_simulation_persistence import NewHouseholdJob, WaitingYear + from policyengine_stage12_persistence import WindowAdvance NOW = CREATED_AT + timedelta(minutes=4) DATABASE_ERROR = OperationalError("SELECT 1", {}, Exception("server closed")) @@ -134,10 +133,12 @@ class FakeRestartStore: # A read of one run that raises, by run ID. read_failures: dict[UUID, Exception] = field(default_factory=dict) marked_elsewhere: bool = False + # Raised by every mark, as when the mark can't be written. + mark_failure: Exception | None = None # Each row's status and call as stored, by run ID; a row not named here is # as the rig's other fakes left it (``holder``). held: dict[UUID, RunCall | None] = field(default_factory=dict) - idle_windows: tuple[UnstartedRun, ...] = () + idle_windows: tuple[IdleBudgetWindow, ...] = () waiting_years: dict[UUID, WaitingYear] = field(default_factory=dict) holder: Callable[[UnstartedRun], RunCall | None] | None = None listings: list[tuple[datetime, datetime, int]] = field(default_factory=list) @@ -145,7 +146,7 @@ class FakeRestartStore: def list_idle_windows( self, *, idle_after: datetime, idle_before: datetime, limit: int - ) -> tuple[UnstartedRun, ...]: + ) -> tuple[IdleBudgetWindow, ...]: self.listings.append((idle_after, idle_before, limit)) return self.idle_windows @@ -169,6 +170,8 @@ def list_unstarted( return self.unstarted def mark_restart(self, run: UnstartedRun) -> bool: + if self.mark_failure is not None: + raise self.mark_failure if self.marked_elsewhere: return False self.marked.append(run) @@ -279,35 +282,14 @@ async def spawn( return "fc-household-restarted" -@dataclass -class FakeWindows: - """Advances a window as ``BudgetWindowStore`` does: its waiting years in order. - - Each waiting year is offered to the starter; the advance ends with - ``status``, or failed when a start raises. - """ - - waiting: tuple[YearToStart, ...] = () - status: RunStatus = RunStatus.PENDING - advanced: list[UUID] = field(default_factory=list) - started_calls: list[str] = field(default_factory=list) +class UnusedWindows: + """A window advancer for checks that list no budget window.""" def advance_window( self, window_run_id: UUID, *, now: datetime, start_year: Any = None ) -> WindowAdvance: - self.advanced.append(window_run_id) - started = [] - for year in self.waiting: - try: - self.started_calls.append(start_year(year)) - except Exception: # noqa: BLE001 # as the store fails a refused year - return WindowAdvance( - status=RunStatus.FAILED, changed=True, started_years=tuple(started) - ) - started.append(year.year) - return WindowAdvance( - status=self.status, changed=False, started_years=tuple(started) - ) + msg = "this check lists no budget window" + raise AssertionError(msg) @dataclass @@ -320,13 +302,15 @@ class RecordingTelemetry: ) @contextmanager - def restart_scope(self, run: UnstartedRun, *, age_seconds: float) -> Iterator[None]: + def restart_scope( + self, run: RestartTarget, *, age_seconds: float + ) -> Iterator[None]: self.scopes.append((run.kind, run.run_id, age_seconds)) yield def record_outcome( self, - run: UnstartedRun, + run: RestartTarget, outcome: RestartOutcome, *, error_type: str | None = None, @@ -342,7 +326,7 @@ class RestartRig: """The restart service and every fake it was built with.""" restarts: FakeRestartStore - windows: FakeWindows + windows: BudgetWindowAdvancer economy: FakeStore households: FakeHouseholdJobs economy_caller: FakeCaller @@ -354,7 +338,7 @@ def restart(self) -> RestartSummary: return asyncio.run(self.service.restart_unstarted_runs()) -def _rig( # noqa: PLR0913 # each fake a test may replace +def restart_rig( # noqa: PLR0913 # each fake a test may replace *, restarts: FakeRestartStore | None = None, economy: FakeStore | None = None, @@ -363,10 +347,10 @@ def _rig( # noqa: PLR0913 # each fake a test may replace household_caller: FakeHouseholdCaller | None = None, manifest: object | None = None, timer: Callable[[], float] | None = None, - windows: FakeWindows | None = None, + windows: BudgetWindowAdvancer | None = None, ) -> RestartRig: restarts = restarts or FakeRestartStore() - windows = windows or FakeWindows() + windows = windows or UnusedWindows() economy = economy or FakeStore() households = households or FakeHouseholdJobs() economy_caller = economy_caller or FakeCaller() @@ -414,7 +398,7 @@ def holder(run: UnstartedRun) -> RunCall: ) -def _submitted_report() -> tuple[StoredReportRun, tuple[dict, dict]]: +def submitted_report() -> tuple[StoredReportRun, tuple[dict, dict]]: """Submit a report as the service does; return its stored run and its call.""" caller = FakeCaller() created = asyncio.run( @@ -515,7 +499,7 @@ def _without_request_id(context: dict[str, Any]) -> dict[str, Any]: def test_the_check_lists_runs_created_three_to_ten_minutes_ago() -> None: - rig = _rig() + rig = restart_rig() summary = rig.restart() @@ -532,9 +516,9 @@ def test_the_check_lists_runs_created_three_to_ten_minutes_ago() -> None: def test_a_report_run_is_started_again_with_the_call_its_submission_started() -> None: - stored, (payload, context) = _submitted_report() + stored, (payload, context) = submitted_report() run = _unstarted(RestartKind.REPORT_RUN, stored.report_run_id) - rig = _rig( + rig = restart_rig( restarts=FakeRestartStore( unstarted=(run,), report_runs={stored.report_run_id: stored} ) @@ -559,7 +543,7 @@ def test_a_standalone_simulation_run_is_started_again_with_its_submissions_call( ): stored, (payload, context) = _submitted_simulation() run_id = stored.run.simulation_run_id - rig = _rig( + rig = restart_rig( restarts=FakeRestartStore( unstarted=(_unstarted(RestartKind.SIMULATION_RUN, run_id),), simulation_runs={run_id: stored}, @@ -582,7 +566,7 @@ def test_a_household_job_is_started_again_with_its_submissions_call() -> None: payload = _submitted_household_job( households, policy_id=str(POLICY_ID), add_missing_variables=True ) - rig = _rig( + rig = restart_rig( restarts=FakeRestartStore( unstarted=(_unstarted(RestartKind.HOUSEHOLD_JOB, HOUSEHOLD_JOB_ID),) ), @@ -600,8 +584,8 @@ def test_a_household_job_is_started_again_with_its_submissions_call() -> None: def test_a_run_another_check_marked_first_is_left_without_a_call() -> None: - stored, _call = _submitted_report() - rig = _rig( + stored, _call = submitted_report() + rig = restart_rig( restarts=FakeRestartStore( unstarted=(_unstarted(RestartKind.REPORT_RUN, stored.report_run_id),), report_runs={stored.report_run_id: stored}, @@ -617,7 +601,7 @@ def test_a_run_another_check_marked_first_is_left_without_a_call() -> None: def test_a_run_gone_before_it_is_read_is_left_alone() -> None: - rig = _rig( + rig = restart_rig( restarts=FakeRestartStore( unstarted=( _unstarted(RestartKind.REPORT_RUN, UUID(int=1)), @@ -634,8 +618,8 @@ def test_a_run_gone_before_it_is_read_is_left_alone() -> None: def test_a_run_whose_rows_cannot_be_read_is_left_for_the_next_check() -> None: - stored, _call = _submitted_report() - rig = _rig( + stored, _call = submitted_report() + rig = restart_rig( restarts=FakeRestartStore( unstarted=(_unstarted(RestartKind.REPORT_RUN, stored.report_run_id),), read_failure=DATABASE_ERROR, @@ -650,8 +634,8 @@ def test_a_run_whose_rows_cannot_be_read_is_left_for_the_next_check() -> None: def test_an_unreadable_manifest_leaves_the_run_for_the_next_check() -> None: - stored, _call = _submitted_report() - rig = _rig( + stored, _call = submitted_report() + rig = restart_rig( restarts=FakeRestartStore( unstarted=(_unstarted(RestartKind.REPORT_RUN, stored.report_run_id),), report_runs={stored.report_run_id: stored}, @@ -666,8 +650,8 @@ def test_an_unreadable_manifest_leaves_the_run_for_the_next_check() -> None: def test_a_run_whose_worker_left_the_manifest_is_failed_as_not_started() -> None: - stored, _call = _submitted_report() - rig = _rig( + stored, _call = submitted_report() + rig = restart_rig( restarts=FakeRestartStore( unstarted=(_unstarted(RestartKind.REPORT_RUN, stored.report_run_id),), report_runs={stored.report_run_id: stored}, @@ -691,7 +675,7 @@ def test_a_run_whose_worker_left_the_manifest_is_failed_as_not_started() -> None def test_a_run_whose_stored_rows_no_longer_describe_it_is_failed() -> None: run_id = UUID(int=7) - rig = _rig( + rig = restart_rig( restarts=FakeRestartStore( unstarted=(_unstarted(RestartKind.SIMULATION_RUN, run_id),), read_failure=StoredRunUnreadableError("simulation run 7 names no region"), @@ -711,7 +695,7 @@ def test_a_household_job_whose_household_changed_is_failed_as_not_started() -> N households = FakeHouseholdJobs() _submitted_household_job(households) households.default_year = 2027 - rig = _rig( + rig = restart_rig( restarts=FakeRestartStore( unstarted=(_unstarted(RestartKind.HOUSEHOLD_JOB, HOUSEHOLD_JOB_ID),) ), @@ -728,8 +712,8 @@ def test_a_household_job_whose_household_changed_is_failed_as_not_started() -> N def test_a_restart_modal_refuses_fails_the_run_as_not_started() -> None: - stored, _call = _submitted_report() - rig = _rig( + stored, _call = submitted_report() + rig = restart_rig( restarts=FakeRestartStore( unstarted=(_unstarted(RestartKind.REPORT_RUN, stored.report_run_id),), report_runs={stored.report_run_id: stored}, @@ -761,7 +745,7 @@ def test_a_restart_modal_refuses_fails_the_run_as_not_started() -> None: def test_a_restart_modal_does_not_confirm_in_time_fails_the_run() -> None: stored, _call = _submitted_simulation() run_id = stored.run.simulation_run_id - rig = _rig( + rig = restart_rig( restarts=FakeRestartStore( unstarted=(_unstarted(RestartKind.SIMULATION_RUN, run_id),), simulation_runs={run_id: stored}, @@ -778,8 +762,8 @@ def test_a_restart_modal_does_not_confirm_in_time_fails_the_run() -> None: def test_a_refused_restart_of_a_run_another_call_claimed_leaves_it() -> None: - stored, _call = _submitted_report() - rig = _rig( + stored, _call = submitted_report() + rig = restart_rig( restarts=FakeRestartStore( unstarted=(_unstarted(RestartKind.REPORT_RUN, stored.report_run_id),), report_runs={stored.report_run_id: stored}, @@ -800,7 +784,7 @@ def test_a_restarted_call_another_call_claimed_first_counts_as_started_meanwhile households = FakeHouseholdJobs() _submitted_household_job(households) households.claimed = True - rig = _rig( + rig = restart_rig( restarts=FakeRestartStore( unstarted=(_unstarted(RestartKind.HOUSEHOLD_JOB, HOUSEHOLD_JOB_ID),) ), @@ -814,8 +798,8 @@ def test_a_restarted_call_another_call_claimed_first_counts_as_started_meanwhile def test_a_restart_whose_call_cannot_be_recorded_still_counts_its_call() -> None: - stored, _call = _submitted_report() - rig = _rig( + stored, _call = submitted_report() + rig = restart_rig( restarts=FakeRestartStore( unstarted=(_unstarted(RestartKind.REPORT_RUN, stored.report_run_id),), report_runs={stored.report_run_id: stored}, @@ -831,14 +815,14 @@ def test_a_restart_whose_call_cannot_be_recorded_still_counts_its_call() -> None def test_each_listed_run_is_one_telemetry_scope_with_its_age_and_outcome() -> None: - report, _report_call = _submitted_report() + report, _report_call = submitted_report() simulation, _simulation_call = _submitted_simulation() simulation_run_id = simulation.run.simulation_run_id report_run = replace( _unstarted(RestartKind.REPORT_RUN, report.report_run_id), created_at=CREATED_AT + timedelta(seconds=30), ) - rig = _rig( + rig = restart_rig( restarts=FakeRestartStore( unstarted=( report_run, @@ -882,7 +866,7 @@ def test_an_unreadable_report_run_is_failed_with_its_simulation_runs() -> None: """As a submission fails a report run, with the simulation runs made for it.""" run_id = UUID(int=8) sides = (UUID(int=81), UUID(int=82)) - rig = _rig( + rig = restart_rig( restarts=FakeRestartStore( unstarted=(_unstarted(RestartKind.REPORT_RUN, run_id),), read_failure=StoredRunUnreadableError("report run 8 names no region"), @@ -899,8 +883,8 @@ def test_an_unreadable_report_run_is_failed_with_its_simulation_runs() -> None: def test_a_restarted_call_claiming_its_run_before_the_record_is_restarted() -> None: - stored, _call = _submitted_report() - rig = _rig( + stored, _call = submitted_report() + rig = restart_rig( restarts=FakeRestartStore( unstarted=(_unstarted(RestartKind.REPORT_RUN, stored.report_run_id),), report_runs={stored.report_run_id: stored}, @@ -917,8 +901,8 @@ def test_a_restarted_call_claiming_its_run_before_the_record_is_restarted() -> N def test_a_run_failed_before_its_restart_was_recorded_is_left() -> None: - stored, _call = _submitted_report() - rig = _rig( + stored, _call = submitted_report() + rig = restart_rig( restarts=FakeRestartStore( unstarted=(_unstarted(RestartKind.REPORT_RUN, stored.report_run_id),), report_runs={stored.report_run_id: stored}, @@ -931,8 +915,8 @@ def test_a_run_failed_before_its_restart_was_recorded_is_left() -> None: def test_a_refused_restart_of_a_run_the_sweep_failed_meanwhile_is_left() -> None: - stored, _call = _submitted_report() - rig = _rig( + stored, _call = submitted_report() + rig = restart_rig( restarts=FakeRestartStore( unstarted=(_unstarted(RestartKind.REPORT_RUN, stored.report_run_id),), report_runs={stored.report_run_id: stored}, @@ -959,10 +943,10 @@ async def start_report_run(self, *arguments: Any, **keywords: Any) -> str: def test_a_check_past_its_time_limit_leaves_the_other_runs_for_the_next() -> None: """No row is marked once the check's time is up, so none waits for the sweep.""" - first, _first_call = _submitted_report() - second, _second_call = _submitted_report() + first, _first_call = submitted_report() + second, _second_call = submitted_report() caller = AdvancingCaller(step_seconds=CHECK_TIME_LIMIT.total_seconds()) - rig = _rig( + rig = restart_rig( restarts=FakeRestartStore( unstarted=( _unstarted(RestartKind.REPORT_RUN, first.report_run_id), @@ -1003,9 +987,9 @@ def __setitem__(self, key: str, value: object) -> None: def test_a_runs_call_is_rebuilt_off_the_event_loop() -> None: """Modal's blocking reads, such as the manifest's, never hold up the loop.""" - stored, _call = _submitted_report() + stored, _call = submitted_report() manifest = LoopWatchingManifestStore(MANIFEST_STORE) - rig = _rig( + rig = restart_rig( restarts=FakeRestartStore( unstarted=(_unstarted(RestartKind.REPORT_RUN, stored.report_run_id),), report_runs={stored.report_run_id: stored}, @@ -1026,8 +1010,8 @@ def test_an_unexpected_rebuild_error_defers_its_run_and_the_check_goes_on( The error is logged with its traceback, so it can be diagnosed. """ broken_run_id = UUID(int=9) - stored, _call = _submitted_report() - rig = _rig( + stored, _call = submitted_report() + rig = restart_rig( restarts=FakeRestartStore( unstarted=( _unstarted(RestartKind.REPORT_RUN, broken_run_id), @@ -1060,134 +1044,3 @@ def test_an_unexpected_rebuild_error_defers_its_run_and_the_check_goes_on( "run_kind": "report_run", "run_id": str(broken_run_id), } - - -WINDOW_RUN_ID = UUID("00000000-0000-4000-8000-0000000000f1") - - -def _idle_window_rig(**options: Any) -> tuple[RestartRig, StoredReportRun, dict]: - """Return a rig with one idle window whose earliest waiting year was submitted. - - Its second waiting year is the following year, on runs of its own. - """ - stored, (payload, _context) = _submitted_report() - first_year = YearToStart( - year=2026, - run_ids=ReportRunIds( - report_run_id=stored.report_run_id, - baseline_simulation_run_id=stored.baseline.simulation_run_id, - reform_simulation_run_id=stored.reform.simulation_run_id, - ), - ) - second_year = YearToStart( - year=2027, - run_ids=ReportRunIds( - report_run_id=UUID(int=2027), - baseline_simulation_run_id=UUID(int=20271), - reform_simulation_run_id=UUID(int=20272), - ), - ) - restarts = options.pop("restarts", None) or FakeRestartStore( - idle_windows=( - UnstartedRun( - kind=RestartKind.BUDGET_WINDOW, - run_id=WINDOW_RUN_ID, - created_at=CREATED_AT, - ), - ), - report_runs={stored.report_run_id: stored}, - waiting_years={ - WINDOW_RUN_ID: WaitingYear(report_run_id=stored.report_run_id, year=2026) - }, - ) - windows = options.pop("windows", None) or FakeWindows( - waiting=(first_year, second_year) - ) - return _rig(restarts=restarts, windows=windows, **options), stored, payload - - -def test_an_idle_window_is_restarted_from_its_earliest_waiting_year() -> None: - """Its first years start as the service starts them, from its first year's call.""" - rig, stored, payload = _idle_window_rig() - - summary = rig.restart() - - assert rig.windows.advanced == [WINDOW_RUN_ID] - assert [run.run_id for run in rig.restarts.marked] == [stored.report_run_id] - first_call, second_call = rig.economy_caller.report_runs - assert first_call[1] == payload - assert second_call[1]["evaluation_id"] == str(UUID(int=2027)) - assert second_call[1]["baseline"]["year"] == 2027 - assert summary == RestartSummary(listed=1, restarted=1) - assert rig.telemetry.outcomes == [ - (RestartKind.BUDGET_WINDOW, WINDOW_RUN_ID, RestartOutcome.RESTARTED, None) - ] - - -def test_the_check_lists_idle_windows_in_the_same_window_of_time() -> None: - rig, _stored, _payload = _idle_window_rig() - - rig.restart() - - assert ( - rig.restarts.listings - == [(NOW - RESTART_UNTIL, NOW - RESTART_AFTER, RESTARTS_PER_CHECK)] * 2 - ) - - -def test_a_window_whose_restarted_year_modal_refuses_counts_as_failed() -> None: - rig, _stored, _payload = _idle_window_rig( - economy_caller=FakeCaller(failure=ConnectionError("Modal is down")) - ) - - summary = rig.restart() - - assert summary == RestartSummary(listed=1, failed=1) - - -def test_a_window_no_longer_waiting_is_left_unmarked() -> None: - rig, _stored, _payload = _idle_window_rig( - restarts=FakeRestartStore( - idle_windows=( - UnstartedRun( - kind=RestartKind.BUDGET_WINDOW, - run_id=WINDOW_RUN_ID, - created_at=CREATED_AT, - ), - ) - ) - ) - - summary = rig.restart() - - assert rig.restarts.marked == [] - assert rig.windows.advanced == [] - assert summary == RestartSummary(listed=1, left=1) - - -def test_a_window_another_check_marked_first_is_left_without_an_advance() -> None: - rig, stored, _payload = _idle_window_rig() - rig.restarts.marked_elsewhere = True - - summary = rig.restart() - - assert rig.windows.advanced == [] - assert summary == RestartSummary(listed=1, left=1) - assert stored.report_run_id not in [run.run_id for run in rig.restarts.marked] - - -def test_a_window_whose_first_years_call_cannot_be_rebuilt_is_left_to_the_sweep() -> ( - None -): - """It is marked, so no later check tries again; the sweep fails it as stalled.""" - rig, stored, _payload = _idle_window_rig() - rig.restarts.read_failures[stored.report_run_id] = StoredRunUnreadableError( - "report run names no region" - ) - - summary = rig.restart() - - assert [run.run_id for run in rig.restarts.marked] == [stored.report_run_id] - assert rig.windows.advanced == [] - assert summary == RestartSummary(listed=1, left=1) - assert rig.telemetry.outcomes[0][3] == "StoredRunUnreadableError" From c63d73327fa6c764bc9335e207dd1f6ed886fbdb Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:41:47 +0400 Subject: [PATCH 23/29] Fix review round 6's budget-window restart findings The window's earliest waiting year is now marked just before the advance's first start, by the starter the window is advanced with, so an advance whose lock or reads fail marks nothing and the next check takes the window up again. An unexpected error in a window's restart is logged with its traceback and ends only that window's restart. A window whose first call can't be built again fails at once through the same advance: the starter refuses that year with YearStartRefused, so the year records run_not_started and why, and the window which year failed. The restart's budget-window store bounds each statement with the check's statement timeout, idle windows are a type of their own (IdleBudgetWindow), the outcome descriptions cover windows, and the docstrings that said only the sweep handles a stopped window now name the restart check. Co-Authored-By: Claude Opus 5.5 --- .../c1-19-restart-unstarted-runs.added.md | 2 +- .../stage-12-modal-worker-foundation.md | 15 +- .../README.md | 9 +- .../__init__.py | 2 + .../run_restart_records.py | 40 ++- .../run_restart_store.py | 3 +- .../README.md | 12 +- .../__init__.py | 2 + .../budget_window_store.py | 66 ++++- .../policyengine-simulation-entry/README.md | 3 +- .../budget_window_submissions.py | 5 +- .../run_restart_calls.py | 7 +- .../run_restart_check.py | 29 +- .../run_restart_outcomes.py | 58 +++- .../run_restart_windows.py | 274 +++++++++++------- .../run_restarts.py | 77 ++--- .../budget_window_scheduling.py | 7 +- 17 files changed, 401 insertions(+), 210 deletions(-) diff --git a/changelog.d/c1-19-restart-unstarted-runs.added.md b/changelog.d/c1-19-restart-unstarted-runs.added.md index a08af1223..77ed9fe64 100644 --- a/changelog.d/c1-19-restart-unstarted-runs.added.md +++ b/changelog.d/c1-19-restart-unstarted-runs.added.md @@ -1 +1 @@ -A restart check in the stuck-run sweep's Modal application starts the Modal call of a household job, economy report run or standalone economy simulation run again when the run stayed pending with no call recorded 3 to 10 minutes after its creation, as when the Simulation Entrypoint stopped before starting its call. It also starts a budget window's waiting years again when the window stopped with none under way, as the Simulation Entrypoint starts a new window's first years. Its telemetry counts every restart and how it ended. The sweep now fails a run still pending with no call at 15 minutes instead of 10. +A restart check in the stuck-run sweep's Modal application starts the Modal call of a household job, economy report run or standalone economy simulation run again when the run stayed pending with no call recorded 3 to 10 minutes after its creation, as when the Simulation Entrypoint stopped before starting its call. It also starts a budget window's waiting years again when the window stopped with none under way, as the Simulation Entrypoint starts a new window's first years, and fails at once, with the reason, a window whose first waiting year's call can't be built again. Its telemetry counts every restart and how it ended. The sweep now fails a run still pending with no call at 15 minutes instead of 10. diff --git a/docs/migration/stage-12-modal-worker-foundation.md b/docs/migration/stage-12-modal-worker-foundation.md index e4134511e..22c9ae91b 100644 --- a/docs/migration/stage-12-modal-worker-foundation.md +++ b/docs/migration/stage-12-modal-worker-foundation.md @@ -821,11 +821,16 @@ no year is under way and years still wait: the entrypoint stopped before starting its first years, or a year's coordinator before starting the next. The check restarts a window that stopped 3 to 10 minutes ago, with its waiting years untouched, as the entrypoint starts a new window: it rebuilds -the earliest waiting year's call from its stored rows, marks that year, and -advances the window (`BudgetWindowStore.advance_window`) with the -entrypoint's year starter (`YearStarts`), which derives each later year's -call from the first's. A window whose first call can't be rebuilt is marked -and left, and the sweep fails it as stalled. Household simulations' runs +the earliest waiting year's call from its stored rows, then advances the +window (`BudgetWindowStore.advance_window`) with the entrypoint's year +starter (`YearStarts`), which derives each later year's call from the +first's. Just before the advance's first start it marks that year, so the +window is restarted once each time it stops; an advance whose lock or reads +fail marks nothing, and the next check takes the window up again. Its +statements are bounded by the check's statement timeout. A window whose +first call can't be rebuilt is failed at once through the same advance: that +year records `run_not_started` and why, and the window which year failed. +Household simulations' runs are left alone: the check can't rebuild their calls yet. Each check is a `v2_run_restart_check` operation and each row it takes up a `v2_run_restart` operation, with the row's kind, ID, age and outcome, and diff --git a/libs/policyengine-simulation-persistence/README.md b/libs/policyengine-simulation-persistence/README.md index bec513aec..dd5fa8dd3 100644 --- a/libs/policyengine-simulation-persistence/README.md +++ b/libs/policyengine-simulation-persistence/README.md @@ -137,10 +137,11 @@ the run tables and their inputs, and `UPDATE` of `updated_at`, still equals `created_at`: household jobs, report runs outside a budget window whose baseline is an economy simulation, and runs of economy simulations that no report run names (standalone runs). -- `list_idle_budget_windows` returns the budget windows that stopped: pending - or running, with no year under way and years still waiting untouched, - last changed in a window. `read_earliest_waiting_year` names the year a - restart starts the window from, and marks. +- `list_idle_budget_windows` returns the budget windows that stopped + (`IdleBudgetWindow`, a type apart from the rows): pending or running, with + no year under way and years still waiting untouched, last changed in a + window. `read_earliest_waiting_year` names the year a restart starts the + window from, and marks just before its first start. - `mark_restart` sets a listed row's `updated_at` to the statement's clock while it is still unstarted, so it is listed and marked only once. - `read_unstarted_report_run` and `read_unstarted_simulation_run` read back diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py index 13dfcc1c3..55b9fbee3 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py @@ -96,6 +96,7 @@ from policyengine_simulation_persistence.run_restart_records import ( RESTART_TABLES, EconomyScope, + IdleBudgetWindow, RestartKind, RunCall, StoredEconomySide, @@ -158,6 +159,7 @@ "HouseholdJobReferenceError", "HouseholdJobStatus", "HouseholdJobStore", + "IdleBudgetWindow", "MissingPolicyParametersError", "NewBudgetWindowReport", "NewEconomyReport", diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py index f939b5aac..b0fce9940 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py @@ -14,9 +14,10 @@ are left out, and household simulations too: the check can't rebuild their calls yet. ``list_idle_budget_windows`` lists, apart, the budget windows that - stopped: no year is under way and years are still waiting, untouched, - since a time in the window. ``read_earliest_waiting_year`` names the year - a restart starts the window from, and marks. + stopped (``IdleBudgetWindow``): no year is under way and years are still + waiting, untouched, since a time in the window. + ``read_earliest_waiting_year`` names the year a restart starts the window + from, and marks just before its first start. 2. ``mark_restart`` sets the row's ``updated_at`` before the check starts its call, so the row is never listed again and is restarted at most once. It uses the statement's clock rather than the transaction's, so the mark @@ -37,7 +38,7 @@ from dataclasses import dataclass from enum import StrEnum -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, ClassVar, Final from sqlalchemy import and_, exists, func, or_, select, update @@ -85,8 +86,9 @@ class RestartKind(StrEnum): HOUSEHOLD_JOB = "household_job" REPORT_RUN = "report_run" SIMULATION_RUN = "simulation_run" - # A budget window none of whose years is under way: a restart starts its - # earliest waiting years, as the service starts a new window's. + # A budget window none of whose years is under way (``IdleBudgetWindow``): + # a restart starts its earliest waiting years, as the service starts a + # new window's. BUDGET_WINDOW = "budget_window" @@ -100,17 +102,27 @@ class RestartKind(StrEnum): @dataclass(frozen=True) class UnstartedRun: - """A row still pending with no Modal call, and since when it has waited. - - ``created_at`` is a row's creation, and a budget window's last change: - the latest change of the window or any of its years. - """ + """A row still pending with no Modal call, and when it was created.""" kind: RestartKind run_id: UUID created_at: datetime +@dataclass(frozen=True) +class IdleBudgetWindow: + """A budget window that stopped with years still waiting, and when. + + ``last_changed_at`` is the latest ``updated_at`` of the window and its + years. A window is restarted as a whole, never as one row, so it is a + type of its own, kept apart from the row-only reads and writes. + """ + + kind: ClassVar[RestartKind] = RestartKind.BUDGET_WINDOW + run_id: UUID + last_changed_at: datetime + + class StoredRunUnreadableError(LookupError): """A run's stored rows no longer describe one calculation to start.""" @@ -299,7 +311,7 @@ def list_idle_budget_windows( idle_after: datetime, idle_before: datetime, limit: int, -) -> tuple[UnstartedRun, ...]: +) -> tuple[IdleBudgetWindow, ...]: """Return the stopped budget windows last changed in the window, oldest first. A window has stopped when it is pending or running, none of its years is @@ -337,9 +349,7 @@ def list_idle_budget_windows( .limit(limit) ) return tuple( - UnstartedRun( - kind=RestartKind.BUDGET_WINDOW, run_id=row.id, created_at=row.last_change - ) + IdleBudgetWindow(run_id=row.id, last_changed_at=row.last_change) for row in connection.execute(statement) ) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_store.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_store.py index 094dd116e..9f2880065 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_store.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_store.py @@ -31,6 +31,7 @@ from sqlalchemy import Connection, Engine from policyengine_simulation_persistence.run_restart_records import ( + IdleBudgetWindow, RunCall, StoredReportRun, StoredSimulationRun, @@ -104,7 +105,7 @@ def read_report_run_simulation_run_ids( def list_idle_windows( self, *, idle_after: datetime, idle_before: datetime, limit: int - ) -> tuple[UnstartedRun, ...]: + ) -> tuple[IdleBudgetWindow, ...]: """Return the stopped budget windows last changed in the window.""" with self._transaction() as connection: return list_idle_budget_windows( diff --git a/libs/policyengine-stage12-persistence/README.md b/libs/policyengine-stage12-persistence/README.md index 686c9d944..a04bca768 100644 --- a/libs/policyengine-stage12-persistence/README.md +++ b/libs/policyengine-stage12-persistence/README.md @@ -39,10 +39,14 @@ revisions and live schema. This package never changes a schema. every year succeeded, or starts the earliest waiting years until `BUDGET_WINDOW_MAX_RUNNING_YEARS` (five) are under way, recording each call while the year is still pending with no call. A year's coordinator - call marks its window running. The stuck-run sweep leaves a waiting year - to its window, fails the window of a year it fails, and settles a window - no year advances any more. The Modal worker role and the simulation - service's role may both advance a window. + call marks its window running. The simulation service's restart check + advances a window that stopped with years still waiting, as the service + starts a new one; its starter may fail a year with a message of its own + (`YearStartRefused`), and its store bounds each statement + (`statement_timeout_milliseconds`). The stuck-run sweep leaves a waiting + year to its window, fails the window of a year it fails, and settles a + window no year advances any more. The Modal worker role and the + simulation service's role may both advance a window. ## Tests diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/__init__.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/__init__.py index 3f1bca46a..6d2834bf9 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/__init__.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/__init__.py @@ -6,6 +6,7 @@ WindowAdvance, YearStartDeferred, YearStarter, + YearStartRefused, YearToStart, ) from policyengine_stage12_persistence.report_run_results import ( @@ -48,6 +49,7 @@ "Stage12RuntimeAccessError", "WindowAdvance", "YearStartDeferred", + "YearStartRefused", "YearStarter", "YearToStart", "create_stage12_engine", diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/budget_window_store.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/budget_window_store.py index 4a025babc..dbb12d3af 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/budget_window_store.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/budget_window_store.py @@ -11,6 +11,8 @@ Each advance locks the budget-window run's row while it starts years and records their calls, so coordinators ending at once start each year once. +The simulation service's restart check advances a window that stopped, +with no year under way and years still waiting, the same way. The lock is held across those Modal calls; claims and the window's start are separate one-statement transactions that never wait on each other, so other years' coordinators wait only for the lock to be released. @@ -96,14 +98,31 @@ class YearStartDeferred(Exception): # noqa: N818 # it names an outcome, not a """ +class YearStartRefused(Exception): # noqa: N818 # it names an outcome, not a fault + """A starter could not start this year; the year records ``error_message``. + + It fails the year, and so the window, as any other error a start raises + does, but the year and its own simulation runs record the starter's + message rather than ``modal_call_refused``. The message is stored, so it + must hold no connection details. + """ + + def __init__(self, error_message: str) -> None: + """Keep the message the refused year records.""" + super().__init__(error_message) + self.error_message = error_message + + class YearStarter(Protocol): """Starts a year's coordinator call and returns the call's Modal ID. - It raises when Modal refuses the call, or ``YearStartDeferred`` to leave - the year waiting. A starter defers only once a year of the window is - under way, whose coordinator starts the waiting years when its year - ends; otherwise nothing starts them until the stuck-run sweep fails the - window as stalled. + It raises when Modal refuses the call, ``YearStartRefused`` to fail the + year with a message of its own, or ``YearStartDeferred`` to leave the + year waiting. A starter defers only once a year of the window is under + way, whose coordinator starts the waiting years when its year ends, or + when it started nothing in this advance; a window left with no year + under way is started again by the simulation service's restart check, + and failed as stalled by the stuck-run sweep when nothing starts it. """ def __call__(self, year_to_start: YearToStart, /) -> str: @@ -175,10 +194,14 @@ def __init__( *, engine: Engine | None = None, open_connection: ConnectionOpener = open_connection_once, + statement_timeout_milliseconds: int | None = None, ) -> None: """Use ``engine`` if given, else build one from ``database_url``. - Raises ValueError when neither is given or the URL is invalid. + With ``statement_timeout_milliseconds``, the database cancels any + statement that runs longer, a wait for the window's lock included, + raising OperationalError. Raises ValueError when neither an engine + nor a URL is given, or the URL is invalid. """ if engine is None: if database_url is None: @@ -186,6 +209,7 @@ def __init__( raise ValueError(msg) engine = create_stage12_engine(database_url) self.engine = engine + self.statement_timeout_milliseconds = statement_timeout_milliseconds self._open_connection = open_connection @contextmanager @@ -197,6 +221,18 @@ def _transaction(self, operation_name: str) -> Iterator[Connection]: operation_name=operation_name, ) with connection, connection.begin(): + if self.statement_timeout_milliseconds is not None: + # Local to the transaction, so it never stays on a pooled + # connection. + connection.execute( + select( + func.set_config( + "statement_timeout", + str(self.statement_timeout_milliseconds), + True, # noqa: FBT003 # set_config's is_local argument + ) + ) + ) yield connection def window_of_year(self, report_run_id: UUID) -> UUID | None: @@ -254,10 +290,11 @@ def advance_window( started until ``max_running_years`` are under way, and each started year's call is recorded while the year is still pending with no call; a call its coordinator already claimed under keeps that claim. A - start that raises fails that year as refused by Modal, and so the - window, unless the year's own call has claimed it meanwhile; a start - deferred with ``YearStartDeferred`` leaves that year and the later - ones waiting. Raises LookupError when the window run does not exist. + start that raises fails that year as refused by Modal (or with the + message of a ``YearStartRefused``), and so the window, unless the + year's own call has claimed it meanwhile; a start deferred with + ``YearStartDeferred`` leaves that year and the later ones waiting. + Raises LookupError when the window run does not exist. """ with self._transaction("advance_window") as connection: window = _lock_window(connection, window_run_id) @@ -446,8 +483,9 @@ def _start_waiting_years( # noqa: PLR0913 # the window's locked state, the sta A start that raises fails that year and the window, and starts nothing more, unless the year's own call claimed it meanwhile (Modal accepted the call after the starter stopped waiting): that year is under way, and - the next ones start. A start deferred with ``YearStartDeferred`` leaves - that year and the later ones waiting. + the next ones start. The failed year records a ``YearStartRefused``'s + message, or else that Modal refused the call. A start deferred with + ``YearStartDeferred`` leaves that year and the later ones waiting. """ started: list[int] = [] for year in [year for year in years if year.waiting][: max(free_places, 0)]: @@ -468,7 +506,9 @@ def _start_waiting_years( # noqa: PLR0913 # the window's locked state, the sta refused_years = _fail_waiting_years( connection, window_run_id, - CALL_REFUSED_MESSAGE.format(error_type=type(error).__name__), + error.error_message + if isinstance(error, YearStartRefused) + else CALL_REFUSED_MESSAGE.format(error_type=type(error).__name__), now=now, only=year.report_run_id, ) diff --git a/projects/policyengine-simulation-entry/README.md b/projects/policyengine-simulation-entry/README.md index 0b4058349..52dca4994 100644 --- a/projects/policyengine-simulation-entry/README.md +++ b/projects/policyengine-simulation-entry/README.md @@ -122,7 +122,8 @@ runs it every two minutes as the Modal worker role. a check, oldest first, with the budget windows that stopped 3 to 10 minutes ago (`run_restart_windows`). A window is restarted as `budget_window_submissions` starts a new one: from its earliest waiting - year's rebuilt call, with `YearStarts`. Household simulations' runs are + year's rebuilt call, with `YearStarts`, marking that year just before the + first start. Household simulations' runs are left out. After three minutes a check marks no more rows and leaves the rest for the next. - It rebuilds each call with this service's own builders diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/budget_window_submissions.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/budget_window_submissions.py index df927df32..46da41586 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/budget_window_submissions.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/budget_window_submissions.py @@ -250,8 +250,9 @@ async def _start_first_years( first in time; a later year not confirmed within the shared wait is left waiting. An advance that can't be written leaves the window pending: the report exists, so it is answered as created, and a year Modal - started claims itself, while the stuck-run sweep fails a window no - year advances. + started claims itself. A window no year advances is started again by + the restart check (``run_restart_windows``), and failed as stalled by + the stuck-run sweep when that start is lost too. """ start_year = YearStarts( worker=worker, diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py index 068cceee3..d184dc0c9 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py @@ -67,6 +67,7 @@ HouseholdCalculationInputs, HouseholdJobRecord, HouseholdJobStatus, + IdleBudgetWindow, NewHouseholdJob, RunCall, RunVersions, @@ -158,7 +159,7 @@ def read_call(self, run: UnstartedRun) -> RunCall | None: def list_idle_windows( self, *, idle_after: datetime, idle_before: datetime, limit: int - ) -> tuple[UnstartedRun, ...]: + ) -> tuple[IdleBudgetWindow, ...]: """Return the stopped budget windows last changed in the window.""" ... @@ -465,8 +466,8 @@ def report_calculation( A budget window is restarted from its earliest waiting year's call, and the service's year starter derives each later year's from it. Raise DeferredRestartError or CallNotRebuiltError as ``build`` does, - the latter naming no simulation runs: a window's years are failed by - the window, never by the restart. + the latter naming no simulation runs: a window's year is failed + through the window's advance, which fails its own simulation runs. """ try: stored = self._restarts.read_report_run(report_run_id) diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py index 981c46e02..1cda73c6c 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py @@ -10,19 +10,20 @@ - each check is one ``v2_run_restart_check`` operation, with the number of rows it listed and how each ended (``restart_listed``, ``restart_failed`` and the rest) set on it; -- each row it takes up is one ``v2_run_restart`` operation, with the row's - kind, ID and age. A row left ``deferred`` is taken up again by the next +- each row or budget window it takes up is one ``v2_run_restart`` + operation, with its kind, ID and age (for a window, the time since its + last change). A row left ``deferred`` is taken up again by the next check, so the operation count metric counts restart attempts, and the distinct ``run_id`` values are the runs whose call never started that a check took up. A row no check took up (more rows at once than the checks in its window had time for) is failed by the sweep with ``run_not_started`` instead; a count of the outcomes other than ``deferred`` also leaves out the rows deferred in every check. -- each row's outcome is set on its operation (``restart_outcome``, with +- each outcome is set on its operation (``restart_outcome``, with ``restart_error_type`` when an error ended it) and logged as a - ``v2_run_restart_outcome`` event, at WARNING when the row could not be - started again or its rows could not be read; an unexpected error - rebuilding a row's call is also logged with its traceback + ``v2_run_restart_outcome`` event, at WARNING when the row or window could + not be started again or its rows could not be read; an unexpected error + in a restart is also logged with its traceback (``v2_run_restart_unexpected_error``). This module imports none of the web service, so the sweep's image, which @@ -69,10 +70,12 @@ from collections.abc import Mapping from contextlib import AbstractContextManager - from policyengine_simulation_persistence import UnstartedRun from sqlalchemy import Engine - from policyengine_simulation_entry.run_restarts import RestartSummary + from policyengine_simulation_entry.run_restarts import ( + RestartSummary, + RestartTarget, + ) # The longest any one statement of a check may take. RESTART_STATEMENT_TIMEOUT_MILLISECONDS: Final = 5_000 @@ -124,9 +127,9 @@ def __init__(self, runtime: RestartRuntime) -> None: self._runtime = runtime def restart_scope( - self, run: UnstartedRun, *, age_seconds: float + self, run: RestartTarget, *, age_seconds: float ) -> AbstractContextManager[Any]: - """Return the operation one row's restart runs in.""" + """Return the operation one row's or window's restart runs in.""" return self._runtime.operation( V2_RUN_RESTART_STAGES.name(Stage.V2_RUN_RESTART), attributes={ @@ -138,7 +141,7 @@ def restart_scope( def record_outcome( self, - run: UnstartedRun, + run: RestartTarget, outcome: RestartOutcome, *, error_type: str | None = None, @@ -216,7 +219,9 @@ def scheduled_restart_service( households=HouseholdJobStore.from_engine( engine, statement_timeout_milliseconds=timeout ), - windows=BudgetWindowStore(engine=engine), + windows=BudgetWindowStore( + engine=engine, statement_timeout_milliseconds=timeout + ), ), workers=RestartWorkers( economy=ManifestEconomyWorkers.from_modal( diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_outcomes.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_outcomes.py index 461eb6240..7720388e0 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_outcomes.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_outcomes.py @@ -1,29 +1,56 @@ -"""How the restart check ended for each row it took up.""" +"""How the restart check ended for each row and budget window it took up.""" from __future__ import annotations +import logging from enum import StrEnum +from typing import TYPE_CHECKING, Final + +if TYPE_CHECKING: + from policyengine_simulation_persistence import IdleBudgetWindow, UnstartedRun + +logger = logging.getLogger(__name__) + +# What one restart takes up: a row whose call never started, or a budget +# window that stopped with years still waiting. +type RestartTarget = UnstartedRun | IdleBudgetWindow + +# The code and summary a row the check could not start again records. +NOT_STARTED_MESSAGE: Final = ( + "run_not_started: The run stayed pending with no Modal call recorded for it" +) class RestartOutcome(StrEnum): - """How the check ended for one listed row. + """How the check ended for one listed row or budget window. - ``restarted``: its new call started, and the row records that call. + ``restarted``: its new call started, and the row records that call; a + window's advance started at least one of its years. ``call_not_recorded``: its new call started, but the check couldn't - record or read the call's ID; the call claims the row itself. + record or read the call's ID; the call claims the row itself. So too a + window whose advance failed after Modal accepted a year's call. ``started_meanwhile``: the row records another call, which claimed it after the check marked it (or the check's own call, accepted after the check stopped waiting for it); the other call then can't claim it. ``left``: the row was gone, changed before the check started a call, or failed meanwhile; also a marked row whose failure couldn't be written, which the sweep fails later, and a row whose call couldn't be read - after Modal didn't confirm the check's call. ``deferred``: the row is - left unmarked for the next check, because its rows or the manifest - couldn't be read, rebuilding its call raised an unexpected error (logged - with its traceback), the failure of a row that can't be rebuilt couldn't - be written, the mark couldn't be written, or the check's time ran out. + after Modal didn't confirm the check's call. A window is left when it is + gone, has no year waiting untouched, finished or was failed for an + earlier failed year before the check started anything, or its year + changed before the mark; and when its marked year's failure couldn't be + written, for the sweep to fail as stalled. + ``deferred``: the row or window is left unmarked for the next check, + because its rows or the manifest couldn't be read, its restart raised + an unexpected error before any mark (logged with its traceback), the + failure of a row that can't be rebuilt couldn't be written, the mark + couldn't be written, a window's lock or reads failed before any start, + or the check's time ran out. ``failed``: it couldn't be started again and was failed with - ``run_not_started``. + ``run_not_started``. A window fails when Modal refuses one of its years + (``modal_call_refused``), or its first year's call can't be rebuilt + (that year records ``run_not_started`` and why); the window then + records which year failed. """ RESTARTED = "restarted" @@ -32,3 +59,14 @@ class RestartOutcome(StrEnum): LEFT = "left" DEFERRED = "deferred" FAILED = "failed" + + +def log_unexpected_error(target: RestartTarget) -> None: + """Log the error being handled, with its traceback, naming the row or window.""" + fields = {"run_kind": target.kind.value, "run_id": str(target.run_id)} + # The observability runtime's log handler writes only + # ``policyengine_attributes``, as route_support's own log does. + logger.exception( + "v2_run_restart_unexpected_error", + extra={**fields, "policyengine_attributes": fields}, + ) diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_windows.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_windows.py index 438371045..7a85cf496 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_windows.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_windows.py @@ -10,24 +10,34 @@ The restart check starts a stopped window again, as the service starts a new window's first years. ``WindowRestarter`` rebuilds the earliest waiting -year's coordinator call from its stored rows, marks that year, so the window -is restarted once each time it stops, and advances the window +year's coordinator call from its stored rows, then advances the window (``BudgetWindowStore.advance_window``) with the service's own year starter (``YearStarts``). That starter derives each later year's call from the first's, and shares one wait for Modal among them; under the window's row lock, the earliest years are started, at most five under way, and their -calls recorded. A year Modal refuses fails the window, as it would at the -service's start. A window whose first call can't be rebuilt is marked and -left, and the sweep fails it as stalled. +calls recorded. + +Just before the advance's first start, the restart marks the earliest +waiting year, so the window is restarted once each time it stops. An +advance whose lock or reads fail starts and marks nothing, so the next +check takes the window up again. A year Modal refuses fails the window, as +it would at the service's start. A window whose first call can't be +rebuilt is failed at once, through the same advance: that year records +``run_not_started`` and why, as a row the check can't rebuild is failed. """ from __future__ import annotations import asyncio +from functools import partial from typing import TYPE_CHECKING from policyengine_simulation_persistence import RestartKind, UnstartedRun -from policyengine_stage12_persistence import RunStatus +from policyengine_stage12_persistence import ( + RunStatus, + YearStartDeferred, + YearStartRefused, +) from sqlalchemy.exc import SQLAlchemyError from policyengine_simulation_entry.budget_window_submissions import YearStarts @@ -35,40 +45,130 @@ CallNotRebuiltError, DeferredRestartError, ) -from policyengine_simulation_entry.run_restart_outcomes import RestartOutcome +from policyengine_simulation_entry.run_restart_outcomes import ( + NOT_STARTED_MESSAGE, + RestartOutcome, + log_unexpected_error, +) if TYPE_CHECKING: from collections.abc import Callable from datetime import datetime - from policyengine_simulation_persistence import WaitingYear - from policyengine_stage12_persistence import WindowAdvance + from policyengine_simulation_persistence import IdleBudgetWindow + from policyengine_stage12_persistence import WindowAdvance, YearToStart from policyengine_simulation_entry.budget_window_submissions import ( BudgetWindowAdvancer, ) - from policyengine_simulation_entry.economy_calculation_inputs import ( - RunCalculation, - ) - from policyengine_simulation_entry.economy_workers import ( - EconomyRunCaller, - EconomyWorker, - ) + from policyengine_simulation_entry.economy_workers import EconomyRunCaller from policyengine_simulation_entry.run_restart_calls import ( RestartCallBuilder, UnstartedRunSource, ) from policyengine_simulation_entry.run_restarts import RestartTelemetry +type _Ended = tuple[RestartOutcome, str | None] + -class _RestartEnded(Exception): # noqa: N818 # it names an outcome, not a fault - """The window's restart ended early, with this outcome.""" +class _MarkedYearStarts: + """Start a window's years with ``YearStarts``, marking the restart first. - def __init__(self, outcome: RestartOutcome, error_type: str | None = None) -> None: - """Keep the outcome and the error type that ended it, if any.""" - super().__init__(outcome.value) - self.outcome = outcome - self.error_type = error_type + Before the first start, it marks the earliest waiting year. A mark that + changes nothing (the year changed meanwhile) or can't be written defers + the year, so the advance starts nothing more. It records what happened, + so the restart can say how it ended. + """ + + def __init__(self, *, mark: Callable[[], bool], start_year: YearStarts) -> None: + """Mark with ``mark``, then start each year with ``start_year``.""" + self._mark = mark + self._start_year = start_year + self.marked = False + self.mark_error_type: str | None = None + self.start_error_type: str | None = None + self.calls_started = 0 + + def __call__(self, year_to_start: YearToStart, /) -> str: + """Start the year's coordinator and return its Modal call ID.""" + if not self.marked: + try: + self.marked = self._mark() + except SQLAlchemyError as error: + self.mark_error_type = type(error).__name__ + raise YearStartDeferred from None + if not self.marked: + raise YearStartDeferred + try: + call_id = self._start_year(year_to_start) + except YearStartDeferred: + raise + except Exception as error: + # The advance fails the year as refused, and logs it. + self.start_error_type = type(error).__name__ + raise + self.calls_started += 1 + return call_id + + def ended(self, advance: WindowAdvance) -> _Ended: + """Return how the restart ended, from the advance it made.""" + if self.mark_error_type is not None: + return RestartOutcome.DEFERRED, self.mark_error_type + if not self.marked: + # Nothing was started: the window finished, or a failed year + # failed it, or its year changed before the mark. + return RestartOutcome.LEFT, None + if advance.status is RunStatus.FAILED: + return RestartOutcome.FAILED, self.start_error_type + if advance.started_years: + return RestartOutcome.RESTARTED, None + return RestartOutcome.LEFT, None + + def ended_by_error(self) -> RestartOutcome: + """Return how the restart ended when its advance raised.""" + if self.calls_started: + # A year whose call Modal accepted claims itself. + return RestartOutcome.CALL_NOT_RECORDED + if self.marked: + # Its failure wasn't written; the sweep fails the window. + return RestartOutcome.LEFT + return RestartOutcome.DEFERRED + + +class _RefusedFirstYear: + """Fail the first year offered with why its call can't be rebuilt. + + The advance fails that year, and so the window, unless the year's own + call claimed it meanwhile; any later year is deferred to its + coordinator. Nothing is marked, so a failure that isn't written leaves + the window for the next check. + """ + + def __init__(self, cannot: CallNotRebuiltError) -> None: + """Refuse with the reason ``cannot`` gives.""" + self._error_message = ( + f"{NOT_STARTED_MESSAGE}, and its call could not be built again: " + f"{cannot.reason}." + ) + self._error_type = cannot.error_type + self.offered = False + + def __call__(self, year_to_start: YearToStart, /) -> str: # noqa: ARG002 # the YearStarter protocol's argument + """Refuse the first year offered; defer any later one.""" + if self.offered: + raise YearStartDeferred + self.offered = True + raise YearStartRefused(self._error_message) + + def ended(self, advance: WindowAdvance) -> _Ended: + """Return how the restart ended, from the advance it made.""" + if self.offered and advance.status is RunStatus.FAILED: + return RestartOutcome.FAILED, self._error_type + return RestartOutcome.LEFT, None + + def ended_by_error(self) -> RestartOutcome: + """Return how the restart ended when its advance raised.""" + return RestartOutcome.DEFERRED class WindowRestarter: @@ -96,106 +196,84 @@ def __init__( # noqa: PLR0913 # the service's seams this restart shares self._out_of_time = out_of_time self._call_start_timeout_seconds = call_start_timeout_seconds - async def restart(self, window: UnstartedRun) -> tuple[RestartOutcome, str | None]: - """Start the window's earliest waiting years; return how it ended.""" - try: - waiting, built = await self._first_year_call(window) - worker, calculation = await self._mark(window, waiting, built) - advance = await self._advance(window, waiting, worker, calculation) - except _RestartEnded as ended: - return ended.outcome, ended.error_type - if advance.status is RunStatus.FAILED: - return RestartOutcome.FAILED, None - if advance.started_years: - return RestartOutcome.RESTARTED, None - return RestartOutcome.LEFT, None - - async def _first_year_call( - self, window: UnstartedRun - ) -> tuple[WaitingYear, tuple[EconomyWorker, RunCalculation] | CallNotRebuiltError]: - """Return the earliest waiting year and its rebuilt call, or why there's none. + async def restart(self, window: IdleBudgetWindow) -> _Ended: + """Start the window's earliest waiting years; return how it ended. - A call that can't be rebuilt is returned, not raised, so the year is - still marked. + Any error no step expects is logged with its traceback, and the + window is left as its marks and starts so far leave it. """ + starts: _MarkedYearStarts | _RefusedFirstYear | None = None + try: + prepared = await self._starts(window) + if isinstance(prepared, tuple): + return prepared + starts = prepared + return await self._advance(window, starts) + except Exception as error: # noqa: BLE001 # one window's fault never ends the check + log_unexpected_error(window) + outcome = ( + RestartOutcome.DEFERRED if starts is None else starts.ended_by_error() + ) + return outcome, type(error).__name__ + + async def _starts( + self, window: IdleBudgetWindow + ) -> _MarkedYearStarts | _RefusedFirstYear | _Ended: + """Return the earliest waiting year's starter, or how the restart ended.""" try: waiting = await asyncio.to_thread( self._restarts.read_earliest_waiting_year, window.run_id ) except SQLAlchemyError as error: - raise _RestartEnded(RestartOutcome.DEFERRED, type(error).__name__) from None + return RestartOutcome.DEFERRED, type(error).__name__ if waiting is None: - raise _RestartEnded(RestartOutcome.LEFT) + return RestartOutcome.LEFT, None try: built = await asyncio.to_thread( self._calls.report_calculation, waiting.report_run_id ) except DeferredRestartError as deferred: - raise _RestartEnded(RestartOutcome.DEFERRED, str(deferred)) from None + return RestartOutcome.DEFERRED, str(deferred) except CallNotRebuiltError as cannot: - return waiting, cannot + return _RefusedFirstYear(cannot) if built is None: - raise _RestartEnded(RestartOutcome.LEFT) - return waiting, built - - async def _mark( - self, - window: UnstartedRun, - waiting: WaitingYear, - built: tuple[EconomyWorker, RunCalculation] | CallNotRebuiltError, - ) -> tuple[EconomyWorker, RunCalculation]: - """Mark the earliest waiting year, so the window restarts once per stop. - - Return the year's rebuilt call. Raise _RestartEnded when time is up, - the year changed meanwhile, the mark can't be written, or the year's - call can't be rebuilt (once marked). - """ - if self._out_of_time(): - raise _RestartEnded(RestartOutcome.DEFERRED) + return RestartOutcome.LEFT, None + worker, calculation = built first_year = UnstartedRun( kind=RestartKind.REPORT_RUN, run_id=waiting.report_run_id, - created_at=window.created_at, + # The mark reads only the row's kind and ID. + created_at=window.last_changed_at, + ) + return _MarkedYearStarts( + mark=partial(self._restarts.mark_restart, first_year), + start_year=YearStarts( + worker=worker, + caller=self._caller, + first_year=waiting.year, + first_calculation=calculation, + observability_context=self._telemetry.capture_context(), + loop=asyncio.get_running_loop(), + timeout_seconds=self._call_start_timeout_seconds, + clock=self._clock, + ), ) - try: - marked = await asyncio.to_thread(self._restarts.mark_restart, first_year) - except SQLAlchemyError as error: - raise _RestartEnded(RestartOutcome.DEFERRED, type(error).__name__) from None - if not marked: - raise _RestartEnded(RestartOutcome.LEFT) - if isinstance(built, CallNotRebuiltError): - raise _RestartEnded(RestartOutcome.LEFT, built.error_type) - return built async def _advance( - self, - window: UnstartedRun, - waiting: WaitingYear, - worker: EconomyWorker, - calculation: RunCalculation, - ) -> WindowAdvance: - """Advance the window with the service's year starter, from the first year.""" - start_year = YearStarts( - worker=worker, - caller=self._caller, - first_year=waiting.year, - first_calculation=calculation, - observability_context=self._telemetry.capture_context(), - loop=asyncio.get_running_loop(), - timeout_seconds=self._call_start_timeout_seconds, - clock=self._clock, - ) + self, window: IdleBudgetWindow, starts: _MarkedYearStarts | _RefusedFirstYear + ) -> _Ended: + """Advance the window with ``starts``; return how the restart ended.""" + if self._out_of_time(): + return RestartOutcome.DEFERRED, None try: - return await asyncio.to_thread( + advance = await asyncio.to_thread( self._windows.advance_window, window.run_id, now=self._clock(), - start_year=start_year, + start_year=starts, ) except SQLAlchemyError as error: - # A year whose call Modal accepted claims itself. - raise _RestartEnded( - RestartOutcome.CALL_NOT_RECORDED, type(error).__name__ - ) from None + return starts.ended_by_error(), type(error).__name__ except LookupError: - raise _RestartEnded(RestartOutcome.LEFT) from None + return RestartOutcome.LEFT, None + return starts.ended(advance) diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py index 6b051e393..0c5642b62 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py @@ -10,11 +10,13 @@ 1. it lists the rows created 3 to 10 minutes ago that are still pending with no call and that nothing has changed since their insert, and the budget - windows that stopped 3 to 10 minutes ago (``run_restart_windows``); + windows that stopped 3 to 10 minutes ago, and takes them up oldest first; 2. for each, it reads back what the service built the call from and builds the same call, on the worker of the .py version the row records; 3. it marks the row, so it is never restarted twice, starts the call, and - records the call's ID while the row is still pending with none. + records the call's ID while the row is still pending with none. A + window is advanced instead, from its earliest waiting year's call + (``run_restart_windows``). A row whose call can't be built again (its worker left the version manifest, or its stored rows changed), or whose new call Modal refuses, is failed at @@ -29,15 +31,15 @@ earlier call's claim is refused; if the earlier call claimed it first, the restart's call is refused instead. -Each listed row is one telemetry scope, ``restart_scope``, with its age, and -its outcome is reported through ``record_outcome``, so how often runs need a -restart, and how each restart ended, can be counted. +Each listed row or window is one telemetry scope, ``restart_scope``, with +how long it has waited, and its outcome is reported through +``record_outcome``, so how often runs need a restart, and how each restart +ended, can be counted. """ from __future__ import annotations import asyncio -import logging import time import uuid from collections import Counter @@ -48,7 +50,7 @@ from itertools import islice from typing import TYPE_CHECKING, Final, Protocol -from policyengine_simulation_persistence import RestartKind +from policyengine_simulation_persistence import IdleBudgetWindow, RestartKind from sqlalchemy.exc import SQLAlchemyError from policyengine_simulation_entry.run_restart_calls import ( @@ -61,7 +63,12 @@ RestartWorkers, UnstartedRunSource, ) -from policyengine_simulation_entry.run_restart_outcomes import RestartOutcome +from policyengine_simulation_entry.run_restart_outcomes import ( + NOT_STARTED_MESSAGE, + RestartOutcome, + RestartTarget, + log_unexpected_error, +) from policyengine_simulation_entry.run_restart_windows import WindowRestarter if TYPE_CHECKING: @@ -93,13 +100,12 @@ "RestartSettings", "RestartStores", "RestartSummary", + "RestartTarget", "RestartTelemetry", "RestartWorkers", "RunRestartService", ] -logger = logging.getLogger(__name__) - # A row is restarted once it has stayed pending with no call this long, which # is far longer than a submission takes to start and record its call. RESTART_AFTER: Final = timedelta(minutes=3) @@ -117,9 +123,6 @@ # bounded statements, so the check ends within the scheduled function's # timeout and no row it marks is left without its call. CHECK_TIME_LIMIT: Final = timedelta(minutes=3) -_NOT_STARTED: Final = ( - "run_not_started: The run stayed pending with no Modal call recorded for it" -) @dataclass(frozen=True) @@ -170,19 +173,19 @@ class RestartTelemetry(Protocol): """Where the check reports each listed row (``ObservabilityRestartTelemetry``).""" def restart_scope( - self, run: UnstartedRun, *, age_seconds: float + self, run: RestartTarget, *, age_seconds: float ) -> AbstractContextManager[None]: - """Return the scope one row's restart runs in.""" + """Return the scope one row's or window's restart runs in.""" ... def record_outcome( self, - run: UnstartedRun, + run: RestartTarget, outcome: RestartOutcome, *, error_type: str | None = None, ) -> None: - """Report how the row's restart ended, and the error type that ended it.""" + """Report how the restart ended, and the error type that ended it.""" ... def capture_context(self) -> dict[str, str] | None: @@ -195,7 +198,7 @@ class NoRestartTelemetry: def restart_scope( self, - run: UnstartedRun, # noqa: ARG002 # the protocol's argument + run: RestartTarget, # noqa: ARG002 # the protocol's argument *, age_seconds: float, # noqa: ARG002 # the protocol's argument ) -> AbstractContextManager[None]: @@ -204,7 +207,7 @@ def restart_scope( def record_outcome( self, - run: UnstartedRun, + run: RestartTarget, outcome: RestartOutcome, *, error_type: str | None = None, @@ -295,18 +298,15 @@ async def restart_unstarted_runs(self) -> RestartSummary: limit=RESTARTS_PER_CHECK, ) # Each listing is oldest first; together, oldest first too. - runs = list( - islice( - merge(rows, windows, key=lambda run: run.created_at), - RESTARTS_PER_CHECK, - ) + runs: list[RestartTarget] = list( + islice(merge(rows, windows, key=_waiting_since), RESTARTS_PER_CHECK) ) outcomes: Counter[RestartOutcome] = Counter() for position, run in enumerate(runs): if self._out_of_time(): outcomes[RestartOutcome.DEFERRED] += len(runs) - position break - age_seconds = (now - run.created_at).total_seconds() + age_seconds = (now - _waiting_since(run)).total_seconds() with self._telemetry.restart_scope(run, age_seconds=age_seconds): outcome, error_type = await self._restart(run) self._telemetry.record_outcome(run, outcome, error_type=error_type) @@ -319,9 +319,9 @@ async def restart_unstarted_runs(self) -> RestartSummary: def _out_of_time(self) -> bool: return self._timer() >= self._deadline - async def _restart(self, run: UnstartedRun) -> tuple[RestartOutcome, str | None]: + async def _restart(self, run: RestartTarget) -> tuple[RestartOutcome, str | None]: """Start the row's call once more; return how it ended and any error type.""" - if run.kind is RestartKind.BUDGET_WINDOW: + if isinstance(run, IdleBudgetWindow): return await self._windows.restart(run) try: rebuilt = await asyncio.to_thread(self._calls.build, run) @@ -329,7 +329,7 @@ async def _restart(self, run: UnstartedRun) -> tuple[RestartOutcome, str | None] return RestartOutcome.DEFERRED, str(deferred) except CallNotRebuiltError as cannot: message = ( - f"{_NOT_STARTED}, and its call could not be built again: " + f"{NOT_STARTED_MESSAGE}, and its call could not be built again: " f"{cannot.reason}." ) failed = self._fail(run, message, cannot.simulation_run_ids) @@ -342,14 +342,8 @@ async def _restart(self, run: UnstartedRun) -> tuple[RestartOutcome, str | None] return outcome, cannot.error_type # Any other error leaves this row unmarked for the next check, and # the check goes on to the next row; its traceback is logged. - except Exception as error: - fields = {"run_kind": run.kind.value, "run_id": str(run.run_id)} - # The observability runtime's log handler writes only - # ``policyengine_attributes``, as route_support's own log does. - logger.exception( - "v2_run_restart_unexpected_error", - extra={**fields, "policyengine_attributes": fields}, - ) + except Exception as error: # noqa: BLE001 # logged with its traceback by log_unexpected_error + log_unexpected_error(run) return RestartOutcome.DEFERRED, type(error).__name__ if rebuilt is None: return RestartOutcome.LEFT, None @@ -378,8 +372,8 @@ async def _start_marked( # call was not confirmed, and the row is failed unless it has started. except Exception as error: # noqa: BLE001 # every unconfirmed call is answered alike message = ( - f"{_NOT_STARTED}, and Modal did not accept the call that started it " - f"again ({type(error).__name__})." + f"{NOT_STARTED_MESSAGE}, and Modal did not accept the call that " + f"started it again ({type(error).__name__})." ) failed = self._fail(run, message, simulation_run_ids) # A failure that changed nothing means a call holds the row now, @@ -449,3 +443,10 @@ def _record(self, run: UnstartedRun, call_id: str) -> None: self._stores.economy.record_simulation_call(run.run_id, call_id) else: self._stores.households.record_call(run.run_id, call_id) + + +def _waiting_since(run: RestartTarget) -> datetime: + """Return since when the row or window has waited: its creation or last change.""" + if isinstance(run, IdleBudgetWindow): + return run.last_changed_at + return run.created_at diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/budget_window_scheduling.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/budget_window_scheduling.py index f4d3b66e2..856156a51 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/budget_window_scheduling.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/budget_window_scheduling.py @@ -8,9 +8,10 @@ the window failed or succeeded (see ``BudgetWindowStore``). Neither step changes the year's own outcome. A failed write is logged, -without the error's text, and the stuck-run sweep finishes a window that -no year advances any more. A pre-emption or cancellation propagates, as -everywhere in the coordinator. +without the error's text. The simulation service's restart check starts a +window that stopped with years still waiting again, and the stuck-run sweep +finishes a window that nothing advances any more. A pre-emption or +cancellation propagates, as everywhere in the coordinator. """ from __future__ import annotations From 52329602b8f2c50c9f33ec69243445c329fd71f3 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:00:37 +0400 Subject: [PATCH 24/29] Test review round 7's budget-window restart findings (failing) A refusal fails only the year whose call couldn't be built again; an unexpected error from the mark defers the window and is logged; a window's year is marked by itself, not as a stand-in row. Also covers a refusal its year's own call overtook, a refusal that can't be written, and a starter that writes its own year during the advance. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_run_restart_records.py | 16 +-- .../tests/test_budget_window_store.py | 44 +++++++- .../tests/test_run_restart_windows.py | 100 ++++++++++++++++-- .../tests/test_run_restarts.py | 9 ++ 4 files changed, 151 insertions(+), 18 deletions(-) diff --git a/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py b/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py index a43c3192d..cd1d6bd9a 100644 --- a/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py +++ b/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py @@ -49,6 +49,7 @@ list_idle_budget_windows, list_unstarted_runs, mark_restart, + mark_waiting_year, read_earliest_waiting_year, read_report_run_simulation_run_ids, read_run_call, @@ -622,15 +623,14 @@ def test_a_window_whose_earliest_waiting_year_was_marked_is_not_listed( insert_budget_window_report(connection, window) created_at = _transaction_start(connection) act_as_the_modal_worker(connection) - first_year = UnstartedRun( - kind=RestartKind.REPORT_RUN, - run_id=window.years[0].report_run_id, - created_at=created_at, - ) + first_year = WaitingYear(report_run_id=window.years[0].report_run_id, year=2026) - assert mark_restart(connection, first_year) is True + assert mark_waiting_year(connection, first_year) is True + assert mark_waiting_year(connection, first_year) is False marked_at = connection.execute( - select(report_runs.c.updated_at).where(report_runs.c.id == first_year.run_id) + select(report_runs.c.updated_at).where( + report_runs.c.id == first_year.report_run_id + ) ).scalar_one() assert _idle_windows(connection, around=created_at) == () assert _idle_windows(connection, around=marked_at) == () @@ -714,6 +714,7 @@ def test_the_store_lists_marks_and_reads_in_transactions_of_its_own( limit=LISTING_LIMIT, ) waiting = savepoint_store.read_earliest_waiting_year(window.window_run_id) + year_marked = waiting is not None and savepoint_store.mark_waiting_year(waiting) sides = savepoint_store.read_report_run_simulation_run_ids(report.run_id) assert set(by_kind) == {RestartKind.REPORT_RUN, RestartKind.SIMULATION_RUN} @@ -722,6 +723,7 @@ def test_the_store_lists_marks_and_reads_in_transactions_of_its_own( assert waiting == WaitingYear( report_run_id=window.years[0].report_run_id, year=2026 ) + assert year_marked is True assert sides == ( report.baseline.simulation_run_id, report.reform.simulation_run_id, diff --git a/libs/policyengine-stage12-persistence/tests/test_budget_window_store.py b/libs/policyengine-stage12-persistence/tests/test_budget_window_store.py index 5f80c543b..af958f37a 100644 --- a/libs/policyengine-stage12-persistence/tests/test_budget_window_store.py +++ b/libs/policyengine-stage12-persistence/tests/test_budget_window_store.py @@ -36,8 +36,9 @@ YearToStart, budget_window_store, ) +from policyengine_stage12_persistence.report_run_tables import report_runs from report_run_database import NOW, SEED_URL_ENV, worker_database_url -from sqlalchemy import Engine, create_engine, event +from sqlalchemy import Engine, create_engine, event, func, select, update from sqlalchemy.exc import OperationalError # The simulation service's role, which the Public API's manifest grants. @@ -394,6 +395,47 @@ def hold_the_lock(year_to_start: YearToStart) -> str: bounded.engine.dispose() +def test_a_starter_writes_its_own_year_on_another_connection_without_waiting( + store, windows, window +) -> None: + """The advance locks only the window's row, not the years it starts. + + The restart check marks the year it restarts a window from on its own + connection, from the starter, before that year's start; a lock on the + year would hold the mark until its statement timeout. + """ + starter = RecordingStarter() + first_year = window.year(2026).report_run_id + marks: list[UUID | None] = [] + + def mark_then_start(year_to_start: YearToStart) -> str: + if not marks: + with store.engine.begin() as connection: + connection.execute( + select(func.set_config("statement_timeout", "2000", True)) # noqa: FBT003 # set_config's is_local argument + ) + marks.append( + connection.execute( + update(report_runs) + .where( + report_runs.c.id == first_year, + report_runs.c.modal_call_id.is_(None), + ) + .values(updated_at=func.clock_timestamp()) + .returning(report_runs.c.id) + ).scalar_one_or_none() + ) + return starter(year_to_start) + + advance = store.advance_window( + window.window_run_id, now=LATER, start_year=mark_then_start + ) + + assert marks == [first_year] + assert advance.started_years == (2026, 2027, 2028, 2029, 2030) + assert windows.year_rows(window)[2026]["modal_call_id"] == "fc-started-1" + + def test_the_service_role_starts_and_fails_a_window(windows, window) -> None: # The simulation service starts a window's first years with the same # store, as its own role, and fails the window when Modal refuses one. diff --git a/projects/policyengine-simulation-entry/tests/test_run_restart_windows.py b/projects/policyengine-simulation-entry/tests/test_run_restart_windows.py index eb79354f5..7119468b2 100644 --- a/projects/policyengine-simulation-entry/tests/test_run_restart_windows.py +++ b/projects/policyengine-simulation-entry/tests/test_run_restart_windows.py @@ -98,7 +98,10 @@ class FakeWindows: as for a window that finished, or that a failed year fails now. ``failure`` is raised before any year is offered, as when the window's lock or reads fail, and ``failure_after_starts`` after every start, as - when the commit fails. + when the commit fails. A start that raises for a year in + ``claimed_meanwhile`` counts that year started and goes on, as when the + year's own call claimed it first; ``refusal_not_written`` is raised in + place of failing the window, as when that failure can't be written. """ waiting: tuple[YearToStart, ...] = () @@ -106,6 +109,8 @@ class FakeWindows: ended: WindowAdvance | None = None failure: BaseException | None = None failure_after_starts: BaseException | None = None + claimed_meanwhile: frozenset[UUID] = frozenset() + refusal_not_written: BaseException | None = None advanced: list[UUID] = field(default_factory=list) started_calls: list[str] = field(default_factory=list) refusals: list[str] = field(default_factory=list) @@ -124,7 +129,12 @@ def advance_window( self.started_calls.append(start_year(year)) except YearStartDeferred: break - except Exception as error: # noqa: BLE001 # as the store fails a refused year + except Exception as error: # as the store fails a refused year + if year.run_ids.report_run_id in self.claimed_meanwhile: + started.append(year.year) + continue + if self.refusal_not_written is not None: + raise self.refusal_not_written from error self.refusals.append( error.error_message if isinstance(error, YearStartRefused) @@ -186,7 +196,8 @@ def _idle_window_rig(**options: Any) -> tuple[RestartRig, StoredReportRun, dict] def _marked_run_ids(rig: RestartRig) -> list[UUID]: - return [run.run_id for run in rig.restarts.marked] + """Return the report runs of the window years the check marked.""" + return [year.report_run_id for year in rig.restarts.marked_years] def test_an_idle_window_is_restarted_from_its_earliest_waiting_year() -> None: @@ -269,7 +280,7 @@ def test_a_window_no_longer_waiting_is_left_unmarked() -> None: summary = rig.restart() - assert rig.restarts.marked == [] + assert rig.restarts.marked_years == [] assert rig.windows.advanced == [] assert summary == RestartSummary(listed=1, left=1) @@ -297,7 +308,7 @@ def test_a_window_whose_first_years_call_cannot_be_rebuilt_fails_with_the_reason assert rig.windows.refusals == [NOT_REBUILT] assert rig.economy_caller.report_runs == [] - assert rig.restarts.marked == [] + assert rig.restarts.marked_years == [] assert summary == RestartSummary(listed=1, failed=1) assert rig.telemetry.outcomes == [ ( @@ -309,6 +320,74 @@ def test_a_window_whose_first_years_call_cannot_be_rebuilt_fails_with_the_reason ] +def test_a_refusal_never_fails_a_year_other_than_the_one_not_rebuilt() -> None: + """The year read was claimed before the advance, which offers the next first.""" + rig, stored, _payload = _idle_window_rig() + rig.restarts.read_failures[stored.report_run_id] = StoredRunUnreadableError( + "report run names no region" + ) + rig.windows.waiting = rig.windows.waiting[1:] + + summary = rig.restart() + + assert rig.windows.refusals == [] + assert rig.economy_caller.report_runs == [] + assert summary == RestartSummary(listed=1, left=1) + + +def test_a_refused_year_its_own_call_claimed_first_is_left() -> None: + """The year is under way after all; the later years wait for its coordinator.""" + rig, stored, _payload = _idle_window_rig() + rig.restarts.read_failures[stored.report_run_id] = StoredRunUnreadableError( + "report run names no region" + ) + rig.windows.claimed_meanwhile = frozenset({stored.report_run_id}) + + summary = rig.restart() + + assert rig.windows.refusals == [] + assert rig.economy_caller.report_runs == [] + assert summary == RestartSummary(listed=1, left=1) + + +def test_a_marked_window_whose_refusal_cannot_be_written_is_left_to_the_sweep() -> None: + rig, stored, _payload = _idle_window_rig( + economy_caller=FakeCaller(failure=ConnectionError("Modal is down")) + ) + rig.windows.refusal_not_written = DATABASE_ERROR + + summary = rig.restart() + + assert _marked_run_ids(rig) == [stored.report_run_id] + assert summary == RestartSummary(listed=1, left=1) + assert rig.telemetry.outcomes[0][3] == "OperationalError" + + +def test_an_unexpected_error_from_the_mark_defers_the_window_and_is_logged( + caplog, +) -> None: + """It is no refusal by Modal, so no year is failed for it.""" + rig, _stored, _payload = _idle_window_rig() + rig.restarts.mark_failure = RuntimeError("unforeseen") + + summary = rig.restart() + + assert rig.windows.refusals == [] + assert rig.economy_caller.report_runs == [] + assert summary == RestartSummary(listed=1, deferred=1) + assert rig.telemetry.outcomes[0][3] == "RuntimeError" + (logged,) = [ + record + for record in caplog.records + if record.getMessage() == "v2_run_restart_unexpected_error" + ] + assert isinstance(logged.exc_info[1], RuntimeError) + assert logged.policyengine_attributes == { + "run_kind": "budget_window", + "run_id": str(WINDOW_RUN_ID), + } + + def test_a_database_error_before_any_start_leaves_the_window_for_the_next_check() -> ( None ): @@ -319,7 +398,7 @@ def test_a_database_error_before_any_start_leaves_the_window_for_the_next_check( summary = rig.restart() - assert rig.restarts.marked == [] + assert rig.restarts.marked_years == [] assert rig.economy_caller.report_runs == [] assert summary == RestartSummary(listed=1, deferred=1) assert rig.telemetry.outcomes[0][2:] == ( @@ -398,7 +477,8 @@ def test_an_unexpected_error_in_a_windows_restart_defers_it_and_the_check_goes_o RestartOutcome.DEFERRED, "RuntimeError", ) - assert _marked_run_ids(rig) == [stored.report_run_id] + assert rig.restarts.marked_years == [] + assert [run.run_id for run in rig.restarts.marked] == [stored.report_run_id] assert summary == RestartSummary(listed=2, restarted=1, deferred=1) (logged,) = [ record @@ -419,7 +499,7 @@ def test_a_window_gone_before_its_advance_is_left() -> None: summary = rig.restart() - assert rig.restarts.marked == [] + assert rig.restarts.marked_years == [] assert summary == RestartSummary(listed=1, left=1) @@ -439,7 +519,7 @@ def test_a_window_that_ends_without_a_start_is_left_unmarked(ended) -> None: summary = rig.restart() - assert rig.restarts.marked == [] + assert rig.restarts.marked_years == [] assert summary == RestartSummary(listed=1, left=1) @@ -454,7 +534,7 @@ def test_a_window_the_check_has_no_time_left_for_is_left_for_the_next_check() -> summary = rig.restart() assert rig.windows.advanced == [] - assert rig.restarts.marked == [] + assert rig.restarts.marked_years == [] assert summary == RestartSummary(listed=1, deferred=1) diff --git a/projects/policyengine-simulation-entry/tests/test_run_restarts.py b/projects/policyengine-simulation-entry/tests/test_run_restarts.py index b40f72f0c..e41edaf84 100644 --- a/projects/policyengine-simulation-entry/tests/test_run_restarts.py +++ b/projects/policyengine-simulation-entry/tests/test_run_restarts.py @@ -143,6 +143,7 @@ class FakeRestartStore: holder: Callable[[UnstartedRun], RunCall | None] | None = None listings: list[tuple[datetime, datetime, int]] = field(default_factory=list) marked: list[UnstartedRun] = field(default_factory=list) + marked_years: list[WaitingYear] = field(default_factory=list) def list_idle_windows( self, *, idle_after: datetime, idle_before: datetime, limit: int @@ -177,6 +178,14 @@ def mark_restart(self, run: UnstartedRun) -> bool: self.marked.append(run) return True + def mark_waiting_year(self, year: WaitingYear) -> bool: + if self.mark_failure is not None: + raise self.mark_failure + if self.marked_elsewhere: + return False + self.marked_years.append(year) + return True + def read_report_run(self, report_run_id: UUID) -> StoredReportRun | None: if self.read_failure is not None: raise self.read_failure From 7a9de4ae4aad07270ff1ae4fb0dc2f1621ef484e Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:02:30 +0400 Subject: [PATCH 25/29] Fix review round 7's budget-window restart findings A window whose first call can't be built again refuses only that year: when the advance offers another first, the year read changed meanwhile, and nothing is failed. An error from the mark other than the database's is logged with its traceback and defers the window, rather than failing its year as refused by Modal. A window's year is marked by itself (mark_waiting_year), not as a stand-in row. Co-Authored-By: Claude Opus 5.5 --- .../stage-12-modal-worker-foundation.md | 3 +- .../README.md | 3 +- .../__init__.py | 2 + .../run_restart_records.py | 19 ++++- .../run_restart_store.py | 6 ++ .../run_restart_calls.py | 4 ++ .../run_restart_outcomes.py | 3 +- .../run_restart_windows.py | 70 +++++++++++-------- 8 files changed, 75 insertions(+), 35 deletions(-) diff --git a/docs/migration/stage-12-modal-worker-foundation.md b/docs/migration/stage-12-modal-worker-foundation.md index 22c9ae91b..a7f751b9c 100644 --- a/docs/migration/stage-12-modal-worker-foundation.md +++ b/docs/migration/stage-12-modal-worker-foundation.md @@ -829,7 +829,8 @@ window is restarted once each time it stops; an advance whose lock or reads fail marks nothing, and the next check takes the window up again. Its statements are bounded by the check's statement timeout. A window whose first call can't be rebuilt is failed at once through the same advance: that -year records `run_not_started` and why, and the window which year failed. +year records `run_not_started` and why, and the window which year failed, +unless the year changed before the advance took the window's lock. Household simulations' runs are left alone: the check can't rebuild their calls yet. Each check is a `v2_run_restart_check` operation and each row it takes up a diff --git a/libs/policyengine-simulation-persistence/README.md b/libs/policyengine-simulation-persistence/README.md index dd5fa8dd3..d7df5a877 100644 --- a/libs/policyengine-simulation-persistence/README.md +++ b/libs/policyengine-simulation-persistence/README.md @@ -141,7 +141,8 @@ the run tables and their inputs, and `UPDATE` of `updated_at`, (`IdleBudgetWindow`, a type apart from the rows): pending or running, with no year under way and years still waiting untouched, last changed in a window. `read_earliest_waiting_year` names the year a restart starts the - window from, and marks just before its first start. + window from, and `mark_waiting_year` marks it just before its first + start. - `mark_restart` sets a listed row's `updated_at` to the statement's clock while it is still unstarted, so it is listed and marked only once. - `read_unstarted_report_run` and `read_unstarted_simulation_run` read back diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py index 55b9fbee3..5eb73f425 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py @@ -108,6 +108,7 @@ list_idle_budget_windows, list_unstarted_runs, mark_restart, + mark_waiting_year, read_earliest_waiting_year, read_report_run_simulation_run_ids, read_run_call, @@ -215,6 +216,7 @@ "list_idle_budget_windows", "list_unstarted_runs", "mark_restart", + "mark_waiting_year", "missing_table_privileges", "normalized_database_url", "read_catalog_version_id", diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py index b0fce9940..cd9f9fa15 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py @@ -17,7 +17,7 @@ stopped (``IdleBudgetWindow``): no year is under way and years are still waiting, untouched, since a time in the window. ``read_earliest_waiting_year`` names the year a restart starts the window - from, and marks just before its first start. + from, and ``mark_waiting_year`` marks it just before its first start. 2. ``mark_restart`` sets the row's ``updated_at`` before the check starts its call, so the row is never listed again and is restarted at most once. It uses the statement's clock rather than the transaction's, so the mark @@ -376,10 +376,23 @@ def mark_restart(connection: Connection, run: UnstartedRun) -> bool: A row given a call, claimed, finished or already marked since it was listed is left as it is, so two checks at once restart it only once. """ - table = RESTART_TABLES[run.kind] + return _mark(connection, RESTART_TABLES[run.kind], run.run_id) + + +def mark_waiting_year(connection: Connection, year: WaitingYear) -> bool: + """Mark a window's waiting year, as ``mark_restart`` marks a row. + + A window with a marked waiting year is not listed again until that year + has started, so the window is restarted once each time it stops. + """ + return _mark(connection, report_runs, year.report_run_id) + + +def _mark(connection: Connection, table: Table, run_id: UUID) -> bool: + """Set the row's ``updated_at`` while it is still unstarted; return whether.""" marked = connection.execute( update(table) - .where(table.c.id == run.run_id, *_unstarted(table)) + .where(table.c.id == run_id, *_unstarted(table)) .values(updated_at=func.clock_timestamp()) .returning(table.c.id) ).scalar_one_or_none() diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_store.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_store.py index 9f2880065..464db7cdf 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_store.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_store.py @@ -15,6 +15,7 @@ list_idle_budget_windows, list_unstarted_runs, mark_restart, + mark_waiting_year, read_earliest_waiting_year, read_report_run_simulation_run_ids, read_run_call, @@ -119,3 +120,8 @@ def read_earliest_waiting_year(self, window_run_id: UUID) -> WaitingYear | None: """Return the window's earliest year still waiting, never changed, or None.""" with self._transaction() as connection: return read_earliest_waiting_year(connection, window_run_id) + + def mark_waiting_year(self, year: WaitingYear) -> bool: + """Mark the window's year while it is still unstarted; return whether it was.""" + with self._transaction() as connection: + return mark_waiting_year(connection, year) diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py index d184dc0c9..3bc59d03e 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py @@ -167,6 +167,10 @@ def read_earliest_waiting_year(self, window_run_id: UUID) -> WaitingYear | None: """Return the window's earliest year still waiting, never changed.""" ... + def mark_waiting_year(self, year: WaitingYear) -> bool: + """Mark the window's year while it is still unstarted; return whether.""" + ... + class HouseholdJobRestarts(Protocol): """The household job reads and writes a restart uses (``HouseholdJobStore``).""" diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_outcomes.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_outcomes.py index 7720388e0..a1f531a89 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_outcomes.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_outcomes.py @@ -38,7 +38,8 @@ class RestartOutcome(StrEnum): after Modal didn't confirm the check's call. A window is left when it is gone, has no year waiting untouched, finished or was failed for an earlier failed year before the check started anything, or its year - changed before the mark; and when its marked year's failure couldn't be + changed before the mark (or before the refusal of a year whose call + can't be rebuilt); and when its marked year's failure couldn't be written, for the sweep to fail as stalled. ``deferred``: the row or window is left unmarked for the next check, because its rows or the manifest couldn't be read, its restart raised diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_windows.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_windows.py index 7a85cf496..c7f50b6cf 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_windows.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_windows.py @@ -23,7 +23,8 @@ check takes the window up again. A year Modal refuses fails the window, as it would at the service's start. A window whose first call can't be rebuilt is failed at once, through the same advance: that year records -``run_not_started`` and why, as a row the check can't rebuild is failed. +``run_not_started`` and why, as a row the check can't rebuild is failed, +unless it changed before the advance took the window's lock. """ from __future__ import annotations @@ -32,7 +33,6 @@ from functools import partial from typing import TYPE_CHECKING -from policyengine_simulation_persistence import RestartKind, UnstartedRun from policyengine_stage12_persistence import ( RunStatus, YearStartDeferred, @@ -54,6 +54,7 @@ if TYPE_CHECKING: from collections.abc import Callable from datetime import datetime + from uuid import UUID from policyengine_simulation_persistence import IdleBudgetWindow from policyengine_stage12_persistence import WindowAdvance, YearToStart @@ -75,13 +76,21 @@ class _MarkedYearStarts: """Start a window's years with ``YearStarts``, marking the restart first. Before the first start, it marks the earliest waiting year. A mark that - changes nothing (the year changed meanwhile) or can't be written defers - the year, so the advance starts nothing more. It records what happened, - so the restart can say how it ended. + changes nothing (the year changed meanwhile) or raises defers the year, + so the advance starts nothing more; an error other than the database's + is logged with its traceback. It records what happened, so the restart + can say how it ended. """ - def __init__(self, *, mark: Callable[[], bool], start_year: YearStarts) -> None: - """Mark with ``mark``, then start each year with ``start_year``.""" + def __init__( + self, + *, + window: IdleBudgetWindow, + mark: Callable[[], bool], + start_year: YearStarts, + ) -> None: + """Mark with ``mark``, then start each of the window's years.""" + self._window = window self._mark = mark self._start_year = start_year self.marked = False @@ -97,6 +106,11 @@ def __call__(self, year_to_start: YearToStart, /) -> str: except SQLAlchemyError as error: self.mark_error_type = type(error).__name__ raise YearStartDeferred from None + # Any other error is no refusal by Modal: nothing was started. + except Exception as error: # noqa: BLE001 # logged with its traceback by log_unexpected_error + log_unexpected_error(self._window) + self.mark_error_type = type(error).__name__ + raise YearStartDeferred from None if not self.marked: raise YearStartDeferred try: @@ -136,33 +150,36 @@ def ended_by_error(self) -> RestartOutcome: class _RefusedFirstYear: - """Fail the first year offered with why its call can't be rebuilt. - - The advance fails that year, and so the window, unless the year's own - call claimed it meanwhile; any later year is deferred to its - coordinator. Nothing is marked, so a failure that isn't written leaves - the window for the next check. + """Fail the year whose call can't be rebuilt, with why. + + The advance offers the earliest year still waiting. When that is the + year the restart read, it is refused, and the advance fails it, and so + the window, unless the year's own call claimed it meanwhile. Any other + year is deferred to its coordinator: the year read has changed since, + so nothing is failed. Nothing is marked, so a failure that isn't written + leaves the window for the next check. """ - def __init__(self, cannot: CallNotRebuiltError) -> None: - """Refuse with the reason ``cannot`` gives.""" + def __init__(self, cannot: CallNotRebuiltError, *, report_run_id: UUID) -> None: + """Refuse the year of ``report_run_id`` with the reason ``cannot`` gives.""" self._error_message = ( f"{NOT_STARTED_MESSAGE}, and its call could not be built again: " f"{cannot.reason}." ) self._error_type = cannot.error_type - self.offered = False + self._report_run_id = report_run_id + self.refused = False - def __call__(self, year_to_start: YearToStart, /) -> str: # noqa: ARG002 # the YearStarter protocol's argument - """Refuse the first year offered; defer any later one.""" - if self.offered: + def __call__(self, year_to_start: YearToStart, /) -> str: + """Refuse the year read, when it is offered first; defer any other.""" + if self.refused or year_to_start.run_ids.report_run_id != self._report_run_id: raise YearStartDeferred - self.offered = True + self.refused = True raise YearStartRefused(self._error_message) def ended(self, advance: WindowAdvance) -> _Ended: """Return how the restart ended, from the advance it made.""" - if self.offered and advance.status is RunStatus.FAILED: + if self.refused and advance.status is RunStatus.FAILED: return RestartOutcome.FAILED, self._error_type return RestartOutcome.LEFT, None @@ -235,18 +252,13 @@ async def _starts( except DeferredRestartError as deferred: return RestartOutcome.DEFERRED, str(deferred) except CallNotRebuiltError as cannot: - return _RefusedFirstYear(cannot) + return _RefusedFirstYear(cannot, report_run_id=waiting.report_run_id) if built is None: return RestartOutcome.LEFT, None worker, calculation = built - first_year = UnstartedRun( - kind=RestartKind.REPORT_RUN, - run_id=waiting.report_run_id, - # The mark reads only the row's kind and ID. - created_at=window.last_changed_at, - ) return _MarkedYearStarts( - mark=partial(self._restarts.mark_restart, first_year), + window=window, + mark=partial(self._restarts.mark_waiting_year, waiting), start_year=YearStarts( worker=worker, caller=self._caller, From 6dd830844b20d0a9b6d6572d4327c27be18f040f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:11:03 +0400 Subject: [PATCH 26/29] Test restarting a household report run whose call never started (failing) A household report run is listed as its own kind, read back from its stored rows, and started again with the call its submission started, rebuilt from the household and policies read against the run's own catalog version. One whose household, policy, catalog version or worker changed is failed with why, with its simulation runs. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_run_restart_records.py | 176 +++++++++-- .../test_run_restart_household_reports.py | 293 ++++++++++++++++++ .../tests/test_run_restarts.py | 51 ++- 3 files changed, 495 insertions(+), 25 deletions(-) create mode 100644 projects/policyengine-simulation-entry/tests/test_run_restart_household_reports.py diff --git a/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py b/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py index cd1d6bd9a..03af0f9bb 100644 --- a/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py +++ b/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py @@ -26,11 +26,13 @@ ) from policyengine_simulation_persistence import ( BudgetWindowYear, + HouseholdReportSimulation, IdleBudgetWindow, NewBudgetWindowReport, NewEconomyReport, NewEconomySimulation, NewHouseholdJob, + NewHouseholdReport, ReportRunTrigger, ReportSimulation, RestartKind, @@ -38,6 +40,8 @@ RunRestartStore, SimulationChoice, SimulationType, + StoredHouseholdReportRun, + StoredHouseholdReportSide, StoredRunUnreadableError, UnstartedRun, WaitingYear, @@ -46,6 +50,7 @@ insert_economy_report, insert_economy_simulation, insert_household_job, + insert_household_report, list_idle_budget_windows, list_unstarted_runs, mark_restart, @@ -53,6 +58,7 @@ read_earliest_waiting_year, read_report_run_simulation_run_ids, read_run_call, + read_unstarted_household_report_run, read_unstarted_report_run, read_unstarted_simulation_run, record_household_call, @@ -63,6 +69,7 @@ ) from public_api_rows import ( StoredValue, + insert_catalog_version, insert_dynamic, insert_household, insert_policy, @@ -343,37 +350,149 @@ def test_a_budget_window_year_is_not_listed(connection) -> None: assert _listed(connection, around=created_at) == () -def test_a_household_simulations_run_and_report_run_are_not_listed( +def _new_household_report( + economy: EconomyCatalog, + household_id: UUID, + *, + reform_policy_id: UUID | None = None, + with_reform: bool = True, +) -> NewHouseholdReport: + reform = ( + HouseholdReportSimulation( + simulation_id=uuid4(), + simulation_run_id=uuid4(), + policy_id=reform_policy_id, + ) + if with_reform + else None + ) + return NewHouseholdReport( + id=uuid4(), + run_id=uuid4(), + country="us", + label="A larger child tax credit for one family", + description=None, + tax_benefit_model_id=economy.catalog.model_id, + tax_benefit_model_version_id=economy.catalog.version_id, + household_id=household_id, + year=2027, + versions=VERSIONS, + inputs={}, + baseline=HouseholdReportSimulation( + simulation_id=uuid4(), simulation_run_id=uuid4(), policy_id=None + ), + reform=reform, + ) + + +def _stored_household(connection: Connection) -> UUID: + return insert_household( + connection, country="us", default_year=2026, document=HOUSEHOLD_DOCUMENT + ) + + +def test_a_household_report_run_is_listed_as_its_own_kind_and_its_sides_are_not( connection, ) -> None: + """Its one household function call claims both simulation runs.""" economy = insert_economy_catalog(connection) - report = _new_report(economy) - insert_economy_report(connection, report) + report = _new_household_report(economy, _stored_household(connection)) + insert_household_report(connection, report) simulation = _new_simulation(economy) insert_economy_simulation(connection, simulation) - household_id = insert_household( - connection, country="us", default_year=2026, document=HOUSEHOLD_DOCUMENT - ) - # Make every simulation a household simulation, as C1 ยท 15's household - # reports and household simulations will be. - for simulation_id in ( - report.baseline.simulation_id, - report.reform.simulation_id, - simulation.id, - ): - connection.execute( - update(simulations) - .where(simulations.c.id == simulation_id) - .values( - simulation_type=SimulationType.HOUSEHOLD, - dataset_id=None, - household_id=household_id, - ) + # A household simulation's run that no report names: there are none + # yet, and the check couldn't rebuild its call. + connection.execute( + update(simulations) + .where(simulations.c.id == simulation.id) + .values( + simulation_type=SimulationType.HOUSEHOLD, + dataset_id=None, + household_id=report.household_id, ) + ) created_at = _transaction_start(connection) act_as_the_modal_worker(connection) - assert _listed(connection, around=created_at) == () + assert _listed(connection, around=created_at) == ( + UnstartedRun( + kind=RestartKind.HOUSEHOLD_REPORT_RUN, + run_id=report.run_id, + created_at=created_at, + ), + ) + + +def test_a_household_report_runs_stored_rows_are_read_back(connection) -> None: + economy = insert_economy_catalog(connection, parameter_names=(CTC_AMOUNT,)) + policy_id = insert_policy( + connection, economy.catalog, [StoredValue(CTC_AMOUNT, "2026-01-01", 3_000)] + ) + report = _new_household_report( + economy, _stored_household(connection), reform_policy_id=policy_id + ) + insert_household_report(connection, report) + created_at = _transaction_start(connection) + act_as_the_modal_worker(connection) + + stored = read_unstarted_household_report_run(connection, report.run_id) + + assert stored == StoredHouseholdReportRun( + report_run_id=report.run_id, + country="us", + versions=VERSIONS, + household_id=report.household_id, + year=2027, + catalog_version_id=economy.catalog.version_id, + baseline=StoredHouseholdReportSide( + simulation_run_id=report.baseline.simulation_run_id, policy_id=None + ), + reform=StoredHouseholdReportSide( + simulation_run_id=report.reform.simulation_run_id, policy_id=policy_id + ), + created_at=created_at, + ) + assert read_unstarted_household_report_run(connection, uuid4()) is None + + +def test_a_household_report_run_of_the_baseline_alone_reads_back_no_reform( + connection, +) -> None: + economy = insert_economy_catalog(connection) + report = _new_household_report( + economy, _stored_household(connection), with_reform=False + ) + insert_household_report(connection, report) + act_as_the_modal_worker(connection) + + stored = read_unstarted_household_report_run(connection, report.run_id) + + assert stored is not None + assert stored.reform is None + assert stored.baseline.simulation_run_id == report.baseline.simulation_run_id + + +def test_a_household_report_run_whose_sides_differ_cannot_be_read( + connection, +) -> None: + economy = insert_economy_catalog(connection) + report = _new_household_report(economy, _stored_household(connection)) + insert_household_report(connection, report) + other_catalog = insert_catalog_version( + connection, + country="us", + model_id=economy.catalog.model_id, + policyengine_version="9.9.9", + ) + connection.execute( + update(simulation_runs) + .where(simulation_runs.c.id == report.reform.simulation_run_id) + .values(tax_benefit_model_version_id=other_catalog.version_id) + ) + act_as_the_modal_worker(connection) + + with pytest.raises(StoredRunUnreadableError, match="catalog versions"): + read_unstarted_household_report_run(connection, report.run_id) def test_a_report_runs_stored_inputs_are_read_back(connection) -> None: @@ -695,6 +814,8 @@ def test_the_store_lists_marks_and_reads_in_transactions_of_its_own( insert_economy_simulation(connection, simulation) window = _new_window(economy) insert_budget_window_report(connection, window) + household_report = _new_household_report(economy, _stored_household(connection)) + insert_household_report(connection, household_report) created_at = _transaction_start(connection) act_as_the_modal_worker(connection) @@ -706,6 +827,9 @@ def test_the_store_lists_marks_and_reads_in_transactions_of_its_own( by_kind = {run.kind: run for run in listed} report_run = savepoint_store.read_report_run(report.run_id) simulation_run = savepoint_store.read_simulation_run(simulation.run_id) + household_report_run = savepoint_store.read_household_report_run( + household_report.run_id + ) marked = savepoint_store.mark_restart(by_kind[RestartKind.REPORT_RUN]) call = savepoint_store.read_call(by_kind[RestartKind.SIMULATION_RUN]) windows = savepoint_store.list_idle_windows( @@ -717,7 +841,13 @@ def test_the_store_lists_marks_and_reads_in_transactions_of_its_own( year_marked = waiting is not None and savepoint_store.mark_waiting_year(waiting) sides = savepoint_store.read_report_run_simulation_run_ids(report.run_id) - assert set(by_kind) == {RestartKind.REPORT_RUN, RestartKind.SIMULATION_RUN} + assert set(by_kind) == { + RestartKind.REPORT_RUN, + RestartKind.SIMULATION_RUN, + RestartKind.HOUSEHOLD_REPORT_RUN, + } + assert household_report_run is not None + assert household_report_run.report_run_id == household_report.run_id assert call == RunCall(status="pending", modal_call_id=None) assert [run.run_id for run in windows] == [window.window_run_id] assert waiting == WaitingYear( diff --git a/projects/policyengine-simulation-entry/tests/test_run_restart_household_reports.py b/projects/policyengine-simulation-entry/tests/test_run_restart_household_reports.py new file mode 100644 index 000000000..e649da122 --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_run_restart_household_reports.py @@ -0,0 +1,293 @@ +"""Starting a household report run again when its household function call never started. + +One household function call calculates both sides of a household report run. +The restart reads back the report run's stored rows, reads the household and +each side's policy as they are now, against the catalog version the run was +created on, and rebuilds the call with the submission's own builder +(``household_report_calculation``). Each test first submits a household +report through the service that creates it, captures the call that +submission started, and requires the restart to start the same call. The +store's reads are tested against PostgreSQL in the persistence library. +""" + +from __future__ import annotations + +import asyncio +from dataclasses import replace +from typing import Any +from uuid import UUID + +from economy_service_fakes import CREATED_AT +from household_report_fakes import ( + POLICY_ID, + FakeHouseholdReportCaller, + FakeHouseholdReportStore, + household_report_request, + numbered_ids, +) +from policyengine_simulation_entry.household_report_submissions import ( + HouseholdReportSubmissionService, +) +from policyengine_simulation_entry.household_workers import ManifestHouseholdWorkers +from policyengine_simulation_entry.run_restarts import RestartOutcome, RestartSummary +from policyengine_simulation_persistence import ( + ReportSideError, + RestartKind, + RunVersions, + StoredHouseholdReportRun, + StoredHouseholdReportSide, + UnknownHouseholdError, + UnknownPolicyError, + UnstartedRun, +) +from test_run_restarts import ( + DATABASE_ERROR, + MANIFEST_STORE, + FakeHouseholdCaller, + FakeRestartStore, + RestartRig, + restart_rig, +) + +NOT_STARTED = ( + "run_not_started: The run stayed pending with no Modal call recorded for it" +) + + +def _submitted_household_report( + **request_fields: object, +) -> tuple[StoredHouseholdReportRun, dict, UUID]: + """Submit a household report; return its stored run, its call's payload and run. + + The report run is stored as the submission created it. + """ + store = FakeHouseholdReportStore() + caller = FakeHouseholdReportCaller() + submissions = HouseholdReportSubmissionService( + store=store, + workers=ManifestHouseholdWorkers(MANIFEST_STORE), + caller=caller, + call_start_timeout_seconds=0.05, + new_id=numbered_ids(), + ) + asyncio.run( + submissions.submit_report( + household_report_request(**request_fields), observability_context=None + ) + ) + (report,) = store.reports + (_worker, payload, report_run_id, _observability) = caller.calls[0] + stored = StoredHouseholdReportRun( + report_run_id=report.run_id, + country=report.country, + versions=report.versions, + household_id=report.household_id, + year=report.year, + catalog_version_id=report.tax_benefit_model_version_id, + baseline=StoredHouseholdReportSide( + simulation_run_id=report.baseline.simulation_run_id, + policy_id=report.baseline.policy_id, + ), + reform=( + None + if report.reform is None + else StoredHouseholdReportSide( + simulation_run_id=report.reform.simulation_run_id, + policy_id=report.reform.policy_id, + ) + ), + created_at=CREATED_AT, + ) + return stored, payload, report_run_id + + +def _household_report_rig( + stored: StoredHouseholdReportRun, **options: Any +) -> RestartRig: + return restart_rig( + restarts=FakeRestartStore( + unstarted=( + UnstartedRun( + kind=RestartKind.HOUSEHOLD_REPORT_RUN, + run_id=stored.report_run_id, + created_at=CREATED_AT, + ), + ), + household_report_runs={stored.report_run_id: stored}, + ), + **options, + ) + + +def _sides(stored: StoredHouseholdReportRun) -> tuple[UUID, ...]: + """Return the report run's simulation runs, the baseline's first.""" + sides = ( + [stored.baseline] if stored.reform is None else [stored.baseline, stored.reform] + ) + return tuple(side.simulation_run_id for side in sides) + + +def test_a_household_report_run_is_started_again_with_its_submissions_call() -> None: + stored, payload, report_run_id = _submitted_household_report() + rig = _household_report_rig(stored) + + summary = rig.restart() + + (restarted,) = rig.household_caller.report_runs + assert restarted["payload"] == payload + assert restarted["report_run_id"] == report_run_id + assert restarted["observability_context"] == {"traceparent": "00-restart-check"} + assert [run.run_id for run in rig.restarts.marked] == [report_run_id] + assert rig.economy.recorded_calls == [ + (report_run_id, "fc-household-report-restarted") + ] + assert summary == RestartSummary(listed=1, restarted=1) + assert rig.telemetry.outcomes == [ + ( + RestartKind.HOUSEHOLD_REPORT_RUN, + report_run_id, + RestartOutcome.RESTARTED, + None, + ) + ] + # The household and each side's policy are read on the run's own .py + # version, the report's year kept as stored. + (read,) = rig.household_reports.reads + assert read["policyengine_version"] == stored.versions.policyengine_version + assert read["policy_ids"] == {"baseline": None, "reform": POLICY_ID} + + +def test_a_household_report_of_the_baseline_alone_is_started_again() -> None: + stored, payload, _report_run_id = _submitted_household_report(reform=None) + rig = _household_report_rig(stored) + + summary = rig.restart() + + assert stored.reform is None + (restarted,) = rig.household_caller.report_runs + assert restarted["payload"] == payload + assert set(rig.household_reports.reads[0]["policy_ids"]) == {"baseline"} + assert summary == RestartSummary(listed=1, restarted=1) + + +def _failed_message(rig: RestartRig, stored: StoredHouseholdReportRun) -> str: + """Return the message the report run was failed with, with its own runs.""" + ((report_run_id, simulation_run_ids, message),) = rig.economy.failed_runs + assert report_run_id == stored.report_run_id + assert simulation_run_ids == _sides(stored) + return message + + +def test_a_household_report_run_whose_catalog_version_changed_is_failed() -> None: + """The .py version's catalog version is no longer the one the run records.""" + stored, _payload, _report_run_id = _submitted_household_report() + stored = replace(stored, catalog_version_id=UUID(int=404)) + rig = _household_report_rig(stored) + + summary = rig.restart() + + assert rig.household_caller.report_runs == [] + assert rig.restarts.marked == [] + assert _failed_message(rig, stored) == ( + f"{NOT_STARTED}, and its call could not be built again: its catalog " + "version changed since it was created." + ) + assert summary == RestartSummary(listed=1, failed=1) + + +def test_a_household_report_run_whose_versions_no_worker_holds_is_failed() -> None: + stored, _payload, _report_run_id = _submitted_household_report() + stored = replace( + stored, + versions=RunVersions( + country_package_version="0.0.1", + policyengine_version=stored.versions.policyengine_version, + ), + ) + rig = _household_report_rig(stored) + + summary = rig.restart() + + assert rig.household_caller.report_runs == [] + assert "calculates on other versions than the run records" in _failed_message( + rig, stored + ) + assert summary == RestartSummary(listed=1, failed=1) + + +def test_a_household_report_run_whose_household_is_gone_is_failed() -> None: + stored, _payload, _report_run_id = _submitted_household_report() + rig = _household_report_rig( + stored, + household_reports=FakeHouseholdReportStore( + read_failure=UnknownHouseholdError("us has no household") + ), + ) + + summary = rig.restart() + + assert _failed_message(rig, stored).endswith(": its household is gone.") + assert summary == RestartSummary(listed=1, failed=1) + assert rig.telemetry.outcomes[0][3] == "UnknownHouseholdError" + + +def test_a_household_report_run_whose_reforms_policy_is_gone_is_failed() -> None: + """A side's refusal is read as its own cause.""" + stored, _payload, _report_run_id = _submitted_household_report() + rig = _household_report_rig( + stored, + household_reports=FakeHouseholdReportStore( + read_failure=ReportSideError("reform", UnknownPolicyError("no policy")) + ), + ) + + summary = rig.restart() + + assert _failed_message(rig, stored).endswith(": its policy is gone.") + assert summary == RestartSummary(listed=1, failed=1) + + +def test_a_household_report_run_whose_household_cannot_be_calculated_is_failed() -> ( + None +): + stored, _payload, _report_run_id = _submitted_household_report() + rig = _household_report_rig( + stored, + household_reports=FakeHouseholdReportStore(document={"people": "unreadable"}), + ) + + summary = rig.restart() + + assert _failed_message(rig, stored).endswith( + ": its household can no longer be calculated." + ) + assert summary == RestartSummary(listed=1, failed=1) + + +def test_unreadable_rows_leave_a_household_report_run_for_the_next_check() -> None: + stored, _payload, _report_run_id = _submitted_household_report() + rig = _household_report_rig( + stored, household_reports=FakeHouseholdReportStore(read_failure=DATABASE_ERROR) + ) + + summary = rig.restart() + + assert rig.restarts.marked == [] + assert rig.economy.failed_runs == [] + assert summary == RestartSummary(listed=1, deferred=1) + + +def test_a_household_report_run_whose_restart_modal_refuses_is_failed() -> None: + stored, _payload, _report_run_id = _submitted_household_report() + rig = _household_report_rig( + stored, household_caller=FakeHouseholdCaller(failure=ConnectionError("down")) + ) + + summary = rig.restart() + + assert [run.run_id for run in rig.restarts.marked] == [stored.report_run_id] + assert _failed_message(rig, stored) == ( + f"{NOT_STARTED}, and Modal did not accept the call that started it again " + "(ConnectionError)." + ) + assert summary == RestartSummary(listed=1, failed=1) diff --git a/projects/policyengine-simulation-entry/tests/test_run_restarts.py b/projects/policyengine-simulation-entry/tests/test_run_restarts.py index e41edaf84..e838408d3 100644 --- a/projects/policyengine-simulation-entry/tests/test_run_restarts.py +++ b/projects/policyengine-simulation-entry/tests/test_run_restarts.py @@ -32,6 +32,7 @@ simulation_request, ) from household_fixtures import manifest_store +from household_report_fakes import FakeHouseholdReportStore from policyengine_simulation_entry.economy_workers import ManifestEconomyWorkers from policyengine_simulation_entry.household_api_models import ( HouseholdCalculationRequest, @@ -82,7 +83,11 @@ ) from policyengine_simulation_entry.household_workers import HouseholdWorker from policyengine_simulation_entry.run_restarts import RestartTarget - from policyengine_simulation_persistence import NewHouseholdJob, WaitingYear + from policyengine_simulation_persistence import ( + NewHouseholdJob, + StoredHouseholdReportRun, + WaitingYear, + ) from policyengine_stage12_persistence import WindowAdvance NOW = CREATED_AT + timedelta(minutes=4) @@ -126,6 +131,9 @@ class FakeRestartStore: unstarted: tuple[UnstartedRun, ...] = () report_runs: dict[UUID, StoredReportRun] = field(default_factory=dict) simulation_runs: dict[UUID, StoredSimulationRun] = field(default_factory=dict) + household_report_runs: dict[UUID, StoredHouseholdReportRun] = field( + default_factory=dict + ) report_simulation_run_ids: dict[UUID, tuple[UUID, ...]] = field( default_factory=dict ) @@ -200,6 +208,15 @@ def read_simulation_run( raise self.read_failure return self.simulation_runs.get(simulation_run_id) + def read_household_report_run( + self, report_run_id: UUID + ) -> StoredHouseholdReportRun | None: + if self.read_failure is not None: + raise self.read_failure + if report_run_id in self.read_failures: + raise self.read_failures[report_run_id] + return self.household_report_runs.get(report_run_id) + @dataclass class FakeHouseholdJobs: @@ -265,10 +282,35 @@ def read(self, job_id: UUID) -> HouseholdJobRecord | None: @dataclass class FakeHouseholdCaller: - """Records each household function call; ``failure`` is raised instead.""" + """Records each household function call; ``failure`` is raised instead. + + A household job's call is ``spawn``, and a household report run's + ``start_report_run``. + """ failure: Exception | None = None calls: list[dict[str, Any]] = field(default_factory=list) + report_runs: list[dict[str, Any]] = field(default_factory=list) + + async def start_report_run( + self, + worker: HouseholdWorker, + payload: dict[str, Any], + *, + report_run_id: UUID, + observability_context: dict[str, Any] | None, + ) -> str: + if self.failure is not None: + raise self.failure + self.report_runs.append( + { + "worker": worker, + "payload": payload, + "report_run_id": report_run_id, + "observability_context": observability_context, + } + ) + return "fc-household-report-restarted" async def spawn( self, @@ -338,6 +380,7 @@ class RestartRig: windows: BudgetWindowAdvancer economy: FakeStore households: FakeHouseholdJobs + household_reports: FakeHouseholdReportStore economy_caller: FakeCaller household_caller: FakeHouseholdCaller telemetry: RecordingTelemetry @@ -357,9 +400,11 @@ def restart_rig( # noqa: PLR0913 # each fake a test may replace manifest: object | None = None, timer: Callable[[], float] | None = None, windows: BudgetWindowAdvancer | None = None, + household_reports: FakeHouseholdReportStore | None = None, ) -> RestartRig: restarts = restarts or FakeRestartStore() windows = windows or UnusedWindows() + household_reports = household_reports or FakeHouseholdReportStore() economy = economy or FakeStore() households = households or FakeHouseholdJobs() economy_caller = economy_caller or FakeCaller() @@ -381,6 +426,7 @@ def holder(run: UnstartedRun) -> RunCall: restarts=restarts, economy=economy, households=households, + household_reports=household_reports, windows=windows, ), workers=RestartWorkers( @@ -400,6 +446,7 @@ def holder(run: UnstartedRun) -> RunCall: windows=windows, economy=economy, households=households, + household_reports=household_reports, economy_caller=economy_caller, household_caller=household_caller, telemetry=telemetry, From da89c8e14a5cb7331b0c0d629b570b448fcfcc55 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:17:01 +0400 Subject: [PATCH 27/29] Restart a household report run whose call never started The restart check lists household report runs as a kind of their own, reads back the report's household and year, the run's versions, each side's simulation run and policy, and the catalog version the sides were created on. It reads the household and the policies' values again with the submission's reader, rebuilds the one household function call with household_report_calculation on the worker of the run's .py version, and starts it with start_report_run. The call is recorded, or the run failed with its simulation runs, with the report run writes economy report runs use. A run whose catalog version, worker, household or policy changed is failed with run_not_started and why. Co-Authored-By: Claude Opus 5.5 --- .../c1-19-restart-unstarted-runs.added.md | 2 +- .../stage-12-modal-worker-foundation.md | 12 +- .../README.md | 8 +- .../__init__.py | 6 + .../run_restart_records.py | 142 +++++++++++- .../run_restart_store.py | 9 + .../tests/test_run_restart_records.py | 2 +- .../policyengine-simulation-entry/README.md | 9 +- .../household_report_submissions.py | 5 +- .../run_restart_calls.py | 208 +++++++++++++++--- .../run_restart_check.py | 4 + .../run_restarts.py | 18 +- 12 files changed, 374 insertions(+), 51 deletions(-) diff --git a/changelog.d/c1-19-restart-unstarted-runs.added.md b/changelog.d/c1-19-restart-unstarted-runs.added.md index 77ed9fe64..abb6a3135 100644 --- a/changelog.d/c1-19-restart-unstarted-runs.added.md +++ b/changelog.d/c1-19-restart-unstarted-runs.added.md @@ -1 +1 @@ -A restart check in the stuck-run sweep's Modal application starts the Modal call of a household job, economy report run or standalone economy simulation run again when the run stayed pending with no call recorded 3 to 10 minutes after its creation, as when the Simulation Entrypoint stopped before starting its call. It also starts a budget window's waiting years again when the window stopped with none under way, as the Simulation Entrypoint starts a new window's first years, and fails at once, with the reason, a window whose first waiting year's call can't be built again. Its telemetry counts every restart and how it ended. The sweep now fails a run still pending with no call at 15 minutes instead of 10. +A restart check in the stuck-run sweep's Modal application starts the Modal call of a household job, economy or household report run, or standalone economy simulation run again when the run stayed pending with no call recorded 3 to 10 minutes after its creation, as when the Simulation Entrypoint stopped before starting its call. It also starts a budget window's waiting years again when the window stopped with none under way, as the Simulation Entrypoint starts a new window's first years, and fails at once, with the reason, a window whose first waiting year's call can't be built again. Its telemetry counts every restart and how it ended. The sweep now fails a run still pending with no call at 15 minutes instead of 10. diff --git a/docs/migration/stage-12-modal-worker-foundation.md b/docs/migration/stage-12-modal-worker-foundation.md index 56f156dbe..c8ec95789 100644 --- a/docs/migration/stage-12-modal-worker-foundation.md +++ b/docs/migration/stage-12-modal-worker-foundation.md @@ -863,8 +863,16 @@ statements are bounded by the check's statement timeout. A window whose first call can't be rebuilt is failed at once through the same advance: that year records `run_not_started` and why, and the window which year failed, unless the year changed before the advance took the window's lock. -Household simulations' runs -are left alone: the check can't rebuild their calls yet. Each check is a + +A household report run is restarted as its own kind: the check reads the +report run's household, year, versions and each side's policy, reads the +household and the policies' values again with the entrypoint's reader +(`HouseholdReportStore.read_inputs`), and rebuilds its one household +function call with `household_report_calculation`, started with +`start_report_run`. A run whose .py version's catalog version is no longer +the one its simulation runs were created on, or whose household or policy +is gone or can no longer be calculated, is failed with `run_not_started` +and why, with its simulation runs. Each check is a `v2_run_restart_check` operation and each row it takes up a `v2_run_restart` operation, with the row's kind, ID, age and outcome, and each outcome is also logged as a `v2_run_restart_outcome` event. A deferred diff --git a/libs/policyengine-simulation-persistence/README.md b/libs/policyengine-simulation-persistence/README.md index f6f637429..8836d0704 100644 --- a/libs/policyengine-simulation-persistence/README.md +++ b/libs/policyengine-simulation-persistence/README.md @@ -158,7 +158,8 @@ the run tables and their inputs, and `UPDATE` of `updated_at`, - `list_unstarted_runs` returns the oldest rows, up to a limit, created in a window that are still `pending` with no Modal call and whose `updated_at` still equals `created_at`: household jobs, report runs outside a budget - window whose baseline is an economy simulation, and runs of economy + window whose baseline is an economy simulation, household report runs + (`household_report_run`, a kind of their own), and runs of economy simulations that no report run names (standalone runs). - `list_idle_budget_windows` returns the budget windows that stopped (`IdleBudgetWindow`, a type apart from the rows): pending or running, with @@ -176,6 +177,11 @@ the run tables and their inputs, and `UPDATE` of `updated_at`, the rows no longer describe one calculation: a side's region or dataset is gone, or the two sides differ. The values are read with `read_simulation_reform`, the reader a submission uses. +- `read_unstarted_household_report_run` reads back a household report run: + its report's household and year, the run's versions, each side's + simulation run and policy, and the catalog version the sides were created + on. The entrypoint reads the household and the policies' values again + with `HouseholdReportStore.read_inputs`. - `read_report_run_simulation_run_ids` returns the simulation runs a report run names, so one that can't be rebuilt is failed with them, and `read_run_call` returns a row's status and recorded call, which says whose diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py index fda593df2..d66aee44c 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py @@ -110,6 +110,8 @@ RestartKind, RunCall, StoredEconomySide, + StoredHouseholdReportRun, + StoredHouseholdReportSide, StoredReportRun, StoredRunUnreadableError, StoredSimulationRun, @@ -122,6 +124,7 @@ read_earliest_waiting_year, read_report_run_simulation_run_ids, read_run_call, + read_unstarted_household_report_run, read_unstarted_report_run, read_unstarted_simulation_run, ) @@ -203,6 +206,8 @@ "SimulationType", "StoredEconomySide", "StoredHousehold", + "StoredHouseholdReportRun", + "StoredHouseholdReportSide", "StoredReportRun", "StoredRunUnreadableError", "StoredSimulationRun", @@ -251,6 +256,7 @@ "read_run_call", "read_simulation_reform", "read_stored_household", + "read_unstarted_household_report_run", "read_unstarted_report_run", "read_unstarted_simulation_run", "record_household_call", diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py index cd9f9fa15..85ce3ec50 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py @@ -8,11 +8,14 @@ 1. ``list_unstarted_runs`` lists the rows still pending with no call that were created within a window and that nothing has changed since their insert - (``updated_at`` still equals ``created_at``): household jobs, report runs, - and standalone simulation runs, which no report run names. A report run's - own simulation runs are its coordinator's to start. Budget-window years - are left out, and household simulations too: the check can't rebuild - their calls yet. + (``updated_at`` still equals ``created_at``): household jobs, economy + report runs, household report runs (a kind of their own, since one + household function call calculates both sides), and standalone economy + simulation runs, which no report run names. A report run's own + simulation runs are its call's to start. Budget-window years are left + out, and so are household simulations' runs no report names (there are + none yet). ``read_unstarted_household_report_run`` reads a household + report run back. ``list_idle_budget_windows`` lists, apart, the budget windows that stopped (``IdleBudgetWindow``): no year is under way and years are still waiting, untouched, since a time in the window. @@ -84,7 +87,11 @@ class RestartKind(StrEnum): """What kind of row the restart check starts again.""" HOUSEHOLD_JOB = "household_job" + # An economy report run. REPORT_RUN = "report_run" + # A household report run: one household function call calculates both + # of its sides. + HOUSEHOLD_REPORT_RUN = "household_report_run" SIMULATION_RUN = "simulation_run" # A budget window none of whose years is under way (``IdleBudgetWindow``): # a restart starts its earliest waiting years, as the service starts a @@ -96,6 +103,7 @@ class RestartKind(StrEnum): RESTART_TABLES: Final[dict[RestartKind, Table]] = { RestartKind.HOUSEHOLD_JOB: household_jobs, RestartKind.REPORT_RUN: report_runs, + RestartKind.HOUSEHOLD_REPORT_RUN: report_runs, RestartKind.SIMULATION_RUN: simulation_runs, } @@ -189,6 +197,39 @@ class StoredReportRun: created_at: datetime +@dataclass(frozen=True) +class StoredHouseholdReportSide: + """One side of a household report run: its simulation run and its policy. + + ``policy_id`` is the side's stored policy, None for current law. + """ + + simulation_run_id: UUID + policy_id: UUID | None + + +@dataclass(frozen=True) +class StoredHouseholdReportRun: + """A household report run as its household function call was built from it. + + ``year`` is the report's, ``catalog_version_id`` the catalog version its + simulation runs were created on, and ``versions`` those the report run + records, which choose the worker. ``reform`` is None for a report of the + baseline alone. The household and the policies' values are read again, + with the submission's own reader. + """ + + report_run_id: UUID + country: str + versions: RunVersions + household_id: UUID + year: int + catalog_version_id: UUID + baseline: StoredHouseholdReportSide + reform: StoredHouseholdReportSide | None + created_at: datetime + + @dataclass(frozen=True) class StoredSimulationRun: """A standalone simulation run as its simulation function's call was built.""" @@ -213,10 +254,17 @@ def _economy_simulation_run( simulation_run_id: ColumnElement[Any], ) -> ColumnElement[bool]: """Return the condition that the simulation run is an economy simulation's.""" + return _simulation_run_of_type(simulation_run_id, SimulationType.ECONOMY) + + +def _simulation_run_of_type( + simulation_run_id: ColumnElement[Any], simulation_type: SimulationType +) -> ColumnElement[bool]: + """Return the condition that the simulation run is a simulation of the type.""" return exists().where( simulations.c.id == simulation_runs.c.simulation_id, simulation_runs.c.id == simulation_run_id, - simulations.c.simulation_type == SimulationType.ECONOMY, + simulations.c.simulation_type == simulation_type, ) @@ -229,6 +277,13 @@ def _restart_candidates(kind: RestartKind) -> Select[Any]: report_runs.c.budget_window_run_id.is_(None), _economy_simulation_run(report_runs.c.baseline_simulation_run_id), ) + if kind is RestartKind.HOUSEHOLD_REPORT_RUN: + return statement.where( + report_runs.c.budget_window_run_id.is_(None), + _simulation_run_of_type( + report_runs.c.baseline_simulation_run_id, SimulationType.HOUSEHOLD + ), + ) if kind is RestartKind.SIMULATION_RUN: naming_report_run = report_runs.alias("naming_report_run") return statement.where( @@ -556,6 +611,81 @@ def read_unstarted_report_run( ) +def read_unstarted_household_report_run( + connection: Connection, report_run_id: UUID +) -> StoredHouseholdReportRun | None: + """Return what a household report run's household function call is built from. + + Return None when there is no such run. Raise StoredRunUnreadableError + when its report names no household or year, a side's run is missing, or + the sides were created on different catalog versions. + """ + run = connection.execute( + select( + report_runs.c.created_at, + report_runs.c.baseline_simulation_run_id, + report_runs.c.reform_simulation_run_id, + report_runs.c.country_package_version, + report_runs.c.policyengine_version, + reports.c.country, + reports.c.household_id, + reports.c.year, + ) + .join(reports, reports.c.id == report_runs.c.report_id) + .where(report_runs.c.id == report_run_id) + ).one_or_none() + if run is None: + return None + if run.household_id is None or run.year is None: + msg = f"report run {report_run_id}'s report names no household or year" + raise StoredRunUnreadableError(msg) + side_ids = [run.baseline_simulation_run_id] + if run.reform_simulation_run_id is not None: + side_ids.append(run.reform_simulation_run_id) + sides = { + row.id: row + for row in connection.execute( + select( + simulation_runs.c.id, + simulation_runs.c.tax_benefit_model_version_id, + simulations.c.policy_id, + ) + .join(simulations, simulations.c.id == simulation_runs.c.simulation_id) + .where(simulation_runs.c.id.in_(side_ids)) + ) + } + if set(sides) != set(side_ids): + msg = f"report run {report_run_id} names a missing simulation run" + raise StoredRunUnreadableError(msg) + catalog_version_ids = {row.tax_benefit_model_version_id for row in sides.values()} + if len(catalog_version_ids) != 1: + msg = ( + f"report run {report_run_id}'s sides were created on different " + "catalog versions" + ) + raise StoredRunUnreadableError(msg) + stored_sides = [ + StoredHouseholdReportSide( + simulation_run_id=side_id, policy_id=sides[side_id].policy_id + ) + for side_id in side_ids + ] + return StoredHouseholdReportRun( + report_run_id=report_run_id, + country=run.country, + versions=RunVersions( + country_package_version=run.country_package_version, + policyengine_version=run.policyengine_version, + ), + household_id=run.household_id, + year=run.year, + catalog_version_id=catalog_version_ids.pop(), + baseline=stored_sides[0], + reform=stored_sides[1] if len(stored_sides) > 1 else None, + created_at=run.created_at, + ) + + def read_unstarted_simulation_run( connection: Connection, simulation_run_id: UUID ) -> StoredSimulationRun | None: diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_store.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_store.py index 464db7cdf..8a78e3723 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_store.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_store.py @@ -19,6 +19,7 @@ read_earliest_waiting_year, read_report_run_simulation_run_ids, read_run_call, + read_unstarted_household_report_run, read_unstarted_report_run, read_unstarted_simulation_run, ) @@ -34,6 +35,7 @@ from policyengine_simulation_persistence.run_restart_records import ( IdleBudgetWindow, RunCall, + StoredHouseholdReportRun, StoredReportRun, StoredSimulationRun, UnstartedRun, @@ -85,6 +87,13 @@ def read_report_run(self, report_run_id: UUID) -> StoredReportRun | None: with self._transaction() as connection: return read_unstarted_report_run(connection, report_run_id) + def read_household_report_run( + self, report_run_id: UUID + ) -> StoredHouseholdReportRun | None: + """Return what the household report run's call is built from, or None.""" + with self._transaction() as connection: + return read_unstarted_household_report_run(connection, report_run_id) + def read_simulation_run( self, simulation_run_id: UUID ) -> StoredSimulationRun | None: diff --git a/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py b/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py index 03af0f9bb..cf570f41a 100644 --- a/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py +++ b/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py @@ -426,7 +426,7 @@ def test_a_household_report_run_is_listed_as_its_own_kind_and_its_sides_are_not( def test_a_household_report_runs_stored_rows_are_read_back(connection) -> None: economy = insert_economy_catalog(connection, parameter_names=(CTC_AMOUNT,)) policy_id = insert_policy( - connection, economy.catalog, [StoredValue(CTC_AMOUNT, "2026-01-01", 3_000)] + connection, economy.catalog, [StoredValue(CTC_AMOUNT, 3_000, START)] ) report = _new_household_report( economy, _stored_household(connection), reform_policy_id=policy_id diff --git a/projects/policyengine-simulation-entry/README.md b/projects/policyengine-simulation-entry/README.md index dd7dbfc93..1fbda5327 100644 --- a/projects/policyengine-simulation-entry/README.md +++ b/projects/policyengine-simulation-entry/README.md @@ -123,12 +123,13 @@ runs it every two minutes as the Modal worker role. minutes ago (`run_restart_windows`). A window is restarted as `budget_window_submissions` starts a new one: from its earliest waiting year's rebuilt call, with `YearStarts`, marking that year just before the - first start. Household simulations' runs are - left out. After three minutes a check marks no more rows and leaves the - rest for the next. + first start. A household report run is its own kind, restarted with its + one household function call. After three minutes a check marks no more + rows and leaves the rest for the next. - It rebuilds each call with this service's own builders (`report_run_calculation`, `simulation_run_calculation`, - `prepare_household_job`) from the stored rows, on the manifest's worker + `prepare_household_job`, `household_report_calculation`) from the stored + rows, on the manifest's worker for the .py version the row records, even when that worker is no longer the default. The context's request ID is the check's own (`run-restart-`), and its creation time is the row's. diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_report_submissions.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_report_submissions.py index 987be2b7d..25779c47f 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_report_submissions.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_report_submissions.py @@ -24,8 +24,9 @@ Whatever is missing or cannot be calculated is refused in steps 1 and 2, before anything is written. The household function then claims the report run and each simulation run, writes each run's household document, and -finishes the report run; the stuck-run sweep fails a pending run no call -was recorded for. The errors are the economy routes' +finishes the report run. The restart check (``run_restart_calls``) starts +a report run whose call was never recorded once more, and the stuck-run +sweep fails it when that start is lost too. The errors are the economy routes' (``economy_submission_errors``), so ``POST /reports`` answers both kinds alike. """ diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py index 3bc59d03e..2b1b33f42 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py @@ -4,10 +4,13 @@ call must be the one the row's submission started: ``RestartCallBuilder`` reads back what the submission built it from and builds it again with the submission's own builders (``report_run_calculation``, -``simulation_run_calculation`` and ``prepare_household_job``), on the worker -of the .py version the row records. A household job whose rebuilt job would -record anything else (another year, catalog version or policy) can't be -rebuilt, so the restart never calculates something its row doesn't record. +``simulation_run_calculation``, ``prepare_household_job`` and +``household_report_calculation``), on the worker of the .py version the row +records. A household job whose rebuilt job would record anything else +(another year, catalog version or policy), or a household report run whose +.py version's catalog version is no longer the one its runs were created +on, can't be rebuilt, so the restart never calculates something its row +doesn't record. """ from __future__ import annotations @@ -22,8 +25,12 @@ from policyengine_simulation_persistence import ( MissingPolicyParametersError, OverlappingReformError, + ReportSideError, RestartKind, + StoredHouseholdReportRun, + StoredReportRun, StoredRunUnreadableError, + StoredSimulationRun, UnknownCatalogVersionError, UnknownDynamicError, UnknownHouseholdError, @@ -41,6 +48,9 @@ simulation_options, simulation_run_calculation, ) +from policyengine_simulation_entry.economy_submission_errors import ( + EconomySubmissionUnprocessableError, +) from policyengine_simulation_entry.economy_workers import ( EconomyWorkerUnavailableError, ) @@ -52,13 +62,20 @@ HouseholdCalculationUnprocessableError, prepare_household_job, ) +from policyengine_simulation_entry.household_report_submissions import ( + HouseholdReportBuildError, + HouseholdReportSideInputs, + household_report_calculation, +) from policyengine_simulation_entry.household_workers import ( + HouseholdFunctionCaller, HouseholdFunctionUnavailableError, + HouseholdReportRunCaller, VersionManifestUnavailableError, ) if TYPE_CHECKING: - from collections.abc import Awaitable, Callable + from collections.abc import Awaitable, Callable, Mapping from datetime import datetime from uuid import UUID @@ -67,12 +84,12 @@ HouseholdCalculationInputs, HouseholdJobRecord, HouseholdJobStatus, + HouseholdReportInputs, IdleBudgetWindow, NewHouseholdJob, + ReportSide, RunCall, RunVersions, - StoredReportRun, - StoredSimulationRun, UnstartedRun, WaitingYear, ) @@ -84,10 +101,7 @@ EconomyRunCaller, EconomyWorker, ) - from policyengine_simulation_entry.household_workers import ( - HouseholdFunctionCaller, - HouseholdWorker, - ) + from policyengine_simulation_entry.household_workers import HouseholdWorker # The deployment each Modal environment holding the v2 workers belongs to, as # the Simulation Entrypoint's deploy pairs them (APP_ENVIRONMENT). A call's @@ -157,6 +171,12 @@ def read_call(self, run: UnstartedRun) -> RunCall | None: """Return the row's status and recorded call, or None when it is gone.""" ... + def read_household_report_run( + self, report_run_id: UUID + ) -> StoredHouseholdReportRun | None: + """Return what the household report run's call is built from, or None.""" + ... + def list_idle_windows( self, *, idle_after: datetime, idle_before: datetime, limit: int ) -> tuple[IdleBudgetWindow, ...]: @@ -201,6 +221,30 @@ def fail_unstarted(self, job_id: UUID, error_message: str) -> bool: ... +class HouseholdReportRestarts(Protocol): + """The household report read a restart uses (``HouseholdReportStore``).""" + + def read_inputs( + self, + *, + country: str, + household_id: UUID, + policy_ids: Mapping[ReportSide, UUID | None], + policyengine_version: str, + ) -> HouseholdReportInputs: + """Read the catalog version, the household and each side's policy together.""" + ... + + +class RestartHouseholdCaller( + HouseholdFunctionCaller, HouseholdReportRunCaller, Protocol +): + """Starts a household job's call and a household report run's. + + ``ModalHouseholdFunctionCaller`` does both. + """ + + class VersionedEconomyWorkers(Protocol): """Find a .py version's economy functions (``ManifestEconomyWorkers``).""" @@ -234,10 +278,17 @@ class RestartCallers: """What starts a restarted call on Modal.""" economy: EconomyRunCaller - households: HouseholdFunctionCaller + households: RestartHouseholdCaller type CallStart = Callable[[dict[str, Any] | None], Awaitable[str]] +# What a row's call is rebuilt from, as the restart reads it back. +type _StoredRow = ( + StoredReportRun + | StoredHouseholdReportRun + | StoredSimulationRun + | HouseholdJobRecord +) class DeferredRestartError(Exception): @@ -275,6 +326,8 @@ class _ChangedRunError(LookupError): EconomyWorkerUnavailableError: None, HouseholdFunctionUnavailableError: None, _ChangedRunError: None, + # A report side's refusal is described by its own cause. + ReportSideError: None, StoredRunUnreadableError: "its stored rows no longer describe one calculation", UnknownCatalogVersionError: "its catalog version is gone", UnknownHouseholdError: "its household is gone", @@ -286,11 +339,15 @@ class _ChangedRunError(LookupError): HouseholdCalculationUnprocessableError: "its household can no longer be read", HouseholdCalculationInternalError: "its household request no longer validates", ValidationError: "its stored request no longer validates", + EconomySubmissionUnprocessableError: "its household can no longer be calculated", + HouseholdReportBuildError: "its household report request no longer validates", } _REBUILD_ERRORS: Final = tuple(_REBUILD_REASONS) def _rebuild_reason(error: Exception) -> str: + if isinstance(error, ReportSideError): + return _rebuild_reason(error.error) reason = next( ( reason @@ -355,14 +412,16 @@ def __init__( # noqa: PLR0913 # each seam tests replace *, restarts: UnstartedRunSource, households: HouseholdJobRestarts, + household_reports: HouseholdReportRestarts, workers: RestartWorkers, callers: RestartCallers, settings: RestartSettings, new_request_id: Callable[[], str] = lambda: f"run-restart-{uuid.uuid4()}", ) -> None: - """Read rows from the two stores; start calls with ``callers``.""" + """Read rows from the stores; start calls with ``callers``.""" self._restarts = restarts self._households = households + self._household_reports = household_reports self._workers = workers self._callers = callers self._settings = settings @@ -376,28 +435,15 @@ def build(self, run: UnstartedRun) -> RebuiltCall | None: """ simulation_run_ids: tuple[UUID, ...] = () try: - if run.kind is RestartKind.REPORT_RUN: - stored_report = self._restarts.read_report_run(run.run_id) - if stored_report is None: - return None - simulation_run_ids = ( - stored_report.baseline.simulation_run_id, - stored_report.reform.simulation_run_id, - ) - return RebuiltCall(self._report_call(stored_report), simulation_run_ids) - if run.kind is RestartKind.SIMULATION_RUN: - stored_simulation = self._restarts.read_simulation_run(run.run_id) - if stored_simulation is None: - return None - return RebuiltCall(self._simulation_call(stored_simulation)) - job = self._households.read(run.run_id) - if job is None: + stored = self._read(run) + if stored is None: return None - return RebuiltCall(self._household_call(job)) + simulation_run_ids = _report_sides(stored) + return RebuiltCall(self._call(stored), simulation_run_ids) except (SQLAlchemyError, VersionManifestUnavailableError) as error: raise DeferredRestartError(type(error).__name__) from error except _REBUILD_ERRORS as error: - if run.kind is RestartKind.REPORT_RUN and not simulation_run_ids: + if run.kind in _REPORT_RUN_KINDS and not simulation_run_ids: simulation_run_ids = self._report_simulation_run_ids(run.run_id) raise CallNotRebuiltError( _rebuild_reason(error), @@ -405,6 +451,26 @@ def build(self, run: UnstartedRun) -> RebuiltCall | None: simulation_run_ids=simulation_run_ids, ) from error + def _read(self, run: UnstartedRun) -> _StoredRow | None: + """Read back what the row's call was built from, or None when it is gone.""" + if run.kind is RestartKind.REPORT_RUN: + return self._restarts.read_report_run(run.run_id) + if run.kind is RestartKind.HOUSEHOLD_REPORT_RUN: + return self._restarts.read_household_report_run(run.run_id) + if run.kind is RestartKind.SIMULATION_RUN: + return self._restarts.read_simulation_run(run.run_id) + return self._households.read(run.run_id) + + def _call(self, stored: _StoredRow) -> CallStart: + """Rebuild the call the stored row's submission started.""" + if isinstance(stored, StoredReportRun): + return self._report_call(stored) + if isinstance(stored, StoredHouseholdReportRun): + return self._household_report_call(stored) + if isinstance(stored, StoredSimulationRun): + return self._simulation_call(stored) + return self._household_call(stored) + def _report_simulation_run_ids(self, report_run_id: UUID) -> tuple[UUID, ...]: """Return the simulation runs a report run its rows can't rebuild names. @@ -565,3 +631,83 @@ def start(observability_context: dict[str, Any] | None) -> Awaitable[str]: ) return start + + def _household_report_call(self, stored: StoredHouseholdReportRun) -> CallStart: + """Rebuild the run's household function call from its rows and its inputs. + + The household and each side's policy are read as the submission read + them, on the run's .py version, and must be on the catalog version + the run's simulation runs were created on. + """ + worker = self._workers.households.worker_for_version( + stored.country, stored.versions.policyengine_version + ) + if ( + worker.bundle.country_package_version + != stored.versions.country_package_version + ): + msg = ( + f"the v2 worker for .py {stored.versions.policyengine_version} " + "calculates on other versions than the run records" + ) + raise _ChangedRunError(msg) + policy_ids: dict[ReportSide, UUID | None] = { + "baseline": stored.baseline.policy_id + } + if stored.reform is not None: + policy_ids["reform"] = stored.reform.policy_id + inputs = self._household_reports.read_inputs( + country=stored.country, + household_id=stored.household_id, + policy_ids=policy_ids, + policyengine_version=stored.versions.policyengine_version, + ) + if inputs.catalog_version_id != stored.catalog_version_id: + msg = "its catalog version changed since it was created" + raise _ChangedRunError(msg) + calculation = household_report_calculation( + household=inputs.household, + year=stored.year, + baseline=HouseholdReportSideInputs( + simulation_run_id=stored.baseline.simulation_run_id, + policy_values=inputs.reforms["baseline"], + ), + reform=( + None + if stored.reform is None + else HouseholdReportSideInputs( + simulation_run_id=stored.reform.simulation_run_id, + policy_values=inputs.reforms["reform"], + ) + ), + worker=worker, + ) + payload = calculation.model_dump(mode="json") + caller = self._callers.households + + def start(observability_context: dict[str, Any] | None) -> Awaitable[str]: + return caller.start_report_run( + worker, + payload, + report_run_id=stored.report_run_id, + observability_context=observability_context, + ) + + return start + + +# The kinds recorded on a report run, which is failed with its simulation runs. +_REPORT_RUN_KINDS: Final = frozenset( + {RestartKind.REPORT_RUN, RestartKind.HOUSEHOLD_REPORT_RUN} +) + + +def _report_sides(stored: _StoredRow) -> tuple[UUID, ...]: + """Return a report run's simulation runs, the baseline's first; else none.""" + if isinstance(stored, StoredReportRun): + return (stored.baseline.simulation_run_id, stored.reform.simulation_run_id) + if isinstance(stored, StoredHouseholdReportRun): + if stored.reform is None: + return (stored.baseline.simulation_run_id,) + return (stored.baseline.simulation_run_id, stored.reform.simulation_run_id) + return () diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py index 1cda73c6c..3807be593 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_check.py @@ -44,6 +44,7 @@ from policyengine_simulation_persistence import ( EconomyRunStore, HouseholdJobStore, + HouseholdReportStore, RunRestartStore, create_simulation_engine, ) @@ -219,6 +220,9 @@ def scheduled_restart_service( households=HouseholdJobStore.from_engine( engine, statement_timeout_milliseconds=timeout ), + household_reports=HouseholdReportStore.from_engine( + engine, statement_timeout_milliseconds=timeout + ), windows=BudgetWindowStore( engine=engine, statement_timeout_milliseconds=timeout ), diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py index 0c5642b62..89f158487 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py @@ -57,6 +57,7 @@ CallNotRebuiltError, DeferredRestartError, HouseholdJobRestarts, + HouseholdReportRestarts, RestartCallBuilder, RestartCallers, RestartSettings, @@ -123,6 +124,12 @@ # bounded statements, so the check ends within the scheduled function's # timeout and no row it marks is left without its call. CHECK_TIME_LIMIT: Final = timedelta(minutes=3) +# The kinds recorded on a report run. A household report run's call and +# failure are written as an economy report run's are, with the same +# conditional writes on ``report_runs``. +_REPORT_RUN_KINDS: Final = frozenset( + {RestartKind.REPORT_RUN, RestartKind.HOUSEHOLD_REPORT_RUN} +) @dataclass(frozen=True) @@ -139,7 +146,10 @@ class RestartSummary: class EconomyRunWrites(Protocol): - """The submission's own call writes on economy runs (``EconomyRunStore``).""" + """The submission's own call writes on report and simulation runs. + + ``EconomyRunStore``'s; a household report run's are the same writes. + """ def record_report_call( self, report_run_id: UUID, modal_call_id: str @@ -226,6 +236,7 @@ class RestartStores: restarts: UnstartedRunSource economy: EconomyRunWrites households: HouseholdJobRestarts + household_reports: HouseholdReportRestarts windows: BudgetWindowAdvancer @@ -255,6 +266,7 @@ def __init__( # noqa: PLR0913 # each seam tests replace self._calls = RestartCallBuilder( restarts=stores.restarts, households=stores.households, + household_reports=stores.household_reports, workers=workers, callers=callers, settings=settings, @@ -421,7 +433,7 @@ def _fail( Return None when the failure couldn't be written. """ try: - if run.kind is RestartKind.REPORT_RUN: + if run.kind in _REPORT_RUN_KINDS: failed = self._stores.economy.fail_unstarted_report( run.run_id, simulation_run_ids, message ) @@ -437,7 +449,7 @@ def _fail( def _record(self, run: UnstartedRun, call_id: str) -> None: """Record the call's ID while the row is pending with no call.""" - if run.kind is RestartKind.REPORT_RUN: + if run.kind in _REPORT_RUN_KINDS: self._stores.economy.record_report_call(run.run_id, call_id) elif run.kind is RestartKind.SIMULATION_RUN: self._stores.economy.record_simulation_call(run.run_id, call_id) From a6753ef9d7b9a010dbc12df2a6e7afd675550afc Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:06:26 +0400 Subject: [PATCH 28/29] Test round 8's household report restart findings (failing) A side's refused policy is counted under its cause's error type. Also covers a policy naming parameters its catalog lacks, a payload the contract refuses, a worker gone from the manifest, rows that no longer describe the run, an unexpected rebuild error, and the Modal worker role reading a household report's inputs. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_run_restart_records.py | 30 +++++ .../test_run_restart_household_reports.py | 105 +++++++++++++++++- 2 files changed, 131 insertions(+), 4 deletions(-) diff --git a/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py b/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py index cf570f41a..481ae938d 100644 --- a/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py +++ b/libs/policyengine-simulation-persistence/tests/test_run_restart_records.py @@ -27,6 +27,7 @@ from policyengine_simulation_persistence import ( BudgetWindowYear, HouseholdReportSimulation, + HouseholdReportStore, IdleBudgetWindow, NewBudgetWindowReport, NewEconomyReport, @@ -804,6 +805,35 @@ def transaction() -> Iterator[Connection]: return RunRestartStore(transaction) +def test_the_modal_worker_role_reads_a_household_reports_inputs_again( + connection, +) -> None: + """The restart reads the household and policies with the submission's reader.""" + economy = insert_economy_catalog(connection, parameter_names=(CTC_AMOUNT,)) + policy_id = insert_policy( + connection, economy.catalog, [StoredValue(CTC_AMOUNT, 3_000, START)] + ) + household_id = _stored_household(connection) + act_as_the_modal_worker(connection) + + @contextmanager + def transaction() -> Iterator[Connection]: + with connection.begin_nested(): + yield connection + + inputs = HouseholdReportStore(transaction).read_inputs( + country="us", + household_id=household_id, + policy_ids={"baseline": None, "reform": policy_id}, + policyengine_version=VERSIONS.policyengine_version, + ) + + assert inputs.catalog_version_id == economy.catalog.version_id + assert inputs.household.id == household_id + assert inputs.reforms["baseline"] == {} + assert set(inputs.reforms["reform"]) == {CTC_AMOUNT} + + def test_the_store_lists_marks_and_reads_in_transactions_of_its_own( connection, savepoint_store ) -> None: diff --git a/projects/policyengine-simulation-entry/tests/test_run_restart_household_reports.py b/projects/policyengine-simulation-entry/tests/test_run_restart_household_reports.py index e649da122..68e009ff6 100644 --- a/projects/policyengine-simulation-entry/tests/test_run_restart_household_reports.py +++ b/projects/policyengine-simulation-entry/tests/test_run_restart_household_reports.py @@ -17,6 +17,7 @@ from typing import Any from uuid import UUID +import pytest from economy_service_fakes import CREATED_AT from household_report_fakes import ( POLICY_ID, @@ -31,11 +32,13 @@ from policyengine_simulation_entry.household_workers import ManifestHouseholdWorkers from policyengine_simulation_entry.run_restarts import RestartOutcome, RestartSummary from policyengine_simulation_persistence import ( + MissingPolicyParametersError, ReportSideError, RestartKind, RunVersions, StoredHouseholdReportRun, StoredHouseholdReportSide, + StoredRunUnreadableError, UnknownHouseholdError, UnknownPolicyError, UnstartedRun, @@ -231,22 +234,116 @@ def test_a_household_report_run_whose_household_is_gone_is_failed() -> None: assert rig.telemetry.outcomes[0][3] == "UnknownHouseholdError" -def test_a_household_report_run_whose_reforms_policy_is_gone_is_failed() -> None: - """A side's refusal is read as its own cause.""" +@pytest.mark.parametrize( + ("refusal", "reason"), + [ + (UnknownPolicyError("no policy"), "its policy is gone"), + ( + MissingPolicyParametersError("no such parameter"), + "its policy names parameters its catalog lacks", + ), + ], + ids=["policy-gone", "parameters-missing"], +) +def test_a_household_report_run_whose_reforms_policy_is_refused_is_failed( + refusal, reason +) -> None: + """A side's refusal is read, and counted, as its own cause.""" stored, _payload, _report_run_id = _submitted_household_report() rig = _household_report_rig( stored, household_reports=FakeHouseholdReportStore( - read_failure=ReportSideError("reform", UnknownPolicyError("no policy")) + read_failure=ReportSideError("reform", refusal) + ), + ) + + summary = rig.restart() + + assert _failed_message(rig, stored).endswith(f": {reason}.") + assert summary == RestartSummary(listed=1, failed=1) + assert rig.telemetry.outcomes[0][3] == type(refusal).__name__ + + +def test_a_household_report_run_whose_sides_name_one_run_is_failed() -> None: + """The contract refuses a payload the service built wrong.""" + stored, _payload, _report_run_id = _submitted_household_report() + assert stored.reform is not None + stored = replace( + stored, + reform=replace( + stored.reform, simulation_run_id=stored.baseline.simulation_run_id + ), + ) + rig = _household_report_rig(stored) + + summary = rig.restart() + + assert rig.household_caller.report_runs == [] + ((_run_id, _sides_failed, message),) = rig.economy.failed_runs + assert message.endswith(": its household report request no longer validates.") + assert summary == RestartSummary(listed=1, failed=1) + assert rig.telemetry.outcomes[0][3] == "HouseholdReportBuildError" + + +def test_a_household_report_run_whose_worker_left_the_manifest_is_failed() -> None: + stored, _payload, _report_run_id = _submitted_household_report() + stored = replace( + stored, + versions=RunVersions( + country_package_version=stored.versions.country_package_version, + policyengine_version="0.0.1", ), ) + rig = _household_report_rig(stored) + + summary = rig.restart() + + assert _failed_message(rig, stored).endswith( + ": the v2 version manifest has no worker for .py 0.0.1." + ) + assert summary == RestartSummary(listed=1, failed=1) + + +def test_a_household_report_run_whose_rows_no_longer_describe_it_is_failed() -> None: + """It is failed with the simulation runs it names, read on their own.""" + stored, _payload, _report_run_id = _submitted_household_report(reform=None) + rig = _household_report_rig(stored) + rig.restarts.read_failures[stored.report_run_id] = StoredRunUnreadableError( + "report run names a missing simulation run" + ) + rig.restarts.report_simulation_run_ids[stored.report_run_id] = ( + stored.baseline.simulation_run_id, + ) summary = rig.restart() - assert _failed_message(rig, stored).endswith(": its policy is gone.") + assert _failed_message(rig, stored).endswith( + ": its stored rows no longer describe one calculation." + ) assert summary == RestartSummary(listed=1, failed=1) +def test_an_unexpected_rebuild_error_defers_a_household_report_run(caplog) -> None: + stored, _payload, _report_run_id = _submitted_household_report() + rig = _household_report_rig( + stored, + household_reports=FakeHouseholdReportStore( + read_failure=RuntimeError("unforeseen") + ), + ) + + summary = rig.restart() + + assert rig.restarts.marked == [] + assert rig.economy.failed_runs == [] + assert summary == RestartSummary(listed=1, deferred=1) + assert [ + record.policyengine_attributes + for record in caplog.records + if record.getMessage() == "v2_run_restart_unexpected_error" + ] == [{"run_kind": "household_report_run", "run_id": str(stored.report_run_id)}] + + def test_a_household_report_run_whose_household_cannot_be_calculated_is_failed() -> ( None ): From c069a2fa54b08433620d3f0c8a84b4172cea8777 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:08:01 +0400 Subject: [PATCH 29/29] Fix round 8's household report restart findings A report side's refused policy is counted under its cause's error type, as its stored reason already names the cause. The kinds recorded on a report run are defined once (REPORT_RUN_KINDS), so a report run is never recorded or failed on household_jobs. The listing's docstring names household report runs, and the household report service names the restart check's own module. Co-Authored-By: Claude Opus 5.5 --- .../stage-12-modal-worker-foundation.md | 5 +++-- .../run_restart_records.py | 7 ++++--- .../household_report_submissions.py | 2 +- .../run_restart_calls.py | 18 ++++++++++++------ .../run_restarts.py | 11 +++-------- 5 files changed, 23 insertions(+), 20 deletions(-) diff --git a/docs/migration/stage-12-modal-worker-foundation.md b/docs/migration/stage-12-modal-worker-foundation.md index c8ec95789..6c96875cd 100644 --- a/docs/migration/stage-12-modal-worker-foundation.md +++ b/docs/migration/stage-12-modal-worker-foundation.md @@ -872,8 +872,9 @@ function call with `household_report_calculation`, started with `start_report_run`. A run whose .py version's catalog version is no longer the one its simulation runs were created on, or whose household or policy is gone or can no longer be calculated, is failed with `run_not_started` -and why, with its simulation runs. Each check is a -`v2_run_restart_check` operation and each row it takes up a +and why, with its simulation runs. + +Each check is a `v2_run_restart_check` operation and each row it takes up a `v2_run_restart` operation, with the row's kind, ID, age and outcome, and each outcome is also logged as a `v2_run_restart_outcome` event. A deferred row is taken up again by the next check, so the distinct run IDs are the diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py index 85ce3ec50..3fb26e93f 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/run_restart_records.py @@ -313,9 +313,10 @@ def list_unstarted_runs( """Return the oldest restartable rows created in the window, at most ``limit``. A row is restartable while it is pending with no Modal call recorded and - nothing has changed it since its insert. Report runs and standalone - simulation runs are listed only for economy simulations, and report runs - only outside a budget window. + nothing has changed it since its insert. Report runs are listed only + outside a budget window: an economy report run as ``report_run``, a + household report run as ``household_report_run``. Standalone simulation + runs are listed only for economy simulations. """ listed: list[UnstartedRun] = [] for kind, table in RESTART_TABLES.items(): diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_report_submissions.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_report_submissions.py index 25779c47f..dd6de0ecb 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_report_submissions.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_report_submissions.py @@ -24,7 +24,7 @@ Whatever is missing or cannot be calculated is refused in steps 1 and 2, before anything is written. The household function then claims the report run and each simulation run, writes each run's household document, and -finishes the report run. The restart check (``run_restart_calls``) starts +finishes the report run. The restart check (``run_restarts``) starts a report run whose call was never recorded once more, and the stuck-run sweep fails it when that start is lost too. The errors are the economy routes' (``economy_submission_errors``), so ``POST /reports`` answers both kinds diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py index 2b1b33f42..1fc27a3eb 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restart_calls.py @@ -345,9 +345,13 @@ class _ChangedRunError(LookupError): _REBUILD_ERRORS: Final = tuple(_REBUILD_REASONS) +def _rebuild_cause(error: Exception) -> Exception: + """Return the error that says why: a report side's refusal, or the error.""" + return error.error if isinstance(error, ReportSideError) else error + + def _rebuild_reason(error: Exception) -> str: - if isinstance(error, ReportSideError): - return _rebuild_reason(error.error) + error = _rebuild_cause(error) reason = next( ( reason @@ -443,11 +447,11 @@ def build(self, run: UnstartedRun) -> RebuiltCall | None: except (SQLAlchemyError, VersionManifestUnavailableError) as error: raise DeferredRestartError(type(error).__name__) from error except _REBUILD_ERRORS as error: - if run.kind in _REPORT_RUN_KINDS and not simulation_run_ids: + if run.kind in REPORT_RUN_KINDS and not simulation_run_ids: simulation_run_ids = self._report_simulation_run_ids(run.run_id) raise CallNotRebuiltError( _rebuild_reason(error), - error_type=type(error).__name__, + error_type=type(_rebuild_cause(error)).__name__, simulation_run_ids=simulation_run_ids, ) from error @@ -696,8 +700,10 @@ def start(observability_context: dict[str, Any] | None) -> Awaitable[str]: return start -# The kinds recorded on a report run, which is failed with its simulation runs. -_REPORT_RUN_KINDS: Final = frozenset( +# The kinds recorded on a report run, failed with its simulation runs. A +# household report run's call and failure are written as an economy report +# run's are, with the same conditional writes on ``report_runs``. +REPORT_RUN_KINDS: Final = frozenset( {RestartKind.REPORT_RUN, RestartKind.HOUSEHOLD_REPORT_RUN} ) diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py index 89f158487..f55abe8e2 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/run_restarts.py @@ -54,6 +54,7 @@ from sqlalchemy.exc import SQLAlchemyError from policyengine_simulation_entry.run_restart_calls import ( + REPORT_RUN_KINDS, CallNotRebuiltError, DeferredRestartError, HouseholdJobRestarts, @@ -124,12 +125,6 @@ # bounded statements, so the check ends within the scheduled function's # timeout and no row it marks is left without its call. CHECK_TIME_LIMIT: Final = timedelta(minutes=3) -# The kinds recorded on a report run. A household report run's call and -# failure are written as an economy report run's are, with the same -# conditional writes on ``report_runs``. -_REPORT_RUN_KINDS: Final = frozenset( - {RestartKind.REPORT_RUN, RestartKind.HOUSEHOLD_REPORT_RUN} -) @dataclass(frozen=True) @@ -433,7 +428,7 @@ def _fail( Return None when the failure couldn't be written. """ try: - if run.kind in _REPORT_RUN_KINDS: + if run.kind in REPORT_RUN_KINDS: failed = self._stores.economy.fail_unstarted_report( run.run_id, simulation_run_ids, message ) @@ -449,7 +444,7 @@ def _fail( def _record(self, run: UnstartedRun, call_id: str) -> None: """Record the call's ID while the row is pending with no call.""" - if run.kind in _REPORT_RUN_KINDS: + if run.kind in REPORT_RUN_KINDS: self._stores.economy.record_report_call(run.run_id, call_id) elif run.kind is RestartKind.SIMULATION_RUN: self._stores.economy.record_simulation_call(run.run_id, call_id)