From 18beb508390675e7e1acf6280c180599fe2d3e05 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:35:30 +0400 Subject: [PATCH 01/24] Report run completion leaves its four cliff columns null (failing) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A cliff impact becomes each simulation run's own cliff result (C1 · 24's answer 4b), so the report run's completion must neither set nor name baseline_cliff_gap, baseline_cliff_share, reform_cliff_gap and reform_cliff_share, and API v1 can then drop them. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_report_run_results.py | 56 ++++++++++++++----- 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/libs/policyengine-stage12-persistence/tests/test_report_run_results.py b/libs/policyengine-stage12-persistence/tests/test_report_run_results.py index e2cfeabdc..3910c7cfc 100644 --- a/libs/policyengine-stage12-persistence/tests/test_report_run_results.py +++ b/libs/policyengine-stage12-persistence/tests/test_report_run_results.py @@ -17,6 +17,7 @@ uk_output_with_withheld_codes, us_output_with_every_part, ) +from policyengine_simulation_contract.stage12_execution import ReportRunIds from policyengine_simulation_persistence.result_tables import BudgetSummary from policyengine_stage12_persistence import ( DistrictCodes, @@ -159,19 +160,6 @@ def _rebuilt_poverty(seeder, run_ids, poverty_types): return {name: parts.get(name) for name in POVERTY_PART_NAMES} -def _rebuilt_cliff_impact(report_run): - """Rebuild today's cliff impact from the report run's four columns.""" - if report_run["baseline_cliff_gap"] is None: - return None - return { - side: { - "cliff_gap": report_run[f"{side}_cliff_gap"], - "cliff_share": report_run[f"{side}_cliff_share"], - } - for side in ("baseline", "reform") - } - - def _rebuilt_labor_supply(seeder, report_run_id): """Rebuild today's labour supply response from its two tables' rows. @@ -283,9 +271,11 @@ def test_a_report_run_stores_its_economy_output_as_rows( output["inequality"]["top_10_pct_share"][side], output["inequality"]["top_1_pct_share"][side], ) - # The cliff impact is four of the report run's own columns. + # A cliff impact is each simulation run's own cliff result, which the + # report run's completion never writes. report_run = seeder.row("report_runs", run_ids.report_run_id) - assert _rebuilt_cliff_impact(report_run) == output["cliff_impact"] + assert output["cliff_impact"] is not None + assert {report_run[column] for column in CLIFF_COLUMNS} == {None} assert report_run["data_version"] == output["data_version"] assert ( _rebuilt_labor_supply(seeder, run_ids.report_run_id) @@ -317,6 +307,42 @@ def test_the_labour_supply_response_is_one_row_and_a_row_per_decile( ) == (-0.05, 0.1, None, None) +def test_a_report_runs_result_columns_never_name_its_cliff_columns() -> None: + """The completion's update names none of them, so the Public API can drop them.""" + run_ids = ReportRunIds( + report_run_id=uuid4(), + baseline_simulation_run_id=uuid4(), + reform_simulation_run_id=uuid4(), + ) + + result = report_run_result( + US_ECONOMY_OUTPUT, + run_ids=run_ids, + country="us", + district_codes=DISTRICT_CODES, + ) + + assert US_ECONOMY_OUTPUT["cliff_impact"] is not None + assert set(result.report_run_columns) == {"spm_provenance", "data_version"} + + +def test_a_completed_report_run_leaves_its_cliff_columns_null( + store, seeder, seeded_run: SeededReportRun +) -> None: + output = deepcopy(US_ECONOMY_OUTPUT) + output["cliff_impact"] = { + "baseline": {"cliff_gap": 120.5, "cliff_share": 0.04}, + "reform": {"cliff_gap": 98.25, "cliff_share": 0.03}, + } + + assert _complete(store, seeded_run, output=output) == RunWrite( + changed=True, status=RunStatus.SUCCEEDED + ) + + report_run = seeder.row("report_runs", seeded_run.run_ids.report_run_id) + assert {report_run[column] for column in CLIFF_COLUMNS} == {None} + + def test_an_output_without_labour_supply_or_cliff_impact_stores_neither( store, seeder, seeded_run: SeededReportRun ) -> None: From 36a5c94037e87d90cd86c8d4479179fb127cad2b Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:36:42 +0400 Subject: [PATCH 02/24] Stop writing report_runs' four cliff columns on completion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A report run's completion now sets only spm_provenance and data_version of its own columns. Its update names none of the cliff columns, which stay null, so API v1 can drop them (C1 · 41). Co-Authored-By: Claude Opus 5.5 --- .../README.md | 4 +-- .../report_run_results.py | 32 +++++-------------- 2 files changed, 10 insertions(+), 26 deletions(-) diff --git a/libs/policyengine-stage12-persistence/README.md b/libs/policyengine-stage12-persistence/README.md index 04543a512..bdeeeb042 100644 --- a/libs/policyengine-stage12-persistence/README.md +++ b/libs/policyengine-stage12-persistence/README.md @@ -30,8 +30,8 @@ and ends the process with a failure if the check runs for 15 minutes. - **The Stage 12 comparison tables** (`Stage12PersistenceStore`): temporary state of the comparison path, removed with it. - **The Public API's run tables** (`ReportRunStore`): `report_runs`, which - names its baseline and reform simulation runs and holds the cliff impact, - SPM receipts and data version, `simulation_runs`, the result tables, the + names its baseline and reform simulation runs and holds the SPM receipts + and data version, `simulation_runs`, the result tables, the two labour supply tables included, and `household_jobs`. As the Modal worker role (`policyengine_v2_modal_worker`), the v2 Modal coordinator claims and records the report run its call names, each v2 worker claims diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_results.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_results.py index 299b02efe..37be46bf6 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_results.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_results.py @@ -4,8 +4,10 @@ in the format API v1 and v3 read today. Each part with a lossless place in a result table becomes rows: per-report parts keyed by the report run, with its two simulation runs where the table names them, and poverty and inequality -keyed by each simulation run. The cliff impact, SPM receipts and data version -are columns of the report run itself, which its completion sets. +keyed by each simulation run. The SPM receipts and data version are columns +of the report run itself, which its completion sets. A cliff impact is not +stored here: each simulation run has its own cliff result, which the +worker that calculates it writes. The result tables' models are the simulation service library's (``policyengine_simulation_persistence.result_tables``), which the CI @@ -87,14 +89,6 @@ "Gain less than 5%": "gain_less_than_5pct", "Gain more than 5%": "gain_more_than_5pct", } -# The report run's columns for the baseline's and reform's cliff impact; all -# four are set or all four null. -CLIFF_COLUMNS = ( - "baseline_cliff_gap", - "baseline_cliff_share", - "reform_cliff_gap", - "reform_cliff_share", -) DISCLOSURE_FIELDS = ( "disclosure_tier", "average_household_income_change_lower", @@ -236,24 +230,14 @@ def report_run_result( def _report_run_columns(economy: SingleYearMacroOutput) -> dict[str, Any]: - """Return the cliff impact, SPM receipts and data version, as report run columns. + """Return the SPM receipts and data version, as report run columns. - The four cliff columns are all null when the output has no cliff impact. + No cliff column: a cliff impact is each simulation run's own cliff + result, so the report run's four cliff columns stay null, and its + completion names none of them, which lets the Public API drop them. """ - cliffs = economy.cliff_impact - cliff_columns: dict[str, float | None] - if cliffs is None: - cliff_columns = dict.fromkeys(CLIFF_COLUMNS) - else: - cliff_columns = { - "baseline_cliff_gap": cliffs.baseline.cliff_gap, - "baseline_cliff_share": cliffs.baseline.cliff_share, - "reform_cliff_gap": cliffs.reform.cliff_gap, - "reform_cliff_share": cliffs.reform.cliff_share, - } receipts = economy.spm_provenance return { - **cliff_columns, "spm_provenance": ( None if receipts is None else receipts.model_dump(mode="json") ), From 1c159cc508ce78152a29b9fd99839207462f13c2 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:43:03 +0400 Subject: [PATCH 03/24] A worker calculates the cliff result its call names, once (failing) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A cliff result belongs to one simulation run (C1 · 24's answer 4b). A simulation worker's call that names one in its context claims it under the call's Modal call ID, checks that its payload is that run's simulation on the run's year and versions, calculates the simulation with the cliff columns, and writes the weighted cliff gap and cliff share, or its failure. A second start, or a restart after success, calculates nothing. A US national calculation is split into the region groups a plain run uses. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_stage12_report_run_ids.py | 44 +- .../tests/stage12_fake_cliff_results.py | 118 ++++++ .../tests/test_stage12_cliff_results.py | 375 ++++++++++++++++++ .../tests/test_stage12_output_planning.py | 43 ++ 4 files changed, 579 insertions(+), 1 deletion(-) create mode 100644 projects/policyengine-simulation-executor/tests/stage12_fake_cliff_results.py create mode 100644 projects/policyengine-simulation-executor/tests/test_stage12_cliff_results.py diff --git a/libs/policyengine-simulation-contract/tests/test_stage12_report_run_ids.py b/libs/policyengine-simulation-contract/tests/test_stage12_report_run_ids.py index 859ccc426..70568bd1f 100644 --- a/libs/policyengine-simulation-contract/tests/test_stage12_report_run_ids.py +++ b/libs/policyengine-simulation-contract/tests/test_stage12_report_run_ids.py @@ -18,6 +18,7 @@ REPORT_RUN_ID = UUID("00000000-0000-0000-0000-0000000000a1") BASELINE_RUN_ID = UUID("00000000-0000-0000-0000-0000000000b1") REFORM_RUN_ID = UUID("00000000-0000-0000-0000-0000000000c1") +CLIFF_RESULT_ID = UUID("00000000-0000-0000-0000-0000000000f1") def _context_fields(**overrides: Any) -> dict[str, Any]: @@ -141,7 +142,7 @@ def test_only_a_detailed_save_call_records_runs() -> None: def test_a_context_names_a_report_run_or_a_simulation_run_not_both() -> None: - with pytest.raises(ValidationError, match="report run or a simulation run"): + with pytest.raises(ValidationError, match="one of a report run, a simulation run"): Stage12InvocationContext( **_context_fields( report_run=_run_ids().model_dump(mode="json"), @@ -150,6 +151,47 @@ def test_a_context_names_a_report_run_or_a_simulation_run_not_both() -> None: ) +def test_a_cliff_workers_context_names_the_cliff_result_its_call_claims() -> None: + context = Stage12InvocationContext( + **_context_fields(cliff_result_id=str(CLIFF_RESULT_ID)) + ) + + assert context.cliff_result_id == CLIFF_RESULT_ID + assert (context.report_run, context.simulation_run_id) == (None, None) + assert context.records_runs + restored = Stage12InvocationContext.model_validate(context.model_dump(mode="json")) + assert restored.cliff_result_id == CLIFF_RESULT_ID + # A context naming no cliff result keeps its JSON, for the deployed workers. + assert "cliff_result_id" not in Stage12InvocationContext( + **_context_fields(simulation_run_id=str(BASELINE_RUN_ID)) + ).model_dump(mode="json", exclude_none=True) + + +@pytest.mark.parametrize( + "named_run", + [ + {"report_run": _run_ids().model_dump(mode="json")}, + {"simulation_run_id": str(BASELINE_RUN_ID)}, + ], + ids=["report-run", "simulation-run"], +) +def test_a_context_naming_a_cliff_result_names_no_run(named_run) -> None: + with pytest.raises(ValidationError, match="one of a report run, a simulation run"): + Stage12InvocationContext( + **_context_fields(cliff_result_id=str(CLIFF_RESULT_ID), **named_run) + ) + + +def test_a_shadow_comparison_run_cannot_record_a_cliff_result() -> None: + with pytest.raises(ValidationError, match="production function call"): + Stage12InvocationContext( + **_context_fields( + production_function_call_id="fc-production", + cliff_result_id=str(CLIFF_RESULT_ID), + ) + ) + + def test_a_shadow_comparison_run_cannot_record_a_simulation_run() -> None: with pytest.raises(ValidationError, match="production function call"): Stage12InvocationContext( diff --git a/projects/policyengine-simulation-executor/tests/stage12_fake_cliff_results.py b/projects/policyengine-simulation-executor/tests/stage12_fake_cliff_results.py new file mode 100644 index 000000000..f40d1f8db --- /dev/null +++ b/projects/policyengine-simulation-executor/tests/stage12_fake_cliff_results.py @@ -0,0 +1,118 @@ +"""In-memory cliff results beside the run tables, for the cliff calculation's tests. + +``FakeCliffResults`` applies the same conditions as ``CliffResultStore``'s +conditional updates, which the persistence library tests against Postgres. +A cliff result names the simulation run it is calculated for; the call's +payload is that run's simulation input. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from uuid import UUID + +from policyengine_simulation_contract.stage12_execution import ( + SimulationExecutionInput, + SimulationRole, + Stage12InvocationContext, +) +from policyengine_stage12_persistence import CliffResultState, RunStatus, RunWrite +from stage12_fake_report_runs import ( + BASELINE_RUN_ID, + REPORT_RUN_PREFIX, + FakeReportRuns, + report_run_simulation, +) +from stage12_runtime_support import invocation_context + +CLIFF_RESULT_ID = UUID("00000000-0000-0000-0000-0000000000f1") + + +def cliff_simulation( + role: SimulationRole = SimulationRole.BASELINE, +) -> SimulationExecutionInput: + """Return the simulation input of the run the cliff result is for.""" + return report_run_simulation(role) + + +def cliff_result_context( + cliff_result_id: UUID = CLIFF_RESULT_ID, +) -> Stage12InvocationContext: + """Return a worker's context, naming the cliff result its call claims.""" + return invocation_context().model_copy( + update={ + "cliff_result_id": cliff_result_id, + "artifact_prefix": REPORT_RUN_PREFIX, + } + ) + + +@dataclass +class FakeCliffResult: + """One cliff result's mutable columns, and the run it is calculated for.""" + + simulation_run_id: UUID = BASELINE_RUN_ID + status: RunStatus = RunStatus.PENDING + modal_call_id: str | None = None + error_message: str | None = None + cliff_gap: float | None = None + cliff_share: float | None = None + writes: list[str] = field(default_factory=list) + + +class FakeCliffResults(FakeReportRuns): + """The run tables and their cliff results, held in memory, written conditionally.""" + + def __init__(self, events: list[str] | None = None) -> None: + super().__init__(events) + self.cliff_results = {CLIFF_RESULT_ID: FakeCliffResult()} + + def get_cliff_result(self, cliff_result_id): + """Read a cliff result; a read is not a write, so it records nothing.""" + result = self._run(self.cliff_results, cliff_result_id, "cliff result") + return CliffResultState( + simulation_run_id=result.simulation_run_id, + status=result.status, + modal_call_id=result.modal_call_id, + cliff_gap=result.cliff_gap, + cliff_share=result.cliff_share, + error_message=result.error_message, + ) + + def claim_cliff_result( + self, cliff_result_id, *, modal_call_id, claimed_at + ) -> RunWrite: + result = self._run(self.cliff_results, cliff_result_id, "cliff result") + return self._apply( + result, + "claim_cliff_result", + condition=lambda: self._claimable(result, modal_call_id), + status=RunStatus.RUNNING, + modal_call_id=modal_call_id, + ) + + def complete_cliff_result( + self, cliff_result_id, *, modal_call_id, cliff_gap, cliff_share, completed_at + ) -> RunWrite: + result = self._run(self.cliff_results, cliff_result_id, "cliff result") + return self._apply( + result, + "complete_cliff_result", + condition=lambda: self._holds(result, modal_call_id), + status=RunStatus.SUCCEEDED, + cliff_gap=cliff_gap, + cliff_share=cliff_share, + error_message=None, + ) + + def fail_cliff_result( + self, cliff_result_id, *, modal_call_id, error_message, failed_at + ) -> RunWrite: + result = self._run(self.cliff_results, cliff_result_id, "cliff result") + return self._apply( + result, + "fail_cliff_result", + condition=lambda: self._holds(result, modal_call_id), + status=RunStatus.FAILED, + error_message=error_message, + ) diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_cliff_results.py b/projects/policyengine-simulation-executor/tests/test_stage12_cliff_results.py new file mode 100644 index 000000000..9bd46a4a7 --- /dev/null +++ b/projects/policyengine-simulation-executor/tests/test_stage12_cliff_results.py @@ -0,0 +1,375 @@ +"""A worker calculates the cliff result its call names, once, under its own call ID. + +A cliff result belongs to one simulation run. The worker claims it, checks +that the call's payload is that run's simulation on the run's versions, +calculates the simulation with the cliff columns, and writes the run's +cliff gap and cliff share, or the failure. It writes nothing to the +simulation run itself, to any report run, or to the comparison store. +""" + +from __future__ import annotations + +import pandas as pd +import pytest +from policyengine_simulation_contract.stage12_execution import ( + EntityOutputPlan, + GeographySelection, + ReportAggregate, + ReportOutputRequirements, + SimulationRole, + Stage12OutputPlan, +) +from policyengine_simulation_executor.stage12_runtime import ( + SimulationCalculation, + SimulationWorkerDependencies, + run_single_simulation, +) +from policyengine_simulation_executor.stage12_runtime import ( + cliff_results as cliff_results_module, +) +from policyengine_stage12_persistence import RunStatus +from stage12_fake_cliff_results import ( + CLIFF_RESULT_ID, + FakeCliffResults, + cliff_result_context, + cliff_simulation, +) +from stage12_fake_report_runs import ( + BASELINE_RUN_ID, + REFORM_RUN_ID, + ComparisonStoreNotUsed, +) +from stage12_fake_stores import FakeArtifacts + +WORKER_CALL_ID = "fc-cliff-worker" +OTHER_CALL_ID = "fc-other-cliff-worker" +CLIFF_VARIABLES = ("cliff_gap", "is_adult", "is_on_cliff") +# Three people with different weights: the gap and both counts are weighted +# sums, so an unweighted calculation gives 125.0 and a share of 1.0. +PERSON_WEIGHTS = (1.0, 3.0, 2.0) +WEIGHTED_CLIFF_GAP = 100.0 * 1.0 + 0.0 * 3.0 + 25.0 * 2.0 +# People on a cliff (the first and third) over adults (the first two). +WEIGHTED_CLIFF_SHARE = (1.0 + 2.0) / (1.0 + 3.0) + + +def cliff_plan(country: str = "us") -> Stage12OutputPlan: + """Return a cliff calculation's plan: the person columns the share reads.""" + return Stage12OutputPlan( + country=country, + requirements=ReportOutputRequirements( + aggregates=tuple(ReportAggregate), + include_cliff_impacts=True, + labor_supply_response_active=False, + ), + entities=( + EntityOutputPlan( + entity="person", + materialized_variables=tuple( + sorted((*CLIFF_VARIABLES, "person_id", "person_weight")) + ), + additional_variables=CLIFF_VARIABLES, + ), + ), + ) + + +def cliff_frames() -> dict[str, pd.DataFrame]: + """Return a calculation's person rows, with their weights and cliff columns.""" + return { + "person": pd.DataFrame( + { + "person_id": [1, 2, 3], + "person_weight": list(PERSON_WEIGHTS), + "cliff_gap": [100.0, 0.0, 25.0], + "is_on_cliff": [True, False, True], + "is_adult": [True, True, False], + } + ) + } + + +class RecordingCalculator: + """A calculator returning ``frames`` and recording each simulation it gets.""" + + def __init__(self, frames=None) -> None: + self.frames = frames if frames is not None else cliff_frames() + self.simulations = [] + + def __call__(self, simulation): + self.simulations.append(simulation) + return self.frames + + +@pytest.fixture(autouse=True) +def _cliff_plan(monkeypatch): + """Plan every cliff calculation without loading a country model.""" + monkeypatch.setattr( + cliff_results_module, + "resolve_cliff_output_plan", + lambda simulation: cliff_plan(simulation.geography.country), + ) + + +def _run_cliff_worker( + cliff_results, *, simulation=None, context=None, **dependencies +) -> dict: + simulation = simulation or cliff_simulation() + dependencies.setdefault("store", ComparisonStoreNotUsed()) + dependencies.setdefault("artifacts", FakeArtifacts()) + dependencies.setdefault("calculator", RecordingCalculator()) + return run_single_simulation( + simulation.model_dump(mode="json"), + (context or cliff_result_context()).model_dump(mode="json"), + required_country=simulation.geography.country, + modal_call_id=dependencies.pop("modal_call_id", WORKER_CALL_ID), + dependencies=SimulationWorkerDependencies( + report_runs=cliff_results, **dependencies + ), + ) + + +def test_a_worker_writes_its_runs_weighted_cliff_gap_and_share() -> None: + cliff_results = FakeCliffResults() + calculator = RecordingCalculator() + + returned = _run_cliff_worker(cliff_results, calculator=calculator) + + result = cliff_results.cliff_results[CLIFF_RESULT_ID] + assert result.status is RunStatus.SUCCEEDED + assert result.modal_call_id == WORKER_CALL_ID + assert result.writes == ["claim_cliff_result", "complete_cliff_result"] + assert result.cliff_gap == pytest.approx(WEIGHTED_CLIFF_GAP) + assert result.cliff_share == pytest.approx(WEIGHTED_CLIFF_SHARE) + assert returned == { + "cliff_result_id": str(CLIFF_RESULT_ID), + "simulation_run_id": str(BASELINE_RUN_ID), + "status": "succeeded", + "cliff_gap": pytest.approx(WEIGHTED_CLIFF_GAP), + "cliff_share": pytest.approx(WEIGHTED_CLIFF_SHARE), + } + # It calculated the run's own simulation, planned with the cliff columns. + (calculated,) = calculator.simulations + assert calculated.simulation_execution_id == BASELINE_RUN_ID + assert calculated.output_plan == cliff_plan() + # The simulation runs and the report run are not the cliff worker's. + assert cliff_results.simulation_runs[BASELINE_RUN_ID].writes == [] + assert cliff_results.simulation_runs[REFORM_RUN_ID].writes == [] + assert all(not run.writes for run in cliff_results.report_runs.values()) + + +def test_a_succeeded_cliff_result_is_returned_without_calculating_again() -> None: + cliff_results = FakeCliffResults() + first_calculator = RecordingCalculator() + _run_cliff_worker(cliff_results, calculator=first_calculator) + later_calculator = RecordingCalculator() + + # Modal restarts the call that completed it, under the same call ID. + returned = _run_cliff_worker(cliff_results, calculator=later_calculator) + + assert later_calculator.simulations == [] + result = cliff_results.cliff_results[CLIFF_RESULT_ID] + assert result.writes == [ + "claim_cliff_result", + "complete_cliff_result", + "claim_cliff_result", + ] + assert returned["status"] == "succeeded" + assert returned["cliff_gap"] == pytest.approx(WEIGHTED_CLIFF_GAP) + assert returned["cliff_share"] == pytest.approx(WEIGHTED_CLIFF_SHARE) + + +def test_two_starts_of_one_cliff_result_calculate_it_once() -> None: + cliff_results = FakeCliffResults() + first_calculator = RecordingCalculator() + second_calculator = RecordingCalculator() + + _run_cliff_worker(cliff_results, calculator=first_calculator) + returned = _run_cliff_worker( + cliff_results, calculator=second_calculator, modal_call_id=OTHER_CALL_ID + ) + + assert len(first_calculator.simulations) == 1 + assert second_calculator.simulations == [] + result = cliff_results.cliff_results[CLIFF_RESULT_ID] + # The second call's claim changed nothing, and it wrote nothing else. + assert result.writes == [ + "claim_cliff_result", + "complete_cliff_result", + "claim_cliff_result", + ] + assert result.modal_call_id == WORKER_CALL_ID + assert returned == { + "deduplicated": True, + "cliff_result_id": str(CLIFF_RESULT_ID), + "status": "succeeded", + } + + +def test_a_cliff_result_another_call_holds_is_left_to_that_call() -> None: + cliff_results = FakeCliffResults() + result = cliff_results.cliff_results[CLIFF_RESULT_ID] + result.status = RunStatus.RUNNING + result.modal_call_id = OTHER_CALL_ID + calculator = RecordingCalculator() + + returned = _run_cliff_worker(cliff_results, calculator=calculator) + + assert calculator.simulations == [] + assert result.status is RunStatus.RUNNING + assert result.modal_call_id == OTHER_CALL_ID + assert result.writes == ["claim_cliff_result"] + assert returned == { + "deduplicated": True, + "cliff_result_id": str(CLIFF_RESULT_ID), + "status": "running", + } + + +def test_a_failed_calculation_records_its_failure_on_the_cliff_result() -> None: + cliff_results = FakeCliffResults() + + def failing_calculation(_simulation): + msg = "the calculation ran out of memory" + raise RuntimeError(msg) + + with pytest.raises(RuntimeError, match=r"^Stage 12 cliff calculation failed$"): + _run_cliff_worker(cliff_results, calculator=failing_calculation) + + result = cliff_results.cliff_results[CLIFF_RESULT_ID] + assert result.status is RunStatus.FAILED + assert result.writes == ["claim_cliff_result", "fail_cliff_result"] + assert result.error_message.startswith("cliff_calculation_failed: ") + assert (result.cliff_gap, result.cliff_share) == (None, None) + assert cliff_results.simulation_runs[BASELINE_RUN_ID].writes == [] + + +def test_a_payload_for_another_simulation_run_fails_the_cliff_result() -> None: + cliff_results = FakeCliffResults() + calculator = RecordingCalculator() + + with pytest.raises(RuntimeError, match="cliff calculation failed"): + _run_cliff_worker( + cliff_results, + simulation=cliff_simulation(SimulationRole.REFORM), + calculator=calculator, + ) + + assert calculator.simulations == [] + result = cliff_results.cliff_results[CLIFF_RESULT_ID] + assert result.status is RunStatus.FAILED + assert result.error_message.startswith("cliff_result_input_mismatch: ") + + +@pytest.mark.parametrize( + "stored_version", + [ + {"policyengine_version": "5.1.0"}, + {"country_package_version": "1.700.0"}, + {"year": 2025}, + ], +) +def test_a_run_calculated_on_other_versions_fails_its_cliff_result( + stored_version, +) -> None: + cliff_results = FakeCliffResults() + run = cliff_results.simulation_runs[BASELINE_RUN_ID] + for column, value in stored_version.items(): + setattr(run, column, value) + calculator = RecordingCalculator() + + with pytest.raises(RuntimeError, match="cliff calculation failed"): + _run_cliff_worker(cliff_results, calculator=calculator) + + assert calculator.simulations == [] + result = cliff_results.cliff_results[CLIFF_RESULT_ID] + assert result.status is RunStatus.FAILED + assert result.error_message.startswith("cliff_result_input_mismatch: ") + + +def test_a_cliff_result_failed_meanwhile_takes_no_values() -> None: + cliff_results = FakeCliffResults() + result = cliff_results.cliff_results[CLIFF_RESULT_ID] + + def calculation_then_sweep(_simulation): + # The stuck-run sweep fails the cliff result while it calculates. + result.status = RunStatus.FAILED + result.error_message = "stuck_run: the call ended" + return cliff_frames() + + with pytest.raises( + RuntimeError, match=r"^Stage 12 cliff calculation was cancelled$" + ): + _run_cliff_worker(cliff_results, calculator=calculation_then_sweep) + + assert result.writes == ["claim_cliff_result", "complete_cliff_result"] + assert result.error_message == "stuck_run: the call ended" + assert (result.cliff_gap, result.cliff_share) == (None, None) + + +def test_a_us_national_cliff_calculation_is_split_by_region_group() -> None: + cliff_results = FakeCliffResults() + segmented = [] + calculator = RecordingCalculator() + + def segmented_calculator(simulation, cancellation_requested): + segmented.append(simulation) + assert cancellation_requested() is False + return SimulationCalculation(frames=cliff_frames()) + + _run_cliff_worker( + cliff_results, + calculator=calculator, + segmented_calculator=segmented_calculator, + ) + + (split,) = segmented + assert split.geography.region == "us" + assert split.output_plan.requirements.include_cliff_impacts is True + assert calculator.simulations == [] + result = cliff_results.cliff_results[CLIFF_RESULT_ID] + assert result.cliff_gap == pytest.approx(WEIGHTED_CLIFF_GAP) + assert result.cliff_share == pytest.approx(WEIGHTED_CLIFF_SHARE) + + +@pytest.mark.parametrize( + ("region", "options"), + [("state/ut", {}), ("us", {"segmented": False})], +) +def test_a_state_or_unsegmented_cliff_calculation_runs_in_one_worker( + region, options +) -> None: + cliff_results = FakeCliffResults() + simulation = cliff_simulation().model_copy( + update={ + "geography": GeographySelection(country="us", region=region), + "options": options, + } + ) + calculator = RecordingCalculator() + + def segmented_calculator(_simulation, _cancellation_requested): + msg = "a state calculation is not split" + raise AssertionError(msg) + + _run_cliff_worker( + cliff_results, + simulation=simulation, + calculator=calculator, + segmented_calculator=segmented_calculator, + ) + + assert len(calculator.simulations) == 1 + assert cliff_results.cliff_results[CLIFF_RESULT_ID].status is (RunStatus.SUCCEEDED) + + +def test_a_dataset_with_no_adults_fails_the_cliff_result() -> None: + cliff_results = FakeCliffResults() + frames = cliff_frames() + frames["person"] = frames["person"].assign(is_adult=False) + + with pytest.raises(RuntimeError, match="cliff calculation failed"): + _run_cliff_worker(cliff_results, calculator=RecordingCalculator(frames)) + + result = cliff_results.cliff_results[CLIFF_RESULT_ID] + assert result.status is RunStatus.FAILED + assert result.error_message.startswith("cliff_share_undefined: ") diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_output_planning.py b/projects/policyengine-simulation-executor/tests/test_stage12_output_planning.py index 50810d1cd..6134db782 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_output_planning.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_output_planning.py @@ -24,6 +24,7 @@ ) from policyengine_simulation_executor.stage12_runtime.output_planning import ( apply_output_plan, + resolve_cliff_output_plan, resolve_report_output_plan, resolve_simulation_output_plan, validate_output_frames, @@ -222,6 +223,48 @@ def test_cliff_variables_are_conditional() -> None: assert "cliff_gap" not in _additional_variables(without_cliffs, "person") +@pytest.mark.parametrize("country", ["us", "uk"]) +def test_a_cliff_calculation_plans_the_cliff_columns_and_no_others_added( + country, +) -> None: + simulation = _simulation(country, SimulationRole.BASELINE) + + plan = resolve_cliff_output_plan(simulation) + + assert plan.country == country + assert plan.requirements.include_cliff_impacts is True + assert plan.requirements.labor_supply_response_active is False + # Only the columns the cliff gap and share read are added to the defaults: + # no labour-supply response or US budget columns. + assert _additional_variables(plan, "person") == { + "cliff_gap", + "is_adult", + "is_on_cliff", + } + assert {"person_id", "person_weight"}.issubset(_variables(plan, "person")) + assert all( + not entity.additional_variables + for entity in plan.entities + if entity.entity != "person" + ) + assert all(not entity.dataset_variables for entity in plan.entities) + + +def test_a_cliff_plan_records_a_labour_supply_response_without_its_columns() -> None: + simulation = _simulation( + "us", + SimulationRole.REFORM, + policy={"gov.simulation.labor_supply_responses.elasticities.income": 0.1}, + ) + + plan = resolve_cliff_output_plan(simulation) + + # The response changes the policy's net incomes, and so its cliffs; its + # own columns aren't needed. The flag keeps the run's own split rule. + assert plan.requirements.labor_supply_response_active is True + assert "income_elasticity_lsr" not in _additional_variables(plan, "person") + + def test_labor_supply_variables_are_shared_when_either_policy_activates_them() -> None: plan = resolve_report_output_plan( _report( From 3a53c7a880e6e2b97dc7684dcc894233e7e22939 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:55:54 +0400 Subject: [PATCH 04/24] Store and grant each simulation run's cliff calculations (failing) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit API v1's cliff_impacts table (C1 · 37's revision bc334f9877b7) holds a simulation run's cliff calculations. Sim models it with the service's tables: the simulation service inserts each pending row and records its call; the Modal worker role claims it, and completes or fails it, each in one conditional update. The Modal worker role no longer needs to update report_runs' four cliff columns. A coordinator's call naming a cliff result is refused before any write. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_access.py | 17 +- .../tests/test_modal_worker_writes.py | 37 ++- .../tests/test_tables.py | 20 ++ .../tests/test_cliff_result_store.py | 234 ++++++++++++++++++ .../test_stage12_report_run_coordinator.py | 24 ++ 5 files changed, 315 insertions(+), 17 deletions(-) create mode 100644 libs/policyengine-stage12-persistence/tests/test_cliff_result_store.py diff --git a/libs/policyengine-simulation-persistence/tests/test_access.py b/libs/policyengine-simulation-persistence/tests/test_access.py index 5b349f064..37bdf43a3 100644 --- a/libs/policyengine-simulation-persistence/tests/test_access.py +++ b/libs/policyengine-simulation-persistence/tests/test_access.py @@ -41,6 +41,8 @@ "report_runs", "budget_window_runs", "household_jobs", + # Each simulation run's cliff calculations, which the service starts. + "cliff_impacts", ) READ_TABLES = ( "policies", @@ -85,6 +87,7 @@ def test_the_service_updates_only_the_call_state_of_each_run() -> None: "report_runs": CALL_START_COLUMNS, "household_jobs": CALL_START_COLUMNS, "budget_window_runs": WINDOW_FAILURE_COLUMNS, + "cliff_impacts": CALL_START_COLUMNS, } @@ -101,6 +104,7 @@ def test_every_column_the_service_updates_is_modelled() -> None: "simulation_runs", "household_jobs", "budget_window_runs", + "cliff_impacts", ) # A run's state, which the Modal call holding it records. RUN_STATE_COLUMNS = frozenset( @@ -132,19 +136,14 @@ def test_the_modal_worker_role_updates_its_runs_state_and_results() -> None: its status, failure and times. """ assert dict(MODAL_WORKER_UPDATABLE_COLUMNS) == { - "report_runs": RUN_STATE_COLUMNS - | { - "baseline_cliff_gap", - "baseline_cliff_share", - "reform_cliff_gap", - "reform_cliff_share", - "spm_provenance", - "data_version", - }, + # A report run's cliff impact is its simulation runs' cliff results, + # so its own four cliff columns are never written. + "report_runs": RUN_STATE_COLUMNS | {"spm_provenance", "data_version"}, "simulation_runs": RUN_STATE_COLUMNS | {"output_uri", "output_sha256", "household_result"}, "household_jobs": RUN_STATE_COLUMNS | {"result"}, "budget_window_runs": RUN_STATE_COLUMNS - {"modal_call_id"}, + "cliff_impacts": RUN_STATE_COLUMNS | {"cliff_gap", "cliff_share"}, } diff --git a/libs/policyengine-simulation-persistence/tests/test_modal_worker_writes.py b/libs/policyengine-simulation-persistence/tests/test_modal_worker_writes.py index 6646a54b7..ee4dddaa4 100644 --- a/libs/policyengine-simulation-persistence/tests/test_modal_worker_writes.py +++ b/libs/policyengine-simulation-persistence/tests/test_modal_worker_writes.py @@ -12,7 +12,7 @@ from datetime import UTC, datetime from typing import TYPE_CHECKING, Any -from uuid import uuid4 +from uuid import UUID, uuid4 import pytest from conftest import ( @@ -25,6 +25,7 @@ MODAL_WORKER_TABLE_PRIVILEGES, MODAL_WORKER_UPDATABLE_COLUMNS, BudgetWindowRun, + CliffImpact, HouseholdJob, Poverty, ReportRun, @@ -62,11 +63,14 @@ def test_the_modal_worker_role_holds_every_privilege_the_modal_calls_need( ) -def _run_state(service_rows: ServiceRows) -> dict[Table, tuple[Any, dict[str, Any]]]: +def _run_state( + service_rows: ServiceRows, cliff_result_id: UUID +) -> dict[Table, tuple[Any, dict[str, Any]]]: """Return, by run table, a run's ID and a value for every column the role updates. A Python ``None`` in a JSON result is written as one, so the test can read - back that it is stored as SQL NULL. + back that it is stored as SQL NULL. ``cliff_result_id`` is a cliff + calculation of the baseline's simulation run. """ finished = { "modal_call_id": MODAL_CALL_ID, @@ -81,10 +85,6 @@ def _run_state(service_rows: ServiceRows) -> dict[Table, tuple[Any, dict[str, An **finished, "status": "succeeded", "error_message": None, - "baseline_cliff_gap": 1.0, - "baseline_cliff_share": 0.1, - "reform_cliff_gap": 2.0, - "reform_cliff_share": 0.2, "spm_provenance": None, "data_version": "test-data-version", }, @@ -109,6 +109,16 @@ def _run_state(service_rows: ServiceRows) -> dict[Table, tuple[Any, dict[str, An "result": None, }, ), + CliffImpact.__table__: ( + cliff_result_id, + { + **finished, + "status": "succeeded", + "error_message": None, + "cliff_gap": 1.0, + "cliff_share": 0.1, + }, + ), BudgetWindowRun.__table__: ( service_rows.budget_window_run_id, { @@ -195,9 +205,20 @@ def test_the_modal_worker_role_records_its_runs_and_inserts_every_result( public_api_rows = insert_public_api_rows(connection) act_as_the_service(connection) service_rows = insert_service_rows(connection, public_api_rows) + # The simulation service inserts each cliff calculation pending. + cliff_result_id = uuid4() + connection.execute( + insert(CliffImpact.__table__).values( + id=cliff_result_id, + simulation_run_id=service_rows.baseline_run_id, + status="pending", + ) + ) act_as_the_modal_worker(connection) - for table, (run_id, values) in _run_state(service_rows).items(): + for table, (run_id, values) in _run_state( + service_rows, cliff_result_id + ).items(): assert set(values) == MODAL_WORKER_UPDATABLE_COLUMNS[table.name], table updated = connection.execute( update(table).where(table.c.id == run_id).values(**values) diff --git a/libs/policyengine-simulation-persistence/tests/test_tables.py b/libs/policyengine-simulation-persistence/tests/test_tables.py index ecac00d78..a0ae4d464 100644 --- a/libs/policyengine-simulation-persistence/tests/test_tables.py +++ b/libs/policyengine-simulation-persistence/tests/test_tables.py @@ -5,6 +5,7 @@ from policyengine_simulation_persistence import ( SERVICE_TABLE_PRIVILEGES, BudgetWindowRun, + CliffImpact, HouseholdJob, Report, ReportRun, @@ -285,6 +286,25 @@ def test_a_definition_and_version_set_has_one_unfailed_unforced_run() -> None: ) +def test_a_simulation_run_has_one_unfailed_cliff_calculation() -> None: + """A failed calculation stays recorded, and a later start makes a new row.""" + table = CliffImpact.__table__ + index = _index(table, "uq_cliff_impacts_unfailed") + + assert index.unique is True + assert tuple(column.name for column in index.columns) == ("simulation_run_id",) + assert str(index.dialect_options["postgresql"]["where"]) == "status <> 'failed'" + # The calculation belongs to its run, and goes with it. + assert _foreign_keys(table) == { + "fk_cliff_impacts_simulation_run_id_simulation_runs": ( + ("simulation_run_id",), + "simulation_runs(id)", + "CASCADE", + ) + } + assert table.c.status.type.enums == list(SimulationStatus) + + def test_a_run_is_unforced_unless_its_insert_marks_it() -> None: for table in (SimulationRun.__table__, ReportRun.__table__): assert table.c.forced.nullable is False, table.name diff --git a/libs/policyengine-stage12-persistence/tests/test_cliff_result_store.py b/libs/policyengine-stage12-persistence/tests/test_cliff_result_store.py new file mode 100644 index 000000000..055ddd601 --- /dev/null +++ b/libs/policyengine-stage12-persistence/tests/test_cliff_result_store.py @@ -0,0 +1,234 @@ +"""A simulation run's cliff calculation, written conditionally as the Modal worker role. + +The simulation service inserts each cliff result pending, and may record the +Modal call it starts while the row is still pending with none; the seed role +does that here. The worker then claims it, and completes or fails it, each in +one ``UPDATE`` conditional on the row's status and recorded call. +""" + +from __future__ import annotations + +from datetime import timedelta +from uuid import uuid4 + +import pytest +from policyengine_simulation_persistence import CliffImpact +from policyengine_stage12_persistence import ( + CliffResultState, + CliffResultStore, + RunStatus, + RunWrite, +) +from report_run_database import NOW, SeededReportRun, worker_database_url +from sqlalchemy import insert, select, update +from sqlalchemy.exc import ProgrammingError + +WORKER_CALL_ID = "fc-cliff-worker" +OTHER_CALL_ID = "fc-other-cliff-worker" +LATER = NOW + timedelta(minutes=2) +CLIFF_GAP = 1_234.5 +CLIFF_SHARE = 0.0375 + + +@pytest.fixture +def store(): + return CliffResultStore(worker_database_url()) + + +def _insert_cliff_result(seeder, seeded_run: SeededReportRun, **values): + """Insert a cliff result of the seeded baseline run, as the service does.""" + cliff_result_id = uuid4() + with seeder.engine.begin() as connection: + connection.execute( + insert(CliffImpact.__table__).values( + id=cliff_result_id, + simulation_run_id=seeded_run.run_ids.baseline_simulation_run_id, + status=values.pop("status", "pending"), + **values, + ) + ) + return cliff_result_id + + +def _cliff_row(seeder, cliff_result_id): + with seeder.engine.connect() as connection: + row = connection.execute( + select(CliffImpact.__table__).where(CliffImpact.id == cliff_result_id) + ).one() + return dict(row._mapping) + + +def _claim(store, cliff_result_id, modal_call_id=WORKER_CALL_ID): + return store.claim_cliff_result( + cliff_result_id, modal_call_id=modal_call_id, claimed_at=NOW + ) + + +def _complete(store, cliff_result_id, modal_call_id=WORKER_CALL_ID): + return store.complete_cliff_result( + cliff_result_id, + modal_call_id=modal_call_id, + cliff_gap=CLIFF_GAP, + cliff_share=CLIFF_SHARE, + completed_at=LATER, + ) + + +def test_a_cliff_result_is_claimed_once_and_resumed_by_the_same_call( + store, seeder, seeded_run +) -> None: + cliff_result_id = _insert_cliff_result(seeder, seeded_run) + + assert _claim(store, cliff_result_id) == RunWrite( + changed=True, status=RunStatus.RUNNING + ) + # Modal restarts a pre-empted call under the same call ID. + assert _claim(store, cliff_result_id) == RunWrite( + changed=True, status=RunStatus.RUNNING + ) + # Another call is the second start of the same calculation. + assert _claim(store, cliff_result_id, OTHER_CALL_ID) == RunWrite( + changed=False, status=RunStatus.RUNNING + ) + + row = _cliff_row(seeder, cliff_result_id) + assert (row["status"], row["modal_call_id"]) == ("running", WORKER_CALL_ID) + assert row["started_at"] == NOW + + +def test_a_pending_cliff_result_is_claimed_only_by_the_call_recorded_on_it( + store, seeder, seeded_run +) -> None: + cliff_result_id = _insert_cliff_result( + seeder, seeded_run, modal_call_id=WORKER_CALL_ID + ) + + assert _claim(store, cliff_result_id, OTHER_CALL_ID) == RunWrite( + changed=False, status=RunStatus.PENDING + ) + assert _claim(store, cliff_result_id) == RunWrite( + changed=True, status=RunStatus.RUNNING + ) + + +def test_a_cliff_result_completes_with_its_values_while_its_call_holds_it( + store, seeder, seeded_run +) -> None: + cliff_result_id = _insert_cliff_result(seeder, seeded_run) + + # Not claimed yet: no call holds it. + assert _complete(store, cliff_result_id) == RunWrite( + changed=False, status=RunStatus.PENDING + ) + _claim(store, cliff_result_id) + assert _complete(store, cliff_result_id, OTHER_CALL_ID) == RunWrite( + changed=False, status=RunStatus.RUNNING + ) + assert _complete(store, cliff_result_id) == RunWrite( + changed=True, status=RunStatus.SUCCEEDED + ) + # A completed calculation is written once. + assert _complete(store, cliff_result_id) == RunWrite( + changed=False, status=RunStatus.SUCCEEDED + ) + assert _claim(store, cliff_result_id) == RunWrite( + changed=False, status=RunStatus.SUCCEEDED + ) + + row = _cliff_row(seeder, cliff_result_id) + assert (row["status"], row["cliff_gap"], row["cliff_share"]) == ( + "succeeded", + CLIFF_GAP, + CLIFF_SHARE, + ) + assert (row["completed_at"], row["error_message"]) == (LATER, None) + + +def test_a_failed_cliff_calculation_records_its_failure_and_never_completes( + store, seeder, seeded_run +) -> None: + cliff_result_id = _insert_cliff_result(seeder, seeded_run) + _claim(store, cliff_result_id) + + failure = { + "error_message": "cliff_calculation_failed: the calculation failed", + "failed_at": LATER, + } + assert store.fail_cliff_result( + cliff_result_id, modal_call_id=OTHER_CALL_ID, **failure + ) == RunWrite(changed=False, status=RunStatus.RUNNING) + assert store.fail_cliff_result( + cliff_result_id, modal_call_id=WORKER_CALL_ID, **failure + ) == RunWrite(changed=True, status=RunStatus.FAILED) + assert _complete(store, cliff_result_id) == RunWrite( + changed=False, status=RunStatus.FAILED + ) + assert _claim(store, cliff_result_id) == RunWrite( + changed=False, status=RunStatus.FAILED + ) + + row = _cliff_row(seeder, cliff_result_id) + assert row["status"] == "failed" + assert row["error_message"] == failure["error_message"] + assert (row["completed_at"], row["cliff_gap"], row["cliff_share"]) == ( + LATER, + None, + None, + ) + + +def test_a_cliff_results_state_names_its_run_call_values_and_failure( + store, seeder, seeded_run +) -> None: + cliff_result_id = _insert_cliff_result(seeder, seeded_run) + _claim(store, cliff_result_id) + _complete(store, cliff_result_id) + + assert store.get_cliff_result(cliff_result_id) == CliffResultState( + simulation_run_id=seeded_run.run_ids.baseline_simulation_run_id, + status=RunStatus.SUCCEEDED, + modal_call_id=WORKER_CALL_ID, + cliff_gap=CLIFF_GAP, + cliff_share=CLIFF_SHARE, + error_message=None, + ) + # The cliff store reads the run it was calculated for, as the worker checks. + run = store.get_simulation_run(seeded_run.run_ids.baseline_simulation_run_id) + assert (run.year, run.status) == (2026, RunStatus.PENDING) + + +def test_a_missing_cliff_result_raises_lookup_error(store, seeded_run) -> None: + missing = uuid4() + + with pytest.raises(LookupError, match=f"cliff result {missing} does not exist"): + store.get_cliff_result(missing) + with pytest.raises(LookupError, match=f"cliff result {missing} does not exist"): + _claim(store, missing) + + +def test_the_modal_worker_role_cannot_insert_or_retarget_a_cliff_result( + store, seeder, seeded_run +) -> None: + cliff_result_id = _insert_cliff_result(seeder, seeded_run) + + with ( + pytest.raises(ProgrammingError, match="permission denied"), + store.sessions.transaction("create_cliff_result") as session, + ): + session.execute( + insert(CliffImpact).values( + id=uuid4(), + simulation_run_id=seeded_run.run_ids.reform_simulation_run_id, + status="pending", + ) + ) + with ( + pytest.raises(ProgrammingError, match="permission denied"), + store.sessions.transaction("retarget_cliff_result") as session, + ): + session.execute( + update(CliffImpact) + .where(CliffImpact.id == cliff_result_id) + .values(simulation_run_id=seeded_run.run_ids.reform_simulation_run_id), + execution_options={"synchronize_session": False}, + ) diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_report_run_coordinator.py b/projects/policyengine-simulation-executor/tests/test_stage12_report_run_coordinator.py index 3676fd582..d5716e593 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_report_run_coordinator.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_report_run_coordinator.py @@ -24,6 +24,7 @@ ) from policyengine_stage12_persistence import RunStatus from stage12_child_call_support import ConcurrentInvoker +from stage12_fake_cliff_results import CLIFF_RESULT_ID from stage12_fake_report_runs import ( BASELINE_RUN_ID, REFORM_RUN_ID, @@ -80,6 +81,29 @@ def _run_through_workers(report_runs=None, **invoker_options): return report_runs, artifacts, invoker +@pytest.mark.parametrize( + "worker_record", + [ + {"simulation_run_id": BASELINE_RUN_ID}, + {"cliff_result_id": CLIFF_RESULT_ID}, + ], + ids=["simulation-run", "cliff-result"], +) +def test_a_coordinator_call_naming_a_workers_record_is_refused_unwritten( + worker_record, +) -> None: + report_runs, artifacts, invoker = _run_through_workers() + context = invocation_context().model_copy( + update={**worker_record, "artifact_prefix": REPORT_RUN_PREFIX} + ) + + with pytest.raises(ValueError, match="names a report run, not a simulation run"): + _coordinate(report_runs, invoker, artifacts, context=context) + + assert report_runs.events == [] + assert invoker.simulations == [] + + def test_the_coordinator_claims_its_report_run_and_each_worker_its_simulation_run() -> ( None ): From 2f6cabf0fdcd42122b97af114149f62bc779870a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:03:36 +0400 Subject: [PATCH 05/24] Calculate a simulation run's cliff gap and share as its cliff result MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A simulation worker's call whose context names a cliff result (Stage12InvocationContext.cliff_result_id) claims it in API v1's cliff_impacts under its Modal call ID, checks that the payload is the simulation of the result's run on the run's year and versions, plans it with .py's cliff columns and no others added, calculates it, and writes .py's weighted cliff gap and cliff share, or the failure, only while its call holds the row. A second start or a finished row calculates nothing. A US national calculation runs as the 20 region groups a plain run uses, whenever its run's own calculation would (should_segment_cliff_calculation): each person's cliff columns depend only on their household, and the share is taken once from the merged rows. A coordinator's call naming a cliff result is refused. Sim models cliff_impacts with the service's tables (C1 · 37's revision bc334f9877b7), with each role's privileges; the Modal worker role no longer needs to update report_runs' cliff columns. Co-Authored-By: Claude Opus 5.5 --- .../stage12_execution/comparison_runs.py | 37 +- .../__init__.py | 2 + .../access.py | 32 +- .../cliff_impact_tables.py | 99 ++++ .../tests/test_public_api_schema.py | 30 + .../__init__.py | 8 +- .../cliff_result_store.py | 176 ++++++ .../report_run_store.py | 8 +- .../stage12_runtime/cliff_results.py | 520 ++++++++++++++++++ .../stage12_runtime/coordination.py | 7 +- .../stage12_runtime/output_planning.py | 117 +++- .../stage12_runtime/partition.py | 21 +- .../stage12_runtime/report_run_access.py | 26 +- .../stage12_runtime/simulation.py | 18 +- .../tests/test_stage12_cliff_results.py | 2 +- 15 files changed, 1030 insertions(+), 73 deletions(-) create mode 100644 libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/cliff_impact_tables.py create mode 100644 libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/cliff_result_store.py create mode 100644 projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/cliff_results.py diff --git a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/stage12_execution/comparison_runs.py b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/stage12_execution/comparison_runs.py index 6f8b2e346..7d2530aa4 100644 --- a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/stage12_execution/comparison_runs.py +++ b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/stage12_execution/comparison_runs.py @@ -73,9 +73,11 @@ class Stage12InvocationContext(StrictContractModel): A detailed-save call works on the Public API's run tables instead of the comparison path's records: a report coordinator's call names its report - run and both simulation runs in ``report_run``, and a simulation worker's - call names the one simulation run it claims in ``simulation_run_id``. - Its simulation output files go to their own bucket, which keeps them. + run and both simulation runs in ``report_run``, a simulation worker's + call names the one simulation run it claims in ``simulation_run_id``, and + a simulation worker's call that calculates a run's cliff gap and share + names the cliff result it claims in ``cliff_result_id``. Its simulation + output files go to their own bucket, which keeps them. """ contract_version: Literal[1] = 1 @@ -97,9 +99,10 @@ class Stage12InvocationContext(StrictContractModel): production_function_call_id: ContractText | None = None created_at: datetime retention_expires_at: datetime - # Comparison runs leave both unset and record only the Stage 12 tables. + # Comparison runs leave all three unset and record only the Stage 12 tables. report_run: ReportRunIds | None = None simulation_run_id: UUID | None = None + cliff_result_id: UUID | None = None @model_validator(mode="after") def validate_retention(self) -> Stage12InvocationContext: @@ -111,21 +114,37 @@ def validate_retention(self) -> Stage12InvocationContext: def validate_report_run_is_direct(self) -> Stage12InvocationContext: """Reject runs on a shadow run compared with a production call. - A call names a report run, for its coordinator, or a simulation run, - for a worker, not both. + A call names at most one of a report run, for its coordinator, a + simulation run, for a worker, and a cliff result, for a worker + calculating a run's cliffs. """ if self.records_runs and self.production_function_call_id: msg = "a run compared with a production function call has no report run" raise ValueError(msg) - if self.report_run is not None and self.simulation_run_id is not None: - msg = "a call names a report run or a simulation run, not both" + if len(self._named_records()) > 1: + msg = ( + "a call names one of a report run, a simulation run and a " + "cliff result, not two" + ) raise ValueError(msg) return self @property def records_runs(self) -> bool: """Whether this is a detailed-save call, which works on the run tables.""" - return self.report_run is not None or self.simulation_run_id is not None + return bool(self._named_records()) + + def _named_records(self) -> tuple[object, ...]: + """Return the report run, simulation run and cliff result the call names.""" + return tuple( + record + for record in ( + self.report_run, + self.simulation_run_id, + self.cliff_result_id, + ) + if record is not None + ) ErrorCode = Annotated[str, Field(min_length=1, max_length=64)] diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py index eb6663fdb..14ae71304 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py @@ -12,6 +12,7 @@ WindowSide, WindowYearRuns, ) +from policyengine_simulation_persistence.cliff_impact_tables import CliffImpact from policyengine_simulation_persistence.definition_keys import ( KEY_VERSION, EconomyReportDefinition, @@ -230,6 +231,7 @@ "BudgetWindowRun", "CatalogDataset", "CatalogRegion", + "CliffImpact", "CongressionalDistrictImpact", "ConstituencyImpact", "Dataset", diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/access.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/access.py index 1ab50e183..dd50b889c 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/access.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/access.py @@ -35,8 +35,8 @@ _READ: Final = frozenset({"SELECT"}) # The service creates simulations, reports, their runs, budget windows' -# calculations and household jobs, and reads the policy, dynamic, household -# and catalog rows a run needs. +# calculations, household jobs and simulation runs' cliff calculations, and +# reads the policy, dynamic, household and catalog rows a run needs. SERVICE_TABLE_PRIVILEGES: Final[Mapping[str, frozenset[str]]] = MappingProxyType( { "simulations": _CREATE, @@ -45,6 +45,7 @@ "report_runs": _CREATE, "budget_window_runs": _CREATE, "household_jobs": _CREATE, + "cliff_impacts": _CREATE, "policies": _READ, "households": _READ, "tax_benefit_models": _READ, @@ -75,6 +76,7 @@ "report_runs": _CALL_START, "household_jobs": _CALL_START, "budget_window_runs": _WINDOW_FAILURE, + "cliff_impacts": _CALL_START, } ) @@ -91,35 +93,33 @@ } ) # The v2 Modal coordinators, workers, household functions and stuck-run sweep -# read the runs they were given and insert their results, once each. A session's -# insert of a result also reads the row's server defaults back. +# read the runs and cliff calculations they were given and insert their results, +# once each. A session's insert of a result also reads the row's server defaults +# back. A cliff calculation's result is in its own row. MODAL_WORKER_TABLE_PRIVILEGES: Final[Mapping[str, frozenset[str]]] = MappingProxyType( { "report_runs": _READ, "simulation_runs": _READ, "household_jobs": _READ, "budget_window_runs": _READ, + "cliff_impacts": _READ, **dict.fromkeys(RESULT_TABLE_NAMES, _CREATE), } ) -# The columns the Modal worker role may UPDATE, by table: each run's state and -# the results its own row holds. A budget-window run has no Modal call of its -# own; its years' coordinators record its status, failure and times. +# The columns the Modal worker role may UPDATE, by table: each run's and cliff +# calculation's state and the results its own row holds. A budget-window run +# has no Modal call of its own; its years' coordinators record its status, +# failure and times. MODAL_WORKER_UPDATABLE_COLUMNS: Final[Mapping[str, frozenset[str]]] = MappingProxyType( { - "report_runs": _RUN_STATE - | { - "baseline_cliff_gap", - "baseline_cliff_share", - "reform_cliff_gap", - "reform_cliff_share", - "spm_provenance", - "data_version", - }, + # A report run's cliff impact is its simulation runs' cliff results, + # so the role never writes its four cliff columns. + "report_runs": _RUN_STATE | {"spm_provenance", "data_version"}, "simulation_runs": _RUN_STATE | {"output_uri", "output_sha256", "household_result"}, "household_jobs": _RUN_STATE | {"result"}, "budget_window_runs": _RUN_STATE - {"modal_call_id"}, + "cliff_impacts": _RUN_STATE | {"cliff_gap", "cliff_share"}, } ) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/cliff_impact_tables.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/cliff_impact_tables.py new file mode 100644 index 000000000..b24d60191 --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/cliff_impact_tables.py @@ -0,0 +1,99 @@ +"""The model of ``cliff_impacts``: each simulation run's cliff calculations. + +A simulation run can have one cliff result, its cliff gap and cliff share, +which every report naming the run reads. Each row is one calculation of it: +the simulation service inserts it pending and records the Modal call it +starts, and the Modal worker role claims it under that call and records the +two numbers or the failure. A run has at most one calculation that hasn't +failed, so identical requests join it; a failed one stays recorded, and a +later start inserts a new row. The row has no versions of its own: it is a +calculation of its run, on the run's versions. Deleting a run deletes its +calculations. + +The Public API's v2 migration chain creates the table (revision +``bc334f9877b7``); this class only describes it, column for column, with its +constraints and indexes under the same names. It is one of the service's +tables (``ServiceBase``), since the service creates its rows. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 # SQLAlchemy reads Mapped annotations +from typing import ClassVar +from uuid import UUID # noqa: TC003 # SQLAlchemy reads Mapped annotations at run time + +from sqlalchemy import ( + CheckConstraint, + Float, + ForeignKeyConstraint, + Index, + PrimaryKeyConstraint, + String, + Uuid, + text, +) +from sqlalchemy.orm import Mapped, mapped_column + +from policyengine_simulation_persistence.orm_base import ( + ServiceBase, + enum_type, + identifier_column, + optional_timestamp_column, + server_timestamp_column, +) +from policyengine_simulation_persistence.tables import ( + MODAL_CALL_ID_LENGTH, + SimulationRun, + SimulationStatus, +) + + +class CliffImpact(ServiceBase): + """One cliff calculation of a simulation run, and its result once it succeeds.""" + + __tablename__ = "cliff_impacts" + __table_args__: ClassVar = ( + PrimaryKeyConstraint("id", name="pk_cliff_impacts"), + ForeignKeyConstraint( + ["simulation_run_id"], + [SimulationRun.id], + name="fk_cliff_impacts_simulation_run_id_simulation_runs", + ondelete="CASCADE", + ), + CheckConstraint( + "status NOT IN ('succeeded', 'failed') OR completed_at IS NOT NULL", + name="ck_cliff_impacts_finished_completion", + ), + CheckConstraint( + "(cliff_gap IS NULL) = (cliff_share IS NULL) AND " + "(status = 'succeeded') = (cliff_gap IS NOT NULL)", + name="ck_cliff_impacts_result_when_succeeded", + ), + Index("ix_cliff_impacts_simulation_run_id", "simulation_run_id"), + # For the stuck-run sweep, which judges a calculation by its age. + Index("ix_cliff_impacts_status_started_at", "status", "started_at"), + # One calculation per run that hasn't failed: find-or-create's conflict + # target, so identical requests join one calculation. + Index( + "uq_cliff_impacts_unfailed", + "simulation_run_id", + unique=True, + postgresql_where=text("status <> 'failed'"), + ), + ) + + id: Mapped[UUID] = identifier_column() + created_at: Mapped[datetime] = server_timestamp_column() + updated_at: Mapped[datetime] = server_timestamp_column() + simulation_run_id: Mapped[UUID] = mapped_column(Uuid) + # A cliff calculation has a simulation run's states, under its type. + status: Mapped[SimulationStatus] = mapped_column( + enum_type(SimulationStatus, "v2_simulation_status") + ) + modal_call_id: Mapped[str | None] = mapped_column(String(MODAL_CALL_ID_LENGTH)) + error_message: Mapped[str | None] = mapped_column(String) + started_at: Mapped[datetime | None] = optional_timestamp_column() + completed_at: Mapped[datetime | None] = optional_timestamp_column() + # Both set exactly when the calculation succeeded. + cliff_gap: Mapped[float | None] = mapped_column(Float) + cliff_share: Mapped[float | None] = mapped_column(Float) diff --git a/libs/policyengine-simulation-persistence/tests/test_public_api_schema.py b/libs/policyengine-simulation-persistence/tests/test_public_api_schema.py index b163dd170..bdf395c93 100644 --- a/libs/policyengine-simulation-persistence/tests/test_public_api_schema.py +++ b/libs/policyengine-simulation-persistence/tests/test_public_api_schema.py @@ -18,6 +18,7 @@ import pytest from policyengine_simulation_persistence import ( + CliffImpact, input_metadata, referenced_metadata, result_metadata, @@ -77,6 +78,35 @@ def test_the_models_match_the_public_api_tables( ) +@pytest.mark.public_api_schema +def test_the_cliff_calculation_model_matches_the_public_api_table( + disposable_database, public_api_database_url +) -> None: + """A simulation run's cliff calculations match the Public API's ``cliff_impacts``. + + The service's other tables are compared with it above; this names the + table alone, so a difference there is reported as the cliff table's. + """ + models_engine = _engine(disposable_database.superuser_url) + real_engine = _engine(public_api_database_url) + try: + referenced_metadata.create_all(models_engine) + service_metadata.create_all(models_engine) + with models_engine.connect() as connection: + models = describe_tables(connection, [CliffImpact.__tablename__]) + with real_engine.connect() as connection: + real = describe_tables(connection, [CliffImpact.__tablename__]) + finally: + models_engine.dispose() + real_engine.dispose() + + assert set(real.tables) == {"cliff_impacts"} + compared_with = os.environ.get(COMMIT_VARIABLE) or "an unrecorded commit" + assert schema_differences(models, real) == [], ( + f"compared with the Public API at {compared_with}" + ) + + @pytest.mark.public_api_schema def test_the_result_models_match_the_public_api_tables( disposable_database, public_api_database_url diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/__init__.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/__init__.py index 6e8311eb3..3c3f031c9 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/__init__.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/__init__.py @@ -1,4 +1,4 @@ -"""Direct DML for Stage 12 comparison state, runs, budget windows and household jobs.""" +"""Direct DML for Stage 12 comparison state, runs, cliffs, windows and households.""" from policyengine_stage12_persistence.budget_window_store import ( BUDGET_WINDOW_MAX_RUNNING_YEARS, @@ -8,6 +8,10 @@ YearStarter, YearToStart, ) +from policyengine_stage12_persistence.cliff_result_store import ( + CliffResultState, + CliffResultStore, +) from policyengine_stage12_persistence.engine import ( SessionOpener, create_stage12_check_session_factory, @@ -39,6 +43,8 @@ "BUDGET_WINDOW_MAX_RUNNING_YEARS", "ActiveRun", "BudgetWindowStore", + "CliffResultState", + "CliffResultStore", "DistrictCodes", "HouseholdReportRunStore", "ReportRunResult", diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/cliff_result_store.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/cliff_result_store.py new file mode 100644 index 000000000..6d874f008 --- /dev/null +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/cliff_result_store.py @@ -0,0 +1,176 @@ +"""Conditional writes of a simulation run's cliff calculation, as the Modal worker role. + +The Public API's ``cliff_impacts`` holds each cliff calculation of a +simulation run, its cliff result. The simulation service inserts it pending +and may record the Modal call it starts while it is still pending with none. +A simulation worker's call then claims it under that call, as a run is +claimed, and records the cliff gap and share, or the failure, only while its +call holds it running. Each write is one ``UPDATE`` conditional on the row's +status and recorded call, so a call Modal restarts, or a second start of the +same calculation, changes nothing already final. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from policyengine_simulation_persistence.cliff_impact_tables import CliffImpact +from sqlalchemy import func, or_, select, update + +from policyengine_stage12_persistence.report_run_store import ( + ACTIVE_STATUSES, + ReportRunStore, + RunStatus, + running_under_call, +) + +if TYPE_CHECKING: + from datetime import datetime + from uuid import UUID + + from policyengine_stage12_persistence.report_run_store import RunWrite + + +@dataclass(frozen=True) +class CliffResultState: + """A cliff result's run, status, recorded call, values and failure. + + ``modal_call_id`` is None while no call is recorded; ``cliff_gap`` and + ``cliff_share`` are set only once it succeeded, and ``error_message`` + only once it failed. + """ + + simulation_run_id: UUID + status: RunStatus + modal_call_id: str | None + cliff_gap: float | None + cliff_share: float | None + error_message: str | None + + +class CliffResultStore(ReportRunStore): + """A run store that also claims and records simulation runs' cliff results. + + The worker reads the cliff result's simulation run with the run store's + ``get_simulation_run``. + """ + + def get_cliff_result(self, cliff_result_id: UUID) -> CliffResultState: + """Return a cliff result's state; it only reads the row. + + Raises LookupError when the cliff result does not exist. + """ + statement = select( + CliffImpact.simulation_run_id, + CliffImpact.status, + CliffImpact.modal_call_id, + CliffImpact.cliff_gap, + CliffImpact.cliff_share, + CliffImpact.error_message, + ).where(CliffImpact.id == cliff_result_id) + with self.sessions.transaction("get_cliff_result") as session: + stored = session.execute(statement).one_or_none() + if stored is None: + msg = f"cliff result {cliff_result_id} does not exist" + raise LookupError(msg) + return CliffResultState( + simulation_run_id=stored.simulation_run_id, + status=RunStatus(stored.status), + modal_call_id=stored.modal_call_id, + cliff_gap=stored.cliff_gap, + cliff_share=stored.cliff_share, + error_message=stored.error_message, + ) + + def claim_cliff_result( + self, + cliff_result_id: UUID, + *, + modal_call_id: str, + claimed_at: datetime, + ) -> RunWrite: + """Mark the cliff result running under the worker's ``modal_call_id``. + + A pending or running cliff result is claimed while it records no call + or this call: the simulation service records the call it starts while + the row is still pending, and Modal restarts a pre-empted call under + the same call ID. One recording another call is that call's. + """ + statement = ( + update(CliffImpact) + .where( + CliffImpact.id == cliff_result_id, + CliffImpact.status.in_(ACTIVE_STATUSES), + or_( + CliffImpact.modal_call_id.is_(None), + CliffImpact.modal_call_id == modal_call_id, + ), + ) + .values( + status=RunStatus.RUNNING.value, + modal_call_id=modal_call_id, + started_at=func.coalesce(CliffImpact.started_at, claimed_at), + updated_at=claimed_at, + ) + ) + return self._write( + CliffImpact, cliff_result_id, statement, "claim_cliff_result" + ) + + def complete_cliff_result( + self, + cliff_result_id: UUID, + *, + modal_call_id: str, + cliff_gap: float, + cliff_share: float, + completed_at: datetime, + ) -> RunWrite: + """Record the cliff gap and share; mark it succeeded while this call holds it. + + A cliff result failed meanwhile, as the stuck-run sweep fails one, + takes no values from the call. + """ + statement = ( + update(CliffImpact) + .where( + CliffImpact.id == cliff_result_id, + running_under_call(CliffImpact, modal_call_id), + ) + .values( + status=RunStatus.SUCCEEDED.value, + cliff_gap=cliff_gap, + cliff_share=cliff_share, + error_message=None, + completed_at=completed_at, + updated_at=completed_at, + ) + ) + return self._write( + CliffImpact, cliff_result_id, statement, "complete_cliff_result" + ) + + def fail_cliff_result( + self, + cliff_result_id: UUID, + *, + modal_call_id: str, + error_message: str, + failed_at: datetime, + ) -> RunWrite: + """Mark the cliff result failed while ``modal_call_id`` holds it running.""" + statement = ( + update(CliffImpact) + .where( + CliffImpact.id == cliff_result_id, + running_under_call(CliffImpact, modal_call_id), + ) + .values( + status=RunStatus.FAILED.value, + error_message=error_message, + completed_at=failed_at, + updated_at=failed_at, + ) + ) + return self._write(CliffImpact, cliff_result_id, statement, "fail_cliff_result") diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py index 1a4fc2b80..f52362092 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py @@ -15,6 +15,7 @@ from enum import StrEnum from typing import TYPE_CHECKING, Any, Final +from policyengine_simulation_persistence.cliff_impact_tables import CliffImpact from policyengine_simulation_persistence.tables import ( BudgetWindowRun, HouseholdJob, @@ -63,7 +64,7 @@ class RunKind(StrEnum): HOUSEHOLD_JOB = "household_job" -type RunModel = type[ReportRun | SimulationRun | HouseholdJob] +type RunModel = type[ReportRun | SimulationRun | HouseholdJob | CliffImpact] RUN_MODELS: Final[dict[RunKind, RunModel]] = { RunKind.REPORT_RUN: ReportRun, @@ -75,6 +76,7 @@ class RunKind(StrEnum): ReportRun: "report run", SimulationRun: "simulation run", HouseholdJob: "household job", + CliffImpact: "cliff result", } # A bulk insert otherwise leaves out each column a row sets to None. Every row # names the same columns, so the insert stays one statement that sets them NULL. @@ -714,8 +716,8 @@ def _uniform_rows(rows: tuple[dict[str, Any], ...]) -> list[dict[str, Any]]: def running_under_call(model: RunModel, modal_call_id: str) -> ColumnElement[bool]: """Return the condition that a row is running under ``modal_call_id``. - A run or household job is completed or failed by a call only while that - call holds it running. + A run, household job or cliff result is completed or failed by a call only + while that call holds it running. """ return and_( model.status == RunStatus.RUNNING.value, diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/cliff_results.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/cliff_results.py new file mode 100644 index 000000000..65190f1dd --- /dev/null +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/cliff_results.py @@ -0,0 +1,520 @@ +"""A worker's cliff calculation of one simulation run: claim, calculate, record. + +Each simulation run can have a cliff result, its cliff gap and cliff share, +which every report naming the run reads. A simulation worker's call whose +context names a cliff result claims it under the call's Modal call ID, +checks that the payload is the simulation of the cliff result's run on the +run's year and versions, calculates that simulation with the cliff columns, +and records the result or its failure, only while its call still holds it. +A cliff result another call holds, or one that finished, is not calculated +again. The call writes nothing to the simulation run, its output file or any +report run. + +The cliff gap and share are .py's: ``cliff_gap`` reads marginal tax rates, +which recalculate each household's net income with extra earnings, so the +simulation is calculated again from its dataset and policy; the run's stored +output file has no cliff columns. The gap is the weighted sum of +``cliff_gap``, and the share the weighted count of people ``is_on_cliff`` +over the weighted count of ``is_adult`` (``policyengine.outputs.cliff_impact``). +A US national calculation runs as the region groups a plain run uses: each +person's cliff columns depend only on their own household, each group keeps +its households' weights, and the share is taken once, from the merged rows. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from datetime import UTC, datetime +from functools import partial +from typing import TYPE_CHECKING, Any, Protocol + +import pandas as pd +from microdf import MicroDataFrame +from policyengine.outputs.cliff_impact import CLIFF_IMPACT_VARIABLES +from policyengine_simulation_contract.stage12_execution import ( + SimulationExecutionInput, +) +from policyengine_simulation_observability.stages import STAGE12_SIMULATION_STAGES +from policyengine_stage12_persistence import RunStatus + +from .failure_causes import ( + StageProgress, + StageTracking, + record_failure_cause, + tracked_stage, +) +from .failure_recording import FailureSite, persist_failure_record, report_failure +from .failures import ( + Stage12CancellationError, + Stage12ExecutionError, + Stage12InputError, + validate_policy_periods, +) +from .output_planning import ( + plan_simulation_input, + resolve_cliff_output_plan, + validate_output_frames, +) +from .partition import should_segment_cliff_calculation +from .report_run_access import cliff_result_store +from .report_run_recording import run_error_message +from .worker_inputs import require_worker_context + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping + from contextlib import AbstractContextManager + from uuid import UUID + + from policyengine_observability import ObservabilityRuntime + from policyengine_simulation_contract.stage12_bundle import CountryId + from policyengine_simulation_contract.stage12_execution import ( + PlannedSimulationExecutionInput, + Stage12InvocationContext, + Stage12OutputPlan, + ) + from policyengine_simulation_observability.stages import Stage + from policyengine_stage12_persistence import ( + CliffResultState, + RunWrite, + SimulationRunState, + ) + + from .failures import Stage12FailureDetail + from .simulation import SimulationCalculation, SimulationWorkerDependencies + +# The payload is not the simulation of the cliff result's run, on its year +# and versions. +CLIFF_RESULT_INPUT_MISMATCH = "cliff_result_input_mismatch" +# The calculation found no adults, or a total that is not a finite number, +# so the cliff share has no value. +CLIFF_SHARE_UNDEFINED = "cliff_share_undefined" +# The column the cliff share divides by. +ADULT_COUNT_VARIABLE = "is_adult" + + +class CliffResultWriter(Protocol): + """Cliff result writes and the reads they check; ``CliffResultStore`` implements it. + + Each write is conditional on the cliff result's status and returns + whether it changed it, and its status afterwards. + """ + + def get_cliff_result(self, cliff_result_id: UUID) -> CliffResultState: + """Return a cliff result's state; LookupError when it does not exist.""" + ... + + def get_simulation_run(self, simulation_run_id: UUID) -> SimulationRunState: + """Return a simulation run's state; LookupError when it does not exist.""" + ... + + def claim_cliff_result( + self, cliff_result_id: UUID, *, modal_call_id: str, claimed_at: datetime + ) -> RunWrite: + """Mark a pending or running cliff result that records no other call running.""" + ... + + def complete_cliff_result( # the result, its call, then its values + self, + cliff_result_id: UUID, + *, + modal_call_id: str, + cliff_gap: float, + cliff_share: float, + completed_at: datetime, + ) -> RunWrite: + """Record the values and mark it succeeded while ``modal_call_id`` holds it.""" + ... + + def fail_cliff_result( + self, + cliff_result_id: UUID, + *, + modal_call_id: str, + error_message: str, + failed_at: datetime, + ) -> RunWrite: + """Mark the cliff result failed while ``modal_call_id`` holds it running.""" + ... + + +@dataclass(frozen=True) +class CliffImpact: + """One simulation's cliff gap and cliff share.""" + + cliff_gap: float + cliff_share: float + + +@dataclass(frozen=True) +class CliffResultRecorder: + """A worker's writes to the cliff result its call claims, under its call ID.""" + + writer: CliffResultWriter + cliff_result_id: UUID + modal_call_id: str + + def claim(self) -> RunWrite: + """Mark the cliff result running under this call, unless another holds it.""" + return self.writer.claim_cliff_result( + self.cliff_result_id, + modal_call_id=self.modal_call_id, + claimed_at=datetime.now(UTC), + ) + + def state(self) -> CliffResultState: + """Return the cliff result's state as it is now.""" + return self.writer.get_cliff_result(self.cliff_result_id) + + def still_held(self) -> bool: + """Return whether this call still holds the cliff result running.""" + state = self.state() + return ( + state.status is RunStatus.RUNNING + and state.modal_call_id == self.modal_call_id + ) + + def complete(self, impact: CliffImpact) -> RunWrite: + """Record the cliff gap and share; mark the cliff result succeeded.""" + return self.writer.complete_cliff_result( + self.cliff_result_id, + modal_call_id=self.modal_call_id, + cliff_gap=impact.cliff_gap, + cliff_share=impact.cliff_share, + completed_at=datetime.now(UTC), + ) + + def fail(self, detail: Stage12FailureDetail) -> RunWrite: + """Mark the cliff result failed with ``detail`` while this call holds it.""" + return self.writer.fail_cliff_result( + self.cliff_result_id, + modal_call_id=self.modal_call_id, + error_message=run_error_message(detail.error_code, detail.error_summary), + failed_at=datetime.now(UTC), + ) + + +class StandardCalculation(Protocol): + """Calculates a planned simulation in one worker; ``simulation._calculate``.""" + + def __call__( + self, + simulation: PlannedSimulationExecutionInput, + dependencies: SimulationWorkerDependencies, + *, + progress: StageProgress, + stage_span: Callable[[Stage], AbstractContextManager[object]], + cancellation_requested: Callable[[], bool], + ) -> SimulationCalculation: + """Return the calculation, timing each stage with ``stage_span``.""" + ... + + +def run_named_cliff_result( # noqa: PLR0913 # the call's inputs, then the worker's seams + payload: object, + context: Stage12InvocationContext, + *, + required_country: CountryId, + dependencies: SimulationWorkerDependencies, + calculate: StandardCalculation, + modal_call_id: str | None, +) -> dict[str, Any]: + """Run a call on the cliff result its context names. + + Raises ValueError, before any write, for a context naming no cliff + result, or a call without its Modal call ID. The store is opened before + the claim, so a missing setting claims nothing. Returns what + ``run_cliff_result`` returns. + """ + cliff_result_id = context.cliff_result_id + if cliff_result_id is None: + msg = "a cliff calculation's call names its cliff result" + raise ValueError(msg) + if modal_call_id is None: + msg = "a cliff calculation needs its Modal call ID" + raise ValueError(msg) + runtime = dependencies.runtime + recorder = CliffResultRecorder( + writer=dependencies.cliff_results or cliff_result_store(runtime), + cliff_result_id=cliff_result_id, + modal_call_id=modal_call_id, + ) + return run_cliff_result( + payload, + context, + required_country=required_country, + recorder=recorder, + dependencies=dependencies, + calculate=calculate, + ) + + +def run_cliff_result( # noqa: PLR0913 # the call's inputs, then the worker's seams + payload: object, + context: Stage12InvocationContext, + *, + required_country: CountryId, + recorder: CliffResultRecorder, + dependencies: SimulationWorkerDependencies, + calculate: StandardCalculation, +) -> dict[str, Any]: + """Claim the cliff result, calculate its run's cliffs, and record them. + + Returns ``{"cliff_result_id", "simulation_run_id", "status": + "succeeded", "cliff_gap", "cliff_share"}``, also without calculating for + a cliff result this call already completed, as when Modal restarts it. A + cliff result another call holds, or one that failed or that another call + completed, returns ``{"deduplicated": True, "cliff_result_id", + "status"}``. Once the call holds it, a failure, of the input or the + calculation, fails the cliff result and raises a ``RuntimeError`` + carrying no calculation detail; one failed meanwhile, as the stuck-run + sweep fails it, records nothing more and raises one that says it was + cancelled. A pre-emption propagates, since Modal runs the call again. + """ + runtime = dependencies.runtime + claimed = recorder.claim() + if not claimed.changed: + state = recorder.state() + if ( + claimed.status is RunStatus.SUCCEEDED + and state.modal_call_id == recorder.modal_call_id + ): + return _succeeded(recorder.cliff_result_id, state) + return { + "deduplicated": True, + "cliff_result_id": str(recorder.cliff_result_id), + "status": claimed.status.value, + } + progress = StageProgress(role=_role(payload)) + stage_span = partial( + tracked_stage, + tracking=StageTracking(STAGE12_SIMULATION_STAGES, progress, runtime), + ) + try: + simulation = _checked_simulation( + payload, context, recorder=recorder, required_country=required_country + ) + calculation = _calculation( + simulation, + dependencies, + calculate=calculate, + progress=progress, + stage_span=stage_span, + cancellation_requested=partial(_taken_from_call, recorder), + ) + validate_output_frames(calculation.frames, simulation.output_plan) + impact = cliff_impact_from_frames(calculation.frames, simulation.output_plan) + _require_completed(recorder, impact) + except Stage12CancellationError: + msg = "Stage 12 cliff calculation was cancelled" + raise RuntimeError(msg) from None + # Persist a redacted cause for any input or calculation failure before + # returning a stable exception to Modal. + except Exception as error: # noqa: BLE001 # every failure is recorded before a stable exception returns to Modal + _record_failure(error, recorder, runtime=runtime, progress=progress) + msg = "Stage 12 cliff calculation failed" + raise RuntimeError(msg) from None + return { + "cliff_result_id": str(recorder.cliff_result_id), + "simulation_run_id": str(simulation.simulation_execution_id), + "status": RunStatus.SUCCEEDED.value, + "cliff_gap": impact.cliff_gap, + "cliff_share": impact.cliff_share, + } + + +def _calculation( # noqa: PLR0913 # the simulation, then the worker's seams + simulation: PlannedSimulationExecutionInput, + dependencies: SimulationWorkerDependencies, + *, + calculate: StandardCalculation, + progress: StageProgress, + stage_span: Callable[[Stage], AbstractContextManager[object]], + cancellation_requested: Callable[[], bool], +) -> SimulationCalculation: + """Calculate the simulation as US region groups when it may be, else in one worker. + + ``cancellation_requested`` says whether the cliff result was taken from + the call, which a segmented calculation checks between polls. + """ + if dependencies.segmented_calculator is not None and ( + should_segment_cliff_calculation(simulation) + ): + return dependencies.segmented_calculator(simulation, cancellation_requested) + return calculate( + simulation, + dependencies, + progress=progress, + stage_span=stage_span, + cancellation_requested=cancellation_requested, + ) + + +def _require_completed(recorder: CliffResultRecorder, impact: CliffImpact) -> None: + """Record the values; raise if the cliff result was taken from this call meanwhile. + + One failed meanwhile, as the stuck-run sweep fails one, takes no write + from it: ``Stage12CancellationError``. + """ + if not recorder.complete(impact).changed: + msg = "Stage 12 cliff calculation was cancelled" + raise Stage12CancellationError(msg) + + +def cliff_impact_from_frames( + frames: Mapping[str, pd.DataFrame], output_plan: Stage12OutputPlan +) -> CliffImpact: + """Return the cliff gap and share of a calculation's weighted output rows. + + Each of .py's cliff variables is summed on the entity the plan + materializes it on, weighted by that entity's weight, as .py's own + cliff impact sums them. Raises ``Stage12InputError`` + (``cliff_share_undefined``) when there are no adults, or a total is not + a finite number. + """ + totals = { + variable: _weighted_total(frames, output_plan, variable) + for variable in CLIFF_IMPACT_VARIABLES + } + adults = totals[ADULT_COUNT_VARIABLE] + if not all(math.isfinite(total) for total in totals.values()) or adults <= 0: + raise Stage12InputError( + CLIFF_SHARE_UNDEFINED, + "The calculation has no adults, or a cliff total that is not a " + "finite number, so its cliff share has no value.", + ) + return CliffImpact( + cliff_gap=totals["cliff_gap"], + cliff_share=totals["is_on_cliff"] / adults, + ) + + +def _weighted_total( + frames: Mapping[str, pd.DataFrame], output_plan: Stage12OutputPlan, variable: str +) -> float: + """Return the weighted sum of ``variable`` on the entity the plan puts it on.""" + entity = next( + plan.entity + for plan in output_plan.entities + if variable in plan.materialized_variables + ) + weighted = MicroDataFrame(frames[entity], weights=f"{entity}_weight") + column = weighted[variable] + if not isinstance(column, pd.Series): + msg = f"{entity} output has more than one {variable!r} column" + raise TypeError(msg) + return float(column.sum()) + + +def _checked_simulation( + payload: object, + context: Stage12InvocationContext, + *, + recorder: CliffResultRecorder, + required_country: CountryId, +) -> PlannedSimulationExecutionInput: + """Return the simulation planned with the cliff columns, checked against its run. + + Raises ``Stage12InputError`` (``cliff_result_input_mismatch``) for a + payload that is not the simulation of the cliff result's run, on the + run's year and versions, ``Stage12InputError`` for a policy with an + invalid period, and ValueError for a context describing another worker. + The calculation then checks the run's versions are the installed ones. + """ + simulation = SimulationExecutionInput.model_validate(_without_output_plan(payload)) + simulation_run_id = recorder.state().simulation_run_id + run = recorder.writer.get_simulation_run(simulation_run_id) + calculated = { + "simulation run": simulation.simulation_execution_id, + "year": simulation.year, + "PolicyEngine.py version": simulation.bundle.policyengine_version, + "country package version": simulation.bundle.country_package_version, + } + recorded = { + "simulation run": simulation_run_id, + "year": run.year, + "PolicyEngine.py version": run.policyengine_version, + "country package version": run.country_package_version, + } + differences = [name for name in calculated if calculated[name] != recorded[name]] + if differences: + raise Stage12InputError( + CLIFF_RESULT_INPUT_MISMATCH, + "The payload is not the simulation of the cliff result's run: its " + + ", ".join(differences) + + " differ.", + ) + planned = plan_simulation_input(simulation, resolve_cliff_output_plan(simulation)) + require_worker_context(planned, context, required_country=required_country) + validate_policy_periods(planned.policy) + return planned + + +def _without_output_plan(payload: object) -> object: + """Return ``payload`` without an output plan: the cliff calculation plans one.""" + if isinstance(payload, dict) and "output_plan" in payload: + return {key: value for key, value in payload.items() if key != "output_plan"} + return payload + + +def _succeeded(cliff_result_id: UUID, state: CliffResultState) -> dict[str, Any]: + """Return what a call returns for a cliff result it completed.""" + return { + "cliff_result_id": str(cliff_result_id), + "simulation_run_id": str(state.simulation_run_id), + "status": RunStatus.SUCCEEDED.value, + "cliff_gap": state.cliff_gap, + "cliff_share": state.cliff_share, + } + + +def _role(payload: object) -> str | None: + """Return the role a payload names, for stage tracking, before it is checked.""" + role = payload.get("role") if isinstance(payload, dict) else None + return role if isinstance(role, str) else None + + +def _taken_from_call(recorder: CliffResultRecorder) -> bool: + """Return whether the cliff result is no longer running under this call. + + A segmented calculation checks it between polls. A failed read is + treated as not taken, so the calculation goes on. + """ + try: + return not recorder.still_held() + except Exception: # noqa: BLE001 # a failed read must not stop the calculation + return False + + +def _record_failure( + error: Exception, + recorder: CliffResultRecorder, + *, + runtime: ObservabilityRuntime | None, + progress: StageProgress, +) -> None: + """Report a failure once, then fail the cliff result while this call holds it.""" + if isinstance(error, Stage12ExecutionError): + # A segmented calculation reports its segment failure where it + # occurs, before raising this error. + detail = error.detail + else: + detail = report_failure( + error, + FailureSite( + scope="cliff_calculation", + default_code="cliff_calculation_failed", + attributes={ + "simulation_role": progress.process_role, + "cliff_result_id": str(recorder.cliff_result_id), + }, + ), + runtime=runtime, + progress=progress, + ) + record_failure_cause(runtime, error, detail.error_summary) + persist_failure_record( + partial(recorder.fail, detail), + runtime=runtime, + failure_scope="cliff_calculation", + ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/coordination.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/coordination.py index 90615d908..056c00e1e 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/coordination.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/coordination.py @@ -86,8 +86,11 @@ def coordinate_report( """ dependencies = dependencies or CoordinatorDependencies() context = Stage12InvocationContext.model_validate(context_payload) - if context.simulation_run_id is not None: - msg = "a coordinator's call names a report run, not a simulation run" + if context.simulation_run_id is not None or context.cliff_result_id is not None: + msg = ( + "a coordinator's call names a report run, not a simulation run or " + "a cliff result" + ) raise ValueError(msg) if context.report_run is not None: if parent_payload is not None: diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/output_planning.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/output_planning.py index fa557ccd6..7a5b398c8 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/output_planning.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/output_planning.py @@ -368,42 +368,99 @@ def _resolve_output_plan( # noqa: PLR0913 # the two simulations, then keyword- labor_supply_response_active=labor_supply_active, ) with stage_scope(Stage.STAGE12_OUTPUT_VARIABLE_RESOLUTION, stage_attributes): - baseline_variables = model.resolve_entity_variables(baseline) - reform_variables = model.resolve_entity_variables(reform) - dataset_variables = _required_dataset_variables( - baseline_input, aggregates=requirements.aggregates + entities = _entity_plans( + model, + (baseline, reform), + dataset_variables=_required_dataset_variables( + baseline_input, aggregates=requirements.aggregates + ), ) - entities = [] - for entity in sorted( - set(baseline_variables) | set(reform_variables) | set(dataset_variables) - ): - entity_dataset_variables = dataset_variables.get(entity, ()) - materialized = tuple( - sorted( - set(baseline_variables.get(entity, ())) - | set(reform_variables.get(entity, ())) - | set(entity_dataset_variables) + return Stage12OutputPlan( + country=country, + requirements=requirements, + entities=entities, + ) + + +def resolve_cliff_output_plan( + simulation: SimulationExecutionInput, + *, + stage_scope: StageScope = _unmonitored_stage_scope, +) -> Stage12OutputPlan: + """Resolve the schema of a cliff calculation of ``simulation``'s run. + + The country model's default columns, and the columns .py's cliff impact + reads (``cliff_gap``, ``is_on_cliff`` and ``is_adult``), with no + labour-supply response, US budget or area columns: a policy's response + changes its net incomes, and so its cliffs, whether or not its columns + are stored. The requirements record whether the simulation's own policy + turns a response on, so ``should_segment_cliff_calculation`` splits the + calculation into region groups exactly when the run's own is split. + """ + country = simulation.geography.country + stage_attributes = {"country": country} + with stage_scope(Stage.STAGE12_COUNTRY_MODEL_LOAD, stage_attributes): + model = _country_model(country) + with stage_scope(Stage.STAGE12_OUTPUT_CONFIGURATION, stage_attributes): + planning = _planning_simulation(simulation, model=model) + requirements = ReportOutputRequirements( + aggregates=tuple(ReportAggregate), + include_cliff_impacts=True, + labor_supply_response_active=labor_supply_response_is_active( + planning, planning, country_code=country + ), + ) + configure_cliff_impact_variables(planning) + with stage_scope(Stage.STAGE12_OUTPUT_VARIABLE_RESOLUTION, stage_attributes): + entities = _entity_plans(model, (planning,), dataset_variables={}) + return Stage12OutputPlan( + country=country, + requirements=requirements, + entities=entities, + ) + + +def _entity_plans( + model: OutputVariableModel, + simulations: tuple[Simulation, ...], + *, + dataset_variables: Mapping[str, tuple[str, ...]], +) -> tuple[EntityOutputPlan, ...]: + """Return each entity's columns: every simulation's, and the dataset's copied. + + A column is additional when a simulation adds it to the model's defaults. + """ + variables = [model.resolve_entity_variables(item) for item in simulations] + entity_names = set(dataset_variables).union(*variables) + entities = [] + for entity in sorted(entity_names): + entity_dataset_variables = dataset_variables.get(entity, ()) + materialized = tuple( + sorted( + set(entity_dataset_variables).union( + *(resolved.get(entity, ()) for resolved in variables) ) ) - additional = tuple( - sorted( - set((baseline.extra_variables or {}).get(entity, ())) - | set((reform.extra_variables or {}).get(entity, ())) + ) + additional = tuple( + sorted( + set().union( + *( + (item.extra_variables or {}).get(entity, ()) + for item in simulations + ) ) ) - entities.append( - EntityOutputPlan( - entity=entity, - materialized_variables=materialized, - additional_variables=additional, - dataset_variables=entity_dataset_variables, - ) + ) + entities.append( + EntityOutputPlan( + entity=entity, + materialized_variables=materialized, + additional_variables=additional, + dataset_variables=entity_dataset_variables, ) - return Stage12OutputPlan( - country=country, - requirements=requirements, - entities=tuple(entities), - ) + ) + return tuple(entities) def plan_simulation_input( diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/partition.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/partition.py index a1e3c0a09..f7821ec3d 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/partition.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/partition.py @@ -73,15 +73,28 @@ def stage12_region_groups_for_model(country: str, model: object) -> list[list[st def should_segment_simulation(simulation: PlannedSimulationExecutionInput) -> bool: """Return whether this logical simulation should use the US partition.""" + if simulation.output_plan.requirements.include_cliff_impacts: + return False + return should_segment_cliff_calculation(simulation) + + +def should_segment_cliff_calculation( + simulation: PlannedSimulationExecutionInput, +) -> bool: + """Return whether a cliff calculation of this simulation uses the US partition. + + It is split exactly when the simulation's own run is: a US national + simulation, not marked ``segmented: false``, whose policy turns no + labour-supply response on. Its cliff columns don't prevent it, since + each person's depend only on their own household's calculation and the + cliff share is taken once from the merged rows (``cliff_results``). + """ if simulation.geography.country != "us": return False if simulation.geography.region.strip().lower() != "us": return False if simulation.options.get("segmented") is False: return False - requirements = simulation.output_plan.requirements - if requirements.include_cliff_impacts: - return False - if requirements.labor_supply_response_active: + if simulation.output_plan.requirements.labor_supply_response_active: return False return stage12_region_groups("us") is not None diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/report_run_access.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/report_run_access.py index 4465ccd57..4e5167108 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/report_run_access.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/report_run_access.py @@ -3,10 +3,10 @@ An operator creates the Modal secret in each Modal environment. The v2 Modal application attaches it to the coordinator, which records the report run its call names, to the simulation workers, each of which claims and records -the simulation run its call names, and to the household functions, which -record the household job their call names. The coordinator also starts a -budget window's waiting years, and it and the stuck-run sweep record the -window's outcome. The secret must exist in a Modal environment before the +the simulation run or cliff result its call names, and to the household +functions, which record the household job their call names. The coordinator +also starts a budget window's waiting years, and it and the stuck-run sweep +record the window's outcome. The secret must exist in a Modal environment before the v2 application deploys there: ``modal deploy`` checks its required key and fails without it. """ @@ -17,7 +17,11 @@ from functools import partial from typing import TYPE_CHECKING, Protocol -from policyengine_stage12_persistence import BudgetWindowStore, ReportRunStore +from policyengine_stage12_persistence import ( + BudgetWindowStore, + CliffResultStore, + ReportRunStore, +) from .database_retry import open_session_with_retry @@ -182,6 +186,18 @@ def report_run_store(runtime: ObservabilityRuntime | None = None) -> ReportRunSt ) +def cliff_result_store(runtime: ObservabilityRuntime | None = None) -> CliffResultStore: + """Build the cliff result store from ``MODAL_WORKER_DATABASE_URL``. + + Raises ``ValueError`` when the variable is unset or not a complete + ``postgresql://`` URL. + """ + return CliffResultStore( + os.environ.get(MODAL_WORKER_DATABASE_URL_ENV, ""), + open_session=partial(open_session_with_retry, runtime=runtime), + ) + + def budget_window_store( runtime: ObservabilityRuntime | None = None, ) -> BudgetWindowStore: diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/simulation.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/simulation.py index c552bdc09..188f3bb76 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/simulation.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/simulation.py @@ -42,6 +42,7 @@ ) from .cached_simulation import Stage12CachedSimulation, qualifying_cache_identity +from .cliff_results import run_named_cliff_result from .dataset_download_retry import load_with_download_retry from .dependencies import ComparisonStore, artifact_store, runtime_store from .failure_causes import ( @@ -91,6 +92,7 @@ RegionResolution, ) + from .cliff_results import CliffResultWriter from .report_run_access import ReportRunWriter from .reused_outputs import OutputFileReader @@ -472,7 +474,8 @@ class SimulationWorkerDependencies: the standard country-package calculation, the deployed region-group segment runner, no telemetry runtime, and, for a call that names a simulation run, the run store, the output planning of a simulation on - its own, and the Cloud Storage reader of a succeeded run's output file. + its own, and the Cloud Storage reader of a succeeded run's output file; + for one naming a cliff result, the cliff result store. """ store: ComparisonStore | None = None @@ -494,6 +497,7 @@ class SimulationWorkerDependencies: resolve_simulation_output_plan ) output_files: OutputFileReader | None = None + cliff_results: CliffResultWriter | None = None def run_single_simulation( @@ -508,7 +512,8 @@ def run_single_simulation( A detailed-save call's context names the simulation run it claims, and ``run_simulation_run`` records that run alone, under ``modal_call_id``, - keeping the output file in the simulation output bucket. A + keeping the output file in the simulation output bucket; one naming a + cliff result calculates that run's cliffs (``cliff_results``). A comparison-path call records its evaluation child record: a child already marked succeeded returns its stored descriptor without recalculating, and a failure after it is marked running marks it FAILED @@ -518,6 +523,15 @@ def run_single_simulation( dependencies = dependencies or SimulationWorkerDependencies() runtime = dependencies.runtime context = Stage12InvocationContext.model_validate(context_payload) + if context.cliff_result_id is not None: + return run_named_cliff_result( + payload, + context, + required_country=required_country, + dependencies=dependencies, + calculate=_calculate, + modal_call_id=modal_call_id, + ) if context.records_runs: return _run_named_simulation_run( payload, diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_cliff_results.py b/projects/policyengine-simulation-executor/tests/test_stage12_cliff_results.py index 9bd46a4a7..f8e5c7eb9 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_cliff_results.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_cliff_results.py @@ -123,7 +123,7 @@ def _run_cliff_worker( required_country=simulation.geography.country, modal_call_id=dependencies.pop("modal_call_id", WORKER_CALL_ID), dependencies=SimulationWorkerDependencies( - report_runs=cliff_results, **dependencies + cliff_results=cliff_results, **dependencies ), ) From 103a6d921075c9f6b11a5fff9b6d611b20a4f9b5 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:05:41 +0400 Subject: [PATCH 06/24] The stuck-run sweep fails a dead cliff calculation (failing) A cliff calculation left pending or running by a call that ended, or one never started, would make every later request for the run's cliffs join it. The sweep judges cliff results as it judges household jobs: after the other kinds, by their recorded call, and by the age limit. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_cliff_result_store.py | 82 +++++++++++++++++++ .../tests/test_stage12_run_sweep.py | 40 ++++++++- 2 files changed, 120 insertions(+), 2 deletions(-) diff --git a/libs/policyengine-stage12-persistence/tests/test_cliff_result_store.py b/libs/policyengine-stage12-persistence/tests/test_cliff_result_store.py index 055ddd601..4547b6063 100644 --- a/libs/policyengine-stage12-persistence/tests/test_cliff_result_store.py +++ b/libs/policyengine-stage12-persistence/tests/test_cliff_result_store.py @@ -14,8 +14,10 @@ import pytest from policyengine_simulation_persistence import CliffImpact from policyengine_stage12_persistence import ( + ActiveRun, CliffResultState, CliffResultStore, + RunKind, RunStatus, RunWrite, ) @@ -28,6 +30,9 @@ LATER = NOW + timedelta(minutes=2) CLIFF_GAP = 1_234.5 CLIFF_SHARE = 0.0375 +# The sweep's cutoff: a cliff result active since before it is listed. +CUTOFF = NOW + timedelta(hours=1) +ENDED_MESSAGE = "modal_call_ended: the call ended without an outcome" @pytest.fixture @@ -232,3 +237,80 @@ def test_the_modal_worker_role_cannot_insert_or_retarget_a_cliff_result( .values(simulation_run_id=seeded_run.run_ids.reform_simulation_run_id), execution_options={"synchronize_session": False}, ) + + +def _listed_cliff_results(store, cliff_result_id, cutoff=CUTOFF) -> set[ActiveRun]: + """Return the sweep's listing of cliff results, narrowed to ``cliff_result_id``.""" + return { + listed + for listed in store.list_active_runs( + RunKind.CLIFF_RESULT, active_since_before=cutoff + ) + if listed.run_id == cliff_result_id + } + + +def test_the_sweep_lists_a_cliff_result_from_its_claim_or_its_creation( + store, seeder, seeded_run +) -> None: + cliff_result_id = _insert_cliff_result(seeder, seeded_run, created_at=NOW) + + assert _listed_cliff_results(store, cliff_result_id) == { + ActiveRun( + kind=RunKind.CLIFF_RESULT, + run_id=cliff_result_id, + status=RunStatus.PENDING, + modal_call_id=None, + ) + } + assert _listed_cliff_results(store, cliff_result_id, NOW) == set() + + # Once claimed, it counts from its claim, after the cutoff here. + store.claim_cliff_result( + cliff_result_id, + modal_call_id=WORKER_CALL_ID, + claimed_at=CUTOFF + timedelta(hours=1), + ) + assert _listed_cliff_results(store, cliff_result_id) == set() + assert _listed_cliff_results( + store, cliff_result_id, CUTOFF + timedelta(hours=2) + ) == { + ActiveRun( + kind=RunKind.CLIFF_RESULT, + run_id=cliff_result_id, + status=RunStatus.RUNNING, + modal_call_id=WORKER_CALL_ID, + ) + } + + +def test_the_sweep_fails_a_cliff_result_only_while_it_is_as_listed( + store, seeder, seeded_run +) -> None: + cliff_result_id = _insert_cliff_result(seeder, seeded_run, created_at=NOW) + (listed_without_call,) = _listed_cliff_results(store, cliff_result_id) + # After the listing, the simulation service recorded the call it started. + with seeder.engine.begin() as connection: + connection.execute( + update(CliffImpact.__table__) + .where(CliffImpact.id == cliff_result_id) + .values(modal_call_id=WORKER_CALL_ID) + ) + late_call = store.fail_stuck_run( + listed_without_call, error_message=ENDED_MESSAGE, failed_at=CUTOFF + ) + (listed_with_call,) = _listed_cliff_results(store, cliff_result_id) + + first = store.fail_stuck_run( + listed_with_call, error_message=ENDED_MESSAGE, failed_at=CUTOFF + ) + again = store.fail_stuck_run( + listed_with_call, error_message=ENDED_MESSAGE, failed_at=CUTOFF + ) + + assert late_call == RunWrite(changed=False, status=RunStatus.PENDING) + assert first == RunWrite(changed=True, status=RunStatus.FAILED) + assert again == RunWrite(changed=False, status=RunStatus.FAILED) + row = _cliff_row(seeder, cliff_result_id) + assert (row["status"], row["error_message"]) == ("failed", ENDED_MESSAGE) + assert (row["completed_at"], row["updated_at"]) == (CUTOFF, CUTOFF) diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep.py b/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep.py index f8991c9b4..28c4aa1f3 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep.py @@ -156,11 +156,13 @@ def test_the_sweep_lists_runs_older_than_its_minimum_age() -> None: def test_report_runs_are_swept_before_simulation_runs_are_listed() -> None: # A simulation run is listed only once no report run naming it is # running, so failing a dead coordinator's report run first lets the same - # sweep reach its children. Household jobs, which no run names, follow. + # sweep reach its children. Household jobs and simulation runs' cliff + # calculations, which no run names, follow. report_run = _run("fc-coordinator", RunKind.REPORT_RUN) simulation_run = _run("fc-worker") household_job = _run("fc-household", RunKind.HOUSEHOLD_JOB) - store = FakeSweepStore([household_job, simulation_run, report_run]) + cliff_result = _run("fc-cliffs", RunKind.CLIFF_RESULT) + store = FakeSweepStore([cliff_result, household_job, simulation_run, report_run]) sweep_stuck_runs(store, call_state=lambda _: CallState.ENDED, now=NOW) @@ -168,15 +170,49 @@ def test_report_runs_are_swept_before_simulation_runs_are_listed() -> None: "list-expired:report_run", "list-expired:simulation_run", "list-expired:household_job", + "list-expired:cliff_result", "list:report_run", "fail:report_run", "list:simulation_run", "fail:simulation_run", "list:household_job", "fail:household_job", + "list:cliff_result", + "fail:cliff_result", ] +def test_cliff_results_are_judged_as_runs_are() -> None: + """A dead cliff calculation fails, so a later request starts another one.""" + # The simulation service stopped before starting the call. + never_started = _run(None, RunKind.CLIFF_RESULT, RunStatus.PENDING) + # The worker ran out of memory after claiming it. + ended_running = _run("fc-ended", RunKind.CLIFF_RESULT) + still_running = _run("fc-running", RunKind.CLIFF_RESULT) + expired = _run("fc-old", RunKind.CLIFF_RESULT) + store = FakeSweepStore( + [never_started, ended_running, still_running], expired=[expired] + ) + states = {"fc-ended": CallState.ENDED, "fc-running": CallState.RUNNING} + + summary = sweep_stuck_runs(store, call_state=states.__getitem__, now=NOW) + + failed = {run: message for run, message, _ in store.failed} + assert set(failed) == {expired, never_started, ended_running} + assert failed[expired].startswith("run_exceeded_maximum_age: ") + assert failed[never_started].startswith("run_not_started: ") + assert failed[ended_running].startswith("modal_call_ended: ") + assert summary == SweepSummary( + checked=3, + failed=1, + running=1, + unknown=0, + left=0, + failed_for_age=1, + failed_without_call=1, + ) + + def test_household_jobs_are_judged_as_runs_are() -> None: # The simulation service stopped before starting the call. never_started = _run(None, RunKind.HOUSEHOLD_JOB, RunStatus.PENDING) From c0875637140e8541caac2769b642ab0b15b492e0 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:06:48 +0400 Subject: [PATCH 07/24] Sweep simulation runs' cliff results left by a call that ended RunKind.CLIFF_RESULT names cliff_impacts, and the sweep judges it last, as it judges household jobs: a cliff result pending with no call, or whose recorded call ended, or past the age limit, is failed while it is as listed, so the next request for the run's cliffs starts another. Co-Authored-By: Claude Opus 5.5 --- .../report_run_store.py | 14 +++++++----- .../stage12_runtime/run_sweep.py | 22 ++++++++++++++----- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py index f52362092..d78aca0fc 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py @@ -45,7 +45,7 @@ class RunStatus(StrEnum): - """A report run's, simulation run's or household job's status.""" + """A report run's, simulation run's, household job's or cliff result's status.""" PENDING = "pending" RUNNING = "running" @@ -57,11 +57,12 @@ class RunStatus(StrEnum): class RunKind(StrEnum): - """Which run table a run, or a household job, is in.""" + """Which table a run, household job or simulation run's cliff result is in.""" REPORT_RUN = "report_run" SIMULATION_RUN = "simulation_run" HOUSEHOLD_JOB = "household_job" + CLIFF_RESULT = "cliff_result" type RunModel = type[ReportRun | SimulationRun | HouseholdJob | CliffImpact] @@ -70,6 +71,7 @@ class RunKind(StrEnum): RunKind.REPORT_RUN: ReportRun, RunKind.SIMULATION_RUN: SimulationRun, RunKind.HOUSEHOLD_JOB: HouseholdJob, + RunKind.CLIFF_RESULT: CliffImpact, } # What a missing row is called. _ROW_NOUNS: Final[dict[RunModel, str]] = { @@ -728,14 +730,16 @@ def running_under_call(model: RunModel, modal_call_id: str) -> ColumnElement[boo def _active_since(kind: RunKind) -> SQLColumnExpression[datetime]: """Return when a run of ``kind`` became active, for the sweep's ages. - A household job counts from its creation. A run counts from its first - claim or, before it, from its creation, except a budget window's year, - which counts from its last change: when its call was recorded. + A household job counts from its creation. A run or cliff result counts from + its first claim or, before it, from its creation, except a budget window's + year, which counts from its last change: when its call was recorded. """ if kind is RunKind.HOUSEHOLD_JOB: return HouseholdJob.created_at if kind is RunKind.SIMULATION_RUN: return func.coalesce(SimulationRun.started_at, SimulationRun.created_at) + if kind is RunKind.CLIFF_RESULT: + return func.coalesce(CliffImpact.started_at, CliffImpact.created_at) unclaimed_since = case( (ReportRun.budget_window_run_id.is_not(None), ReportRun.updated_at), else_=ReportRun.created_at, diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/run_sweep.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/run_sweep.py index b08c32346..cbb58ceba 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/run_sweep.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/run_sweep.py @@ -30,7 +30,11 @@ Household jobs follow the same rule: the simulation service records the call it starts on a pending job, and a household function claims the job under that call. A household job counts from its creation, while pending or -running, for both ages. +running, for both ages. A simulation run's cliff result, which the +simulation service starts and a worker claims, is judged as a run is, from +its first claim or, while pending, its creation. No run names it, so it is +judged last; once failed it never counts, and the next request for its +run's cliffs starts another calculation. A budget window's years are report runs naming its budget-window run. A year still pending with no call, in a window that hasn't finished, waits @@ -161,8 +165,13 @@ def settle_idle_window( # The order the sweep judges each kind in: report runs before simulation # runs, which are listed only once no unfinished report run names them, then -# household jobs, which no run names. -SWEPT_KINDS = (RunKind.REPORT_RUN, RunKind.SIMULATION_RUN, RunKind.HOUSEHOLD_JOB) +# household jobs and simulation runs' cliff results, which no run names. +SWEPT_KINDS = ( + RunKind.REPORT_RUN, + RunKind.SIMULATION_RUN, + RunKind.HOUSEHOLD_JOB, + RunKind.CLIFF_RESULT, +) @dataclass(frozen=True) @@ -334,9 +343,10 @@ def sweep_stuck_runs( # noqa: PLR0913 # the stores, then keyword-only seams an ) -> SweepSummary: """Fail every listed run whose recorded Modal call has ended or was never started. - Household jobs are judged as runs are. Runs older than ``maximum_age`` - are failed first, report runs, then simulation runs, then household - jobs, before any Modal lookup can use up the scheduled function's time. + Household jobs and cliff results are judged as runs are. Runs older + than ``maximum_age`` are failed first, report runs, then simulation + runs, then household jobs, then cliff results, before any Modal lookup + can use up the scheduled function's time. Then, in the same order, so a dead coordinator's simulation runs are listed in the same sweep, each run older than ``minimum_age`` is judged: a pending run with no call recorded is failed, and each recorded call is From 3e5cdcdb6be83431854d6d8079d6e06cb189e391 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:08:45 +0400 Subject: [PATCH 08/24] Document each simulation run's cliff calculation The Stage 12 foundation document, the two persistence libraries' READMEs, the executor's README and the v2 app's workers say how a worker call naming a cliff result calculates and records its run's cliff gap and share, how a US national calculation is split, and that the sweep judges cliff results. Adds the changelog fragment. Co-Authored-By: Claude Opus 5.5 --- changelog.d/c1-38-run-cliff-results.added.md | 1 + .../stage-12-modal-worker-foundation.md | 44 ++++++++++++++++--- .../README.md | 36 ++++++++++----- .../README.md | 4 ++ .../README.md | 5 ++- .../src/modal/v2_app.py | 12 +++-- 6 files changed, 81 insertions(+), 21 deletions(-) create mode 100644 changelog.d/c1-38-run-cliff-results.added.md diff --git a/changelog.d/c1-38-run-cliff-results.added.md b/changelog.d/c1-38-run-cliff-results.added.md new file mode 100644 index 000000000..51a418bbf --- /dev/null +++ b/changelog.d/c1-38-run-cliff-results.added.md @@ -0,0 +1 @@ +A v2 simulation worker can now calculate a simulation run's own cliff result: a call whose context names a row of the Public API's `cliff_impacts` calculates that run's simulation again with .py's cliff columns and records its weighted cliff gap and cliff share, once, on that row, and the stuck-run sweep fails one whose call ended. A report run's completion no longer writes `report_runs`' four cliff columns. Nothing starts a cliff calculation yet, and a report asking for cliffs is still refused. diff --git a/docs/migration/stage-12-modal-worker-foundation.md b/docs/migration/stage-12-modal-worker-foundation.md index 9113dcb74..c38752cc2 100644 --- a/docs/migration/stage-12-modal-worker-foundation.md +++ b/docs/migration/stage-12-modal-worker-foundation.md @@ -367,6 +367,37 @@ stored output's descriptor without calculating, and for a run another call holds, or one that failed, it returns `{"deduplicated": true, "simulation_run_id", "status"}`. +**A simulation run's cliff result** (C1 · 24, answer 4b). Each simulation run +can have one cliff result, its cliff gap and cliff share, which every report +naming the run reads: a row of the Public API's `cliff_impacts`, one per +calculation. The simulation service inserts it pending, starts the country's +simulation worker (`run_single_simulation_us` or `run_single_simulation_uk`) +with the run's own simulation input and a context naming the row +(`Stage12InvocationContext.cliff_result_id`), and records the call. The worker +claims the row as a run is claimed, and refuses, failing it with +`cliff_result_input_mismatch`, an input that is not the simulation of the +row's run on that run's year and versions; the calculation then checks those +versions are the installed bundle's, so a result never comes from other +versions than its run's. It plans the simulation with the country model's +default columns and .py's three cliff columns (`resolve_cliff_output_plan`), +calculates it again from its dataset and policy, since `cliff_gap` reads +marginal tax rates, which recalculate each household's net income with extra +earnings, and writes .py's cliff gap (the weighted sum of `cliff_gap`) and +cliff share (the weighted count of people `is_on_cliff` over the weighted +count of `is_adult`) only while it holds the row; a calculation with no adults +fails with `cliff_share_undefined`. A US national calculation is split into +the 20 region groups whenever its run's own calculation would be +(`should_segment_cliff_calculation`): each person's cliff columns depend only +on their own household's calculation, each group keeps its households' +weights, and the share is taken once, from the merged rows. A second start, +or a row another call holds or that finished, calculates nothing and returns +`{"deduplicated": true, "cliff_result_id", "status"}`; the call that completed +it returns its values again when Modal restarts it. A failure fails the row +with its redacted cause; the stuck-run sweep fails one whose call ended +without an outcome, and a failed row never counts, so the next request starts +another calculation. The worker writes nothing to the simulation run, its +output file or any report run. + **What a simulation run stores** (C1 · 24, answer 4). A run's output file depends on its simulation alone, never on a report naming it (`resolve_simulation_output_plan`): every column that doesn't depend on the @@ -375,10 +406,11 @@ columns; the labour-supply response columns only when the simulation's own policy turns a response on; in the US, the two inputs .py's labour-supply output reads from either side (`self_employment_income`, `weekly_hours_worked`); and never the cliff columns, so a US national run keeps -its region-group split. Until cliffs come back as a calculation of each -simulation run (a later stage), a report run asking for them -(`include_cliffs: true`) fails before any worker starts, with -`cliff_impacts_unavailable`, and touches none of its simulation runs. The +its region-group split. Cliffs are a calculation of each simulation run of +their own (below). A report run asking for them (`include_cliffs: true`) still +fails before any worker starts, with `cliff_impacts_unavailable`, and touches +none of its simulation runs, and its completion never writes `report_runs`' +four cliff columns. The coordinator gives each worker its simulation's stored plan, and keeps the report's own plan, from its pair, for what the aggregation reads. A reused or waited-for file must hold what the report reads @@ -947,7 +979,9 @@ report run and its simulation runs are), counting a job's age from its creation: a pending job with no call recorded ten minutes after its creation, a pending or running job once Modal reports its recorded call ended, and any job still unfinished 24 hours -after its creation, whatever Modal reports. +after its creation, whatever Modal reports. It judges simulation runs' cliff +results last, by the same rule, counting from a result's first claim or, +while it is pending, its creation. - Image: the CA store, the frozen `modal-simulation-image` dependency group and the source packages, without the bundle's datasets or the Stage 12 diff --git a/libs/policyengine-simulation-persistence/README.md b/libs/policyengine-simulation-persistence/README.md index 2c4115e6f..dcedad60e 100644 --- a/libs/policyengine-simulation-persistence/README.md +++ b/libs/policyengine-simulation-persistence/README.md @@ -81,6 +81,14 @@ household report with only a baseline). A report with a `budget_window_size` is calculated as a `budget_window_runs` row with one report run per year; each year's report run names that row and its year, and belongs to the same report. +`cliff_impact_tables.py` maps a seventh table the service creates rows in, on +`ServiceBase` too: `cliff_impacts` (`CliffImpact`), each cliff calculation of a +simulation run. The service inserts a row pending and records the Modal call +it starts; the Modal worker role claims it and records the run's cliff gap +and cliff share, or the failure. A run has at most one calculation that +hasn't failed (`uq_cliff_impacts_unfailed`), so identical requests join it, +and a failed one stays recorded. Deleting a run deletes its calculations. + ## Result tables `result_tables.py` models the twelve result tables the Modal worker role @@ -434,10 +442,11 @@ uv run pytest tests/ -m public_api_schema `SERVICE_TABLE_PRIVILEGES` lists the table privileges the service's role needs: `SELECT` and `INSERT` on the tables it creates rows in (`simulations`, `reports`, `simulation_runs`, `report_runs`, -`budget_window_runs` and `household_jobs`), and `SELECT` on the policy, -dynamic, household and catalog tables a run reads. `SERVICE_UPDATABLE_COLUMNS` -lists the columns it may `UPDATE` on `simulation_runs`, `report_runs` and -`household_jobs`: `status`, `modal_call_id`, `error_message`, `completed_at` +`budget_window_runs`, `household_jobs` and `cliff_impacts`, a simulation +run's cliff calculations), and `SELECT` on the policy, dynamic, household and +catalog tables a run reads. `SERVICE_UPDATABLE_COLUMNS` lists the columns it +may `UPDATE` on `simulation_runs`, `report_runs`, `household_jobs` and +`cliff_impacts`: `status`, `modal_call_id`, `error_message`, `completed_at` and `updated_at`, to record a Modal call's ID once Modal accepts the call, or to fail the run when Modal refuses it. On `budget_window_runs` it lists `status`, `error_message`, `completed_at` and `updated_at`, to fail a window @@ -447,14 +456,17 @@ refuses any other change to a run. The Public API's privilege manifest grants them all. `MODAL_WORKER_TABLE_PRIVILEGES` and `MODAL_WORKER_UPDATABLE_COLUMNS` list what -the Modal worker role needs: `SELECT` on the four run tables, `SELECT` and -`INSERT` on every result table, because a session's insert reads the row's -server defaults back, and `UPDATE` of each run's state (`status`, -`modal_call_id`, `error_message`, `started_at`, `completed_at` and -`updated_at`; a budget-window run has no call of its own) and of the results -its own row holds: a report run's cliff impact, `spm_provenance` and -`data_version`, a simulation run's output file and `household_result`, and a -household job's `result`. The Public API's manifest grants them; +the Modal worker role needs: `SELECT` on the four run tables and +`cliff_impacts`, `SELECT` and `INSERT` on every result table, because a +session's insert reads the row's server defaults back, and `UPDATE` of each +run's and cliff calculation's state (`status`, `modal_call_id`, +`error_message`, `started_at`, `completed_at` and `updated_at`; a budget-window +run has no call of its own) and of the results its own row holds: a report +run's `spm_provenance` and `data_version`, a simulation run's output file and +`household_result`, a household job's `result`, and a cliff calculation's +`cliff_gap` and `cliff_share`. It no longer writes a report run's four cliff +columns, which the Public API is to drop: a report run's cliff impact is its +two simulation runs' cliff results. The Public API's manifest grants them; `test_modal_worker_writes` checks that in CI. `missing_table_privileges` returns each one the connected role lacks, ordered diff --git a/libs/policyengine-stage12-persistence/README.md b/libs/policyengine-stage12-persistence/README.md index bdeeeb042..b64031e11 100644 --- a/libs/policyengine-stage12-persistence/README.md +++ b/libs/policyengine-stage12-persistence/README.md @@ -38,6 +38,10 @@ and ends the process with a failure if the check runs for 15 minutes. and records the one simulation run its call names, the v2 household functions record the household job their call names, and the scheduled stuck-run sweep fails the runs and household jobs nothing will finish. + `CliffResultStore` adds a simulation run's cliff calculation + (`cliff_impacts`): a v2 worker whose call names one claims it and records + the run's cliff gap and share, or its failure, and the sweep fails one + nothing will finish. The coordinator also reads a simulation run's status, year, versions, dataset, region, output file, recorded call and failure, so a report run can use a run that already succeeded instead of calculating it again, or diff --git a/projects/policyengine-simulation-executor/README.md b/projects/policyengine-simulation-executor/README.md index acf16e000..b4c1669ea 100644 --- a/projects/policyengine-simulation-executor/README.md +++ b/projects/policyengine-simulation-executor/README.md @@ -375,7 +375,10 @@ dtype-preserving entity tables. This produces 40 region-group calculation invocations per report while retaining two simulation records and two private simulation artifacts. `segmented=false`, US subnational requests, UK requests, cliff analysis, and labor-supply-response calculations use the -single-container Stage 12 path. +single-container Stage 12 path. A simulation run's own cliff calculation (a +worker call whose context names its cliff result) is split into the same 20 +region groups whenever the run's own calculation is: its cliff gap and share +are weighted sums of the merged person rows. A comparison run keeps both simulation output files in the Stage 12 artifact bucket (`STAGE12_ARTIFACT_BUCKET`), which deletes every object 30 days after it diff --git a/projects/policyengine-simulation-executor/src/modal/v2_app.py b/projects/policyengine-simulation-executor/src/modal/v2_app.py index 0885a3030..a134a655f 100644 --- a/projects/policyengine-simulation-executor/src/modal/v2_app.py +++ b/projects/policyengine-simulation-executor/src/modal/v2_app.py @@ -1,7 +1,9 @@ # pyright: reportAttributeAccessIssue=false """Separately named v2 Modal application: report coordinator and country workers. -Besides the Stage 12 report coordinator and simulation workers, it defines a +A simulation worker's call calculates a simulation run, or, when its context +names a cliff result, that run's cliff gap and share. Besides the Stage 12 +report coordinator and simulation workers, it defines a household function for each country, ``calculate_household_us`` and ``calculate_household_uk``, in an image without datasets. A call calculates one household under one policy, and records it on the household job it @@ -388,7 +390,9 @@ def run_single_simulation_us( A ``context`` naming a simulation run claims that run under this call's Modal call ID; a simulation started on its own, without an output plan, - is planned here. + is planned here. A ``context`` naming a cliff result instead calculates + the cliff gap and share of that result's simulation run, ``payload``, + and records them on the result (``stage12_runtime.cliff_results``). """ runtime = init_process_observability( service_name="policyengine-stage12-us-worker", @@ -461,7 +465,9 @@ def run_single_simulation_uk( A ``context`` naming a simulation run claims that run under this call's Modal call ID; a simulation started on its own, without an output plan, - is planned here. + is planned here. A ``context`` naming a cliff result instead calculates + the cliff gap and share of that result's simulation run, ``payload``, + and records them on the result (``stage12_runtime.cliff_results``). """ runtime = init_process_observability( service_name="policyengine-stage12-uk-worker", From e37c443165a27f98cd99d6d1dd4add3c7d02909d Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:33:41 +0400 Subject: [PATCH 09/24] A context naming no cliff result keeps its JSON (failing) Stage12InvocationContext is strict, so a v2 application built before cliff_result_id refuses a context carrying the field, even as null. The simulation service serializes contexts with model_dump(mode="json"), so until the new application is the default, every report run, simulation run and budget-window year it starts would fail. Review round 1, bug 1. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_stage12_report_run_ids.py | 23 +++++++++++++++---- .../tests/test_economy_calculation_inputs.py | 23 +++++++++++++++++++ 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/libs/policyengine-simulation-contract/tests/test_stage12_report_run_ids.py b/libs/policyengine-simulation-contract/tests/test_stage12_report_run_ids.py index 70568bd1f..66ff08c18 100644 --- a/libs/policyengine-simulation-contract/tests/test_stage12_report_run_ids.py +++ b/libs/policyengine-simulation-contract/tests/test_stage12_report_run_ids.py @@ -161,10 +161,25 @@ def test_a_cliff_workers_context_names_the_cliff_result_its_call_claims() -> Non assert context.records_runs restored = Stage12InvocationContext.model_validate(context.model_dump(mode="json")) assert restored.cliff_result_id == CLIFF_RESULT_ID - # A context naming no cliff result keeps its JSON, for the deployed workers. - assert "cliff_result_id" not in Stage12InvocationContext( - **_context_fields(simulation_run_id=str(BASELINE_RUN_ID)) - ).model_dump(mode="json", exclude_none=True) + + +@pytest.mark.parametrize( + "named_run", + [ + {}, + {"report_run": _run_ids().model_dump(mode="json")}, + {"simulation_run_id": str(BASELINE_RUN_ID)}, + ], + ids=["comparison", "report-run", "simulation-run"], +) +def test_a_context_naming_no_cliff_result_keeps_its_json(named_run) -> None: + """Workers deployed before the field refuse it, even as null (extra fields).""" + context = Stage12InvocationContext(**_context_fields(**named_run)) + + serialized = context.model_dump(mode="json") + + assert "cliff_result_id" not in serialized + assert Stage12InvocationContext.model_validate(serialized) == context @pytest.mark.parametrize( diff --git a/projects/policyengine-simulation-entry/tests/test_economy_calculation_inputs.py b/projects/policyengine-simulation-entry/tests/test_economy_calculation_inputs.py index 722ee115c..188c8a8f6 100644 --- a/projects/policyengine-simulation-entry/tests/test_economy_calculation_inputs.py +++ b/projects/policyengine-simulation-entry/tests/test_economy_calculation_inputs.py @@ -129,6 +129,29 @@ def test_a_report_run_is_named_by_its_run_ids_with_no_evaluation_record() -> Non assert context.retention_expires_at - context.created_at == timedelta(days=30) +def test_the_contexts_the_service_sends_name_no_cliff_result() -> None: + """A v2 application deployed before cliff results refuses the field, even null.""" + worker = _worker() + report = report_run_calculation( + worker, + _scope(), + ORIGIN, + run_ids=RUN_IDS, + baseline_policy={}, + reform_policy=REFORM, + ) + standalone = simulation_run_calculation( + worker, + _scope(options={}), + ORIGIN, + simulation_run_id=BASELINE_RUN_ID, + policy=REFORM, + ) + + assert "cliff_result_id" not in report.context + assert "cliff_result_id" not in standalone.context + + def test_a_standalone_run_is_named_in_its_functions_context() -> None: worker = _worker() From dbda6e214074d7c42eae2ceff5bf235bc492293a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:34:35 +0400 Subject: [PATCH 10/24] Leave an unset cliff_result_id out of the call context's JSON The field is excluded when None (exclude_if), as the manifest's optional local_area_reports is, so a context naming no cliff result serializes as before and the v2 applications deployed before it accept the contexts the simulation service sends. Co-Authored-By: Claude Opus 5.5 --- .../stage12_execution/comparison_runs.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/stage12_execution/comparison_runs.py b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/stage12_execution/comparison_runs.py index 7d2530aa4..3dd5fc7ed 100644 --- a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/stage12_execution/comparison_runs.py +++ b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/stage12_execution/comparison_runs.py @@ -102,7 +102,11 @@ class Stage12InvocationContext(StrictContractModel): # Comparison runs leave all three unset and record only the Stage 12 tables. report_run: ReportRunIds | None = None simulation_run_id: UUID | None = None - cliff_result_id: UUID | None = None + # Left out of the JSON when unset: a worker deployed before the field + # refuses it, even as null, so every other call keeps its JSON. + cliff_result_id: UUID | None = Field( + default=None, exclude_if=lambda value: value is None + ) @model_validator(mode="after") def validate_retention(self) -> Stage12InvocationContext: From 64fbba22c7d863ef9de5ebd060fc239e71643021 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:36:03 +0400 Subject: [PATCH 11/24] A failed read during a split cliff calculation is logged (failing) As for a simulation run, a cliff calculation split into region groups checks between polls whether its cliff result was taken from its call; a read that fails is treated as not taken, and must leave a warning. Review round 1 also asked for cover of a call without its Modal call ID, a payload for another country's worker, the totals against .py's own calculate_cliff_impact, and the cliff result's shared model in RUN_TABLES; those tests pass already. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_shared_table_models.py | 6 ++ .../tests/test_stage12_cliff_results.py | 86 ++++++++++++++++++- 2 files changed, 91 insertions(+), 1 deletion(-) diff --git a/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py b/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py index d696f2865..434e3e04e 100644 --- a/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py +++ b/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py @@ -30,10 +30,14 @@ result_metadata, service_metadata, ) +from policyengine_simulation_persistence import ( + cliff_impact_tables as shared_cliff_tables, +) from policyengine_simulation_persistence import tables as shared_tables from policyengine_stage12_persistence import ( DistrictCodes, budget_window_store, + cliff_result_store, household_report_run_store, report_run_result, report_run_store, @@ -61,6 +65,8 @@ def test_the_run_stores_write_the_shared_run_table_models() -> None: assert RUN_MODELS[RunKind.REPORT_RUN] is shared_tables.ReportRun assert RUN_MODELS[RunKind.SIMULATION_RUN] is shared_tables.SimulationRun assert RUN_MODELS[RunKind.HOUSEHOLD_JOB] is shared_tables.HouseholdJob + assert RUN_MODELS[RunKind.CLIFF_RESULT] is shared_cliff_tables.CliffImpact + assert cliff_result_store.CliffImpact is shared_cliff_tables.CliffImpact assert report_run_store.BudgetWindowRun is shared_tables.BudgetWindowRun diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_cliff_results.py b/projects/policyengine-simulation-executor/tests/test_stage12_cliff_results.py index f8e5c7eb9..5959d4044 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_cliff_results.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_cliff_results.py @@ -9,8 +9,13 @@ from __future__ import annotations +import logging +from types import SimpleNamespace + import pandas as pd import pytest +from microdf import MicroDataFrame +from policyengine.outputs.cliff_impact import calculate_cliff_impact from policyengine_simulation_contract.stage12_execution import ( EntityOutputPlan, GeographySelection, @@ -27,6 +32,9 @@ from policyengine_simulation_executor.stage12_runtime import ( cliff_results as cliff_results_module, ) +from policyengine_simulation_executor.stage12_runtime.cliff_results import ( + cliff_impact_from_frames, +) from policyengine_stage12_persistence import RunStatus from stage12_fake_cliff_results import ( CLIFF_RESULT_ID, @@ -120,7 +128,9 @@ def _run_cliff_worker( return run_single_simulation( simulation.model_dump(mode="json"), (context or cliff_result_context()).model_dump(mode="json"), - required_country=simulation.geography.country, + required_country=dependencies.pop( + "required_country", simulation.geography.country + ), modal_call_id=dependencies.pop("modal_call_id", WORKER_CALL_ID), dependencies=SimulationWorkerDependencies( cliff_results=cliff_results, **dependencies @@ -373,3 +383,77 @@ def test_a_dataset_with_no_adults_fails_the_cliff_result() -> None: result = cliff_results.cliff_results[CLIFF_RESULT_ID] assert result.status is RunStatus.FAILED assert result.error_message.startswith("cliff_share_undefined: ") + + +def test_a_call_without_its_modal_call_id_claims_nothing() -> None: + cliff_results = FakeCliffResults() + + with pytest.raises(ValueError, match="needs its Modal call ID"): + _run_cliff_worker(cliff_results, modal_call_id=None) + + assert cliff_results.cliff_results[CLIFF_RESULT_ID].writes == [] + + +def test_a_payload_for_another_countrys_worker_fails_the_claimed_cliff_result() -> None: + cliff_results = FakeCliffResults() + calculator = RecordingCalculator() + + with pytest.raises(RuntimeError, match="cliff calculation failed"): + _run_cliff_worker(cliff_results, calculator=calculator, required_country="uk") + + assert calculator.simulations == [] + result = cliff_results.cliff_results[CLIFF_RESULT_ID] + assert result.writes == ["claim_cliff_result", "fail_cliff_result"] + assert result.error_message.startswith("cliff_calculation_failed: ") + + +class PersonVariables: + """Stands in for a country model version: every cliff variable is a person's.""" + + def get_variable(self, _name): + return SimpleNamespace(entity="person") + + +def test_the_cliff_gap_and_share_are_pys_own_from_the_same_rows() -> None: + frames = cliff_frames() + calculated = SimpleNamespace( + id="cliff-test", + tax_benefit_model_version=PersonVariables(), + output_dataset=SimpleNamespace( + data=SimpleNamespace( + person=MicroDataFrame(frames["person"], weights="person_weight") + ) + ), + ) + + impact = cliff_impact_from_frames(frames, cliff_plan()) + + expected = calculate_cliff_impact(calculated, calculated).baseline + assert (impact.cliff_gap, impact.cliff_share) == ( + pytest.approx(expected.cliff_gap), + pytest.approx(expected.cliff_share), + ) + + +def test_a_failed_read_during_a_split_calculation_is_logged_and_ignored( + caplog, +) -> None: + cliff_results = FakeCliffResults() + readable = cliff_results.get_cliff_result + + def unreadable(_cliff_result_id): + msg = "the session pooler has no free connections" + raise ConnectionError(msg) + + def segmented_calculator(_simulation, cancellation_requested): + cliff_results.get_cliff_result = unreadable + taken = cancellation_requested() + cliff_results.get_cliff_result = readable + assert taken is False + return SimulationCalculation(frames=cliff_frames()) + + with caplog.at_level(logging.WARNING): + _run_cliff_worker(cliff_results, segmented_calculator=segmented_calculator) + + assert cliff_results.cliff_results[CLIFF_RESULT_ID].status is (RunStatus.SUCCEEDED) + assert f"Could not read cliff result {CLIFF_RESULT_ID}" in caplog.text From c225773c9aac93323362d307be38e21b78624066 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:37:36 +0400 Subject: [PATCH 12/24] Log a failed cliff result read during a split calculation The check whether the cliff result was taken from the call logs a warning when its read fails, as the simulation-run worker's check does, before treating it as not taken. Co-Authored-By: Claude Opus 5.5 --- .../stage12_runtime/cliff_results.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/cliff_results.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/cliff_results.py index 65190f1dd..d0717ef27 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/cliff_results.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/cliff_results.py @@ -23,6 +23,7 @@ from __future__ import annotations +import logging import math from dataclasses import dataclass from datetime import UTC, datetime @@ -83,6 +84,8 @@ from .failures import Stage12FailureDetail from .simulation import SimulationCalculation, SimulationWorkerDependencies +logger = logging.getLogger(__name__) + # The payload is not the simulation of the cliff result's run, on its year # and versions. CLIFF_RESULT_INPUT_MISMATCH = "cliff_result_input_mismatch" @@ -478,11 +481,16 @@ def _taken_from_call(recorder: CliffResultRecorder) -> bool: """Return whether the cliff result is no longer running under this call. A segmented calculation checks it between polls. A failed read is - treated as not taken, so the calculation goes on. + logged and treated as not taken, so the calculation goes on. """ try: return not recorder.still_held() - except Exception: # noqa: BLE001 # a failed read must not stop the calculation + except Exception as error: # noqa: BLE001 # a failed read must not stop the calculation + logger.warning( + "Could not read cliff result %s (%s)", + recorder.cliff_result_id, + type(error).__name__, + ) return False From ea9032ee00b5081dc862481da043e06b63c71fb4 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:37:36 +0400 Subject: [PATCH 13/24] Name cliff results wherever the docs list the sweep's and roles' tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stuck-run sweep's application, store and summary, the executor README's deployment step, the database-access skill and the v2 app's secret comment now include simulation runs' cliff results. The Stage 12 foundation document says nothing starts a cliff calculation yet, how C1 · 39 is to start one, and that API v1's cliff_impacts revision must be applied in an environment before this release deploys there. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/skills/database-access.md | 13 ++++++------ .../stage-12-modal-worker-foundation.md | 17 +++++++++------ .../report_run_store.py | 2 +- .../README.md | 3 ++- .../src/modal/run_sweep_app.py | 21 ++++++++++--------- .../src/modal/v2_app.py | 6 +++--- .../stage12_runtime/run_sweep.py | 2 +- 7 files changed, 36 insertions(+), 28 deletions(-) diff --git a/docs/engineering/skills/database-access.md b/docs/engineering/skills/database-access.md index 25f9ce793..d283300a6 100644 --- a/docs/engineering/skills/database-access.md +++ b/docs/engineering/skills/database-access.md @@ -170,8 +170,8 @@ and this repository has no migration chain of its own - the Stage 12 comparison tables, whose live schema deployment validation compares with the Stage 12 library's SQLAlchemy mappings. - the tables the simulation service writes (`simulations`, `reports`, - `simulation_runs`, `report_runs`, `budget_window_runs` and - `household_jobs`), modelled in `libs/policyengine-simulation-persistence`. + `simulation_runs`, `report_runs`, `budget_window_runs`, `household_jobs` + and `cliff_impacts`), modelled in `libs/policyengine-simulation-persistence`. The service connects as its own role, which policyengine-api's privilege manifest grants. - the policyengine-api tables a calculation is read from (`households`, @@ -182,10 +182,11 @@ and this repository has no migration chain of its own models of the tables it writes. - the run tables the v2 Modal coordinator, workers, household functions and stuck-run sweep write as the Modal worker role: `report_runs`, - `simulation_runs`, `budget_window_runs`, the result tables and - `household_jobs`. They use the same models, in - `libs/policyengine-simulation-persistence` (`tables.py` and - `result_tables.py`, which `sim.persistence.models-match-public-api` + `simulation_runs`, `budget_window_runs`, the result tables, + `household_jobs` and `cliff_impacts`. They use the same models, in + `libs/policyengine-simulation-persistence` (`tables.py`, + `cliff_impact_tables.py` and `result_tables.py`, which + `sim.persistence.models-match-public-api` compares with the Public API's tables), through the stores in `libs/policyengine-stage12-persistence`. The v1 `run_simulation` code imports neither library. diff --git a/docs/migration/stage-12-modal-worker-foundation.md b/docs/migration/stage-12-modal-worker-foundation.md index c38752cc2..9b1d79872 100644 --- a/docs/migration/stage-12-modal-worker-foundation.md +++ b/docs/migration/stage-12-modal-worker-foundation.md @@ -370,10 +370,12 @@ holds, or one that failed, it returns `{"deduplicated": true, **A simulation run's cliff result** (C1 · 24, answer 4b). Each simulation run can have one cliff result, its cliff gap and cliff share, which every report naming the run reads: a row of the Public API's `cliff_impacts`, one per -calculation. The simulation service inserts it pending, starts the country's -simulation worker (`run_single_simulation_us` or `run_single_simulation_uk`) -with the run's own simulation input and a context naming the row -(`Stage12InvocationContext.cliff_result_id`), and records the call. The worker +calculation. Nothing starts one yet. The simulation service is to insert it +pending, start the country's simulation worker (`run_single_simulation_us` or +`run_single_simulation_uk`) with the run's own simulation input, built as a +standalone run's, and a context naming the row +(`Stage12InvocationContext.cliff_result_id`, left out of a context that names +none, which older workers refuse), and record the call (C1 · 39). The worker claims the row as a run is claimed, and refuses, failing it with `cliff_result_input_mismatch`, an input that is not the simulation of the row's run on that run's year and versions; the calculation then checks those @@ -396,7 +398,10 @@ it returns its values again when Modal restarts it. A failure fails the row with its redacted cause; the stuck-run sweep fails one whose call ended without an outcome, and a failed row never counts, so the next request starts another calculation. The worker writes nothing to the simulation run, its -output file or any report run. +output file or any report run. The Public API's revision that adds +`cliff_impacts` must be applied in an environment before this release deploys +there: the stuck-run sweep lists the table on every pass, and the service's +`/ready` privilege check names it. **What a simulation run stores** (C1 · 24, answer 4). A run's output file depends on its simulation alone, never on a report naming it @@ -407,7 +412,7 @@ policy turns a response on; in the US, the two inputs .py's labour-supply output reads from either side (`self_employment_income`, `weekly_hours_worked`); and never the cliff columns, so a US national run keeps its region-group split. Cliffs are a calculation of each simulation run of -their own (below). A report run asking for them (`include_cliffs: true`) still +their own (above). A report run asking for them (`include_cliffs: true`) still fails before any worker starts, with `cliff_impacts_unavailable`, and touches none of its simulation runs, and its completion never writes `report_runs`' four cliff columns. The diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py index d78aca0fc..bfc1deb54 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py @@ -87,7 +87,7 @@ class RunKind(StrEnum): @dataclass(frozen=True) class ActiveRun: - """A pending or running run or household job, and its recorded Modal call. + """A pending or running run, household job or cliff result, and its Modal call. ``modal_call_id`` is None for a run with no call recorded. ``budget_window_run_id`` is the budget-window run a report run is a year diff --git a/projects/policyengine-simulation-executor/README.md b/projects/policyengine-simulation-executor/README.md index b4c1669ea..6e2e21a41 100644 --- a/projects/policyengine-simulation-executor/README.md +++ b/projects/policyengine-simulation-executor/README.md @@ -154,7 +154,8 @@ with its own mechanism, separate from the v1 routing jobs above: while the stored manifest is still what it wrote (`--restore-plan`). In a deployment from `main`, the same job then deploys the unversioned stuck-run sweep (`src/modal/run_sweep_app.py`), which fails the report - runs, simulation runs and household jobs nothing will finish, and whose + runs, simulation runs, household jobs and simulation runs' cliff results + nothing will finish, and whose schedule starts as soon as it deploys; a failed sweep deploy, including a failed build of its image, fails the job and so puts the previous default back. A deployment from another branch leaves the sweep as it is. diff --git a/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py b/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py index 04381c48c..e5123b25e 100644 --- a/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py +++ b/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py @@ -1,18 +1,19 @@ """A scheduled Modal function that fails runs and household jobs nothing will finish. -Every ten minutes it lists the Public API's report runs, simulation runs and -household jobs that have been pending or running for at least ten minutes, -counted from a run's first claim or, while it is pending, from its creation -(a budget window's year from when its call was recorded), and from a -household job's creation. It asks Modal for the status of each -recorded call and marks a run or job failed once its call has ended, and -fails a pending one that has no call recorded, because nothing will run it. +Every ten minutes it lists the Public API's report runs, simulation runs, +household jobs and simulation runs' cliff results that have been pending or +running for at least ten minutes, counted from a run's or cliff result's +first claim or, while it is pending, from its creation (a budget window's +year from when its call was recorded), and from a household job's creation. +It asks Modal for the status of each recorded call and marks a run or job +failed once its call has ended, and fails a pending one that has no call +recorded, because nothing will run it. It also fails any still unfinished 24 hours after it was created or started, longer than any Modal call of it can run, whatever Modal reports. A failed year of a budget window fails its window, and a window no year has advanced for ten minutes is finished from its years, or failed as stalled. -Its result counts runs and household jobs together, and the windows it -finished. It connects as the Modal worker +Its result counts runs, household jobs and cliff results together, and the +windows it finished. It connects as the Modal worker role, from the same Modal secret the v2 coordinator, workers and household functions use. This application is not versioned with the .py bundle: one copy runs per Modal environment. @@ -83,7 +84,7 @@ secrets=[modal_worker_database_secret], ) def sweep_stuck_runs() -> dict: - """Fail the runs, household jobs and windows nothing will finish; count them.""" + """Fail the runs, household jobs, cliff results and windows nothing will finish.""" from policyengine_simulation_executor.stage12_runtime import ( # noqa: PLC0415 # Modal function body: loaded in the container, not at deploy time report_run_access, run_sweep, diff --git a/projects/policyengine-simulation-executor/src/modal/v2_app.py b/projects/policyengine-simulation-executor/src/modal/v2_app.py index a134a655f..a2f554324 100644 --- a/projects/policyengine-simulation-executor/src/modal/v2_app.py +++ b/projects/policyengine-simulation-executor/src/modal/v2_app.py @@ -157,9 +157,9 @@ def _external_assertions(environment: dict[str, str]) -> dict[str, str]: hf_secret, comparison_runtime_secret, ] -# Only the functions that record a report run's, simulation run's or -# household job's status and results hold the Modal worker role's database -# credentials. +# Only the functions that record a report run's, simulation run's, cliff +# result's or household job's status and results hold the Modal worker role's +# database credentials. modal_worker_database_secret = modal.Secret.from_name( MODAL_WORKER_DATABASE_SECRET_NAME, required_keys=[MODAL_WORKER_DATABASE_URL_ENV], diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/run_sweep.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/run_sweep.py index cbb58ceba..436fb31a9 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/run_sweep.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/run_sweep.py @@ -176,7 +176,7 @@ def settle_idle_window( @dataclass(frozen=True) class SweepSummary: - """What the sweep did with the runs and household jobs it listed, together. + """What the sweep did with the runs, household jobs and cliff results it listed. ``checked`` runs were judged after the minimum age: ``failed`` because their call ended, ``failed_without_call`` because they stayed pending with From beeb2d21f903c0c23810d866efeca9d6741eaa02 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:08:00 +0400 Subject: [PATCH 14/24] Expect the sweep's fourth kind and cliff_impacts' status column Three sweep tests counted three kinds of run per pass, and the enum test seven enum columns among the service's tables; the sweep now also lists cliff results, and cliff_impacts.status is an eighth. Review round 2, bug 1. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_tables.py | 3 ++- .../tests/test_stage12_run_sweep.py | 15 ++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/libs/policyengine-simulation-persistence/tests/test_tables.py b/libs/policyengine-simulation-persistence/tests/test_tables.py index a0ae4d464..09b73d8e7 100644 --- a/libs/policyengine-simulation-persistence/tests/test_tables.py +++ b/libs/policyengine-simulation-persistence/tests/test_tables.py @@ -133,7 +133,8 @@ def test_every_enum_type_names_the_public_schema() -> None: if isinstance(column.type, Enum) ] - assert len(enum_columns) == 7 + # Including cliff_impacts.status, which reuses the simulation runs' type. + assert len(enum_columns) == 8 for column in enum_columns: assert column.type.schema == "public", f"{column.table.name}.{column.name}" # The helper every enum column uses, before any table adopts its type. diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep.py b/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep.py index 28c4aa1f3..c66dcbbf2 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_run_sweep.py @@ -149,8 +149,8 @@ def test_the_sweep_lists_runs_older_than_its_minimum_age() -> None: minimum_age=timedelta(minutes=15), ) - # Report runs, simulation runs and household jobs. - assert store.cutoffs == [NOW - timedelta(minutes=15)] * 3 + # Report runs, simulation runs, household jobs and cliff results. + assert store.cutoffs == [NOW - timedelta(minutes=15)] * 4 def test_report_runs_are_swept_before_simulation_runs_are_listed() -> None: @@ -440,9 +440,9 @@ def test_the_age_limit_fails_a_simulation_run_whatever_report_runs_name_it() -> sweep_stuck_runs(store, call_state=lambda _: CallState.ENDED, now=NOW) assert [run for run, _, _ in store.failed] == [expired, judged] - # The age limit's three listings and its failure ignore report runs; the - # minimum age's three listings and failure don't. - assert store.listed_regardless == [True] * 3 + [False] * 3 + # The age limit's four listings and its failure ignore report runs; the + # minimum age's four listings and failure don't. + assert store.listed_regardless == [True] * 4 + [False] * 4 assert store.failed_regardless == [True, False] @@ -474,8 +474,8 @@ def call_state(call_id): maximum_age=timedelta(hours=24), ) - assert store.expired_cutoffs == [NOW - timedelta(hours=24)] * 3 - assert store.cutoffs == [NOW - timedelta(minutes=10)] * 3 + assert store.expired_cutoffs == [NOW - timedelta(hours=24)] * 4 + assert store.cutoffs == [NOW - timedelta(minutes=10)] * 4 # Every kind's age check comes before any lookup. assert asked == [ ( @@ -484,6 +484,7 @@ def call_state(call_id): "list-expired:report_run", "list-expired:simulation_run", "list-expired:household_job", + "list-expired:cliff_result", "list:report_run", ], ) From e9c9f508975c9c8b385b5dd385eff76d3d59e0ca Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:09:24 +0400 Subject: [PATCH 15/24] Cover year and service contexts' cliff rows; tidy sweep wording A budget-window year's context leaves out an unset cliff_result_id, and the service role inserts a cliff_impacts row in its writes test. The sweep's and run store's summaries, and the simulation workers' container limit comment, now name cliff results. Review round 2's code-quality findings. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_stage12_budget_window_years.py | 15 +++++++++++++++ .../tests/test_service_writes.py | 14 +++++++++++++- .../report_run_store.py | 3 ++- .../src/modal/run_sweep_app.py | 2 +- .../src/modal/v2_app.py | 8 ++++++-- .../stage12_runtime/run_sweep.py | 4 ++-- 6 files changed, 39 insertions(+), 7 deletions(-) diff --git a/libs/policyengine-simulation-contract/tests/test_stage12_budget_window_years.py b/libs/policyengine-simulation-contract/tests/test_stage12_budget_window_years.py index 3797fabbd..fded1c590 100644 --- a/libs/policyengine-simulation-contract/tests/test_stage12_budget_window_years.py +++ b/libs/policyengine-simulation-contract/tests/test_stage12_budget_window_years.py @@ -159,6 +159,21 @@ def test_a_later_year_is_calculated_on_its_own_year_and_runs() -> None: assert later_context.retention_expires_at == LATER_CREATED_AT + timedelta(days=30) +def test_a_later_years_context_names_no_cliff_result_in_its_json() -> None: + """A v2 application deployed before cliff results refuses the field, even null.""" + report, context = _first_year() + + _, later_context = year_calculation( + report, + context, + year=2029, + run_ids=LATER_YEAR_RUNS, + created_at=LATER_CREATED_AT, + ) + + assert "cliff_result_id" not in later_context.model_dump(mode="json") + + def test_nothing_else_differs_between_the_years() -> None: report, context = _first_year() diff --git a/libs/policyengine-simulation-persistence/tests/test_service_writes.py b/libs/policyengine-simulation-persistence/tests/test_service_writes.py index 8bf76b69e..7a19fb586 100644 --- a/libs/policyengine-simulation-persistence/tests/test_service_writes.py +++ b/libs/policyengine-simulation-persistence/tests/test_service_writes.py @@ -9,6 +9,8 @@ from __future__ import annotations +from uuid import uuid4 + import pytest from conftest import ( act_as_the_service, @@ -18,6 +20,7 @@ ) from policyengine_simulation_persistence import ( BudgetWindowRun, + CliffImpact, HouseholdJob, Report, ReportRun, @@ -26,7 +29,7 @@ missing_table_privileges, ) from service_rows import insert_public_api_rows, insert_service_rows -from sqlalchemy import func, select +from sqlalchemy import func, insert, select pytestmark = pytest.mark.public_api_schema @@ -45,6 +48,14 @@ def test_the_service_role_creates_and_reads_a_row_in_every_table_it_writes( public_api_rows = insert_public_api_rows(connection) act_as_the_service(connection) service_rows = insert_service_rows(connection, public_api_rows) + # A cliff calculation of a simulation run, which the service starts. + connection.execute( + insert(CliffImpact.__table__).values( + id=uuid4(), + simulation_run_id=service_rows.baseline_run_id, + status="pending", + ) + ) compared_runs = connection.execute( select( @@ -65,6 +76,7 @@ def test_the_service_role_creates_and_reads_a_row_in_every_table_it_writes( BudgetWindowRun.__table__, ReportRun.__table__, HouseholdJob.__table__, + CliffImpact.__table__, ): assert ( connection.execute(select(func.count()).select_from(table)).scalar_one() diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py index bfc1deb54..3e4b6d601 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py @@ -136,7 +136,8 @@ class ReportRunStore: """Status writes to the Public API's run tables and household jobs. It also reads a simulation run's state, for a report run that names one - that already succeeded. + that already succeeded. The sweep's listing and failure of stuck rows + serve every ``RunKind``, simulation runs' cliff results included. """ def __init__( diff --git a/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py b/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py index e5123b25e..ee858f094 100644 --- a/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py +++ b/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py @@ -1,4 +1,4 @@ -"""A scheduled Modal function that fails runs and household jobs nothing will finish. +"""A scheduled Modal function that fails the runs nothing will finish. Every ten minutes it lists the Public API's report runs, simulation runs, household jobs and simulation runs' cliff results that have been pending or diff --git a/projects/policyengine-simulation-executor/src/modal/v2_app.py b/projects/policyengine-simulation-executor/src/modal/v2_app.py index a2f554324..689f356f5 100644 --- a/projects/policyengine-simulation-executor/src/modal/v2_app.py +++ b/projects/policyengine-simulation-executor/src/modal/v2_app.py @@ -376,7 +376,9 @@ def run_single_simulation_segment_us( memory=32768, timeout=US_SIMULATION_TIMEOUT_SECONDS, retries=0, - # Both simulations of every coordinator call that may run at once. + # Both simulations of every coordinator call that may run at once. The + # simulation runs and cliff calculations the simulation service starts + # directly share the same containers. max_containers=50, secrets=report_run_writer_secrets, ) @@ -451,7 +453,9 @@ def segmented_calculator( memory=32768, timeout=UK_SIMULATION_TIMEOUT_SECONDS, retries=0, - # Both simulations of every coordinator call that may run at once. + # Both simulations of every coordinator call that may run at once. The + # simulation runs and cliff calculations the simulation service starts + # directly share the same containers. max_containers=50, secrets=report_run_writer_secrets, ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/run_sweep.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/run_sweep.py index 436fb31a9..e25da9193 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/run_sweep.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/run_sweep.py @@ -1,4 +1,4 @@ -"""The sweep that fails runs and household jobs nothing will finish. +"""The sweep that fails runs, household jobs and cliff results nothing will finish. A coordinator or worker records the Modal call running each run it claims, and the simulation service is to record the call it starts on a pending @@ -120,7 +120,7 @@ def list_active_runs( active_since_before: datetime, regardless_of_report_runs: bool = False, ) -> tuple[ActiveRun, ...]: - """Return the pending and running runs, or household jobs, older than a time. + """Return the pending and running runs of ``kind`` older than a time. A simulation run an unfinished report run names is left out, unless ``regardless_of_report_runs`` and it records a call. From c57a1f72e0a1ead1ba47ffab40b8e786fdb148bb Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:38:05 +0400 Subject: [PATCH 16/24] Test that merged region groups give the national cliffs; fix two texts Twenty region-group results merged by the real segment code give the same cliff gap and share as the unsplit rows. The foundation document's segmentation section now names a simulation run's own cliff calculation as split, and the Modal worker credentials' summary names cliffs. Review round 3's code-quality findings. Co-Authored-By: Claude Opus 5.5 --- .../stage-12-modal-worker-foundation.md | 9 ++-- .../stage12_runtime/report_run_access.py | 2 +- .../tests/test_stage12_cliff_results.py | 53 +++++++++++++++++++ 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/docs/migration/stage-12-modal-worker-foundation.md b/docs/migration/stage-12-modal-worker-foundation.md index 9b1d79872..e78209416 100644 --- a/docs/migration/stage-12-modal-worker-foundation.md +++ b/docs/migration/stage-12-modal-worker-foundation.md @@ -652,9 +652,12 @@ Segmentation applies by default when both simulation inputs describe a US national society-wide calculation and the shared output plan does not require cliff analysis or labor-supply responses. `segmented=false` selects the single-container calculation explicitly. UK calculations, subnational US -calculations, cliff analysis, and labor-supply-response calculations also use -the single-container path. `segmented=true` is accepted, but it does not -override those numerical eligibility conditions. A US national simulation run +calculations, a report's cliff analysis, and labor-supply-response +calculations also use the single-container path. A simulation run's own cliff +calculation is the exception: it is split whenever its run's own calculation +is (`should_segment_cliff_calculation`), and its cliff gap and share are taken +from the merged rows. `segmented=true` is accepted, but it does not override +those numerical eligibility conditions. A US national simulation run started on its own (role `standalone`) is segmented by the same rule. It never uses the precomputed baseline files, which serve only a report run's current-law baseline, planned as its simulation run stores its output, so it diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/report_run_access.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/report_run_access.py index 4e5167108..4d9468734 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/report_run_access.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/report_run_access.py @@ -1,4 +1,4 @@ -"""The Modal worker role's database credentials, which runs and household jobs use. +"""The Modal worker role's database credentials, for runs, cliffs and household jobs. An operator creates the Modal secret in each Modal environment. The v2 Modal application attaches it to the coordinator, which records the report run diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_cliff_results.py b/projects/policyengine-simulation-executor/tests/test_stage12_cliff_results.py index 5959d4044..ddb7c52d1 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_cliff_results.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_cliff_results.py @@ -10,6 +10,7 @@ from __future__ import annotations import logging +from hashlib import sha256 from types import SimpleNamespace import pandas as pd @@ -24,9 +25,15 @@ SimulationRole, Stage12OutputPlan, ) +from policyengine_simulation_executor.stage12_artifacts import ( + serialize_simulation_frames, +) from policyengine_simulation_executor.stage12_runtime import ( SimulationCalculation, SimulationWorkerDependencies, + Stage12SegmentResult, + build_segment_inputs, + merge_segment_results, run_single_simulation, ) from policyengine_simulation_executor.stage12_runtime import ( @@ -35,6 +42,9 @@ from policyengine_simulation_executor.stage12_runtime.cliff_results import ( cliff_impact_from_frames, ) +from policyengine_simulation_executor.stage12_runtime.output_planning import ( + plan_simulation_input, +) from policyengine_stage12_persistence import RunStatus from stage12_fake_cliff_results import ( CLIFF_RESULT_ID, @@ -457,3 +467,46 @@ def segmented_calculator(_simulation, cancellation_requested): assert cliff_results.cliff_results[CLIFF_RESULT_ID].status is (RunStatus.SUCCEEDED) assert f"Could not read cliff result {CLIFF_RESULT_ID}" in caplog.text + + +def test_merged_region_group_rows_give_the_cliffs_of_the_whole_country() -> None: + """The split is exact: the share is taken from the merged rows, not averaged.""" + people = 400 + country = pd.DataFrame( + { + "person_id": list(range(people)), + "person_weight": [0.5 + (person * 37) % 900 for person in range(people)], + "cliff_gap": [ + float((person * 53) % 3_000) if person % 5 == 0 else 0.0 + for person in range(people) + ], + "is_on_cliff": [person % 7 == 0 for person in range(people)], + "is_adult": [person % 10 < 7 for person in range(people)], + } + ) + plan = cliff_plan() + simulation = plan_simulation_input(cliff_simulation(), plan) + groups = [[f"state/group-{index}"] for index in range(20)] + segments = build_segment_inputs(simulation, groups=groups) + group_size = people // len(groups) + payloads = [] + for segment in segments: + start = segment.segment_index * group_size + rows = country.iloc[start : start + group_size].reset_index(drop=True) + parquet, _ = serialize_simulation_frames({"person": rows}) + payloads.append( + Stage12SegmentResult( + segment_index=segment.segment_index, + role=simulation.role, + region_codes=segment.region_codes, + parquet_payload=parquet, + payload_sha256=sha256(parquet).hexdigest(), + ).model_dump(mode="python") + ) + + merged = merge_segment_results(simulation, payloads, groups=groups) + + split = cliff_impact_from_frames(merged.frames, plan) + whole = cliff_impact_from_frames({"person": country}, plan) + assert split.cliff_gap == pytest.approx(whole.cliff_gap, rel=1e-12) + assert split.cliff_share == pytest.approx(whole.cliff_share, rel=1e-12) From 6fdd18cc3db4f26eb12e83b98a7017e87cf69ca8 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:46:45 +0400 Subject: [PATCH 17/24] Grant cliff_impacts in the missing-privileges rule's proof fixtures The fixtures list every privilege the simulation service needs, which now include SELECT and INSERT on cliff_impacts and UPDATE of its call-start columns. The bad grants still leave out the same two privileges. Co-Authored-By: Claude Opus 5.5 --- .../bad_grants.txt | 7 +++++++ .../good_grants.txt | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/rules/fixtures/sim.persistence.missing-privileges-reported/bad_grants.txt b/rules/fixtures/sim.persistence.missing-privileges-reported/bad_grants.txt index ef31953e1..a735a5ac7 100644 --- a/rules/fixtures/sim.persistence.missing-privileges-reported/bad_grants.txt +++ b/rules/fixtures/sim.persistence.missing-privileges-reported/bad_grants.txt @@ -12,6 +12,8 @@ INSERT budget_window_runs SELECT budget_window_runs INSERT household_jobs SELECT household_jobs +INSERT cliff_impacts +SELECT cliff_impacts SELECT policies SELECT households SELECT tax_benefit_models @@ -39,3 +41,8 @@ UPDATE household_jobs error_message UPDATE household_jobs modal_call_id UPDATE household_jobs status UPDATE household_jobs updated_at +UPDATE cliff_impacts completed_at +UPDATE cliff_impacts error_message +UPDATE cliff_impacts modal_call_id +UPDATE cliff_impacts status +UPDATE cliff_impacts updated_at diff --git a/rules/fixtures/sim.persistence.missing-privileges-reported/good_grants.txt b/rules/fixtures/sim.persistence.missing-privileges-reported/good_grants.txt index bba121dfb..afbe388ba 100644 --- a/rules/fixtures/sim.persistence.missing-privileges-reported/good_grants.txt +++ b/rules/fixtures/sim.persistence.missing-privileges-reported/good_grants.txt @@ -11,6 +11,8 @@ INSERT budget_window_runs SELECT budget_window_runs INSERT household_jobs SELECT household_jobs +INSERT cliff_impacts +SELECT cliff_impacts SELECT policies SELECT households SELECT tax_benefit_models @@ -39,3 +41,8 @@ UPDATE household_jobs error_message UPDATE household_jobs modal_call_id UPDATE household_jobs status UPDATE household_jobs updated_at +UPDATE cliff_impacts completed_at +UPDATE cliff_impacts error_message +UPDATE cliff_impacts modal_call_id +UPDATE cliff_impacts status +UPDATE cliff_impacts updated_at From 19889bd3e9a336823f0b63452b963300c6b63230 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 19 Nov 2026 01:43:23 +0400 Subject: [PATCH 18/24] Test CliffImpact with the service's mapped classes (failing) CliffImpact joins the mapped-class test's service classes, and the shared model test expects it in tables, beside the other classes the service creates rows in. The worker's cliff result store retries a refused connection, and concurrent starts leave its row with one Modal call. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_mapped_classes.py | 2 ++ .../tests/test_cliff_result_store.py | 26 +++++++++++++++++++ .../tests/test_shared_table_models.py | 7 ++--- .../test_stage12_runtime_store_retries.py | 9 +++++++ 4 files changed, 39 insertions(+), 5 deletions(-) diff --git a/libs/policyengine-simulation-persistence/tests/test_mapped_classes.py b/libs/policyengine-simulation-persistence/tests/test_mapped_classes.py index cfaf93b03..8441a2242 100644 --- a/libs/policyengine-simulation-persistence/tests/test_mapped_classes.py +++ b/libs/policyengine-simulation-persistence/tests/test_mapped_classes.py @@ -16,6 +16,7 @@ from policyengine_simulation_persistence import ( BudgetSummary, BudgetWindowRun, + CliffImpact, CongressionalDistrictImpact, ConstituencyImpact, Dataset, @@ -72,6 +73,7 @@ "budget_window_runs": BudgetWindowRun, "report_runs": ReportRun, "household_jobs": HouseholdJob, + "cliff_impacts": CliffImpact, } REFERENCED_CLASSES = { "datasets": referenced_tables.ReferencedDataset, diff --git a/libs/policyengine-stage12-persistence/tests/test_cliff_result_store.py b/libs/policyengine-stage12-persistence/tests/test_cliff_result_store.py index 4547b6063..ec8bcfd21 100644 --- a/libs/policyengine-stage12-persistence/tests/test_cliff_result_store.py +++ b/libs/policyengine-stage12-persistence/tests/test_cliff_result_store.py @@ -8,7 +8,9 @@ from __future__ import annotations +from concurrent.futures import ThreadPoolExecutor from datetime import timedelta +from threading import Barrier from uuid import uuid4 import pytest @@ -101,6 +103,30 @@ def test_a_cliff_result_is_claimed_once_and_resumed_by_the_same_call( assert row["started_at"] == NOW +def test_concurrent_starts_leave_the_cliff_result_with_one_call( + worker_sessions, statements, seeder, seeded_run +) -> None: + """Two starts calculate once, however close together they claim the row.""" + assert statements.sent == [] + cliff_result_id = _insert_cliff_result(seeder, seeded_run) + claimants = 6 + start_together = Barrier(claimants) + + def claim(claimant: int) -> tuple[str, RunWrite]: + claiming_store = CliffResultStore(session_factory=worker_sessions) + call = f"fc-claimant-{claimant}" + start_together.wait(timeout=10) + return call, _claim(claiming_store, cliff_result_id, call) + + with ThreadPoolExecutor(max_workers=claimants) as executor: + outcomes = list(executor.map(claim, range(claimants))) + + winners = [call for call, write in outcomes if write.changed] + assert len(winners) == 1 + assert _cliff_row(seeder, cliff_result_id)["modal_call_id"] == winners[0] + assert all(write.status is RunStatus.RUNNING for _call, write in outcomes) + + def test_a_pending_cliff_result_is_claimed_only_by_the_call_recorded_on_it( store, seeder, seeded_run ) -> None: diff --git a/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py b/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py index 434e3e04e..e30196335 100644 --- a/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py +++ b/libs/policyengine-stage12-persistence/tests/test_shared_table_models.py @@ -30,9 +30,6 @@ result_metadata, service_metadata, ) -from policyengine_simulation_persistence import ( - cliff_impact_tables as shared_cliff_tables, -) from policyengine_simulation_persistence import tables as shared_tables from policyengine_stage12_persistence import ( DistrictCodes, @@ -65,8 +62,8 @@ def test_the_run_stores_write_the_shared_run_table_models() -> None: assert RUN_MODELS[RunKind.REPORT_RUN] is shared_tables.ReportRun assert RUN_MODELS[RunKind.SIMULATION_RUN] is shared_tables.SimulationRun assert RUN_MODELS[RunKind.HOUSEHOLD_JOB] is shared_tables.HouseholdJob - assert RUN_MODELS[RunKind.CLIFF_RESULT] is shared_cliff_tables.CliffImpact - assert cliff_result_store.CliffImpact is shared_cliff_tables.CliffImpact + assert RUN_MODELS[RunKind.CLIFF_RESULT] is shared_tables.CliffImpact + assert cliff_result_store.CliffImpact is shared_tables.CliffImpact assert report_run_store.BudgetWindowRun is shared_tables.BudgetWindowRun diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_runtime_store_retries.py b/projects/policyengine-simulation-executor/tests/test_stage12_runtime_store_retries.py index 8fd7fd8c4..ee98c1886 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_runtime_store_retries.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_runtime_store_retries.py @@ -16,6 +16,7 @@ from policyengine_simulation_executor.stage12_runtime.report_run_access import ( MODAL_WORKER_DATABASE_URL_ENV, budget_window_store, + cliff_result_store, report_run_store, ) from policyengine_stage12_persistence import engine as persistence_engine_module @@ -29,6 +30,7 @@ ) REPORT_RUN_ID = UUID("00000000-0000-0000-0000-00000000000a") +CLIFF_RESULT_ID = UUID("00000000-0000-0000-0000-00000000000c") CLAIMED_AT = datetime(2026, 11, 4, tzinfo=UTC) @@ -176,10 +178,17 @@ def _claim_a_report_run(store) -> Any: ) +def _claim_a_cliff_result(store) -> Any: + return store.claim_cliff_result( + CLIFF_RESULT_ID, modal_call_id="fc-call", claimed_at=CLAIMED_AT + ) + + @pytest.mark.parametrize( ("build_store", "write"), [ pytest.param(report_run_store, _claim_a_report_run, id="report-runs"), + pytest.param(cliff_result_store, _claim_a_cliff_result, id="cliff-results"), pytest.param( household_report_recording.household_report_run_store, _claim_a_report_run, From a72ac62a5e223062141ffc9e257bc8df2fe42a04 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 19 Nov 2026 01:52:41 +0400 Subject: [PATCH 19/24] Map CliffImpact in tables, beside the service's other classes The class joins the tables the service creates rows in, so ServiceBase's module, the schema rule's model list and the docs describe it without a module of its own. The runtime's two calculated numbers are CliffTotals, so they aren't confused with the mapped row or .py's CliffImpact output. The run store's docstrings say its sweep also serves cliff results. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/skills/database-access.md | 5 +- .../README.md | 20 ++-- .../__init__.py | 2 +- .../cliff_impact_tables.py | 99 ------------------- .../tables.py | 57 +++++++++++ .../cliff_result_store.py | 2 +- .../report_run_store.py | 9 +- .../stage12_runtime/cliff_results.py | 28 +++--- .../tests/test_stage12_cliff_results.py | 8 +- 9 files changed, 94 insertions(+), 136 deletions(-) delete mode 100644 libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/cliff_impact_tables.py diff --git a/docs/engineering/skills/database-access.md b/docs/engineering/skills/database-access.md index d283300a6..978ba9a4d 100644 --- a/docs/engineering/skills/database-access.md +++ b/docs/engineering/skills/database-access.md @@ -184,9 +184,8 @@ and this repository has no migration chain of its own stuck-run sweep write as the Modal worker role: `report_runs`, `simulation_runs`, `budget_window_runs`, the result tables, `household_jobs` and `cliff_impacts`. They use the same models, in - `libs/policyengine-simulation-persistence` (`tables.py`, - `cliff_impact_tables.py` and `result_tables.py`, which - `sim.persistence.models-match-public-api` + `libs/policyengine-simulation-persistence` (`tables.py` and + `result_tables.py`, which `sim.persistence.models-match-public-api` compares with the Public API's tables), through the stores in `libs/policyengine-stage12-persistence`. The v1 `run_simulation` code imports neither library. diff --git a/libs/policyengine-simulation-persistence/README.md b/libs/policyengine-simulation-persistence/README.md index dcedad60e..eb64996b2 100644 --- a/libs/policyengine-simulation-persistence/README.md +++ b/libs/policyengine-simulation-persistence/README.md @@ -47,10 +47,11 @@ referenced row before adding the row that names it. ## Tables -`tables.py` models, as SQLAlchemy ORM mapped classes, the six tables the +`tables.py` models, as SQLAlchemy ORM mapped classes, the seven tables the service creates rows in: `Simulation`, `Report`, `SimulationRun`, `ReportRun`, -`BudgetWindowRun` and `HouseholdJob`, for `simulations`, `reports`, -`simulation_runs`, `report_runs`, `budget_window_runs` and `household_jobs`. +`BudgetWindowRun`, `HouseholdJob` and `CliffImpact`, for `simulations`, +`reports`, `simulation_runs`, `report_runs`, `budget_window_runs`, +`household_jobs` and `cliff_impacts`. They derive from `ServiceBase` in `orm_base.py`, and their tables are in `service_metadata`, which names the `public` schema so writes reach those tables whatever the role's `search_path`. @@ -81,13 +82,12 @@ household report with only a baseline). A report with a `budget_window_size` is calculated as a `budget_window_runs` row with one report run per year; each year's report run names that row and its year, and belongs to the same report. -`cliff_impact_tables.py` maps a seventh table the service creates rows in, on -`ServiceBase` too: `cliff_impacts` (`CliffImpact`), each cliff calculation of a -simulation run. The service inserts a row pending and records the Modal call -it starts; the Modal worker role claims it and records the run's cliff gap -and cliff share, or the failure. A run has at most one calculation that -hasn't failed (`uq_cliff_impacts_unfailed`), so identical requests join it, -and a failed one stays recorded. Deleting a run deletes its calculations. +`CliffImpact` is one cliff calculation of a simulation run. The service inserts +a row pending and records the Modal call it starts; the Modal worker role claims +it and records the run's cliff gap and cliff share, or the failure. A run has at +most one calculation that hasn't failed (`uq_cliff_impacts_unfailed`), so +identical requests join it, and a failed one stays recorded. Deleting a run +deletes its calculations. ## Result tables diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py index 14ae71304..0c0ca9823 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py @@ -12,7 +12,6 @@ WindowSide, WindowYearRuns, ) -from policyengine_simulation_persistence.cliff_impact_tables import CliffImpact from policyengine_simulation_persistence.definition_keys import ( KEY_VERSION, EconomyReportDefinition, @@ -204,6 +203,7 @@ LATEST_YEAR, MAX_BUDGET_WINDOW_YEARS, BudgetWindowRun, + CliffImpact, HouseholdJob, HouseholdJobStatus, Report, diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/cliff_impact_tables.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/cliff_impact_tables.py deleted file mode 100644 index b24d60191..000000000 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/cliff_impact_tables.py +++ /dev/null @@ -1,99 +0,0 @@ -"""The model of ``cliff_impacts``: each simulation run's cliff calculations. - -A simulation run can have one cliff result, its cliff gap and cliff share, -which every report naming the run reads. Each row is one calculation of it: -the simulation service inserts it pending and records the Modal call it -starts, and the Modal worker role claims it under that call and records the -two numbers or the failure. A run has at most one calculation that hasn't -failed, so identical requests join it; a failed one stays recorded, and a -later start inserts a new row. The row has no versions of its own: it is a -calculation of its run, on the run's versions. Deleting a run deletes its -calculations. - -The Public API's v2 migration chain creates the table (revision -``bc334f9877b7``); this class only describes it, column for column, with its -constraints and indexes under the same names. It is one of the service's -tables (``ServiceBase``), since the service creates its rows. -""" - -from __future__ import annotations - -from datetime import datetime # noqa: TC003 # SQLAlchemy reads Mapped annotations -from typing import ClassVar -from uuid import UUID # noqa: TC003 # SQLAlchemy reads Mapped annotations at run time - -from sqlalchemy import ( - CheckConstraint, - Float, - ForeignKeyConstraint, - Index, - PrimaryKeyConstraint, - String, - Uuid, - text, -) -from sqlalchemy.orm import Mapped, mapped_column - -from policyengine_simulation_persistence.orm_base import ( - ServiceBase, - enum_type, - identifier_column, - optional_timestamp_column, - server_timestamp_column, -) -from policyengine_simulation_persistence.tables import ( - MODAL_CALL_ID_LENGTH, - SimulationRun, - SimulationStatus, -) - - -class CliffImpact(ServiceBase): - """One cliff calculation of a simulation run, and its result once it succeeds.""" - - __tablename__ = "cliff_impacts" - __table_args__: ClassVar = ( - PrimaryKeyConstraint("id", name="pk_cliff_impacts"), - ForeignKeyConstraint( - ["simulation_run_id"], - [SimulationRun.id], - name="fk_cliff_impacts_simulation_run_id_simulation_runs", - ondelete="CASCADE", - ), - CheckConstraint( - "status NOT IN ('succeeded', 'failed') OR completed_at IS NOT NULL", - name="ck_cliff_impacts_finished_completion", - ), - CheckConstraint( - "(cliff_gap IS NULL) = (cliff_share IS NULL) AND " - "(status = 'succeeded') = (cliff_gap IS NOT NULL)", - name="ck_cliff_impacts_result_when_succeeded", - ), - Index("ix_cliff_impacts_simulation_run_id", "simulation_run_id"), - # For the stuck-run sweep, which judges a calculation by its age. - Index("ix_cliff_impacts_status_started_at", "status", "started_at"), - # One calculation per run that hasn't failed: find-or-create's conflict - # target, so identical requests join one calculation. - Index( - "uq_cliff_impacts_unfailed", - "simulation_run_id", - unique=True, - postgresql_where=text("status <> 'failed'"), - ), - ) - - id: Mapped[UUID] = identifier_column() - created_at: Mapped[datetime] = server_timestamp_column() - updated_at: Mapped[datetime] = server_timestamp_column() - simulation_run_id: Mapped[UUID] = mapped_column(Uuid) - # A cliff calculation has a simulation run's states, under its type. - status: Mapped[SimulationStatus] = mapped_column( - enum_type(SimulationStatus, "v2_simulation_status") - ) - modal_call_id: Mapped[str | None] = mapped_column(String(MODAL_CALL_ID_LENGTH)) - error_message: Mapped[str | None] = mapped_column(String) - started_at: Mapped[datetime | None] = optional_timestamp_column() - completed_at: Mapped[datetime | None] = optional_timestamp_column() - # Both set exactly when the calculation succeeded. - cliff_gap: Mapped[float | None] = mapped_column(Float) - cliff_share: Mapped[float | None] = mapped_column(Float) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/tables.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/tables.py index 31823e2ba..1c4fc22c0 100644 --- a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/tables.py +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/tables.py @@ -729,4 +729,61 @@ class HouseholdJob(ServiceBase): modal_call_id: Mapped[str | None] = mapped_column(String(MODAL_CALL_ID_LENGTH)) +class CliffImpact(ServiceBase): + """One cliff calculation of a simulation run, and its result once it succeeds. + + The service inserts it pending and records the Modal call it starts; the + Modal worker role claims it and records the two numbers or the failure. A + run has at most one that hasn't failed, so identical requests join it, and + a failed one stays recorded. It has the versions of its run. + """ + + __tablename__ = "cliff_impacts" + __table_args__: ClassVar = ( + PrimaryKeyConstraint("id", name="pk_cliff_impacts"), + ForeignKeyConstraint( + ["simulation_run_id"], + [SimulationRun.id], + name="fk_cliff_impacts_simulation_run_id_simulation_runs", + ondelete="CASCADE", + ), + CheckConstraint( + "status NOT IN ('succeeded', 'failed') OR completed_at IS NOT NULL", + name="ck_cliff_impacts_finished_completion", + ), + CheckConstraint( + "(cliff_gap IS NULL) = (cliff_share IS NULL) AND " + "(status = 'succeeded') = (cliff_gap IS NOT NULL)", + name="ck_cliff_impacts_result_when_succeeded", + ), + Index("ix_cliff_impacts_simulation_run_id", "simulation_run_id"), + # For the stuck-run sweep, which judges a calculation by its age. + Index("ix_cliff_impacts_status_started_at", "status", "started_at"), + # One calculation per run that hasn't failed: find-or-create's conflict + # target, so identical requests join one calculation. + Index( + "uq_cliff_impacts_unfailed", + "simulation_run_id", + unique=True, + postgresql_where=text("status <> 'failed'"), + ), + ) + + id: Mapped[UUID] = identifier_column() + created_at: Mapped[datetime] = server_timestamp_column() + updated_at: Mapped[datetime] = server_timestamp_column() + simulation_run_id: Mapped[UUID] = mapped_column(Uuid) + # A cliff calculation has a simulation run's states, under its type. + status: Mapped[SimulationStatus] = mapped_column( + enum_type(SimulationStatus, "v2_simulation_status") + ) + modal_call_id: Mapped[str | None] = mapped_column(String(MODAL_CALL_ID_LENGTH)) + error_message: Mapped[str | None] = mapped_column(String) + started_at: Mapped[datetime | None] = optional_timestamp_column() + completed_at: Mapped[datetime | None] = optional_timestamp_column() + # Both set exactly when the calculation succeeded. + cliff_gap: Mapped[float | None] = mapped_column(Float) + cliff_share: Mapped[float | None] = mapped_column(Float) + + service_metadata = ServiceBase.metadata diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/cliff_result_store.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/cliff_result_store.py index 6d874f008..02e540a64 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/cliff_result_store.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/cliff_result_store.py @@ -15,7 +15,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING -from policyengine_simulation_persistence.cliff_impact_tables import CliffImpact +from policyengine_simulation_persistence.tables import CliffImpact from sqlalchemy import func, or_, select, update from policyengine_stage12_persistence.report_run_store import ( diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py index 3e4b6d601..70c4e2a6a 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_store.py @@ -6,7 +6,8 @@ result columns in the same update, and inserts its result rows in the same transaction, only when its update changed the run. A household job is written the same way, and its result document is set in the update that -finishes it. +finishes it. The stuck-run sweep's reads and failures here also serve cliff +results, which ``CliffResultStore`` otherwise writes. """ from __future__ import annotations @@ -15,9 +16,9 @@ from enum import StrEnum from typing import TYPE_CHECKING, Any, Final -from policyengine_simulation_persistence.cliff_impact_tables import CliffImpact from policyengine_simulation_persistence.tables import ( BudgetWindowRun, + CliffImpact, HouseholdJob, ReportRun, SimulationRun, @@ -564,8 +565,8 @@ def list_active_runs( ) -> tuple[ActiveRun, ...]: """Return the pending and running runs of ``kind`` active since before a time. - A run counts from its first claim, or from its creation while it is - pending; a household job counts from its creation, as the table's + A run or cliff result counts from its first claim, or from its creation + while it is pending; a household job counts from its creation, as the table's status and creation index allows. Runs are oldest first, with or without a recorded Modal call. A simulation run is listed only while no pending or running report run names it as its baseline or reform: diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/cliff_results.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/cliff_results.py index d0717ef27..dc5aed7fe 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/cliff_results.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/cliff_results.py @@ -142,8 +142,8 @@ def fail_cliff_result( @dataclass(frozen=True) -class CliffImpact: - """One simulation's cliff gap and cliff share.""" +class CliffTotals: + """One simulation's calculated cliff gap and cliff share.""" cliff_gap: float cliff_share: float @@ -177,13 +177,13 @@ def still_held(self) -> bool: and state.modal_call_id == self.modal_call_id ) - def complete(self, impact: CliffImpact) -> RunWrite: + def complete(self, totals: CliffTotals) -> RunWrite: """Record the cliff gap and share; mark the cliff result succeeded.""" return self.writer.complete_cliff_result( self.cliff_result_id, modal_call_id=self.modal_call_id, - cliff_gap=impact.cliff_gap, - cliff_share=impact.cliff_share, + cliff_gap=totals.cliff_gap, + cliff_share=totals.cliff_share, completed_at=datetime.now(UTC), ) @@ -306,8 +306,8 @@ def run_cliff_result( # noqa: PLR0913 # the call's inputs, then the worker's s cancellation_requested=partial(_taken_from_call, recorder), ) validate_output_frames(calculation.frames, simulation.output_plan) - impact = cliff_impact_from_frames(calculation.frames, simulation.output_plan) - _require_completed(recorder, impact) + totals = cliff_totals_from_frames(calculation.frames, simulation.output_plan) + _require_completed(recorder, totals) except Stage12CancellationError: msg = "Stage 12 cliff calculation was cancelled" raise RuntimeError(msg) from None @@ -321,8 +321,8 @@ def run_cliff_result( # noqa: PLR0913 # the call's inputs, then the worker's s "cliff_result_id": str(recorder.cliff_result_id), "simulation_run_id": str(simulation.simulation_execution_id), "status": RunStatus.SUCCEEDED.value, - "cliff_gap": impact.cliff_gap, - "cliff_share": impact.cliff_share, + "cliff_gap": totals.cliff_gap, + "cliff_share": totals.cliff_share, } @@ -353,20 +353,20 @@ def _calculation( # noqa: PLR0913 # the simulation, then the worker's seams ) -def _require_completed(recorder: CliffResultRecorder, impact: CliffImpact) -> None: +def _require_completed(recorder: CliffResultRecorder, totals: CliffTotals) -> None: """Record the values; raise if the cliff result was taken from this call meanwhile. One failed meanwhile, as the stuck-run sweep fails one, takes no write from it: ``Stage12CancellationError``. """ - if not recorder.complete(impact).changed: + if not recorder.complete(totals).changed: msg = "Stage 12 cliff calculation was cancelled" raise Stage12CancellationError(msg) -def cliff_impact_from_frames( +def cliff_totals_from_frames( frames: Mapping[str, pd.DataFrame], output_plan: Stage12OutputPlan -) -> CliffImpact: +) -> CliffTotals: """Return the cliff gap and share of a calculation's weighted output rows. Each of .py's cliff variables is summed on the entity the plan @@ -386,7 +386,7 @@ def cliff_impact_from_frames( "The calculation has no adults, or a cliff total that is not a " "finite number, so its cliff share has no value.", ) - return CliffImpact( + return CliffTotals( cliff_gap=totals["cliff_gap"], cliff_share=totals["is_on_cliff"] / adults, ) diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_cliff_results.py b/projects/policyengine-simulation-executor/tests/test_stage12_cliff_results.py index ddb7c52d1..f04e7b7b5 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_cliff_results.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_cliff_results.py @@ -40,7 +40,7 @@ cliff_results as cliff_results_module, ) from policyengine_simulation_executor.stage12_runtime.cliff_results import ( - cliff_impact_from_frames, + cliff_totals_from_frames, ) from policyengine_simulation_executor.stage12_runtime.output_planning import ( plan_simulation_input, @@ -436,7 +436,7 @@ def test_the_cliff_gap_and_share_are_pys_own_from_the_same_rows() -> None: ), ) - impact = cliff_impact_from_frames(frames, cliff_plan()) + impact = cliff_totals_from_frames(frames, cliff_plan()) expected = calculate_cliff_impact(calculated, calculated).baseline assert (impact.cliff_gap, impact.cliff_share) == ( @@ -506,7 +506,7 @@ def test_merged_region_group_rows_give_the_cliffs_of_the_whole_country() -> None merged = merge_segment_results(simulation, payloads, groups=groups) - split = cliff_impact_from_frames(merged.frames, plan) - whole = cliff_impact_from_frames({"person": country}, plan) + split = cliff_totals_from_frames(merged.frames, plan) + whole = cliff_totals_from_frames({"person": country}, plan) assert split.cliff_gap == pytest.approx(whole.cliff_gap, rel=1e-12) assert split.cliff_share == pytest.approx(whole.cliff_share, rel=1e-12) From d7bb73534939dce2f18e213582f307340f754d7e Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 19 Nov 2026 02:35:35 +0400 Subject: [PATCH 20/24] Test a split cliff calculation's stop and its one-worker case A region-group calculation learns its cliff result was taken from its call, and a US national calculation whose policy turns on a labour-supply response stays in one worker, as its run does. The deployment workflow's comment on the sweep names cliff results, and two small wording leftovers go. Co-Authored-By: Claude Opus 5.5 --- .../workflows/simulation-deploy.reusable.yml | 4 +- .../tests/test_public_api_schema.py | 2 +- .../stage12_runtime/cliff_results.py | 2 +- .../tests/test_stage12_cliff_results.py | 60 +++++++++++++++++-- 4 files changed, 60 insertions(+), 8 deletions(-) diff --git a/.github/workflows/simulation-deploy.reusable.yml b/.github/workflows/simulation-deploy.reusable.yml index 309092f45..dadba2e61 100644 --- a/.github/workflows/simulation-deploy.reusable.yml +++ b/.github/workflows/simulation-deploy.reusable.yml @@ -1187,8 +1187,8 @@ jobs: MODAL_ENVIRONMENT: ${{ inputs.modal_environment }} run: ../../.github/scripts/stage12-v2-default-verify.sh - # The scheduled sweep that fails report runs, simulation runs and - # household jobs nothing will finish. Its app is unversioned and its + # The scheduled sweep that fails report runs, simulation runs, household + # jobs and cliff results nothing will finish. Its app is unversioned and its # schedule starts when it deploys, so it deploys here, after every # check, with the release's Stage 12 default; if it fails, the next # step puts the default back. diff --git a/libs/policyengine-simulation-persistence/tests/test_public_api_schema.py b/libs/policyengine-simulation-persistence/tests/test_public_api_schema.py index bdf395c93..297307c2d 100644 --- a/libs/policyengine-simulation-persistence/tests/test_public_api_schema.py +++ b/libs/policyengine-simulation-persistence/tests/test_public_api_schema.py @@ -84,7 +84,7 @@ def test_the_cliff_calculation_model_matches_the_public_api_table( ) -> None: """A simulation run's cliff calculations match the Public API's ``cliff_impacts``. - The service's other tables are compared with it above; this names the + The comparison of all the service's tables above includes it; this names the table alone, so a difference there is reported as the cliff table's. """ models_engine = _engine(disposable_database.superuser_url) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/cliff_results.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/cliff_results.py index dc5aed7fe..120fb6f49 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/cliff_results.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/cliff_results.py @@ -117,7 +117,7 @@ def claim_cliff_result( """Mark a pending or running cliff result that records no other call running.""" ... - def complete_cliff_result( # the result, its call, then its values + def complete_cliff_result( self, cliff_result_id: UUID, *, diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_cliff_results.py b/projects/policyengine-simulation-executor/tests/test_stage12_cliff_results.py index f04e7b7b5..fce99480f 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_cliff_results.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_cliff_results.py @@ -70,14 +70,16 @@ WEIGHTED_CLIFF_SHARE = (1.0 + 2.0) / (1.0 + 3.0) -def cliff_plan(country: str = "us") -> Stage12OutputPlan: +def cliff_plan( + country: str = "us", *, labor_supply_response_active: bool = False +) -> Stage12OutputPlan: """Return a cliff calculation's plan: the person columns the share reads.""" return Stage12OutputPlan( country=country, requirements=ReportOutputRequirements( aggregates=tuple(ReportAggregate), include_cliff_impacts=True, - labor_supply_response_active=False, + labor_supply_response_active=labor_supply_response_active, ), entities=( EntityOutputPlan( @@ -382,6 +384,56 @@ def segmented_calculator(_simulation, _cancellation_requested): assert cliff_results.cliff_results[CLIFF_RESULT_ID].status is (RunStatus.SUCCEEDED) +def test_a_national_calculation_with_a_labour_supply_response_stays_in_one_worker( + monkeypatch, +) -> None: + """It is split exactly when its run's own calculation is, which this one isn't.""" + monkeypatch.setattr( + cliff_results_module, + "resolve_cliff_output_plan", + lambda simulation: cliff_plan( + simulation.geography.country, labor_supply_response_active=True + ), + ) + cliff_results = FakeCliffResults() + calculator = RecordingCalculator() + + def segmented_calculator(_simulation, _cancellation_requested): + msg = "a calculation with a labour-supply response is not split" + raise AssertionError(msg) + + _run_cliff_worker( + cliff_results, + calculator=calculator, + segmented_calculator=segmented_calculator, + ) + + assert len(calculator.simulations) == 1 + assert cliff_results.cliff_results[CLIFF_RESULT_ID].status is RunStatus.SUCCEEDED + + +def test_a_split_calculation_learns_its_cliff_result_was_taken_from_it() -> None: + cliff_results = FakeCliffResults() + result = cliff_results.cliff_results[CLIFF_RESULT_ID] + checks = [] + + def segmented_calculator(_simulation, cancellation_requested): + checks.append(cancellation_requested()) + # The stuck-run sweep fails the cliff result while its groups run. + result.status = RunStatus.FAILED + result.error_message = "modal_call_ended: the call ended" + checks.append(cancellation_requested()) + return SimulationCalculation(frames=cliff_frames()) + + with pytest.raises( + RuntimeError, match=r"^Stage 12 cliff calculation was cancelled$" + ): + _run_cliff_worker(cliff_results, segmented_calculator=segmented_calculator) + + assert checks == [False, True] + assert (result.cliff_gap, result.cliff_share) == (None, None) + + def test_a_dataset_with_no_adults_fails_the_cliff_result() -> None: cliff_results = FakeCliffResults() frames = cliff_frames() @@ -436,10 +488,10 @@ def test_the_cliff_gap_and_share_are_pys_own_from_the_same_rows() -> None: ), ) - impact = cliff_totals_from_frames(frames, cliff_plan()) + totals = cliff_totals_from_frames(frames, cliff_plan()) expected = calculate_cliff_impact(calculated, calculated).baseline - assert (impact.cliff_gap, impact.cliff_share) == ( + assert (totals.cliff_gap, totals.cliff_share) == ( pytest.approx(expected.cliff_gap), pytest.approx(expected.cliff_share), ) From b249742b810de1842929d7c5ce857214a3012c50 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 19 Nov 2026 02:37:55 +0400 Subject: [PATCH 21/24] A cliff calculation's operation names its cliff result (failing) A simulation worker's call naming a cliff result records cliff_impacts and that row's ID in its operation, as a call naming a simulation run records that run, so an interrupted calculation leads to its row in telemetry. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_stage12_modal_app.py | 50 ++++++++++++++++--- 1 file changed, 44 insertions(+), 6 deletions(-) diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py b/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py index 365a0a379..461297e06 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py @@ -941,6 +941,7 @@ def test_modal_app_name_cannot_override_bundle_selection(monkeypatch) -> None: REPORT_RUN_ID = "00000000-0000-4000-8000-00000000b002" HOUSEHOLD_JOB_ID = "00000000-0000-4000-8000-00000000a001" HOUSEHOLD_REPORT_RUN_ID = "00000000-0000-4000-8000-00000000a002" +CLIFF_RESULT_ID = "00000000-0000-4000-8000-00000000c001" OPERATION_RECORD_KEYS = ("database_table", "database_row_id", "modal_call_id") @@ -967,10 +968,14 @@ def _invoke_with_record( def succeed(*_arguments, **_keywords) -> dict: return {"ok": True} - if call_name in ("run_single_simulation_us", "run_single_simulation_uk"): + function_name = call_name.removesuffix(":cliffs") + if function_name in ("run_single_simulation_us", "run_single_simulation_uk"): monkeypatch.setattr(stage12_runtime, "run_single_simulation", succeed) - context = {"simulation_run_id": SIMULATION_RUN_ID} if names_record else {} - return getattr(module, call_name)( + if call_name.endswith(":cliffs"): + context = {"cliff_result_id": CLIFF_RESULT_ID} + else: + context = {"simulation_run_id": SIMULATION_RUN_ID} if names_record else {} + return getattr(module, function_name)( {"role": "baseline"}, context, observability_context=None ) if call_name == "run_single_simulation_segment_us": @@ -1045,11 +1050,44 @@ def record_metric_attributes(values, config): return captured +@pytest.mark.parametrize( + ("context", "expected_row"), + [ + ({"cliff_result_id": CLIFF_RESULT_ID}, ("cliff_impacts", CLIFF_RESULT_ID)), + ( + {"simulation_run_id": SIMULATION_RUN_ID}, + ("simulation_runs", SIMULATION_RUN_ID), + ), + ({"cliff_result_id": "not-a-uuid"}, None), + ({}, None), + ("not-a-context", None), + ], + ids=["cliff-result", "simulation-run", "malformed-id", "no-row", "malformed"], +) +def test_a_simulation_workers_record_is_the_row_its_context_names( + context, expected_row +) -> None: + from policyengine_simulation_executor import stage12_runtime + + attributes = stage12_runtime.simulation_worker_record_attributes( + context, "fc-operation" + ) + + assert attributes.pop("modal_call_id") == "fc-operation" + assert attributes == ( + {} + if expected_row is None + else {"database_table": expected_row[0], "database_row_id": expected_row[1]} + ) + + @pytest.mark.parametrize( ("call_name", "expected_table", "expected_row_id"), [ ("run_single_simulation_us", "simulation_runs", SIMULATION_RUN_ID), ("run_single_simulation_uk", "simulation_runs", SIMULATION_RUN_ID), + ("run_single_simulation_us:cliffs", "cliff_impacts", CLIFF_RESULT_ID), + ("run_single_simulation_uk:cliffs", "cliff_impacts", CLIFF_RESULT_ID), ("run_single_simulation_segment_us", "simulation_runs", SIMULATION_RUN_ID), ("coordinate_report", "report_runs", REPORT_RUN_ID), ("calculate_household_us:job", "household_jobs", HOUSEHOLD_JOB_ID), @@ -1074,9 +1112,9 @@ def test_v2_functions_name_their_database_row_and_modal_call_in_the_operation_re ) metric_attributes = _capture_metric_attributes(monkeypatch) - assert _invoke_with_record( - monkeypatch, module, call_name, names_record=True - ) == {"ok": True} + assert _invoke_with_record(monkeypatch, module, call_name, names_record=True) == { + "ok": True + } [completion] = [ record From 4b7ece5087f6ee51d91bb45a7c5b7c24723ae2bb Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 19 Nov 2026 02:38:48 +0400 Subject: [PATCH 22/24] Name the cliff result in a cliff calculation's operation A simulation worker's operation names cliff_impacts and the cliff result its context names, or else the simulation run, through one helper both workers use. cliff_impacts joins the tables telemetry may name, and the observability guidance says a cliff calculation's segments link to it through the trace. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/skills/observability.md | 9 ++--- .../log_records.py | 7 +++- .../src/modal/v2_app.py | 10 +++--- .../stage12_runtime/__init__.py | 2 ++ .../stage12_runtime/operation_records.py | 33 +++++++++++++++++-- 5 files changed, 47 insertions(+), 14 deletions(-) diff --git a/docs/engineering/skills/observability.md b/docs/engineering/skills/observability.md index 86528740d..e32366e1d 100644 --- a/docs/engineering/skills/observability.md +++ b/docs/engineering/skills/observability.md @@ -16,16 +16,17 @@ Use each identifier for its defined scope: | `evaluation_id` | One Stage 12 report | Stage 12 report construction | Yes, as functional report state | | `simulation_execution_id` | One Stage 12 baseline or reform simulation | Stage 12 coordinator | Yes, as functional simulation state | | `submission_claim_id` | One API v1 attempt to acquire ownership of a simulation submission | API v1 | Only where the simulation request is stored | -| `database_table` | The API v1 table holding the row a v2 Modal function call, or a Simulation Entrypoint log record, is about: `household_jobs`, `report_runs` (a report run or a household report run), `simulation_runs`, `reports` or `simulations` | The code writing the record, as a fixed name | No: telemetry only | +| `database_table` | The API v1 table holding the row a v2 Modal function call, or a Simulation Entrypoint log record, is about: `household_jobs`, `report_runs` (a report run or a household report run), `simulation_runs`, `cliff_impacts`, `reports` or `simulations` | The code writing the record, as a fixed name | No: telemetry only | | `database_row_id` | That row's UUID | The simulation service, which creates the row | Yes, as the row's ID | | `modal_call_id` | One Modal function call of a v2 Modal function | Modal | Only where the row records the call that claimed it; a segment worker's call is recorded nowhere, and its `database_row_id` names the simulation run its simulation worker's call claimed | Each v2 Modal function's operation (`src/modal/v2_app.py`) names its `modal_call_id` and, when its call works on a detailed-save row, that row's `database_table` and `database_row_id`; a US segment worker names the -simulation run its simulation worker records. They go to the operation's log -records and span only, never to metrics, where each distinct ID would make a -separate series. The observability package drops every attribute whose key +simulation run its simulation worker records, and a segment of a cliff +calculation names no row, linking to its cliff result through the trace. They +go to the operation's log records and span only, never to metrics, where each +distinct ID would make a separate series. The observability package drops every attribute whose key contains `household`, `reform`, `payload` or `person`, among others, so no key for these IDs names one. Do not use `job_id` for a v2 call: it names one annual Modal invocation of the v1 executor. diff --git a/libs/policyengine-simulation-observability/src/policyengine_simulation_observability/log_records.py b/libs/policyengine-simulation-observability/src/policyengine_simulation_observability/log_records.py index a35f1c07c..734f2e54a 100644 --- a/libs/policyengine-simulation-observability/src/policyengine_simulation_observability/log_records.py +++ b/libs/policyengine-simulation-observability/src/policyengine_simulation_observability/log_records.py @@ -23,7 +23,12 @@ # The API v1 tables whose rows telemetry names. DatabaseTable = Literal[ - "household_jobs", "report_runs", "reports", "simulation_runs", "simulations" + "cliff_impacts", + "household_jobs", + "report_runs", + "reports", + "simulation_runs", + "simulations", ] LogFieldValue = str | int | float | bool diff --git a/projects/policyengine-simulation-executor/src/modal/v2_app.py b/projects/policyengine-simulation-executor/src/modal/v2_app.py index 689f356f5..972735629 100644 --- a/projects/policyengine-simulation-executor/src/modal/v2_app.py +++ b/projects/policyengine-simulation-executor/src/modal/v2_app.py @@ -427,8 +427,8 @@ def segmented_calculator( attributes={ "runner_name": "stage12", "simulation_role": payload.get("role"), - **stage12_runtime.operation_record_attributes( - "simulation_runs", simulation_run_id, modal_call_id + **stage12_runtime.simulation_worker_record_attributes( + context, modal_call_id ), }, remote_context=propagated, @@ -487,10 +487,8 @@ def run_single_simulation_uk( attributes={ "runner_name": "stage12", "simulation_role": payload.get("role"), - **stage12_runtime.operation_record_attributes( - "simulation_runs", - stage12_runtime.simulation_run_named_by(context), - modal_call_id, + **stage12_runtime.simulation_worker_record_attributes( + context, modal_call_id ), }, remote_context=propagated, diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/__init__.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/__init__.py index 723ebc8d8..f8b574159 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/__init__.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/__init__.py @@ -33,6 +33,7 @@ report_run_named_by, row_id_or_none, simulation_run_named_by, + simulation_worker_record_attributes, ) from .partition import ( should_segment_simulation, @@ -96,6 +97,7 @@ "should_segment_simulation", "simulation_input_sha256", "simulation_run_named_by", + "simulation_worker_record_attributes", "stage12_region_groups", "stage12_region_groups_for_model", "validate_policy_periods", diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/operation_records.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/operation_records.py index 18b2f6010..02e7d3d21 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/operation_records.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/operation_records.py @@ -6,8 +6,8 @@ was working on: - ``database_table``: the table holding that row, ``household_jobs``, - ``report_runs`` (a report run, or a household report run) or - ``simulation_runs``; + ``report_runs`` (a report run, or a household report run), + ``simulation_runs`` or ``cliff_impacts``; - ``database_row_id``: the row's UUID; - ``modal_call_id``: the function's own Modal function call ID. @@ -30,7 +30,9 @@ ) # The tables a v2 Modal function's call works on. -OperationTable = Literal["household_jobs", "report_runs", "simulation_runs"] +OperationTable = Literal[ + "cliff_impacts", "household_jobs", "report_runs", "simulation_runs" +] def operation_record_attributes( @@ -79,3 +81,28 @@ def simulation_run_named_by(context: object) -> UUID | None: if not isinstance(context, Mapping): return None return row_id_or_none(context.get("simulation_run_id")) + + +def cliff_result_named_by(context: object) -> UUID | None: + """Return the cliff result a simulation worker call's context names, if any.""" + if not isinstance(context, Mapping): + return None + return row_id_or_none(context.get("cliff_result_id")) + + +def simulation_worker_record_attributes( + context: object, modal_call_id: str | None +) -> dict[str, str]: + """Return the operation attributes of a simulation worker's call. + + The call works on the cliff result its context names, or else on the + simulation run it names. + """ + cliff_result_id = cliff_result_named_by(context) + if cliff_result_id is not None: + return operation_record_attributes( + "cliff_impacts", cliff_result_id, modal_call_id + ) + return operation_record_attributes( + "simulation_runs", simulation_run_named_by(context), modal_call_id + ) From 8f0a0f734bda514c6073e0bde18eeccc3a2db8f5 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 19 Nov 2026 03:10:07 +0400 Subject: [PATCH 23/24] Pin that a cliff calculation's segments name no row The US worker passes a segment no simulation run for a cliff calculation. The telemetry guidance and docstrings say why, and how a segment still leads to its cliff result; the changelog fragment names the new table value. Co-Authored-By: Claude Opus 5.5 --- changelog.d/c1-38-run-cliff-results.added.md | 2 +- docs/engineering/skills/observability.md | 10 ++++++---- .../stage12_runtime/fanout.py | 4 ++-- .../stage12_runtime/operation_records.py | 4 +++- .../tests/test_stage12_modal_app.py | 17 ++++++++++++----- 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/changelog.d/c1-38-run-cliff-results.added.md b/changelog.d/c1-38-run-cliff-results.added.md index 51a418bbf..cc4cf1814 100644 --- a/changelog.d/c1-38-run-cliff-results.added.md +++ b/changelog.d/c1-38-run-cliff-results.added.md @@ -1 +1 @@ -A v2 simulation worker can now calculate a simulation run's own cliff result: a call whose context names a row of the Public API's `cliff_impacts` calculates that run's simulation again with .py's cliff columns and records its weighted cliff gap and cliff share, once, on that row, and the stuck-run sweep fails one whose call ended. A report run's completion no longer writes `report_runs`' four cliff columns. Nothing starts a cliff calculation yet, and a report asking for cliffs is still refused. +A v2 simulation worker can now calculate a simulation run's own cliff result: a call whose context names a row of the Public API's `cliff_impacts` calculates that run's simulation again with .py's cliff columns and records its weighted cliff gap and cliff share, once, on that row, and the stuck-run sweep fails one whose call ended. The worker's operation record names `cliff_impacts` and the row in telemetry. A report run's completion no longer writes `report_runs`' four cliff columns. Nothing starts a cliff calculation yet, and a report asking for cliffs is still refused. diff --git a/docs/engineering/skills/observability.md b/docs/engineering/skills/observability.md index e32366e1d..068f8a04e 100644 --- a/docs/engineering/skills/observability.md +++ b/docs/engineering/skills/observability.md @@ -23,10 +23,12 @@ Use each identifier for its defined scope: Each v2 Modal function's operation (`src/modal/v2_app.py`) names its `modal_call_id` and, when its call works on a detailed-save row, that row's `database_table` and `database_row_id`; a US segment worker names the -simulation run its simulation worker records, and a segment of a cliff -calculation names no row, linking to its cliff result through the trace. They -go to the operation's log records and span only, never to metrics, where each -distinct ID would make a separate series. The observability package drops every attribute whose key +simulation run its simulation worker records. A segment of a cliff calculation +names no row: it joins the simulation worker's trace, whose operation names the +cliff result, or, if it starts too late to join, links to that trace with a span +link, so its log records then carry another trace ID. The attributes go to the +operation's log records and span only, never to metrics, where each distinct ID +would make a separate series. The observability package drops every attribute whose key contains `household`, `reform`, `payload` or `person`, among others, so no key for these IDs names one. Do not use `job_id` for a v2 call: it names one annual Modal invocation of the v1 executor. diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/fanout.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/fanout.py index c2a3c9788..38162d3d8 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/fanout.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/fanout.py @@ -81,7 +81,7 @@ def __init__( # noqa: PLR0913 # the simulation, then keyword-only app, runtime ``cancellation_requested`` is checked between polls. Each segment's call is passed ``simulation_run_id``, the detailed-save simulation run - this simulation records, for its telemetry. + this simulation records, if it records one, for its telemetry. """ seams = seams or SegmentRunnerSeams() self.simulation = simulation @@ -232,7 +232,7 @@ def _collect(self, handles: list[tuple[int, Any]]) -> list[object]: def _segment_record_keywords(self) -> dict[str, str]: """Return the keyword naming the simulation run, when there is one. - A comparison-path simulation records no simulation run. + A comparison-path simulation and a cliff calculation record none. """ if self.simulation_run_id is None: return {} diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/operation_records.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/operation_records.py index 02e7d3d21..366069de9 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/operation_records.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/operation_records.py @@ -15,7 +15,9 @@ ``household``, among other words, so no key here names one. A call that works on no detailed-save row (a comparison-path call, a standalone simulation, a release validation's household calculation) names only its -Modal call. +Modal call. So does a US region-group segment of a cliff calculation: the +segment's record can name only a simulation run, and its cliff result is named +by the simulation worker's operation, which its trace context leads to. """ from __future__ import annotations diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py b/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py index 461297e06..88fcae044 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py @@ -1198,8 +1198,17 @@ def test_a_malformed_simulation_run_id_is_left_out_of_the_operation_record( assert "database_table" not in completion["attributes"] +@pytest.mark.parametrize( + ("context", "expected_simulation_run_id"), + [ + ({"simulation_run_id": SIMULATION_RUN_ID}, UUID(SIMULATION_RUN_ID)), + # A segment's record names simulation runs, so a cliff result isn't one. + ({"cliff_result_id": CLIFF_RESULT_ID}, None), + ], + ids=["simulation-run", "cliff-result"], +) def test_the_us_simulation_worker_passes_its_simulation_run_to_its_segments( - monkeypatch, + monkeypatch, context, expected_simulation_run_id ) -> None: from policyengine_simulation_executor import stage12_runtime @@ -1228,13 +1237,11 @@ def run_single_simulation(payload, context, **keywords): monkeypatch.setattr(stage12_runtime, "run_single_simulation", run_single_simulation) module.run_single_simulation_us( - {"role": "baseline"}, - {"simulation_run_id": SIMULATION_RUN_ID}, - observability_context=None, + {"role": "baseline"}, context, observability_context=None ) (segmented_run,) = segmented_runs - assert segmented_run["simulation_run_id"] == UUID(SIMULATION_RUN_ID) + assert segmented_run["simulation_run_id"] == expected_simulation_run_id @pytest.mark.parametrize( From 495bf7eaea09e83ae98b07cc849fb028a46c3c4e Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 19 Nov 2026 20:41:42 +0400 Subject: [PATCH 24/24] Call API v1 by its name in the cliff result's text The changelog fragment, the migration and library documents, the docstrings, the cliff table's schema test and its failure message now name API v1. The existing pytest marker and fixture names stay. Co-Authored-By: Claude Opus 5.5 --- changelog.d/c1-38-run-cliff-results.added.md | 2 +- docs/migration/stage-12-modal-worker-foundation.md | 4 ++-- libs/policyengine-simulation-persistence/README.md | 4 ++-- .../tests/test_public_api_schema.py | 6 +++--- .../policyengine_stage12_persistence/cliff_result_store.py | 2 +- .../policyengine_stage12_persistence/report_run_results.py | 2 +- .../tests/test_report_run_results.py | 2 +- .../src/modal/run_sweep_app.py | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/changelog.d/c1-38-run-cliff-results.added.md b/changelog.d/c1-38-run-cliff-results.added.md index cc4cf1814..bdc08f234 100644 --- a/changelog.d/c1-38-run-cliff-results.added.md +++ b/changelog.d/c1-38-run-cliff-results.added.md @@ -1 +1 @@ -A v2 simulation worker can now calculate a simulation run's own cliff result: a call whose context names a row of the Public API's `cliff_impacts` calculates that run's simulation again with .py's cliff columns and records its weighted cliff gap and cliff share, once, on that row, and the stuck-run sweep fails one whose call ended. The worker's operation record names `cliff_impacts` and the row in telemetry. A report run's completion no longer writes `report_runs`' four cliff columns. Nothing starts a cliff calculation yet, and a report asking for cliffs is still refused. +A v2 simulation worker can now calculate a simulation run's own cliff result: a call whose context names a row of API v1's `cliff_impacts` calculates that run's simulation again with .py's cliff columns and records its weighted cliff gap and cliff share, once, on that row, and the stuck-run sweep fails one whose call ended. The worker's operation record names `cliff_impacts` and the row in telemetry. A report run's completion no longer writes `report_runs`' four cliff columns. Nothing starts a cliff calculation yet, and a report asking for cliffs is still refused. diff --git a/docs/migration/stage-12-modal-worker-foundation.md b/docs/migration/stage-12-modal-worker-foundation.md index e78209416..e267e15d1 100644 --- a/docs/migration/stage-12-modal-worker-foundation.md +++ b/docs/migration/stage-12-modal-worker-foundation.md @@ -369,7 +369,7 @@ holds, or one that failed, it returns `{"deduplicated": true, **A simulation run's cliff result** (C1 · 24, answer 4b). Each simulation run can have one cliff result, its cliff gap and cliff share, which every report -naming the run reads: a row of the Public API's `cliff_impacts`, one per +naming the run reads: a row of API v1's `cliff_impacts`, one per calculation. Nothing starts one yet. The simulation service is to insert it pending, start the country's simulation worker (`run_single_simulation_us` or `run_single_simulation_uk`) with the run's own simulation input, built as a @@ -398,7 +398,7 @@ it returns its values again when Modal restarts it. A failure fails the row with its redacted cause; the stuck-run sweep fails one whose call ended without an outcome, and a failed row never counts, so the next request starts another calculation. The worker writes nothing to the simulation run, its -output file or any report run. The Public API's revision that adds +output file or any report run. API v1's revision that adds `cliff_impacts` must be applied in an environment before this release deploys there: the stuck-run sweep lists the table on every pass, and the service's `/ready` privilege check names it. diff --git a/libs/policyengine-simulation-persistence/README.md b/libs/policyengine-simulation-persistence/README.md index eb64996b2..528fe58b0 100644 --- a/libs/policyengine-simulation-persistence/README.md +++ b/libs/policyengine-simulation-persistence/README.md @@ -465,8 +465,8 @@ run has no call of its own) and of the results its own row holds: a report run's `spm_provenance` and `data_version`, a simulation run's output file and `household_result`, a household job's `result`, and a cliff calculation's `cliff_gap` and `cliff_share`. It no longer writes a report run's four cliff -columns, which the Public API is to drop: a report run's cliff impact is its -two simulation runs' cliff results. The Public API's manifest grants them; +columns, which API v1 is to drop: a report run's cliff impact is its +two simulation runs' cliff results. The same privilege manifest grants them; `test_modal_worker_writes` checks that in CI. `missing_table_privileges` returns each one the connected role lacks, ordered diff --git a/libs/policyengine-simulation-persistence/tests/test_public_api_schema.py b/libs/policyengine-simulation-persistence/tests/test_public_api_schema.py index 297307c2d..b4e5a399b 100644 --- a/libs/policyengine-simulation-persistence/tests/test_public_api_schema.py +++ b/libs/policyengine-simulation-persistence/tests/test_public_api_schema.py @@ -79,10 +79,10 @@ def test_the_models_match_the_public_api_tables( @pytest.mark.public_api_schema -def test_the_cliff_calculation_model_matches_the_public_api_table( +def test_the_cliff_calculation_model_matches_the_api_v1_table( disposable_database, public_api_database_url ) -> None: - """A simulation run's cliff calculations match the Public API's ``cliff_impacts``. + """A simulation run's cliff calculations match API v1's ``cliff_impacts``. The comparison of all the service's tables above includes it; this names the table alone, so a difference there is reported as the cliff table's. @@ -103,7 +103,7 @@ def test_the_cliff_calculation_model_matches_the_public_api_table( assert set(real.tables) == {"cliff_impacts"} compared_with = os.environ.get(COMMIT_VARIABLE) or "an unrecorded commit" assert schema_differences(models, real) == [], ( - f"compared with the Public API at {compared_with}" + f"compared with API v1 at {compared_with}" ) diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/cliff_result_store.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/cliff_result_store.py index 02e540a64..11418f532 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/cliff_result_store.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/cliff_result_store.py @@ -1,6 +1,6 @@ """Conditional writes of a simulation run's cliff calculation, as the Modal worker role. -The Public API's ``cliff_impacts`` holds each cliff calculation of a +API v1's ``cliff_impacts`` holds each cliff calculation of a simulation run, its cliff result. The simulation service inserts it pending and may record the Modal call it starts while it is still pending with none. A simulation worker's call then claims it under that call, as a run is diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_results.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_results.py index 37be46bf6..1252711b3 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_results.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/report_run_results.py @@ -234,7 +234,7 @@ def _report_run_columns(economy: SingleYearMacroOutput) -> dict[str, Any]: No cliff column: a cliff impact is each simulation run's own cliff result, so the report run's four cliff columns stay null, and its - completion names none of them, which lets the Public API drop them. + completion names none of them, which lets API v1 drop them. """ receipts = economy.spm_provenance return { diff --git a/libs/policyengine-stage12-persistence/tests/test_report_run_results.py b/libs/policyengine-stage12-persistence/tests/test_report_run_results.py index 3910c7cfc..226ebe0bd 100644 --- a/libs/policyengine-stage12-persistence/tests/test_report_run_results.py +++ b/libs/policyengine-stage12-persistence/tests/test_report_run_results.py @@ -308,7 +308,7 @@ def test_the_labour_supply_response_is_one_row_and_a_row_per_decile( def test_a_report_runs_result_columns_never_name_its_cliff_columns() -> None: - """The completion's update names none of them, so the Public API can drop them.""" + """The completion's update names none of them, so API v1 can drop them.""" run_ids = ReportRunIds( report_run_id=uuid4(), baseline_simulation_run_id=uuid4(), diff --git a/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py b/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py index ee858f094..4873363dc 100644 --- a/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py +++ b/projects/policyengine-simulation-executor/src/modal/run_sweep_app.py @@ -1,6 +1,6 @@ """A scheduled Modal function that fails the runs nothing will finish. -Every ten minutes it lists the Public API's report runs, simulation runs, +Every ten minutes it lists API v1's report runs, simulation runs, household jobs and simulation runs' cliff results that have been pending or running for at least ten minutes, counted from a run's or cliff result's first claim or, while it is pending, from its creation (a budget window's