From c772f0c1344aa37e036696c5e5b4ddfa242f5f64 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:00:58 +0400 Subject: [PATCH 01/10] Test that v2 operation records name their database row and Modal call (failing) Each v2 Modal function's operation record should carry `database_table`, `database_row_id` and `modal_call_id` after the observability package's attribute filter, and its metrics none of them. The US simulation worker should pass its simulation run to the segments it starts. Co-Authored-By: Claude Opus 5.5 --- .../fixtures/fake_modal.py | 2 + .../tests/test_stage12_modal_app.py | 262 ++++++++++++++++++ .../tests/test_stage12_segment_fanout.py | 59 ++++ 3 files changed, 323 insertions(+) diff --git a/projects/policyengine-simulation-executor/fixtures/fake_modal.py b/projects/policyengine-simulation-executor/fixtures/fake_modal.py index bc64d36f5..d1f0fd591 100644 --- a/projects/policyengine-simulation-executor/fixtures/fake_modal.py +++ b/projects/policyengine-simulation-executor/fixtures/fake_modal.py @@ -142,5 +142,7 @@ def install_fake_modal(monkeypatch: pytest.MonkeyPatch) -> None: modal.App = FakeApp modal.Cron = lambda schedule: {"schedule": schedule} modal.is_local = lambda: True + # Modal answers None outside a function call. + modal.current_function_call_id = lambda: None modal.asgi_app = lambda: lambda function: function monkeypatch.setitem(sys.modules, "modal", modal) diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py b/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py index e3f6ef998..32476087a 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py @@ -922,3 +922,265 @@ def test_modal_app_name_cannot_override_bundle_selection(monkeypatch) -> None: with pytest.raises(RuntimeError, match="does not match"): importlib.import_module("src.modal.v2_app") + + +SIMULATION_RUN_ID = "00000000-0000-4000-8000-00000000b001" +REPORT_RUN_ID = "00000000-0000-4000-8000-00000000b002" +HOUSEHOLD_JOB_ID = "00000000-0000-4000-8000-00000000a001" +HOUSEHOLD_REPORT_RUN_ID = "00000000-0000-4000-8000-00000000a002" +OPERATION_RECORD_KEYS = ("database_table", "database_row_id", "modal_call_id") + + +def _report_run_context() -> dict: + """Return a detailed-save coordinator call's context naming its report run.""" + return { + "report_run": { + "report_run_id": REPORT_RUN_ID, + "baseline_simulation_run_id": "00000000-0000-4000-8000-00000000b003", + "reform_simulation_run_id": "00000000-0000-4000-8000-00000000b004", + } + } + + +def _invoke_with_record( + monkeypatch, module, call_name: str, *, names_record: bool +) -> dict: + """Call one v2 Modal function, naming its database row when ``names_record``. + + Every application function is replaced with one returning ``{"ok": True}``. + """ + from policyengine_simulation_executor import household_calculation, stage12_runtime + + def succeed(*_arguments, **_keywords) -> dict: + return {"ok": True} + + if call_name in ("run_single_simulation_us", "run_single_simulation_uk"): + monkeypatch.setattr(stage12_runtime, "run_single_simulation", succeed) + context = {"simulation_run_id": SIMULATION_RUN_ID} if names_record else {} + return getattr(module, call_name)( + {"role": "baseline"}, context, observability_context=None + ) + if call_name == "run_single_simulation_segment_us": + + class SegmentResult: + def model_dump(self, *, mode): + return {"ok": True} + + monkeypatch.setattr( + stage12_runtime, "calculate_segment", lambda *_, **__: SegmentResult() + ) + keywords = {"simulation_run_id": SIMULATION_RUN_ID} if names_record else {} + return module.run_single_simulation_segment_us( + {"simulation": {"role": "baseline"}, "segment_index": 0}, + observability_context=None, + **keywords, + ) + if call_name == "coordinate_report": + monkeypatch.setattr(stage12_runtime, "coordinate_report", succeed) + monkeypatch.setattr( + stage12_runtime, + "coordinate_window_year", + lambda coordinate, **_keywords: coordinate(), + ) + context = _report_run_context() if names_record else {} + return module.coordinate_report({}, context, observability_context=None) + + function_name, record_kind = call_name.split(":") + country = function_name.rsplit("_", 1)[1] + monkeypatch.setattr(stage12_runtime, "run_household_job", succeed) + monkeypatch.setattr(stage12_runtime, "run_household_report", succeed) + + class HouseholdOutcome: + def model_dump(self, *, mode): + return {"ok": True} + + monkeypatch.setattr( + household_calculation, + "calculate_household", + lambda *_, **__: HouseholdOutcome(), + ) + if not names_record: + return getattr(module, function_name)( + _household_payload(country), observability_context=None + ) + if record_kind == "job": + return getattr(module, function_name)( + _household_payload(country), + observability_context=None, + household_job_id=HOUSEHOLD_JOB_ID, + ) + return getattr(module, function_name)( + _household_report_payload(country), + observability_context=None, + household_report_run_id=HOUSEHOLD_REPORT_RUN_ID, + ) + + +def _capture_metric_attributes(monkeypatch) -> list[dict]: + """Return the list every metric's attributes are appended to, after filtering.""" + from policyengine_observability import otel + + captured: list[dict] = [] + filter_metric_attributes = otel.metric_attributes + + def record_metric_attributes(values, config): + filtered = filter_metric_attributes(values, config) + captured.append(filtered) + return filtered + + monkeypatch.setattr(otel, "metric_attributes", record_metric_attributes) + return captured + + +@pytest.mark.parametrize( + ("call_name", "expected_table", "expected_row_id"), + [ + ("run_single_simulation_us", "simulation_runs", SIMULATION_RUN_ID), + ("run_single_simulation_uk", "simulation_runs", SIMULATION_RUN_ID), + ("run_single_simulation_segment_us", "simulation_runs", SIMULATION_RUN_ID), + ("coordinate_report", "report_runs", REPORT_RUN_ID), + ("calculate_household_us:job", "household_jobs", HOUSEHOLD_JOB_ID), + ("calculate_household_uk:job", "household_jobs", HOUSEHOLD_JOB_ID), + ("calculate_household_us:report", "report_runs", HOUSEHOLD_REPORT_RUN_ID), + ("calculate_household_uk:report", "report_runs", HOUSEHOLD_REPORT_RUN_ID), + ], +) +def test_v2_functions_name_their_database_row_and_modal_call_in_the_operation_record( + monkeypatch, + recorded_observability_runtime, + call_name, + expected_table, + expected_row_id, +) -> None: + # H7 · 1 links an interrupted operation in production's logs to its row. + runtime, written_records = recorded_observability_runtime + module = _load(monkeypatch) + monkeypatch.setattr(module, "init_process_observability", lambda **_: runtime) + monkeypatch.setattr( + module.modal, "current_function_call_id", lambda: "fc-operation" + ) + metric_attributes = _capture_metric_attributes(monkeypatch) + + assert _invoke_with_record( + monkeypatch, module, call_name, names_record=True + ) == {"ok": True} + + [completion] = [ + record + for record in written_records + if record.get("event.name") == "operation.completed" + ] + # Read after the package's attribute filter, which drops a key naming a + # household, a reform or a payload, among others. + assert completion["attributes"]["database_table"] == expected_table + assert completion["attributes"]["database_row_id"] == expected_row_id + assert completion["attributes"]["modal_call_id"] == "fc-operation" + # Each distinct ID would make a separate metric series. + assert metric_attributes + for attributes in metric_attributes: + assert not set(OPERATION_RECORD_KEYS) & set(attributes) + + +@pytest.mark.parametrize( + "call_name", + [ + "run_single_simulation_us", + "run_single_simulation_uk", + "run_single_simulation_segment_us", + "coordinate_report", + "calculate_household_us:none", + "calculate_household_uk:none", + ], +) +def test_v2_functions_naming_no_row_name_only_their_modal_call( + monkeypatch, recorded_observability_runtime, call_name +) -> None: + # A comparison-path call, a standalone simulation, or a release + # validation's household calculation works on no detailed-save row. + runtime, written_records = recorded_observability_runtime + module = _load(monkeypatch) + monkeypatch.setattr(module, "init_process_observability", lambda **_: runtime) + monkeypatch.setattr( + module.modal, "current_function_call_id", lambda: "fc-operation" + ) + + _invoke_with_record(monkeypatch, module, call_name, names_record=False) + + [completion] = [ + record + for record in written_records + if record.get("event.name") == "operation.completed" + ] + assert completion["attributes"]["modal_call_id"] == "fc-operation" + assert "database_table" not in completion["attributes"] + assert "database_row_id" not in completion["attributes"] + + +def test_a_malformed_simulation_run_id_is_left_out_of_the_operation_record( + monkeypatch, recorded_observability_runtime +) -> None: + # The worker refuses the context itself; telemetry names only a UUID. + from policyengine_simulation_executor import stage12_runtime + + runtime, written_records = recorded_observability_runtime + module = _load(monkeypatch) + monkeypatch.setattr(module, "init_process_observability", lambda **_: runtime) + monkeypatch.setattr( + module.modal, "current_function_call_id", lambda: "fc-operation" + ) + monkeypatch.setattr( + stage12_runtime, "run_single_simulation", lambda *_, **__: {"ok": True} + ) + + module.run_single_simulation_us( + {"role": "baseline"}, + {"simulation_run_id": "not a run ID"}, + observability_context=None, + ) + + [completion] = [ + record + for record in written_records + if record.get("event.name") == "operation.completed" + ] + assert "database_row_id" not in completion["attributes"] + assert "database_table" not in completion["attributes"] + + +def test_the_us_simulation_worker_passes_its_simulation_run_to_its_segments( + monkeypatch, +) -> None: + from policyengine_simulation_executor import stage12_runtime + + module = _load(monkeypatch) + monkeypatch.setattr( + module, "init_process_observability", lambda **_: RecordingRuntime() + ) + monkeypatch.setattr( + module.modal, "current_function_call_id", lambda: "fc-simulation-worker" + ) + segmented_runs: list[dict] = [] + + def run_segmented_simulation(simulation, **keywords): + segmented_runs.append({"simulation": simulation, **keywords}) + return "calculated" + + def run_single_simulation(payload, context, **keywords): + planned_simulation = object() + return keywords["dependencies"].segmented_calculator( + planned_simulation, lambda: False + ) + + monkeypatch.setattr( + stage12_runtime, "run_segmented_simulation", run_segmented_simulation + ) + monkeypatch.setattr(stage12_runtime, "run_single_simulation", run_single_simulation) + + module.run_single_simulation_us( + {"role": "baseline"}, + {"simulation_run_id": SIMULATION_RUN_ID}, + observability_context=None, + ) + + (segmented_run,) = segmented_runs + assert segmented_run["simulation_run_id"] == UUID(SIMULATION_RUN_ID) diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_segment_fanout.py b/projects/policyengine-simulation-executor/tests/test_stage12_segment_fanout.py index 6506c0a8f..ce42040e0 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_segment_fanout.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_segment_fanout.py @@ -5,6 +5,7 @@ from contextlib import contextmanager from types import SimpleNamespace from typing import TYPE_CHECKING, cast +from uuid import UUID import pandas as pd import pytest @@ -335,3 +336,61 @@ def test_merge_error_is_raised_from_the_real_merge_span( with pytest.raises(ValueError, match="duplicate index"): runner.run() + + +class FunctionRecordingSpawnKeywords(FakeFunction): + """A segment function that also records each spawn's keyword arguments.""" + + def __init__(self, results: list[dict]) -> None: + super().__init__(results, {}) + self.spawn_keywords: list[dict] = [] + + def spawn(self, payload: dict, **keywords): + self.spawn_keywords.append(keywords) + return super().spawn( + payload, observability_context=keywords["observability_context"] + ) + + +@pytest.mark.parametrize( + ("simulation_run_id", "expected_keyword"), + [ + (UUID("00000000-0000-4000-8000-00000000b001"), { + "simulation_run_id": "00000000-0000-4000-8000-00000000b001" + }), + (None, {}), + ], + ids=["simulation-run", "comparison-path"], +) +def test_segments_are_started_naming_the_simulation_run_of_their_worker( + simulation_run_id, expected_keyword +) -> None: + # Each segment's telemetry names the simulation run it calculates part of. + simulation = planned_simulation(SimulationRole.BASELINE) + function = FunctionRecordingSpawnKeywords(_results(simulation)) + + class FunctionLookup: + @staticmethod + def from_name(_app_name, _function_name): + return function + + runner = Stage12SegmentRunner( + simulation, + app_name="stage12-app", + runtime=cast("ObservabilityRuntime", FakeRuntime()), + simulation_run_id=simulation_run_id, + seams=SegmentRunnerSeams( + modal_module=SimpleNamespace(Function=FunctionLookup), + groups=US_REGION_GROUPS, + sleep=lambda _seconds: None, + ), + ) + + runner.run() + + assert len(function.spawn_keywords) == 20 + for keywords in function.spawn_keywords: + assert set(keywords) == {"observability_context", *expected_keyword} + assert keywords.get("simulation_run_id") == expected_keyword.get( + "simulation_run_id" + ) From cb0f3ba6d9afdaec41d36ac1c80b4924b3f92819 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:05:25 +0400 Subject: [PATCH 02/10] Name the database row and Modal call in each v2 operation's telemetry Each v2 Modal function's operation now carries `modal_call_id` and, when its call works on a detailed-save row, `database_table` (`household_jobs`, `report_runs` or `simulation_runs`) and `database_row_id`. The US simulation worker passes its simulation run to the segments it starts. The keys avoid the words the observability package's filter drops, and the IDs stay off metrics. The observability guide lists the new identifiers. Co-Authored-By: Claude Opus 5.5 --- .../c1-34-v2-telemetry-record-ids.added.md | 1 + docs/engineering/skills/observability.md | 13 +++ .../src/modal/v2_app.py | 55 +++++++++++-- .../stage12_runtime/__init__.py | 10 +++ .../budget_window_scheduling.py | 21 ++--- .../stage12_runtime/fanout.py | 26 +++++- .../stage12_runtime/operation_records.py | 80 +++++++++++++++++++ .../tests/test_stage12_modal_app.py | 5 +- 8 files changed, 184 insertions(+), 27 deletions(-) create mode 100644 changelog.d/c1-34-v2-telemetry-record-ids.added.md create mode 100644 projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/operation_records.py diff --git a/changelog.d/c1-34-v2-telemetry-record-ids.added.md b/changelog.d/c1-34-v2-telemetry-record-ids.added.md new file mode 100644 index 000000000..d328e77f7 --- /dev/null +++ b/changelog.d/c1-34-v2-telemetry-record-ids.added.md @@ -0,0 +1 @@ +Each v2 Modal function's telemetry operation now names the Public API row its call works on and its own Modal call, so an operation that Modal interrupted leads straight to its row: `database_table` (`household_jobs`, `report_runs` for a report run or a household report run, or `simulation_runs`), `database_row_id` and `modal_call_id`. A US segment worker names the simulation run of the worker that started it. The IDs go to the operation's log records and span, not to metrics. diff --git a/docs/engineering/skills/observability.md b/docs/engineering/skills/observability.md index 77fbd9423..62660d31d 100644 --- a/docs/engineering/skills/observability.md +++ b/docs/engineering/skills/observability.md @@ -16,6 +16,19 @@ Use each identifier for its defined scope: | `evaluation_id` | One Stage 12 report | Stage 12 report construction | Yes, as functional report state | | `simulation_execution_id` | One Stage 12 baseline or reform simulation | Stage 12 coordinator | Yes, as functional simulation state | | `submission_claim_id` | One API v1 attempt to acquire ownership of a simulation submission | API v1 | Only where the simulation request is stored | +| `database_table` | The Public API table holding the row one v2 Modal function call works on: `household_jobs`, `report_runs` (a report run or a household report run) or `simulation_runs` | The simulation service, which creates the row | Yes, as the row | +| `database_row_id` | That row's UUID | The simulation service, which creates the row | Yes, as the row | +| `modal_call_id` | One Modal function call of a v2 Modal function | Modal | Yes, where the row records the call that claimed it | + +Each v2 Modal function's operation (`src/modal/v2_app.py`) names its +`modal_call_id` and, when its call works on a detailed-save row, that row's +`database_table` and `database_row_id`; a US segment worker names the +simulation run its simulation worker records. They go to the operation's log +records and span only, never to metrics, where each distinct ID would make a +separate series. The observability package drops every attribute whose key +contains `household`, `reform`, `payload` or `person`, among others, so no key +for these IDs names one. Do not use `job_id` for a v2 call: it names one +annual Modal invocation of the v1 executor. `observability_id` is a canonical UUID string and has diagnostic meaning only. Do not use it for idempotency, authorization, database identity, routing, or diff --git a/projects/policyengine-simulation-executor/src/modal/v2_app.py b/projects/policyengine-simulation-executor/src/modal/v2_app.py index ea7b4b5be..304124a7b 100644 --- a/projects/policyengine-simulation-executor/src/modal/v2_app.py +++ b/projects/policyengine-simulation-executor/src/modal/v2_app.py @@ -330,8 +330,13 @@ def run_single_simulation_segment_us( payload: dict, *, observability_context: dict | None = None, + simulation_run_id: str | None = None, ) -> dict: - """Calculate one US region-group segment and return its artifact descriptor.""" + """Calculate one US region-group segment and return its artifact descriptor. + + ``simulation_run_id`` names the detailed-save simulation run the segment + calculates part of, for its telemetry only. + """ runtime = init_process_observability( service_name="policyengine-stage12-us-segment-worker", service_role="stage12_segment_worker", @@ -347,6 +352,11 @@ def run_single_simulation_segment_us( "runner_name": "stage12", "simulation_role": payload.get("simulation", {}).get("role"), "segment_index": payload.get("segment_index"), + **stage12_runtime.operation_record_attributes( + "simulation_runs", + stage12_runtime.row_id_or_none(simulation_run_id), + modal.current_function_call_id(), + ), }, remote_context=propagated, ): @@ -386,6 +396,9 @@ def run_single_simulation_us( environment=os.getenv("MODAL_ENVIRONMENT", "local"), ) + modal_call_id = modal.current_function_call_id() + simulation_run_id = stage12_runtime.simulation_run_named_by(context) + def segmented_calculator( simulation: PlannedSimulationExecutionInput, cancellation_requested: Callable[[], bool], @@ -395,6 +408,7 @@ def segmented_calculator( app_name=APP_NAME, runtime=runtime, cancellation_requested=cancellation_requested, + simulation_run_id=simulation_run_id, ) try: @@ -404,6 +418,9 @@ def segmented_calculator( attributes={ "runner_name": "stage12", "simulation_role": payload.get("role"), + **stage12_runtime.operation_record_attributes( + "simulation_runs", simulation_run_id, modal_call_id + ), }, remote_context=propagated, ): @@ -415,7 +432,7 @@ def segmented_calculator( segmented_calculator=segmented_calculator, runtime=runtime, ), - modal_call_id=modal.current_function_call_id(), + modal_call_id=modal_call_id, ) finally: runtime.shutdown() @@ -449,6 +466,7 @@ def run_single_simulation_uk( platform="modal", environment=os.getenv("MODAL_ENVIRONMENT", "local"), ) + modal_call_id = modal.current_function_call_id() try: propagated = normalize_observability_context(observability_context) with runtime.operation( @@ -456,6 +474,11 @@ def run_single_simulation_uk( attributes={ "runner_name": "stage12", "simulation_role": payload.get("role"), + **stage12_runtime.operation_record_attributes( + "simulation_runs", + stage12_runtime.simulation_run_named_by(context), + modal_call_id, + ), }, remote_context=propagated, ): @@ -466,7 +489,7 @@ def run_single_simulation_uk( dependencies=stage12_runtime.SimulationWorkerDependencies( runtime=runtime ), - modal_call_id=modal.current_function_call_id(), + modal_call_id=modal_call_id, ) finally: runtime.shutdown() @@ -529,6 +552,11 @@ def coordinate_report( if context.get("production_function_call_id") else "authoritative" ), + **stage12_runtime.operation_record_attributes( + "report_runs", + stage12_runtime.report_run_named_by(context), + coordinator_invocation_id, + ), }, remote_context=propagated, ): @@ -634,10 +662,25 @@ def calculate() -> HouseholdExecutionResult | HouseholdExecutionFailure: msg = "a household call names either a household job or a report run" raise ValueError(msg) propagated = normalize_observability_context(observability_context) + modal_call_id = modal.current_function_call_id() + # A household report run is a ``report_runs`` row. + record_attributes = ( + stage12_runtime.operation_record_attributes( + "report_runs", + stage12_runtime.row_id_or_none(household_report_run_id), + modal_call_id, + ) + if household_report_run_id is not None + else stage12_runtime.operation_record_attributes( + "household_jobs", + stage12_runtime.row_id_or_none(household_job_id), + modal_call_id, + ) + ) result = None with runtime.operation( V2_HOUSEHOLD_STAGES.name(Stage.V2_HOUSEHOLD_EXECUTION), - attributes={"country": country}, + attributes={"country": country, **record_attributes}, remote_context=propagated, ): if household_report_run_id is not None: @@ -647,7 +690,7 @@ def calculate() -> HouseholdExecutionResult | HouseholdExecutionFailure: UUID(household_report_run_id), payload, calculate_request, - modal_call_id=modal.current_function_call_id(), + modal_call_id=modal_call_id, runtime=runtime, ) elif household_job_id is None: @@ -656,7 +699,7 @@ def calculate() -> HouseholdExecutionResult | HouseholdExecutionFailure: result = stage12_runtime.run_household_job( UUID(household_job_id), calculate, - modal_call_id=modal.current_function_call_id(), + modal_call_id=modal_call_id, runtime=runtime, ) if result is None: diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/__init__.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/__init__.py index e946f69a5..44b6a58a0 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/__init__.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/__init__.py @@ -26,6 +26,12 @@ ) from .household_job_recording import run_household_job from .household_report_recording import run_household_report +from .operation_records import ( + operation_record_attributes, + report_run_named_by, + row_id_or_none, + simulation_run_named_by, +) from .partition import ( should_segment_simulation, stage12_region_groups, @@ -72,13 +78,17 @@ "coordinate_report", "coordinate_window_year", "merge_segment_results", + "operation_record_attributes", "redacted_failure_detail", + "report_run_named_by", + "row_id_or_none", "run_household_job", "run_household_report", "run_segmented_simulation", "run_single_simulation", "should_segment_simulation", "simulation_input_sha256", + "simulation_run_named_by", "stage12_region_groups", "stage12_region_groups_for_model", "validate_policy_periods", diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/budget_window_scheduling.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/budget_window_scheduling.py index f4d3b66e2..4f9fcf4c5 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/budget_window_scheduling.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/budget_window_scheduling.py @@ -16,12 +16,10 @@ from __future__ import annotations import logging -from collections.abc import Mapping from dataclasses import dataclass from datetime import UTC, datetime from importlib import import_module from typing import TYPE_CHECKING, Any, Protocol -from uuid import UUID from policyengine_simulation_contract.stage12_execution import ( ReportExecutionInput, @@ -29,8 +27,11 @@ year_calculation, ) +from .operation_records import report_run_named_by + if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Mapping + from uuid import UUID from policyengine_stage12_persistence import ( WindowAdvance, @@ -132,7 +133,7 @@ def coordinate_window_year[CoordinatorResult]( itself fails, nothing is marked and the window is looked up again once the year ends. """ - report_run_id = _report_run_named_by(context) + report_run_id = report_run_named_by(context) window_store = None if report_run_id is None else _built(windows) if report_run_id is None or window_store is None: return coordinate() @@ -211,18 +212,6 @@ def _captured( return None -def _report_run_named_by(context: Mapping[str, Any]) -> UUID | None: - """Return the report run a coordinator call's context names, if any.""" - report_run = context.get("report_run") - if not isinstance(report_run, Mapping): - return None - try: - return UUID(str(report_run["report_run_id"])) - except (KeyError, ValueError): - # The coordinator refuses the malformed context itself. - return None - - def _built(windows: Callable[[], YearWindowWriter]) -> YearWindowWriter | None: """Build the window store, or log why it can't be built and return None.""" try: diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/fanout.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/fanout.py index 570810913..c2a3c9788 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/fanout.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/fanout.py @@ -23,6 +23,7 @@ if TYPE_CHECKING: from collections.abc import Callable, Sequence from types import ModuleType + from uuid import UUID from policyengine_observability import ObservabilityRuntime from policyengine_simulation_contract.stage12_execution import ( @@ -66,21 +67,25 @@ class SegmentRunnerSeams: class Stage12SegmentRunner: """Start, collect, and merge the 20 parts of one logical simulation.""" - def __init__( + def __init__( # noqa: PLR0913 # the simulation, then keyword-only app, runtime, cancellation, run and seams self, simulation: PlannedSimulationExecutionInput, *, app_name: str, runtime: ObservabilityRuntime, cancellation_requested: Callable[[], bool] | None = None, + simulation_run_id: UUID | None = None, seams: SegmentRunnerSeams | None = None, ) -> None: """Look up the segment function deployed in ``app_name``. - ``cancellation_requested`` is checked between polls. + ``cancellation_requested`` is checked between polls. Each segment's + call is passed ``simulation_run_id``, the detailed-save simulation run + this simulation records, for its telemetry. """ seams = seams or SegmentRunnerSeams() self.simulation = simulation + self.simulation_run_id = simulation_run_id self.runtime = runtime self.sleep = seams.sleep self.poll_interval_initial_seconds = seams.poll_interval_seconds @@ -131,6 +136,7 @@ def run(self) -> SimulationCalculation: call = self.child_function.spawn( segment.model_dump(mode="python"), observability_context=self.runtime.capture_context(), + **self._segment_record_keywords(), ) # Kept inside the span, so an interruption as it closes # still finds the call to cancel. @@ -223,6 +229,15 @@ def _collect(self, handles: list[tuple[int, Any]]) -> list[object]: raise RuntimeError(msg) return [result for result in results if result is not None] + def _segment_record_keywords(self) -> dict[str, str]: + """Return the keyword naming the simulation run, when there is one. + + A comparison-path simulation records no simulation run. + """ + if self.simulation_run_id is None: + return {} + return {"simulation_run_id": str(self.simulation_run_id)} + def _wait_with_cancellation( self, seconds: float, @@ -265,11 +280,16 @@ def run_segmented_simulation( app_name: str, runtime: ObservabilityRuntime, cancellation_requested: Callable[[], bool] | None = None, + simulation_run_id: UUID | None = None, ) -> SimulationCalculation: - """Calculate ``simulation`` as region-group segments in ``app_name``; merge them.""" + """Calculate ``simulation`` as region-group segments in ``app_name``; merge them. + + Each segment's telemetry names ``simulation_run_id``, when given. + """ return Stage12SegmentRunner( simulation, app_name=app_name, runtime=runtime, cancellation_requested=cancellation_requested, + simulation_run_id=simulation_run_id, ).run() diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/operation_records.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/operation_records.py new file mode 100644 index 000000000..a380dbe98 --- /dev/null +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/operation_records.py @@ -0,0 +1,80 @@ +"""Name the database row and Modal call of a v2 Modal function's operation. + +Each v2 Modal function opens one telemetry operation. Its attributes reach +the operation's log records and span, but not its metrics, so an operation +that Modal interrupted in production leads straight to the Public API row it +was working on: + +- ``database_table``: the table holding that row, ``household_jobs``, + ``report_runs`` (a report run, or a household report run) or + ``simulation_runs``; +- ``database_row_id``: the row's UUID; +- ``modal_call_id``: the function's own Modal function call ID. + +The observability package drops every attribute whose key contains +``household``, among other words, so no key here names one. A call that +works on no detailed-save row (a comparison-path call, a standalone +simulation, a release validation's household calculation) names only its +Modal call. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Literal +from uuid import UUID + +DATABASE_TABLE_ATTRIBUTE = "database_table" +DATABASE_ROW_ID_ATTRIBUTE = "database_row_id" +MODAL_CALL_ID_ATTRIBUTE = "modal_call_id" + +DatabaseTable = Literal["household_jobs", "report_runs", "simulation_runs"] + + +def operation_record_attributes( + table: DatabaseTable, + row_id: UUID | None, + modal_call_id: str | None, +) -> dict[str, str]: + """Return the operation attributes naming ``row_id`` in ``table`` and the call. + + A missing row or Modal call ID is left out. + """ + attributes: dict[str, str] = {} + if row_id is not None: + attributes[DATABASE_TABLE_ATTRIBUTE] = table + attributes[DATABASE_ROW_ID_ATTRIBUTE] = str(row_id) + if modal_call_id: + attributes[MODAL_CALL_ID_ATTRIBUTE] = modal_call_id + return attributes + + +def row_id_or_none(value: object) -> UUID | None: + """Return ``value`` as a UUID, or None when it is missing or not a UUID. + + Telemetry names only a well-formed ID; the function refuses a malformed + one itself. + """ + if value is None: + return None + try: + return UUID(str(value)) + except ValueError: + return None + + +def report_run_named_by(context: object) -> UUID | None: + """Return the report run a coordinator call's context names, if any.""" + if not isinstance(context, Mapping): + return None + report_run = context.get("report_run") + if not isinstance(report_run, Mapping): + return None + return row_id_or_none(report_run.get("report_run_id")) + + +def simulation_run_named_by(context: object) -> UUID | None: + """Return the simulation run a simulation worker call's context names, if any.""" + if not isinstance(context, Mapping): + return None + return row_id_or_none(context.get("simulation_run_id")) diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py b/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py index 32476087a..3d629b2a8 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py @@ -94,8 +94,9 @@ def model_dump(self, *, mode): ) # The US and UK workers pass their own Modal call ID to the runtime, so the - # stand-in ``modal`` module needs one; a test that set its own keeps it. - if not hasattr(module.modal, "current_function_call_id"): + # stand-in ``modal`` module, which answers None, needs one; a test that set + # its own keeps it. + if module.modal.current_function_call_id() is None: monkeypatch.setattr( module.modal, "current_function_call_id", From b92b10533af53039a63c3042b0a079725d5cf428 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:10:16 +0400 Subject: [PATCH 03/10] Shut the simulation workers' runtime down if their IDs can't be read, after review round 1 The US and UK simulation workers read their Modal call ID and simulation run inside `try`, as the other v2 functions do. The observability guide now says `database_table` is a telemetry-only name and that no row records a segment worker's own Modal call. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/skills/observability.md | 6 ++--- .../src/modal/v2_app.py | 8 ++++--- .../tests/test_stage12_modal_app.py | 24 +++++++++++++++++++ 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/docs/engineering/skills/observability.md b/docs/engineering/skills/observability.md index 62660d31d..81f49c14e 100644 --- a/docs/engineering/skills/observability.md +++ b/docs/engineering/skills/observability.md @@ -16,9 +16,9 @@ Use each identifier for its defined scope: | `evaluation_id` | One Stage 12 report | Stage 12 report construction | Yes, as functional report state | | `simulation_execution_id` | One Stage 12 baseline or reform simulation | Stage 12 coordinator | Yes, as functional simulation state | | `submission_claim_id` | One API v1 attempt to acquire ownership of a simulation submission | API v1 | Only where the simulation request is stored | -| `database_table` | The Public API table holding the row one v2 Modal function call works on: `household_jobs`, `report_runs` (a report run or a household report run) or `simulation_runs` | The simulation service, which creates the row | Yes, as the row | -| `database_row_id` | That row's UUID | The simulation service, which creates the row | Yes, as the row | -| `modal_call_id` | One Modal function call of a v2 Modal function | Modal | Yes, where the row records the call that claimed it | +| `database_table` | The Public API table holding the row one v2 Modal function call works on: `household_jobs`, `report_runs` (a report run or a household report run) or `simulation_runs` | The v2 Modal function, as a fixed name for its kind of call | No: telemetry only | +| `database_row_id` | That row's UUID | The simulation service, which creates the row | Yes, as the row's ID | +| `modal_call_id` | One Modal function call of a v2 Modal function | Modal | Only where the row records the call that claimed it; a segment worker's call is recorded nowhere, and its `database_row_id` names the simulation run its simulation worker's call claimed | Each v2 Modal function's operation (`src/modal/v2_app.py`) names its `modal_call_id` and, when its call works on a detailed-save row, that row's diff --git a/projects/policyengine-simulation-executor/src/modal/v2_app.py b/projects/policyengine-simulation-executor/src/modal/v2_app.py index 304124a7b..520f32c98 100644 --- a/projects/policyengine-simulation-executor/src/modal/v2_app.py +++ b/projects/policyengine-simulation-executor/src/modal/v2_app.py @@ -396,8 +396,8 @@ def run_single_simulation_us( environment=os.getenv("MODAL_ENVIRONMENT", "local"), ) - modal_call_id = modal.current_function_call_id() - simulation_run_id = stage12_runtime.simulation_run_named_by(context) + # Read inside ``try``, so the runtime is shut down whatever happens. + simulation_run_id = None def segmented_calculator( simulation: PlannedSimulationExecutionInput, @@ -412,6 +412,8 @@ def segmented_calculator( ) try: + modal_call_id = modal.current_function_call_id() + simulation_run_id = stage12_runtime.simulation_run_named_by(context) propagated = normalize_observability_context(observability_context) with runtime.operation( STAGE12_SIMULATION_STAGES.name(Stage.STAGE12_SIMULATION_EXECUTION), @@ -466,8 +468,8 @@ def run_single_simulation_uk( platform="modal", environment=os.getenv("MODAL_ENVIRONMENT", "local"), ) - modal_call_id = modal.current_function_call_id() try: + modal_call_id = modal.current_function_call_id() propagated = normalize_observability_context(observability_context) with runtime.operation( STAGE12_SIMULATION_STAGES.name(Stage.STAGE12_SIMULATION_EXECUTION), diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py b/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py index 3d629b2a8..f35a91b9c 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py @@ -1185,3 +1185,27 @@ def run_single_simulation(payload, context, **keywords): (segmented_run,) = segmented_runs assert segmented_run["simulation_run_id"] == UUID(SIMULATION_RUN_ID) + + +@pytest.mark.parametrize( + "function_name", ["run_single_simulation_us", "run_single_simulation_uk"] +) +def test_simulation_workers_close_runtime_when_their_modal_call_id_is_unreadable( + monkeypatch, function_name +) -> None: + module = _load(monkeypatch) + runtime = RecordingRuntime() + monkeypatch.setattr(module, "init_process_observability", lambda **_: runtime) + + def unreadable_call_id(): + msg = "Modal call ID is unreadable" + raise RuntimeError(msg) + + monkeypatch.setattr(module.modal, "current_function_call_id", unreadable_call_id) + + with pytest.raises(RuntimeError, match="unreadable"): + getattr(module, function_name)( + {"role": "baseline"}, {}, observability_context=None + ) + + assert runtime.shutdown_calls == 1 From deb60fab88a8d529c6d30ce7887fa9c3f045a53c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:32:00 +0400 Subject: [PATCH 04/10] Test that every Simulation Entrypoint log call's fields reach the logs (failing) The observability runtime's log handler emits only a record's `policyengine_attributes`, and drops keys naming a household. A `log_extra` helper in the shared observability library should pass each field there; every Entrypoint log call with `extra=` should use it, with no field named by a word the package drops; and the household job logs should name their job as a `household_jobs` row. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_log_records.py | 121 ++++++++++++++++++ .../tests/test_budget_window_submissions.py | 11 +- .../tests/test_household_calculations.py | 52 ++++++++ .../tests/test_log_fields.py | 94 ++++++++++++++ 4 files changed, 274 insertions(+), 4 deletions(-) create mode 100644 libs/policyengine-simulation-observability/tests/test_log_records.py create mode 100644 projects/policyengine-simulation-entry/tests/test_log_fields.py diff --git a/libs/policyengine-simulation-observability/tests/test_log_records.py b/libs/policyengine-simulation-observability/tests/test_log_records.py new file mode 100644 index 000000000..5232fffc7 --- /dev/null +++ b/libs/policyengine-simulation-observability/tests/test_log_records.py @@ -0,0 +1,121 @@ +"""Standard-library log fields that reach the observability runtime's records.""" + +from __future__ import annotations + +import logging +from uuid import UUID + +import pytest +from policyengine_observability import ( + CustomLogDestination, + DeploymentIdentity, + LoggingConfig, + ObservabilityConfig, + OTelConfig, + ServiceIdentity, + configure, +) + +ROW_ID = UUID("00000000-0000-4000-8000-00000000a001") + + +@pytest.fixture +def written_records(): + """Configure a runtime that keeps every record, standard-library ones too.""" + records: list[dict] = [] + + class ListRecordWriter: + def write(self, record: dict) -> None: + records.append(record) + + runtime = configure( + ObservabilityConfig( + service=ServiceIdentity( + name="policyengine-simulation-observability-test", + namespace="policyengine.test", + version="test", + role="test", + ), + deployment=DeploymentIdentity(environment="test", platform="local"), + logging=LoggingConfig( + destinations=( + CustomLogDestination( + name="test-records", + writer_factory=ListRecordWriter, + delivery="inline", + ), + ), + capture_standard_library=True, + ), + otel=OTelConfig(enabled=False), + ) + ) + yield records + runtime.shutdown() + + +def _record_with_message(records: list[dict], message: str) -> dict: + (record,) = [item for item in records if item.get("message") == message] + return record + + +def test_fields_in_log_extra_reach_the_emitted_record(written_records) -> None: + from policyengine_simulation_observability.log_records import ( + database_row_attributes, + log_extra, + ) + + logging.getLogger("policyengine_simulation_entry.test").warning( + "household_call_unconfirmed", + extra=log_extra( + error_type="ConnectionError", + **database_row_attributes("household_jobs", ROW_ID), + ), + ) + + record = _record_with_message(written_records, "household_call_unconfirmed") + assert record["attributes"] == { + "error_type": "ConnectionError", + "database_table": "household_jobs", + "database_row_id": str(ROW_ID), + } + + +def test_log_extra_keeps_its_fields_on_the_standard_library_record(caplog) -> None: + # Other handlers, and tests, read them there. + from policyengine_simulation_observability.log_records import log_extra + + with caplog.at_level(logging.INFO): + logging.getLogger("policyengine_simulation_entry.test").info( + "budget_window_year_deferred", extra=log_extra(year=2027) + ) + + (record,) = caplog.records + assert record.year == 2027 + assert record.policyengine_attributes == {"year": 2027} + + +def test_plain_extra_fields_are_not_emitted(written_records) -> None: + # Why ``log_extra`` exists: the runtime's log handler keeps only + # ``policyengine_attributes``. + logging.getLogger("policyengine_simulation_entry.test").warning( + "economy_state_unreadable", extra={"error_type": "OperationalError"} + ) + + record = _record_with_message(written_records, "economy_state_unreadable") + assert "attributes" not in record + + +def test_a_household_key_is_dropped_even_in_policyengine_attributes( + written_records, +) -> None: + # Why rows are named by ``database_table`` and ``database_row_id``. + from policyengine_simulation_observability.log_records import log_extra + + logging.getLogger("policyengine_simulation_entry.test").warning( + "household_job_not_failed", + extra=log_extra(household_job_id=str(ROW_ID), error_type="OperationalError"), + ) + + record = _record_with_message(written_records, "household_job_not_failed") + assert record["attributes"] == {"error_type": "OperationalError"} diff --git a/projects/policyengine-simulation-entry/tests/test_budget_window_submissions.py b/projects/policyengine-simulation-entry/tests/test_budget_window_submissions.py index 1717f11d8..c8eb34f91 100644 --- a/projects/policyengine-simulation-entry/tests/test_budget_window_submissions.py +++ b/projects/policyengine-simulation-entry/tests/test_budget_window_submissions.py @@ -496,7 +496,10 @@ def test_a_later_year_is_deferred_without_a_call_once_the_wait_has_run_out( if record.getMessage() == "budget_window_year_deferred" ] second_year = store.windows[0].years[1] - assert (deferral.year, deferral.record_id) == ( - 2027, - str(second_year.report_run_id), - ) + # These fields reach the emitted log record; a report run is a + # ``report_runs`` row. + assert deferral.policyengine_attributes == { + "year": 2027, + "database_table": "report_runs", + "database_row_id": str(second_year.report_run_id), + } diff --git a/projects/policyengine-simulation-entry/tests/test_household_calculations.py b/projects/policyengine-simulation-entry/tests/test_household_calculations.py index dd7aa455c..656deeda8 100644 --- a/projects/policyengine-simulation-entry/tests/test_household_calculations.py +++ b/projects/policyengine-simulation-entry/tests/test_household_calculations.py @@ -7,6 +7,8 @@ from __future__ import annotations import asyncio +import logging +from contextlib import suppress from dataclasses import dataclass, field from datetime import UTC, datetime from typing import TYPE_CHECKING, Any @@ -802,3 +804,53 @@ async def test_a_job_that_cannot_be_read_is_unavailable(): await service.read(JOB_ID) assert raised.value.code == "database_unavailable" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("store", "caller", "event", "error_type"), + [ + ( + FakeStore(), + FakeCaller(failure=ConnectionError("Modal is unreachable")), + "household_call_unconfirmed", + "ConnectionError", + ), + ( + FakeStore(fail_failure=DATABASE_ERROR), + FakeCaller(failure=ConnectionError("Modal is unreachable")), + "household_job_not_failed", + "OperationalError", + ), + ( + FakeStore(claimed_by_function=True, job_read_failure=DATABASE_ERROR), + FakeCaller(failure=ConnectionError("Modal is unreachable")), + "household_job_not_read", + "OperationalError", + ), + ( + FakeStore(record_failure=DATABASE_ERROR), + FakeCaller(), + "household_call_not_recorded", + "OperationalError", + ), + ], + ids=["unconfirmed", "not-failed", "not-read", "not-recorded"], +) +async def test_a_household_jobs_log_names_its_row_in_the_emitted_fields( + caplog, store, caller, event, error_type +): + # The runtime's log handler emits only ``policyengine_attributes``, and + # drops a key naming a household, so the job is named by its table and row. + service, _, _, _ = _service(store, caller=caller) + + # A failed job is answered with an error; the others are accepted. + with caplog.at_level(logging.INFO), suppress(HouseholdCalculationUnavailableError): + await service.submit(_request(), observability_context=None) + + (record,) = [item for item in caplog.records if item.getMessage() == event] + assert record.policyengine_attributes == { + "database_table": "household_jobs", + "database_row_id": str(JOB_ID), + "error_type": error_type, + } diff --git a/projects/policyengine-simulation-entry/tests/test_log_fields.py b/projects/policyengine-simulation-entry/tests/test_log_fields.py new file mode 100644 index 000000000..029191d6b --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_log_fields.py @@ -0,0 +1,94 @@ +"""Every Simulation Entrypoint log call's fields reach the emitted log record. + +The observability runtime's log handler keeps only a record's +``policyengine_attributes``, and drops each of those whose key contains +``household``, ``reform``, ``payload`` or ``person``, among others. +""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import policyengine_simulation_entry +from policyengine_observability import ( + DeploymentIdentity, + ObservabilityConfig, + ServiceIdentity, +) +from policyengine_observability.schema import normalize_attributes + +PACKAGE_DIRECTORY = Path(policyengine_simulation_entry.__file__).parent +LOG_METHODS = frozenset( + {"debug", "info", "warning", "error", "exception", "critical", "log"} +) + + +def _log_calls_with_extra() -> list[tuple[str, ast.Call, ast.expr]]: + """Return each logging call in the package that passes ``extra=``.""" + calls = [] + for path in sorted(PACKAGE_DIRECTORY.rglob("*.py")): + tree = ast.parse(path.read_text(), filename=str(path)) + for node in ast.walk(tree): + if not ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr in LOG_METHODS + ): + continue + for keyword in node.keywords: + if keyword.arg == "extra": + location = f"{path.relative_to(PACKAGE_DIRECTORY)}:{node.lineno}" + calls.append((location, node, keyword.value)) + return calls + + +def _is_log_extra_call(expression: ast.expr) -> bool: + return ( + isinstance(expression, ast.Call) + and isinstance(expression.func, ast.Name) + and expression.func.id == "log_extra" + ) + + +def test_the_package_has_log_calls_with_fields() -> None: + # Guards the next tests against finding nothing to check. + assert len(_log_calls_with_extra()) >= 30 + + +def test_every_log_call_passes_its_fields_through_log_extra() -> None: + plain_extra = [ + location + for location, _, extra in _log_calls_with_extra() + if not _is_log_extra_call(extra) + ] + + assert plain_extra == [] + + +def test_no_log_field_is_named_with_a_word_the_package_drops() -> None: + config = ObservabilityConfig( + service=ServiceIdentity( + name="policyengine-simulation-entry-test", + namespace="policyengine.test", + version="test", + role="test", + ), + deployment=DeploymentIdentity(environment="test", platform="local"), + ) + field_names = { + (location, keyword.arg) + for location, _, extra in _log_calls_with_extra() + if isinstance(extra, ast.Call) + for keyword in extra.keywords + if keyword.arg is not None + } + assert field_names + + dropped = sorted( + (location, name) + for location, name in field_names + if not normalize_attributes({name: "value"}, config)[0] + ) + + assert dropped == [] From 9e94b7d5f9f2262d592fb0eae52439f053a7f9bf Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:35:34 +0400 Subject: [PATCH 05/10] Emit every Simulation Entrypoint log call's fields The observability runtime's log handler emits only a log record's `policyengine_attributes`, so 31 of the Entrypoint's 33 log calls, which passed their fields only as `extra=` keys, wrote records naming no job, report, run or error type. A new `log_extra` helper in the shared observability library puts the fields in both places, and every log call now uses it. Rows are named with `database_table` and `database_row_id` (`database_row_attributes`), since the runtime drops a key such as `household_job_id`; the v2 Modal functions' operations take the same names from the shared module. A budget window's started years are joined into one string, since the runtime drops lists. Co-Authored-By: Claude Opus 5.5 --- .../c1-34-v2-telemetry-record-ids.fixed.md | 1 + docs/engineering/skills/observability.md | 15 ++++++- .../log_records.py | 38 +++++++++++++++++ .../src/policyengine_simulation_entry/app.py | 29 ++++++------- .../src/policyengine_simulation_entry/auth.py | 3 +- .../budget_window_submissions.py | 35 +++++++++------- .../economy_routes.py | 3 +- .../economy_submission_errors.py | 7 +++- .../economy_submissions.py | 40 +++++++++--------- .../entry_runtime.py | 23 ++++++----- .../household_calculations.py | 41 +++++++++++-------- .../household_report_submissions.py | 31 ++++++++------ .../route_support.py | 4 +- .../stage12_runtime/operation_records.py | 14 +++---- 14 files changed, 185 insertions(+), 99 deletions(-) create mode 100644 changelog.d/c1-34-v2-telemetry-record-ids.fixed.md create mode 100644 libs/policyengine-simulation-observability/src/policyengine_simulation_observability/log_records.py diff --git a/changelog.d/c1-34-v2-telemetry-record-ids.fixed.md b/changelog.d/c1-34-v2-telemetry-record-ids.fixed.md new file mode 100644 index 000000000..51fa8218a --- /dev/null +++ b/changelog.d/c1-34-v2-telemetry-record-ids.fixed.md @@ -0,0 +1 @@ +The Simulation Entrypoint's log records now carry their fields. 31 of its 33 log calls passed their fields only as standard-library `extra=` keys, which the observability runtime doesn't emit, so their records in Cloud Logging named no job, report, run or error type. Every log call now passes its fields through `log_extra` in the shared observability library, and names a Public API row with `database_table` (`household_jobs`, `reports`, `report_runs` or `simulations`) and `database_row_id`: a key such as `household_job_id` would be dropped by the runtime's key filter. A budget window's `started_years` are now joined into one string, since the runtime drops lists. diff --git a/docs/engineering/skills/observability.md b/docs/engineering/skills/observability.md index 81f49c14e..b297b1fd0 100644 --- a/docs/engineering/skills/observability.md +++ b/docs/engineering/skills/observability.md @@ -16,7 +16,7 @@ Use each identifier for its defined scope: | `evaluation_id` | One Stage 12 report | Stage 12 report construction | Yes, as functional report state | | `simulation_execution_id` | One Stage 12 baseline or reform simulation | Stage 12 coordinator | Yes, as functional simulation state | | `submission_claim_id` | One API v1 attempt to acquire ownership of a simulation submission | API v1 | Only where the simulation request is stored | -| `database_table` | The Public API table holding the row one v2 Modal function call works on: `household_jobs`, `report_runs` (a report run or a household report run) or `simulation_runs` | The v2 Modal function, as a fixed name for its kind of call | No: telemetry only | +| `database_table` | The Public API table holding the row a v2 Modal function call, or a Simulation Entrypoint log record, is about: `household_jobs`, `report_runs` (a report run or a household report run), `simulation_runs`, `reports` or `simulations` | The code writing the record, as a fixed name | No: telemetry only | | `database_row_id` | That row's UUID | The simulation service, which creates the row | Yes, as the row's ID | | `modal_call_id` | One Modal function call of a v2 Modal function | Modal | Only where the row records the call that claimed it; a segment worker's call is recorded nowhere, and its `database_row_id` names the simulation run its simulation worker's call claimed | @@ -30,6 +30,19 @@ contains `household`, `reform`, `payload` or `person`, among others, so no key for these IDs names one. Do not use `job_id` for a v2 call: it names one annual Modal invocation of the v1 executor. +## Standard-library log fields + +The observability runtime's standard-library log handler emits only a log +record's `policyengine_attributes`: any other key passed in `extra=` stays on +the Python `LogRecord` and never reaches Cloud Logging. Pass every log call's +fields through `log_extra(...)` from +`policyengine_simulation_observability.log_records`, which puts them in both +places, and name a row with `database_row_attributes(table, row_id)`. Each +field must be a string, a number or a boolean; the runtime drops a list. In +the Simulation Entrypoint, `tests/test_log_fields.py` checks that every log call +with `extra=` uses `log_extra` and that no literal field name contains a word +the package drops. + `observability_id` is a canonical UUID string and has diagnostic meaning only. Do not use it for idempotency, authorization, database identity, routing, or business logic. Do not create one for health checks, version queries, invalid diff --git a/libs/policyengine-simulation-observability/src/policyengine_simulation_observability/log_records.py b/libs/policyengine-simulation-observability/src/policyengine_simulation_observability/log_records.py new file mode 100644 index 000000000..ea691aae9 --- /dev/null +++ b/libs/policyengine-simulation-observability/src/policyengine_simulation_observability/log_records.py @@ -0,0 +1,38 @@ +"""Telemetry fields that name a Public API row, and log fields that are emitted. + +``policyengine-observability``'s standard-library log handler emits only a +log record's ``policyengine_attributes``; every other key passed in +``extra=`` stays on the standard-library record alone. It then drops each +attribute whose key contains ``household``, ``reform``, ``payload`` or +``person``, among other words. So a row is named by the table holding it +(``database_table``) and its UUID (``database_row_id``), never by a key such +as ``household_job_id``. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from uuid import UUID + +DATABASE_TABLE_ATTRIBUTE = "database_table" +DATABASE_ROW_ID_ATTRIBUTE = "database_row_id" +MODAL_CALL_ID_ATTRIBUTE = "modal_call_id" + +LogFieldValue = str | int | float | bool + + +def database_row_attributes(table: str, row_id: UUID) -> dict[str, str]: + """Return the attributes naming row ``row_id`` of the Public API's ``table``.""" + return {DATABASE_TABLE_ATTRIBUTE: table, DATABASE_ROW_ID_ATTRIBUTE: str(row_id)} + + +def log_extra(**fields: LogFieldValue) -> dict[str, object]: + """Return a logging call's ``extra`` whose ``fields`` reach the emitted record. + + Each field also stays on the standard-library record, where other + handlers and tests read it. A field must be a string, a number or a + boolean: the runtime drops anything else. + """ + return {**fields, "policyengine_attributes": dict(fields)} diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py index ba75fa1d8..53a824dbe 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py @@ -38,6 +38,7 @@ OBSERVABILITY_ID_HEADER, normalize_observability_id, ) +from policyengine_simulation_observability.log_records import log_extra from policyengine_simulation_observability.observability import ( init_simulation_observability, ) @@ -174,12 +175,12 @@ async def request_context( except Exception: logger.exception( "simulation_entry_unhandled_request", - extra={ - "request_id": request_id, - "method": request.method, - "path": route_template(request), + extra=log_extra( + request_id=request_id, + method=request.method, + path=route_template(request), **request_identifiers(request), - }, + ), ) response = Response( content="Internal Server Error", @@ -257,10 +258,10 @@ async def submit_temporary_stage12_report( except Exception as error: # noqa: BLE001 # every integration failure becomes a 503 logger.error( # noqa: TRY400 # Stage 12 failure logs carry only the error type; a traceback could expose request or calculation data "stage12_direct_submission_failed", - extra={ - "request_id": request.state.request_id, - "error_type": type(error).__name__, - }, + extra=log_extra( + request_id=request.state.request_id, + error_type=type(error).__name__, + ), ) return _stage12_unavailable_response( "Stage 12 direct execution is unavailable." @@ -314,11 +315,11 @@ async def get_temporary_stage12_report( except Exception as error: # noqa: BLE001 # any status-read failure becomes a 503 logger.error( # noqa: TRY400 # Stage 12 failure logs carry only the error type; a traceback could expose request or calculation data "stage12_direct_status_failed", - extra={ - "request_id": request.state.request_id, - "evaluation_id": str(evaluation_id), - "error_type": type(error).__name__, - }, + extra=log_extra( + request_id=request.state.request_id, + evaluation_id=str(evaluation_id), + error_type=type(error).__name__, + ), ) return _stage12_unavailable_response( "Stage 12 report status is unavailable." diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/auth.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/auth.py index 6f865dd18..4c6351c58 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/auth.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/auth.py @@ -11,6 +11,7 @@ import jwt from fastapi import Depends, HTTPException, status from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer +from policyengine_simulation_observability.log_records import log_extra from policyengine_simulation_entry.schemas import CallerIdentity @@ -79,7 +80,7 @@ def __call__( reason = type(error).__name__ logger.info( "invalid_simulation_entry_bearer_token", - extra={"error_type": reason}, + extra=log_extra(error_type=reason), ) runtime.event( "simulation_entry_auth_rejected", diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/budget_window_submissions.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/budget_window_submissions.py index ba8d84d64..e62752f6c 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/budget_window_submissions.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/budget_window_submissions.py @@ -38,6 +38,10 @@ Stage12InvocationContext, year_calculation, ) +from policyengine_simulation_observability.log_records import ( + database_row_attributes, + log_extra, +) from policyengine_simulation_persistence import ( EconomyReportDefinition, KeyedBudgetWindow, @@ -302,22 +306,21 @@ async def _start_first_years( # noqa: PLR0913 # the call, then the window it s except SQLAlchemyError as error: logger.warning( "budget_window_not_started", - extra={ - "record": "report", - "record_id": str(report_id), - "error_type": type(error).__name__, - }, + extra=log_extra( + error_type=type(error).__name__, + **database_row_attributes("reports", report_id), + ), ) return ReportRunStatus.PENDING if advance.status is RunStatus.FAILED: # The budget-window store logs the refused call's error type. logger.error( "budget_window_call_unconfirmed", - extra={ - "record": "report", - "record_id": str(report_id), - "started_years": list(advance.started_years), - }, + extra=log_extra( + # The runtime emits no list, so the years are joined. + started_years=",".join(str(year) for year in advance.started_years), + **database_row_attributes("reports", report_id), + ), ) refused = call_refused( "report", report_id, rerun_id=window_run_id if rerun else None @@ -372,7 +375,10 @@ def __call__(self, year_to_start: YearToStart, /) -> str: later_year = year_to_start.year != self._first_year remaining_seconds = self._wait_ends_at - time.monotonic() if later_year and remaining_seconds <= 0: - logger.info("budget_window_year_deferred", extra=_deferral(year_to_start)) + logger.info( + "budget_window_year_deferred", + extra=log_extra(**_deferral(year_to_start)), + ) raise YearStartDeferred payload, context = self._calculation(year_to_start) started = asyncio.run_coroutine_threadsafe( @@ -390,7 +396,8 @@ def __call__(self, year_to_start: YearToStart, /) -> str: started.cancel() if later_year: logger.info( - "budget_window_year_deferred", extra=_deferral(year_to_start) + "budget_window_year_deferred", + extra=log_extra(**_deferral(year_to_start)), ) raise YearStartDeferred from None raise @@ -414,11 +421,11 @@ def _calculation( return year_report.model_dump(mode="json"), year_context.model_dump(mode="json") -def _deferral(year_to_start: YearToStart) -> dict[str, Any]: +def _deferral(year_to_start: YearToStart) -> dict[str, int | str]: """Return a deferred year's log fields: its year and its report run.""" return { "year": year_to_start.year, - "record_id": str(year_to_start.run_ids.report_run_id), + **database_row_attributes("report_runs", year_to_start.run_ids.report_run_id), } diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_routes.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_routes.py index 2462d6023..e859a1676 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_routes.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_routes.py @@ -52,6 +52,7 @@ from fastapi import APIRouter, Depends, Request, status from fastapi.responses import JSONResponse +from policyengine_simulation_observability.log_records import log_extra from policyengine_simulation_entry.budget_window_api_models import ( BudgetWindowReportCreated, @@ -339,7 +340,7 @@ def _outcome_unknown_response( written = f"the {error.record}'s new run" if error.rerun else f"the {error.record}" logger.error( "economy_record_outcome_unknown", - extra={**fields, "policyengine_attributes": fields}, + extra=log_extra(**fields), ) return error_response( status.HTTP_500_INTERNAL_SERVER_ERROR, diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_submission_errors.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_submission_errors.py index 10c882ee4..3c6f62a79 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_submission_errors.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_submission_errors.py @@ -15,7 +15,7 @@ from contextlib import contextmanager from dataclasses import dataclass -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING, Final, Literal from policyengine_simulation_persistence import ( MissingPolicyParametersError, @@ -64,6 +64,11 @@ } # What a submission creates, or a re-run calculates again. RecordName = Literal["report", "simulation"] +# The Public API table holding each record, as telemetry names it. +RECORD_TABLES: Final[dict[RecordName, str]] = { + "report": "reports", + "simulation": "simulations", +} @dataclass(frozen=True) diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_submissions.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_submissions.py index 644b99d41..46e1561c8 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_submissions.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_submissions.py @@ -48,6 +48,10 @@ from uuid import UUID, uuid4 from policyengine_simulation_contract.stage12_execution import ReportRunIds +from policyengine_simulation_observability.log_records import ( + database_row_attributes, + log_extra, +) from policyengine_simulation_persistence import ( EconomyRecordOutcomeUnknownError, EconomyRecordReferenceError, @@ -81,6 +85,7 @@ stored_options, ) from policyengine_simulation_entry.economy_submission_errors import ( + RECORD_TABLES, REQUEST_BODY, EconomySubmissionOutcomeUnknownError, EconomySubmissionRefusedError, @@ -412,7 +417,7 @@ async def _read_state[State](self, read: Callable[[], State]) -> State: except SQLAlchemyError as error: logger.warning( "economy_state_unreadable", - extra={"error_type": type(error).__name__}, + extra=log_extra(error_type=type(error).__name__), ) raise _database_unavailable() from None @@ -444,7 +449,7 @@ async def _default_worker(self, country: str) -> EconomyWorker: except VersionManifestUnavailableError as error: logger.warning( "economy_manifest_unavailable", - extra={"error_type": type(error.__cause__).__name__}, + extra=log_extra(error_type=type(error.__cause__).__name__), ) raise EconomySubmissionUnavailableError( code="manifest_unavailable", @@ -464,7 +469,7 @@ async def _read_inputs[Inputs]( except SQLAlchemyError as error: logger.warning( "economy_inputs_unreadable", - extra={"error_type": type(error).__name__}, + extra=log_extra(error_type=type(error).__name__), ) raise _database_unavailable() from None except Exception as error: @@ -503,7 +508,7 @@ async def _create[Created]( except (SharedRowNotSettledError, SQLAlchemyError) as error: logger.warning( "economy_records_not_created", - extra={"error_type": type(error).__name__}, + extra=log_extra(error_type=type(error).__name__), ) raise _database_unavailable() from None @@ -621,11 +626,10 @@ async def _fail_unconfirmed( # The log names the error type only; its text could repeat request data. logger.error( "economy_call_unconfirmed", - extra={ - "record": record, - "record_id": str(record_id), - "error_type": type(error).__name__, - }, + extra=log_extra( + error_type=type(error).__name__, + **database_row_attributes(RECORD_TABLES[record], record_id), + ), ) message = ( f"modal_call_refused: Modal did not accept the {record}'s call " @@ -636,11 +640,10 @@ async def _fail_unconfirmed( except SQLAlchemyError as write_error: logger.warning( "economy_run_not_failed", - extra={ - "record": record, - "record_id": str(record_id), - "error_type": type(write_error).__name__, - }, + extra=log_extra( + error_type=type(write_error).__name__, + **database_row_attributes(RECORD_TABLES[record], record_id), + ), ) return False @@ -660,11 +663,10 @@ async def _record_call( except SQLAlchemyError as error: logger.warning( "economy_call_not_recorded", - extra={ - "record": record, - "record_id": str(record_id), - "error_type": type(error).__name__, - }, + extra=log_extra( + error_type=type(error).__name__, + **database_row_attributes(RECORD_TABLES[record], record_id), + ), ) return None return None if status is None else str(status) diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/entry_runtime.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/entry_runtime.py index cbe7a2258..0fee42245 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/entry_runtime.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/entry_runtime.py @@ -16,6 +16,7 @@ normalize_observability_id, resolve_observability_id, ) +from policyengine_simulation_observability.log_records import log_extra from pydantic import BaseModel, TypeAdapter, ValidationError from policyengine_simulation_entry.backend import ( @@ -123,9 +124,9 @@ def route_template(request: Request) -> str: return getattr(request.scope.get("route"), "path", request.url.path) -def request_identifiers(request: Request) -> RequestIdentifiers: - """Return the job, batch and evaluation IDs in the request's path.""" - identifiers: RequestIdentifiers = {} +def request_identifiers(request: Request) -> dict[str, str]: + """Return the job, batch and evaluation IDs in the request's path, as log fields.""" + identifiers: dict[str, str] = {} job_id = request.path_params.get("job_id") batch_job_id = request.path_params.get("batch_job_id") evaluation_id = request.path_params.get("evaluation_id") @@ -486,18 +487,18 @@ async def dispatch_comparison_run( except TimeoutError: logger.error( # noqa: TRY400 # an expected timeout, fully described by timeout_seconds; the traceback would show only asyncio cancellation "stage12_comparison_dispatch_timed_out", - extra={ - "request_id": request.state.request_id, - "timeout_seconds": STAGE12_MODAL_SUBMISSION_TIMEOUT_SECONDS, - }, + extra=log_extra( + request_id=request.state.request_id, + timeout_seconds=STAGE12_MODAL_SUBMISSION_TIMEOUT_SECONDS, + ), ) # Automatic Stage 12 work must never change the production v1 # response, including for unexpected integration failures. except Exception as error: # noqa: BLE001 # Stage 12 dispatch must never change the v1 response logger.error( # noqa: TRY400 # Stage 12 failure logs carry only the error type; a traceback could expose request or calculation data "stage12_comparison_dispatch_failed", - extra={ - "request_id": request.state.request_id, - "error_type": type(error).__name__, - }, + extra=log_extra( + request_id=request.state.request_id, + error_type=type(error).__name__, + ), ) diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_calculations.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_calculations.py index bc085df3e..7d44b0fcf 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_calculations.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_calculations.py @@ -34,6 +34,10 @@ from policyengine_simulation_contract.household_execution import ( HouseholdExecutionRequest, ) +from policyengine_simulation_observability.log_records import ( + database_row_attributes, + log_extra, +) from policyengine_simulation_persistence import ( HouseholdJobReferenceError, HouseholdJobStatus, @@ -386,7 +390,7 @@ async def read(self, job_id: UUID) -> HouseholdJobRecord | None: except SQLAlchemyError as error: logger.warning( "household_job_read_failed", - extra={"error_type": type(error).__name__}, + extra=log_extra(error_type=type(error).__name__), ) raise _database_unavailable() from None @@ -400,7 +404,7 @@ async def _default_worker(self, country: str) -> HouseholdWorker: except VersionManifestUnavailableError as error: logger.warning( "household_manifest_unavailable", - extra={"error_type": type(error.__cause__).__name__}, + extra=log_extra(error_type=type(error.__cause__).__name__), ) raise HouseholdCalculationUnavailableError( code="manifest_unavailable", @@ -449,7 +453,7 @@ async def _prepare( except SQLAlchemyError as error: logger.warning( "household_inputs_unreadable", - extra={"error_type": type(error).__name__}, + extra=log_extra(error_type=type(error).__name__), ) raise _database_unavailable() from None return prepare_household_job(request, worker, inputs, job_id=self._new_job_id()) @@ -475,7 +479,7 @@ async def _create(self, job: NewHouseholdJob) -> None: except SQLAlchemyError as error: logger.warning( "household_job_not_created", - extra={"error_type": type(error).__name__}, + extra=log_extra(error_type=type(error).__name__), ) raise _database_unavailable() from None @@ -496,7 +500,10 @@ async def _answer_unconfirmed_call( # The log names the error type only; its text could repeat request data. logger.error( "household_call_unconfirmed", - extra={"household_job_id": str(job_id), "error_type": type(error).__name__}, + extra=log_extra( + error_type=type(error).__name__, + **database_row_attributes("household_jobs", job_id), + ), ) message = ( "Modal did not accept the household function's call " @@ -510,10 +517,10 @@ async def _answer_unconfirmed_call( # The stuck-run sweep fails the job later if it stays pending. logger.warning( "household_job_not_failed", - extra={ - "household_job_id": str(job_id), - "error_type": type(write_error).__name__, - }, + extra=log_extra( + error_type=type(write_error).__name__, + **database_row_attributes("household_jobs", job_id), + ), ) return HouseholdCalculationAccepted( job_id=job_id, status=HouseholdJobStatus.PENDING @@ -532,10 +539,10 @@ async def _answer_unconfirmed_call( except SQLAlchemyError as read_error: logger.warning( "household_job_not_read", - extra={ - "household_job_id": str(job_id), - "error_type": type(read_error).__name__, - }, + extra=log_extra( + error_type=type(read_error).__name__, + **database_row_attributes("household_jobs", job_id), + ), ) else: if record is not None: @@ -555,10 +562,10 @@ async def _record_call(self, job_id: UUID, call_id: str) -> HouseholdJobStatus: except SQLAlchemyError as error: logger.warning( "household_call_not_recorded", - extra={ - "household_job_id": str(job_id), - "error_type": type(error).__name__, - }, + extra=log_extra( + error_type=type(error).__name__, + **database_row_attributes("household_jobs", job_id), + ), ) return HouseholdJobStatus.PENDING return HouseholdJobStatus.PENDING if status is None else status diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_report_submissions.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_report_submissions.py index 6bc1aceca..8aba5228f 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_report_submissions.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_report_submissions.py @@ -46,6 +46,10 @@ from typing import TYPE_CHECKING, Any, Protocol from uuid import UUID, uuid4 +from policyengine_simulation_observability.log_records import ( + database_row_attributes, + log_extra, +) from policyengine_simulation_persistence import ( EconomyRecordOutcomeUnknownError, EconomyRecordReferenceError, @@ -380,7 +384,7 @@ async def _default_worker(self, country: str) -> HouseholdWorker: except VersionManifestUnavailableError as error: logger.warning( "household_report_manifest_unavailable", - extra={"error_type": type(error.__cause__).__name__}, + extra=log_extra(error_type=type(error.__cause__).__name__), ) raise EconomySubmissionUnavailableError( code="manifest_unavailable", @@ -410,7 +414,7 @@ async def _read_inputs( except SQLAlchemyError as error: logger.warning( "household_report_inputs_unreadable", - extra={"error_type": type(error).__name__}, + extra=log_extra(error_type=type(error).__name__), ) raise _database_unavailable() from None except Exception as error: @@ -498,7 +502,7 @@ async def _create[Created]( except (SharedRowNotSettledError, SQLAlchemyError) as error: logger.warning( "household_report_not_created", - extra={"error_type": type(error).__name__}, + extra=log_extra(error_type=type(error).__name__), ) raise _database_unavailable() from None @@ -554,10 +558,10 @@ async def _start_report_run( # noqa: PLR0913 # the call, then the runs it calc # itself, so the calculation goes ahead either way. logger.warning( "household_report_call_not_recorded", - extra={ - "report_id": str(report_id), - "error_type": type(error).__name__, - }, + extra=log_extra( + error_type=type(error).__name__, + **database_row_attributes("reports", report_id), + ), ) return ReportRunStatus.PENDING return ReportRunStatus.PENDING if status is None else status @@ -581,7 +585,10 @@ async def _fail_unconfirmed( # The log names the error type only; its text could repeat request data. logger.error( "household_report_call_unconfirmed", - extra={"report_id": str(report_id), "error_type": type(error).__name__}, + extra=log_extra( + error_type=type(error).__name__, + **database_row_attributes("reports", report_id), + ), ) message = ( "modal_call_refused: Modal did not accept the report's call " @@ -597,10 +604,10 @@ async def _fail_unconfirmed( except SQLAlchemyError as write_error: logger.warning( "household_report_run_not_failed", - extra={ - "report_id": str(report_id), - "error_type": type(write_error).__name__, - }, + extra=log_extra( + error_type=type(write_error).__name__, + **database_row_attributes("reports", report_id), + ), ) return False diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/route_support.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/route_support.py index 457e467cd..8b7880e82 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/route_support.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/route_support.py @@ -12,6 +12,8 @@ import traceback from typing import TYPE_CHECKING, Any +from policyengine_simulation_observability.log_records import log_extra + if TYPE_CHECKING: import logging @@ -108,7 +110,7 @@ def log_unexpected_failure( "stack_frames": frames, "stack_frames_omitted": omitted_frame_count, } - logger.error(event, extra={**fields, "policyengine_attributes": fields}) + logger.error(event, extra=log_extra(**fields)) def observability_context(entry: EntryRuntime, observability_id: str) -> dict[str, Any]: diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/operation_records.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/operation_records.py index a380dbe98..1a357ff71 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/operation_records.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/operation_records.py @@ -24,9 +24,10 @@ from typing import Literal from uuid import UUID -DATABASE_TABLE_ATTRIBUTE = "database_table" -DATABASE_ROW_ID_ATTRIBUTE = "database_row_id" -MODAL_CALL_ID_ATTRIBUTE = "modal_call_id" +from policyengine_simulation_observability.log_records import ( + MODAL_CALL_ID_ATTRIBUTE, + database_row_attributes, +) DatabaseTable = Literal["household_jobs", "report_runs", "simulation_runs"] @@ -40,10 +41,9 @@ def operation_record_attributes( A missing row or Modal call ID is left out. """ - attributes: dict[str, str] = {} - if row_id is not None: - attributes[DATABASE_TABLE_ATTRIBUTE] = table - attributes[DATABASE_ROW_ID_ATTRIBUTE] = str(row_id) + attributes: dict[str, str] = ( + {} if row_id is None else database_row_attributes(table, row_id) + ) if modal_call_id: attributes[MODAL_CALL_ID_ATTRIBUTE] = modal_call_id return attributes From b8eb69e7f6d27603c0f7b280e2fe2d38e6547c1d Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:51:53 +0400 Subject: [PATCH 06/10] Test INFO-level Entrypoint logs, reserved field names and each row table (failing) After review round 2: the Entrypoint's INFO records should reach the observability runtime's handler, which the root logger's WARNING level stops today; a field named like a LogRecord attribute should be emitted instead of making the log call raise; the economy, household report and budget window logs should name their `reports` or `simulations` row; and the static check gets proof fixtures for its rule. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_log_records.py | 16 +++ .../tests/test_budget_window_submissions.py | 25 ++++ .../tests/test_economy_submissions.py | 29 ++++ .../test_household_report_submissions.py | 23 ++++ .../tests/test_log_fields.py | 128 +++++++++++++----- .../bad_entry_logs.py | 30 ++++ .../good_entry_logs.py | 30 ++++ 7 files changed, 245 insertions(+), 36 deletions(-) create mode 100644 rules/fixtures/sim.observability.log-fields-through-log-extra/bad_entry_logs.py create mode 100644 rules/fixtures/sim.observability.log-fields-through-log-extra/good_entry_logs.py diff --git a/libs/policyengine-simulation-observability/tests/test_log_records.py b/libs/policyengine-simulation-observability/tests/test_log_records.py index 5232fffc7..d14c0ca0e 100644 --- a/libs/policyengine-simulation-observability/tests/test_log_records.py +++ b/libs/policyengine-simulation-observability/tests/test_log_records.py @@ -119,3 +119,19 @@ def test_a_household_key_is_dropped_even_in_policyengine_attributes( record = _record_with_message(written_records, "household_job_not_failed") assert record["attributes"] == {"error_type": "OperationalError"} + + +def test_a_field_named_like_a_log_record_attribute_is_emitted_without_raising( + written_records, +) -> None: + # ``logging`` refuses an ``extra`` key that names a LogRecord attribute, + # and these calls sit in error handlers, where raising would turn a + # handled failure into an unhandled one. + from policyengine_simulation_observability.log_records import log_extra + + logging.getLogger("policyengine_simulation_entry.test").warning( + "budget_window_not_started", extra=log_extra(name="window", module="entry") + ) + + record = _record_with_message(written_records, "budget_window_not_started") + assert record["attributes"] == {"name": "window", "module": "entry"} diff --git a/projects/policyengine-simulation-entry/tests/test_budget_window_submissions.py b/projects/policyengine-simulation-entry/tests/test_budget_window_submissions.py index c8eb34f91..bca9c8aaf 100644 --- a/projects/policyengine-simulation-entry/tests/test_budget_window_submissions.py +++ b/projects/policyengine-simulation-entry/tests/test_budget_window_submissions.py @@ -503,3 +503,28 @@ def test_a_later_year_is_deferred_without_a_call_once_the_wait_has_run_out( "database_table": "report_runs", "database_row_id": str(second_year.report_run_id), } + + +def test_a_later_year_modal_refuses_is_logged_naming_the_report(caplog) -> None: + store = FakeWindowStore() + + with ( + caplog.at_level(logging.INFO, logger=budget_window_submissions.__name__), + pytest.raises(EconomySubmissionUnavailableError), + ): + _submit( + _service(store, caller=RefusingAfterFirstCaller()), budget_window_size=3 + ) + + (record,) = [ + item + for item in caplog.records + if item.getMessage() == "budget_window_call_unconfirmed" + ] + fields = record.policyengine_attributes + assert (fields["database_table"], fields["database_row_id"]) == ( + "reports", + str(store.windows[0].report_id), + ) + # The runtime emits no list, so the years started are one string. + assert isinstance(fields["started_years"], str) diff --git a/projects/policyengine-simulation-entry/tests/test_economy_submissions.py b/projects/policyengine-simulation-entry/tests/test_economy_submissions.py index 47b9f3137..7cd5d3dcc 100644 --- a/projects/policyengine-simulation-entry/tests/test_economy_submissions.py +++ b/projects/policyengine-simulation-entry/tests/test_economy_submissions.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import logging from dataclasses import replace from uuid import UUID @@ -810,3 +811,31 @@ def test_a_create_whose_commit_is_unconfirmed_may_have_written_the_record( "OperationalError", ) assert (caller.report_runs, caller.simulation_runs) == ([], []) + + +@pytest.mark.parametrize( + ("submit", "expected_table"), + [(_submit_report, "reports"), (_submit_simulation, "simulations")], + ids=["report", "simulation"], +) +def test_an_unconfirmed_call_is_logged_naming_its_report_or_simulation( + caplog, submit, expected_table +) -> None: + store = FakeStore() + + with ( + caplog.at_level(logging.INFO), + pytest.raises(EconomySubmissionUnavailableError) as unavailable, + ): + submit(service(store, caller=FakeCaller(failure=ConnectionError("refused")))) + + (record,) = [ + item + for item in caplog.records + if item.getMessage() == "economy_call_unconfirmed" + ] + assert record.policyengine_attributes == { + "error_type": "ConnectionError", + "database_table": expected_table, + "database_row_id": str(unavailable.value.record_id), + } diff --git a/projects/policyengine-simulation-entry/tests/test_household_report_submissions.py b/projects/policyengine-simulation-entry/tests/test_household_report_submissions.py index 04d4ebc15..5a0112d21 100644 --- a/projects/policyengine-simulation-entry/tests/test_household_report_submissions.py +++ b/projects/policyengine-simulation-entry/tests/test_household_report_submissions.py @@ -8,6 +8,7 @@ from __future__ import annotations import asyncio +import logging from uuid import UUID import pytest @@ -595,3 +596,25 @@ def test_a_refusal_of_the_missing_variables_option_is_the_services_fault(): with pytest.raises(HouseholdReportBuildError, match="add_missing_variables"): household_report_calculations._contract_reasons(refused.value, "baseline") + + +def test_an_unconfirmed_call_is_logged_naming_its_report(caplog): + store = FakeHouseholdReportStore() + caller = FakeHouseholdReportCaller(failure=RuntimeError("refused")) + + with ( + caplog.at_level(logging.INFO), + pytest.raises(EconomySubmissionUnavailableError), + ): + _submit(household_report_service(store, caller=caller)) + + (record,) = [ + item + for item in caplog.records + if item.getMessage() == "household_report_call_unconfirmed" + ] + assert record.policyengine_attributes == { + "error_type": "RuntimeError", + "database_table": "reports", + "database_row_id": str(REPORT_ID), + } diff --git a/projects/policyengine-simulation-entry/tests/test_log_fields.py b/projects/policyengine-simulation-entry/tests/test_log_fields.py index 029191d6b..cad9e1b65 100644 --- a/projects/policyengine-simulation-entry/tests/test_log_fields.py +++ b/projects/policyengine-simulation-entry/tests/test_log_fields.py @@ -8,26 +8,47 @@ from __future__ import annotations import ast +import logging from pathlib import Path import policyengine_simulation_entry +from conftest import FakeBackend, make_settings from policyengine_observability import ( DeploymentIdentity, ObservabilityConfig, ServiceIdentity, ) from policyengine_observability.schema import normalize_attributes +from policyengine_simulation_entry.app import create_app PACKAGE_DIRECTORY = Path(policyengine_simulation_entry.__file__).parent +PROOF_FIXTURES = ( + Path(__file__).resolve().parents[3] + / "rules" + / "fixtures" + / "sim.observability.log-fields-through-log-extra" +) LOG_METHODS = frozenset( {"debug", "info", "warning", "error", "exception", "critical", "log"} ) +FILTER_CONFIG = ObservabilityConfig( + service=ServiceIdentity( + name="policyengine-simulation-entry-test", + namespace="policyengine.test", + version="test", + role="test", + ), + deployment=DeploymentIdentity(environment="test", platform="local"), +) -def _log_calls_with_extra() -> list[tuple[str, ast.Call, ast.expr]]: - """Return each logging call in the package that passes ``extra=``.""" +def _log_calls_with_extra(paths: list[Path]) -> list[tuple[str, ast.expr]]: + """Return each logging call in ``paths`` that passes ``extra=``, with its value. + + Each call is named ``file:line``. + """ calls = [] - for path in sorted(PACKAGE_DIRECTORY.rglob("*.py")): + for path in sorted(paths): tree = ast.parse(path.read_text(), filename=str(path)) for node in ast.walk(tree): if not ( @@ -36,10 +57,11 @@ def _log_calls_with_extra() -> list[tuple[str, ast.Call, ast.expr]]: and node.func.attr in LOG_METHODS ): continue - for keyword in node.keywords: - if keyword.arg == "extra": - location = f"{path.relative_to(PACKAGE_DIRECTORY)}:{node.lineno}" - calls.append((location, node, keyword.value)) + calls.extend( + (f"{path.name}:{node.lineno}", keyword.value) + for keyword in node.keywords + if keyword.arg == "extra" + ) return calls @@ -51,44 +73,78 @@ def _is_log_extra_call(expression: ast.expr) -> bool: ) -def test_the_package_has_log_calls_with_fields() -> None: - # Guards the next tests against finding nothing to check. - assert len(_log_calls_with_extra()) >= 30 - - -def test_every_log_call_passes_its_fields_through_log_extra() -> None: - plain_extra = [ - location - for location, _, extra in _log_calls_with_extra() - if not _is_log_extra_call(extra) - ] +def _plain_extra_calls(calls: list[tuple[str, ast.expr]]) -> list[str]: + """Return the calls whose fields don't go through ``log_extra``.""" + return [location for location, extra in calls if not _is_log_extra_call(extra)] - assert plain_extra == [] +def _dropped_field_names(calls: list[tuple[str, ast.expr]]) -> list[tuple[str, str]]: + """Return each field named in a ``log_extra`` call that the package drops. -def test_no_log_field_is_named_with_a_word_the_package_drops() -> None: - config = ObservabilityConfig( - service=ServiceIdentity( - name="policyengine-simulation-entry-test", - namespace="policyengine.test", - version="test", - role="test", - ), - deployment=DeploymentIdentity(environment="test", platform="local"), - ) + Only names written as keywords are seen, not keys unpacked with ``**``. + """ field_names = { (location, keyword.arg) - for location, _, extra in _log_calls_with_extra() - if isinstance(extra, ast.Call) + for location, extra in calls + if _is_log_extra_call(extra) and isinstance(extra, ast.Call) for keyword in extra.keywords if keyword.arg is not None } - assert field_names - - dropped = sorted( + return sorted( (location, name) for location, name in field_names - if not normalize_attributes({name: "value"}, config)[0] + if not normalize_attributes({name: "value"}, FILTER_CONFIG)[0] ) - assert dropped == [] + +def _package_log_calls() -> list[tuple[str, ast.expr]]: + return _log_calls_with_extra(list(PACKAGE_DIRECTORY.rglob("*.py"))) + + +def test_the_package_has_log_calls_with_fields() -> None: + # Guards the next tests against finding nothing to check. + assert len(_package_log_calls()) >= 30 + + +def test_every_log_call_passes_its_fields_through_log_extra() -> None: + assert _plain_extra_calls(_package_log_calls()) == [] + + +def test_no_log_field_is_named_with_a_word_the_package_drops() -> None: + assert _dropped_field_names(_package_log_calls()) == [] + + +def test_rule_proof_log_fields_through_log_extra() -> None: + """Proof for rule sim.observability.log-fields-through-log-extra.""" + bad_calls = _log_calls_with_extra([PROOF_FIXTURES / "bad_entry_logs.py"]) + good_calls = _log_calls_with_extra([PROOF_FIXTURES / "good_entry_logs.py"]) + + assert _plain_extra_calls(bad_calls) == ["bad_entry_logs.py:22"] + assert _dropped_field_names(bad_calls) == [ + ("bad_entry_logs.py:30", "household_job_id") + ] + assert _plain_extra_calls(good_calls) == [] + assert _dropped_field_names(good_calls) == [] + assert good_calls + + +def test_the_entrypoint_emits_its_info_level_log_records() -> None: + # The root logger stays at WARNING, so without a level of its own an INFO + # record, such as ``budget_window_year_deferred``, is discarded before the + # observability runtime's handler sees it. + package_logger = logging.getLogger("policyengine_simulation_entry") + previous_level = package_logger.level + # ``setLevel`` also clears the loggers' cached answers to ``isEnabledFor``. + package_logger.setLevel(logging.NOTSET) + try: + create_app( + settings=make_settings(), + backend=FakeBackend(), + auth_dependency=lambda: None, + ) + + assert logging.getLogger( + "policyengine_simulation_entry.budget_window_submissions" + ).isEnabledFor(logging.INFO) + finally: + package_logger.setLevel(previous_level) diff --git a/rules/fixtures/sim.observability.log-fields-through-log-extra/bad_entry_logs.py b/rules/fixtures/sim.observability.log-fields-through-log-extra/bad_entry_logs.py new file mode 100644 index 000000000..9cb41910c --- /dev/null +++ b/rules/fixtures/sim.observability.log-fields-through-log-extra/bad_entry_logs.py @@ -0,0 +1,30 @@ +"""Proof fixture: log calls whose fields never reach Cloud Logging. + +The rule must reject both. The first is a real past mistake: until this rule, +the Simulation Entrypoint's household job logs passed `household_job_id` as a +plain `extra=` key, which the observability runtime's log handler ignores. The +second passes the key through `log_extra`, but the runtime drops it by name. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Callable + +logger = logging.getLogger(__name__) + + +def record_call_failed(job_id: str, error: Exception) -> None: + """Log a call ID that could not be recorded, with plain extra fields.""" + logger.warning( + "household_call_not_recorded", + extra={"household_job_id": job_id, "error_type": type(error).__name__}, + ) + + +def fail_unconfirmed(log_extra: Callable[..., dict[str, object]], job_id: str) -> None: + """Log an unconfirmed call, naming the job by a key the runtime drops.""" + logger.error("household_call_unconfirmed", extra=log_extra(household_job_id=job_id)) diff --git a/rules/fixtures/sim.observability.log-fields-through-log-extra/good_entry_logs.py b/rules/fixtures/sim.observability.log-fields-through-log-extra/good_entry_logs.py new file mode 100644 index 000000000..d0cce8cb2 --- /dev/null +++ b/rules/fixtures/sim.observability.log-fields-through-log-extra/good_entry_logs.py @@ -0,0 +1,30 @@ +"""Proof fixture: a log call whose fields reach Cloud Logging. + +The rule must accept it. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +from policyengine_simulation_observability.log_records import ( + database_row_attributes, + log_extra, +) + +if TYPE_CHECKING: + from uuid import UUID + +logger = logging.getLogger(__name__) + + +def record_call_failed(job_id: UUID, error: Exception) -> None: + """Log a call ID that could not be recorded, naming the job's row.""" + logger.warning( + "household_call_not_recorded", + extra=log_extra( + error_type=type(error).__name__, + **database_row_attributes("household_jobs", job_id), + ), + ) From 1c2bf90b9da1f0d7476a91425636eaa66a1724fc Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:53:35 +0400 Subject: [PATCH 07/10] Emit the Entrypoint's INFO logs and register its log-fields rule, after review round 2 The Entrypoint sets its package logger to INFO: the root logger's WARNING level discarded its INFO records, such as a deferred budget-window year, before the observability runtime's handler saw them. `log_extra` keeps a field named like a LogRecord attribute only in `policyengine_attributes`, since `logging` raises for it. One `DatabaseTable` type in the shared module lists the tables telemetry names. Rule `sim.observability.log-fields-through-log-extra` registers the static check, and the observability guide's new section follows the identifier table instead of splitting it. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 1 + .../c1-34-v2-telemetry-record-ids.fixed.md | 2 +- docs/engineering/skills/observability.md | 25 ++++++++--------- docs/generated/rules.md | 10 +++++-- .../log_records.py | 26 +++++++++++++++--- .../src/policyengine_simulation_entry/app.py | 5 ++++ .../economy_submission_errors.py | 3 ++- .../stage12_runtime/operation_records.py | 5 ++-- rules/registry.toml | 27 +++++++++++++++++++ 9 files changed, 82 insertions(+), 22 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 12ce348df..c55c0c699 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -150,6 +150,7 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`. ### Observability +- Checked: `sim.observability.log-fields-through-log-extra` - `sim.observability.failures-never-break-requests`: Invalid observability configuration fails at build or deploy time, and telemetry failures at runtime never fail a request or computation. (doc-only, not checked: follow it yourself) - `sim.observability.header-transport-only`: observability_id travels only in the X-PolicyEngine-Observability-Id header, never in a simulation JSON body, and polling never invents one. (doc-only, not checked: follow it yourself) - `sim.observability.id-is-diagnostic-only`: observability_id has diagnostic meaning only: never use it for idempotency, authorization, database identity, routing or business logic. (doc-only, not checked: follow it yourself) diff --git a/changelog.d/c1-34-v2-telemetry-record-ids.fixed.md b/changelog.d/c1-34-v2-telemetry-record-ids.fixed.md index 51fa8218a..a78a9fc7a 100644 --- a/changelog.d/c1-34-v2-telemetry-record-ids.fixed.md +++ b/changelog.d/c1-34-v2-telemetry-record-ids.fixed.md @@ -1 +1 @@ -The Simulation Entrypoint's log records now carry their fields. 31 of its 33 log calls passed their fields only as standard-library `extra=` keys, which the observability runtime doesn't emit, so their records in Cloud Logging named no job, report, run or error type. Every log call now passes its fields through `log_extra` in the shared observability library, and names a Public API row with `database_table` (`household_jobs`, `reports`, `report_runs` or `simulations`) and `database_row_id`: a key such as `household_job_id` would be dropped by the runtime's key filter. A budget window's `started_years` are now joined into one string, since the runtime drops lists. +The Simulation Entrypoint's log records now carry their fields. 31 of its 33 log calls passed their fields only as standard-library `extra=` keys, which the observability runtime doesn't emit, so their records in Cloud Logging named no job, report, run or error type. Every log call now passes its fields through `log_extra` in the shared observability library, and names a Public API row with `database_table` (`household_jobs`, `reports`, `report_runs` or `simulations`) and `database_row_id`: a key such as `household_job_id` would be dropped by the runtime's key filter. A budget window's `started_years` are now joined into one string, since the runtime drops lists. The Entrypoint also sets its package logger to INFO, so its INFO records (a deferred budget-window year, a refused bearer token) reach the runtime: the root logger's WARNING level discarded them. diff --git a/docs/engineering/skills/observability.md b/docs/engineering/skills/observability.md index b297b1fd0..a7d8acdd7 100644 --- a/docs/engineering/skills/observability.md +++ b/docs/engineering/skills/observability.md @@ -30,23 +30,24 @@ contains `household`, `reform`, `payload` or `person`, among others, so no key for these IDs names one. Do not use `job_id` for a v2 call: it names one annual Modal invocation of the v1 executor. +`observability_id` is a canonical UUID string and has diagnostic meaning only. +Do not use it for idempotency, authorization, database identity, routing, or +business logic. Do not create one for health checks, version queries, invalid +requests, missing jobs, or ordinary status requests. + ## Standard-library log fields The observability runtime's standard-library log handler emits only a log record's `policyengine_attributes`: any other key passed in `extra=` stays on -the Python `LogRecord` and never reaches Cloud Logging. Pass every log call's -fields through `log_extra(...)` from +the Python `LogRecord` and never reaches Cloud Logging. In the Simulation +Entrypoint, pass every log call's fields through `log_extra(...)` from `policyengine_simulation_observability.log_records`, which puts them in both -places, and name a row with `database_row_attributes(table, row_id)`. Each -field must be a string, a number or a boolean; the runtime drops a list. In -the Simulation Entrypoint, `tests/test_log_fields.py` checks that every log call -with `extra=` uses `log_extra` and that no literal field name contains a word -the package drops. - -`observability_id` is a canonical UUID string and has diagnostic meaning only. -Do not use it for idempotency, authorization, database identity, routing, or -business logic. Do not create one for health checks, version queries, invalid -requests, missing jobs, or ordinary status requests. +places, and name a row with `database_row_attributes(table, row_id)` (rule +`sim.observability.log-fields-through-log-extra`). Each field must be a +string, a number or a boolean; the runtime drops a list. The Entrypoint sets +its package logger to INFO: the root logger stays at WARNING, which would +discard INFO records before the handler sees them. Some Stage 12 runtime and +contract-library logs still pass plain `extra=` fields, which are not emitted. ## HTTP transport diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 0b43a4a3f..d1f861eb8 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -8,11 +8,11 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf | Tier | enforced | ratcheted | advisory | doc-only | Total | | --- | ---: | ---: | ---: | ---: | ---: | -| mechanical | 64 | 0 | 0 | 14 | 78 | +| mechanical | 65 | 0 | 0 | 14 | 79 | | metric | 1 | 0 | 0 | 0 | 1 | | judgment | 0 | 0 | 0 | 0 | 0 | -**Blocking:** 65 of 79 rules. +**Blocking:** 66 of 80 rules. ## Gaps @@ -306,6 +306,7 @@ flowchart LR | `sim.observability.failures-never-break-requests` | Invalid observability configuration fails at build or deploy time, and telemetry failures at runtime never fail a request or computation. | mechanical | doc-only | no | nothing | — | — | | `sim.observability.header-transport-only` | observability_id travels only in the X-PolicyEngine-Observability-Id header, never in a simulation JSON body, and polling never invents one. | mechanical | doc-only | no | nothing | — | — | | `sim.observability.id-is-diagnostic-only` | observability_id has diagnostic meaning only: never use it for idempotency, authorization, database identity, routing or business logic. | mechanical | doc-only | no | nothing | — | — | +| `sim.observability.log-fields-through-log-extra` | Every Simulation Entrypoint log call that passes fields passes them through log_extra, so they reach the emitted record, and names none with a word the observability package drops (household, reform, payload, person and others). | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_log_fields.py` | `pr.yml:test` | yes | | `sim.observability.registered-stages-only` | Span names come from the stage plans in policyengine_simulation_observability.stages; runtime code never adds span-name string literals. | mechanical | doc-only | no | nothing | — | — | ```mermaid @@ -314,10 +315,15 @@ flowchart LR gap_unenforced>"not enforced"] r_sim_observability_header_transport_only("sim.observability.header-transport-only (doc-only)") r_sim_observability_id_is_diagnostic_only("sim.observability.id-is-diagnostic-only (doc-only)") + r_sim_observability_log_fields_through_log_extra("sim.observability.log-fields-through-log-extra (enforced)") + e_pytest_projects_policyengine_simulation_entry_tests_test_log_fields_py["pytest projects/policyengine-simulation-entry/tests/test_log_fields.py"] + j_CI_pr_yml_test["CI pr.yml:test"] r_sim_observability_registered_stages_only("sim.observability.registered-stages-only (doc-only)") r_sim_observability_failures_never_break_requests -.-> gap_unenforced r_sim_observability_header_transport_only -.-> gap_unenforced r_sim_observability_id_is_diagnostic_only -.-> gap_unenforced + r_sim_observability_log_fields_through_log_extra --> e_pytest_projects_policyengine_simulation_entry_tests_test_log_fields_py + e_pytest_projects_policyengine_simulation_entry_tests_test_log_fields_py --> j_CI_pr_yml_test r_sim_observability_registered_stages_only -.-> gap_unenforced ``` diff --git a/libs/policyengine-simulation-observability/src/policyengine_simulation_observability/log_records.py b/libs/policyengine-simulation-observability/src/policyengine_simulation_observability/log_records.py index ea691aae9..0d422d4bd 100644 --- a/libs/policyengine-simulation-observability/src/policyengine_simulation_observability/log_records.py +++ b/libs/policyengine-simulation-observability/src/policyengine_simulation_observability/log_records.py @@ -11,7 +11,8 @@ from __future__ import annotations -from typing import TYPE_CHECKING +import logging +from typing import TYPE_CHECKING, Final, Literal if TYPE_CHECKING: from uuid import UUID @@ -20,10 +21,20 @@ DATABASE_ROW_ID_ATTRIBUTE = "database_row_id" MODAL_CALL_ID_ATTRIBUTE = "modal_call_id" +# The Public API tables whose rows telemetry names. +DatabaseTable = Literal[ + "household_jobs", "report_runs", "reports", "simulation_runs", "simulations" +] LogFieldValue = str | int | float | bool +# ``logging`` raises on an ``extra`` key naming one of a record's own +# attributes, or the two it sets while formatting. +_LOG_RECORD_ATTRIBUTES: Final = frozenset( + vars(logging.LogRecord("", logging.INFO, "", 0, "", None, None)) +) | {"message", "asctime"} -def database_row_attributes(table: str, row_id: UUID) -> dict[str, str]: + +def database_row_attributes(table: DatabaseTable, row_id: UUID) -> dict[str, str]: """Return the attributes naming row ``row_id`` of the Public API's ``table``.""" return {DATABASE_TABLE_ATTRIBUTE: table, DATABASE_ROW_ID_ATTRIBUTE: str(row_id)} @@ -32,7 +43,14 @@ def log_extra(**fields: LogFieldValue) -> dict[str, object]: """Return a logging call's ``extra`` whose ``fields`` reach the emitted record. Each field also stays on the standard-library record, where other - handlers and tests read it. A field must be a string, a number or a + handlers and tests read it, unless its name is one of the record's own + attributes: ``logging`` would raise for that one, and these calls are + often in error handlers. A field must be a string, a number or a boolean: the runtime drops anything else. """ - return {**fields, "policyengine_attributes": dict(fields)} + record_fields = { + name: value + for name, value in fields.items() + if name not in _LOG_RECORD_ATTRIBUTES + } + return {**record_fields, "policyengine_attributes": dict(fields)} diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py index 53a824dbe..30dc6456c 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py @@ -127,6 +127,8 @@ ] logger = logging.getLogger(__name__) +# Every module of the package logs under this logger. +PACKAGE_LOGGER_NAME = "policyengine_simulation_entry" SIMULATION_DATABASE_READINESS_TIMEOUT_SECONDS = 3.0 # The database ends each statement of a household route that runs longer. HOUSEHOLD_STATEMENT_TIMEOUT_MILLISECONDS = 5000 @@ -718,6 +720,9 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: platform="google_cloud_run", environment=runtime_settings.environment, ) + # The root logger stays at WARNING, which would discard this package's + # INFO records before the observability runtime's handler sees them. + logging.getLogger(PACKAGE_LOGGER_NAME).setLevel(logging.INFO) runtime_backend = backend or OldGatewayBackend(runtime_settings, runtime=runtime) authenticate = auth_dependency or CallerAuthenticator(runtime_settings, runtime) diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_submission_errors.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_submission_errors.py index 3c6f62a79..98bd3548a 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_submission_errors.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_submission_errors.py @@ -33,6 +33,7 @@ from collections.abc import Iterator from uuid import UUID + from policyengine_simulation_observability.log_records import DatabaseTable from policyengine_simulation_persistence import ( EconomyRecordReferenceError, EconomyRequest, @@ -65,7 +66,7 @@ # What a submission creates, or a re-run calculates again. RecordName = Literal["report", "simulation"] # The Public API table holding each record, as telemetry names it. -RECORD_TABLES: Final[dict[RecordName, str]] = { +RECORD_TABLES: Final[dict[RecordName, DatabaseTable]] = { "report": "reports", "simulation": "simulations", } diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/operation_records.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/operation_records.py index 1a357ff71..0ef1e860b 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/operation_records.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/operation_records.py @@ -29,11 +29,12 @@ database_row_attributes, ) -DatabaseTable = Literal["household_jobs", "report_runs", "simulation_runs"] +# The tables a v2 Modal function's call works on. +OperationTable = Literal["household_jobs", "report_runs", "simulation_runs"] def operation_record_attributes( - table: DatabaseTable, + table: OperationTable, row_id: UUID | None, modal_call_id: str | None, ) -> dict[str, str]: diff --git a/rules/registry.toml b/rules/registry.toml index b0100b932..e5a8209d4 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -973,6 +973,33 @@ autofix = false [rule.enforcer] kind = "none" +[[rule]] +id = "sim.observability.log-fields-through-log-extra" +category = "observability" +statement = "Every Simulation Entrypoint log call that passes fields passes them through log_extra, so they reach the emitted record, and names none with a word the observability package drops (household, reform, payload, person and others)." +rationale = "docs/engineering/skills/observability.md#standard-library-log-fields" +scope = ["projects/policyengine-simulation-entry/src/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = [ + "tests/test_log_fields.py::test_every_log_call_passes_its_fields_through_log_extra", + "tests/test_log_fields.py::test_no_log_field_is_named_with_a_word_the_package_drops", + "tests/test_log_fields.py::test_the_package_has_log_calls_with_fields", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run --frozen pytest tests/" +[rule.proof] +bad_fixture = "rules/fixtures/sim.observability.log-fields-through-log-extra/bad_entry_logs.py" +good_fixture = "rules/fixtures/sim.observability.log-fields-through-log-extra/good_entry_logs.py" +project = "projects/policyengine-simulation-entry" +test = "tests/test_log_fields.py::test_rule_proof_log_fields_through_log_extra" + [[rule]] id = "sim.observability.header-transport-only" category = "observability" From 4d54cd9592ed7c27ac50762f7b7f88d2f5be3d1a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 14:06:47 +0400 Subject: [PATCH 08/10] Tie the package logger's name to the package and say what the log check reads, after review round 3 `PACKAGE_LOGGER_NAME` comes from the module name, and the INFO-level test imports it. The guide and the rule's statement say the check reads only field names written as keywords in `log_extra(...)`, and that a field named like a LogRecord attribute goes only to `policyengine_attributes`. The budget-window log test checks the started years' value. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/skills/observability.md | 7 +++++-- docs/generated/rules.md | 2 +- .../src/policyengine_simulation_entry/app.py | 2 +- .../tests/test_budget_window_submissions.py | 5 +++-- .../policyengine-simulation-entry/tests/test_log_fields.py | 6 +++--- rules/registry.toml | 2 +- 6 files changed, 14 insertions(+), 10 deletions(-) diff --git a/docs/engineering/skills/observability.md b/docs/engineering/skills/observability.md index a7d8acdd7..9510a5a9f 100644 --- a/docs/engineering/skills/observability.md +++ b/docs/engineering/skills/observability.md @@ -42,8 +42,11 @@ record's `policyengine_attributes`: any other key passed in `extra=` stays on the Python `LogRecord` and never reaches Cloud Logging. In the Simulation Entrypoint, pass every log call's fields through `log_extra(...)` from `policyengine_simulation_observability.log_records`, which puts them in both -places, and name a row with `database_row_attributes(table, row_id)` (rule -`sim.observability.log-fields-through-log-extra`). Each field must be a +places (a field named like a `LogRecord` attribute, such as `name`, goes only +to `policyengine_attributes`, since `logging` refuses it), and name a row with `database_row_attributes(table, row_id)` (rule +`sim.observability.log-fields-through-log-extra`; its check reads only the +field names written as keywords in the `log_extra(...)` call, not keys unpacked +from a dictionary with `**`). Each field must be a string, a number or a boolean; the runtime drops a list. The Entrypoint sets its package logger to INFO: the root logger stays at WARNING, which would discard INFO records before the handler sees them. Some Stage 12 runtime and diff --git a/docs/generated/rules.md b/docs/generated/rules.md index d1f861eb8..451c78fb5 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -306,7 +306,7 @@ flowchart LR | `sim.observability.failures-never-break-requests` | Invalid observability configuration fails at build or deploy time, and telemetry failures at runtime never fail a request or computation. | mechanical | doc-only | no | nothing | — | — | | `sim.observability.header-transport-only` | observability_id travels only in the X-PolicyEngine-Observability-Id header, never in a simulation JSON body, and polling never invents one. | mechanical | doc-only | no | nothing | — | — | | `sim.observability.id-is-diagnostic-only` | observability_id has diagnostic meaning only: never use it for idempotency, authorization, database identity, routing or business logic. | mechanical | doc-only | no | nothing | — | — | -| `sim.observability.log-fields-through-log-extra` | Every Simulation Entrypoint log call that passes fields passes them through log_extra, so they reach the emitted record, and names none with a word the observability package drops (household, reform, payload, person and others). | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_log_fields.py` | `pr.yml:test` | yes | +| `sim.observability.log-fields-through-log-extra` | Every Simulation Entrypoint log call that passes fields passes them through log_extra, so they reach the emitted record, and no field name written as a keyword in that call contains a word the observability package drops (household, reform, payload, person and others). | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_log_fields.py` | `pr.yml:test` | yes | | `sim.observability.registered-stages-only` | Span names come from the stage plans in policyengine_simulation_observability.stages; runtime code never adds span-name string literals. | mechanical | doc-only | no | nothing | — | — | ```mermaid diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py index 30dc6456c..6387f2a7d 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py @@ -128,7 +128,7 @@ logger = logging.getLogger(__name__) # Every module of the package logs under this logger. -PACKAGE_LOGGER_NAME = "policyengine_simulation_entry" +PACKAGE_LOGGER_NAME = __name__.rpartition(".")[0] SIMULATION_DATABASE_READINESS_TIMEOUT_SECONDS = 3.0 # The database ends each statement of a household route that runs longer. HOUSEHOLD_STATEMENT_TIMEOUT_MILLISECONDS = 5000 diff --git a/projects/policyengine-simulation-entry/tests/test_budget_window_submissions.py b/projects/policyengine-simulation-entry/tests/test_budget_window_submissions.py index bca9c8aaf..d799c7548 100644 --- a/projects/policyengine-simulation-entry/tests/test_budget_window_submissions.py +++ b/projects/policyengine-simulation-entry/tests/test_budget_window_submissions.py @@ -526,5 +526,6 @@ def test_a_later_year_modal_refuses_is_logged_naming_the_report(caplog) -> None: "reports", str(store.windows[0].report_id), ) - # The runtime emits no list, so the years started are one string. - assert isinstance(fields["started_years"], str) + # The runtime emits no list, so the years started (only the first, whose + # call Modal accepted) are one string. + assert fields["started_years"] == "2026" diff --git a/projects/policyengine-simulation-entry/tests/test_log_fields.py b/projects/policyengine-simulation-entry/tests/test_log_fields.py index cad9e1b65..8ee71f980 100644 --- a/projects/policyengine-simulation-entry/tests/test_log_fields.py +++ b/projects/policyengine-simulation-entry/tests/test_log_fields.py @@ -19,7 +19,7 @@ ServiceIdentity, ) from policyengine_observability.schema import normalize_attributes -from policyengine_simulation_entry.app import create_app +from policyengine_simulation_entry.app import PACKAGE_LOGGER_NAME, create_app PACKAGE_DIRECTORY = Path(policyengine_simulation_entry.__file__).parent PROOF_FIXTURES = ( @@ -132,7 +132,7 @@ def test_the_entrypoint_emits_its_info_level_log_records() -> None: # The root logger stays at WARNING, so without a level of its own an INFO # record, such as ``budget_window_year_deferred``, is discarded before the # observability runtime's handler sees it. - package_logger = logging.getLogger("policyengine_simulation_entry") + package_logger = logging.getLogger(PACKAGE_LOGGER_NAME) previous_level = package_logger.level # ``setLevel`` also clears the loggers' cached answers to ``isEnabledFor``. package_logger.setLevel(logging.NOTSET) @@ -144,7 +144,7 @@ def test_the_entrypoint_emits_its_info_level_log_records() -> None: ) assert logging.getLogger( - "policyengine_simulation_entry.budget_window_submissions" + f"{PACKAGE_LOGGER_NAME}.budget_window_submissions" ).isEnabledFor(logging.INFO) finally: package_logger.setLevel(previous_level) diff --git a/rules/registry.toml b/rules/registry.toml index e5a8209d4..3b8f56492 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -976,7 +976,7 @@ kind = "none" [[rule]] id = "sim.observability.log-fields-through-log-extra" category = "observability" -statement = "Every Simulation Entrypoint log call that passes fields passes them through log_extra, so they reach the emitted record, and names none with a word the observability package drops (household, reform, payload, person and others)." +statement = "Every Simulation Entrypoint log call that passes fields passes them through log_extra, so they reach the emitted record, and no field name written as a keyword in that call contains a word the observability package drops (household, reform, payload, person and others)." rationale = "docs/engineering/skills/observability.md#standard-library-log-fields" scope = ["projects/policyengine-simulation-entry/src/**/*.py"] tier = "mechanical" From ed4d5505bf2d8cd08eee1d9255b1000e903051b4 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 19 Nov 2026 02:05:15 +0400 Subject: [PATCH 09/10] Send the household outcome-unknown log through log_extra after merging main #1090 added `household_job_outcome_unknown`, which built `policyengine_attributes` by hand; the rule this branch adds requires `log_extra`. The emitted fields don't change. Its docstring now gives the current reason it names no job. The text this branch added says API v1. Co-Authored-By: Claude Opus 5.5 --- changelog.d/c1-34-v2-telemetry-record-ids.added.md | 2 +- changelog.d/c1-34-v2-telemetry-record-ids.fixed.md | 2 +- docs/engineering/skills/observability.md | 2 +- docs/generated/rules.md | 4 ++-- .../policyengine_simulation_observability/log_records.py | 6 +++--- .../economy_submission_errors.py | 2 +- .../src/policyengine_simulation_entry/household_routes.py | 7 +++---- .../stage12_runtime/operation_records.py | 2 +- 8 files changed, 13 insertions(+), 14 deletions(-) diff --git a/changelog.d/c1-34-v2-telemetry-record-ids.added.md b/changelog.d/c1-34-v2-telemetry-record-ids.added.md index d328e77f7..19464ae41 100644 --- a/changelog.d/c1-34-v2-telemetry-record-ids.added.md +++ b/changelog.d/c1-34-v2-telemetry-record-ids.added.md @@ -1 +1 @@ -Each v2 Modal function's telemetry operation now names the Public API row its call works on and its own Modal call, so an operation that Modal interrupted leads straight to its row: `database_table` (`household_jobs`, `report_runs` for a report run or a household report run, or `simulation_runs`), `database_row_id` and `modal_call_id`. A US segment worker names the simulation run of the worker that started it. The IDs go to the operation's log records and span, not to metrics. +Each v2 Modal function's telemetry operation now names the API v1 row its call works on and its own Modal call, so an operation that Modal interrupted leads straight to its row: `database_table` (`household_jobs`, `report_runs` for a report run or a household report run, or `simulation_runs`), `database_row_id` and `modal_call_id`. A US segment worker names the simulation run of the worker that started it. The IDs go to the operation's log records and span, not to metrics. diff --git a/changelog.d/c1-34-v2-telemetry-record-ids.fixed.md b/changelog.d/c1-34-v2-telemetry-record-ids.fixed.md index a78a9fc7a..1a0c8f88c 100644 --- a/changelog.d/c1-34-v2-telemetry-record-ids.fixed.md +++ b/changelog.d/c1-34-v2-telemetry-record-ids.fixed.md @@ -1 +1 @@ -The Simulation Entrypoint's log records now carry their fields. 31 of its 33 log calls passed their fields only as standard-library `extra=` keys, which the observability runtime doesn't emit, so their records in Cloud Logging named no job, report, run or error type. Every log call now passes its fields through `log_extra` in the shared observability library, and names a Public API row with `database_table` (`household_jobs`, `reports`, `report_runs` or `simulations`) and `database_row_id`: a key such as `household_job_id` would be dropped by the runtime's key filter. A budget window's `started_years` are now joined into one string, since the runtime drops lists. The Entrypoint also sets its package logger to INFO, so its INFO records (a deferred budget-window year, a refused bearer token) reach the runtime: the root logger's WARNING level discarded them. +The Simulation Entrypoint's log records now carry their fields. 31 of its 33 log calls passed their fields only as standard-library `extra=` keys, which the observability runtime doesn't emit, so their records in Cloud Logging named no job, report, run or error type. Every log call now passes its fields through `log_extra` in the shared observability library, and names an API v1 row with `database_table` (`household_jobs`, `reports`, `report_runs` or `simulations`) and `database_row_id`: a key such as `household_job_id` would be dropped by the runtime's key filter. A budget window's `started_years` are now joined into one string, since the runtime drops lists. The Entrypoint also sets its package logger to INFO, so its INFO records (a deferred budget-window year, a refused bearer token) reach the runtime: the root logger's WARNING level discarded them. diff --git a/docs/engineering/skills/observability.md b/docs/engineering/skills/observability.md index 9510a5a9f..46b5e628f 100644 --- a/docs/engineering/skills/observability.md +++ b/docs/engineering/skills/observability.md @@ -16,7 +16,7 @@ Use each identifier for its defined scope: | `evaluation_id` | One Stage 12 report | Stage 12 report construction | Yes, as functional report state | | `simulation_execution_id` | One Stage 12 baseline or reform simulation | Stage 12 coordinator | Yes, as functional simulation state | | `submission_claim_id` | One API v1 attempt to acquire ownership of a simulation submission | API v1 | Only where the simulation request is stored | -| `database_table` | The Public API table holding the row a v2 Modal function call, or a Simulation Entrypoint log record, is about: `household_jobs`, `report_runs` (a report run or a household report run), `simulation_runs`, `reports` or `simulations` | The code writing the record, as a fixed name | No: telemetry only | +| `database_table` | The API v1 table holding the row a v2 Modal function call, or a Simulation Entrypoint log record, is about: `household_jobs`, `report_runs` (a report run or a household report run), `simulation_runs`, `reports` or `simulations` | The code writing the record, as a fixed name | No: telemetry only | | `database_row_id` | That row's UUID | The simulation service, which creates the row | Yes, as the row's ID | | `modal_call_id` | One Modal function call of a v2 Modal function | Modal | Only where the row records the call that claimed it; a segment worker's call is recorded nowhere, and its `database_row_id` names the simulation run its simulation worker's call claimed | diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 5511fd8d6..310ba2c36 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -8,11 +8,11 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf | Tier | enforced | ratcheted | advisory | doc-only | Total | | --- | ---: | ---: | ---: | ---: | ---: | -| mechanical | 65 | 0 | 0 | 14 | 79 | +| mechanical | 66 | 0 | 0 | 14 | 80 | | metric | 1 | 0 | 0 | 0 | 1 | | judgment | 0 | 0 | 0 | 0 | 0 | -**Blocking:** 66 of 80 rules. +**Blocking:** 67 of 81 rules. ## Gaps diff --git a/libs/policyengine-simulation-observability/src/policyengine_simulation_observability/log_records.py b/libs/policyengine-simulation-observability/src/policyengine_simulation_observability/log_records.py index 0d422d4bd..a35f1c07c 100644 --- a/libs/policyengine-simulation-observability/src/policyengine_simulation_observability/log_records.py +++ b/libs/policyengine-simulation-observability/src/policyengine_simulation_observability/log_records.py @@ -1,4 +1,4 @@ -"""Telemetry fields that name a Public API row, and log fields that are emitted. +"""Telemetry fields that name an API v1 row, and log fields that are emitted. ``policyengine-observability``'s standard-library log handler emits only a log record's ``policyengine_attributes``; every other key passed in @@ -21,7 +21,7 @@ DATABASE_ROW_ID_ATTRIBUTE = "database_row_id" MODAL_CALL_ID_ATTRIBUTE = "modal_call_id" -# The Public API tables whose rows telemetry names. +# The API v1 tables whose rows telemetry names. DatabaseTable = Literal[ "household_jobs", "report_runs", "reports", "simulation_runs", "simulations" ] @@ -35,7 +35,7 @@ def database_row_attributes(table: DatabaseTable, row_id: UUID) -> dict[str, str]: - """Return the attributes naming row ``row_id`` of the Public API's ``table``.""" + """Return the attributes naming row ``row_id`` of API v1's ``table``.""" return {DATABASE_TABLE_ATTRIBUTE: table, DATABASE_ROW_ID_ATTRIBUTE: str(row_id)} diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_submission_errors.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_submission_errors.py index 07ff877e1..e3fda6233 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_submission_errors.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/economy_submission_errors.py @@ -67,7 +67,7 @@ } # What a submission creates, or a re-run calculates again. RecordName = Literal["report", "simulation"] -# The Public API table holding each record, as telemetry names it. +# The API v1 table holding each record, as telemetry names it. RECORD_TABLES: Final[dict[RecordName, DatabaseTable]] = { "report": "reports", "simulation": "simulations", diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_routes.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_routes.py index c58b172e1..3164b559a 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_routes.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/household_routes.py @@ -33,6 +33,7 @@ from fastapi import APIRouter, Depends, Request, status from fastapi.responses import JSONResponse +from policyengine_simulation_observability.log_records import log_extra from policyengine_simulation_entry.household_api_models import ( HouseholdCalculationAccepted, @@ -184,13 +185,11 @@ def _outcome_unknown_response( nothing was created. The log is the fixed event ``household_job_outcome_unknown`` with the commit error's type, never its text, as ``economy_record_outcome_unknown`` logs. It names no job: - the observability runtime drops every attribute whose key contains - ``household``, and ``job_id`` names a Modal invocation. + the error carries only that type. """ - fields = {"error_type": error.error_type} logger.error( "household_job_outcome_unknown", - extra={**fields, "policyengine_attributes": fields}, + extra=log_extra(error_type=error.error_type), ) return _error_response( status.HTTP_500_INTERNAL_SERVER_ERROR, diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/operation_records.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/operation_records.py index 0ef1e860b..18b2f6010 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/operation_records.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/operation_records.py @@ -2,7 +2,7 @@ Each v2 Modal function opens one telemetry operation. Its attributes reach the operation's log records and span, but not its metrics, so an operation -that Modal interrupted in production leads straight to the Public API row it +that Modal interrupted in production leads straight to the API v1 row it was working on: - ``database_table``: the table holding that row, ``household_jobs``, From 77c425d4db76a089e9293970ee34b80ee5b7769b Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 19 Nov 2026 02:16:30 +0400 Subject: [PATCH 10/10] Correct the log-fields changelog after merging main With #1090 merged, 31 of the Entrypoint's 34 log calls that pass fields were affected, and only the calls about a row name one. Co-Authored-By: Claude Opus 5.5 --- changelog.d/c1-34-v2-telemetry-record-ids.fixed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/c1-34-v2-telemetry-record-ids.fixed.md b/changelog.d/c1-34-v2-telemetry-record-ids.fixed.md index 1a0c8f88c..418e48ff5 100644 --- a/changelog.d/c1-34-v2-telemetry-record-ids.fixed.md +++ b/changelog.d/c1-34-v2-telemetry-record-ids.fixed.md @@ -1 +1 @@ -The Simulation Entrypoint's log records now carry their fields. 31 of its 33 log calls passed their fields only as standard-library `extra=` keys, which the observability runtime doesn't emit, so their records in Cloud Logging named no job, report, run or error type. Every log call now passes its fields through `log_extra` in the shared observability library, and names an API v1 row with `database_table` (`household_jobs`, `reports`, `report_runs` or `simulations`) and `database_row_id`: a key such as `household_job_id` would be dropped by the runtime's key filter. A budget window's `started_years` are now joined into one string, since the runtime drops lists. The Entrypoint also sets its package logger to INFO, so its INFO records (a deferred budget-window year, a refused bearer token) reach the runtime: the root logger's WARNING level discarded them. +The Simulation Entrypoint's log records now carry their fields. 31 of its 34 log calls that pass fields passed them only as standard-library `extra=` keys, which the observability runtime doesn't emit, so their records in Cloud Logging named no job, report, run or error type. Every log call now passes its fields through `log_extra` in the shared observability library, and a call about an API v1 row names it with `database_table` (`household_jobs`, `reports`, `report_runs` or `simulations`) and `database_row_id`: a key such as `household_job_id` would be dropped by the runtime's key filter. A budget window's `started_years` are now joined into one string, since the runtime drops lists. The Entrypoint also sets its package logger to INFO, so its INFO records (a deferred budget-window year, a refused bearer token) reach the runtime: the root logger's WARNING level discarded them.