diff --git a/changelog.d/c1-34-stage12-log-fields.fixed.md b/changelog.d/c1-34-stage12-log-fields.fixed.md new file mode 100644 index 000000000..8ca15b3a4 --- /dev/null +++ b/changelog.d/c1-34-stage12-log-fields.fixed.md @@ -0,0 +1 @@ +The Stage 12 runtime's log records now carry their fields: the result comparison's three logs and a child simulation's failed state update or cancellation passed them only as standard-library `extra=` keys, which the observability runtime doesn't emit. They now go through `log_extra`. The Stage 12 runtime's logger is also set to INFO, so its INFO records, such as `stage12_result_comparison_completed`, are no longer discarded by the root logger's WARNING level. diff --git a/docs/engineering/skills/observability.md b/docs/engineering/skills/observability.md index 46b5e628f..86528740d 100644 --- a/docs/engineering/skills/observability.md +++ b/docs/engineering/skills/observability.md @@ -40,17 +40,20 @@ requests, missing jobs, or ordinary status requests. The observability runtime's standard-library log handler emits only a log record's `policyengine_attributes`: any other key passed in `extra=` stays on the Python `LogRecord` and never reaches Cloud Logging. In the Simulation -Entrypoint, pass every log call's fields through `log_extra(...)` from +Entrypoint and in Stage 12 (the executor's `stage12_*` modules and the +Stage 12 and v2 Modal applications), pass every log call's fields through `log_extra(...)` from `policyengine_simulation_observability.log_records`, which puts them in both places (a field named like a `LogRecord` attribute, such as `name`, goes only to `policyengine_attributes`, since `logging` refuses it), and name a row with `database_row_attributes(table, row_id)` (rule `sim.observability.log-fields-through-log-extra`; its check reads only the field names written as keywords in the `log_extra(...)` call, not keys unpacked from a dictionary with `**`). Each field must be a -string, a number or a boolean; the runtime drops a list. The Entrypoint sets -its package logger to INFO: the root logger stays at WARNING, which would -discard INFO records before the handler sees them. Some Stage 12 runtime and -contract-library logs still pass plain `extra=` fields, which are not emitted. +string, a number or a boolean; the runtime drops a list. The Entrypoint's +package logger and `stage12_runtime`'s are set to INFO: the root logger stays +at WARNING, which would discard INFO records before the handler sees them. The +v1 code's logs are left as they are, so `log_and_redact_exception` (whose +fields also reach telemetry as its `gateway_error_redacted` event) and the +contract library's `budget_window_state` still pass plain `extra=` fields. ## HTTP transport diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 310ba2c36..4e43f8679 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -311,7 +311,7 @@ flowchart LR | `sim.observability.failures-never-break-requests` | Invalid observability configuration fails at build or deploy time, and telemetry failures at runtime never fail a request or computation. | mechanical | doc-only | no | nothing | — | — | | `sim.observability.header-transport-only` | observability_id travels only in the X-PolicyEngine-Observability-Id header, never in a simulation JSON body, and polling never invents one. | mechanical | doc-only | no | nothing | — | — | | `sim.observability.id-is-diagnostic-only` | observability_id has diagnostic meaning only: never use it for idempotency, authorization, database identity, routing or business logic. | mechanical | doc-only | no | nothing | — | — | -| `sim.observability.log-fields-through-log-extra` | Every Simulation Entrypoint log call that passes fields passes them through log_extra, so they reach the emitted record, and no field name written as a keyword in that call contains a word the observability package drops (household, reform, payload, person and others). | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_log_fields.py` | `pr.yml:test` | yes | +| `sim.observability.log-fields-through-log-extra` | Every Simulation Entrypoint and Stage 12 (the executor's stage12_* modules and the Stage 12 and v2 Modal applications) log call that passes fields passes them through log_extra, so they reach the emitted record, and no field name written as a keyword in that call contains a word the observability package drops (household, reform, payload, person and others). | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_log_fields.py` | `pr.yml:test` | yes | | `sim.observability.registered-stages-only` | Span names come from the stage plans in policyengine_simulation_observability.stages; runtime code never adds span-name string literals. | mechanical | doc-only | no | nothing | — | — | ```mermaid diff --git a/projects/policyengine-simulation-entry/tests/test_log_fields.py b/projects/policyengine-simulation-entry/tests/test_log_fields.py index 8ee71f980..451a76fcc 100644 --- a/projects/policyengine-simulation-entry/tests/test_log_fields.py +++ b/projects/policyengine-simulation-entry/tests/test_log_fields.py @@ -1,4 +1,7 @@ -"""Every Simulation Entrypoint log call's fields reach the emitted log record. +"""Every Simulation Entrypoint and Stage 12 log call's fields reach the log record. + +Stage 12 here is every executor ``stage12_*`` module and package, and the +Stage 12 and v2 Modal applications. The v1 code's logs are left as they are. The observability runtime's log handler keeps only a record's ``policyengine_attributes``, and drops each of those whose key contains @@ -22,8 +25,29 @@ from policyengine_simulation_entry.app import PACKAGE_LOGGER_NAME, create_app PACKAGE_DIRECTORY = Path(policyengine_simulation_entry.__file__).parent +REPOSITORY_ROOT = Path(__file__).resolve().parents[3] +EXECUTOR_SOURCE = ( + REPOSITORY_ROOT / "projects" / "policyengine-simulation-executor" / "src" +) +EXECUTOR_PACKAGE = EXECUTOR_SOURCE / "policyengine_simulation_executor" +# Every Stage 12 module: the executor package's ``stage12_*`` modules and +# packages, and the Stage 12 and v2 Modal applications. +STAGE12_SOURCES = sorted( + { + *( + path + for stage12_path in EXECUTOR_PACKAGE.glob("stage12_*") + for path in ( + stage12_path.rglob("*.py") if stage12_path.is_dir() else [stage12_path] + ) + ), + *(EXECUTOR_SOURCE / "modal").glob("stage12_*.py"), + *(EXECUTOR_SOURCE / "modal").glob("v2_*.py"), + EXECUTOR_SOURCE / "modal" / "run_sweep_app.py", + } +) PROOF_FIXTURES = ( - Path(__file__).resolve().parents[3] + REPOSITORY_ROOT / "rules" / "fixtures" / "sim.observability.log-fields-through-log-extra" @@ -98,12 +122,15 @@ def _dropped_field_names(calls: list[tuple[str, ast.expr]]) -> list[tuple[str, s def _package_log_calls() -> list[tuple[str, ast.expr]]: - return _log_calls_with_extra(list(PACKAGE_DIRECTORY.rglob("*.py"))) + return _log_calls_with_extra([*PACKAGE_DIRECTORY.rglob("*.py"), *STAGE12_SOURCES]) def test_the_package_has_log_calls_with_fields() -> None: - # Guards the next tests against finding nothing to check. - assert len(_package_log_calls()) >= 30 + # Guards the next tests against finding nothing to check: the + # Entrypoint's 33 calls and the Stage 12 runtime's four, counted apart so + # a moved Stage 12 package is noticed. + assert len(_log_calls_with_extra(list(PACKAGE_DIRECTORY.rglob("*.py")))) >= 33 + assert len(_log_calls_with_extra(STAGE12_SOURCES)) >= 4 def test_every_log_call_passes_its_fields_through_log_extra() -> None: diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/__init__.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/__init__.py index 44b6a58a0..723ebc8d8 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/__init__.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/__init__.py @@ -1,5 +1,7 @@ """Independent Stage 12 simulation execution and report coordination.""" +import logging + from .aggregation import ( RetainedSimulationOutput, build_aggregate_report, @@ -56,6 +58,11 @@ # locating the implementation with the rest of the aggregation logic. _build_spm_result = build_spm_result +# The root logger stays at WARNING in a worker container, which would discard +# this package's INFO records before the observability runtime's handler sees +# them. Only Stage 12 processes import this package; the v1 code does not. +logging.getLogger(__name__).setLevel(logging.INFO) + __all__ = [ "CoordinatorDependencies", "CoordinatorInvocation", diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/comparison.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/comparison.py index fefc8281b..297c21f3e 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/comparison.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/comparison.py @@ -13,6 +13,7 @@ ResultComparisonStatus, Stage12InvocationContext, ) +from policyengine_simulation_observability.log_records import log_extra from policyengine_simulation_observability.stages import Stage, StagePlan from policyengine_simulation_executor.stage12_result_comparison import compare_results @@ -111,14 +112,14 @@ def compare_completed_report( ) logger.info( "stage12_result_comparison_completed", - extra={ - "evaluation_id": str(parent.evaluation_id), - "production_job_id": production_job_id, - "comparison_status": comparison_status.value, - "difference_count": comparison.difference_count, - "production_result_sha256": comparison.production_result_sha256, - "stage12_result_sha256": comparison.stage12_result_sha256, - }, + extra=log_extra( + evaluation_id=str(parent.evaluation_id), + production_job_id=production_job_id, + comparison_status=comparison_status.value, + difference_count=comparison.difference_count, + production_result_sha256=comparison.production_result_sha256, + stage12_result_sha256=comparison.stage12_result_sha256, + ), ) # Result retrieval, validation, artifact storage, and persistence are all # deliberately isolated from the successful Stage 12 calculation. @@ -147,16 +148,16 @@ def compare_completed_report( except Exception: logger.exception( "stage12_result_comparison_state_update_failed", - extra={ - "evaluation_id": str(parent.evaluation_id), - "production_job_id": production_job_id, - }, + extra=log_extra( + evaluation_id=str(parent.evaluation_id), + production_job_id=production_job_id, + ), ) logger.error( # noqa: TRY400 # Stage 12 failure logs carry only the error type; a traceback could expose request or calculation data "stage12_result_comparison_failed", - extra={ - "evaluation_id": str(parent.evaluation_id), - "production_job_id": production_job_id, - "error_type": type(error).__name__, - }, + extra=log_extra( + evaluation_id=str(parent.evaluation_id), + production_job_id=production_job_id, + error_type=type(error).__name__, + ), ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/coordinated_simulations.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/coordinated_simulations.py index 86016e19b..22540fef2 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/coordinated_simulations.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/coordinated_simulations.py @@ -12,6 +12,7 @@ PlannedSimulationExecutionInput, Stage12InvocationContext, ) +from policyengine_simulation_observability.log_records import log_extra from .failure_recording import FailureSite, report_failure from .failures import Stage12FailureDetail, failure_detail_from_record @@ -51,12 +52,12 @@ def _log_child_operation_failure( simulation.simulation_execution_id, simulation.evaluation_id, error_type, - extra={ - "evaluation_id": str(simulation.evaluation_id), - "simulation_execution_id": str(simulation.simulation_execution_id), - "simulation_role": simulation.role.value, - "error_type": error_type, - }, + extra=log_extra( + evaluation_id=str(simulation.evaluation_id), + simulation_execution_id=str(simulation.simulation_execution_id), + simulation_role=simulation.role.value, + error_type=error_type, + ), ) diff --git a/projects/policyengine-simulation-executor/tests/stage12_child_call_support.py b/projects/policyengine-simulation-executor/tests/stage12_child_call_support.py index 097888f11..e56344ea3 100644 --- a/projects/policyengine-simulation-executor/tests/stage12_child_call_support.py +++ b/projects/policyengine-simulation-executor/tests/stage12_child_call_support.py @@ -402,6 +402,13 @@ def assert_log_identifies_child(record, *, role: str, error_type: str) -> None: assert record.simulation_execution_id == simulation_execution_id assert record.simulation_role == role assert record.error_type == error_type + # The observability runtime's log handler emits only these. + assert record.policyengine_attributes == { + "evaluation_id": str(EVALUATION_ID), + "simulation_execution_id": simulation_execution_id, + "simulation_role": role, + "error_type": error_type, + } message = record.getMessage() assert str(EVALUATION_ID) in message assert simulation_execution_id in message diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_coordination.py b/projects/policyengine-simulation-executor/tests/test_stage12_coordination.py index 8af76d3a2..cba59a96f 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_coordination.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_coordination.py @@ -635,6 +635,15 @@ def test_shadow_comparison_logs_its_outcome_and_times_its_stage(caplog): assert record.production_job_id == "production-job-1" assert record.comparison_status == "matched" assert record.difference_count == 0 + # The observability runtime's log handler emits only these. + assert set(record.policyengine_attributes) == { + "evaluation_id", + "production_job_id", + "comparison_status", + "difference_count", + "production_result_sha256", + "stage12_result_sha256", + } assert store.parent.comparison_status is ResultComparisonStatus.MATCHED @@ -675,3 +684,13 @@ def unavailable_report(_evaluation_id): assert failure_record.levelno == logging.ERROR assert failure_record.exc_info is None assert failure_record.error_type == "ValidationError" + # The observability runtime's log handler emits only these. + assert state_record.policyengine_attributes == { + "evaluation_id": str(EVALUATION_ID), + "production_job_id": "production-job-1", + } + assert failure_record.policyengine_attributes == { + "evaluation_id": str(EVALUATION_ID), + "production_job_id": "production-job-1", + "error_type": "ValidationError", + } diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_log_level.py b/projects/policyengine-simulation-executor/tests/test_stage12_log_level.py new file mode 100644 index 000000000..558793607 --- /dev/null +++ b/projects/policyengine-simulation-executor/tests/test_stage12_log_level.py @@ -0,0 +1,21 @@ +"""Stage 12 runtime INFO records reach the observability runtime's handler.""" + +from __future__ import annotations + +import logging + + +def test_stage12_runtime_info_records_are_not_discarded_by_the_root_level() -> None: + # The root logger stays at WARNING in a worker container, which would + # discard INFO records, such as ``stage12_result_comparison_completed``, + # before the observability runtime's handler sees them. + import policyengine_simulation_executor.stage12_runtime # noqa: F401 # the import sets the level + + # The package's own level, not a lower root level, enables them. + assert ( + logging.getLogger("policyengine_simulation_executor.stage12_runtime").level + == logging.INFO + ) + assert logging.getLogger( + "policyengine_simulation_executor.stage12_runtime.comparison" + ).isEnabledFor(logging.INFO) diff --git a/rules/registry.toml b/rules/registry.toml index 9a29f2f7c..27c5beb7b 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -1011,9 +1011,9 @@ kind = "none" [[rule]] id = "sim.observability.log-fields-through-log-extra" category = "observability" -statement = "Every Simulation Entrypoint log call that passes fields passes them through log_extra, so they reach the emitted record, and no field name written as a keyword in that call contains a word the observability package drops (household, reform, payload, person and others)." +statement = "Every Simulation Entrypoint and Stage 12 (the executor's stage12_* modules and the Stage 12 and v2 Modal applications) log call that passes fields passes them through log_extra, so they reach the emitted record, and no field name written as a keyword in that call contains a word the observability package drops (household, reform, payload, person and others)." rationale = "docs/engineering/skills/observability.md#standard-library-log-fields" -scope = ["projects/policyengine-simulation-entry/src/**/*.py"] +scope = ["projects/policyengine-simulation-entry/src/**/*.py", "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_*", "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_*/**/*.py", "projects/policyengine-simulation-executor/src/modal/stage12_*.py", "projects/policyengine-simulation-executor/src/modal/v2_*.py", "projects/policyengine-simulation-executor/src/modal/run_sweep_app.py"] tier = "mechanical" status = "enforced" autofix = false