From f317711f05de2142189b2b3ec79b29417285aee2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 16 Nov 2026 16:23:09 +0000 Subject: [PATCH 01/18] Update PolicyEngine bundle to 6.20.0 --- ...date-policyengine-bundle-6.20.0.changed.md | 1 + docker/Dockerfile | 2 +- pyproject.toml | 2 +- uv.lock | 20 +++++++++---------- 4 files changed, 13 insertions(+), 12 deletions(-) create mode 100644 changelog.d/update-policyengine-bundle-6.20.0.changed.md diff --git a/changelog.d/update-policyengine-bundle-6.20.0.changed.md b/changelog.d/update-policyengine-bundle-6.20.0.changed.md new file mode 100644 index 000000000..a2a4c53aa --- /dev/null +++ b/changelog.d/update-policyengine-bundle-6.20.0.changed.md @@ -0,0 +1 @@ +Update the PolicyEngine bundle to 6.20.0. diff --git a/docker/Dockerfile b/docker/Dockerfile index 131fdc900..0152908eb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ FROM python:3.12 # Match the API bundle in pyproject.toml; the bundle updater changes both pins. # The models extra pins the complete certified runtime package set. -RUN pip install "policyengine[models]==6.17.1" ipython +RUN pip install "policyengine[models]==6.20.0" ipython diff --git a/pyproject.toml b/pyproject.toml index 2a96b7045..e0b77aeba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ dependencies = [ # versions certified by this PolicyEngine bundle. Cloud Run installs the # complete frozen environment from uv.lock; the generic image installs # this requirement directly. - "policyengine[models]==6.17.1", + "policyengine[models]==6.20.0", "pydantic", "pymysql", "python-dotenv", diff --git a/uv.lock b/uv.lock index 9199ff28d..a749c9c2f 100644 --- a/uv.lock +++ b/uv.lock @@ -2954,7 +2954,7 @@ wheels = [ [[package]] name = "policyengine" -version = "6.17.1" +version = "6.20.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "h5py" }, @@ -2966,9 +2966,9 @@ dependencies = [ { name = "pydantic" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/da/45/483142f35ce0dfa02db01e541f730843b9108ae4b60a8bdae92ac223f3b2/policyengine-6.17.1.tar.gz", hash = "sha256:07a2617c738a77d2f1fd8ff2c5f4d89b040a54f44e9f931331a7ae34fdbbfc9f", size = 1629731, upload-time = "2026-11-10T16:26:24.672Z" } +sdist = { url = "https://files.pythonhosted.org/packages/14/e8/4373c716f428ed885392ff1776dc1be1f9e9a3a6f7608afcc1f39d683546/policyengine-6.20.0.tar.gz", hash = "sha256:65dad563d92c17485c5943b2bb8114d8f16cd49c977acbce6412cd8b8bcfadfd", size = 1863269, upload-time = "2026-11-16T16:21:45.756Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1f/d5/e8b6181f512e5c3b14bbd0638a4c179515a9f479a68a6eaba640005b0fad/policyengine-6.17.1-py3-none-any.whl", hash = "sha256:7fa63f9f6524d911ff7a490448a2b202341487dc09d5c945fc0e0c816893df1e", size = 562776, upload-time = "2026-11-10T16:26:22.717Z" }, + { url = "https://files.pythonhosted.org/packages/64/b6/9ebed9a78c755c25893a20c924bc6952100bc3376608511fbcdb065145c3/policyengine-6.20.0-py3-none-any.whl", hash = "sha256:cc2da1b7448ae229294090318140bfe39440e8899415b8e465459c7e89bb7135", size = 774390, upload-time = "2026-11-16T16:21:43.414Z" }, ] [package.optional-dependencies] @@ -3072,7 +3072,7 @@ requires-dist = [ { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.15,<2" }, { name = "openai" }, { name = "packaging", specifier = ">=24,<27" }, - { name = "policyengine", extras = ["models"], specifier = "==6.17.1" }, + { name = "policyengine", extras = ["models"], specifier = "==6.20.0" }, { name = "policyengine-observability", extras = ["flask", "google", "httpx", "otlp-grpc"], specifier = ">=3.0.3,<4" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.3,<4" }, { name = "pydantic" }, @@ -3169,7 +3169,7 @@ wheels = [ [[package]] name = "policyengine-us" -version = "2.2.1" +version = "2.2.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -3179,9 +3179,9 @@ dependencies = [ { name = "tables" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2c/09/763b21371a62635e15e62b53fb696e299f01460907c27313d40f63031eb2/policyengine_us-2.2.1.tar.gz", hash = "sha256:d5300c91f3eae2e47a61f485fdfde91a892b80e95db3e46ff2c4bbd417d76501", size = 11946510, upload-time = "2026-09-15T04:45:50.33Z" } +sdist = { url = "https://files.pythonhosted.org/packages/72/86/1bbc7ab8d689987b364e624ef550a7a43ec49343b863be39b267a7c3477b/policyengine_us-2.2.2.tar.gz", hash = "sha256:a7cd4383d82e2453a3433e49d5501fd21b01b43b3910a66954d0a3cfe65f422b", size = 11947023, upload-time = "2026-11-09T20:52:43.712Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/09/86ac3968d2a18e28fce3ba1186fc21728e9f299b96a806d6f204f1e0aaed/policyengine_us-2.2.1-py3-none-any.whl", hash = "sha256:0993a6c73fcdfbe171a796aca9d302741bd8e00ab83388ad81c15a08740318c6", size = 14885537, upload-time = "2026-09-15T04:45:46.649Z" }, + { url = "https://files.pythonhosted.org/packages/84/89/12a44d68e4131cc6ad92cfa22959254c69306d1f25fa6760ea1aab851bcb/policyengine_us-2.2.2-py3-none-any.whl", hash = "sha256:82ac620444dc0f246559fcbf7b34daee5a921a57684b63121f319365820811ce", size = 14885748, upload-time = "2026-11-09T20:52:38.979Z" }, ] [[package]] @@ -4109,7 +4109,7 @@ wheels = [ [[package]] name = "spm-calculator" -version = "1.0.0" +version = "1.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "census" }, @@ -4119,9 +4119,9 @@ dependencies = [ { name = "requests" }, { name = "us" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c2/5e/3f28b212c401795990250bb3daec741f699a8c7dc2a23a3db7005df6feb6/spm_calculator-1.0.0.tar.gz", hash = "sha256:a99aac8c2c0bf81a9455105bbf872366f1ea9066a99cd793714881cd3db846dc", size = 7150825, upload-time = "2026-09-11T15:45:12.732Z" } +sdist = { url = "https://files.pythonhosted.org/packages/33/84/19057fa04e6c8cce85494458b2164d473ce82cbf54608a3b2951af2c36b1/spm_calculator-1.1.0.tar.gz", hash = "sha256:1be418c6a1baaba86f63c3287b38efecf7f94af563b864a73934ecbf60a01f38", size = 9625137, upload-time = "2026-11-02T18:48:45.002Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/19/a0/c484f69a0ebf88a9b9fd0ac28176f8df66550f46714f1b0c5ec0b360824e/spm_calculator-1.0.0-py3-none-any.whl", hash = "sha256:e354937a5e1a4045d4966ed594a528d8b02866fabaac9bb5672017004b627305", size = 7395384, upload-time = "2026-09-11T15:45:10.492Z" }, + { url = "https://files.pythonhosted.org/packages/35/91/12a744eb7f3445db8899dc187bbd193c884f9fe4c373851c0ddb24cc0f5e/spm_calculator-1.1.0-py3-none-any.whl", hash = "sha256:c528ec0f06eeb1188b7d6cf456ff3a9b73a0fd8856b3938358251cc5f9be83e8", size = 9881340, upload-time = "2026-11-02T18:48:42.665Z" }, ] [[package]] From fa5b1c72422a7e3a0e40b7c53aed4c90853373f5 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 00:37:41 +0400 Subject: [PATCH 02/18] Test revoking whole-table privileges declared for named columns (failing) A new table's default privileges give the runtime role UPDATE on the whole table. Where the manifest declares UPDATE on named columns only, as for user_type_changes, apply reports that privilege but never revokes it, so verify fails every deploy. These tests expect apply to revoke it, grant the declared columns again, and leave every other extra privilege. Co-Authored-By: Claude Opus 5.5 --- .../test_v2_privilege_reconciliation.py | 49 +++++++ .../unit/v2/test_privilege_reconciliation.py | 121 +++++++++++++++++- 2 files changed, 167 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_v2_privilege_reconciliation.py b/tests/integration/test_v2_privilege_reconciliation.py index 97fdb3f34..c2a03b727 100644 --- a/tests/integration/test_v2_privilege_reconciliation.py +++ b/tests/integration/test_v2_privilege_reconciliation.py @@ -151,6 +151,55 @@ def test_verify_fails_when_a_role_can_update_the_whole_table( ) in summary +def test_apply_revokes_a_new_tables_whole_table_update_and_verify_passes( + disposable_v2_database_url, superuser_engine, temporary_role, monkeypatch, tmp_path +) -> None: + """A new table's default privileges give UPDATE on the whole table. + + ``apply`` revokes it and grants the declared column again in the same + transaction, so ``verify`` passes. The extra ``INSERT`` and ``DELETE`` + stay, reported but not revoked. + """ + role = temporary_role() + declared = {role: {"regions": READ | {update_columns("label")}}} + monkeypatch.setattr( + privilege_reconciliation, "declared_privileges", lambda _environment: declared + ) + monkeypatch.setattr( + privilege_reconciliation, "ROLES_AWAITING_PROVISIONING", frozenset() + ) + with superuser_engine.begin() as connection: + connection.execute( + text( + "GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE public.regions " + f'TO "{role}"' + ) + ) + summary_path = tmp_path / "summary.md" + environ = { + V2_MIGRATION_DATABASE_URL: disposable_v2_database_url, + V2_ALEMBIC_DISPOSABLE_TEST: "1", + DEPLOYMENT_ENVIRONMENT: "test", + "GITHUB_STEP_SUMMARY": str(summary_path), + } + + apply_status = run("apply", environ=environ) + verify_status = run("verify", environ=environ) + summary = summary_path.read_text(encoding="utf-8") + with superuser_engine.begin() as connection: + second_plan, _defaults = apply_declared_privileges(connection, declared) + + assert (apply_status, verify_status) == (0, 0) + assert f"| `{role}` | `regions` | UPDATE |" in summary + assert not _has_privilege(superuser_engine, role, "regions", "UPDATE") + assert _has_column_privilege(superuser_engine, role, "regions", "label", "UPDATE") + assert not _has_column_privilege( + superuser_engine, role, "regions", "code", "UPDATE" + ) + assert _has_privilege(superuser_engine, role, "regions", "DELETE") + assert (second_plan.revocations, second_plan.grants) == ((), ()) + + def test_apply_grants_nothing_the_second_time_and_verify_passes( disposable_v2_database_url, managed_v2_roles, capsys ) -> None: diff --git a/tests/unit/v2/test_privilege_reconciliation.py b/tests/unit/v2/test_privilege_reconciliation.py index 208a5a1e5..5d185f39d 100644 --- a/tests/unit/v2/test_privilege_reconciliation.py +++ b/tests/unit/v2/test_privilege_reconciliation.py @@ -16,6 +16,7 @@ grant_statements, plan_reconciliation, render_summary, + revoke_statements, table_wide_overrides, verification_failures, ) @@ -296,7 +297,7 @@ def test_column_and_whole_table_privileges_beyond_the_manifest_are_reported() -> plan = plan_reconciliation( WORKER_DECLARED, granted=GrantedPrivileges( - tables={WORKER: {"simulation_runs": {"SELECT", "UPDATE"}}}, + tables={WORKER: {"simulation_runs": {"SELECT", "DELETE"}}}, columns={ WORKER: { "simulation_runs": { @@ -311,9 +312,9 @@ def test_column_and_whole_table_privileges_beyond_the_manifest_are_reported() -> existing_tables={"simulation_runs"}, ) - assert plan.grants == () + assert (plan.grants, plan.revocations) == ((), ()) assert plan.undeclared == ( - PrivilegeEntry(WORKER, "simulation_runs", "UPDATE"), + PrivilegeEntry(WORKER, "simulation_runs", "DELETE"), PrivilegeEntry(WORKER, "simulation_runs", "UPDATE", column="simulation_id"), ) @@ -386,6 +387,120 @@ def has_privilege(role: str, table: str, privilege: str) -> bool: ) +def test_a_whole_table_privilege_declared_for_columns_is_revoked() -> None: + """A new table's default privileges give UPDATE on the whole table. + + Revoking it also removes the role's column grants of UPDATE, so the plan + grants the declared columns again and reports none of the removed ones. + """ + plan = plan_reconciliation( + WORKER_DECLARED, + granted=GrantedPrivileges( + tables={WORKER: {"simulation_runs": {"SELECT", "UPDATE", "DELETE"}}}, + columns={ + WORKER: { + "simulation_runs": { + "status": {"UPDATE"}, + "simulation_id": {"UPDATE"}, + } + } + }, + ), + existing_roles={WORKER}, + existing_tables={"simulation_runs"}, + ) + + assert plan.revocations == (PrivilegeEntry(WORKER, "simulation_runs", "UPDATE"),) + assert plan.grants == ( + PrivilegeEntry(WORKER, "simulation_runs", "UPDATE", column="completed_at"), + PrivilegeEntry(WORKER, "simulation_runs", "UPDATE", column="status"), + ) + assert plan.undeclared == (PrivilegeEntry(WORKER, "simulation_runs", "DELETE"),) + + +def test_nothing_is_revoked_while_a_declared_role_or_table_is_missing() -> None: + plan = plan_reconciliation( + WORKER_DECLARED, + granted=GrantedPrivileges( + tables={WORKER: {"simulation_runs": {"SELECT", "UPDATE"}}} + ), + existing_roles={WORKER}, + existing_tables=set(), + ) + + assert plan.missing_tables == ("simulation_runs",) + assert plan.revocations == () + + +def test_revoke_statements_revoke_each_role_and_tables_privileges() -> None: + plan = ReconciliationPlan( + grants=(), + undeclared=(), + missing_roles=(), + missing_tables=(), + revocations=( + PrivilegeEntry(WORKER, "simulation_runs", "UPDATE"), + PrivilegeEntry(RUNTIME, "user_type_changes", "UPDATE"), + ), + ) + + assert revoke_statements(plan, _quote) == [ + 'REVOKE UPDATE ON TABLE "public"."simulation_runs" ' + 'FROM "policyengine_v2_modal_worker"', + 'REVOKE UPDATE ON TABLE "public"."user_type_changes" ' + 'FROM "policyengine_v2_runtime"', + ] + + +def test_revoke_statements_refuse_an_unsafe_table_name() -> None: + plan = ReconciliationPlan( + grants=(), + undeclared=(), + missing_roles=(), + missing_tables=(), + revocations=(PrivilegeEntry(WORKER, 'runs"; DROP TABLE users; --', "UPDATE"),), + ) + + with pytest.raises(privilege_reconciliation.UnsafeIdentifierError): + revoke_statements(plan, _quote) + + +def test_summary_lists_revoked_whole_table_privileges_only_when_there_are_some() -> ( + None +): + revoking_plan = ReconciliationPlan( + grants=( + PrivilegeEntry(RUNTIME, "user_type_changes", "UPDATE", "dismissed_at"), + ), + undeclared=(), + missing_roles=(), + missing_tables=(), + revocations=(PrivilegeEntry(RUNTIME, "user_type_changes", "UPDATE"),), + ) + plain_plan = ReconciliationPlan( + grants=(), undeclared=(), missing_roles=(), missing_tables=() + ) + + applied = render_summary( + revoking_plan, (), deployment_environment="staging", applied=True + ) + planned = render_summary( + revoking_plan, (), deployment_environment="staging", applied=False + ) + plain = render_summary( + plain_plan, (), deployment_environment="staging", applied=True + ) + + revoked_row = "| `policyengine_v2_runtime` | `user_type_changes` | UPDATE |" + assert "### Revoked: whole-table privileges declared for named columns" in applied + assert revoked_row in applied + assert "### Would revoke: whole-table privileges declared for named columns" in ( + planned + ) + assert "### Revoked" not in plain + assert "### Would revoke" not in plain + + def test_summary_lists_column_grants_with_their_columns() -> None: plan = ReconciliationPlan( grants=( From f965ed17cd28ecda49fbe7839c6b7c16ed4540bc Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 00:40:08 +0400 Subject: [PATCH 03/18] Revoke whole-table privileges the manifest declares for named columns The projects' default privileges give the runtime role UPDATE on every new table, including user_type_changes, where the manifest allows it to update only dismissed_at. apply reported that privilege without revoking it, so verify failed every staging deploy after the invitations revision. apply now revokes a whole-table privilege a role holds directly when the manifest declares it for named columns only, then grants those columns again in the same transaction, since PostgreSQL's REVOKE on a table also removes the role's column grants of it. The job summary lists the revoke. Every other undeclared privilege is still reported, not revoked, and verify still fails if a role keeps the whole-table privilege another way. Co-Authored-By: Claude Opus 5.5 --- changelog.d/v2-column-update-grants.fixed.md | 1 + docs/engineering/skills/alembic-migrations.md | 23 +++--- docs/generated/rules.md | 2 +- policyengine_api/data/v2/privilege_plans.py | 70 +++++++++++++++++-- .../data/v2/privilege_reconciliation.py | 26 ++++--- rules/registry.toml | 2 +- 6 files changed, 98 insertions(+), 26 deletions(-) create mode 100644 changelog.d/v2-column-update-grants.fixed.md diff --git a/changelog.d/v2-column-update-grants.fixed.md b/changelog.d/v2-column-update-grants.fixed.md new file mode 100644 index 000000000..62cb52b0a --- /dev/null +++ b/changelog.d/v2-column-update-grants.fixed.md @@ -0,0 +1 @@ +The v2 privilege reconciliation now revokes a privilege a role holds on a whole table when the manifest grants it on named columns only, such as the `UPDATE` new tables' default privileges give the runtime role on `user_type_changes`, and grants those columns again, so the deploy's privilege verification passes. diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index bf5f9dc27..8d5b3bd81 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -283,14 +283,21 @@ check the declarations: then `verify`, before the catalog is seeded. `apply` grants each declared privilege a role lacks, as one `GRANT` per role -and table and one per column privilege. It never revokes: it reports -privileges the file doesn't declare, on whole tables or on columns, and the -projects' default privileges, in the job summary. It grants nothing when a -declared role or table is missing. `verify` checks each column of a column -privilege with `has_column_privilege`, and fails when a role holds the same -privilege on the whole table. The roles themselves are provisioned outside -the repository; `--create-missing-roles` is accepted only for the disposable -test database. +and table and one per column privilege. It revokes one kind of privilege +only: one a role holds directly on a whole table that the file declares for +named columns only. The projects' default privileges give the runtime role +`UPDATE` on every new table, so `user_type_changes`, where it may update only +`dismissed_at`, starts with one. PostgreSQL's `REVOKE` on a table also removes +the role's column grants of that privilege, so `apply` grants the declared +columns again, in the same transaction, and the job summary lists the revoke. +It reports every other privilege the file doesn't declare, on whole tables or +on columns, and the projects' default privileges, without revoking them. It +changes nothing when a declared role or table is missing. `verify` checks each +column of a column privilege with `has_column_privilege`, and fails when a role +still holds the same privilege on the whole table, for instance through a role +it belongs to. The roles themselves are provisioned outside the repository; +`--create-missing-roles` is accepted only for the disposable test database, +which has no default privileges. A pull request may declare a role before an operator creates it by listing it in `ROLES_AWAITING_PROVISIONING` in `privileges.py`. While a listed role is diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 79b3012d4..07d85a6cd 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -285,7 +285,7 @@ flowchart LR | `api.migration.bundle-supported-by-simulation-api` | The installed PolicyEngine bundle is the latest one the simulation API has published, checked on every pull request and before each staging and production deploy. | mechanical | enforced | yes | CI job `ensure-policyengine-bundle-supported-by-simulation-api` | `pr.yml:ensure-policyengine-bundle-supported-by-simulation-api` | — | | `api.migration.contracts-consistent` | Migration route groups and workflow contracts are internally consistent, and their generated docs are current. | mechanical | enforced | yes | quality guard `migration-contracts` | `pr.yml:quality-guards` | **unproven** | | `api.migration.cross-database-integration` | Cross-database integration tests pass against real MySQL and Postgres. | mechanical | enforced | yes | CI job `v2-integration-check` | `pr.yml:v2-integration-check` | — | -| `api.v2.deployed-privileges-match` | Every v2 deploy grants the managed roles each privilege policyengine_api/data/v2/privileges.py declares, on the whole table or on the columns it names, right after the schema upgrade, reports privileges it doesn't declare without revoking them, and stops before the catalog is seeded if a declared privilege is still missing or a role holds on a whole table a privilege declared for only some of its columns. A declared role listed in ROLES_AWAITING_PROVISIONING that doesn't exist yet is named in the job summary and skipped instead. | mechanical | enforced | yes | CI job `seed`, `catalog-runtime-integration` | `seed-v2-database.yml:seed`, `v2-integration-check.yml:catalog-runtime-integration` | yes | +| `api.v2.deployed-privileges-match` | Every v2 deploy grants the managed roles each privilege policyengine_api/data/v2/privileges.py declares, on the whole table or on the columns it names, right after the schema upgrade, revokes a privilege a role holds on a whole table that is declared for only some of its columns and grants those columns again, reports every other privilege it doesn't declare without revoking it, and stops before the catalog is seeded if a declared privilege is still missing or a role still holds on a whole table a privilege declared for only some of its columns. A declared role listed in ROLES_AWAITING_PROVISIONING that doesn't exist yet is named in the job summary and skipped instead. | mechanical | enforced | yes | CI job `seed`, `catalog-runtime-integration` | `seed-v2-database.yml:seed`, `v2-integration-check.yml:catalog-runtime-integration` | yes | | `api.v2.runtime-privileges-declared` | Every v2 table has an entry for each managed database role in policyengine_api/data/v2/privileges.py; a privilege declared on only some of a table's columns names columns of that table and is never also declared on the whole table; and every privilege a v2 connector module needs (from its reads, creates, updates or deletes module name and its row locks) is declared for the role that runs it, on the whole table. | mechanical | enforced | yes | pytest `tests/unit/v2/test_privilege_manifest.py` | `pr.yml:test` | yes | | `api.v2.runtime-privileges-tested-as-runtime-role` | The cross-database integration tests run the v2 user services and the parameter tree read as policyengine_v2_runtime, catalog publication as policyengine_v2_catalog_writer, the comparison-run services as policyengine_stage12_, and the simulation API's table writes as policyengine_v2_simulation_service and policyengine_v2_modal_worker, holding only their declared privileges, so a statement those privileges don't allow fails CI. | mechanical | enforced | yes | CI job `v2-integration-check`, `catalog-runtime-integration` | `pr.yml:v2-integration-check`, `v2-integration-check.yml:catalog-runtime-integration` | — | diff --git a/policyengine_api/data/v2/privilege_plans.py b/policyengine_api/data/v2/privilege_plans.py index 671243cc8..819106500 100644 --- a/policyengine_api/data/v2/privilege_plans.py +++ b/policyengine_api/data/v2/privilege_plans.py @@ -1,9 +1,10 @@ """Plan, state and report the table privileges the v2 manifest declares. The pure half of ``privilege_reconciliation``: it compares the declared -privileges with what a database grants, writes the GRANT statements, finds the -privileges a role lacks or holds too widely, and renders the reports. It runs -no SQL; ``privilege_reconciliation`` reads the database and runs the result. +privileges with what a database grants, writes the REVOKE and GRANT statements, +finds the privileges a role lacks or holds too widely, and renders the reports. +It runs no SQL; ``privilege_reconciliation`` reads the database and runs the +result. """ from __future__ import annotations @@ -75,10 +76,12 @@ class DefaultPrivilege: @dataclass(frozen=True) class ReconciliationPlan: - """What ``apply`` grants and reports for one database. + """What ``apply`` revokes, grants and reports for one database. ``skipped_roles`` are missing roles awaiting creation by an operator; they - don't block the plan, and nothing is granted to them. + don't block the plan, and nothing is granted to them. ``revocations`` are + whole-table privileges a role holds directly that the manifest declares + for named columns only; ``apply`` revokes them before it grants. """ grants: tuple[PrivilegeEntry, ...] @@ -86,6 +89,7 @@ class ReconciliationPlan: missing_roles: tuple[str, ...] missing_tables: tuple[str, ...] skipped_roles: tuple[str, ...] = () + revocations: tuple[PrivilegeEntry, ...] = () @property def can_apply(self) -> bool: @@ -120,6 +124,11 @@ def plan_reconciliation( """Compare declared privileges with what the database grants. A missing role in ``roles_awaiting_creation`` is skipped instead of missing. + A whole-table privilege the manifest declares for named columns only, such + as the ``UPDATE`` a new table's default privileges give, is revoked. + PostgreSQL's ``REVOKE`` on a table also removes the role's column grants of + that privilege, so the plan grants the declared columns again and reports + none of the removed grants as undeclared. """ absent_roles = set(declared) - set(existing_roles) skipped_roles = tuple(sorted(absent_roles & set(roles_awaiting_creation))) @@ -144,18 +153,35 @@ def plan_reconciliation( for column, privileges in columns.items() for privilege in privileges } + revocations = sorted( + PrivilegeEntry(role, table, privilege.privilege.value) + for role, tables in declared.items() + for table, privileges in tables.items() + for privilege in privileges + if isinstance(privilege, ColumnPrivilege) + and role not in absent_roles + and table not in missing_tables + and privilege.privilege.value in granted.tables.get(role, {}).get(table, ()) + ) + revoked = {(entry.role, entry.table, entry.privilege) for entry in revocations} + held_entries = { + entry + for entry in granted_entries + if (entry.role, entry.table, entry.privilege) not in revoked + } grants = sorted( entry - for entry in declared_entries - granted_entries + for entry in declared_entries - held_entries if entry.role not in absent_roles and entry.table not in missing_tables ) - undeclared = sorted(granted_entries - declared_entries) + undeclared = sorted(held_entries - declared_entries) return ReconciliationPlan( grants=tuple(grants), undeclared=tuple(undeclared), missing_roles=missing_roles, missing_tables=missing_tables, skipped_roles=skipped_roles, + revocations=tuple(revocations), ) @@ -167,6 +193,22 @@ def safe_identifier(identifier: str) -> str: return identifier +def revoke_statements( + plan: ReconciliationPlan, quote_identifier: Callable[[str], str] +) -> list[str]: + """Return the REVOKEs for a plan: one per role and table, in that order.""" + privileges_by_target: dict[tuple[str, str], set[str]] = defaultdict(set) + for entry in plan.revocations: + target = (safe_identifier(entry.role), safe_identifier(entry.table)) + privileges_by_target[target].add(TablePrivilege(entry.privilege).value) + return [ + f"REVOKE {', '.join(sorted(privileges))} ON TABLE " + f"{quote_identifier(SCHEMA)}.{quote_identifier(table)} " + f"FROM {quote_identifier(role)}" + for (role, table), privileges in sorted(privileges_by_target.items()) + ] + + def grant_statements( plan: ReconciliationPlan, quote_identifier: Callable[[str], str] ) -> list[str]: @@ -310,6 +352,20 @@ def render_summary( + ", ".join(f"`{table}`" for table in plan.missing_tables), "", ] + if plan.revocations: + revoke_heading = "Revoked" if applied else "Would revoke" + lines += [ + f"### {revoke_heading}: whole-table privileges declared for named " + "columns only", + "", + "Revoking one also removes the role's column grants of it; the " + "declared columns are granted again below.", + "", + "| Role | Table | Privileges |", + "| --- | --- | --- |", + *_privilege_rows(plan.revocations), + "", + ] grant_heading = "Granted" if applied else "Would grant" lines += [f"### {grant_heading}", ""] if plan.grants: diff --git a/policyengine_api/data/v2/privilege_reconciliation.py b/policyengine_api/data/v2/privilege_reconciliation.py index 008e33e87..84c3e524b 100644 --- a/policyengine_api/data/v2/privilege_reconciliation.py +++ b/policyengine_api/data/v2/privilege_reconciliation.py @@ -6,8 +6,11 @@ - ``apply`` reads the managed roles' current grants, on whole tables and on columns, grants every declared privilege a role lacks, and reports privileges - the manifest doesn't declare. It never revokes, and it changes nothing when a - declared role or table is missing. Running it again grants nothing more. + the manifest doesn't declare. It revokes only a whole-table privilege the + manifest declares for named columns only, such as the ``UPDATE`` a new + table's default privileges give, and then grants those columns again, in the + same transaction. It changes nothing when a declared role or table is + missing. Running it again changes nothing more. - ``verify`` checks every declared privilege with ``has_table_privilege``, or ``has_column_privilege`` for each column of a column privilege (rule ``api.v2.deployed-privileges-match``), and fails on any that is missing. It @@ -61,6 +64,7 @@ grant_statements, plan_reconciliation, render_summary, + revoke_statements, safe_identifier, skipped_roles_line, table_wide_overrides, @@ -105,6 +109,7 @@ "read_granted_privileges", "read_public_tables", "render_summary", + "revoke_statements", "run", "skipped_roles", "table_wide_overrides", @@ -274,10 +279,14 @@ def apply_declared_privileges( *, roles_awaiting_creation: Collection[str] = (), ) -> tuple[ReconciliationPlan, list[DefaultPrivilege]]: - """Grant every declared privilege a role lacks; never revoke. - - Grants nothing when a declared role or table is missing, unless the missing - role awaits creation, which is skipped. + """Grant every declared privilege a role lacks. + + First revokes each whole-table privilege the manifest declares for named + columns only, which also removes the role's column grants of it, so the + grants that follow include those columns again. Every other privilege the + manifest doesn't declare is reported, not revoked. Changes nothing when a + declared role or table is missing, unless the missing role awaits creation, + which is skipped. """ roles = list(declared) plan = plan_reconciliation( @@ -291,9 +300,8 @@ def apply_declared_privileges( roles_awaiting_creation=roles_awaiting_creation, ) if plan.can_apply: - for statement in grant_statements( - plan, connection.dialect.identifier_preparer.quote - ): + quote = connection.dialect.identifier_preparer.quote + for statement in revoke_statements(plan, quote) + grant_statements(plan, quote): connection.execute(text(statement)) return plan, read_default_privileges(connection, roles) diff --git a/rules/registry.toml b/rules/registry.toml index c31a7ab57..6df8f5ffe 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -1067,7 +1067,7 @@ invocation = "reconcile_v2_privileges.py apply --create-missing-roles" [[rule]] id = "api.v2.deployed-privileges-match" category = "migration" -statement = "Every v2 deploy grants the managed roles each privilege policyengine_api/data/v2/privileges.py declares, on the whole table or on the columns it names, right after the schema upgrade, reports privileges it doesn't declare without revoking them, and stops before the catalog is seeded if a declared privilege is still missing or a role holds on a whole table a privilege declared for only some of its columns. A declared role listed in ROLES_AWAITING_PROVISIONING that doesn't exist yet is named in the job summary and skipped instead." +statement = "Every v2 deploy grants the managed roles each privilege policyengine_api/data/v2/privileges.py declares, on the whole table or on the columns it names, right after the schema upgrade, revokes a privilege a role holds on a whole table that is declared for only some of its columns and grants those columns again, reports every other privilege it doesn't declare without revoking it, and stops before the catalog is seeded if a declared privilege is still missing or a role still holds on a whole table a privilege declared for only some of its columns. A declared role listed in ROLES_AWAITING_PROVISIONING that doesn't exist yet is named in the job summary and skipped instead." rationale = "docs/engineering/skills/alembic-migrations.md#table-privileges" scope = ["policyengine_api/data/v2/privileges.py", "policyengine_api/data/v2/privilege_plans.py", "policyengine_api/data/v2/privilege_reconciliation.py", "scripts/reconcile_v2_privileges.py", ".github/workflows/seed-v2-database.yml"] tier = "mechanical" From 758050b71cf4f6beef2299f6320bbab773894414 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 00:50:33 +0400 Subject: [PATCH 04/18] Fix review round 1's findings on the column-grant revoke The job summary now lists the undeclared column grants a revoke removes, the PostgreSQL test starts from staging's state (whole-table privileges plus a declared and an undeclared column grant), the planned-revoke test uses a state that can occur, and the privileges module, rule statement and skill document say which grants the revoke removes and that a grant another role made survives it. Co-Authored-By: Claude Opus 5.5 --- changelog.d/v2-column-update-grants.fixed.md | 2 +- docs/engineering/skills/alembic-migrations.md | 14 ++++++----- docs/generated/rules.md | 2 +- policyengine_api/data/v2/privilege_plans.py | 21 +++++++++++++---- .../data/v2/privilege_reconciliation.py | 7 +++--- policyengine_api/data/v2/privileges.py | 10 +++++--- rules/registry.toml | 2 +- .../test_v2_privilege_reconciliation.py | 16 +++++++++---- .../unit/v2/test_privilege_reconciliation.py | 23 ++++++++++++++----- 9 files changed, 67 insertions(+), 30 deletions(-) diff --git a/changelog.d/v2-column-update-grants.fixed.md b/changelog.d/v2-column-update-grants.fixed.md index 62cb52b0a..6c030449a 100644 --- a/changelog.d/v2-column-update-grants.fixed.md +++ b/changelog.d/v2-column-update-grants.fixed.md @@ -1 +1 @@ -The v2 privilege reconciliation now revokes a privilege a role holds on a whole table when the manifest grants it on named columns only, such as the `UPDATE` new tables' default privileges give the runtime role on `user_type_changes`, and grants those columns again, so the deploy's privilege verification passes. +The v2 privilege reconciliation now revokes a privilege a role holds on a whole table when the manifest declares it for named columns only, such as the `UPDATE` new tables' default privileges give the runtime role on `user_type_changes`, and grants those columns again, so the deploy's privilege verification passes. diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index 8d5b3bd81..28fe9927b 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -288,14 +288,16 @@ only: one a role holds directly on a whole table that the file declares for named columns only. The projects' default privileges give the runtime role `UPDATE` on every new table, so `user_type_changes`, where it may update only `dismissed_at`, starts with one. PostgreSQL's `REVOKE` on a table also removes -the role's column grants of that privilege, so `apply` grants the declared -columns again, in the same transaction, and the job summary lists the revoke. -It reports every other privilege the file doesn't declare, on whole tables or -on columns, and the projects' default privileges, without revoking them. It +the role's column grants of that privilege, undeclared ones included, so +`apply` grants the declared columns again, in the same transaction; the job +summary lists the revoke with the undeclared column grants it removed. It +reports every other privilege the file doesn't declare, on whole tables or on +columns, and the projects' default privileges, without revoking them. It changes nothing when a declared role or table is missing. `verify` checks each column of a column privilege with `has_column_privilege`, and fails when a role -still holds the same privilege on the whole table, for instance through a role -it belongs to. The roles themselves are provisioned outside the repository; +still holds the same privilege on the whole table: through a role it belongs +to, or through a grant another role made, which the owner's `REVOKE` leaves in +place. The roles themselves are provisioned outside the repository; `--create-missing-roles` is accepted only for the disposable test database, which has no default privileges. diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 07d85a6cd..3b17654f4 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -285,7 +285,7 @@ flowchart LR | `api.migration.bundle-supported-by-simulation-api` | The installed PolicyEngine bundle is the latest one the simulation API has published, checked on every pull request and before each staging and production deploy. | mechanical | enforced | yes | CI job `ensure-policyengine-bundle-supported-by-simulation-api` | `pr.yml:ensure-policyengine-bundle-supported-by-simulation-api` | — | | `api.migration.contracts-consistent` | Migration route groups and workflow contracts are internally consistent, and their generated docs are current. | mechanical | enforced | yes | quality guard `migration-contracts` | `pr.yml:quality-guards` | **unproven** | | `api.migration.cross-database-integration` | Cross-database integration tests pass against real MySQL and Postgres. | mechanical | enforced | yes | CI job `v2-integration-check` | `pr.yml:v2-integration-check` | — | -| `api.v2.deployed-privileges-match` | Every v2 deploy grants the managed roles each privilege policyengine_api/data/v2/privileges.py declares, on the whole table or on the columns it names, right after the schema upgrade, revokes a privilege a role holds on a whole table that is declared for only some of its columns and grants those columns again, reports every other privilege it doesn't declare without revoking it, and stops before the catalog is seeded if a declared privilege is still missing or a role still holds on a whole table a privilege declared for only some of its columns. A declared role listed in ROLES_AWAITING_PROVISIONING that doesn't exist yet is named in the job summary and skipped instead. | mechanical | enforced | yes | CI job `seed`, `catalog-runtime-integration` | `seed-v2-database.yml:seed`, `v2-integration-check.yml:catalog-runtime-integration` | yes | +| `api.v2.deployed-privileges-match` | Every v2 deploy grants the managed roles each privilege policyengine_api/data/v2/privileges.py declares, on the whole table or on the columns it names, right after the schema upgrade, revokes a privilege a role holds on a whole table that is declared for only some of its columns, with the role's column grants of that privilege, and grants the declared columns again, reports every other privilege it doesn't declare without revoking it, and stops before the catalog is seeded if a declared privilege is still missing or a role still holds on a whole table a privilege declared for only some of its columns. A declared role listed in ROLES_AWAITING_PROVISIONING that doesn't exist yet is named in the job summary and skipped instead. | mechanical | enforced | yes | CI job `seed`, `catalog-runtime-integration` | `seed-v2-database.yml:seed`, `v2-integration-check.yml:catalog-runtime-integration` | yes | | `api.v2.runtime-privileges-declared` | Every v2 table has an entry for each managed database role in policyengine_api/data/v2/privileges.py; a privilege declared on only some of a table's columns names columns of that table and is never also declared on the whole table; and every privilege a v2 connector module needs (from its reads, creates, updates or deletes module name and its row locks) is declared for the role that runs it, on the whole table. | mechanical | enforced | yes | pytest `tests/unit/v2/test_privilege_manifest.py` | `pr.yml:test` | yes | | `api.v2.runtime-privileges-tested-as-runtime-role` | The cross-database integration tests run the v2 user services and the parameter tree read as policyengine_v2_runtime, catalog publication as policyengine_v2_catalog_writer, the comparison-run services as policyengine_stage12_, and the simulation API's table writes as policyengine_v2_simulation_service and policyengine_v2_modal_worker, holding only their declared privileges, so a statement those privileges don't allow fails CI. | mechanical | enforced | yes | CI job `v2-integration-check`, `catalog-runtime-integration` | `pr.yml:v2-integration-check`, `v2-integration-check.yml:catalog-runtime-integration` | — | diff --git a/policyengine_api/data/v2/privilege_plans.py b/policyengine_api/data/v2/privilege_plans.py index 819106500..13dea33f2 100644 --- a/policyengine_api/data/v2/privilege_plans.py +++ b/policyengine_api/data/v2/privilege_plans.py @@ -81,7 +81,9 @@ class ReconciliationPlan: ``skipped_roles`` are missing roles awaiting creation by an operator; they don't block the plan, and nothing is granted to them. ``revocations`` are whole-table privileges a role holds directly that the manifest declares - for named columns only; ``apply`` revokes them before it grants. + for named columns only; ``apply`` revokes them before it grants. Revoking + one also removes the role's column grants of it: ``revoked_columns`` are + the undeclared ones among them, and the declared ones are in ``grants``. """ grants: tuple[PrivilegeEntry, ...] @@ -90,6 +92,7 @@ class ReconciliationPlan: missing_tables: tuple[str, ...] skipped_roles: tuple[str, ...] = () revocations: tuple[PrivilegeEntry, ...] = () + revoked_columns: tuple[PrivilegeEntry, ...] = () @property def can_apply(self) -> bool: @@ -127,8 +130,9 @@ def plan_reconciliation( A whole-table privilege the manifest declares for named columns only, such as the ``UPDATE`` a new table's default privileges give, is revoked. PostgreSQL's ``REVOKE`` on a table also removes the role's column grants of - that privilege, so the plan grants the declared columns again and reports - none of the removed grants as undeclared. + that privilege, so the plan grants the declared columns again and lists the + undeclared ones it removes in ``revoked_columns``. The owner's ``REVOKE`` + leaves a grant another role made; ``verify`` then still fails. """ absent_roles = set(declared) - set(existing_roles) skipped_roles = tuple(sorted(absent_roles & set(roles_awaiting_creation))) @@ -169,6 +173,11 @@ def plan_reconciliation( for entry in granted_entries if (entry.role, entry.table, entry.privilege) not in revoked } + revoked_columns = sorted( + entry + for entry in granted_entries - declared_entries + if entry.column and (entry.role, entry.table, entry.privilege) in revoked + ) grants = sorted( entry for entry in declared_entries - held_entries @@ -182,6 +191,7 @@ def plan_reconciliation( missing_tables=missing_tables, skipped_roles=skipped_roles, revocations=tuple(revocations), + revoked_columns=tuple(revoked_columns), ) @@ -359,11 +369,12 @@ def render_summary( "columns only", "", "Revoking one also removes the role's column grants of it; the " - "declared columns are granted again below.", + "undeclared ones are listed with it, and the declared ones are in " + "the grants below.", "", "| Role | Table | Privileges |", "| --- | --- | --- |", - *_privilege_rows(plan.revocations), + *_privilege_rows((*plan.revocations, *plan.revoked_columns)), "", ] grant_heading = "Granted" if applied else "Would grant" diff --git a/policyengine_api/data/v2/privilege_reconciliation.py b/policyengine_api/data/v2/privilege_reconciliation.py index 84c3e524b..692fccfad 100644 --- a/policyengine_api/data/v2/privilege_reconciliation.py +++ b/policyengine_api/data/v2/privilege_reconciliation.py @@ -8,9 +8,10 @@ columns, grants every declared privilege a role lacks, and reports privileges the manifest doesn't declare. It revokes only a whole-table privilege the manifest declares for named columns only, such as the ``UPDATE`` a new - table's default privileges give, and then grants those columns again, in the - same transaction. It changes nothing when a declared role or table is - missing. Running it again changes nothing more. + table's default privileges give, which also removes the role's column grants + of it, and then grants the declared columns again, in the same transaction. + It changes nothing when a declared role or table is missing. Running it + again changes nothing more. - ``verify`` checks every declared privilege with ``has_table_privilege``, or ``has_column_privilege`` for each column of a column privilege (rule ``api.v2.deployed-privileges-match``), and fails on any that is missing. It diff --git a/policyengine_api/data/v2/privileges.py b/policyengine_api/data/v2/privileges.py index f8047aff0..44ea0efb9 100644 --- a/policyengine_api/data/v2/privileges.py +++ b/policyengine_api/data/v2/privileges.py @@ -57,9 +57,13 @@ The migration role (``policyengine_v2_migrator``) owns every table, and Supabase's ``postgres`` role administers the project; neither is managed here. -Extra privileges a database holds beyond this manifest are reported, never -revoked. The v2 schema has no sequences (every key is a UUID); a test fails if -one appears, so the manifest can be extended to cover it. +Extra privileges a database holds beyond this manifest are reported, not +revoked, with one exception: a privilege a role holds on a whole table that +this manifest declares for named columns only is revoked, with the role's +column grants of it, and the declared columns are granted again. + +The v2 schema has no sequences (every key is a UUID); a test fails if one +appears, so the manifest can be extended to cover it. """ from __future__ import annotations diff --git a/rules/registry.toml b/rules/registry.toml index 6df8f5ffe..87a5cf6a8 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -1067,7 +1067,7 @@ invocation = "reconcile_v2_privileges.py apply --create-missing-roles" [[rule]] id = "api.v2.deployed-privileges-match" category = "migration" -statement = "Every v2 deploy grants the managed roles each privilege policyengine_api/data/v2/privileges.py declares, on the whole table or on the columns it names, right after the schema upgrade, revokes a privilege a role holds on a whole table that is declared for only some of its columns and grants those columns again, reports every other privilege it doesn't declare without revoking it, and stops before the catalog is seeded if a declared privilege is still missing or a role still holds on a whole table a privilege declared for only some of its columns. A declared role listed in ROLES_AWAITING_PROVISIONING that doesn't exist yet is named in the job summary and skipped instead." +statement = "Every v2 deploy grants the managed roles each privilege policyengine_api/data/v2/privileges.py declares, on the whole table or on the columns it names, right after the schema upgrade, revokes a privilege a role holds on a whole table that is declared for only some of its columns, with the role's column grants of that privilege, and grants the declared columns again, reports every other privilege it doesn't declare without revoking it, and stops before the catalog is seeded if a declared privilege is still missing or a role still holds on a whole table a privilege declared for only some of its columns. A declared role listed in ROLES_AWAITING_PROVISIONING that doesn't exist yet is named in the job summary and skipped instead." rationale = "docs/engineering/skills/alembic-migrations.md#table-privileges" scope = ["policyengine_api/data/v2/privileges.py", "policyengine_api/data/v2/privilege_plans.py", "policyengine_api/data/v2/privilege_reconciliation.py", "scripts/reconcile_v2_privileges.py", ".github/workflows/seed-v2-database.yml"] tier = "mechanical" diff --git a/tests/integration/test_v2_privilege_reconciliation.py b/tests/integration/test_v2_privilege_reconciliation.py index c2a03b727..4cf7dac64 100644 --- a/tests/integration/test_v2_privilege_reconciliation.py +++ b/tests/integration/test_v2_privilege_reconciliation.py @@ -156,9 +156,12 @@ def test_apply_revokes_a_new_tables_whole_table_update_and_verify_passes( ) -> None: """A new table's default privileges give UPDATE on the whole table. - ``apply`` revokes it and grants the declared column again in the same - transaction, so ``verify`` passes. The extra ``INSERT`` and ``DELETE`` - stay, reported but not revoked. + The role starts as staging's runtime role did on ``user_type_changes``: + the whole-table privileges, the declared column grant an earlier ``apply`` + made, and an undeclared column grant. ``apply`` revokes the whole-table + ``UPDATE``, which removes both column grants, and grants the declared one + again in the same transaction, so ``verify`` passes. The extra ``INSERT`` + and ``DELETE`` stay, reported but not revoked. """ role = temporary_role() declared = {role: {"regions": READ | {update_columns("label")}}} @@ -175,6 +178,9 @@ def test_apply_revokes_a_new_tables_whole_table_update_and_verify_passes( f'TO "{role}"' ) ) + connection.execute( + text(f'GRANT UPDATE (label, code) ON TABLE public.regions TO "{role}"') + ) summary_path = tmp_path / "summary.md" environ = { V2_MIGRATION_DATABASE_URL: disposable_v2_database_url, @@ -190,7 +196,9 @@ def test_apply_revokes_a_new_tables_whole_table_update_and_verify_passes( second_plan, _defaults = apply_declared_privileges(connection, declared) assert (apply_status, verify_status) == (0, 0) - assert f"| `{role}` | `regions` | UPDATE |" in summary + assert f"| `{role}` | `regions` | UPDATE, UPDATE (code) |" in summary + assert f"| `{role}` | `regions` | UPDATE (label) |" in summary + assert f"| `{role}` | `regions` | DELETE, INSERT |" in summary assert not _has_privilege(superuser_engine, role, "regions", "UPDATE") assert _has_column_privilege(superuser_engine, role, "regions", "label", "UPDATE") assert not _has_column_privilege( diff --git a/tests/unit/v2/test_privilege_reconciliation.py b/tests/unit/v2/test_privilege_reconciliation.py index 5d185f39d..b5d13b474 100644 --- a/tests/unit/v2/test_privilege_reconciliation.py +++ b/tests/unit/v2/test_privilege_reconciliation.py @@ -411,6 +411,9 @@ def test_a_whole_table_privilege_declared_for_columns_is_revoked() -> None: ) assert plan.revocations == (PrivilegeEntry(WORKER, "simulation_runs", "UPDATE"),) + assert plan.revoked_columns == ( + PrivilegeEntry(WORKER, "simulation_runs", "UPDATE", column="simulation_id"), + ) assert plan.grants == ( PrivilegeEntry(WORKER, "simulation_runs", "UPDATE", column="completed_at"), PrivilegeEntry(WORKER, "simulation_runs", "UPDATE", column="status"), @@ -418,18 +421,21 @@ def test_a_whole_table_privilege_declared_for_columns_is_revoked() -> None: assert plan.undeclared == (PrivilegeEntry(WORKER, "simulation_runs", "DELETE"),) -def test_nothing_is_revoked_while_a_declared_role_or_table_is_missing() -> None: +def test_a_revoke_is_only_planned_while_a_declared_table_is_missing() -> None: + """Like a grant, a revoke waits until every declared table exists.""" + declared = {WORKER: {**WORKER_DECLARED[WORKER], "household_jobs": READ}} plan = plan_reconciliation( - WORKER_DECLARED, + declared, granted=GrantedPrivileges( tables={WORKER: {"simulation_runs": {"SELECT", "UPDATE"}}} ), existing_roles={WORKER}, - existing_tables=set(), + existing_tables={"simulation_runs"}, ) - assert plan.missing_tables == ("simulation_runs",) - assert plan.revocations == () + assert plan.missing_tables == ("household_jobs",) + assert not plan.can_apply + assert plan.revocations == (PrivilegeEntry(WORKER, "simulation_runs", "UPDATE"),) def test_revoke_statements_revoke_each_role_and_tables_privileges() -> None: @@ -476,6 +482,9 @@ def test_summary_lists_revoked_whole_table_privileges_only_when_there_are_some() missing_roles=(), missing_tables=(), revocations=(PrivilegeEntry(RUNTIME, "user_type_changes", "UPDATE"),), + revoked_columns=( + PrivilegeEntry(RUNTIME, "user_type_changes", "UPDATE", "user_id"), + ), ) plain_plan = ReconciliationPlan( grants=(), undeclared=(), missing_roles=(), missing_tables=() @@ -491,7 +500,9 @@ def test_summary_lists_revoked_whole_table_privileges_only_when_there_are_some() plain_plan, (), deployment_environment="staging", applied=True ) - revoked_row = "| `policyengine_v2_runtime` | `user_type_changes` | UPDATE |" + revoked_row = ( + "| `policyengine_v2_runtime` | `user_type_changes` | UPDATE, UPDATE (user_id) |" + ) assert "### Revoked: whole-table privileges declared for named columns" in applied assert revoked_row in applied assert "### Would revoke: whole-table privileges declared for named columns" in ( From 46a3005a5e12cd23d4cca63b5505d5e75fd5a923 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 00:51:41 +0400 Subject: [PATCH 05/18] Fix review round 2's test docstring wording Co-Authored-By: Claude Opus 5.5 --- tests/unit/v2/test_privilege_reconciliation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/v2/test_privilege_reconciliation.py b/tests/unit/v2/test_privilege_reconciliation.py index b5d13b474..b7d731682 100644 --- a/tests/unit/v2/test_privilege_reconciliation.py +++ b/tests/unit/v2/test_privilege_reconciliation.py @@ -391,7 +391,8 @@ def test_a_whole_table_privilege_declared_for_columns_is_revoked() -> None: """A new table's default privileges give UPDATE on the whole table. Revoking it also removes the role's column grants of UPDATE, so the plan - grants the declared columns again and reports none of the removed ones. + grants the declared columns again and lists the removed undeclared ones + with the revoke rather than as undeclared. """ plan = plan_reconciliation( WORKER_DECLARED, From cbf3e42e718bab2377c0d349109bb2e753ab774c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 01:34:56 +0400 Subject: [PATCH 06/18] Test recording why a v2 metadata read answered 503 or 500 (failing) On 2026-11-16 staging answered GET /v2/parameter-values with the generic 503 "V2 metadata catalog is unavailable" after the database connection was lost mid-query, and nothing recorded the error behind it. These tests expect one runtime event per generic 503 or 500 with the route, the caught error's class, its cause chain's classes, the SQLSTATE and SQLAlchemy's lost-connection flag, and no error message. Co-Authored-By: Claude Opus 5.5 --- .../v2/test_metadata_read_failure_events.py | 200 ++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 tests/unit/v2/test_metadata_read_failure_events.py diff --git a/tests/unit/v2/test_metadata_read_failure_events.py b/tests/unit/v2/test_metadata_read_failure_events.py new file mode 100644 index 000000000..78b9886df --- /dev/null +++ b/tests/unit/v2/test_metadata_read_failure_events.py @@ -0,0 +1,200 @@ +"""Runtime events for v2 metadata reads that fail with a generic 503 or a 500. + +The response stays the generic one, but the observability runtime records one +event that says what failed: the route and request ID, which the runtime takes +from the request it is serving, then the response status, the caught error's +class, the class of each error in its ``__cause__`` chain, the driver's +SQLSTATE and whether SQLAlchemy found the connection lost. It never records an +error's message, which can carry the SQL statement, its parameters, host +names, addresses or credentials. These tests read the event the process's real +runtime writes to standard output. +""" + +from __future__ import annotations + +import json +from typing import Any + +from sqlalchemy.exc import OperationalError + +from policyengine_api import observability +from policyengine_api.data.v2.catalog.catalog_selection_validation import ( + MetadataCatalogUnavailableError, +) +from policyengine_api.services.v2.metadata.validators import ( + CatalogValuesPrunedError, +) +from tests.fixtures.v2_metadata_routes import ResourceReader, metadata_route_client + +ROUTE = "/v2/parameter-values" +PATH = f"{ROUTE}?country_id=us&limit=1" +FAILURE_EVENT = "v2_metadata_read_failed" +STATEMENT = "SELECT parameter_values.id FROM parameter_values WHERE country_id = %(c)s" +PARAMETER_VALUE = "us-secret-parameter-value" +SENSITIVE_TEXT = ( + STATEMENT, + PARAMETER_VALUE, + "pooler.example.internal", + "10.12.0.7", + "hunter2", + "postgresql://", +) + + +class DriverAdminShutdownError(Exception): + """Stands in for psycopg's error for SQLSTATE 57P01, a terminated connection.""" + + sqlstate = "57P01" + + +class FailingRuntime: + def event(self, *_arguments: object, **_details: object) -> None: + msg = "the observability runtime is down" + raise RuntimeError(msg) + + +def _qualified_name(error_class: type[BaseException]) -> str: + return f"{error_class.__module__}.{error_class.__qualname__}" + + +def _lost_connection() -> OperationalError: + """Return SQLAlchemy's error for a connection lost mid-query, as psycopg raises it. + + Every message in the chain names the statement, a parameter, the pooler's + host and address and a password, as a driver's message can. + """ + driver_error = DriverAdminShutdownError( + "terminating connection due to administrator command: " + "postgresql://reader:hunter2@pooler.example.internal (10.12.0.7)" + ) + database_error = OperationalError( + STATEMENT, {"c": PARAMETER_VALUE}, driver_error, connection_invalidated=True + ) + database_error.__cause__ = driver_error + return database_error + + +def _caused_by(error: BaseException, cause: BaseException) -> BaseException: + error.__cause__ = cause + return error + + +def _failure_records(standard_output: str) -> list[dict[str, Any]]: + """Return the failure events among the runtime's JSON records.""" + records = [] + for printed_line in standard_output.splitlines(): + try: + record = json.loads(printed_line) + except ValueError: + continue + if isinstance(record, dict) and record.get("event.name") == FAILURE_EVENT: + records.append(record) + return records + + +def _assert_nothing_sensitive(record: dict[str, Any]) -> None: + recorded = json.dumps(record) + for text in SENSITIVE_TEXT: + assert text not in recorded + + +def test_a_lost_connection_answers_the_same_503_and_records_its_cause( + capsys, +) -> None: + error = _caused_by( + MetadataCatalogUnavailableError(f"catalog read failed: {STATEMENT}"), + _lost_connection(), + ) + reader = ResourceReader({"list_parameter_values": None}, error=error) + + response = metadata_route_client(lambda: reader).get(PATH) + [record] = _failure_records(capsys.readouterr().out) + + assert response.status_code == 503 + assert response.json() == { + "status": "error", + "message": "V2 metadata catalog is unavailable", + } + assert record["severity"] == "ERROR" + assert record["http.route"] == ROUTE + assert record["request.id"] == response.headers["x-policyengine-request-id"] + assert record["attributes"] == { + "http.response.status_code": 503, + "error.type": _qualified_name(MetadataCatalogUnavailableError), + "error.cause_types": ( + "sqlalchemy.exc.OperationalError, " + f"{_qualified_name(DriverAdminShutdownError)}" + ), + "database.sqlstate": "57P01", + "database.connection_invalidated": True, + } + _assert_nothing_sensitive(record) + + +def test_an_unexpected_failure_answers_the_same_500_and_records_its_cause( + capsys, +) -> None: + error = _caused_by(RuntimeError(f"failed on {STATEMENT}"), _lost_connection()) + reader = ResourceReader({"list_parameter_values": None}, error=error) + + response = metadata_route_client(lambda: reader).get(PATH) + [record] = _failure_records(capsys.readouterr().out) + + assert response.status_code == 500 + assert response.json() == {"status": "error", "message": "V2 metadata query failed"} + assert (record["severity"], record["http.route"]) == ("ERROR", ROUTE) + assert record["attributes"] == { + "http.response.status_code": 500, + "error.type": "builtins.RuntimeError", + "error.cause_types": ( + "sqlalchemy.exc.OperationalError, " + f"{_qualified_name(DriverAdminShutdownError)}" + ), + "database.sqlstate": "57P01", + "database.connection_invalidated": True, + } + _assert_nothing_sensitive(record) + + +def test_a_failure_without_a_database_cause_records_only_its_class(capsys) -> None: + reader = ResourceReader( + {"list_parameter_values": None}, + error=MetadataCatalogUnavailableError("catalog connection refused"), + ) + + response = metadata_route_client(lambda: reader).get(PATH) + [record] = _failure_records(capsys.readouterr().out) + + assert response.status_code == 503 + assert record["attributes"] == { + "http.response.status_code": 503, + "error.type": _qualified_name(MetadataCatalogUnavailableError), + } + + +def test_a_pruned_version_answer_records_no_failure(capsys) -> None: + reader = ResourceReader( + {"list_parameter_values": None}, + error=CatalogValuesPrunedError("pruned", running_version=True), + ) + + response = metadata_route_client(lambda: reader).get(PATH) + + assert response.status_code == 503 + assert _failure_records(capsys.readouterr().out) == [] + + +def test_a_failing_runtime_leaves_the_503_unchanged(monkeypatch) -> None: + monkeypatch.setattr(observability, "get_runtime", FailingRuntime) + error = _caused_by( + MetadataCatalogUnavailableError("catalog read failed"), _lost_connection() + ) + reader = ResourceReader({"list_parameter_values": None}, error=error) + + response = metadata_route_client(lambda: reader).get(PATH) + + assert response.status_code == 503 + assert response.json() == { + "status": "error", + "message": "V2 metadata catalog is unavailable", + } From 19e0055a125673eae80cd907ff5e8bcbd17d9788 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 01:38:52 +0400 Subject: [PATCH 07/18] Read the metadata failure event from a test runtime (still failing) The process's runtime writes to the stream it saved at start-up, which pytest's output capture can't read, so the tests serve their requests through a runtime configured the same way but writing to a list. The route fixture takes that runtime as an optional argument. Co-Authored-By: Claude Opus 5.5 --- tests/fixtures/v2_metadata_routes.py | 5 +- .../v2/test_metadata_read_failure_events.py | 123 +++++++++++++----- 2 files changed, 96 insertions(+), 32 deletions(-) diff --git a/tests/fixtures/v2_metadata_routes.py b/tests/fixtures/v2_metadata_routes.py index 2ececbb89..6b04a3194 100644 --- a/tests/fixtures/v2_metadata_routes.py +++ b/tests/fixtures/v2_metadata_routes.py @@ -287,7 +287,9 @@ def detail(item: object) -> MetadataDetailResult: ) -def metadata_route_client(factory, *, response_cache=None) -> TestClient: +def metadata_route_client( + factory, *, response_cache=None, observability_runtime=None +) -> TestClient: flask_app = Flask(__name__) @flask_app.get("//metadata") @@ -317,6 +319,7 @@ def v1_metadata(country_id: str): flask_app, dependencies=dependencies, route_settings=settings, + observability_runtime=observability_runtime, ), raise_server_exceptions=False, ) diff --git a/tests/unit/v2/test_metadata_read_failure_events.py b/tests/unit/v2/test_metadata_read_failure_events.py index 78b9886df..f8ebfbdd6 100644 --- a/tests/unit/v2/test_metadata_read_failure_events.py +++ b/tests/unit/v2/test_metadata_read_failure_events.py @@ -6,26 +6,42 @@ class, the class of each error in its ``__cause__`` chain, the driver's SQLSTATE and whether SQLAlchemy found the connection lost. It never records an error's message, which can carry the SQL statement, its parameters, host -names, addresses or credentials. These tests read the event the process's real -runtime writes to standard output. +names, addresses or credentials. These tests serve the request through a runtime +configured as the process's is, but writing its records to a list, so they see +the event exactly as the runtime writes it, request context included. """ from __future__ import annotations -import json -from typing import Any +from typing import TYPE_CHECKING, Any +import pytest +from policyengine_observability import ( + CustomLogDestination, + LoggingConfig, + ObservabilityConfig, + configure, +) from sqlalchemy.exc import OperationalError from policyengine_api import observability from policyengine_api.data.v2.catalog.catalog_selection_validation import ( MetadataCatalogUnavailableError, ) +from policyengine_api.observability_identity import ( + deployment_identity, + service_identity, +) from policyengine_api.services.v2.metadata.validators import ( CatalogValuesPrunedError, ) from tests.fixtures.v2_metadata_routes import ResourceReader, metadata_route_client +if TYPE_CHECKING: + from collections.abc import Iterator + + from policyengine_observability import ObservabilityRuntime + ROUTE = "/v2/parameter-values" PATH = f"{ROUTE}?country_id=us&limit=1" FAILURE_EVENT = "v2_metadata_read_failed" @@ -47,12 +63,60 @@ class DriverAdminShutdownError(Exception): sqlstate = "57P01" +class RecordList: + """A runtime destination that keeps every record it is given.""" + + def __init__(self) -> None: + self.records: list[dict[str, Any]] = [] + + def write(self, record: dict[str, Any]) -> None: + self.records.append(record) + + def failures(self) -> list[dict[str, Any]]: + return [ + record + for record in self.records + if record.get("event.name") == FAILURE_EVENT + ] + + class FailingRuntime: def event(self, *_arguments: object, **_details: object) -> None: msg = "the observability runtime is down" raise RuntimeError(msg) +@pytest.fixture +def recorded(monkeypatch) -> Iterator[tuple[RecordList, ObservabilityRuntime]]: + """Serve requests through a runtime whose records the test can read.""" + record_list = RecordList() + runtime = configure( + ObservabilityConfig.from_env( + service=service_identity(), + deployment=deployment_identity(), + logging=LoggingConfig( + destinations=( + CustomLogDestination( + name="test-records", + writer_factory=lambda: record_list, + delivery="inline", + ), + ), + ), + ) + ) + monkeypatch.setattr(observability, "get_runtime", lambda: runtime) + try: + yield record_list, runtime + finally: + runtime.shutdown() + + +def _get(reader: ResourceReader, runtime: ObservabilityRuntime): + client = metadata_route_client(lambda: reader, observability_runtime=runtime) + return client.get(PATH) + + def _qualified_name(error_class: type[BaseException]) -> str: return f"{error_class.__module__}.{error_class.__qualname__}" @@ -79,27 +143,14 @@ def _caused_by(error: BaseException, cause: BaseException) -> BaseException: return error -def _failure_records(standard_output: str) -> list[dict[str, Any]]: - """Return the failure events among the runtime's JSON records.""" - records = [] - for printed_line in standard_output.splitlines(): - try: - record = json.loads(printed_line) - except ValueError: - continue - if isinstance(record, dict) and record.get("event.name") == FAILURE_EVENT: - records.append(record) - return records - - def _assert_nothing_sensitive(record: dict[str, Any]) -> None: - recorded = json.dumps(record) + written = repr(record) for text in SENSITIVE_TEXT: - assert text not in recorded + assert text not in written def test_a_lost_connection_answers_the_same_503_and_records_its_cause( - capsys, + recorded, ) -> None: error = _caused_by( MetadataCatalogUnavailableError(f"catalog read failed: {STATEMENT}"), @@ -107,8 +158,10 @@ def test_a_lost_connection_answers_the_same_503_and_records_its_cause( ) reader = ResourceReader({"list_parameter_values": None}, error=error) - response = metadata_route_client(lambda: reader).get(PATH) - [record] = _failure_records(capsys.readouterr().out) + record_list, runtime = recorded + + response = _get(reader, runtime) + [record] = record_list.failures() assert response.status_code == 503 assert response.json() == { @@ -132,13 +185,15 @@ def test_a_lost_connection_answers_the_same_503_and_records_its_cause( def test_an_unexpected_failure_answers_the_same_500_and_records_its_cause( - capsys, + recorded, ) -> None: error = _caused_by(RuntimeError(f"failed on {STATEMENT}"), _lost_connection()) reader = ResourceReader({"list_parameter_values": None}, error=error) - response = metadata_route_client(lambda: reader).get(PATH) - [record] = _failure_records(capsys.readouterr().out) + record_list, runtime = recorded + + response = _get(reader, runtime) + [record] = record_list.failures() assert response.status_code == 500 assert response.json() == {"status": "error", "message": "V2 metadata query failed"} @@ -156,14 +211,18 @@ def test_an_unexpected_failure_answers_the_same_500_and_records_its_cause( _assert_nothing_sensitive(record) -def test_a_failure_without_a_database_cause_records_only_its_class(capsys) -> None: +def test_a_failure_without_a_database_cause_records_only_its_class( + recorded, +) -> None: reader = ResourceReader( {"list_parameter_values": None}, error=MetadataCatalogUnavailableError("catalog connection refused"), ) - response = metadata_route_client(lambda: reader).get(PATH) - [record] = _failure_records(capsys.readouterr().out) + record_list, runtime = recorded + + response = _get(reader, runtime) + [record] = record_list.failures() assert response.status_code == 503 assert record["attributes"] == { @@ -172,16 +231,18 @@ def test_a_failure_without_a_database_cause_records_only_its_class(capsys) -> No } -def test_a_pruned_version_answer_records_no_failure(capsys) -> None: +def test_a_pruned_version_answer_records_no_failure(recorded) -> None: reader = ResourceReader( {"list_parameter_values": None}, error=CatalogValuesPrunedError("pruned", running_version=True), ) - response = metadata_route_client(lambda: reader).get(PATH) + record_list, runtime = recorded + + response = _get(reader, runtime) assert response.status_code == 503 - assert _failure_records(capsys.readouterr().out) == [] + assert record_list.failures() == [] def test_a_failing_runtime_leaves_the_503_unchanged(monkeypatch) -> None: From 9826036f81b04d4924b9efdcb86a9419205dfd62 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 01:40:28 +0400 Subject: [PATCH 08/18] Record why a v2 metadata read answered the generic 503 or a 500 run_metadata_read turned a lost database connection into the generic 503 "V2 metadata catalog is unavailable", and any unexpected error into the 500 "V2 metadata query failed", recording neither. Each such answer now records one runtime event, v2_metadata_read_failed, at ERROR, with the response status, the caught error's class, the classes of its cause chain, the first SQLSTATE in the chain and SQLAlchemy's lost-connection flag; the runtime adds the request's ID, method and route. It records no error message, since SQLAlchemy's and the driver's carry the statement, its parameters and connection details. A pruned version or a missing parameter tree or search file still answers with its own message and records nothing. Recording never changes the response. Co-Authored-By: Claude Opus 5.5 --- .../v2-metadata-read-failure-event.changed.md | 1 + docs/engineering/skills/observability.md | 16 +++ .../v2/metadata/read_failure_support.py | 101 ++++++++++++++++++ .../v2/metadata/route_support.py | 30 +++++- 4 files changed, 145 insertions(+), 3 deletions(-) create mode 100644 changelog.d/v2-metadata-read-failure-event.changed.md create mode 100644 policyengine_api/fastapi_routes/v2/metadata/read_failure_support.py diff --git a/changelog.d/v2-metadata-read-failure-event.changed.md b/changelog.d/v2-metadata-read-failure-event.changed.md new file mode 100644 index 000000000..085783865 --- /dev/null +++ b/changelog.d/v2-metadata-read-failure-event.changed.md @@ -0,0 +1 @@ +A v2 metadata read that answers the generic 503 or the 500 now records one runtime event, `v2_metadata_read_failed`, naming the route, the error classes in its cause chain, the database's SQLSTATE and whether the connection was lost, without any error message. diff --git a/docs/engineering/skills/observability.md b/docs/engineering/skills/observability.md index 1ce250ede..ffc7628de 100644 --- a/docs/engineering/skills/observability.md +++ b/docs/engineering/skills/observability.md @@ -84,6 +84,22 @@ own level is dropped before the runtime's handler sees it. Attribute names must avoid the runtime's prohibited key parts (such as `authorization`, `credential` and `token`), which it drops. +## v2 metadata read failures + +A v2 metadata read that answers the generic 503 "V2 metadata catalog is +unavailable", or the 500 "V2 metadata query failed", records one runtime +event at ERROR, `v2_metadata_read_failed` +(`policyengine_api.fastapi_routes.v2.metadata.read_failure_support`). The +runtime adds the request's ID, method and matched route. The event adds +`http.response.status_code`, `error.type` (the caught error's class as +`module.QualifiedName`), `error.cause_types` (the classes of its `__cause__` +chain, outermost first), `database.sqlstate` (the first SQLSTATE in the chain) +and `database.connection_invalidated` (SQLAlchemy's lost-connection flag). A +pruned catalog version, or a missing parameter tree or search file, answers +with its own message and records nothing. The event never carries an error's +message: SQLAlchemy's and the driver's name the SQL statement and its +parameters, and can name hosts, addresses and credentials. + ## Simulation client transport The simulation HTTP client reads identifiers from request context. Its request diff --git a/policyengine_api/fastapi_routes/v2/metadata/read_failure_support.py b/policyengine_api/fastapi_routes/v2/metadata/read_failure_support.py new file mode 100644 index 000000000..6aeba30a8 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/metadata/read_failure_support.py @@ -0,0 +1,101 @@ +"""Record why a v2 metadata read answered the generic 503 or a 500. + +A catalog the API can't read answers the generic 503 "V2 metadata catalog is +unavailable", and any other unexpected failure the 500 "V2 metadata query +failed", so the response names no cause. Each such answer records one runtime +event, ``v2_metadata_read_failed``, at ERROR. The runtime adds the request it +is serving: its ID, method and matched route. The event adds: + +- ``http.response.status_code``: 503 or 500; +- ``error.type``: the caught error's class, as ``module.QualifiedName``; +- ``error.cause_types``: the class of each error in its ``__cause__`` chain, + outermost first and joined by ", ", or absent when it has no cause; +- ``database.sqlstate``: the first SQLSTATE an error in the chain carries, as + psycopg's errors do in ``sqlstate``; +- ``database.connection_invalidated``: whether SQLAlchemy found the connection + lost, from the first error in the chain with ``connection_invalidated``, as + ``sqlalchemy.exc.DBAPIError`` has. + +It records no error message: SQLAlchemy's and the driver's messages carry the +SQL statement and its parameters, and can carry host names, addresses and +credentials. The attributes are read by name, so this module imports neither +SQLAlchemy nor a driver. Recording never changes the response. +""" + +from __future__ import annotations + +import contextlib + +METADATA_READ_FAILURE_EVENT = "v2_metadata_read_failed" +MAX_NAMED_CAUSES = 10 +"""How many errors of a ``__cause__`` chain the event names at most.""" + + +def record_metadata_read_failure(error: BaseException, status_code: int) -> None: + """Record why one metadata read failed; recording never fails the request.""" + # Telemetry must never change a metadata response. + with contextlib.suppress(Exception): + from policyengine_api import ( # noqa: PLC0415 # importing it builds the process-wide runtime, which importing the route modules alone must not + observability, + ) + + observability.get_runtime().event( + METADATA_READ_FAILURE_EVENT, + severity="ERROR", + attributes=read_failure_attributes(error, status_code), + ) + + +def read_failure_attributes( + error: BaseException, status_code: int +) -> dict[str, str | int | bool]: + """Return the event's attributes: class names, SQLSTATE and lost-connection flag.""" + causes = _cause_chain(error) + attributes: dict[str, str | int | bool] = { + "http.response.status_code": status_code, + "error.type": _class_name(error), + } + if causes: + attributes["error.cause_types"] = ", ".join( + _class_name(cause) for cause in causes + ) + chain = (error, *causes) + sqlstate = next( + ( + code + for code in (getattr(item, "sqlstate", None) for item in chain) + if isinstance(code, str) and code + ), + None, + ) + if sqlstate is not None: + attributes["database.sqlstate"] = sqlstate + invalidated = next( + ( + flag + for flag in ( + getattr(item, "connection_invalidated", None) for item in chain + ) + if isinstance(flag, bool) + ), + None, + ) + if invalidated is not None: + attributes["database.connection_invalidated"] = invalidated + return attributes + + +def _cause_chain(error: BaseException) -> list[BaseException]: + causes: list[BaseException] = [] + cause = error.__cause__ + while cause is not None and cause is not error and len(causes) < MAX_NAMED_CAUSES: + if cause in causes: + break + causes.append(cause) + cause = cause.__cause__ + return causes + + +def _class_name(error: BaseException) -> str: + error_class = type(error) + return f"{error_class.__module__}.{error_class.__qualname__}" diff --git a/policyengine_api/fastapi_routes/v2/metadata/route_support.py b/policyengine_api/fastapi_routes/v2/metadata/route_support.py index e8a5d005f..05887f8d4 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/route_support.py +++ b/policyengine_api/fastapi_routes/v2/metadata/route_support.py @@ -22,6 +22,9 @@ V2ErrorResponse, v2_error_response, ) +from policyengine_api.fastapi_routes.v2.metadata.read_failure_support import ( + record_metadata_read_failure, +) from policyengine_api.fastapi_routes.v2.metadata.response_cache_support import ( BuiltResponse, KeptResponse, @@ -424,7 +427,8 @@ def run_metadata_read( """Run one metadata read and return its result unchanged. Failures become the typed v2 error responses described by read_resource, - and the reader is closed whatever happens. + and the reader is closed whatever happens. A generic 503 or a 500 also + records why, without the error's message (``read_failure_support``). """ reader = None try: @@ -447,8 +451,12 @@ def run_metadata_read( ) as error: return v2_error_response(404, str(error)) except (V2ConfigurationError, MetadataCatalogUnavailableError) as error: - return _catalog_unavailable_response(error) - except Exception: # noqa: BLE001 # preview must return typed errors + response = _catalog_unavailable_response(error) + if _is_generic_unavailable_answer(error): + record_metadata_read_failure(error, response.status_code) + return response + except Exception as error: # noqa: BLE001 # preview must return typed errors + record_metadata_read_failure(error, 500) return v2_error_response(500, "V2 metadata query failed") finally: if reader is not None: @@ -457,6 +465,22 @@ def run_metadata_read( reader.close() +def _is_generic_unavailable_answer( + error: V2ConfigurationError | MetadataCatalogUnavailableError, +) -> bool: + """Whether ``_catalog_unavailable_response`` answers with no message of its own. + + A pruned version, or a missing parameter tree or search file, is a known + state whose message names the catalog version; only the others hide why. + """ + return not isinstance( + error, + CatalogValuesPrunedError + | ParameterTreeUnavailableError + | ParameterSearchFileUnavailableError, + ) + + def _catalog_unavailable_response( error: V2ConfigurationError | MetadataCatalogUnavailableError, ) -> JSONResponse: From ec6cee41a1d6997cb8797edc17a30ae4f74f270f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 01:46:03 +0400 Subject: [PATCH 09/18] Test refreshing the catalog tables' statistics after publication (failing) PostgreSQL re-collects a table's statistics only after about a tenth of its rows change, and parameters keeps every published version, so one release's rows never triggered it on staging: after 6.17.1 the planner estimated 1 US parameter row instead of 102,530, and the parameter-values list took 43-47 s. These tests expect publication, running as the catalog writer, to ANALYZE the catalog tables after a run that adds a version, to analyze nothing when nothing changed, and to fail before writing when its role can't analyze them. The new test file joins the v2 integration job. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/v2-integration-check.yml | 2 +- .../integration/test_v2_catalog_statistics.py | 189 ++++++++++++++++++ tests/unit/v2/test_privilege_manifest.py | 27 +++ 3 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 tests/integration/test_v2_catalog_statistics.py diff --git a/.github/workflows/v2-integration-check.yml b/.github/workflows/v2-integration-check.yml index c21bf595c..ece1ea3e1 100644 --- a/.github/workflows/v2-integration-check.yml +++ b/.github/workflows/v2-integration-check.yml @@ -74,7 +74,7 @@ jobs: - name: Verify the managed v2 roles hold their declared privileges run: uv run --frozen --extra dev python scripts/reconcile_v2_privileges.py verify - name: Test v2 metadata publication, resource routes, the countries list, reports and simulations - run: uv run --frozen --extra dev coverage run --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_catalog_publication_time_periods.py tests/integration/test_v2_catalog_publication_pruning.py tests/integration/test_v2_catalog_spm_availability.py tests/integration/test_v2_metadata_routes.py tests/integration/test_v2_country_routes.py tests/integration/test_v2_report_reads.py tests/integration/test_v2_report_creation.py tests/integration/test_v2_budget_window_report_reads.py tests/integration/test_v2_budget_window_report_creation.py tests/integration/test_v2_report_runs.py + run: uv run --frozen --extra dev coverage run --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_catalog_publication_time_periods.py tests/integration/test_v2_catalog_publication_pruning.py tests/integration/test_v2_catalog_spm_availability.py tests/integration/test_v2_catalog_statistics.py tests/integration/test_v2_metadata_routes.py tests/integration/test_v2_country_routes.py tests/integration/test_v2_report_reads.py tests/integration/test_v2_report_creation.py tests/integration/test_v2_budget_window_report_reads.py tests/integration/test_v2_budget_window_report_creation.py tests/integration/test_v2_report_runs.py - name: Test v2 policy persistence and immediate v1 mirroring run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_policy_persistence.py tests/integration/test_v1_policy_dual_write.py tests/integration/test_mysql_policy_dual_write.py tests/integration/test_v2_user_policy_mirroring.py tests/integration/test_v1_user_policy_dual_write.py - name: Test signed-in identity provisioning, user associations, saved detailed-save records and user types diff --git a/tests/integration/test_v2_catalog_statistics.py b/tests/integration/test_v2_catalog_statistics.py new file mode 100644 index 000000000..3eb1ddae2 --- /dev/null +++ b/tests/integration/test_v2_catalog_statistics.py @@ -0,0 +1,189 @@ +"""PostgreSQL tests for refreshing the catalog tables' statistics after publication. + +PostgreSQL re-collects a table's statistics on its own only after about a tenth +of its rows change. ``parameters`` keeps every published version, so on staging +one release's rows stayed under that share: after 6.17.1 was published on +2026-11-10, the planner estimated 1 parameter row for US 6.17.1 instead of +102,530 and served ``GET /v2/parameter-values?country_id=us&limit=1`` in 43-47 +seconds. Publication now runs ``ANALYZE`` on the catalog tables after a run that +adds, restores or prunes a version. ``ANALYZE`` skips, with only a warning, a +table the role neither owns nor holds ``MAINTAIN`` on, so publication checks +that privilege first and fails before writing anything without it. +""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import TYPE_CHECKING +from uuid import uuid4 + +import pytest +from alembic import command +from alembic.config import Config +from sqlalchemy import create_engine, text +from sqlalchemy.engine import make_url +from sqlalchemy.pool import NullPool + +from policyengine_api.data.v2.catalog.publication import publish_catalog +from policyengine_api.data.v2.catalog.publication_types import ( + CatalogPublicationError, +) +from policyengine_api.data.v2.privilege_reconciliation import ( + apply_declared_privileges, +) +from policyengine_api.data.v2.privileges import ( + CATALOG_WRITER_ROLE, + declared_privileges, +) +from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL +from tests.fixtures.v2_catalog import normalized_catalog + +if TYPE_CHECKING: + from collections.abc import Iterator + from datetime import datetime + + from sqlalchemy.engine import Engine + +REPO = Path(__file__).parents[2] +DISPOSABLE_DATABASE = "policyengine_v2_alembic_test" +ANALYZED_TABLES = ( + "datasets", + "parameter_nodes", + "parameter_tree_children", + "parameter_values", + "parameters", + "regions", + "variables", +) + + +def _disposable_url() -> str: + database_url = os.environ.get(V2_MIGRATION_DATABASE_URL, "") + if not database_url: + pytest.skip(f"{V2_MIGRATION_DATABASE_URL} is not set") + url = make_url(database_url) + if url.database != DISPOSABLE_DATABASE or url.host not in { + "localhost", + "127.0.0.1", + "::1", + "postgres", + }: + pytest.fail("catalog publication tests require disposable local Postgres") + return database_url + + +@pytest.fixture +def superuser_engine() -> Iterator[Engine]: + """Upgrade the disposable database and empty its catalog before and after.""" + database_url = _disposable_url() + command.upgrade(Config(str(REPO / "alembic-v2.ini")), "head") + engine = create_engine(database_url, poolclass=NullPool) + with engine.begin() as connection: + connection.execute(text("TRUNCATE tax_benefit_models CASCADE")) + try: + yield engine + finally: + with engine.begin() as connection: + connection.execute(text("TRUNCATE tax_benefit_models CASCADE")) + engine.dispose() + + +@pytest.fixture +def writer_engine(superuser_engine, v2_engine_as_role) -> Iterator[Engine]: + engine = v2_engine_as_role(CATALOG_WRITER_ROLE, poolclass=NullPool) + try: + yield engine + finally: + engine.dispose() + + +def _database_time(engine: Engine) -> datetime: + with engine.connect() as connection: + return connection.execute(text("SELECT clock_timestamp()")).scalar_one() + + +def _last_manual_analyze(engine: Engine) -> dict[str, datetime | None]: + with engine.connect() as connection: + rows = connection.execute( + text( + "SELECT relname, last_analyze FROM pg_stat_user_tables " + "WHERE schemaname = 'public' AND relname = ANY(:tables)" + ), + {"tables": list(ANALYZED_TABLES)}, + ) + return {row.relname: row.last_analyze for row in rows} + + +def test_publishing_a_new_version_as_the_catalog_writer_analyzes_the_catalog( + superuser_engine, writer_engine +) -> None: + started = _database_time(superuser_engine) + + evidence = publish_catalog(writer_engine, normalized_catalog()) + last_analyze = _last_manual_analyze(superuser_engine) + + assert evidence.catalog_tables_analyzed == ANALYZED_TABLES + assert evidence.as_dict()["catalog_tables_analyzed"] == list(ANALYZED_TABLES) + assert sorted(last_analyze) == list(ANALYZED_TABLES) + assert all( + analyzed_at is not None and analyzed_at >= started + for analyzed_at in last_analyze.values() + ), last_analyze + + +def test_republishing_an_unchanged_catalog_analyzes_nothing( + superuser_engine, writer_engine +) -> None: + catalog = normalized_catalog() + publish_catalog(writer_engine, catalog) + after_first = _last_manual_analyze(superuser_engine) + + second = publish_catalog(writer_engine, catalog) + + assert second.catalog_tables_analyzed == () + assert second.as_dict()["catalog_tables_analyzed"] == [] + assert _last_manual_analyze(superuser_engine) == after_first + + +@pytest.fixture +def writer_without_maintain(superuser_engine) -> Iterator[str]: + """Create a role with the catalog writer's declared privileges except MAINTAIN.""" + role = f"policyengine_catalog_test_{uuid4().hex[:12]}" + writer_tables = declared_privileges("test")[CATALOG_WRITER_ROLE] + declared = { + role: { + table: frozenset( + privilege for privilege in privileges if privilege != "MAINTAIN" + ) + for table, privileges in writer_tables.items() + } + } + with superuser_engine.begin() as connection: + connection.execute(text(f'CREATE ROLE "{role}" NOLOGIN')) + apply_declared_privileges(connection, declared) + try: + yield role + finally: + with superuser_engine.begin() as connection: + connection.execute(text(f'DROP OWNED BY "{role}"')) + connection.execute(text(f'DROP ROLE "{role}"')) + + +def test_a_role_that_cannot_analyze_the_catalog_fails_before_writing( + superuser_engine, v2_engine_as_role, writer_without_maintain +) -> None: + engine = v2_engine_as_role(writer_without_maintain, poolclass=NullPool) + try: + with pytest.raises(CatalogPublicationError) as raised: + publish_catalog(engine, normalized_catalog()) + finally: + engine.dispose() + with superuser_engine.connect() as connection: + stored_versions = connection.execute( + text("SELECT count(*) FROM tax_benefit_model_versions") + ).scalar_one() + + assert "MAINTAIN" in str(raised.value) + assert "parameters" in str(raised.value) + assert stored_versions == 0 diff --git a/tests/unit/v2/test_privilege_manifest.py b/tests/unit/v2/test_privilege_manifest.py index 2f81268c1..a1b457037 100644 --- a/tests/unit/v2/test_privilege_manifest.py +++ b/tests/unit/v2/test_privilege_manifest.py @@ -393,6 +393,33 @@ def test_rule_proof_runtime_privileges_declared() -> None: assert undeclared_requirements(good, runtime_before_3977) == [] +def test_the_catalog_writer_may_analyze_the_tables_publication_refreshes() -> None: + """Publication runs ANALYZE on these tables, which needs MAINTAIN or ownership. + + The migration role owns every table, so the catalog writer needs MAINTAIN + (PostgreSQL 17); without it, ANALYZE skips the table with only a warning. + """ + writer_tables = declared_privileges(TEST_ENVIRONMENT)[CATALOG_WRITER_ROLE] + analyzed = ( + "datasets", + "parameter_nodes", + "parameter_tree_children", + "parameter_values", + "parameters", + "regions", + "variables", + ) + + assert { + table for table in analyzed if "MAINTAIN" not in writer_tables[table] + } == set() + assert { + table + for table, privileges in writer_tables.items() + if "MAINTAIN" in privileges and table not in analyzed + } == set() + + def test_the_catalog_writer_deletes_only_the_rows_catalog_pruning_removes() -> None: """Catalog pruning deletes from three tables and marks the pruned version. From fb8d8dd8b15501f83fc5f7de906da76edbaf9dcd Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 01:49:20 +0400 Subject: [PATCH 10/18] Refresh the catalog tables' statistics after a publication PostgreSQL's autovacuum analyzes a table again only after about a tenth of its rows change, and parameters keeps every published version, so on staging the planner never saw 6.17.1's 102,530 US parameter rows and GET /v2/parameter-values?country_id=us&limit=1 took 43-47 s on every release from 2026-11-10 to 2026-11-16; after a manual ANALYZE it takes 4.1 ms. A publication that publishes, restores or prunes a version now runs one ANALYZE over datasets, parameter_nodes, parameter_tree_children, parameter_values, parameters, regions and variables after it commits and prunes, and lists them in the evidence's catalog_tables_analyzed. ANALYZE needs the owner or MAINTAIN, and PostgreSQL skips any other table with only a warning; the migration role owns the tables, so the manifest grants the catalog writer MAINTAIN on these seven (PostgreSQL 17), and publication fails before writing anything if its role lacks it. Co-Authored-By: Claude Opus 5.5 --- changelog.d/v2-catalog-analyze.fixed.md | 1 + docs/engineering/skills/alembic-migrations.md | 2 +- docs/migration/stage-9-v2-metadata.md | 27 ++++++- .../data/v2/catalog/publication.py | 19 ++++- .../data/v2/catalog/publication_statistics.py | 78 +++++++++++++++++++ .../data/v2/catalog/publication_types.py | 4 + policyengine_api/data/v2/privileges.py | 30 ++++--- 7 files changed, 149 insertions(+), 12 deletions(-) create mode 100644 changelog.d/v2-catalog-analyze.fixed.md create mode 100644 policyengine_api/data/v2/catalog/publication_statistics.py diff --git a/changelog.d/v2-catalog-analyze.fixed.md b/changelog.d/v2-catalog-analyze.fixed.md new file mode 100644 index 000000000..a62fb5f03 --- /dev/null +++ b/changelog.d/v2-catalog-analyze.fixed.md @@ -0,0 +1 @@ +Catalog publication now refreshes PostgreSQL's statistics of the catalog tables after it publishes, restores or prunes a version, so a new version's rows no longer leave `GET /v2/parameter-values` planned as if the version had one parameter (43 to 47 seconds on staging, 4.1 ms after `ANALYZE`). The catalog writer role gains `MAINTAIN` on those tables. diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index 28fe9927b..046344556 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -253,7 +253,7 @@ and a revision runs once per database, so it can't repair drift. | Role | Used by | | --- | --- | | `policyengine_v2_runtime` | The Public API on Cloud Run | -| `policyengine_v2_catalog_writer` | `scripts/publish_v2_metadata_catalog.py` | +| `policyengine_v2_catalog_writer` | `scripts/publish_v2_metadata_catalog.py`, which also runs `ANALYZE` on the catalog tables it grows, with `MAINTAIN` (PostgreSQL 17) | | `policyengine_stage12_` | The simulation API and its Modal worker, on the temporary comparison-run tables | | `policyengine_v2_simulation_service` | The simulation API on Cloud Run: creates simulations, reports, their runs, budget-window runs and household jobs, and reads the policy, dynamic, household and catalog rows a run needs. It updates only the status, Modal call ID, error and completion columns of the runs and household jobs whose Modal calls it starts, and the status, error and completion columns of a budget-window run, which it fails at once when Modal refuses the calls of the window's first years | | `policyengine_v2_modal_worker` | The simulation API's Modal workers: write their runs' and household jobs' status and results, and budget-window runs' status, and read the rows a run needs. It updates only a run's status, Modal call ID, error, time and result columns, and a budget-window run's status, error and time columns: each year's coordinator starts the window's next unstarted year when its own year finishes, the last year to finish marks the window succeeded, and a failed year fails it. The workers' code, not the database, keeps a worker to its own run | diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 62e445528..3f36ac39b 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -192,7 +192,7 @@ Use three separately managed database credentials: | Operation | Configuration | Required database access | | --- | --- | --- | | Alembic schema upgrade | `V2_MIGRATION_DATABASE_URL` | Reviewed v2 schema changes; no application-runtime use | -| Catalog publication | `V2_DATA_WRITE_DATABASE_URL` | Row insertion, update, selection, deletion from `parameter_values`, `parameter_tree_children` and `parameter_search_files` (catalog pruning), temporary tables, and transaction-scoped advisory locks; no persistent schema changes | +| Catalog publication | `V2_DATA_WRITE_DATABASE_URL` | Row insertion, update, selection, deletion from `parameter_values`, `parameter_tree_children` and `parameter_search_files` (catalog pruning), `ANALYZE` of the catalog tables (`MAINTAIN`), temporary tables, and transaction-scoped advisory locks; no persistent schema changes | | Preview GET requests | `V2_RUNTIME_DATABASE_URL` or `V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE` | Catalog selection only during Stage 9; no migration access | Do not place more than one database URL in the environment of either the @@ -626,6 +626,31 @@ Deleted rows free space inside the tables for later publications to reuse; PostgreSQL returns it to the operating system only after a `VACUUM FULL` or an equivalent rewrite, which pruning doesn't run. +### Catalog table statistics + +PostgreSQL's autovacuum analyzes a table again only after about a tenth of its +rows have changed. `parameters` keeps every published version (about 1.25 +million rows on staging), and one release adds about 105,000, so a new +version's rows can stay out of the planner's statistics. On staging, after +6.17.1 was published on 2026-11-10, the planner estimated 1 parameter row for +US 6.17.1 instead of 102,530, chose to fetch and sort every parameter value of +that version, and `GET /v2/parameter-values?country_id=us&limit=1` took 43 to +47 seconds on every release until 2026-11-16. After a manual `ANALYZE +parameters` it walks `uq_parameters_model_version_name` and takes 4.1 ms +(UK: 4.9 ms). + +A publication that publishes, restores or prunes a version therefore runs one +`ANALYZE` over `datasets`, `parameter_nodes`, `parameter_tree_children`, +`parameter_values`, `parameters`, `regions` and `variables` once it has +committed and pruned, and lists them in the evidence's +`catalog_tables_analyzed`; a run that changed none lists none. `ANALYZE` needs +the table's owner or the `MAINTAIN` privilege (PostgreSQL 17), and PostgreSQL +skips any other table with only a warning. The migration role owns the tables, +so the manifest grants the catalog writer `MAINTAIN` on these seven, the +deploy's `apply` step grants it before publication runs, and publication +fails before writing anything if its role lacks it. The code is +`policyengine_api/data/v2/catalog/publication_statistics.py`. + ## Validation evidence and retry Successful JSON evidence contains: diff --git a/policyengine_api/data/v2/catalog/publication.py b/policyengine_api/data/v2/catalog/publication.py index 97f1358a1..f55ed9eb5 100644 --- a/policyengine_api/data/v2/catalog/publication.py +++ b/policyengine_api/data/v2/catalog/publication.py @@ -41,6 +41,10 @@ create_staging_tables, stage_catalog, ) +from policyengine_api.data.v2.catalog.publication_statistics import ( + analyze_catalog_tables, + require_analyze_privilege, +) from policyengine_api.data.v2.catalog.publication_types import ( CatalogPruningFailure, CatalogPublicationError, @@ -163,12 +167,16 @@ def publish_catalog( After the transaction commits, it prunes the older stored versions of the catalog's models (``publication_pruning``), at most two per run, one transaction each; a failure of any kind rolls back only that version and - is reported in the evidence. + is reported in the evidence. Then, if the run published, restored or + pruned a version, it analyzes the catalog tables, whose planner statistics + PostgreSQL would otherwise leave out of date (``publication_statistics``). + Before it writes anything it checks that its role may analyze them. """ started = time.monotonic() with engine.begin() as connection: _verify_expected_revision(connection) _acquire_publication_lock(connection) + require_analyze_privilege(connection) before = _protected_row_counts(connection) create_staging_tables(connection) stage_catalog(connection, catalog, checkpoint=checkpoint) @@ -208,6 +216,14 @@ def publish_catalog( checkpoint("after_validation", connection) pruning = _prune_older_versions(engine, catalog) + published_new_version = any( + country.country_id not in existing for country in catalog.countries + ) + tables_analyzed = ( + analyze_catalog_tables(engine) + if published_new_version or restored_versions or pruning.pruned + else () + ) return replace( _evidence( catalog, @@ -220,6 +236,7 @@ def publish_catalog( pruned_catalog_versions=pruning.pruned, catalog_pruning_failures=pruning.failures, catalog_versions_left_to_prune=pruning.left_to_prune, + catalog_tables_analyzed=tables_analyzed, ) diff --git a/policyengine_api/data/v2/catalog/publication_statistics.py b/policyengine_api/data/v2/catalog/publication_statistics.py new file mode 100644 index 000000000..83fbd1817 --- /dev/null +++ b/policyengine_api/data/v2/catalog/publication_statistics.py @@ -0,0 +1,78 @@ +"""Refresh the planner statistics of the catalog tables after a publication. + +PostgreSQL's autovacuum analyzes a table again only after about a tenth of its +rows have changed since the last time. ``parameters`` and ``parameter_values`` +keep every published catalog version, so as they grow one release's rows stay +under that share. On staging, after 6.17.1 was published on 2026-11-10, the +planner estimated 1 parameter row for US 6.17.1 instead of 102,530, and +``GET /v2/parameter-values?country_id=us&limit=1`` took 43 to 47 seconds until +``ANALYZE parameters`` brought it to 4.1 ms. + +So publication runs one ``ANALYZE`` over the catalog tables it writes after a +run that publishes, restores or prunes a catalog version +(``analyze_catalog_tables``). ``ANALYZE`` needs the table's owner or the +``MAINTAIN`` privilege, and PostgreSQL skips any other table with only a +warning. The migration role owns every table, so the manifest declares +``MAINTAIN`` for the catalog writer on these tables, and publication checks +that its role holds it before it writes anything +(``require_analyze_privilege``). +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Final + +import sqlalchemy as sa + +from policyengine_api.data.v2.catalog.publication_types import ( + CatalogPublicationError, +) + +if TYPE_CHECKING: + from sqlalchemy import Connection, Engine + +CATALOG_STATISTICS_TABLES: Final = ( + "datasets", + "parameter_nodes", + "parameter_tree_children", + "parameter_values", + "parameters", + "regions", + "variables", +) +"""The catalog tables publication analyzes, in name order.""" + +_HAS_MAINTAIN = sa.text( + "SELECT has_table_privilege(current_user, :qualified_table, 'MAINTAIN')" +) + + +def require_analyze_privilege(connection: Connection) -> None: + """Raise unless the connection's role may ``ANALYZE`` every catalog table. + + An owner holds every privilege, so ``has_table_privilege`` answers for + ownership and a ``MAINTAIN`` grant alike. + """ + missing = [ + table + for table in CATALOG_STATISTICS_TABLES + if not connection.execute( + _HAS_MAINTAIN, {"qualified_table": f"public.{table}"} + ).scalar_one() + ] + if missing: + msg = ( + "the publication role lacks MAINTAIN on " + + ", ".join(missing) + + ", so ANALYZE would skip them with only a warning; nothing was " + "published" + ) + raise CatalogPublicationError(msg) + + +def analyze_catalog_tables(engine: Engine) -> tuple[str, ...]: + """Run one ``ANALYZE`` over the catalog tables and return their names.""" + tables = ", ".join(f"public.{table}" for table in CATALOG_STATISTICS_TABLES) + with engine.begin() as connection: + connection.execute(sa.text(f"ANALYZE {tables}")) + return CATALOG_STATISTICS_TABLES diff --git a/policyengine_api/data/v2/catalog/publication_types.py b/policyengine_api/data/v2/catalog/publication_types.py index e46f478c1..ea998428b 100644 --- a/policyengine_api/data/v2/catalog/publication_types.py +++ b/policyengine_api/data/v2/catalog/publication_types.py @@ -110,6 +110,9 @@ class PublicationEvidence: catalog_pruning_failures: tuple[CatalogPruningFailure, ...] = () # Chosen versions beyond the run's limit, which the next publications prune. catalog_versions_left_to_prune: int = 0 + # The catalog tables analyzed after a run that published, restored or + # pruned a version (``publication_statistics``); empty when it changed none. + catalog_tables_analyzed: tuple[str, ...] = () # The SPM availability recorded after the publication committed; None # until initialization records it. spm_availability: SPMAvailabilityRecording | None = None @@ -202,6 +205,7 @@ def as_dict(self) -> dict[str, object]: for failure in self.catalog_pruning_failures ], "catalog_versions_left_to_prune": self.catalog_versions_left_to_prune, + "catalog_tables_analyzed": list(self.catalog_tables_analyzed), "spm_availability": ( None if self.spm_availability is None diff --git a/policyengine_api/data/v2/privileges.py b/policyengine_api/data/v2/privileges.py index 44ea0efb9..5b841b5b2 100644 --- a/policyengine_api/data/v2/privileges.py +++ b/policyengine_api/data/v2/privileges.py @@ -25,8 +25,9 @@ temporary Stage 12 tables (``docs/migration/stage-12-worker-contract.md``). - ``policyengine_v2_catalog_writer``: the metadata catalog publication (``scripts/publish_v2_metadata_catalog.py``). Its privileges follow what - publication writes and reads; its code isn't organized as connector modules, - so the deploy check, not the connector analysis, covers it. + publication writes and reads, plus ``MAINTAIN`` on the catalog tables it + analyzes afterwards; its code isn't organized as connector modules, so the + deploy check, not the connector analysis, covers it. - ``policyengine_v2_simulation_service``: the simulation API on Cloud Run. It creates simulations, reports, their runs, budget-window runs and household jobs, and reads the policy, dynamic, household and catalog rows a run needs. @@ -95,6 +96,8 @@ class TablePrivilege(StrEnum): INSERT = "INSERT" UPDATE = "UPDATE" DELETE = "DELETE" + # PostgreSQL 17: lets a role that doesn't own the table run ANALYZE on it. + MAINTAIN = "MAINTAIN" @dataclass(frozen=True) @@ -133,6 +136,10 @@ def update_columns(*columns: str) -> ColumnPrivilege: TablePrivilege.DELETE, } ) +# ANALYZE after publication (``catalog.publication_statistics``): the migration +# role owns the catalog tables, and PostgreSQL skips a table the role neither +# owns nor holds MAINTAIN on, with only a warning. +_ANALYZE: Final = frozenset({TablePrivilege.MAINTAIN}) @dataclass(frozen=True) @@ -272,29 +279,34 @@ def _run_table( # The UK areas disclosure control withheld from a local-area run's results. "withheld_area_codes": _RESULT, # Metadata catalog: the catalog writer publishes it, the Public API reads it. + # The writer also analyzes the tables publication grows (_ANALYZE). "dataset_versions": _access(catalog_writer=READ), "datasets": _access( - runtime=READ, catalog_writer=READ_INSERT_UPDATE, **_RUN_INPUT_READS + runtime=READ, catalog_writer=READ_INSERT_UPDATE | _ANALYZE, **_RUN_INPUT_READS + ), + "parameter_nodes": _access( + runtime=READ, catalog_writer=READ_INSERT_UPDATE | _ANALYZE ), - "parameter_nodes": _access(runtime=READ, catalog_writer=READ_INSERT_UPDATE), # Written by catalog publication for each version it publishes, from the # installed packages' catalog; never rewritten. Catalog pruning deletes an # older version's file, which publishing that version again writes back. "parameter_search_files": _access(runtime=READ, catalog_writer=READ_INSERT_DELETE), # Written with each catalog version that publication publishes; catalog # pruning deletes an older version's rows. - "parameter_tree_children": _access(runtime=READ, catalog_writer=READ_INSERT_DELETE), + "parameter_tree_children": _access( + runtime=READ, catalog_writer=READ_INSERT_DELETE | _ANALYZE + ), # The Public API also inserts a v2 policy's own parameter values. Catalog # pruning deletes an older version's current-law values (those with no # policy or dynamic), never a policy's or a dynamic's. "parameter_values": _access( - runtime=READ_INSERT, catalog_writer=READ_WRITE, **_RUN_INPUT_READS + runtime=READ_INSERT, catalog_writer=READ_WRITE | _ANALYZE, **_RUN_INPUT_READS ), "parameters": _access( - runtime=READ, catalog_writer=READ_INSERT_UPDATE, **_RUN_INPUT_READS + runtime=READ, catalog_writer=READ_INSERT_UPDATE | _ANALYZE, **_RUN_INPUT_READS ), "regions": _access( - runtime=READ, catalog_writer=READ_INSERT_UPDATE, **_RUN_INPUT_READS + runtime=READ, catalog_writer=READ_INSERT_UPDATE | _ANALYZE, **_RUN_INPUT_READS ), # Catalog publication updates a version's time periods, # catalog_last_published_at and spm_availability, and catalog pruning its @@ -305,7 +317,7 @@ def _run_table( "tax_benefit_models": _access( runtime=READ, catalog_writer=READ_INSERT_UPDATE, **_RUN_INPUT_READS ), - "variables": _access(runtime=READ, catalog_writer=READ_INSERT_UPDATE), + "variables": _access(runtime=READ, catalog_writer=READ_INSERT_UPDATE | _ANALYZE), # v2 policies and households, created by the native routes and by Stage 10 # dual-write. A copy locks its mapping row (FOR UPDATE), which needs UPDATE. "households": _access(runtime=READ_INSERT, **_RUN_INPUT_READS), From 0171b1424f4f73e9493684091d78ad4ba6c5d5b7 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 01:53:01 +0400 Subject: [PATCH 11/18] Test the metadata failure event's grouping fields (failing) The scheduled error-issue job groups an event by its route and its attributes.error_type and attributes.failure_category, and the failure event had neither, so every 503 and 500 on a route would have merged into one issue. The tests now expect error_type and a failure category that tells a catalog outage from an unexpected failure. They also use psycopg's real AdminShutdown wrapped as SQLAlchemy wraps it, keep the process runtime's settings, and cover a configuration error, the parameter tree and search file answers, the 500 with a failing runtime, and looping and long cause chains. Co-Authored-By: Claude Opus 5.5 --- .../v2/test_metadata_read_failure_events.py | 217 +++++++++++------- 1 file changed, 130 insertions(+), 87 deletions(-) diff --git a/tests/unit/v2/test_metadata_read_failure_events.py b/tests/unit/v2/test_metadata_read_failure_events.py index f8ebfbdd6..3ef9478e9 100644 --- a/tests/unit/v2/test_metadata_read_failure_events.py +++ b/tests/unit/v2/test_metadata_read_failure_events.py @@ -3,37 +3,41 @@ The response stays the generic one, but the observability runtime records one event that says what failed: the route and request ID, which the runtime takes from the request it is serving, then the response status, the caught error's -class, the class of each error in its ``__cause__`` chain, the driver's -SQLSTATE and whether SQLAlchemy found the connection lost. It never records an -error's message, which can carry the SQL statement, its parameters, host -names, addresses or credentials. These tests serve the request through a runtime -configured as the process's is, but writing its records to a list, so they see -the event exactly as the runtime writes it, request context included. +class (``error_type``), a failure category that tells a catalog outage from an +unexpected failure, the class of each error in its ``__cause__`` chain, the +driver's SQLSTATE and whether SQLAlchemy found the connection lost. The +scheduled error-issue job groups such an event by route, ``error_type`` and +``failure_category``. The event never records an error's message, which can +carry the SQL statement, its parameters, host names, addresses or credentials. + +The tests serve each request through a runtime with the process runtime's +settings whose records go to a list, so they see the event as the runtime +writes it, request context included. """ from __future__ import annotations +from dataclasses import replace from typing import TYPE_CHECKING, Any import pytest -from policyengine_observability import ( - CustomLogDestination, - LoggingConfig, - ObservabilityConfig, - configure, -) -from sqlalchemy.exc import OperationalError +from policyengine_observability import CustomLogDestination, OTelConfig, configure +from psycopg import errors as psycopg_errors +from sqlalchemy.exc import DBAPIError from policyengine_api import observability from policyengine_api.data.v2.catalog.catalog_selection_validation import ( MetadataCatalogUnavailableError, ) -from policyengine_api.observability_identity import ( - deployment_identity, - service_identity, +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.fastapi_routes.v2.metadata.read_failure_support import ( + MAX_NAMED_CAUSES, + read_failure_attributes, ) from policyengine_api.services.v2.metadata.validators import ( CatalogValuesPrunedError, + ParameterSearchFileUnavailableError, + ParameterTreeUnavailableError, ) from tests.fixtures.v2_metadata_routes import ResourceReader, metadata_route_client @@ -45,6 +49,8 @@ ROUTE = "/v2/parameter-values" PATH = f"{ROUTE}?country_id=us&limit=1" FAILURE_EVENT = "v2_metadata_read_failed" +GENERIC_503 = {"status": "error", "message": "V2 metadata catalog is unavailable"} +GENERIC_500 = {"status": "error", "message": "V2 metadata query failed"} STATEMENT = "SELECT parameter_values.id FROM parameter_values WHERE country_id = %(c)s" PARAMETER_VALUE = "us-secret-parameter-value" SENSITIVE_TEXT = ( @@ -55,12 +61,7 @@ "hunter2", "postgresql://", ) - - -class DriverAdminShutdownError(Exception): - """Stands in for psycopg's error for SQLSTATE 57P01, a terminated connection.""" - - sqlstate = "57P01" +LOST_CONNECTION_CAUSES = "sqlalchemy.exc.OperationalError, psycopg.errors.AdminShutdown" class RecordList: @@ -88,13 +89,18 @@ def event(self, *_arguments: object, **_details: object) -> None: @pytest.fixture def recorded(monkeypatch) -> Iterator[tuple[RecordList, ObservabilityRuntime]]: - """Serve requests through a runtime whose records the test can read.""" + """Serve requests through a runtime whose records the test can read. + + It keeps the process runtime's settings and replaces only where records go, + with OpenTelemetry export off. + """ record_list = RecordList() + process_config = observability.get_runtime().config runtime = configure( - ObservabilityConfig.from_env( - service=service_identity(), - deployment=deployment_identity(), - logging=LoggingConfig( + replace( + process_config, + logging=replace( + process_config.logging, destinations=( CustomLogDestination( name="test-records", @@ -103,6 +109,7 @@ def recorded(monkeypatch) -> Iterator[tuple[RecordList, ObservabilityRuntime]]: ), ), ), + otel=OTelConfig(enabled=False), ) ) monkeypatch.setattr(observability, "get_runtime", lambda: runtime) @@ -112,27 +119,37 @@ def recorded(monkeypatch) -> Iterator[tuple[RecordList, ObservabilityRuntime]]: runtime.shutdown() -def _get(reader: ResourceReader, runtime: ObservabilityRuntime): +def _get(reader: ResourceReader, runtime: ObservabilityRuntime | None = None): client = metadata_route_client(lambda: reader, observability_runtime=runtime) return client.get(PATH) +def _reader_failing_with(error: BaseException) -> ResourceReader: + return ResourceReader({"list_parameter_values": None}, error=error) + + def _qualified_name(error_class: type[BaseException]) -> str: return f"{error_class.__module__}.{error_class.__qualname__}" -def _lost_connection() -> OperationalError: - """Return SQLAlchemy's error for a connection lost mid-query, as psycopg raises it. +def _lost_connection() -> DBAPIError: + """Return SQLAlchemy's error for a connection the server ended mid-query. - Every message in the chain names the statement, a parameter, the pooler's - host and address and a password, as a driver's message can. + psycopg raises ``AdminShutdown`` (SQLSTATE 57P01), and SQLAlchemy wraps it + as it does, raising its error from psycopg's. Every message names the + statement, a parameter, the pooler's host and address and a password, as a + driver's message can. """ - driver_error = DriverAdminShutdownError( + driver_error = psycopg_errors.AdminShutdown( "terminating connection due to administrator command: " "postgresql://reader:hunter2@pooler.example.internal (10.12.0.7)" ) - database_error = OperationalError( - STATEMENT, {"c": PARAMETER_VALUE}, driver_error, connection_invalidated=True + database_error = DBAPIError.instance( + STATEMENT, + {"c": PARAMETER_VALUE}, + driver_error, + psycopg_errors.Error, + connection_invalidated=True, ) database_error.__cause__ = driver_error return database_error @@ -152,32 +169,24 @@ def _assert_nothing_sensitive(record: dict[str, Any]) -> None: def test_a_lost_connection_answers_the_same_503_and_records_its_cause( recorded, ) -> None: + record_list, runtime = recorded error = _caused_by( MetadataCatalogUnavailableError(f"catalog read failed: {STATEMENT}"), _lost_connection(), ) - reader = ResourceReader({"list_parameter_values": None}, error=error) - - record_list, runtime = recorded - response = _get(reader, runtime) + response = _get(_reader_failing_with(error), runtime) [record] = record_list.failures() - assert response.status_code == 503 - assert response.json() == { - "status": "error", - "message": "V2 metadata catalog is unavailable", - } + assert (response.status_code, response.json()) == (503, GENERIC_503) assert record["severity"] == "ERROR" assert record["http.route"] == ROUTE assert record["request.id"] == response.headers["x-policyengine-request-id"] assert record["attributes"] == { "http.response.status_code": 503, - "error.type": _qualified_name(MetadataCatalogUnavailableError), - "error.cause_types": ( - "sqlalchemy.exc.OperationalError, " - f"{_qualified_name(DriverAdminShutdownError)}" - ), + "error_type": _qualified_name(MetadataCatalogUnavailableError), + "failure_category": "v2_metadata_catalog_unavailable", + "error.cause_types": LOST_CONNECTION_CAUSES, "database.sqlstate": "57P01", "database.connection_invalidated": True, } @@ -187,75 +196,109 @@ def test_a_lost_connection_answers_the_same_503_and_records_its_cause( def test_an_unexpected_failure_answers_the_same_500_and_records_its_cause( recorded, ) -> None: - error = _caused_by(RuntimeError(f"failed on {STATEMENT}"), _lost_connection()) - reader = ResourceReader({"list_parameter_values": None}, error=error) - record_list, runtime = recorded + error = _caused_by(RuntimeError(f"failed on {STATEMENT}"), _lost_connection()) - response = _get(reader, runtime) + response = _get(_reader_failing_with(error), runtime) [record] = record_list.failures() - assert response.status_code == 500 - assert response.json() == {"status": "error", "message": "V2 metadata query failed"} + assert (response.status_code, response.json()) == (500, GENERIC_500) assert (record["severity"], record["http.route"]) == ("ERROR", ROUTE) assert record["attributes"] == { "http.response.status_code": 500, - "error.type": "builtins.RuntimeError", - "error.cause_types": ( - "sqlalchemy.exc.OperationalError, " - f"{_qualified_name(DriverAdminShutdownError)}" - ), + "error_type": "builtins.RuntimeError", + "failure_category": "v2_metadata_query_failed", + "error.cause_types": LOST_CONNECTION_CAUSES, "database.sqlstate": "57P01", "database.connection_invalidated": True, } _assert_nothing_sensitive(record) +@pytest.mark.parametrize( + "error", + [ + MetadataCatalogUnavailableError("catalog connection refused"), + V2ConfigurationError("V2_DATABASE_URL is not set"), + ], + ids=["catalog-unavailable", "configuration"], +) def test_a_failure_without_a_database_cause_records_only_its_class( - recorded, + recorded, error: BaseException ) -> None: - reader = ResourceReader( - {"list_parameter_values": None}, - error=MetadataCatalogUnavailableError("catalog connection refused"), - ) - record_list, runtime = recorded - response = _get(reader, runtime) + response = _get(_reader_failing_with(error), runtime) [record] = record_list.failures() - assert response.status_code == 503 + assert (response.status_code, response.json()) == (503, GENERIC_503) assert record["attributes"] == { "http.response.status_code": 503, - "error.type": _qualified_name(MetadataCatalogUnavailableError), + "error_type": _qualified_name(type(error)), + "failure_category": "v2_metadata_catalog_unavailable", } -def test_a_pruned_version_answer_records_no_failure(recorded) -> None: - reader = ResourceReader( - {"list_parameter_values": None}, - error=CatalogValuesPrunedError("pruned", running_version=True), - ) - +@pytest.mark.parametrize( + "error", + [ + CatalogValuesPrunedError("pruned", running_version=True), + ParameterTreeUnavailableError("5.0.1 has no parameter tree"), + ParameterSearchFileUnavailableError("5.0.1 has no parameter search file"), + ], + ids=["pruned-version", "no-parameter-tree", "no-search-file"], +) +def test_an_answer_with_its_own_message_records_no_failure( + recorded, error: BaseException +) -> None: record_list, runtime = recorded - response = _get(reader, runtime) + response = _get(_reader_failing_with(error), runtime) assert response.status_code == 503 + assert response.json()["message"] == str(error) assert record_list.failures() == [] -def test_a_failing_runtime_leaves_the_503_unchanged(monkeypatch) -> None: +@pytest.mark.parametrize( + ("error", "status_code", "body"), + [ + (MetadataCatalogUnavailableError("catalog read failed"), 503, GENERIC_503), + (RuntimeError("failed"), 500, GENERIC_500), + ], + ids=["503", "500"], +) +def test_a_failing_runtime_leaves_the_answer_unchanged( + monkeypatch, error: BaseException, status_code: int, body: dict[str, str] +) -> None: monkeypatch.setattr(observability, "get_runtime", FailingRuntime) - error = _caused_by( - MetadataCatalogUnavailableError("catalog read failed"), _lost_connection() - ) - reader = ResourceReader({"list_parameter_values": None}, error=error) - response = metadata_route_client(lambda: reader).get(PATH) + response = _get(_reader_failing_with(_caused_by(error, _lost_connection()))) - assert response.status_code == 503 - assert response.json() == { - "status": "error", - "message": "V2 metadata catalog is unavailable", - } + assert (response.status_code, response.json()) == (status_code, body) + + +def test_a_cause_chain_that_loops_back_names_each_error_once() -> None: + first = RuntimeError("first") + second = ValueError("second") + first.__cause__ = second + second.__cause__ = first + + attributes = read_failure_attributes(first, 500, "v2_metadata_query_failed") + + assert attributes["error.cause_types"] == "builtins.ValueError" + + +def test_a_long_cause_chain_names_its_first_errors_only() -> None: + error = RuntimeError("outermost") + current = error + for _depth in range(MAX_NAMED_CAUSES + 2): + cause = KeyError("cause") + current.__cause__ = cause + current = cause + + attributes = read_failure_attributes(error, 500, "v2_metadata_query_failed") + + assert attributes["error.cause_types"] == ", ".join( + ["builtins.KeyError"] * MAX_NAMED_CAUSES + ) From a10175ed13d25f575eae1e3d5ed39e023d584c52 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 01:53:57 +0400 Subject: [PATCH 12/18] Group the metadata failure event as the error-issue job expects The event now carries error_type and failure_category, the keys the API's other events use and the scheduled error-issue job groups by, so 503s and 500s on a route, and different causes, stay apart. One helper, _caller_message, now decides both which answers have their own message and which record the event. The observability and v2 code-organization guides describe the event and its module, including that requests waiting on a shared whole-version read answer with a copy and record nothing. Co-Authored-By: Claude Opus 5.5 --- .../v2-metadata-read-failure-event.changed.md | 2 +- docs/engineering/skills/observability.md | 21 ++++++----- .../skills/v2-code-organization.md | 5 ++- .../v2/metadata/read_failure_support.py | 19 +++++++--- .../v2/metadata/route_support.py | 35 +++++++++---------- 5 files changed, 48 insertions(+), 34 deletions(-) diff --git a/changelog.d/v2-metadata-read-failure-event.changed.md b/changelog.d/v2-metadata-read-failure-event.changed.md index 085783865..24b41a9e2 100644 --- a/changelog.d/v2-metadata-read-failure-event.changed.md +++ b/changelog.d/v2-metadata-read-failure-event.changed.md @@ -1 +1 @@ -A v2 metadata read that answers the generic 503 or the 500 now records one runtime event, `v2_metadata_read_failed`, naming the route, the error classes in its cause chain, the database's SQLSTATE and whether the connection was lost, without any error message. +A v2 metadata read that answers the generic 503 or the 500 now records one runtime event, `v2_metadata_read_failed`, naming the route, the error's class and failure category, the error classes in its cause chain, the database's SQLSTATE and whether the connection was lost, without any error message. diff --git a/docs/engineering/skills/observability.md b/docs/engineering/skills/observability.md index ffc7628de..423b09c5e 100644 --- a/docs/engineering/skills/observability.md +++ b/docs/engineering/skills/observability.md @@ -91,14 +91,19 @@ unavailable", or the 500 "V2 metadata query failed", records one runtime event at ERROR, `v2_metadata_read_failed` (`policyengine_api.fastapi_routes.v2.metadata.read_failure_support`). The runtime adds the request's ID, method and matched route. The event adds -`http.response.status_code`, `error.type` (the caught error's class as -`module.QualifiedName`), `error.cause_types` (the classes of its `__cause__` -chain, outermost first), `database.sqlstate` (the first SQLSTATE in the chain) -and `database.connection_invalidated` (SQLAlchemy's lost-connection flag). A -pruned catalog version, or a missing parameter tree or search file, answers -with its own message and records nothing. The event never carries an error's -message: SQLAlchemy's and the driver's name the SQL statement and its -parameters, and can name hosts, addresses and credentials. +`http.response.status_code`, `error_type` (the caught error's class as +`module.QualifiedName`), `failure_category` (`v2_metadata_catalog_unavailable` +or `v2_metadata_query_failed`), `error.cause_types` (the classes of its +`__cause__` chain, outermost first), `database.sqlstate` (the first SQLSTATE in +the chain) and `database.connection_invalidated` (SQLAlchemy's lost-connection +flag). The scheduled error-issue job groups the event by route, `error_type` +and `failure_category`. A pruned catalog version, or a missing parameter tree +or search file, answers with its own message and records nothing. On the two +routes that share one read of a whole catalog version +(`response_cache_support`), only the request that ran the failed read records +it; requests that waited for it answer with a copy of its 503. The event never +carries an error's message: SQLAlchemy's and the driver's name the SQL +statement and its parameters, and can name hosts, addresses and credentials. ## Simulation client transport diff --git a/docs/engineering/skills/v2-code-organization.md b/docs/engineering/skills/v2-code-organization.md index c9cf4d0dc..223560d0a 100644 --- a/docs/engineering/skills/v2-code-organization.md +++ b/docs/engineering/skills/v2-code-organization.md @@ -72,6 +72,7 @@ user_policies/ metadata/ response_models.py route_support.py + read_failure_support.py query_support.py response_cache_support.py search_file_support.py @@ -80,7 +81,9 @@ metadata/ `route_support.py` runs the metadata reads for the route modules, maps their failures to HTTP, and holds the route class that gives metadata responses -their caching headers. `query_support.py` defines the typed FastAPI +their caching headers. `read_failure_support.py` records why a read answered +the generic 503 or a 500, without the error's message. `query_support.py` +defines the typed FastAPI dependencies for the shared query models once, for every metadata route module to import. `response_cache_support.py` keeps serialized whole-version responses (bodies and ETags) for the process; it is HTTP-level, so it lives diff --git a/policyengine_api/fastapi_routes/v2/metadata/read_failure_support.py b/policyengine_api/fastapi_routes/v2/metadata/read_failure_support.py index 6aeba30a8..e9d7af5b0 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/read_failure_support.py +++ b/policyengine_api/fastapi_routes/v2/metadata/read_failure_support.py @@ -7,7 +7,11 @@ is serving: its ID, method and matched route. The event adds: - ``http.response.status_code``: 503 or 500; -- ``error.type``: the caught error's class, as ``module.QualifiedName``; +- ``error_type``: the caught error's class, as ``module.QualifiedName``; +- ``failure_category``: ``v2_metadata_catalog_unavailable`` for the 503 or + ``v2_metadata_query_failed`` for the 500. The scheduled error-issue job + groups such an event by its route, ``error_type`` and ``failure_category``, + the keys the API's other events use; - ``error.cause_types``: the class of each error in its ``__cause__`` chain, outermost first and joined by ", ", or absent when it has no cause; - ``database.sqlstate``: the first SQLSTATE an error in the chain carries, as @@ -27,11 +31,15 @@ import contextlib METADATA_READ_FAILURE_EVENT = "v2_metadata_read_failed" +CATALOG_UNAVAILABLE_FAILURE = "v2_metadata_catalog_unavailable" +QUERY_FAILED_FAILURE = "v2_metadata_query_failed" MAX_NAMED_CAUSES = 10 """How many errors of a ``__cause__`` chain the event names at most.""" -def record_metadata_read_failure(error: BaseException, status_code: int) -> None: +def record_metadata_read_failure( + error: BaseException, status_code: int, failure_category: str +) -> None: """Record why one metadata read failed; recording never fails the request.""" # Telemetry must never change a metadata response. with contextlib.suppress(Exception): @@ -42,18 +50,19 @@ def record_metadata_read_failure(error: BaseException, status_code: int) -> None observability.get_runtime().event( METADATA_READ_FAILURE_EVENT, severity="ERROR", - attributes=read_failure_attributes(error, status_code), + attributes=read_failure_attributes(error, status_code, failure_category), ) def read_failure_attributes( - error: BaseException, status_code: int + error: BaseException, status_code: int, failure_category: str ) -> dict[str, str | int | bool]: """Return the event's attributes: class names, SQLSTATE and lost-connection flag.""" causes = _cause_chain(error) attributes: dict[str, str | int | bool] = { "http.response.status_code": status_code, - "error.type": _class_name(error), + "error_type": _class_name(error), + "failure_category": failure_category, } if causes: attributes["error.cause_types"] = ", ".join( diff --git a/policyengine_api/fastapi_routes/v2/metadata/route_support.py b/policyengine_api/fastapi_routes/v2/metadata/route_support.py index 05887f8d4..9c4368d5f 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/route_support.py +++ b/policyengine_api/fastapi_routes/v2/metadata/route_support.py @@ -23,6 +23,8 @@ v2_error_response, ) from policyengine_api.fastapi_routes.v2.metadata.read_failure_support import ( + CATALOG_UNAVAILABLE_FAILURE, + QUERY_FAILED_FAILURE, record_metadata_read_failure, ) from policyengine_api.fastapi_routes.v2.metadata.response_cache_support import ( @@ -451,12 +453,11 @@ def run_metadata_read( ) as error: return v2_error_response(404, str(error)) except (V2ConfigurationError, MetadataCatalogUnavailableError) as error: - response = _catalog_unavailable_response(error) - if _is_generic_unavailable_answer(error): - record_metadata_read_failure(error, response.status_code) - return response + if _caller_message(error) is None: + record_metadata_read_failure(error, 503, CATALOG_UNAVAILABLE_FAILURE) + return _catalog_unavailable_response(error) except Exception as error: # noqa: BLE001 # preview must return typed errors - record_metadata_read_failure(error, 500) + record_metadata_read_failure(error, 500, QUERY_FAILED_FAILURE) return v2_error_response(500, "V2 metadata query failed") finally: if reader is not None: @@ -465,20 +466,23 @@ def run_metadata_read( reader.close() -def _is_generic_unavailable_answer( +def _caller_message( error: V2ConfigurationError | MetadataCatalogUnavailableError, -) -> bool: - """Whether ``_catalog_unavailable_response`` answers with no message of its own. +) -> str | None: + """Return the message written for callers, or None for the generic 503. A pruned version, or a missing parameter tree or search file, is a known - state whose message names the catalog version; only the others hide why. + state whose message names the catalog version. Any other error answers the + generic 503, which records why (``read_failure_support``). """ - return not isinstance( + if isinstance( error, CatalogValuesPrunedError | ParameterTreeUnavailableError | ParameterSearchFileUnavailableError, - ) + ): + return str(error) + return None def _catalog_unavailable_response( @@ -496,12 +500,5 @@ def _catalog_unavailable_response( if isinstance(error, CatalogValuesPrunedError): status = 503 if error.running_version else 404 return v2_error_response(status, str(error), headers=NO_STORE_HEADERS) - message = ( - str(error) - if isinstance( - error, - ParameterTreeUnavailableError | ParameterSearchFileUnavailableError, - ) - else "V2 metadata catalog is unavailable" - ) + message = _caller_message(error) or "V2 metadata catalog is unavailable" return v2_error_response(503, message) From e20e0529804f11a88aaba90b8906443b65303036 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 01:55:36 +0400 Subject: [PATCH 13/18] Treat only a missing caller message as the generic 503 Co-Authored-By: Claude Opus 5.5 --- policyengine_api/fastapi_routes/v2/metadata/route_support.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/policyengine_api/fastapi_routes/v2/metadata/route_support.py b/policyengine_api/fastapi_routes/v2/metadata/route_support.py index 9c4368d5f..3efeecf20 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/route_support.py +++ b/policyengine_api/fastapi_routes/v2/metadata/route_support.py @@ -500,5 +500,7 @@ def _catalog_unavailable_response( if isinstance(error, CatalogValuesPrunedError): status = 503 if error.running_version else 404 return v2_error_response(status, str(error), headers=NO_STORE_HEADERS) - message = _caller_message(error) or "V2 metadata catalog is unavailable" + message = _caller_message(error) + if message is None: + message = "V2 metadata catalog is unavailable" return v2_error_response(503, message) From a70d2ac1d37a216cfdaad2ea90f366ab38f7da78 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 02:03:17 +0400 Subject: [PATCH 14/18] Test analyzing on every publication and recording a failure (failing) A failed ANALYZE after the catalog committed failed the command, and the retry, which changed nothing, analyzed nothing, leaving the statistics stale. The tests now expect every publication to analyze, a failed ANALYZE to be recorded by its class while the publication is kept, and the evidence to carry both. The catalog writer's pinned privileges and the evidence's pinned fields now include MAINTAIN and the new fields, and the manifest test checks the grant against the module's table list. Co-Authored-By: Claude Opus 5.5 --- .../integration/test_v2_catalog_statistics.py | 47 +++++++++++++++---- tests/unit/v2/test_catalog_publication.py | 4 ++ tests/unit/v2/test_privilege_manifest.py | 29 +++++------- 3 files changed, 56 insertions(+), 24 deletions(-) diff --git a/tests/integration/test_v2_catalog_statistics.py b/tests/integration/test_v2_catalog_statistics.py index 3eb1ddae2..cbeceb509 100644 --- a/tests/integration/test_v2_catalog_statistics.py +++ b/tests/integration/test_v2_catalog_statistics.py @@ -5,8 +5,9 @@ one release's rows stayed under that share: after 6.17.1 was published on 2026-11-10, the planner estimated 1 parameter row for US 6.17.1 instead of 102,530 and served ``GET /v2/parameter-values?country_id=us&limit=1`` in 43-47 -seconds. Publication now runs ``ANALYZE`` on the catalog tables after a run that -adds, restores or prunes a version. ``ANALYZE`` skips, with only a warning, a +seconds. Every publication now runs ``ANALYZE`` on the catalog tables once it +has committed and pruned; a failed ``ANALYZE`` is recorded in the evidence and +the next publication runs it again. ``ANALYZE`` skips, with only a warning, a table the role neither owns nor holds ``MAINTAIN`` on, so publication checks that privilege first and fails before writing anything without it. """ @@ -21,7 +22,8 @@ import pytest from alembic import command from alembic.config import Config -from sqlalchemy import create_engine, text +from psycopg import errors as psycopg_errors +from sqlalchemy import create_engine, event, text from sqlalchemy.engine import make_url from sqlalchemy.pool import NullPool @@ -132,18 +134,47 @@ def test_publishing_a_new_version_as_the_catalog_writer_analyzes_the_catalog( ), last_analyze -def test_republishing_an_unchanged_catalog_analyzes_nothing( +def test_republishing_an_unchanged_catalog_analyzes_it_again( superuser_engine, writer_engine ) -> None: + """Every run analyzes, so a run after a failed ``ANALYZE`` repeats it.""" catalog = normalized_catalog() publish_catalog(writer_engine, catalog) - after_first = _last_manual_analyze(superuser_engine) + second_started = _database_time(superuser_engine) second = publish_catalog(writer_engine, catalog) + last_analyze = _last_manual_analyze(superuser_engine) + + assert second.catalog_tables_analyzed == ANALYZED_TABLES + assert second.catalog_statistics_failure is None + assert all( + analyzed_at is not None and analyzed_at >= second_started + for analyzed_at in last_analyze.values() + ), last_analyze + + +def test_a_failed_analyze_is_recorded_and_the_publication_kept( + superuser_engine, writer_engine +) -> None: + def cancel_analyze(_connection, _cursor, statement, *_arguments) -> None: + if statement.startswith("ANALYZE"): + msg = "canceling statement due to statement timeout" + raise psycopg_errors.QueryCanceled(msg) + + event.listen(writer_engine, "before_cursor_execute", cancel_analyze) + + evidence = publish_catalog(writer_engine, normalized_catalog()) + with superuser_engine.connect() as connection: + stored_versions = connection.execute( + text("SELECT count(*) FROM tax_benefit_model_versions") + ).scalar_one() - assert second.catalog_tables_analyzed == () - assert second.as_dict()["catalog_tables_analyzed"] == [] - assert _last_manual_analyze(superuser_engine) == after_first + assert stored_versions > 0 + assert evidence.catalog_tables_analyzed == () + assert evidence.catalog_statistics_failure == "psycopg.errors.QueryCanceled" + assert evidence.as_dict()["catalog_statistics_failure"] == ( + "psycopg.errors.QueryCanceled" + ) @pytest.fixture diff --git a/tests/unit/v2/test_catalog_publication.py b/tests/unit/v2/test_catalog_publication.py index 338a6d97c..b537ca5cc 100644 --- a/tests/unit/v2/test_catalog_publication.py +++ b/tests/unit/v2/test_catalog_publication.py @@ -321,6 +321,8 @@ def test_completion_evidence_contains_only_reviewed_non_secret_fields() -> None: ), ), catalog_versions_left_to_prune=3, + catalog_tables_analyzed=("parameter_values", "parameters"), + catalog_statistics_failure="psycopg.errors.QueryCanceled", spm_availability=publication_types.SPMAvailabilityRecording( recorded=(("uk", "2.50.0", False),), not_recorded=(("us", "1.450.0"),), @@ -399,6 +401,8 @@ def test_completion_evidence_contains_only_reviewed_non_secret_fields() -> None: } ], "catalog_versions_left_to_prune": 3, + "catalog_tables_analyzed": ["parameter_values", "parameters"], + "catalog_statistics_failure": "psycopg.errors.QueryCanceled", "spm_availability": { "recorded": [{"country_id": "uk", "version": "2.50.0", "available": False}], "not_recorded": [{"country_id": "us", "version": "1.450.0"}], diff --git a/tests/unit/v2/test_privilege_manifest.py b/tests/unit/v2/test_privilege_manifest.py index a1b457037..0a20c74a3 100644 --- a/tests/unit/v2/test_privilege_manifest.py +++ b/tests/unit/v2/test_privilege_manifest.py @@ -11,6 +11,9 @@ import sqlalchemy as sa from policyengine_api.data.v2 import models +from policyengine_api.data.v2.catalog.publication_statistics import ( + CATALOG_STATISTICS_TABLES, +) from policyengine_api.data.v2.models import V2_METADATA from policyengine_api.data.v2.privilege_requirements import ( required_privileges, @@ -400,24 +403,12 @@ def test_the_catalog_writer_may_analyze_the_tables_publication_refreshes() -> No (PostgreSQL 17); without it, ANALYZE skips the table with only a warning. """ writer_tables = declared_privileges(TEST_ENVIRONMENT)[CATALOG_WRITER_ROLE] - analyzed = ( - "datasets", - "parameter_nodes", - "parameter_tree_children", - "parameter_values", - "parameters", - "regions", - "variables", - ) - assert { - table for table in analyzed if "MAINTAIN" not in writer_tables[table] - } == set() assert { table for table, privileges in writer_tables.items() - if "MAINTAIN" in privileges and table not in analyzed - } == set() + if TablePrivilege.MAINTAIN in privileges + } == set(CATALOG_STATISTICS_TABLES) def test_the_catalog_writer_deletes_only_the_rows_catalog_pruning_removes() -> None: @@ -437,13 +428,19 @@ def test_the_catalog_writer_deletes_only_the_rows_catalog_pruning_removes() -> N if TablePrivilege.DELETE in privileges } == {"parameter_values", "parameter_tree_children", "parameter_search_files"} assert TablePrivilege.UPDATE in catalog_writer["tax_benefit_model_versions"] + # MAINTAIN is for ANALYZE after publication, not for deleting. assert catalog_writer["parameter_tree_children"] == frozenset( - {TablePrivilege.SELECT, TablePrivilege.INSERT, TablePrivilege.DELETE} + { + TablePrivilege.SELECT, + TablePrivilege.INSERT, + TablePrivilege.DELETE, + TablePrivilege.MAINTAIN, + } ) assert catalog_writer["parameter_search_files"] == frozenset( {TablePrivilege.SELECT, TablePrivilege.INSERT, TablePrivilege.DELETE} ) - assert catalog_writer["parameter_values"] == READ_WRITE + assert catalog_writer["parameter_values"] == READ_WRITE | {TablePrivilege.MAINTAIN} def test_the_public_api_records_user_type_changes_but_never_rewrites_them() -> None: From 6a772aa8b68ea4f7398964d833961374c9d88985 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 02:04:37 +0400 Subject: [PATCH 15/18] Analyze the catalog on every publication and record a failed ANALYZE A failed ANALYZE after the catalog committed failed the command and was never repeated, because the retry changed nothing and so analyzed nothing. Every publication now analyzes the catalog tables once it has committed and pruned, which takes seconds and blocks no reads, and a failure is returned rather than raised: the evidence lists no tables and names the failure's class in catalog_statistics_failure, as a failed pruning is reported, and the next publication analyzes again. The privileges comment and the Stage 9 document now say that MAINTAIN also allows VACUUM, CLUSTER, REINDEX, REFRESH MATERIALIZED VIEW and LOCK TABLE, PostgreSQL 17 having no narrower privilege, and the evidence list names the two new fields. Co-Authored-By: Claude Opus 5.5 --- changelog.d/v2-catalog-analyze.fixed.md | 2 +- docs/migration/stage-9-v2-metadata.md | 31 +++++++---- .../data/v2/catalog/publication.py | 21 +++----- .../data/v2/catalog/publication_statistics.py | 54 ++++++++++++++----- .../data/v2/catalog/publication_types.py | 7 ++- policyengine_api/data/v2/privileges.py | 5 +- 6 files changed, 79 insertions(+), 41 deletions(-) diff --git a/changelog.d/v2-catalog-analyze.fixed.md b/changelog.d/v2-catalog-analyze.fixed.md index a62fb5f03..d7c29e75b 100644 --- a/changelog.d/v2-catalog-analyze.fixed.md +++ b/changelog.d/v2-catalog-analyze.fixed.md @@ -1 +1 @@ -Catalog publication now refreshes PostgreSQL's statistics of the catalog tables after it publishes, restores or prunes a version, so a new version's rows no longer leave `GET /v2/parameter-values` planned as if the version had one parameter (43 to 47 seconds on staging, 4.1 ms after `ANALYZE`). The catalog writer role gains `MAINTAIN` on those tables. +Catalog publication now refreshes PostgreSQL's statistics of the catalog tables after every publication, so a new version's rows no longer leave `GET /v2/parameter-values` planned as if the version had one parameter (43 to 47 seconds on staging, 4.1 ms after `ANALYZE`); a failed refresh is reported in the publication's summary and repeated by the next one. The catalog writer role gains `MAINTAIN` on those tables. diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 3f36ac39b..9139b5ae2 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -192,7 +192,7 @@ Use three separately managed database credentials: | Operation | Configuration | Required database access | | --- | --- | --- | | Alembic schema upgrade | `V2_MIGRATION_DATABASE_URL` | Reviewed v2 schema changes; no application-runtime use | -| Catalog publication | `V2_DATA_WRITE_DATABASE_URL` | Row insertion, update, selection, deletion from `parameter_values`, `parameter_tree_children` and `parameter_search_files` (catalog pruning), `ANALYZE` of the catalog tables (`MAINTAIN`), temporary tables, and transaction-scoped advisory locks; no persistent schema changes | +| Catalog publication | `V2_DATA_WRITE_DATABASE_URL` | Row insertion, update, selection, deletion from `parameter_values`, `parameter_tree_children` and `parameter_search_files` (catalog pruning), `ANALYZE` of the catalog tables (`MAINTAIN`, which also allows `VACUUM`, `CLUSTER`, `REINDEX` and `LOCK TABLE` on them), temporary tables, and transaction-scoped advisory locks; no persistent schema changes | | Preview GET requests | `V2_RUNTIME_DATABASE_URL` or `V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE` | Catalog selection only during Stage 9; no migration access | Do not place more than one database URL in the environment of either the @@ -639,16 +639,22 @@ that version, and `GET /v2/parameter-values?country_id=us&limit=1` took 43 to parameters` it walks `uq_parameters_model_version_name` and takes 4.1 ms (UK: 4.9 ms). -A publication that publishes, restores or prunes a version therefore runs one -`ANALYZE` over `datasets`, `parameter_nodes`, `parameter_tree_children`, -`parameter_values`, `parameters`, `regions` and `variables` once it has -committed and pruned, and lists them in the evidence's -`catalog_tables_analyzed`; a run that changed none lists none. `ANALYZE` needs -the table's owner or the `MAINTAIN` privilege (PostgreSQL 17), and PostgreSQL -skips any other table with only a warning. The migration role owns the tables, -so the manifest grants the catalog writer `MAINTAIN` on these seven, the -deploy's `apply` step grants it before publication runs, and publication -fails before writing anything if its role lacks it. The code is +Every publication therefore runs one `ANALYZE` over `datasets`, +`parameter_nodes`, `parameter_tree_children`, `parameter_values`, +`parameters`, `regions` and `variables` once it has committed and pruned, and +lists them in the evidence's `catalog_tables_analyzed`. It takes a few seconds +and blocks no reads. A failed `ANALYZE` doesn't undo the publication: the +evidence lists no tables and names the failure's class in +`catalog_statistics_failure`, and the next publication analyzes again. + +`ANALYZE` needs the table's owner or the `MAINTAIN` privilege (PostgreSQL 17), +and PostgreSQL skips any other table with only a warning. The migration role +owns the tables, so the manifest grants the catalog writer `MAINTAIN` on these +seven, the deploy's `apply` step grants it before publication runs, and +publication fails before writing anything if its role lacks it. PostgreSQL 17 +has no narrower privilege: `MAINTAIN` also allows `VACUUM` (`FULL` included), +`CLUSTER`, `REINDEX`, `REFRESH MATERIALIZED VIEW` and `LOCK TABLE` on those +tables, though publication runs only `ANALYZE`. The code is `policyengine_api/data/v2/catalog/publication_statistics.py`. ## Validation evidence and retry @@ -675,6 +681,9 @@ Successful JSON evidence contains: (`catalog_pruning_failures`; see "Catalog pruning"); - the number of chosen versions left for later publications (`catalog_versions_left_to_prune`); +- the catalog tables analyzed after the publication + (`catalog_tables_analyzed`), and the class of a failed `ANALYZE` + (`catalog_statistics_failure`; see "Catalog table statistics"); - elapsed publication time; and - a success outcome. diff --git a/policyengine_api/data/v2/catalog/publication.py b/policyengine_api/data/v2/catalog/publication.py index f55ed9eb5..75c51c6f4 100644 --- a/policyengine_api/data/v2/catalog/publication.py +++ b/policyengine_api/data/v2/catalog/publication.py @@ -167,10 +167,11 @@ def publish_catalog( After the transaction commits, it prunes the older stored versions of the catalog's models (``publication_pruning``), at most two per run, one transaction each; a failure of any kind rolls back only that version and - is reported in the evidence. Then, if the run published, restored or - pruned a version, it analyzes the catalog tables, whose planner statistics - PostgreSQL would otherwise leave out of date (``publication_statistics``). - Before it writes anything it checks that its role may analyze them. + is reported in the evidence. Then it analyzes the catalog tables, whose + planner statistics PostgreSQL would otherwise leave out of date + (``publication_statistics``); a failure is reported in the evidence too, + and the next publication analyzes again. Before it writes anything it + checks that its role may analyze them. """ started = time.monotonic() with engine.begin() as connection: @@ -216,14 +217,7 @@ def publish_catalog( checkpoint("after_validation", connection) pruning = _prune_older_versions(engine, catalog) - published_new_version = any( - country.country_id not in existing for country in catalog.countries - ) - tables_analyzed = ( - analyze_catalog_tables(engine) - if published_new_version or restored_versions or pruning.pruned - else () - ) + statistics = analyze_catalog_tables(engine) return replace( _evidence( catalog, @@ -236,7 +230,8 @@ def publish_catalog( pruned_catalog_versions=pruning.pruned, catalog_pruning_failures=pruning.failures, catalog_versions_left_to_prune=pruning.left_to_prune, - catalog_tables_analyzed=tables_analyzed, + catalog_tables_analyzed=statistics.analyzed, + catalog_statistics_failure=statistics.failure_type, ) diff --git a/policyengine_api/data/v2/catalog/publication_statistics.py b/policyengine_api/data/v2/catalog/publication_statistics.py index 83fbd1817..ab9fb871d 100644 --- a/policyengine_api/data/v2/catalog/publication_statistics.py +++ b/policyengine_api/data/v2/catalog/publication_statistics.py @@ -8,18 +8,23 @@ ``GET /v2/parameter-values?country_id=us&limit=1`` took 43 to 47 seconds until ``ANALYZE parameters`` brought it to 4.1 ms. -So publication runs one ``ANALYZE`` over the catalog tables it writes after a -run that publishes, restores or prunes a catalog version -(``analyze_catalog_tables``). ``ANALYZE`` needs the table's owner or the -``MAINTAIN`` privilege, and PostgreSQL skips any other table with only a -warning. The migration role owns every table, so the manifest declares -``MAINTAIN`` for the catalog writer on these tables, and publication checks -that its role holds it before it writes anything -(``require_analyze_privilege``). +So every publication runs one ``ANALYZE`` over the catalog tables it writes, +once it has committed and pruned (``analyze_catalog_tables``). It takes a few +seconds and blocks no reads. Running it every time, rather than only after a +run that changed a version, means a failed ``ANALYZE``, which is recorded in +the evidence by its error class like a failed pruning, is repeated by the +next publication instead of leaving the statistics stale. + +``ANALYZE`` needs the table's owner or the ``MAINTAIN`` privilege, and +PostgreSQL skips any other table with only a warning. The migration role owns +every table, so the manifest declares ``MAINTAIN`` for the catalog writer on +these tables, and publication checks that its role holds it before it writes +anything (``require_analyze_privilege``). """ from __future__ import annotations +from dataclasses import dataclass from typing import TYPE_CHECKING, Final import sqlalchemy as sa @@ -70,9 +75,32 @@ def require_analyze_privilege(connection: Connection) -> None: raise CatalogPublicationError(msg) -def analyze_catalog_tables(engine: Engine) -> tuple[str, ...]: - """Run one ``ANALYZE`` over the catalog tables and return their names.""" +@dataclass(frozen=True, slots=True) +class CatalogStatisticsRefresh: + """What the ``ANALYZE`` after a publication did. + + ``analyzed`` names the tables, empty when it failed; ``failure_type`` is the + failure's class as ``module.QualifiedName``, never its message. + """ + + analyzed: tuple[str, ...] + failure_type: str | None = None + + +def analyze_catalog_tables(engine: Engine) -> CatalogStatisticsRefresh: + """Run one ``ANALYZE`` over the catalog tables; a failure is returned, not raised. + + It runs after the publication committed, so a failure, of any kind, never + undoes it; the next publication analyzes again. + """ tables = ", ".join(f"public.{table}" for table in CATALOG_STATISTICS_TABLES) - with engine.begin() as connection: - connection.execute(sa.text(f"ANALYZE {tables}")) - return CATALOG_STATISTICS_TABLES + try: + with engine.begin() as connection: + connection.execute(sa.text(f"ANALYZE {tables}")) + except Exception as error: # noqa: BLE001 # the publication has committed; the next one analyzes again + error_class = type(error) + return CatalogStatisticsRefresh( + analyzed=(), + failure_type=f"{error_class.__module__}.{error_class.__qualname__}", + ) + return CatalogStatisticsRefresh(analyzed=CATALOG_STATISTICS_TABLES) diff --git a/policyengine_api/data/v2/catalog/publication_types.py b/policyengine_api/data/v2/catalog/publication_types.py index ea998428b..6bde1ca71 100644 --- a/policyengine_api/data/v2/catalog/publication_types.py +++ b/policyengine_api/data/v2/catalog/publication_types.py @@ -110,9 +110,11 @@ class PublicationEvidence: catalog_pruning_failures: tuple[CatalogPruningFailure, ...] = () # Chosen versions beyond the run's limit, which the next publications prune. catalog_versions_left_to_prune: int = 0 - # The catalog tables analyzed after a run that published, restored or - # pruned a version (``publication_statistics``); empty when it changed none. + # The catalog tables analyzed once the publication committed and pruned + # (``publication_statistics``), empty when ANALYZE failed; then + # ``catalog_statistics_failure`` names the failure's class, never its message. catalog_tables_analyzed: tuple[str, ...] = () + catalog_statistics_failure: str | None = None # The SPM availability recorded after the publication committed; None # until initialization records it. spm_availability: SPMAvailabilityRecording | None = None @@ -206,6 +208,7 @@ def as_dict(self) -> dict[str, object]: ], "catalog_versions_left_to_prune": self.catalog_versions_left_to_prune, "catalog_tables_analyzed": list(self.catalog_tables_analyzed), + "catalog_statistics_failure": self.catalog_statistics_failure, "spm_availability": ( None if self.spm_availability is None diff --git a/policyengine_api/data/v2/privileges.py b/policyengine_api/data/v2/privileges.py index 5b841b5b2..1445d234e 100644 --- a/policyengine_api/data/v2/privileges.py +++ b/policyengine_api/data/v2/privileges.py @@ -138,7 +138,10 @@ def update_columns(*columns: str) -> ColumnPrivilege: ) # ANALYZE after publication (``catalog.publication_statistics``): the migration # role owns the catalog tables, and PostgreSQL skips a table the role neither -# owns nor holds MAINTAIN on, with only a warning. +# owns nor holds MAINTAIN on, with only a warning. PostgreSQL 17 has no narrower +# privilege: MAINTAIN also allows VACUUM (FULL included), CLUSTER, REINDEX, +# REFRESH MATERIALIZED VIEW and LOCK TABLE on them, of which publication runs +# only ANALYZE. _ANALYZE: Final = frozenset({TablePrivilege.MAINTAIN}) From 50406c7d7ba53c590e169316feb0b057d0f79374 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 02:08:14 +0400 Subject: [PATCH 16/18] Test reporting a failed ANALYZE by its driver error and on stderr (failing) Co-Authored-By: Claude Opus 5.5 --- .../integration/test_v2_catalog_statistics.py | 26 ++++++++++------ tests/unit/v2/test_catalog_initialization.py | 30 +++++++++++++++++++ 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/tests/integration/test_v2_catalog_statistics.py b/tests/integration/test_v2_catalog_statistics.py index cbeceb509..cb60d91bd 100644 --- a/tests/integration/test_v2_catalog_statistics.py +++ b/tests/integration/test_v2_catalog_statistics.py @@ -22,7 +22,6 @@ import pytest from alembic import command from alembic.config import Config -from psycopg import errors as psycopg_errors from sqlalchemy import create_engine, event, text from sqlalchemy.engine import make_url from sqlalchemy.pool import NullPool @@ -49,6 +48,7 @@ REPO = Path(__file__).parents[2] DISPOSABLE_DATABASE = "policyengine_v2_alembic_test" +UNDEFINED_TABLE_FAILURE = "psycopg.errors.UndefinedTable (SQLSTATE 42P01)" ANALYZED_TABLES = ( "datasets", "parameter_nodes", @@ -156,12 +156,22 @@ def test_republishing_an_unchanged_catalog_analyzes_it_again( def test_a_failed_analyze_is_recorded_and_the_publication_kept( superuser_engine, writer_engine ) -> None: - def cancel_analyze(_connection, _cursor, statement, *_arguments) -> None: + """The database refuses the ``ANALYZE``, and SQLAlchemy wraps its error. + + The evidence names the driver's error and SQLSTATE, as a failed pruning's + does, not SQLAlchemy's wrapper, which is the same for any database error. + """ + + def analyze_a_missing_table( + _connection, _cursor, statement, parameters, *_arguments + ): if statement.startswith("ANALYZE"): - msg = "canceling statement due to statement timeout" - raise psycopg_errors.QueryCanceled(msg) + return "ANALYZE public.no_such_catalog_table", parameters + return statement, parameters - event.listen(writer_engine, "before_cursor_execute", cancel_analyze) + event.listen( + writer_engine, "before_cursor_execute", analyze_a_missing_table, retval=True + ) evidence = publish_catalog(writer_engine, normalized_catalog()) with superuser_engine.connect() as connection: @@ -171,10 +181,8 @@ def cancel_analyze(_connection, _cursor, statement, *_arguments) -> None: assert stored_versions > 0 assert evidence.catalog_tables_analyzed == () - assert evidence.catalog_statistics_failure == "psycopg.errors.QueryCanceled" - assert evidence.as_dict()["catalog_statistics_failure"] == ( - "psycopg.errors.QueryCanceled" - ) + assert evidence.catalog_statistics_failure == UNDEFINED_TABLE_FAILURE + assert evidence.as_dict()["catalog_statistics_failure"] == UNDEFINED_TABLE_FAILURE @pytest.fixture diff --git a/tests/unit/v2/test_catalog_initialization.py b/tests/unit/v2/test_catalog_initialization.py index 7543dfc75..ad5bc4f95 100644 --- a/tests/unit/v2/test_catalog_initialization.py +++ b/tests/unit/v2/test_catalog_initialization.py @@ -315,6 +315,36 @@ def test_main_succeeds_and_prints_each_pruning_failure_to_stderr( ] +def test_main_succeeds_and_prints_a_failed_analyze_to_stderr( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """A failed ANALYZE leaves the committed publication's command successful. + + The failure is in the evidence on stdout and on one stderr line, which says + the next publication analyzes again. + """ + failure = "psycopg.errors.QueryCanceled (SQLSTATE 57014)" + evidence = PublicationEvidence( + policyengine_version=POLICYENGINE_VERSION, + dependency_versions=(), + entity_counts={"models": 2}, + fallback_summaries=(), + elapsed_seconds=1.0, + catalog_statistics_failure=failure, + ) + monkeypatch.setattr(initialization, "initialize_catalog", lambda: evidence) + + assert initialization.main() == 0 + + output = capsys.readouterr() + assert json.loads(output.out)["catalog_statistics_failure"] == failure + assert output.err.splitlines() == [ + f"catalog table statistics were not refreshed: {failure}; the next " + "publication analyzes the catalog tables again" + ] + + def test_main_prints_a_pruning_failure_that_chose_no_version_to_stderr( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], From 21ce1d1aed63583d9cffb9ba3cc245aed53ab390 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 02:08:56 +0400 Subject: [PATCH 17/18] Report a failed ANALYZE by its driver error, and on stderr catalog_statistics_failure now names the class of the error the database driver raised and its SQLSTATE, as a failed pruning is reported, since SQLAlchemy's own class is the same for every database error, and the command prints the failure to stderr while still succeeding. Two descriptions still said the writer analyzes only the tables publication grows; it analyzes seven tables after every publication. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/skills/alembic-migrations.md | 2 +- docs/migration/stage-9-v2-metadata.md | 6 ++-- .../data/v2/catalog/initialization.py | 15 +++++++--- .../data/v2/catalog/publication_statistics.py | 29 ++++++++++++++----- policyengine_api/data/v2/privileges.py | 2 +- 5 files changed, 39 insertions(+), 15 deletions(-) diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index 046344556..157f8aecf 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -253,7 +253,7 @@ and a revision runs once per database, so it can't repair drift. | Role | Used by | | --- | --- | | `policyengine_v2_runtime` | The Public API on Cloud Run | -| `policyengine_v2_catalog_writer` | `scripts/publish_v2_metadata_catalog.py`, which also runs `ANALYZE` on the catalog tables it grows, with `MAINTAIN` (PostgreSQL 17) | +| `policyengine_v2_catalog_writer` | `scripts/publish_v2_metadata_catalog.py`, which also runs `ANALYZE` on seven catalog tables after every publication, with `MAINTAIN` (PostgreSQL 17) | | `policyengine_stage12_` | The simulation API and its Modal worker, on the temporary comparison-run tables | | `policyengine_v2_simulation_service` | The simulation API on Cloud Run: creates simulations, reports, their runs, budget-window runs and household jobs, and reads the policy, dynamic, household and catalog rows a run needs. It updates only the status, Modal call ID, error and completion columns of the runs and household jobs whose Modal calls it starts, and the status, error and completion columns of a budget-window run, which it fails at once when Modal refuses the calls of the window's first years | | `policyengine_v2_modal_worker` | The simulation API's Modal workers: write their runs' and household jobs' status and results, and budget-window runs' status, and read the rows a run needs. It updates only a run's status, Modal call ID, error, time and result columns, and a budget-window run's status, error and time columns: each year's coordinator starts the window's next unstarted year when its own year finishes, the last year to finish marks the window succeeded, and a failed year fails it. The workers' code, not the database, keeps a worker to its own run | diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 9139b5ae2..96d410aea 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -644,8 +644,10 @@ Every publication therefore runs one `ANALYZE` over `datasets`, `parameters`, `regions` and `variables` once it has committed and pruned, and lists them in the evidence's `catalog_tables_analyzed`. It takes a few seconds and blocks no reads. A failed `ANALYZE` doesn't undo the publication: the -evidence lists no tables and names the failure's class in -`catalog_statistics_failure`, and the next publication analyzes again. +evidence lists no tables and names the failure in `catalog_statistics_failure` +(the database driver's error class and SQLSTATE, never its message), the +command prints it to stderr and still succeeds, and the next publication +analyzes again. `ANALYZE` needs the table's owner or the `MAINTAIN` privilege (PostgreSQL 17), and PostgreSQL skips any other table with only a warning. The migration role diff --git a/policyengine_api/data/v2/catalog/initialization.py b/policyengine_api/data/v2/catalog/initialization.py index a73fcc0a7..3f357abdd 100644 --- a/policyengine_api/data/v2/catalog/initialization.py +++ b/policyengine_api/data/v2/catalog/initialization.py @@ -177,10 +177,10 @@ def _error_payload(error: Exception) -> dict[str, object]: def main() -> int: """Run explicit initialization and return a shell-compatible status. - A failure to prune an older catalog version, or to read the simulation - service's versions to record SPM availability, leaves the status - successful, since the publication had committed; each one is in the - evidence and is also printed to stderr. + A failure to prune an older catalog version, to analyze the catalog + tables, or to read the simulation service's versions to record SPM + availability, leaves the status successful, since the publication had + committed; each one is in the evidence and is also printed to stderr. """ try: evidence = initialize_catalog() @@ -199,6 +199,13 @@ def main() -> int: f"{failure.error_type}: {failure.message}", file=sys.stderr, ) + if evidence.catalog_statistics_failure is not None: + print( + "catalog table statistics were not refreshed: " + f"{evidence.catalog_statistics_failure}; the next publication " + "analyzes the catalog tables again", + file=sys.stderr, + ) spm_recording = evidence.spm_availability if ( spm_recording is not None diff --git a/policyengine_api/data/v2/catalog/publication_statistics.py b/policyengine_api/data/v2/catalog/publication_statistics.py index ab9fb871d..a13ad89b7 100644 --- a/policyengine_api/data/v2/catalog/publication_statistics.py +++ b/policyengine_api/data/v2/catalog/publication_statistics.py @@ -28,6 +28,7 @@ from typing import TYPE_CHECKING, Final import sqlalchemy as sa +from sqlalchemy.exc import SQLAlchemyError from policyengine_api.data.v2.catalog.publication_types import ( CatalogPublicationError, @@ -79,8 +80,10 @@ def require_analyze_privilege(connection: Connection) -> None: class CatalogStatisticsRefresh: """What the ``ANALYZE`` after a publication did. - ``analyzed`` names the tables, empty when it failed; ``failure_type`` is the - failure's class as ``module.QualifiedName``, never its message. + ``analyzed`` names the tables, empty when it failed. ``failure_type`` then + names the failure without its message: for a database error, the class of + the error the driver raised and its SQLSTATE, as a failed pruning reports + it, since SQLAlchemy's own class is the same for every database error. """ analyzed: tuple[str, ...] @@ -98,9 +101,21 @@ def analyze_catalog_tables(engine: Engine) -> CatalogStatisticsRefresh: with engine.begin() as connection: connection.execute(sa.text(f"ANALYZE {tables}")) except Exception as error: # noqa: BLE001 # the publication has committed; the next one analyzes again - error_class = type(error) - return CatalogStatisticsRefresh( - analyzed=(), - failure_type=f"{error_class.__module__}.{error_class.__qualname__}", - ) + return CatalogStatisticsRefresh(analyzed=(), failure_type=_failure(error)) return CatalogStatisticsRefresh(analyzed=CATALOG_STATISTICS_TABLES) + + +def _failure(error: Exception) -> str: + """Name a failure by its class and a database error's SQLSTATE, not its message.""" + driver_error = getattr(error, "orig", None) + reported = ( + driver_error + if isinstance(error, SQLAlchemyError) and isinstance(driver_error, Exception) + else error + ) + reported_class = type(reported) + name = f"{reported_class.__module__}.{reported_class.__qualname__}" + sqlstate = getattr(reported, "sqlstate", None) + if isinstance(sqlstate, str) and sqlstate: + return f"{name} (SQLSTATE {sqlstate})" + return name diff --git a/policyengine_api/data/v2/privileges.py b/policyengine_api/data/v2/privileges.py index 1445d234e..ba116de22 100644 --- a/policyengine_api/data/v2/privileges.py +++ b/policyengine_api/data/v2/privileges.py @@ -282,7 +282,7 @@ def _run_table( # The UK areas disclosure control withheld from a local-area run's results. "withheld_area_codes": _RESULT, # Metadata catalog: the catalog writer publishes it, the Public API reads it. - # The writer also analyzes the tables publication grows (_ANALYZE). + # The writer also analyzes seven of them after every publication (_ANALYZE). "dataset_versions": _access(catalog_writer=READ), "datasets": _access( runtime=READ, catalog_writer=READ_INSERT_UPDATE | _ANALYZE, **_RUN_INPUT_READS From e3f7cb2e26768bc3ffecab898b4ca6f769cad9e0 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 02:18:02 +0400 Subject: [PATCH 18/18] Cover the two remaining branches of the failure reporting Codecov reported changed lines no test ran. Locally, two of them were: an ANALYZE failure that isn't a database error, named by its class alone, and a cause chain that loops back to an error partway along. Co-Authored-By: Claude Opus 5.5 --- .../v2/test_metadata_read_failure_events.py | 13 +++++++++ tests/unit/v2/test_publication_statistics.py | 27 +++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 tests/unit/v2/test_publication_statistics.py diff --git a/tests/unit/v2/test_metadata_read_failure_events.py b/tests/unit/v2/test_metadata_read_failure_events.py index 3ef9478e9..5d2983625 100644 --- a/tests/unit/v2/test_metadata_read_failure_events.py +++ b/tests/unit/v2/test_metadata_read_failure_events.py @@ -289,6 +289,19 @@ def test_a_cause_chain_that_loops_back_names_each_error_once() -> None: assert attributes["error.cause_types"] == "builtins.ValueError" +def test_a_cause_chain_that_loops_back_partway_names_each_error_once() -> None: + error = RuntimeError("outermost") + middle = ValueError("middle") + inner = KeyError("inner") + error.__cause__ = middle + middle.__cause__ = inner + inner.__cause__ = middle + + attributes = read_failure_attributes(error, 500, "v2_metadata_query_failed") + + assert attributes["error.cause_types"] == "builtins.ValueError, builtins.KeyError" + + def test_a_long_cause_chain_names_its_first_errors_only() -> None: error = RuntimeError("outermost") current = error diff --git a/tests/unit/v2/test_publication_statistics.py b/tests/unit/v2/test_publication_statistics.py new file mode 100644 index 000000000..5f8601ba3 --- /dev/null +++ b/tests/unit/v2/test_publication_statistics.py @@ -0,0 +1,27 @@ +"""How the ANALYZE after a catalog publication reports a failure. + +The PostgreSQL tests (``tests/integration/test_v2_catalog_statistics.py``) cover +a database error, which is named by the driver's class and SQLSTATE. Any other +failure is named by its class alone, never its message. +""" + +from __future__ import annotations + +from policyengine_api.data.v2.catalog.publication_statistics import ( + analyze_catalog_tables, +) + + +class EngineThatCannotConnect: + """Stands in for an engine whose connection fails before any statement.""" + + def begin(self) -> None: + msg = "postgresql://writer:hunter2@db.example.internal refused the connection" + raise RuntimeError(msg) + + +def test_a_failure_that_is_not_a_database_error_is_named_by_its_class() -> None: + refresh = analyze_catalog_tables(EngineThatCannotConnect()) + + assert refresh.analyzed == () + assert refresh.failure_type == "builtins.RuntimeError"