diff --git a/.github/workflows/v2-integration-check.yml b/.github/workflows/v2-integration-check.yml index c21bf595c..ece1ea3e1 100644 --- a/.github/workflows/v2-integration-check.yml +++ b/.github/workflows/v2-integration-check.yml @@ -74,7 +74,7 @@ jobs: - name: Verify the managed v2 roles hold their declared privileges run: uv run --frozen --extra dev python scripts/reconcile_v2_privileges.py verify - name: Test v2 metadata publication, resource routes, the countries list, reports and simulations - run: uv run --frozen --extra dev coverage run --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_catalog_publication_time_periods.py tests/integration/test_v2_catalog_publication_pruning.py tests/integration/test_v2_catalog_spm_availability.py tests/integration/test_v2_metadata_routes.py tests/integration/test_v2_country_routes.py tests/integration/test_v2_report_reads.py tests/integration/test_v2_report_creation.py tests/integration/test_v2_budget_window_report_reads.py tests/integration/test_v2_budget_window_report_creation.py tests/integration/test_v2_report_runs.py + run: uv run --frozen --extra dev coverage run --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_catalog_publication_time_periods.py tests/integration/test_v2_catalog_publication_pruning.py tests/integration/test_v2_catalog_spm_availability.py tests/integration/test_v2_catalog_statistics.py tests/integration/test_v2_metadata_routes.py tests/integration/test_v2_country_routes.py tests/integration/test_v2_report_reads.py tests/integration/test_v2_report_creation.py tests/integration/test_v2_budget_window_report_reads.py tests/integration/test_v2_budget_window_report_creation.py tests/integration/test_v2_report_runs.py - name: Test v2 policy persistence and immediate v1 mirroring run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_policy_persistence.py tests/integration/test_v1_policy_dual_write.py tests/integration/test_mysql_policy_dual_write.py tests/integration/test_v2_user_policy_mirroring.py tests/integration/test_v1_user_policy_dual_write.py - name: Test signed-in identity provisioning, user associations, saved detailed-save records and user types diff --git a/changelog.d/update-policyengine-bundle-6.20.0.changed.md b/changelog.d/update-policyengine-bundle-6.20.0.changed.md new file mode 100644 index 000000000..a2a4c53aa --- /dev/null +++ b/changelog.d/update-policyengine-bundle-6.20.0.changed.md @@ -0,0 +1 @@ +Update the PolicyEngine bundle to 6.20.0. diff --git a/changelog.d/v2-catalog-analyze.fixed.md b/changelog.d/v2-catalog-analyze.fixed.md new file mode 100644 index 000000000..d7c29e75b --- /dev/null +++ b/changelog.d/v2-catalog-analyze.fixed.md @@ -0,0 +1 @@ +Catalog publication now refreshes PostgreSQL's statistics of the catalog tables after every publication, so a new version's rows no longer leave `GET /v2/parameter-values` planned as if the version had one parameter (43 to 47 seconds on staging, 4.1 ms after `ANALYZE`); a failed refresh is reported in the publication's summary and repeated by the next one. The catalog writer role gains `MAINTAIN` on those tables. diff --git a/changelog.d/v2-column-update-grants.fixed.md b/changelog.d/v2-column-update-grants.fixed.md new file mode 100644 index 000000000..6c030449a --- /dev/null +++ b/changelog.d/v2-column-update-grants.fixed.md @@ -0,0 +1 @@ +The v2 privilege reconciliation now revokes a privilege a role holds on a whole table when the manifest declares it for named columns only, such as the `UPDATE` new tables' default privileges give the runtime role on `user_type_changes`, and grants those columns again, so the deploy's privilege verification passes. diff --git a/changelog.d/v2-metadata-read-failure-event.changed.md b/changelog.d/v2-metadata-read-failure-event.changed.md new file mode 100644 index 000000000..24b41a9e2 --- /dev/null +++ b/changelog.d/v2-metadata-read-failure-event.changed.md @@ -0,0 +1 @@ +A v2 metadata read that answers the generic 503 or the 500 now records one runtime event, `v2_metadata_read_failed`, naming the route, the error's class and failure category, the error classes in its cause chain, the database's SQLSTATE and whether the connection was lost, without any error message. diff --git a/docker/Dockerfile b/docker/Dockerfile index 131fdc900..0152908eb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ FROM python:3.12 # Match the API bundle in pyproject.toml; the bundle updater changes both pins. # The models extra pins the complete certified runtime package set. -RUN pip install "policyengine[models]==6.17.1" ipython +RUN pip install "policyengine[models]==6.20.0" ipython diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index bf5f9dc27..157f8aecf 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -253,7 +253,7 @@ and a revision runs once per database, so it can't repair drift. | Role | Used by | | --- | --- | | `policyengine_v2_runtime` | The Public API on Cloud Run | -| `policyengine_v2_catalog_writer` | `scripts/publish_v2_metadata_catalog.py` | +| `policyengine_v2_catalog_writer` | `scripts/publish_v2_metadata_catalog.py`, which also runs `ANALYZE` on seven catalog tables after every publication, with `MAINTAIN` (PostgreSQL 17) | | `policyengine_stage12_` | The simulation API and its Modal worker, on the temporary comparison-run tables | | `policyengine_v2_simulation_service` | The simulation API on Cloud Run: creates simulations, reports, their runs, budget-window runs and household jobs, and reads the policy, dynamic, household and catalog rows a run needs. It updates only the status, Modal call ID, error and completion columns of the runs and household jobs whose Modal calls it starts, and the status, error and completion columns of a budget-window run, which it fails at once when Modal refuses the calls of the window's first years | | `policyengine_v2_modal_worker` | The simulation API's Modal workers: write their runs' and household jobs' status and results, and budget-window runs' status, and read the rows a run needs. It updates only a run's status, Modal call ID, error, time and result columns, and a budget-window run's status, error and time columns: each year's coordinator starts the window's next unstarted year when its own year finishes, the last year to finish marks the window succeeded, and a failed year fails it. The workers' code, not the database, keeps a worker to its own run | @@ -283,14 +283,23 @@ check the declarations: then `verify`, before the catalog is seeded. `apply` grants each declared privilege a role lacks, as one `GRANT` per role -and table and one per column privilege. It never revokes: it reports -privileges the file doesn't declare, on whole tables or on columns, and the -projects' default privileges, in the job summary. It grants nothing when a -declared role or table is missing. `verify` checks each column of a column -privilege with `has_column_privilege`, and fails when a role holds the same -privilege on the whole table. The roles themselves are provisioned outside -the repository; `--create-missing-roles` is accepted only for the disposable -test database. +and table and one per column privilege. It revokes one kind of privilege +only: one a role holds directly on a whole table that the file declares for +named columns only. The projects' default privileges give the runtime role +`UPDATE` on every new table, so `user_type_changes`, where it may update only +`dismissed_at`, starts with one. PostgreSQL's `REVOKE` on a table also removes +the role's column grants of that privilege, undeclared ones included, so +`apply` grants the declared columns again, in the same transaction; the job +summary lists the revoke with the undeclared column grants it removed. It +reports every other privilege the file doesn't declare, on whole tables or on +columns, and the projects' default privileges, without revoking them. It +changes nothing when a declared role or table is missing. `verify` checks each +column of a column privilege with `has_column_privilege`, and fails when a role +still holds the same privilege on the whole table: through a role it belongs +to, or through a grant another role made, which the owner's `REVOKE` leaves in +place. The roles themselves are provisioned outside the repository; +`--create-missing-roles` is accepted only for the disposable test database, +which has no default privileges. A pull request may declare a role before an operator creates it by listing it in `ROLES_AWAITING_PROVISIONING` in `privileges.py`. While a listed role is diff --git a/docs/engineering/skills/observability.md b/docs/engineering/skills/observability.md index 1ce250ede..423b09c5e 100644 --- a/docs/engineering/skills/observability.md +++ b/docs/engineering/skills/observability.md @@ -84,6 +84,27 @@ own level is dropped before the runtime's handler sees it. Attribute names must avoid the runtime's prohibited key parts (such as `authorization`, `credential` and `token`), which it drops. +## v2 metadata read failures + +A v2 metadata read that answers the generic 503 "V2 metadata catalog is +unavailable", or the 500 "V2 metadata query failed", records one runtime +event at ERROR, `v2_metadata_read_failed` +(`policyengine_api.fastapi_routes.v2.metadata.read_failure_support`). The +runtime adds the request's ID, method and matched route. The event adds +`http.response.status_code`, `error_type` (the caught error's class as +`module.QualifiedName`), `failure_category` (`v2_metadata_catalog_unavailable` +or `v2_metadata_query_failed`), `error.cause_types` (the classes of its +`__cause__` chain, outermost first), `database.sqlstate` (the first SQLSTATE in +the chain) and `database.connection_invalidated` (SQLAlchemy's lost-connection +flag). The scheduled error-issue job groups the event by route, `error_type` +and `failure_category`. A pruned catalog version, or a missing parameter tree +or search file, answers with its own message and records nothing. On the two +routes that share one read of a whole catalog version +(`response_cache_support`), only the request that ran the failed read records +it; requests that waited for it answer with a copy of its 503. The event never +carries an error's message: SQLAlchemy's and the driver's name the SQL +statement and its parameters, and can name hosts, addresses and credentials. + ## Simulation client transport The simulation HTTP client reads identifiers from request context. Its request diff --git a/docs/engineering/skills/v2-code-organization.md b/docs/engineering/skills/v2-code-organization.md index c9cf4d0dc..223560d0a 100644 --- a/docs/engineering/skills/v2-code-organization.md +++ b/docs/engineering/skills/v2-code-organization.md @@ -72,6 +72,7 @@ user_policies/ metadata/ response_models.py route_support.py + read_failure_support.py query_support.py response_cache_support.py search_file_support.py @@ -80,7 +81,9 @@ metadata/ `route_support.py` runs the metadata reads for the route modules, maps their failures to HTTP, and holds the route class that gives metadata responses -their caching headers. `query_support.py` defines the typed FastAPI +their caching headers. `read_failure_support.py` records why a read answered +the generic 503 or a 500, without the error's message. `query_support.py` +defines the typed FastAPI dependencies for the shared query models once, for every metadata route module to import. `response_cache_support.py` keeps serialized whole-version responses (bodies and ETags) for the process; it is HTTP-level, so it lives diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 79b3012d4..3b17654f4 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -285,7 +285,7 @@ flowchart LR | `api.migration.bundle-supported-by-simulation-api` | The installed PolicyEngine bundle is the latest one the simulation API has published, checked on every pull request and before each staging and production deploy. | mechanical | enforced | yes | CI job `ensure-policyengine-bundle-supported-by-simulation-api` | `pr.yml:ensure-policyengine-bundle-supported-by-simulation-api` | — | | `api.migration.contracts-consistent` | Migration route groups and workflow contracts are internally consistent, and their generated docs are current. | mechanical | enforced | yes | quality guard `migration-contracts` | `pr.yml:quality-guards` | **unproven** | | `api.migration.cross-database-integration` | Cross-database integration tests pass against real MySQL and Postgres. | mechanical | enforced | yes | CI job `v2-integration-check` | `pr.yml:v2-integration-check` | — | -| `api.v2.deployed-privileges-match` | Every v2 deploy grants the managed roles each privilege policyengine_api/data/v2/privileges.py declares, on the whole table or on the columns it names, right after the schema upgrade, reports privileges it doesn't declare without revoking them, and stops before the catalog is seeded if a declared privilege is still missing or a role holds on a whole table a privilege declared for only some of its columns. A declared role listed in ROLES_AWAITING_PROVISIONING that doesn't exist yet is named in the job summary and skipped instead. | mechanical | enforced | yes | CI job `seed`, `catalog-runtime-integration` | `seed-v2-database.yml:seed`, `v2-integration-check.yml:catalog-runtime-integration` | yes | +| `api.v2.deployed-privileges-match` | Every v2 deploy grants the managed roles each privilege policyengine_api/data/v2/privileges.py declares, on the whole table or on the columns it names, right after the schema upgrade, revokes a privilege a role holds on a whole table that is declared for only some of its columns, with the role's column grants of that privilege, and grants the declared columns again, reports every other privilege it doesn't declare without revoking it, and stops before the catalog is seeded if a declared privilege is still missing or a role still holds on a whole table a privilege declared for only some of its columns. A declared role listed in ROLES_AWAITING_PROVISIONING that doesn't exist yet is named in the job summary and skipped instead. | mechanical | enforced | yes | CI job `seed`, `catalog-runtime-integration` | `seed-v2-database.yml:seed`, `v2-integration-check.yml:catalog-runtime-integration` | yes | | `api.v2.runtime-privileges-declared` | Every v2 table has an entry for each managed database role in policyengine_api/data/v2/privileges.py; a privilege declared on only some of a table's columns names columns of that table and is never also declared on the whole table; and every privilege a v2 connector module needs (from its reads, creates, updates or deletes module name and its row locks) is declared for the role that runs it, on the whole table. | mechanical | enforced | yes | pytest `tests/unit/v2/test_privilege_manifest.py` | `pr.yml:test` | yes | | `api.v2.runtime-privileges-tested-as-runtime-role` | The cross-database integration tests run the v2 user services and the parameter tree read as policyengine_v2_runtime, catalog publication as policyengine_v2_catalog_writer, the comparison-run services as policyengine_stage12_, and the simulation API's table writes as policyengine_v2_simulation_service and policyengine_v2_modal_worker, holding only their declared privileges, so a statement those privileges don't allow fails CI. | mechanical | enforced | yes | CI job `v2-integration-check`, `catalog-runtime-integration` | `pr.yml:v2-integration-check`, `v2-integration-check.yml:catalog-runtime-integration` | — | diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 62e445528..96d410aea 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -192,7 +192,7 @@ Use three separately managed database credentials: | Operation | Configuration | Required database access | | --- | --- | --- | | Alembic schema upgrade | `V2_MIGRATION_DATABASE_URL` | Reviewed v2 schema changes; no application-runtime use | -| Catalog publication | `V2_DATA_WRITE_DATABASE_URL` | Row insertion, update, selection, deletion from `parameter_values`, `parameter_tree_children` and `parameter_search_files` (catalog pruning), temporary tables, and transaction-scoped advisory locks; no persistent schema changes | +| Catalog publication | `V2_DATA_WRITE_DATABASE_URL` | Row insertion, update, selection, deletion from `parameter_values`, `parameter_tree_children` and `parameter_search_files` (catalog pruning), `ANALYZE` of the catalog tables (`MAINTAIN`, which also allows `VACUUM`, `CLUSTER`, `REINDEX` and `LOCK TABLE` on them), temporary tables, and transaction-scoped advisory locks; no persistent schema changes | | Preview GET requests | `V2_RUNTIME_DATABASE_URL` or `V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE` | Catalog selection only during Stage 9; no migration access | Do not place more than one database URL in the environment of either the @@ -626,6 +626,39 @@ Deleted rows free space inside the tables for later publications to reuse; PostgreSQL returns it to the operating system only after a `VACUUM FULL` or an equivalent rewrite, which pruning doesn't run. +### Catalog table statistics + +PostgreSQL's autovacuum analyzes a table again only after about a tenth of its +rows have changed. `parameters` keeps every published version (about 1.25 +million rows on staging), and one release adds about 105,000, so a new +version's rows can stay out of the planner's statistics. On staging, after +6.17.1 was published on 2026-11-10, the planner estimated 1 parameter row for +US 6.17.1 instead of 102,530, chose to fetch and sort every parameter value of +that version, and `GET /v2/parameter-values?country_id=us&limit=1` took 43 to +47 seconds on every release until 2026-11-16. After a manual `ANALYZE +parameters` it walks `uq_parameters_model_version_name` and takes 4.1 ms +(UK: 4.9 ms). + +Every publication therefore runs one `ANALYZE` over `datasets`, +`parameter_nodes`, `parameter_tree_children`, `parameter_values`, +`parameters`, `regions` and `variables` once it has committed and pruned, and +lists them in the evidence's `catalog_tables_analyzed`. It takes a few seconds +and blocks no reads. A failed `ANALYZE` doesn't undo the publication: the +evidence lists no tables and names the failure in `catalog_statistics_failure` +(the database driver's error class and SQLSTATE, never its message), the +command prints it to stderr and still succeeds, and the next publication +analyzes again. + +`ANALYZE` needs the table's owner or the `MAINTAIN` privilege (PostgreSQL 17), +and PostgreSQL skips any other table with only a warning. The migration role +owns the tables, so the manifest grants the catalog writer `MAINTAIN` on these +seven, the deploy's `apply` step grants it before publication runs, and +publication fails before writing anything if its role lacks it. PostgreSQL 17 +has no narrower privilege: `MAINTAIN` also allows `VACUUM` (`FULL` included), +`CLUSTER`, `REINDEX`, `REFRESH MATERIALIZED VIEW` and `LOCK TABLE` on those +tables, though publication runs only `ANALYZE`. The code is +`policyengine_api/data/v2/catalog/publication_statistics.py`. + ## Validation evidence and retry Successful JSON evidence contains: @@ -650,6 +683,9 @@ Successful JSON evidence contains: (`catalog_pruning_failures`; see "Catalog pruning"); - the number of chosen versions left for later publications (`catalog_versions_left_to_prune`); +- the catalog tables analyzed after the publication + (`catalog_tables_analyzed`), and the class of a failed `ANALYZE` + (`catalog_statistics_failure`; see "Catalog table statistics"); - elapsed publication time; and - a success outcome. diff --git a/policyengine_api/data/v2/catalog/initialization.py b/policyengine_api/data/v2/catalog/initialization.py index a73fcc0a7..3f357abdd 100644 --- a/policyengine_api/data/v2/catalog/initialization.py +++ b/policyengine_api/data/v2/catalog/initialization.py @@ -177,10 +177,10 @@ def _error_payload(error: Exception) -> dict[str, object]: def main() -> int: """Run explicit initialization and return a shell-compatible status. - A failure to prune an older catalog version, or to read the simulation - service's versions to record SPM availability, leaves the status - successful, since the publication had committed; each one is in the - evidence and is also printed to stderr. + A failure to prune an older catalog version, to analyze the catalog + tables, or to read the simulation service's versions to record SPM + availability, leaves the status successful, since the publication had + committed; each one is in the evidence and is also printed to stderr. """ try: evidence = initialize_catalog() @@ -199,6 +199,13 @@ def main() -> int: f"{failure.error_type}: {failure.message}", file=sys.stderr, ) + if evidence.catalog_statistics_failure is not None: + print( + "catalog table statistics were not refreshed: " + f"{evidence.catalog_statistics_failure}; the next publication " + "analyzes the catalog tables again", + file=sys.stderr, + ) spm_recording = evidence.spm_availability if ( spm_recording is not None diff --git a/policyengine_api/data/v2/catalog/publication.py b/policyengine_api/data/v2/catalog/publication.py index 97f1358a1..75c51c6f4 100644 --- a/policyengine_api/data/v2/catalog/publication.py +++ b/policyengine_api/data/v2/catalog/publication.py @@ -41,6 +41,10 @@ create_staging_tables, stage_catalog, ) +from policyengine_api.data.v2.catalog.publication_statistics import ( + analyze_catalog_tables, + require_analyze_privilege, +) from policyengine_api.data.v2.catalog.publication_types import ( CatalogPruningFailure, CatalogPublicationError, @@ -163,12 +167,17 @@ def publish_catalog( After the transaction commits, it prunes the older stored versions of the catalog's models (``publication_pruning``), at most two per run, one transaction each; a failure of any kind rolls back only that version and - is reported in the evidence. + is reported in the evidence. Then it analyzes the catalog tables, whose + planner statistics PostgreSQL would otherwise leave out of date + (``publication_statistics``); a failure is reported in the evidence too, + and the next publication analyzes again. Before it writes anything it + checks that its role may analyze them. """ started = time.monotonic() with engine.begin() as connection: _verify_expected_revision(connection) _acquire_publication_lock(connection) + require_analyze_privilege(connection) before = _protected_row_counts(connection) create_staging_tables(connection) stage_catalog(connection, catalog, checkpoint=checkpoint) @@ -208,6 +217,7 @@ def publish_catalog( checkpoint("after_validation", connection) pruning = _prune_older_versions(engine, catalog) + statistics = analyze_catalog_tables(engine) return replace( _evidence( catalog, @@ -220,6 +230,8 @@ def publish_catalog( pruned_catalog_versions=pruning.pruned, catalog_pruning_failures=pruning.failures, catalog_versions_left_to_prune=pruning.left_to_prune, + catalog_tables_analyzed=statistics.analyzed, + catalog_statistics_failure=statistics.failure_type, ) diff --git a/policyengine_api/data/v2/catalog/publication_statistics.py b/policyengine_api/data/v2/catalog/publication_statistics.py new file mode 100644 index 000000000..a13ad89b7 --- /dev/null +++ b/policyengine_api/data/v2/catalog/publication_statistics.py @@ -0,0 +1,121 @@ +"""Refresh the planner statistics of the catalog tables after a publication. + +PostgreSQL's autovacuum analyzes a table again only after about a tenth of its +rows have changed since the last time. ``parameters`` and ``parameter_values`` +keep every published catalog version, so as they grow one release's rows stay +under that share. On staging, after 6.17.1 was published on 2026-11-10, the +planner estimated 1 parameter row for US 6.17.1 instead of 102,530, and +``GET /v2/parameter-values?country_id=us&limit=1`` took 43 to 47 seconds until +``ANALYZE parameters`` brought it to 4.1 ms. + +So every publication runs one ``ANALYZE`` over the catalog tables it writes, +once it has committed and pruned (``analyze_catalog_tables``). It takes a few +seconds and blocks no reads. Running it every time, rather than only after a +run that changed a version, means a failed ``ANALYZE``, which is recorded in +the evidence by its error class like a failed pruning, is repeated by the +next publication instead of leaving the statistics stale. + +``ANALYZE`` needs the table's owner or the ``MAINTAIN`` privilege, and +PostgreSQL skips any other table with only a warning. The migration role owns +every table, so the manifest declares ``MAINTAIN`` for the catalog writer on +these tables, and publication checks that its role holds it before it writes +anything (``require_analyze_privilege``). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Final + +import sqlalchemy as sa +from sqlalchemy.exc import SQLAlchemyError + +from policyengine_api.data.v2.catalog.publication_types import ( + CatalogPublicationError, +) + +if TYPE_CHECKING: + from sqlalchemy import Connection, Engine + +CATALOG_STATISTICS_TABLES: Final = ( + "datasets", + "parameter_nodes", + "parameter_tree_children", + "parameter_values", + "parameters", + "regions", + "variables", +) +"""The catalog tables publication analyzes, in name order.""" + +_HAS_MAINTAIN = sa.text( + "SELECT has_table_privilege(current_user, :qualified_table, 'MAINTAIN')" +) + + +def require_analyze_privilege(connection: Connection) -> None: + """Raise unless the connection's role may ``ANALYZE`` every catalog table. + + An owner holds every privilege, so ``has_table_privilege`` answers for + ownership and a ``MAINTAIN`` grant alike. + """ + missing = [ + table + for table in CATALOG_STATISTICS_TABLES + if not connection.execute( + _HAS_MAINTAIN, {"qualified_table": f"public.{table}"} + ).scalar_one() + ] + if missing: + msg = ( + "the publication role lacks MAINTAIN on " + + ", ".join(missing) + + ", so ANALYZE would skip them with only a warning; nothing was " + "published" + ) + raise CatalogPublicationError(msg) + + +@dataclass(frozen=True, slots=True) +class CatalogStatisticsRefresh: + """What the ``ANALYZE`` after a publication did. + + ``analyzed`` names the tables, empty when it failed. ``failure_type`` then + names the failure without its message: for a database error, the class of + the error the driver raised and its SQLSTATE, as a failed pruning reports + it, since SQLAlchemy's own class is the same for every database error. + """ + + analyzed: tuple[str, ...] + failure_type: str | None = None + + +def analyze_catalog_tables(engine: Engine) -> CatalogStatisticsRefresh: + """Run one ``ANALYZE`` over the catalog tables; a failure is returned, not raised. + + It runs after the publication committed, so a failure, of any kind, never + undoes it; the next publication analyzes again. + """ + tables = ", ".join(f"public.{table}" for table in CATALOG_STATISTICS_TABLES) + try: + with engine.begin() as connection: + connection.execute(sa.text(f"ANALYZE {tables}")) + except Exception as error: # noqa: BLE001 # the publication has committed; the next one analyzes again + return CatalogStatisticsRefresh(analyzed=(), failure_type=_failure(error)) + return CatalogStatisticsRefresh(analyzed=CATALOG_STATISTICS_TABLES) + + +def _failure(error: Exception) -> str: + """Name a failure by its class and a database error's SQLSTATE, not its message.""" + driver_error = getattr(error, "orig", None) + reported = ( + driver_error + if isinstance(error, SQLAlchemyError) and isinstance(driver_error, Exception) + else error + ) + reported_class = type(reported) + name = f"{reported_class.__module__}.{reported_class.__qualname__}" + sqlstate = getattr(reported, "sqlstate", None) + if isinstance(sqlstate, str) and sqlstate: + return f"{name} (SQLSTATE {sqlstate})" + return name diff --git a/policyengine_api/data/v2/catalog/publication_types.py b/policyengine_api/data/v2/catalog/publication_types.py index e46f478c1..6bde1ca71 100644 --- a/policyengine_api/data/v2/catalog/publication_types.py +++ b/policyengine_api/data/v2/catalog/publication_types.py @@ -110,6 +110,11 @@ class PublicationEvidence: catalog_pruning_failures: tuple[CatalogPruningFailure, ...] = () # Chosen versions beyond the run's limit, which the next publications prune. catalog_versions_left_to_prune: int = 0 + # The catalog tables analyzed once the publication committed and pruned + # (``publication_statistics``), empty when ANALYZE failed; then + # ``catalog_statistics_failure`` names the failure's class, never its message. + catalog_tables_analyzed: tuple[str, ...] = () + catalog_statistics_failure: str | None = None # The SPM availability recorded after the publication committed; None # until initialization records it. spm_availability: SPMAvailabilityRecording | None = None @@ -202,6 +207,8 @@ def as_dict(self) -> dict[str, object]: for failure in self.catalog_pruning_failures ], "catalog_versions_left_to_prune": self.catalog_versions_left_to_prune, + "catalog_tables_analyzed": list(self.catalog_tables_analyzed), + "catalog_statistics_failure": self.catalog_statistics_failure, "spm_availability": ( None if self.spm_availability is None diff --git a/policyengine_api/data/v2/privilege_plans.py b/policyengine_api/data/v2/privilege_plans.py index 671243cc8..13dea33f2 100644 --- a/policyengine_api/data/v2/privilege_plans.py +++ b/policyengine_api/data/v2/privilege_plans.py @@ -1,9 +1,10 @@ """Plan, state and report the table privileges the v2 manifest declares. The pure half of ``privilege_reconciliation``: it compares the declared -privileges with what a database grants, writes the GRANT statements, finds the -privileges a role lacks or holds too widely, and renders the reports. It runs -no SQL; ``privilege_reconciliation`` reads the database and runs the result. +privileges with what a database grants, writes the REVOKE and GRANT statements, +finds the privileges a role lacks or holds too widely, and renders the reports. +It runs no SQL; ``privilege_reconciliation`` reads the database and runs the +result. """ from __future__ import annotations @@ -75,10 +76,14 @@ class DefaultPrivilege: @dataclass(frozen=True) class ReconciliationPlan: - """What ``apply`` grants and reports for one database. + """What ``apply`` revokes, grants and reports for one database. ``skipped_roles`` are missing roles awaiting creation by an operator; they - don't block the plan, and nothing is granted to them. + don't block the plan, and nothing is granted to them. ``revocations`` are + whole-table privileges a role holds directly that the manifest declares + for named columns only; ``apply`` revokes them before it grants. Revoking + one also removes the role's column grants of it: ``revoked_columns`` are + the undeclared ones among them, and the declared ones are in ``grants``. """ grants: tuple[PrivilegeEntry, ...] @@ -86,6 +91,8 @@ class ReconciliationPlan: missing_roles: tuple[str, ...] missing_tables: tuple[str, ...] skipped_roles: tuple[str, ...] = () + revocations: tuple[PrivilegeEntry, ...] = () + revoked_columns: tuple[PrivilegeEntry, ...] = () @property def can_apply(self) -> bool: @@ -120,6 +127,12 @@ def plan_reconciliation( """Compare declared privileges with what the database grants. A missing role in ``roles_awaiting_creation`` is skipped instead of missing. + A whole-table privilege the manifest declares for named columns only, such + as the ``UPDATE`` a new table's default privileges give, is revoked. + PostgreSQL's ``REVOKE`` on a table also removes the role's column grants of + that privilege, so the plan grants the declared columns again and lists the + undeclared ones it removes in ``revoked_columns``. The owner's ``REVOKE`` + leaves a grant another role made; ``verify`` then still fails. """ absent_roles = set(declared) - set(existing_roles) skipped_roles = tuple(sorted(absent_roles & set(roles_awaiting_creation))) @@ -144,18 +157,41 @@ def plan_reconciliation( for column, privileges in columns.items() for privilege in privileges } + revocations = sorted( + PrivilegeEntry(role, table, privilege.privilege.value) + for role, tables in declared.items() + for table, privileges in tables.items() + for privilege in privileges + if isinstance(privilege, ColumnPrivilege) + and role not in absent_roles + and table not in missing_tables + and privilege.privilege.value in granted.tables.get(role, {}).get(table, ()) + ) + revoked = {(entry.role, entry.table, entry.privilege) for entry in revocations} + held_entries = { + entry + for entry in granted_entries + if (entry.role, entry.table, entry.privilege) not in revoked + } + revoked_columns = sorted( + entry + for entry in granted_entries - declared_entries + if entry.column and (entry.role, entry.table, entry.privilege) in revoked + ) grants = sorted( entry - for entry in declared_entries - granted_entries + for entry in declared_entries - held_entries if entry.role not in absent_roles and entry.table not in missing_tables ) - undeclared = sorted(granted_entries - declared_entries) + undeclared = sorted(held_entries - declared_entries) return ReconciliationPlan( grants=tuple(grants), undeclared=tuple(undeclared), missing_roles=missing_roles, missing_tables=missing_tables, skipped_roles=skipped_roles, + revocations=tuple(revocations), + revoked_columns=tuple(revoked_columns), ) @@ -167,6 +203,22 @@ def safe_identifier(identifier: str) -> str: return identifier +def revoke_statements( + plan: ReconciliationPlan, quote_identifier: Callable[[str], str] +) -> list[str]: + """Return the REVOKEs for a plan: one per role and table, in that order.""" + privileges_by_target: dict[tuple[str, str], set[str]] = defaultdict(set) + for entry in plan.revocations: + target = (safe_identifier(entry.role), safe_identifier(entry.table)) + privileges_by_target[target].add(TablePrivilege(entry.privilege).value) + return [ + f"REVOKE {', '.join(sorted(privileges))} ON TABLE " + f"{quote_identifier(SCHEMA)}.{quote_identifier(table)} " + f"FROM {quote_identifier(role)}" + for (role, table), privileges in sorted(privileges_by_target.items()) + ] + + def grant_statements( plan: ReconciliationPlan, quote_identifier: Callable[[str], str] ) -> list[str]: @@ -310,6 +362,21 @@ def render_summary( + ", ".join(f"`{table}`" for table in plan.missing_tables), "", ] + if plan.revocations: + revoke_heading = "Revoked" if applied else "Would revoke" + lines += [ + f"### {revoke_heading}: whole-table privileges declared for named " + "columns only", + "", + "Revoking one also removes the role's column grants of it; the " + "undeclared ones are listed with it, and the declared ones are in " + "the grants below.", + "", + "| Role | Table | Privileges |", + "| --- | --- | --- |", + *_privilege_rows((*plan.revocations, *plan.revoked_columns)), + "", + ] grant_heading = "Granted" if applied else "Would grant" lines += [f"### {grant_heading}", ""] if plan.grants: diff --git a/policyengine_api/data/v2/privilege_reconciliation.py b/policyengine_api/data/v2/privilege_reconciliation.py index 008e33e87..692fccfad 100644 --- a/policyengine_api/data/v2/privilege_reconciliation.py +++ b/policyengine_api/data/v2/privilege_reconciliation.py @@ -6,8 +6,12 @@ - ``apply`` reads the managed roles' current grants, on whole tables and on columns, grants every declared privilege a role lacks, and reports privileges - the manifest doesn't declare. It never revokes, and it changes nothing when a - declared role or table is missing. Running it again grants nothing more. + the manifest doesn't declare. It revokes only a whole-table privilege the + manifest declares for named columns only, such as the ``UPDATE`` a new + table's default privileges give, which also removes the role's column grants + of it, and then grants the declared columns again, in the same transaction. + It changes nothing when a declared role or table is missing. Running it + again changes nothing more. - ``verify`` checks every declared privilege with ``has_table_privilege``, or ``has_column_privilege`` for each column of a column privilege (rule ``api.v2.deployed-privileges-match``), and fails on any that is missing. It @@ -61,6 +65,7 @@ grant_statements, plan_reconciliation, render_summary, + revoke_statements, safe_identifier, skipped_roles_line, table_wide_overrides, @@ -105,6 +110,7 @@ "read_granted_privileges", "read_public_tables", "render_summary", + "revoke_statements", "run", "skipped_roles", "table_wide_overrides", @@ -274,10 +280,14 @@ def apply_declared_privileges( *, roles_awaiting_creation: Collection[str] = (), ) -> tuple[ReconciliationPlan, list[DefaultPrivilege]]: - """Grant every declared privilege a role lacks; never revoke. - - Grants nothing when a declared role or table is missing, unless the missing - role awaits creation, which is skipped. + """Grant every declared privilege a role lacks. + + First revokes each whole-table privilege the manifest declares for named + columns only, which also removes the role's column grants of it, so the + grants that follow include those columns again. Every other privilege the + manifest doesn't declare is reported, not revoked. Changes nothing when a + declared role or table is missing, unless the missing role awaits creation, + which is skipped. """ roles = list(declared) plan = plan_reconciliation( @@ -291,9 +301,8 @@ def apply_declared_privileges( roles_awaiting_creation=roles_awaiting_creation, ) if plan.can_apply: - for statement in grant_statements( - plan, connection.dialect.identifier_preparer.quote - ): + quote = connection.dialect.identifier_preparer.quote + for statement in revoke_statements(plan, quote) + grant_statements(plan, quote): connection.execute(text(statement)) return plan, read_default_privileges(connection, roles) diff --git a/policyengine_api/data/v2/privileges.py b/policyengine_api/data/v2/privileges.py index f8047aff0..ba116de22 100644 --- a/policyengine_api/data/v2/privileges.py +++ b/policyengine_api/data/v2/privileges.py @@ -25,8 +25,9 @@ temporary Stage 12 tables (``docs/migration/stage-12-worker-contract.md``). - ``policyengine_v2_catalog_writer``: the metadata catalog publication (``scripts/publish_v2_metadata_catalog.py``). Its privileges follow what - publication writes and reads; its code isn't organized as connector modules, - so the deploy check, not the connector analysis, covers it. + publication writes and reads, plus ``MAINTAIN`` on the catalog tables it + analyzes afterwards; its code isn't organized as connector modules, so the + deploy check, not the connector analysis, covers it. - ``policyengine_v2_simulation_service``: the simulation API on Cloud Run. It creates simulations, reports, their runs, budget-window runs and household jobs, and reads the policy, dynamic, household and catalog rows a run needs. @@ -57,9 +58,13 @@ The migration role (``policyengine_v2_migrator``) owns every table, and Supabase's ``postgres`` role administers the project; neither is managed here. -Extra privileges a database holds beyond this manifest are reported, never -revoked. The v2 schema has no sequences (every key is a UUID); a test fails if -one appears, so the manifest can be extended to cover it. +Extra privileges a database holds beyond this manifest are reported, not +revoked, with one exception: a privilege a role holds on a whole table that +this manifest declares for named columns only is revoked, with the role's +column grants of it, and the declared columns are granted again. + +The v2 schema has no sequences (every key is a UUID); a test fails if one +appears, so the manifest can be extended to cover it. """ from __future__ import annotations @@ -91,6 +96,8 @@ class TablePrivilege(StrEnum): INSERT = "INSERT" UPDATE = "UPDATE" DELETE = "DELETE" + # PostgreSQL 17: lets a role that doesn't own the table run ANALYZE on it. + MAINTAIN = "MAINTAIN" @dataclass(frozen=True) @@ -129,6 +136,13 @@ def update_columns(*columns: str) -> ColumnPrivilege: TablePrivilege.DELETE, } ) +# ANALYZE after publication (``catalog.publication_statistics``): the migration +# role owns the catalog tables, and PostgreSQL skips a table the role neither +# owns nor holds MAINTAIN on, with only a warning. PostgreSQL 17 has no narrower +# privilege: MAINTAIN also allows VACUUM (FULL included), CLUSTER, REINDEX, +# REFRESH MATERIALIZED VIEW and LOCK TABLE on them, of which publication runs +# only ANALYZE. +_ANALYZE: Final = frozenset({TablePrivilege.MAINTAIN}) @dataclass(frozen=True) @@ -268,29 +282,34 @@ def _run_table( # The UK areas disclosure control withheld from a local-area run's results. "withheld_area_codes": _RESULT, # Metadata catalog: the catalog writer publishes it, the Public API reads it. + # The writer also analyzes seven of them after every publication (_ANALYZE). "dataset_versions": _access(catalog_writer=READ), "datasets": _access( - runtime=READ, catalog_writer=READ_INSERT_UPDATE, **_RUN_INPUT_READS + runtime=READ, catalog_writer=READ_INSERT_UPDATE | _ANALYZE, **_RUN_INPUT_READS + ), + "parameter_nodes": _access( + runtime=READ, catalog_writer=READ_INSERT_UPDATE | _ANALYZE ), - "parameter_nodes": _access(runtime=READ, catalog_writer=READ_INSERT_UPDATE), # Written by catalog publication for each version it publishes, from the # installed packages' catalog; never rewritten. Catalog pruning deletes an # older version's file, which publishing that version again writes back. "parameter_search_files": _access(runtime=READ, catalog_writer=READ_INSERT_DELETE), # Written with each catalog version that publication publishes; catalog # pruning deletes an older version's rows. - "parameter_tree_children": _access(runtime=READ, catalog_writer=READ_INSERT_DELETE), + "parameter_tree_children": _access( + runtime=READ, catalog_writer=READ_INSERT_DELETE | _ANALYZE + ), # The Public API also inserts a v2 policy's own parameter values. Catalog # pruning deletes an older version's current-law values (those with no # policy or dynamic), never a policy's or a dynamic's. "parameter_values": _access( - runtime=READ_INSERT, catalog_writer=READ_WRITE, **_RUN_INPUT_READS + runtime=READ_INSERT, catalog_writer=READ_WRITE | _ANALYZE, **_RUN_INPUT_READS ), "parameters": _access( - runtime=READ, catalog_writer=READ_INSERT_UPDATE, **_RUN_INPUT_READS + runtime=READ, catalog_writer=READ_INSERT_UPDATE | _ANALYZE, **_RUN_INPUT_READS ), "regions": _access( - runtime=READ, catalog_writer=READ_INSERT_UPDATE, **_RUN_INPUT_READS + runtime=READ, catalog_writer=READ_INSERT_UPDATE | _ANALYZE, **_RUN_INPUT_READS ), # Catalog publication updates a version's time periods, # catalog_last_published_at and spm_availability, and catalog pruning its @@ -301,7 +320,7 @@ def _run_table( "tax_benefit_models": _access( runtime=READ, catalog_writer=READ_INSERT_UPDATE, **_RUN_INPUT_READS ), - "variables": _access(runtime=READ, catalog_writer=READ_INSERT_UPDATE), + "variables": _access(runtime=READ, catalog_writer=READ_INSERT_UPDATE | _ANALYZE), # v2 policies and households, created by the native routes and by Stage 10 # dual-write. A copy locks its mapping row (FOR UPDATE), which needs UPDATE. "households": _access(runtime=READ_INSERT, **_RUN_INPUT_READS), diff --git a/policyengine_api/fastapi_routes/v2/metadata/read_failure_support.py b/policyengine_api/fastapi_routes/v2/metadata/read_failure_support.py new file mode 100644 index 000000000..e9d7af5b0 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/metadata/read_failure_support.py @@ -0,0 +1,110 @@ +"""Record why a v2 metadata read answered the generic 503 or a 500. + +A catalog the API can't read answers the generic 503 "V2 metadata catalog is +unavailable", and any other unexpected failure the 500 "V2 metadata query +failed", so the response names no cause. Each such answer records one runtime +event, ``v2_metadata_read_failed``, at ERROR. The runtime adds the request it +is serving: its ID, method and matched route. The event adds: + +- ``http.response.status_code``: 503 or 500; +- ``error_type``: the caught error's class, as ``module.QualifiedName``; +- ``failure_category``: ``v2_metadata_catalog_unavailable`` for the 503 or + ``v2_metadata_query_failed`` for the 500. The scheduled error-issue job + groups such an event by its route, ``error_type`` and ``failure_category``, + the keys the API's other events use; +- ``error.cause_types``: the class of each error in its ``__cause__`` chain, + outermost first and joined by ", ", or absent when it has no cause; +- ``database.sqlstate``: the first SQLSTATE an error in the chain carries, as + psycopg's errors do in ``sqlstate``; +- ``database.connection_invalidated``: whether SQLAlchemy found the connection + lost, from the first error in the chain with ``connection_invalidated``, as + ``sqlalchemy.exc.DBAPIError`` has. + +It records no error message: SQLAlchemy's and the driver's messages carry the +SQL statement and its parameters, and can carry host names, addresses and +credentials. The attributes are read by name, so this module imports neither +SQLAlchemy nor a driver. Recording never changes the response. +""" + +from __future__ import annotations + +import contextlib + +METADATA_READ_FAILURE_EVENT = "v2_metadata_read_failed" +CATALOG_UNAVAILABLE_FAILURE = "v2_metadata_catalog_unavailable" +QUERY_FAILED_FAILURE = "v2_metadata_query_failed" +MAX_NAMED_CAUSES = 10 +"""How many errors of a ``__cause__`` chain the event names at most.""" + + +def record_metadata_read_failure( + error: BaseException, status_code: int, failure_category: str +) -> None: + """Record why one metadata read failed; recording never fails the request.""" + # Telemetry must never change a metadata response. + with contextlib.suppress(Exception): + from policyengine_api import ( # noqa: PLC0415 # importing it builds the process-wide runtime, which importing the route modules alone must not + observability, + ) + + observability.get_runtime().event( + METADATA_READ_FAILURE_EVENT, + severity="ERROR", + attributes=read_failure_attributes(error, status_code, failure_category), + ) + + +def read_failure_attributes( + error: BaseException, status_code: int, failure_category: str +) -> dict[str, str | int | bool]: + """Return the event's attributes: class names, SQLSTATE and lost-connection flag.""" + causes = _cause_chain(error) + attributes: dict[str, str | int | bool] = { + "http.response.status_code": status_code, + "error_type": _class_name(error), + "failure_category": failure_category, + } + if causes: + attributes["error.cause_types"] = ", ".join( + _class_name(cause) for cause in causes + ) + chain = (error, *causes) + sqlstate = next( + ( + code + for code in (getattr(item, "sqlstate", None) for item in chain) + if isinstance(code, str) and code + ), + None, + ) + if sqlstate is not None: + attributes["database.sqlstate"] = sqlstate + invalidated = next( + ( + flag + for flag in ( + getattr(item, "connection_invalidated", None) for item in chain + ) + if isinstance(flag, bool) + ), + None, + ) + if invalidated is not None: + attributes["database.connection_invalidated"] = invalidated + return attributes + + +def _cause_chain(error: BaseException) -> list[BaseException]: + causes: list[BaseException] = [] + cause = error.__cause__ + while cause is not None and cause is not error and len(causes) < MAX_NAMED_CAUSES: + if cause in causes: + break + causes.append(cause) + cause = cause.__cause__ + return causes + + +def _class_name(error: BaseException) -> str: + error_class = type(error) + return f"{error_class.__module__}.{error_class.__qualname__}" diff --git a/policyengine_api/fastapi_routes/v2/metadata/route_support.py b/policyengine_api/fastapi_routes/v2/metadata/route_support.py index e8a5d005f..3efeecf20 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/route_support.py +++ b/policyengine_api/fastapi_routes/v2/metadata/route_support.py @@ -22,6 +22,11 @@ V2ErrorResponse, v2_error_response, ) +from policyengine_api.fastapi_routes.v2.metadata.read_failure_support import ( + CATALOG_UNAVAILABLE_FAILURE, + QUERY_FAILED_FAILURE, + record_metadata_read_failure, +) from policyengine_api.fastapi_routes.v2.metadata.response_cache_support import ( BuiltResponse, KeptResponse, @@ -424,7 +429,8 @@ def run_metadata_read( """Run one metadata read and return its result unchanged. Failures become the typed v2 error responses described by read_resource, - and the reader is closed whatever happens. + and the reader is closed whatever happens. A generic 503 or a 500 also + records why, without the error's message (``read_failure_support``). """ reader = None try: @@ -447,8 +453,11 @@ def run_metadata_read( ) as error: return v2_error_response(404, str(error)) except (V2ConfigurationError, MetadataCatalogUnavailableError) as error: + if _caller_message(error) is None: + record_metadata_read_failure(error, 503, CATALOG_UNAVAILABLE_FAILURE) return _catalog_unavailable_response(error) - except Exception: # noqa: BLE001 # preview must return typed errors + except Exception as error: # noqa: BLE001 # preview must return typed errors + record_metadata_read_failure(error, 500, QUERY_FAILED_FAILURE) return v2_error_response(500, "V2 metadata query failed") finally: if reader is not None: @@ -457,6 +466,25 @@ def run_metadata_read( reader.close() +def _caller_message( + error: V2ConfigurationError | MetadataCatalogUnavailableError, +) -> str | None: + """Return the message written for callers, or None for the generic 503. + + A pruned version, or a missing parameter tree or search file, is a known + state whose message names the catalog version. Any other error answers the + generic 503, which records why (``read_failure_support``). + """ + if isinstance( + error, + CatalogValuesPrunedError + | ParameterTreeUnavailableError + | ParameterSearchFileUnavailableError, + ): + return str(error) + return None + + def _catalog_unavailable_response( error: V2ConfigurationError | MetadataCatalogUnavailableError, ) -> JSONResponse: @@ -472,12 +500,7 @@ def _catalog_unavailable_response( if isinstance(error, CatalogValuesPrunedError): status = 503 if error.running_version else 404 return v2_error_response(status, str(error), headers=NO_STORE_HEADERS) - message = ( - str(error) - if isinstance( - error, - ParameterTreeUnavailableError | ParameterSearchFileUnavailableError, - ) - else "V2 metadata catalog is unavailable" - ) + message = _caller_message(error) + if message is None: + message = "V2 metadata catalog is unavailable" return v2_error_response(503, message) diff --git a/pyproject.toml b/pyproject.toml index 36a130236..29d8c9f2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ dependencies = [ # versions certified by this PolicyEngine bundle. Cloud Run installs the # complete frozen environment from uv.lock; the generic image installs # this requirement directly. - "policyengine[models]==6.17.1", + "policyengine[models]==6.20.0", "pydantic", "pymysql", "python-dotenv", diff --git a/rules/registry.toml b/rules/registry.toml index c31a7ab57..87a5cf6a8 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -1067,7 +1067,7 @@ invocation = "reconcile_v2_privileges.py apply --create-missing-roles" [[rule]] id = "api.v2.deployed-privileges-match" category = "migration" -statement = "Every v2 deploy grants the managed roles each privilege policyengine_api/data/v2/privileges.py declares, on the whole table or on the columns it names, right after the schema upgrade, reports privileges it doesn't declare without revoking them, and stops before the catalog is seeded if a declared privilege is still missing or a role holds on a whole table a privilege declared for only some of its columns. A declared role listed in ROLES_AWAITING_PROVISIONING that doesn't exist yet is named in the job summary and skipped instead." +statement = "Every v2 deploy grants the managed roles each privilege policyengine_api/data/v2/privileges.py declares, on the whole table or on the columns it names, right after the schema upgrade, revokes a privilege a role holds on a whole table that is declared for only some of its columns, with the role's column grants of that privilege, and grants the declared columns again, reports every other privilege it doesn't declare without revoking it, and stops before the catalog is seeded if a declared privilege is still missing or a role still holds on a whole table a privilege declared for only some of its columns. A declared role listed in ROLES_AWAITING_PROVISIONING that doesn't exist yet is named in the job summary and skipped instead." rationale = "docs/engineering/skills/alembic-migrations.md#table-privileges" scope = ["policyengine_api/data/v2/privileges.py", "policyengine_api/data/v2/privilege_plans.py", "policyengine_api/data/v2/privilege_reconciliation.py", "scripts/reconcile_v2_privileges.py", ".github/workflows/seed-v2-database.yml"] tier = "mechanical" diff --git a/tests/fixtures/v2_metadata_routes.py b/tests/fixtures/v2_metadata_routes.py index 2ececbb89..6b04a3194 100644 --- a/tests/fixtures/v2_metadata_routes.py +++ b/tests/fixtures/v2_metadata_routes.py @@ -287,7 +287,9 @@ def detail(item: object) -> MetadataDetailResult: ) -def metadata_route_client(factory, *, response_cache=None) -> TestClient: +def metadata_route_client( + factory, *, response_cache=None, observability_runtime=None +) -> TestClient: flask_app = Flask(__name__) @flask_app.get("//metadata") @@ -317,6 +319,7 @@ def v1_metadata(country_id: str): flask_app, dependencies=dependencies, route_settings=settings, + observability_runtime=observability_runtime, ), raise_server_exceptions=False, ) diff --git a/tests/integration/test_v2_catalog_statistics.py b/tests/integration/test_v2_catalog_statistics.py new file mode 100644 index 000000000..cb60d91bd --- /dev/null +++ b/tests/integration/test_v2_catalog_statistics.py @@ -0,0 +1,228 @@ +"""PostgreSQL tests for refreshing the catalog tables' statistics after publication. + +PostgreSQL re-collects a table's statistics on its own only after about a tenth +of its rows change. ``parameters`` keeps every published version, so on staging +one release's rows stayed under that share: after 6.17.1 was published on +2026-11-10, the planner estimated 1 parameter row for US 6.17.1 instead of +102,530 and served ``GET /v2/parameter-values?country_id=us&limit=1`` in 43-47 +seconds. Every publication now runs ``ANALYZE`` on the catalog tables once it +has committed and pruned; a failed ``ANALYZE`` is recorded in the evidence and +the next publication runs it again. ``ANALYZE`` skips, with only a warning, a +table the role neither owns nor holds ``MAINTAIN`` on, so publication checks +that privilege first and fails before writing anything without it. +""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import TYPE_CHECKING +from uuid import uuid4 + +import pytest +from alembic import command +from alembic.config import Config +from sqlalchemy import create_engine, event, text +from sqlalchemy.engine import make_url +from sqlalchemy.pool import NullPool + +from policyengine_api.data.v2.catalog.publication import publish_catalog +from policyengine_api.data.v2.catalog.publication_types import ( + CatalogPublicationError, +) +from policyengine_api.data.v2.privilege_reconciliation import ( + apply_declared_privileges, +) +from policyengine_api.data.v2.privileges import ( + CATALOG_WRITER_ROLE, + declared_privileges, +) +from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL +from tests.fixtures.v2_catalog import normalized_catalog + +if TYPE_CHECKING: + from collections.abc import Iterator + from datetime import datetime + + from sqlalchemy.engine import Engine + +REPO = Path(__file__).parents[2] +DISPOSABLE_DATABASE = "policyengine_v2_alembic_test" +UNDEFINED_TABLE_FAILURE = "psycopg.errors.UndefinedTable (SQLSTATE 42P01)" +ANALYZED_TABLES = ( + "datasets", + "parameter_nodes", + "parameter_tree_children", + "parameter_values", + "parameters", + "regions", + "variables", +) + + +def _disposable_url() -> str: + database_url = os.environ.get(V2_MIGRATION_DATABASE_URL, "") + if not database_url: + pytest.skip(f"{V2_MIGRATION_DATABASE_URL} is not set") + url = make_url(database_url) + if url.database != DISPOSABLE_DATABASE or url.host not in { + "localhost", + "127.0.0.1", + "::1", + "postgres", + }: + pytest.fail("catalog publication tests require disposable local Postgres") + return database_url + + +@pytest.fixture +def superuser_engine() -> Iterator[Engine]: + """Upgrade the disposable database and empty its catalog before and after.""" + database_url = _disposable_url() + command.upgrade(Config(str(REPO / "alembic-v2.ini")), "head") + engine = create_engine(database_url, poolclass=NullPool) + with engine.begin() as connection: + connection.execute(text("TRUNCATE tax_benefit_models CASCADE")) + try: + yield engine + finally: + with engine.begin() as connection: + connection.execute(text("TRUNCATE tax_benefit_models CASCADE")) + engine.dispose() + + +@pytest.fixture +def writer_engine(superuser_engine, v2_engine_as_role) -> Iterator[Engine]: + engine = v2_engine_as_role(CATALOG_WRITER_ROLE, poolclass=NullPool) + try: + yield engine + finally: + engine.dispose() + + +def _database_time(engine: Engine) -> datetime: + with engine.connect() as connection: + return connection.execute(text("SELECT clock_timestamp()")).scalar_one() + + +def _last_manual_analyze(engine: Engine) -> dict[str, datetime | None]: + with engine.connect() as connection: + rows = connection.execute( + text( + "SELECT relname, last_analyze FROM pg_stat_user_tables " + "WHERE schemaname = 'public' AND relname = ANY(:tables)" + ), + {"tables": list(ANALYZED_TABLES)}, + ) + return {row.relname: row.last_analyze for row in rows} + + +def test_publishing_a_new_version_as_the_catalog_writer_analyzes_the_catalog( + superuser_engine, writer_engine +) -> None: + started = _database_time(superuser_engine) + + evidence = publish_catalog(writer_engine, normalized_catalog()) + last_analyze = _last_manual_analyze(superuser_engine) + + assert evidence.catalog_tables_analyzed == ANALYZED_TABLES + assert evidence.as_dict()["catalog_tables_analyzed"] == list(ANALYZED_TABLES) + assert sorted(last_analyze) == list(ANALYZED_TABLES) + assert all( + analyzed_at is not None and analyzed_at >= started + for analyzed_at in last_analyze.values() + ), last_analyze + + +def test_republishing_an_unchanged_catalog_analyzes_it_again( + superuser_engine, writer_engine +) -> None: + """Every run analyzes, so a run after a failed ``ANALYZE`` repeats it.""" + catalog = normalized_catalog() + publish_catalog(writer_engine, catalog) + second_started = _database_time(superuser_engine) + + second = publish_catalog(writer_engine, catalog) + last_analyze = _last_manual_analyze(superuser_engine) + + assert second.catalog_tables_analyzed == ANALYZED_TABLES + assert second.catalog_statistics_failure is None + assert all( + analyzed_at is not None and analyzed_at >= second_started + for analyzed_at in last_analyze.values() + ), last_analyze + + +def test_a_failed_analyze_is_recorded_and_the_publication_kept( + superuser_engine, writer_engine +) -> None: + """The database refuses the ``ANALYZE``, and SQLAlchemy wraps its error. + + The evidence names the driver's error and SQLSTATE, as a failed pruning's + does, not SQLAlchemy's wrapper, which is the same for any database error. + """ + + def analyze_a_missing_table( + _connection, _cursor, statement, parameters, *_arguments + ): + if statement.startswith("ANALYZE"): + return "ANALYZE public.no_such_catalog_table", parameters + return statement, parameters + + event.listen( + writer_engine, "before_cursor_execute", analyze_a_missing_table, retval=True + ) + + evidence = publish_catalog(writer_engine, normalized_catalog()) + with superuser_engine.connect() as connection: + stored_versions = connection.execute( + text("SELECT count(*) FROM tax_benefit_model_versions") + ).scalar_one() + + assert stored_versions > 0 + assert evidence.catalog_tables_analyzed == () + assert evidence.catalog_statistics_failure == UNDEFINED_TABLE_FAILURE + assert evidence.as_dict()["catalog_statistics_failure"] == UNDEFINED_TABLE_FAILURE + + +@pytest.fixture +def writer_without_maintain(superuser_engine) -> Iterator[str]: + """Create a role with the catalog writer's declared privileges except MAINTAIN.""" + role = f"policyengine_catalog_test_{uuid4().hex[:12]}" + writer_tables = declared_privileges("test")[CATALOG_WRITER_ROLE] + declared = { + role: { + table: frozenset( + privilege for privilege in privileges if privilege != "MAINTAIN" + ) + for table, privileges in writer_tables.items() + } + } + with superuser_engine.begin() as connection: + connection.execute(text(f'CREATE ROLE "{role}" NOLOGIN')) + apply_declared_privileges(connection, declared) + try: + yield role + finally: + with superuser_engine.begin() as connection: + connection.execute(text(f'DROP OWNED BY "{role}"')) + connection.execute(text(f'DROP ROLE "{role}"')) + + +def test_a_role_that_cannot_analyze_the_catalog_fails_before_writing( + superuser_engine, v2_engine_as_role, writer_without_maintain +) -> None: + engine = v2_engine_as_role(writer_without_maintain, poolclass=NullPool) + try: + with pytest.raises(CatalogPublicationError) as raised: + publish_catalog(engine, normalized_catalog()) + finally: + engine.dispose() + with superuser_engine.connect() as connection: + stored_versions = connection.execute( + text("SELECT count(*) FROM tax_benefit_model_versions") + ).scalar_one() + + assert "MAINTAIN" in str(raised.value) + assert "parameters" in str(raised.value) + assert stored_versions == 0 diff --git a/tests/integration/test_v2_privilege_reconciliation.py b/tests/integration/test_v2_privilege_reconciliation.py index 97fdb3f34..4cf7dac64 100644 --- a/tests/integration/test_v2_privilege_reconciliation.py +++ b/tests/integration/test_v2_privilege_reconciliation.py @@ -151,6 +151,63 @@ def test_verify_fails_when_a_role_can_update_the_whole_table( ) in summary +def test_apply_revokes_a_new_tables_whole_table_update_and_verify_passes( + disposable_v2_database_url, superuser_engine, temporary_role, monkeypatch, tmp_path +) -> None: + """A new table's default privileges give UPDATE on the whole table. + + The role starts as staging's runtime role did on ``user_type_changes``: + the whole-table privileges, the declared column grant an earlier ``apply`` + made, and an undeclared column grant. ``apply`` revokes the whole-table + ``UPDATE``, which removes both column grants, and grants the declared one + again in the same transaction, so ``verify`` passes. The extra ``INSERT`` + and ``DELETE`` stay, reported but not revoked. + """ + role = temporary_role() + declared = {role: {"regions": READ | {update_columns("label")}}} + monkeypatch.setattr( + privilege_reconciliation, "declared_privileges", lambda _environment: declared + ) + monkeypatch.setattr( + privilege_reconciliation, "ROLES_AWAITING_PROVISIONING", frozenset() + ) + with superuser_engine.begin() as connection: + connection.execute( + text( + "GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE public.regions " + f'TO "{role}"' + ) + ) + connection.execute( + text(f'GRANT UPDATE (label, code) ON TABLE public.regions TO "{role}"') + ) + summary_path = tmp_path / "summary.md" + environ = { + V2_MIGRATION_DATABASE_URL: disposable_v2_database_url, + V2_ALEMBIC_DISPOSABLE_TEST: "1", + DEPLOYMENT_ENVIRONMENT: "test", + "GITHUB_STEP_SUMMARY": str(summary_path), + } + + apply_status = run("apply", environ=environ) + verify_status = run("verify", environ=environ) + summary = summary_path.read_text(encoding="utf-8") + with superuser_engine.begin() as connection: + second_plan, _defaults = apply_declared_privileges(connection, declared) + + assert (apply_status, verify_status) == (0, 0) + assert f"| `{role}` | `regions` | UPDATE, UPDATE (code) |" in summary + assert f"| `{role}` | `regions` | UPDATE (label) |" in summary + assert f"| `{role}` | `regions` | DELETE, INSERT |" in summary + assert not _has_privilege(superuser_engine, role, "regions", "UPDATE") + assert _has_column_privilege(superuser_engine, role, "regions", "label", "UPDATE") + assert not _has_column_privilege( + superuser_engine, role, "regions", "code", "UPDATE" + ) + assert _has_privilege(superuser_engine, role, "regions", "DELETE") + assert (second_plan.revocations, second_plan.grants) == ((), ()) + + def test_apply_grants_nothing_the_second_time_and_verify_passes( disposable_v2_database_url, managed_v2_roles, capsys ) -> None: diff --git a/tests/unit/v2/test_catalog_initialization.py b/tests/unit/v2/test_catalog_initialization.py index 7543dfc75..ad5bc4f95 100644 --- a/tests/unit/v2/test_catalog_initialization.py +++ b/tests/unit/v2/test_catalog_initialization.py @@ -315,6 +315,36 @@ def test_main_succeeds_and_prints_each_pruning_failure_to_stderr( ] +def test_main_succeeds_and_prints_a_failed_analyze_to_stderr( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """A failed ANALYZE leaves the committed publication's command successful. + + The failure is in the evidence on stdout and on one stderr line, which says + the next publication analyzes again. + """ + failure = "psycopg.errors.QueryCanceled (SQLSTATE 57014)" + evidence = PublicationEvidence( + policyengine_version=POLICYENGINE_VERSION, + dependency_versions=(), + entity_counts={"models": 2}, + fallback_summaries=(), + elapsed_seconds=1.0, + catalog_statistics_failure=failure, + ) + monkeypatch.setattr(initialization, "initialize_catalog", lambda: evidence) + + assert initialization.main() == 0 + + output = capsys.readouterr() + assert json.loads(output.out)["catalog_statistics_failure"] == failure + assert output.err.splitlines() == [ + f"catalog table statistics were not refreshed: {failure}; the next " + "publication analyzes the catalog tables again" + ] + + def test_main_prints_a_pruning_failure_that_chose_no_version_to_stderr( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], diff --git a/tests/unit/v2/test_catalog_publication.py b/tests/unit/v2/test_catalog_publication.py index 338a6d97c..b537ca5cc 100644 --- a/tests/unit/v2/test_catalog_publication.py +++ b/tests/unit/v2/test_catalog_publication.py @@ -321,6 +321,8 @@ def test_completion_evidence_contains_only_reviewed_non_secret_fields() -> None: ), ), catalog_versions_left_to_prune=3, + catalog_tables_analyzed=("parameter_values", "parameters"), + catalog_statistics_failure="psycopg.errors.QueryCanceled", spm_availability=publication_types.SPMAvailabilityRecording( recorded=(("uk", "2.50.0", False),), not_recorded=(("us", "1.450.0"),), @@ -399,6 +401,8 @@ def test_completion_evidence_contains_only_reviewed_non_secret_fields() -> None: } ], "catalog_versions_left_to_prune": 3, + "catalog_tables_analyzed": ["parameter_values", "parameters"], + "catalog_statistics_failure": "psycopg.errors.QueryCanceled", "spm_availability": { "recorded": [{"country_id": "uk", "version": "2.50.0", "available": False}], "not_recorded": [{"country_id": "us", "version": "1.450.0"}], diff --git a/tests/unit/v2/test_metadata_read_failure_events.py b/tests/unit/v2/test_metadata_read_failure_events.py new file mode 100644 index 000000000..5d2983625 --- /dev/null +++ b/tests/unit/v2/test_metadata_read_failure_events.py @@ -0,0 +1,317 @@ +"""Runtime events for v2 metadata reads that fail with a generic 503 or a 500. + +The response stays the generic one, but the observability runtime records one +event that says what failed: the route and request ID, which the runtime takes +from the request it is serving, then the response status, the caught error's +class (``error_type``), a failure category that tells a catalog outage from an +unexpected failure, the class of each error in its ``__cause__`` chain, the +driver's SQLSTATE and whether SQLAlchemy found the connection lost. The +scheduled error-issue job groups such an event by route, ``error_type`` and +``failure_category``. The event never records an error's message, which can +carry the SQL statement, its parameters, host names, addresses or credentials. + +The tests serve each request through a runtime with the process runtime's +settings whose records go to a list, so they see the event as the runtime +writes it, request context included. +""" + +from __future__ import annotations + +from dataclasses import replace +from typing import TYPE_CHECKING, Any + +import pytest +from policyengine_observability import CustomLogDestination, OTelConfig, configure +from psycopg import errors as psycopg_errors +from sqlalchemy.exc import DBAPIError + +from policyengine_api import observability +from policyengine_api.data.v2.catalog.catalog_selection_validation import ( + MetadataCatalogUnavailableError, +) +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.fastapi_routes.v2.metadata.read_failure_support import ( + MAX_NAMED_CAUSES, + read_failure_attributes, +) +from policyengine_api.services.v2.metadata.validators import ( + CatalogValuesPrunedError, + ParameterSearchFileUnavailableError, + ParameterTreeUnavailableError, +) +from tests.fixtures.v2_metadata_routes import ResourceReader, metadata_route_client + +if TYPE_CHECKING: + from collections.abc import Iterator + + from policyengine_observability import ObservabilityRuntime + +ROUTE = "/v2/parameter-values" +PATH = f"{ROUTE}?country_id=us&limit=1" +FAILURE_EVENT = "v2_metadata_read_failed" +GENERIC_503 = {"status": "error", "message": "V2 metadata catalog is unavailable"} +GENERIC_500 = {"status": "error", "message": "V2 metadata query failed"} +STATEMENT = "SELECT parameter_values.id FROM parameter_values WHERE country_id = %(c)s" +PARAMETER_VALUE = "us-secret-parameter-value" +SENSITIVE_TEXT = ( + STATEMENT, + PARAMETER_VALUE, + "pooler.example.internal", + "10.12.0.7", + "hunter2", + "postgresql://", +) +LOST_CONNECTION_CAUSES = "sqlalchemy.exc.OperationalError, psycopg.errors.AdminShutdown" + + +class RecordList: + """A runtime destination that keeps every record it is given.""" + + def __init__(self) -> None: + self.records: list[dict[str, Any]] = [] + + def write(self, record: dict[str, Any]) -> None: + self.records.append(record) + + def failures(self) -> list[dict[str, Any]]: + return [ + record + for record in self.records + if record.get("event.name") == FAILURE_EVENT + ] + + +class FailingRuntime: + def event(self, *_arguments: object, **_details: object) -> None: + msg = "the observability runtime is down" + raise RuntimeError(msg) + + +@pytest.fixture +def recorded(monkeypatch) -> Iterator[tuple[RecordList, ObservabilityRuntime]]: + """Serve requests through a runtime whose records the test can read. + + It keeps the process runtime's settings and replaces only where records go, + with OpenTelemetry export off. + """ + record_list = RecordList() + process_config = observability.get_runtime().config + runtime = configure( + replace( + process_config, + logging=replace( + process_config.logging, + destinations=( + CustomLogDestination( + name="test-records", + writer_factory=lambda: record_list, + delivery="inline", + ), + ), + ), + otel=OTelConfig(enabled=False), + ) + ) + monkeypatch.setattr(observability, "get_runtime", lambda: runtime) + try: + yield record_list, runtime + finally: + runtime.shutdown() + + +def _get(reader: ResourceReader, runtime: ObservabilityRuntime | None = None): + client = metadata_route_client(lambda: reader, observability_runtime=runtime) + return client.get(PATH) + + +def _reader_failing_with(error: BaseException) -> ResourceReader: + return ResourceReader({"list_parameter_values": None}, error=error) + + +def _qualified_name(error_class: type[BaseException]) -> str: + return f"{error_class.__module__}.{error_class.__qualname__}" + + +def _lost_connection() -> DBAPIError: + """Return SQLAlchemy's error for a connection the server ended mid-query. + + psycopg raises ``AdminShutdown`` (SQLSTATE 57P01), and SQLAlchemy wraps it + as it does, raising its error from psycopg's. Every message names the + statement, a parameter, the pooler's host and address and a password, as a + driver's message can. + """ + driver_error = psycopg_errors.AdminShutdown( + "terminating connection due to administrator command: " + "postgresql://reader:hunter2@pooler.example.internal (10.12.0.7)" + ) + database_error = DBAPIError.instance( + STATEMENT, + {"c": PARAMETER_VALUE}, + driver_error, + psycopg_errors.Error, + connection_invalidated=True, + ) + database_error.__cause__ = driver_error + return database_error + + +def _caused_by(error: BaseException, cause: BaseException) -> BaseException: + error.__cause__ = cause + return error + + +def _assert_nothing_sensitive(record: dict[str, Any]) -> None: + written = repr(record) + for text in SENSITIVE_TEXT: + assert text not in written + + +def test_a_lost_connection_answers_the_same_503_and_records_its_cause( + recorded, +) -> None: + record_list, runtime = recorded + error = _caused_by( + MetadataCatalogUnavailableError(f"catalog read failed: {STATEMENT}"), + _lost_connection(), + ) + + response = _get(_reader_failing_with(error), runtime) + [record] = record_list.failures() + + assert (response.status_code, response.json()) == (503, GENERIC_503) + assert record["severity"] == "ERROR" + assert record["http.route"] == ROUTE + assert record["request.id"] == response.headers["x-policyengine-request-id"] + assert record["attributes"] == { + "http.response.status_code": 503, + "error_type": _qualified_name(MetadataCatalogUnavailableError), + "failure_category": "v2_metadata_catalog_unavailable", + "error.cause_types": LOST_CONNECTION_CAUSES, + "database.sqlstate": "57P01", + "database.connection_invalidated": True, + } + _assert_nothing_sensitive(record) + + +def test_an_unexpected_failure_answers_the_same_500_and_records_its_cause( + recorded, +) -> None: + record_list, runtime = recorded + error = _caused_by(RuntimeError(f"failed on {STATEMENT}"), _lost_connection()) + + response = _get(_reader_failing_with(error), runtime) + [record] = record_list.failures() + + assert (response.status_code, response.json()) == (500, GENERIC_500) + assert (record["severity"], record["http.route"]) == ("ERROR", ROUTE) + assert record["attributes"] == { + "http.response.status_code": 500, + "error_type": "builtins.RuntimeError", + "failure_category": "v2_metadata_query_failed", + "error.cause_types": LOST_CONNECTION_CAUSES, + "database.sqlstate": "57P01", + "database.connection_invalidated": True, + } + _assert_nothing_sensitive(record) + + +@pytest.mark.parametrize( + "error", + [ + MetadataCatalogUnavailableError("catalog connection refused"), + V2ConfigurationError("V2_DATABASE_URL is not set"), + ], + ids=["catalog-unavailable", "configuration"], +) +def test_a_failure_without_a_database_cause_records_only_its_class( + recorded, error: BaseException +) -> None: + record_list, runtime = recorded + + response = _get(_reader_failing_with(error), runtime) + [record] = record_list.failures() + + assert (response.status_code, response.json()) == (503, GENERIC_503) + assert record["attributes"] == { + "http.response.status_code": 503, + "error_type": _qualified_name(type(error)), + "failure_category": "v2_metadata_catalog_unavailable", + } + + +@pytest.mark.parametrize( + "error", + [ + CatalogValuesPrunedError("pruned", running_version=True), + ParameterTreeUnavailableError("5.0.1 has no parameter tree"), + ParameterSearchFileUnavailableError("5.0.1 has no parameter search file"), + ], + ids=["pruned-version", "no-parameter-tree", "no-search-file"], +) +def test_an_answer_with_its_own_message_records_no_failure( + recorded, error: BaseException +) -> None: + record_list, runtime = recorded + + response = _get(_reader_failing_with(error), runtime) + + assert response.status_code == 503 + assert response.json()["message"] == str(error) + assert record_list.failures() == [] + + +@pytest.mark.parametrize( + ("error", "status_code", "body"), + [ + (MetadataCatalogUnavailableError("catalog read failed"), 503, GENERIC_503), + (RuntimeError("failed"), 500, GENERIC_500), + ], + ids=["503", "500"], +) +def test_a_failing_runtime_leaves_the_answer_unchanged( + monkeypatch, error: BaseException, status_code: int, body: dict[str, str] +) -> None: + monkeypatch.setattr(observability, "get_runtime", FailingRuntime) + + response = _get(_reader_failing_with(_caused_by(error, _lost_connection()))) + + assert (response.status_code, response.json()) == (status_code, body) + + +def test_a_cause_chain_that_loops_back_names_each_error_once() -> None: + first = RuntimeError("first") + second = ValueError("second") + first.__cause__ = second + second.__cause__ = first + + attributes = read_failure_attributes(first, 500, "v2_metadata_query_failed") + + assert attributes["error.cause_types"] == "builtins.ValueError" + + +def test_a_cause_chain_that_loops_back_partway_names_each_error_once() -> None: + error = RuntimeError("outermost") + middle = ValueError("middle") + inner = KeyError("inner") + error.__cause__ = middle + middle.__cause__ = inner + inner.__cause__ = middle + + attributes = read_failure_attributes(error, 500, "v2_metadata_query_failed") + + assert attributes["error.cause_types"] == "builtins.ValueError, builtins.KeyError" + + +def test_a_long_cause_chain_names_its_first_errors_only() -> None: + error = RuntimeError("outermost") + current = error + for _depth in range(MAX_NAMED_CAUSES + 2): + cause = KeyError("cause") + current.__cause__ = cause + current = cause + + attributes = read_failure_attributes(error, 500, "v2_metadata_query_failed") + + assert attributes["error.cause_types"] == ", ".join( + ["builtins.KeyError"] * MAX_NAMED_CAUSES + ) diff --git a/tests/unit/v2/test_privilege_manifest.py b/tests/unit/v2/test_privilege_manifest.py index 2f81268c1..0a20c74a3 100644 --- a/tests/unit/v2/test_privilege_manifest.py +++ b/tests/unit/v2/test_privilege_manifest.py @@ -11,6 +11,9 @@ import sqlalchemy as sa from policyengine_api.data.v2 import models +from policyengine_api.data.v2.catalog.publication_statistics import ( + CATALOG_STATISTICS_TABLES, +) from policyengine_api.data.v2.models import V2_METADATA from policyengine_api.data.v2.privilege_requirements import ( required_privileges, @@ -393,6 +396,21 @@ def test_rule_proof_runtime_privileges_declared() -> None: assert undeclared_requirements(good, runtime_before_3977) == [] +def test_the_catalog_writer_may_analyze_the_tables_publication_refreshes() -> None: + """Publication runs ANALYZE on these tables, which needs MAINTAIN or ownership. + + The migration role owns every table, so the catalog writer needs MAINTAIN + (PostgreSQL 17); without it, ANALYZE skips the table with only a warning. + """ + writer_tables = declared_privileges(TEST_ENVIRONMENT)[CATALOG_WRITER_ROLE] + + assert { + table + for table, privileges in writer_tables.items() + if TablePrivilege.MAINTAIN in privileges + } == set(CATALOG_STATISTICS_TABLES) + + def test_the_catalog_writer_deletes_only_the_rows_catalog_pruning_removes() -> None: """Catalog pruning deletes from three tables and marks the pruned version. @@ -410,13 +428,19 @@ def test_the_catalog_writer_deletes_only_the_rows_catalog_pruning_removes() -> N if TablePrivilege.DELETE in privileges } == {"parameter_values", "parameter_tree_children", "parameter_search_files"} assert TablePrivilege.UPDATE in catalog_writer["tax_benefit_model_versions"] + # MAINTAIN is for ANALYZE after publication, not for deleting. assert catalog_writer["parameter_tree_children"] == frozenset( - {TablePrivilege.SELECT, TablePrivilege.INSERT, TablePrivilege.DELETE} + { + TablePrivilege.SELECT, + TablePrivilege.INSERT, + TablePrivilege.DELETE, + TablePrivilege.MAINTAIN, + } ) assert catalog_writer["parameter_search_files"] == frozenset( {TablePrivilege.SELECT, TablePrivilege.INSERT, TablePrivilege.DELETE} ) - assert catalog_writer["parameter_values"] == READ_WRITE + assert catalog_writer["parameter_values"] == READ_WRITE | {TablePrivilege.MAINTAIN} def test_the_public_api_records_user_type_changes_but_never_rewrites_them() -> None: diff --git a/tests/unit/v2/test_privilege_reconciliation.py b/tests/unit/v2/test_privilege_reconciliation.py index 208a5a1e5..b7d731682 100644 --- a/tests/unit/v2/test_privilege_reconciliation.py +++ b/tests/unit/v2/test_privilege_reconciliation.py @@ -16,6 +16,7 @@ grant_statements, plan_reconciliation, render_summary, + revoke_statements, table_wide_overrides, verification_failures, ) @@ -296,7 +297,7 @@ def test_column_and_whole_table_privileges_beyond_the_manifest_are_reported() -> plan = plan_reconciliation( WORKER_DECLARED, granted=GrantedPrivileges( - tables={WORKER: {"simulation_runs": {"SELECT", "UPDATE"}}}, + tables={WORKER: {"simulation_runs": {"SELECT", "DELETE"}}}, columns={ WORKER: { "simulation_runs": { @@ -311,9 +312,9 @@ def test_column_and_whole_table_privileges_beyond_the_manifest_are_reported() -> existing_tables={"simulation_runs"}, ) - assert plan.grants == () + assert (plan.grants, plan.revocations) == ((), ()) assert plan.undeclared == ( - PrivilegeEntry(WORKER, "simulation_runs", "UPDATE"), + PrivilegeEntry(WORKER, "simulation_runs", "DELETE"), PrivilegeEntry(WORKER, "simulation_runs", "UPDATE", column="simulation_id"), ) @@ -386,6 +387,132 @@ def has_privilege(role: str, table: str, privilege: str) -> bool: ) +def test_a_whole_table_privilege_declared_for_columns_is_revoked() -> None: + """A new table's default privileges give UPDATE on the whole table. + + Revoking it also removes the role's column grants of UPDATE, so the plan + grants the declared columns again and lists the removed undeclared ones + with the revoke rather than as undeclared. + """ + plan = plan_reconciliation( + WORKER_DECLARED, + granted=GrantedPrivileges( + tables={WORKER: {"simulation_runs": {"SELECT", "UPDATE", "DELETE"}}}, + columns={ + WORKER: { + "simulation_runs": { + "status": {"UPDATE"}, + "simulation_id": {"UPDATE"}, + } + } + }, + ), + existing_roles={WORKER}, + existing_tables={"simulation_runs"}, + ) + + assert plan.revocations == (PrivilegeEntry(WORKER, "simulation_runs", "UPDATE"),) + assert plan.revoked_columns == ( + PrivilegeEntry(WORKER, "simulation_runs", "UPDATE", column="simulation_id"), + ) + assert plan.grants == ( + PrivilegeEntry(WORKER, "simulation_runs", "UPDATE", column="completed_at"), + PrivilegeEntry(WORKER, "simulation_runs", "UPDATE", column="status"), + ) + assert plan.undeclared == (PrivilegeEntry(WORKER, "simulation_runs", "DELETE"),) + + +def test_a_revoke_is_only_planned_while_a_declared_table_is_missing() -> None: + """Like a grant, a revoke waits until every declared table exists.""" + declared = {WORKER: {**WORKER_DECLARED[WORKER], "household_jobs": READ}} + plan = plan_reconciliation( + declared, + granted=GrantedPrivileges( + tables={WORKER: {"simulation_runs": {"SELECT", "UPDATE"}}} + ), + existing_roles={WORKER}, + existing_tables={"simulation_runs"}, + ) + + assert plan.missing_tables == ("household_jobs",) + assert not plan.can_apply + assert plan.revocations == (PrivilegeEntry(WORKER, "simulation_runs", "UPDATE"),) + + +def test_revoke_statements_revoke_each_role_and_tables_privileges() -> None: + plan = ReconciliationPlan( + grants=(), + undeclared=(), + missing_roles=(), + missing_tables=(), + revocations=( + PrivilegeEntry(WORKER, "simulation_runs", "UPDATE"), + PrivilegeEntry(RUNTIME, "user_type_changes", "UPDATE"), + ), + ) + + assert revoke_statements(plan, _quote) == [ + 'REVOKE UPDATE ON TABLE "public"."simulation_runs" ' + 'FROM "policyengine_v2_modal_worker"', + 'REVOKE UPDATE ON TABLE "public"."user_type_changes" ' + 'FROM "policyengine_v2_runtime"', + ] + + +def test_revoke_statements_refuse_an_unsafe_table_name() -> None: + plan = ReconciliationPlan( + grants=(), + undeclared=(), + missing_roles=(), + missing_tables=(), + revocations=(PrivilegeEntry(WORKER, 'runs"; DROP TABLE users; --', "UPDATE"),), + ) + + with pytest.raises(privilege_reconciliation.UnsafeIdentifierError): + revoke_statements(plan, _quote) + + +def test_summary_lists_revoked_whole_table_privileges_only_when_there_are_some() -> ( + None +): + revoking_plan = ReconciliationPlan( + grants=( + PrivilegeEntry(RUNTIME, "user_type_changes", "UPDATE", "dismissed_at"), + ), + undeclared=(), + missing_roles=(), + missing_tables=(), + revocations=(PrivilegeEntry(RUNTIME, "user_type_changes", "UPDATE"),), + revoked_columns=( + PrivilegeEntry(RUNTIME, "user_type_changes", "UPDATE", "user_id"), + ), + ) + plain_plan = ReconciliationPlan( + grants=(), undeclared=(), missing_roles=(), missing_tables=() + ) + + applied = render_summary( + revoking_plan, (), deployment_environment="staging", applied=True + ) + planned = render_summary( + revoking_plan, (), deployment_environment="staging", applied=False + ) + plain = render_summary( + plain_plan, (), deployment_environment="staging", applied=True + ) + + revoked_row = ( + "| `policyengine_v2_runtime` | `user_type_changes` | UPDATE, UPDATE (user_id) |" + ) + assert "### Revoked: whole-table privileges declared for named columns" in applied + assert revoked_row in applied + assert "### Would revoke: whole-table privileges declared for named columns" in ( + planned + ) + assert "### Revoked" not in plain + assert "### Would revoke" not in plain + + def test_summary_lists_column_grants_with_their_columns() -> None: plan = ReconciliationPlan( grants=( diff --git a/tests/unit/v2/test_publication_statistics.py b/tests/unit/v2/test_publication_statistics.py new file mode 100644 index 000000000..5f8601ba3 --- /dev/null +++ b/tests/unit/v2/test_publication_statistics.py @@ -0,0 +1,27 @@ +"""How the ANALYZE after a catalog publication reports a failure. + +The PostgreSQL tests (``tests/integration/test_v2_catalog_statistics.py``) cover +a database error, which is named by the driver's class and SQLSTATE. Any other +failure is named by its class alone, never its message. +""" + +from __future__ import annotations + +from policyengine_api.data.v2.catalog.publication_statistics import ( + analyze_catalog_tables, +) + + +class EngineThatCannotConnect: + """Stands in for an engine whose connection fails before any statement.""" + + def begin(self) -> None: + msg = "postgresql://writer:hunter2@db.example.internal refused the connection" + raise RuntimeError(msg) + + +def test_a_failure_that_is_not_a_database_error_is_named_by_its_class() -> None: + refresh = analyze_catalog_tables(EngineThatCannotConnect()) + + assert refresh.analyzed == () + assert refresh.failure_type == "builtins.RuntimeError" diff --git a/uv.lock b/uv.lock index dc3aef586..796a266d5 100644 --- a/uv.lock +++ b/uv.lock @@ -2954,7 +2954,7 @@ wheels = [ [[package]] name = "policyengine" -version = "6.17.1" +version = "6.20.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "h5py" }, @@ -2966,9 +2966,9 @@ dependencies = [ { name = "pydantic" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/da/45/483142f35ce0dfa02db01e541f730843b9108ae4b60a8bdae92ac223f3b2/policyengine-6.17.1.tar.gz", hash = "sha256:07a2617c738a77d2f1fd8ff2c5f4d89b040a54f44e9f931331a7ae34fdbbfc9f", size = 1629731, upload-time = "2026-11-10T16:26:24.672Z" } +sdist = { url = "https://files.pythonhosted.org/packages/14/e8/4373c716f428ed885392ff1776dc1be1f9e9a3a6f7608afcc1f39d683546/policyengine-6.20.0.tar.gz", hash = "sha256:65dad563d92c17485c5943b2bb8114d8f16cd49c977acbce6412cd8b8bcfadfd", size = 1863269, upload-time = "2026-11-16T16:21:45.756Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1f/d5/e8b6181f512e5c3b14bbd0638a4c179515a9f479a68a6eaba640005b0fad/policyengine-6.17.1-py3-none-any.whl", hash = "sha256:7fa63f9f6524d911ff7a490448a2b202341487dc09d5c945fc0e0c816893df1e", size = 562776, upload-time = "2026-11-10T16:26:22.717Z" }, + { url = "https://files.pythonhosted.org/packages/64/b6/9ebed9a78c755c25893a20c924bc6952100bc3376608511fbcdb065145c3/policyengine-6.20.0-py3-none-any.whl", hash = "sha256:cc2da1b7448ae229294090318140bfe39440e8899415b8e465459c7e89bb7135", size = 774390, upload-time = "2026-11-16T16:21:43.414Z" }, ] [package.optional-dependencies] @@ -3072,7 +3072,7 @@ requires-dist = [ { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.15,<2" }, { name = "openai" }, { name = "packaging", specifier = ">=24,<27" }, - { name = "policyengine", extras = ["models"], specifier = "==6.17.1" }, + { name = "policyengine", extras = ["models"], specifier = "==6.20.0" }, { name = "policyengine-observability", extras = ["flask", "google", "httpx", "otlp-grpc"], specifier = ">=3.0.3,<4" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.3,<4" }, { name = "pydantic" }, @@ -3169,7 +3169,7 @@ wheels = [ [[package]] name = "policyengine-us" -version = "2.2.1" +version = "2.2.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -3179,9 +3179,9 @@ dependencies = [ { name = "tables" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2c/09/763b21371a62635e15e62b53fb696e299f01460907c27313d40f63031eb2/policyengine_us-2.2.1.tar.gz", hash = "sha256:d5300c91f3eae2e47a61f485fdfde91a892b80e95db3e46ff2c4bbd417d76501", size = 11946510, upload-time = "2026-09-15T04:45:50.33Z" } +sdist = { url = "https://files.pythonhosted.org/packages/72/86/1bbc7ab8d689987b364e624ef550a7a43ec49343b863be39b267a7c3477b/policyengine_us-2.2.2.tar.gz", hash = "sha256:a7cd4383d82e2453a3433e49d5501fd21b01b43b3910a66954d0a3cfe65f422b", size = 11947023, upload-time = "2026-11-09T20:52:43.712Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/09/86ac3968d2a18e28fce3ba1186fc21728e9f299b96a806d6f204f1e0aaed/policyengine_us-2.2.1-py3-none-any.whl", hash = "sha256:0993a6c73fcdfbe171a796aca9d302741bd8e00ab83388ad81c15a08740318c6", size = 14885537, upload-time = "2026-09-15T04:45:46.649Z" }, + { url = "https://files.pythonhosted.org/packages/84/89/12a44d68e4131cc6ad92cfa22959254c69306d1f25fa6760ea1aab851bcb/policyengine_us-2.2.2-py3-none-any.whl", hash = "sha256:82ac620444dc0f246559fcbf7b34daee5a921a57684b63121f319365820811ce", size = 14885748, upload-time = "2026-11-09T20:52:38.979Z" }, ] [[package]] @@ -4109,7 +4109,7 @@ wheels = [ [[package]] name = "spm-calculator" -version = "1.0.0" +version = "1.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "census" }, @@ -4119,9 +4119,9 @@ dependencies = [ { name = "requests" }, { name = "us" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c2/5e/3f28b212c401795990250bb3daec741f699a8c7dc2a23a3db7005df6feb6/spm_calculator-1.0.0.tar.gz", hash = "sha256:a99aac8c2c0bf81a9455105bbf872366f1ea9066a99cd793714881cd3db846dc", size = 7150825, upload-time = "2026-09-11T15:45:12.732Z" } +sdist = { url = "https://files.pythonhosted.org/packages/33/84/19057fa04e6c8cce85494458b2164d473ce82cbf54608a3b2951af2c36b1/spm_calculator-1.1.0.tar.gz", hash = "sha256:1be418c6a1baaba86f63c3287b38efecf7f94af563b864a73934ecbf60a01f38", size = 9625137, upload-time = "2026-11-02T18:48:45.002Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/19/a0/c484f69a0ebf88a9b9fd0ac28176f8df66550f46714f1b0c5ec0b360824e/spm_calculator-1.0.0-py3-none-any.whl", hash = "sha256:e354937a5e1a4045d4966ed594a528d8b02866fabaac9bb5672017004b627305", size = 7395384, upload-time = "2026-09-11T15:45:10.492Z" }, + { url = "https://files.pythonhosted.org/packages/35/91/12a744eb7f3445db8899dc187bbd193c884f9fe4c373851c0ddb24cc0f5e/spm_calculator-1.1.0-py3-none-any.whl", hash = "sha256:c528ec0f06eeb1188b7d6cf456ff3a9b73a0fd8856b3938358251cc5f9be83e8", size = 9881340, upload-time = "2026-11-02T18:48:42.665Z" }, ] [[package]]