From 646963a39598eb816774c477aa1a76b4503decd8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 18 Nov 2026 13:58:26 -0500 Subject: [PATCH] Add Microcosm dataset and certification selectors --- .../microcosm-dataset-selectors.added.md | 1 + docs/bundles.md | 6 +- .../build-m-us-populace-certification.md | 6 +- .../runbooks/canonical-spm-development.md | 2 +- docs/engineering/skills/data-certification.md | 14 ++-- docs/microsim.md | 15 ++-- docs/release-bundles.md | 2 +- scripts/bundle.py | 2 +- scripts/certify_data_release.py | 11 ++- src/policyengine/provenance/certification.py | 12 ++- .../provenance/dataset_materialization.py | 6 +- src/policyengine/provenance/manifest.py | 22 ++++- .../tax_benefit_models/us/datasets.py | 21 +++-- tests/test_certify_data_release.py | 19 +++-- tests/test_microcosm_compatibility.py | 82 ++++++++++++++++++- tests/test_us_legacy_spm_integration.py | 36 +++++--- 16 files changed, 198 insertions(+), 59 deletions(-) create mode 100644 changelog.d/microcosm-dataset-selectors.added.md diff --git a/changelog.d/microcosm-dataset-selectors.added.md b/changelog.d/microcosm-dataset-selectors.added.md new file mode 100644 index 000000000..1a7a08e28 --- /dev/null +++ b/changelog.d/microcosm-dataset-selectors.added.md @@ -0,0 +1 @@ +Add Microcosm selectors for the certified US national and local-area datasets and accept `--data-producer microcosm` in the release certification CLI. Existing selectors and certification metadata remain supported. diff --git a/docs/bundles.md b/docs/bundles.md index acaf3ab39..09f3015e7 100644 --- a/docs/bundles.md +++ b/docs/bundles.md @@ -51,7 +51,7 @@ package scaffold: ```python from policyengine.provenance import materialize_dataset -result = materialize_dataset("us", "populace_us_2024") +result = materialize_dataset("us", "microcosm_us_2024") print(result.path) print(result.bundle_dataset.sha256) ``` @@ -202,7 +202,7 @@ To certify a new data release from a data-producer manifest, run: ```bash python scripts/bundle.py certify-data \ --country uk \ - --data-producer populace \ + --data-producer microcosm \ --manifest-uri hf://dataset/policyengine/populace-uk-private@/releases//release_manifest.json ``` @@ -211,7 +211,7 @@ For US Microcosm releases, certify the Microcosm release manifest directly: ```bash python scripts/bundle.py certify-data \ --country us \ - --data-producer populace \ + --data-producer microcosm \ --manifest-uri hf://dataset/policyengine/populace-us@/releases//release_manifest.json \ --regional-manifest-uri hf://dataset/policyengine/populace-us@/releases//release_manifest.json \ --model-version diff --git a/docs/engineering/runbooks/build-m-us-populace-certification.md b/docs/engineering/runbooks/build-m-us-populace-certification.md index c69f77986..99077ee2f 100644 --- a/docs/engineering/runbooks/build-m-us-populace-certification.md +++ b/docs/engineering/runbooks/build-m-us-populace-certification.md @@ -33,8 +33,8 @@ Set values from the two published release manifests rather than copying an older certification: ``` -NATIONAL_RELEASE_ID = populace-us-2024---Z -LOCAL_AREA_RELEASE_ID = populace-us-2024---Z +NATIONAL_RELEASE_ID = +LOCAL_AREA_RELEASE_ID = MODEL_VERSION = ``` @@ -47,7 +47,7 @@ artifact. ```bash python scripts/bundle.py certify-data \ --country us \ - --data-producer populace \ + --data-producer microcosm \ --model-version "$MODEL_VERSION" \ --manifest-uri "hf://dataset/policyengine/populace-us@$NATIONAL_RELEASE_ID/releases/$NATIONAL_RELEASE_ID/release_manifest.json" \ --regional-manifest-uri "hf://dataset/policyengine/populace-us@$LOCAL_AREA_RELEASE_ID/releases/$LOCAL_AREA_RELEASE_ID/release_manifest.json" diff --git a/docs/engineering/runbooks/canonical-spm-development.md b/docs/engineering/runbooks/canonical-spm-development.md index 4e094bc52..9437fe6dd 100644 --- a/docs/engineering/runbooks/canonical-spm-development.md +++ b/docs/engineering/runbooks/canonical-spm-development.md @@ -168,7 +168,7 @@ Ordinary `check` passes for the staged draft; this is metadata consistency only. coordinator; do not invent future versions to break that dependency. Refresh country and wrapper locks against real registry bytes. 5. Certify the published data release through `scripts/bundle.py certify-data - --country us --data-producer populace --model-version + --country us --data-producer microcosm --model-version "$PUBLISHED_COUNTRY_VERSION" --manifest-uri "$PUBLISHED_RELEASE_MANIFEST_URI"`. The URI must identify the actual immutable producer release. This command updates the country pin and data metadata from real compatibility evidence. diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index 545b1aad4..91760940d 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -22,7 +22,7 @@ Open the work on a fresh branch from current `main` (use a clean worktree if the checkout is dirty). ```bash -python scripts/bundle.py certify-data --country uk --data-producer populace \ +python scripts/bundle.py certify-data --country uk --data-producer microcosm \ --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases//release_manifest.json" ``` @@ -46,7 +46,7 @@ its `release` and `build` as strings, for example its own folder, at the commit that published it: ```bash -python scripts/bundle.py certify-data --country uk --data-producer populace \ +python scripts/bundle.py certify-data --country uk --data-producer microcosm \ --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases///release_manifest.json" ``` @@ -104,7 +104,7 @@ Microcosm national release with one of: `dataset_role: non_default_local_area`, as for the US below. ```bash - python scripts/bundle.py certify-data --country uk --data-producer populace \ + python scripts/bundle.py certify-data --country uk --data-producer microcosm \ --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases/microcosm-uk-2024-25-national/release_manifest.json" \ --regional-manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases//release_manifest.json" \ --model-version "" @@ -118,7 +118,7 @@ Microcosm national release with one of: release whose households carry area codes: ```bash - python scripts/bundle.py certify-data --country uk --data-producer populace \ + python scripts/bundle.py certify-data --country uk --data-producer microcosm \ --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases/microcosm-uk-2024-25-national/release_manifest.json" \ --regional-manifest-uri "hf://model/policyengine/policyengine-uk-data-private@/releases//release_manifest.json" \ --regional-release-kind area_coded_national \ @@ -152,7 +152,7 @@ Microcosm national release with one of: --limitation "the dense cut's VAT predates the policyengine-uk VAT factor fix (#2187, 2.135.7); the dashboard holds a VAT limit" \ --output regional-input-surface.json - .venv/bin/python scripts/bundle.py certify-data --country uk --data-producer populace \ + .venv/bin/python scripts/bundle.py certify-data --country uk --data-producer microcosm \ --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases/microcosm-uk-2024-25-national/release_manifest.json" \ --regional-manifest-uri "" \ --accept-regional-model-mismatch "" \ @@ -320,7 +320,7 @@ folder (it has `releases.json`), so give each manifest's folder at the commit that published it: ```bash -python scripts/bundle.py certify-data --country us --data-producer populace \ +python scripts/bundle.py certify-data --country us --data-producer microcosm \ --manifest-uri "hf://dataset/policyengine/populace-us@/releases/microcosm-us-2024-national//release_manifest.json" \ --regional-manifest-uri "hf://dataset/policyengine/populace-us@/releases/microcosm-us-2024-local//release_manifest.json" \ --model-version "" @@ -362,7 +362,7 @@ exact target installed, then certify with the reason and the evidence. --limitation "" \ --output regional-input-surface.json -.venv/bin/python scripts/bundle.py certify-data --country us --data-producer populace \ +.venv/bin/python scripts/bundle.py certify-data --country us --data-producer microcosm \ --manifest-uri "hf://dataset/policyengine/populace-us@/releases/microcosm-us-2024-national//release_manifest.json" \ --regional-manifest-uri "hf://dataset/policyengine/populace-us@/releases/microcosm-us-2024-local//release_manifest.json" \ --accept-regional-model-mismatch "" \ diff --git a/docs/microsim.md b/docs/microsim.md index 15b12ba25..1d9787fd4 100644 --- a/docs/microsim.md +++ b/docs/microsim.md @@ -128,10 +128,11 @@ Microdata is stored as HDF5 on Hugging Face. `ensure_datasets` downloads, caches ```python datasets = pe.us.ensure_datasets( + datasets=["microcosm_us_2024"], years=[2024, 2026], data_folder="./data", # local cache directory ) -dataset = datasets["populace_us_2024_2026"] +dataset = datasets["microcosm_us_2024_2026"] ``` The default US dataset is **Microcosm US 2024** — a Microcosm-built dataset @@ -144,7 +145,9 @@ remain available as named, non-default bundle datasets. PolicyEngine.py obtains the repository type, immutable revision, and SHA-256 from the installed release bundle. An existing file in the configured data -directory is reused only after hash verification. +directory is reused only after hash verification. The original US selectors +`populace_us_2024` and `populace_us_2024_acs_local` remain supported and select +the same certified artifacts. List datasets already known to the country: @@ -155,7 +158,7 @@ pe.us.load_datasets() # or pe.uk.load_datasets() ### US local-area dataset Alongside the certified national default, the bundle registers a US dataset for -finer geographic work: `populace_us_2024_acs_local`. It is a +finer geographic work: `microcosm_us_2024_acs_local`. It is a Microcosm US 2024 build of roughly **1.6 million households** on an **ACS 2024 multispine**, with each household **PUMA-assigned** to a 119th-Congress congressional district, county, and state, and calibrated to **state @@ -170,14 +173,14 @@ Two-line load: ```python import policyengine as pe -sim = pe.us.managed_microsimulation(dataset="populace_us_2024_acs_local") +sim = pe.us.managed_microsimulation(dataset="microcosm_us_2024_acs_local") ``` Or materialize it as a `PolicyEngineUSDataset` for `Simulation`: ```python -datasets = pe.us.ensure_datasets(datasets=["populace_us_2024_acs_local"], years=[2024]) -dataset = datasets["populace_us_2024_acs_local_2024"] +datasets = pe.us.ensure_datasets(datasets=["microcosm_us_2024_acs_local"], years=[2024]) +dataset = datasets["microcosm_us_2024_acs_local_2024"] ``` Because this file carries PUMA-assigned district, county, and state identifiers diff --git a/docs/release-bundles.md b/docs/release-bundles.md index b4f8199ec..d9be2cdf3 100644 --- a/docs/release-bundles.md +++ b/docs/release-bundles.md @@ -199,7 +199,7 @@ python scripts/bundle.py certify-data --country us \ US Microcosm certification uses the Microcosm release manifest directly: ```bash -python scripts/bundle.py certify-data --country us --data-producer populace \ +python scripts/bundle.py certify-data --country us --data-producer microcosm \ --manifest-uri "hf://dataset/policyengine/populace-us@/releases//release_manifest.json" \ --regional-manifest-uri "hf://dataset/policyengine/populace-us@/releases//release_manifest.json" \ --model-version "" diff --git a/scripts/bundle.py b/scripts/bundle.py index b05ffa589..0cf097e9d 100644 --- a/scripts/bundle.py +++ b/scripts/bundle.py @@ -198,7 +198,7 @@ def _parser() -> argparse.ArgumentParser: certify.add_argument("--country", required=True, choices=["us", "uk"]) certify.add_argument( "--data-producer", - choices=["legacy", "populace"], + choices=["legacy", "microcosm", "populace"], help="Data-producer strategy. Defaults to the certification script default.", ) certify.add_argument( diff --git a/scripts/certify_data_release.py b/scripts/certify_data_release.py index 341297457..6db8e5475 100644 --- a/scripts/certify_data_release.py +++ b/scripts/certify_data_release.py @@ -7,7 +7,7 @@ Usage:: - python scripts/bundle.py certify-data --country uk --data-producer populace \\ + python scripts/bundle.py certify-data --country uk --data-producer microcosm \\ --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases//release_manifest.json" Pin the cut tag, never a branch: each cut rewrites ``releases//``. @@ -47,8 +47,11 @@ def main(argv=None) -> int: parser.add_argument( "--data-producer", default=None, - choices=["legacy", "populace"], - help="Data-producer strategy. Defaults to populace for UK, legacy otherwise.", + choices=["legacy", "microcosm", "populace"], + help=( + "Data-producer strategy. Defaults to microcosm for UK, legacy otherwise. " + "The previous name populace remains accepted." + ), ) parser.add_argument( "--manifest-uri", @@ -189,7 +192,7 @@ def main(argv=None) -> int: / f"certify-{args.country}-{result.build_id or 'data'}.changed.md" ) fragment.write_text( - f"Certify the {args.country.upper()} {result.data_producer} " + f"Certify the {args.country.upper()} {result.display_data_producer} " f"data release `{result.build_id}` ({result.default_dataset}, " f"{result.model_package} {result.model_version}) into the " "PolicyEngine bundle manifest.\n" diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index 72a278903..ed52b4825 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -17,7 +17,7 @@ result = certify_data_release( country="uk", - data_producer="populace", + data_producer="microcosm", manifest_uri=( "hf://dataset/policyengine/populace-uk-private" "@microcosm-uk-2024-25-national-20261002T230158Z-5c6b3f68" @@ -216,6 +216,10 @@ class CertificationResult: model_version: str warnings: list[str] = field(default_factory=list) + @property + def display_data_producer(self) -> str: + return "microcosm" if self.data_producer == "populace" else self.data_producer + def summary(self) -> str: lines = [ f"certified {self.country}: {self.default_dataset} " @@ -223,7 +227,7 @@ def summary(self) -> str: f" manifest: {self.manifest_uri}", f" manifest sha256: {self.manifest_sha256}", f" model: {self.model_package}=={self.model_version}", - f" data-producer: {self.data_producer}", + f" data-producer: {self.display_data_producer}", f" wrote: {self.bundle_path}", ] for warning in self.warnings: @@ -1769,8 +1773,8 @@ def certify( def certification_strategy( country: str, data_producer: Optional[str] = None ) -> DataProducerCertificationStrategy: - producer = data_producer or ("populace" if country == "uk" else "legacy") - if producer == "populace": + producer = data_producer or ("microcosm" if country == "uk" else "legacy") + if producer in ("microcosm", "populace"): return MicrocosmDataProducerCertificationStrategy() if producer == "legacy": return LegacyDataProducerCertificationStrategy() diff --git a/src/policyengine/provenance/dataset_materialization.py b/src/policyengine/provenance/dataset_materialization.py index 28e8f60b9..2d34f8080 100644 --- a/src/policyengine/provenance/dataset_materialization.py +++ b/src/policyengine/provenance/dataset_materialization.py @@ -123,7 +123,9 @@ def _resolve_bundle_dataset( manifest: Optional[CountryReleaseManifest] = None, ) -> _BundleDatasetSpec: country_manifest = manifest or get_release_manifest(country_id) - dataset_name = dataset or country_manifest.default_dataset + dataset_name = country_manifest.bundled_dataset_key( + dataset or country_manifest.default_dataset + ) reference = country_manifest.datasets.get(dataset_name) if reference is None: raise DatasetMaterializationError( @@ -169,7 +171,7 @@ def materialize_dataset( data_dir=data_dir, manifest=manifest, ) - if dataset in manifest.datasets: + if manifest.bundled_dataset_key(dataset) in manifest.datasets: return _use_bundle_dataset( country_id, dataset, diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index 9b3aa3827..601762482 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -27,6 +27,14 @@ PYPI_REQUEST_TIMEOUT_SECONDS = 30 # Producer paths for these artifacts are relative to the release directory. RELEASE_SCOPED_ARTIFACT_KINDS = frozenset({"diagnostics", "evidence"}) +# Public selectors and their counterpart keys in earlier and newer bundles. +# This index is for lookup only. The manifest records the actual artifact key. +US_MICROCOSM_DATASET_ALIASES = { + "microcosm_us_2024": "populace_us_2024", + "populace_us_2024": "microcosm_us_2024", + "microcosm_us_2024_acs_local": "populace_us_2024_acs_local", + "populace_us_2024_acs_local": "microcosm_us_2024_acs_local", +} #: Microcosm publishes each build once, in its own folder #: ``releases///``, and never changes it afterwards; a root @@ -460,6 +468,14 @@ class CountryReleaseManifest(BaseModel): source_sha256: Optional[str] = Field(default=None, exclude=True) """Byte sha256 of the bundled manifest before Pydantic normalization.""" + def bundled_dataset_key(self, dataset: str) -> str: + """Resolve a public selector to the key recorded in this bundle.""" + if self.country_id == "us" and dataset not in self.datasets: + alias = US_MICROCOSM_DATASET_ALIASES.get(dataset) + if alias is not None and alias in self.datasets: + return alias + return dataset + @property def default_dataset_uri(self) -> str: if ( @@ -1330,7 +1346,7 @@ def resolve_dataset_reference(country_id: str, dataset: str) -> str: return dataset manifest = get_release_manifest(country_id) - path_reference = manifest.datasets.get(dataset) + path_reference = manifest.datasets.get(manifest.bundled_dataset_key(dataset)) if path_reference is not None: return build_hf_uri( repo_id=path_reference.repo_id or manifest.data_package.repo_id, @@ -1366,7 +1382,7 @@ def resolve_managed_dataset_reference( release manifest. Callers can: - omit `dataset` to use the certified default dataset for the bundle - - pass a logical dataset name present in the bundled/data-release manifests + - pass a logical dataset name in the bundle or its supported alias Direct URLs or raw Hugging Face references are treated as unmanaged unless `allow_unmanaged=True` is set explicitly. @@ -1376,7 +1392,7 @@ def resolve_managed_dataset_reference( if dataset is None: return manifest.default_dataset_uri - if dataset in manifest.datasets: + if manifest.bundled_dataset_key(dataset) in manifest.datasets: return resolve_dataset_reference(country_id, dataset) local_path = Path(dataset).expanduser() diff --git a/src/policyengine/tax_benefit_models/us/datasets.py b/src/policyengine/tax_benefit_models/us/datasets.py index 320a538a9..287502a30 100644 --- a/src/policyengine/tax_benefit_models/us/datasets.py +++ b/src/policyengine/tax_benefit_models/us/datasets.py @@ -18,6 +18,7 @@ materialize_dataset, ) from policyengine.provenance.manifest import ( + US_MICROCOSM_DATASET_ALIASES, dataset_logical_name, get_release_manifest, resolve_dataset_reference, @@ -422,7 +423,8 @@ def create_datasets( data_folder: Directory to save the dataset files Returns: - Dictionary mapping dataset keys (e.g., "populace_us_2024") to PolicyEngineUSDataset objects + Datasets keyed by selector and year for named Microcosm US datasets, + or by artifact filename stem and year otherwise. """ from policyengine_us import Microsimulation @@ -438,6 +440,9 @@ def create_datasets( data_dir=Path(data_folder), ) dataset_stem = source.name + result_stem = dataset_stem + if dataset is not None and dataset in US_MICROCOSM_DATASET_ALIASES: + result_stem = dataset sim = Microsimulation(dataset=source.path, spm=resolve_spm_selection()) # The register the mapping below uses, stamped into each year file. stamp = year_file_stamp() @@ -621,7 +626,7 @@ def create_datasets( # whose write stopped part-way has no stamp and is cut again. write_year_file_stamp(us_dataset.filepath, stamp) - dataset_key = f"{dataset_stem}_{year}" + dataset_key = f"{result_stem}_{year}" result[dataset_key] = us_dataset return result @@ -657,8 +662,8 @@ def _managed_source_sha256(dataset: Optional[str]) -> Optional[str]: manifest = get_release_manifest("us") if dataset is None or dataset == manifest.default_dataset_uri: name: Optional[str] = manifest.default_dataset - elif dataset in manifest.datasets: - name = dataset + elif (key := manifest.bundled_dataset_key(dataset)) in manifest.datasets: + name = key else: name = next( ( @@ -699,13 +704,17 @@ def load_datasets( data_folder: Directory containing the dataset files Returns: - Dictionary mapping dataset keys (e.g., "populace_us_2024") to PolicyEngineUSDataset objects + Datasets keyed by selector and year for named Microcosm US datasets, + or by artifact filename stem and year otherwise. """ datasets = datasets or [get_release_manifest("us").default_dataset] result = {} for dataset in datasets: resolved_dataset = resolve_dataset_reference("us", dataset) dataset_stem = dataset_logical_name(resolved_dataset) + result_stem = ( + dataset if dataset in US_MICROCOSM_DATASET_ALIASES else dataset_stem + ) for year in years: filepath = f"{data_folder}/{dataset_stem}_year_{year}.h5" if Path(filepath).exists(): @@ -735,7 +744,7 @@ def load_datasets( ) us_dataset.load() - dataset_key = f"{dataset_stem}_{year}" + dataset_key = f"{result_stem}_{year}" result[dataset_key] = us_dataset return result diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index b3bc472b7..c81ac73eb 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -630,8 +630,9 @@ def test__given_unknown_regional_role__then_rejects_it(self): class TestCertifyDataRelease: + @pytest.mark.parametrize("data_producer", ["microcosm", "populace"]) def test__given_fetched_microcosm_manifest__then_updates_bundle_manifest( - self, tmp_path + self, tmp_path, data_producer ): bundle_path = tmp_path / "manifest.json" bundle_path.write_text(json.dumps(_bundle_source_payload()) + "\n") @@ -667,7 +668,7 @@ def test__given_fetched_microcosm_manifest__then_updates_bundle_manifest( ): result = certify_data_release( country="uk", - data_producer="populace", + data_producer=data_producer, manifest_uri=UK_MANIFEST_URI, model_version="2.89.2", bundle_path=bundle_path, @@ -682,6 +683,7 @@ def test__given_fetched_microcosm_manifest__then_updates_bundle_manifest( assert release["source_manifest_uri"] == UK_MANIFEST_URI assert written["packages"]["policyengine-uk"]["version"] == "2.89.2" assert result.data_producer == "populace" + assert "data-producer: microcosm" in result.summary() assert result.dataset_count == 3 assert result.build_id == UK_TAG assert result.bundle_path == bundle_path @@ -771,8 +773,9 @@ def test__given_us_without_data_producer__then_legacy_update_is_explicitly_unsup bundle_path=bundle_path, ) + @pytest.mark.parametrize("producer", ["populace", "microcosm"]) def test__given_bundle_wrapper_regional_args__then_forwards_to_certifier( - self, monkeypatch + self, monkeypatch, producer ): bundle_script = _load_bundle_script(monkeypatch) captured: dict[str, list[str]] = {} @@ -791,7 +794,7 @@ def fake_main(argv: list[str]) -> int: "--country", "us", "--data-producer", - "populace", + producer, "--manifest-uri", MANIFEST_URI, "--regional-manifest-uri", @@ -814,7 +817,7 @@ def fake_main(argv: list[str]) -> int: "--manifest-uri", MANIFEST_URI, "--data-producer", - "populace", + producer, "--model-version", "1.723.0", "--regional-manifest-uri", @@ -1634,8 +1637,9 @@ def fake_main(argv): == "regional-audit.json" ) + @pytest.mark.parametrize("data_producer", ["microcosm", "populace"]) def test__given_bundle_wrapper_empty_reason__then_certifier_refuses_it( - self, monkeypatch, tmp_path, capsys + self, monkeypatch, tmp_path, capsys, data_producer ): bundle_script = _load_bundle_script(monkeypatch) spec = importlib.util.spec_from_file_location( @@ -1673,6 +1677,8 @@ def certify_temporary_bundle(**kwargs): "certify-data", "--country", "uk", + "--data-producer", + data_producer, "--manifest-uri", UK_NATIONAL_URI, "--model-version", @@ -1691,6 +1697,7 @@ def certify_temporary_bundle(**kwargs): assert isinstance(error.value.__context__, CertificationError) assert "needs a reason" in str(error.value.__context__) assert capsys.readouterr().out == "" + assert captured["data_producer"] == data_producer assert captured["regional_model_mismatch_reason"] == "" assert bundle_path.read_bytes() == original_bundle diff --git a/tests/test_microcosm_compatibility.py b/tests/test_microcosm_compatibility.py index 89f96a7ec..816396fc9 100644 --- a/tests/test_microcosm_compatibility.py +++ b/tests/test_microcosm_compatibility.py @@ -13,6 +13,18 @@ MicrocosmDataProducerCertificationStrategy, certification_strategy, ) +from policyengine.provenance.dataset_materialization import ( + MaterializedDataset, + materialize_dataset, +) +from policyengine.provenance.manifest import ( + get_release_manifest, + resolve_dataset_reference, + resolve_managed_dataset_reference, +) +from policyengine.tax_benefit_models.common.model_version import ( + build_runtime_dataset_provenance, +) @pytest.mark.parametrize( @@ -75,7 +87,13 @@ def test_previous_public_names_resolve_to_current_objects( @pytest.mark.parametrize( "country,data_producer", - [("uk", None), ("uk", "populace"), ("us", "populace")], + [ + ("uk", None), + ("uk", "populace"), + ("us", "populace"), + ("uk", "microcosm"), + ("us", "microcosm"), + ], ) def test_current_certification_strategy_preserves_the_published_producer( country, data_producer @@ -86,6 +104,68 @@ def test_current_certification_strategy_preserves_the_published_producer( assert strategy.data_producer == "populace" +@pytest.mark.parametrize( + "current_name,legacy_name", + [ + ("microcosm_us_2024", "populace_us_2024"), + ("microcosm_us_2024_acs_local", "populace_us_2024_acs_local"), + ], +) +def test_us_dataset_alias_preserves_bundle_provenance( + monkeypatch, tmp_path, current_name, legacy_name +): + from policyengine.provenance import dataset_materialization + + def verified_file(spec): + return MaterializedDataset( + data_package_name=spec.data_package_name, + repo_type=spec.repo_type, + revision=spec.revision, + source_uri=spec.source_uri, + sha256=spec.sha256, + path=spec.destination, + ) + + monkeypatch.setattr( + dataset_materialization, "_reuse_or_download_bundle_files", verified_file + ) + + manifest = get_release_manifest("us") + bundle_key = manifest.bundled_dataset_key(current_name) + reference = manifest.datasets[bundle_key] + source = materialize_dataset("us", current_name, data_dir=tmp_path) + existing = materialize_dataset("us", legacy_name, data_dir=tmp_path) + + assert source == existing + assert resolve_dataset_reference("us", current_name) == existing.source_uri + assert resolve_managed_dataset_reference("us", current_name) == existing.source_uri + assert source.bundle_dataset.sha256 == reference.sha256 + provenance = build_runtime_dataset_provenance( + source.source_uri, source.path, source.bundle_dataset + ) + assert provenance["runtime_dataset"] == bundle_key + assert provenance["runtime_dataset_sha256"] == reference.sha256 + if manifest.default_dataset == bundle_key: + assert materialize_dataset("us", data_dir=tmp_path) == source + + +def test_previous_us_selector_resolves_when_bundle_uses_current_key(): + manifest = get_release_manifest("us") + legacy = "populace_us_2024" + current = "microcosm_us_2024" + updated = manifest.model_copy( + update={ + "default_dataset": current, + "datasets": { + current: manifest.datasets[manifest.bundled_dataset_key(legacy)] + }, + } + ) + + assert updated.bundled_dataset_key(legacy) == current + assert updated.bundled_dataset_key(current) == current + + @pytest.mark.parametrize( "current_path,legacy_path,expected_path", [ diff --git a/tests/test_us_legacy_spm_integration.py b/tests/test_us_legacy_spm_integration.py index 7d4c8c776..fc100ae14 100644 --- a/tests/test_us_legacy_spm_integration.py +++ b/tests/test_us_legacy_spm_integration.py @@ -598,18 +598,24 @@ def __init__(self, source, requests): self.requests = requests -def test_year_files_carry_the_declarations_for_every_year(tmp_path, legacy_files): +@pytest.mark.parametrize( + "selector", ["populace_us_2024_acs_local", "microcosm_us_2024_acs_local"] +) +def test_year_files_carry_the_declarations_for_every_year( + tmp_path, legacy_files, selector +): data_folder = str(tmp_path / "data") years = [2025, 2026, 2027] - created = create_datasets( - datasets=["populace_us_2024_acs_local"], years=years, data_folder=data_folder - ) + created = create_datasets(datasets=[selector], years=years, data_folder=data_folder) assert legacy_files.requests == [ - "populace_us_2024_acs_local", + selector, LIST_PIN.bundle_dataset, ] for year in years: - dataset = created[f"populace_us_2024_acs_local_{year}"] + dataset = created[f"{selector}_{year}"] + assert ( + Path(dataset.filepath).name == f"populace_us_2024_acs_local_year_{year}.h5" + ) spm_unit = pd.DataFrame(dataset.data.spm_unit).set_index("spm_unit_id") person = pd.DataFrame(dataset.data.person).set_index("person_id") assert spm_unit[UNIVERSE_VARIABLE].tolist() == [ @@ -630,9 +636,10 @@ def test_year_files_carry_the_declarations_for_every_year(tmp_path, legacy_files ] assert LegacySPMRecord.model_validate(dataset.metadata[RECORD_KEY]) == _record() - (loaded,) = load_datasets( - datasets=["populace_us_2024_acs_local"], years=[YEAR], data_folder=data_folder - ).values() + loaded = ensure_datasets( + datasets=[selector], years=[YEAR], data_folder=data_folder + )[f"{selector}_{YEAR}"] + assert legacy_files.requests == [selector, LIST_PIN.bundle_dataset] simulation = Simulation(dataset=loaded, tax_benefit_model_version=pe.us.model) simulation.run() spm_unit = _frame(simulation, "spm_unit") @@ -728,10 +735,17 @@ def test_a_run_straight_over_a_legacy_source_derives_the_declarations( assert restored.output_dataset.metadata[RECORD_KEY]["model_mode"] == "override" +@pytest.mark.parametrize( + "selector", ["populace_us_2024_acs_local", "microcosm_us_2024_acs_local"] +) def test_a_managed_simulation_of_a_legacy_source_applies_the_declarations( - legacy_files, + legacy_files, selector ): - simulation = pe.us.managed_microsimulation(dataset="populace_us_2024_acs_local") + simulation = pe.us.managed_microsimulation(dataset=selector) + assert legacy_files.requests == [selector, LIST_PIN.bundle_dataset] + assert simulation.policyengine_bundle["runtime_dataset"] == ( + "populace_us_2024_acs_local" + ) thresholds = simulation.calculate("spm_unit_spm_threshold", YEAR).values assert np.isnan(thresholds[[1, 2]]).all() assert np.isfinite(thresholds[[0, 3, 4]]).all()