Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/757.added.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Add optional household row filtering to US managed microsimulations while retaining every native projected year and the source dataset provenance.
134 changes: 134 additions & 0 deletions src/policyengine/tax_benefit_models/us/managed_scope.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
"""Scope a native projected US dataset without dropping supplied years."""

from __future__ import annotations

import hashlib

import pandas as pd

from policyengine.core.scoping_strategy import RowFilterStrategy
from policyengine.utils.entity_utils import _resolve_id_column

GROUPS = ("household", "tax_unit", "spm_unit", "family", "marital_unit")
ENTITIES = ("person", *GROUPS)


def _digest(values: pd.Series) -> str:
"""Hash native values, dtype and order without exposing household IDs."""
digest = hashlib.sha256(str(values.dtype).encode())
digest.update(pd.util.hash_pandas_object(values, index=False).values.tobytes())
return digest.hexdigest()


def scope_native_history(dataset, strategy: RowFilterStrategy, variables=None):
"""Return an official native dataset for one fixed, closed cohort.

The caller obtains ``dataset`` from the ordinary native constructor, so
the country package owns projection and its parameter/reform timing.
This function only filters the stored tables. It never calculates a
variable, derives a weight or writes an input file.
"""
from policyengine_us.data import USMultiYearDataset, USSingleYearDataset

if type(strategy) is not RowFilterStrategy:
raise ValueError("Managed scoping supports only RowFilterStrategy.")
if not isinstance(dataset, USMultiYearDataset) or not dataset.datasets:
raise ValueError("Managed scoping requires a native USMultiYearDataset.")

scoped_years = []
reference_ids = None
reference_memberships = None
eternity_inputs = {}
receipt_years = {}
for year, single in sorted(dataset.datasets.items()):
tables = {entity: getattr(single, entity) for entity in ENTITIES}
for entity, table in tables.items():
id_column = f"{entity}_id"
if (
not isinstance(table, pd.DataFrame)
or not table.columns.is_unique
or id_column not in table
or table[id_column].isna().any()
or not table[id_column].is_unique
):
raise ValueError(f"Invalid native {entity} IDs or columns for {year}.")

person = tables["person"]
household_column = _resolve_id_column(person, "household")
for group in GROUPS:
column = _resolve_id_column(person, group)
if not person[column].isin(tables[group][f"{group}_id"]).all():
raise ValueError(f"Dangling {group} membership for {year}.")
if set(person[column]) != set(tables[group][f"{group}_id"]):
raise ValueError(f"Managed scoping requires members in every {group}.")
# A group spanning selected and unselected households cannot be
# calculated on either subset without changing its population.
membership = pd.DataFrame(
{"group": person[column], "household": person[household_column]}
)
if (
membership.groupby("group", dropna=False)
.household.nunique()
.gt(1)
.any()
):
raise ValueError(f"Managed scoping requires household-closed {group}s.")

scoped = strategy.apply(tables, list(GROUPS), int(year))
# RowFilterStrategy's temporary MicroDataFrame metadata never supplies
# statistical weights here. Keep only the original table columns.
scoped = {entity: pd.DataFrame(scoped[entity]).copy() for entity in ENTITIES}
for table in scoped.values():
for name in table:
variable = (variables or {}).get(name)
if (
variable is not None
and str(variable.definition_period) == "eternity"
):
values = table[name].reset_index(drop=True)
if name in eternity_inputs and not values.equals(
eternity_inputs[name]
):
raise ValueError(
f"Conflicting ETERNITY input {name} across years."
)
eternity_inputs[name] = values
ids = {
entity: table[f"{entity}_id"].reset_index(drop=True)
for entity, table in scoped.items()
}
scoped_memberships = {
group: scoped["person"][
_resolve_id_column(scoped["person"], group)
].reset_index(drop=True)
for group in GROUPS
}
if reference_ids is not None and (
any(not ids[entity].equals(reference_ids[entity]) for entity in ENTITIES)
or any(
not scoped_memberships[group].equals(reference_memberships[group])
for group in GROUPS
)
):
raise ValueError(
"Managed scoping requires the same ordered cohort every year."
)
reference_ids = ids
reference_memberships = scoped_memberships
scoped_years.append(USSingleYearDataset(**scoped, time_period=int(year)))
receipt_years[str(year)] = {
"source_rows": {entity: len(table) for entity, table in tables.items()},
"scoped_rows": {entity: len(table) for entity, table in scoped.items()},
"id_sha256": {entity: _digest(values) for entity, values in ids.items()},
"household_weight_sha256": (
_digest(scoped["household"]["household_weight"])
if "household_weight" in scoped["household"]
else None
),
}
return USMultiYearDataset(datasets=scoped_years), {
"strategy": strategy.model_dump(mode="json"),
"cache_key": strategy.cache_key,
"years": receipt_years,
"method": "native_projected_history_row_filter",
}
44 changes: 42 additions & 2 deletions src/policyengine/tax_benefit_models/us/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from microdf import MicroDataFrame

from policyengine.core import TaxBenefitModel
from policyengine.core.scoping_strategy import RowFilterStrategy
from policyengine.provenance.dataset_materialization import (
materialize_dataset,
)
Expand Down Expand Up @@ -537,6 +538,7 @@ def managed_microsimulation(
dataset: Optional[str] = None,
allow_unmanaged: bool = False,
spm: Optional[SPMSelection] = None,
scoping_strategy: Optional[RowFilterStrategy] = None,
**kwargs,
):
"""Construct a country-package Microsimulation pinned to this bundle.
Expand All @@ -548,6 +550,11 @@ def managed_microsimulation(
Stored inputs the engine has since renamed are mapped onto their live
inputs (see ``legacy_inputs.LEGACY_INPUT_RENAMES``), and the renames
applied are recorded as ``policyengine_bundle["legacy_input_renames"]``.

``scoping_strategy=RowFilterStrategy(...)`` selects a fixed, closed
household cohort from every native projected year before financial
calculations. This retains earlier supplied income for lookbacks. The
unscoped constructor continues to own projection and reform timing.
"""

from policyengine_us import Microsimulation
Expand All @@ -564,15 +571,46 @@ def managed_microsimulation(
dataset,
allow_unmanaged=allow_unmanaged,
)
microsim = Microsimulation(dataset=source.path, spm=selection, **kwargs)
legacy_input_renames = apply_legacy_input_renames_to_microsimulation(microsim)
# BACKWARD COMPATIBILITY (#597, #598): a legacy certified source carries
# no SPM declarations, so derive them from its stored evidence and set
# them on the simulation and its branches (see ``legacy_spm``).
legacy_spm_record = None
legacy_source = legacy_spm_source(
source.bundle_dataset.sha256 if source.bundle_dataset else None
)
if scoping_strategy is not None:
if type(scoping_strategy) is not RowFilterStrategy:
raise ValueError("Managed scoping supports only RowFilterStrategy.")
if legacy_source is not None:
raise ValueError(
"Managed scoping does not support legacy SPM declarations."
)

microsim = Microsimulation(dataset=source.path, spm=selection, **kwargs)
scoping_record = None
if scoping_strategy is not None:
import gc

from policyengine.tax_benefit_models.us.managed_scope import (
scope_native_history,
)

scoped_dataset, scoping_record = scope_native_history(
microsim.dataset, scoping_strategy, microsim.tax_benefit_system.variables
)
# Release the national input-only simulation before constructing the
# scoped simulation. All retained data belong to copied scoped tables.
del microsim
gc.collect()
scoped_kwargs = dict(kwargs)
# A native multi-year dataset already contains the exact projected
# horizon; country constructors accept this option only for single-year
# inputs. Do not project or extend the scoped data a second time.
scoped_kwargs.pop("dataset_end_year", None)
microsim = Microsimulation(
dataset=scoped_dataset, spm=selection, **scoped_kwargs
)
legacy_input_renames = apply_legacy_input_renames_to_microsimulation(microsim)
if legacy_source is not None:
declarations, years = derive_from_stored_tables(
legacy_source, microsim.dataset, data_dir=Path(source.path).parent
Expand All @@ -596,6 +634,8 @@ def managed_microsimulation(
)
if legacy_spm_record is not None:
microsim.policyengine_bundle[LEGACY_SPM_RECORD_KEY] = legacy_spm_record
if scoping_record is not None:
microsim.policyengine_bundle["scoping"] = scoping_record
return microsim


Expand Down
Loading
Loading