From 6d839a148cee47981e8289de37da276f3e7beeec Mon Sep 17 00:00:00 2001 From: ZimingHua Date: Wed, 7 Oct 2026 15:51:11 -0400 Subject: [PATCH 1/5] Add country situation preprocessing and membership provenance --- changelog.d/situation-preprocessing.added.md | 1 + docs/python_api/simulations.md | 18 ++++- policyengine_core/simulations/simulation.py | 15 +++++ .../simulations/simulation_builder.py | 10 +++ .../taxbenefitsystems/tax_benefit_system.py | 14 ++++ .../test_dataset_unknown_column_warning.py | 1 + tests/core/test_simulation_builder.py | 66 +++++++++++++++++++ 7 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 changelog.d/situation-preprocessing.added.md diff --git a/changelog.d/situation-preprocessing.added.md b/changelog.d/situation-preprocessing.added.md new file mode 100644 index 000000000..a8dfee87d --- /dev/null +++ b/changelog.d/situation-preprocessing.added.md @@ -0,0 +1 @@ +Add country-specific entity-situation preprocessing and retain which group memberships were explicitly supplied by a situation or dataset. diff --git a/docs/python_api/simulations.md b/docs/python_api/simulations.md index 0b65c1dd9..ab300a966 100644 --- a/docs/python_api/simulations.md +++ b/docs/python_api/simulations.md @@ -24,6 +24,23 @@ The `policyengine_core.simulations` module contains the definition of `Simulatio ## SimulationBuilder +Country models can override `TaxBenefitSystem.preprocess_situation(situation, +default_period)` to fill missing groups before an entity-form situation is +built. The hook receives a deep copy after singular entity aliases have been +expanded, and returns a situation for the builder's normal validation. Both +`Simulation(situation=...)` and the YAML runner use this path. Dataset and +variable-only construction do not call the hook. The default hook returns its +input unchanged. + +`Simulation.input_group_entities` is an immutable set of singular group keys +supplied before preprocessing. For datasets, it identifies groups with person +membership columns, before loader defaults are inserted. An explicitly empty +group mapping counts as supplied. Synthesized groups and the singleton groups +created for variable-only input do not. Country formulas can use this provenance +to distinguish reported relationships from defaults without inspecting group +sizes or identifier values. Population-only construction does not assert input +provenance; simulation clones preserve their source's set. + ```{eval-rst} .. autoclass:: policyengine_core.simulations.simulation_builder.SimulationBuilder :members: @@ -31,4 +48,3 @@ The `policyengine_core.simulations` module contains the definition of `Simulatio :inherited-members: :show-inheritance: ``` - diff --git a/policyengine_core/simulations/simulation.py b/policyengine_core/simulations/simulation.py index 123a5f1ae..9637f4f67 100644 --- a/policyengine_core/simulations/simulation.py +++ b/policyengine_core/simulations/simulation.py @@ -220,6 +220,15 @@ class Simulation: is_over_dataset: bool = False """Whether this simulation is built over a dataset.""" + input_group_entities: frozenset[str] = frozenset() + """Singular keys of group entities explicitly supplied by the caller. + + Entity-form situations record groups before country preprocessing; + datasets record groups with person membership columns before defaults + are inserted. Variable-only and population-only construction do not + declare input groups. This immutable provenance survives cloning. + """ + macro_cache_read: bool = False """Whether to read from the macro cache.""" @@ -577,6 +586,12 @@ def get_eternity_array(name): entity_ids = get_eternity_array(entity_id_field) builder.declare_person_entity(person_entity.key, entity_ids) + membership_columns = {column.split("__")[0] for column in data} + self.input_group_entities = frozenset( + group_entity.key + for group_entity in self.tax_benefit_system.group_entities + if f"{person_entity.key}_{group_entity.key}_id" in membership_columns + ) for group_entity in self.tax_benefit_system.group_entities: entity_id_field = f"{group_entity.key}_id" if self.dataset.data_format != Dataset.FLAT_FILE: diff --git a/policyengine_core/simulations/simulation_builder.py b/policyengine_core/simulations/simulation_builder.py index e3dee0cc5..f728f99dc 100644 --- a/policyengine_core/simulations/simulation_builder.py +++ b/policyengine_core/simulations/simulation_builder.py @@ -126,6 +126,15 @@ def build_from_entities( simulation.get_variable_population(variable_name).entity, ) + check_type(input_dict, dict, ["error"]) + simulation.input_group_entities = frozenset( + entity.key + for entity in tax_benefit_system.group_entities + if input_dict.get(entity.plural) is not None + ) + input_dict = tax_benefit_system.preprocess_situation( + input_dict, self.default_period + ) check_type(input_dict, dict, ["error"]) axes = input_dict.pop("axes", None) @@ -243,6 +252,7 @@ def build_default_simulation( tax_benefit_system=tax_benefit_system, populations=tax_benefit_system.instantiate_entities(), ) + simulation.input_group_entities = frozenset() for population in simulation.populations.values(): population.count = count population.ids = np.array(range(count)) diff --git a/policyengine_core/taxbenefitsystems/tax_benefit_system.py b/policyengine_core/taxbenefitsystems/tax_benefit_system.py index b7073081d..7c45dfdd3 100644 --- a/policyengine_core/taxbenefitsystems/tax_benefit_system.py +++ b/policyengine_core/taxbenefitsystems/tax_benefit_system.py @@ -101,6 +101,20 @@ class TaxBenefitSystem: modelled_policies: str = None """A YAML filepath containing metadata describing the modelled policies.""" + def preprocess_situation(self, situation: dict, default_period=None) -> dict: + """Normalize entity-form input before the builder creates populations. + + Country models can override this to fill missing entity definitions. + The builder passes a deep copy of the caller's situation, with singular + entity aliases expanded and the default input period (possibly None). + Return an entity-form situation for the builder's usual validation. + Dataset and variable-only inputs do not use this hook. + + ``Simulation.input_group_entities`` records the groups supplied by + the caller before this hook runs, separately from inferred groups. + """ + return situation + def __init__(self, entities: Sequence[Entity] = None, reform=None) -> None: if entities is None: entities = self.entities diff --git a/tests/core/test_dataset_unknown_column_warning.py b/tests/core/test_dataset_unknown_column_warning.py index 7450b92dc..d621cfa70 100644 --- a/tests/core/test_dataset_unknown_column_warning.py +++ b/tests/core/test_dataset_unknown_column_warning.py @@ -38,6 +38,7 @@ def test__given_unknown_dataset_column__then_warns_and_still_loads(caplog): # And the known columns still load normally np.testing.assert_array_equal(salary.values, np.array([100.0, 200.0, 300.0])) + assert simulation.input_group_entities == frozenset({"household"}) def test__given_only_known_columns__then_no_unknown_column_warning(caplog): diff --git a/tests/core/test_simulation_builder.py b/tests/core/test_simulation_builder.py index f838d6b3c..f11f34dc8 100644 --- a/tests/core/test_simulation_builder.py +++ b/tests/core/test_simulation_builder.py @@ -1,4 +1,5 @@ import datetime +import copy from typing import Iterable import pytest @@ -95,6 +96,71 @@ def test_explicit_singular_entities(tax_benefit_system): } +def test_country_preprocessing_preserves_caller_input_and_group_provenance( + tax_benefit_system, monkeypatch +): + seen_periods = [] + + def preprocess(situation, default_period): + seen_periods.append(default_period) + situation.setdefault( + "households", {"inferred": {"parents": ["Alicia", "Javier"]}} + ) + return situation + + monkeypatch.setattr(tax_benefit_system, "preprocess_situation", preprocess) + situation = {"persons": {"Alicia": {}, "Javier": {}}} + original = copy.deepcopy(situation) + builder = SimulationBuilder() + builder.set_default_period("2024-01") + inferred = builder.build_from_dict(tax_benefit_system, situation) + assert list(inferred.household.ids) == ["inferred"] + assert inferred.input_group_entities == frozenset() + assert situation == original + assert seen_periods == ["2024-01"] + + # Singular aliases are normalized before the hook; explicit membership + # remains distinguishable even if it has the same shape as the default. + explicit = builder.build_from_dict( + tax_benefit_system, + {**situation, "household": {"parents": ["Alicia", "Javier"]}}, + ) + assert explicit.input_group_entities == frozenset({"household"}) + assert explicit.clone().input_group_entities == explicit.input_group_entities + + # Reusing a simulation for variable-only construction resets provenance + # and does not run entity preprocessing. + calls = len(seen_periods) + builder.build_default_simulation(tax_benefit_system, 2, explicit) + assert explicit.input_group_entities == frozenset() + assert len(seen_periods) == calls + + +def test_country_preprocessing_applies_to_simulation_api( + tax_benefit_system, monkeypatch +): + def preprocess(situation, default_period): + situation["households"] = { + name: {"parents": [name]} for name in situation["persons"] + } + return situation + + monkeypatch.setattr(tax_benefit_system, "preprocess_situation", preprocess) + simulation = Simulation( + tax_benefit_system=tax_benefit_system, + situation={"persons": {"Alicia": {}, "Javier": {}}}, + ) + assert simulation.household.members_entity_id.tolist() == [0, 1] + assert simulation.input_group_entities == frozenset() + + +def test_explicit_empty_group_mapping_is_recorded(tax_benefit_system): + simulation = SimulationBuilder().build_from_dict( + tax_benefit_system, {"persons": {"Alicia": {}}, "households": {}} + ) + assert simulation.input_group_entities == frozenset({"household"}) + + def test_add_person_entity(persons): persons_json = {"Alicia": {"salary": {}}, "Javier": {}} simulation_builder = SimulationBuilder() From 8537b6ee21dca4b5a537aedb60ae4fcbdabe3b99 Mon Sep 17 00:00:00 2001 From: ZimingHua Date: Fri, 30 Oct 2026 21:55:39 -0400 Subject: [PATCH 2/5] Delegate the situation hook through reforms and fix provenance gaps A Reform inherited TaxBenefitSystem's no-op preprocess_situation, so reformed systems, including the YAML runner's inline-parameter and reforms: cases, skipped the country's hook. Reform now delegates to its baseline. Flat files that declare a group only through its ID column now record it as supplied, matching the loader. IndividualSim's placeholder and default groups are no longer recorded as supplied. A hook that returns something other than a dict raises a TypeError naming the hook. The docs describe dataset provenance, axes and default_period accurately. Co-Authored-By: Claude Opus 5.5 --- docs/python_api/simulations.md | 28 +++++---- policyengine_core/reforms/reform.py | 5 ++ .../simulations/individual_sim.py | 10 +++ policyengine_core/simulations/simulation.py | 17 +++-- .../simulations/simulation_builder.py | 6 +- .../taxbenefitsystems/tax_benefit_system.py | 8 ++- tests/core/test_individual_sim.py | 26 ++++++++ tests/core/test_join_with_persons_identity.py | 3 + tests/core/test_simulation_builder.py | 63 ++++++++++++++++++- 9 files changed, 146 insertions(+), 20 deletions(-) create mode 100644 tests/core/test_individual_sim.py diff --git a/docs/python_api/simulations.md b/docs/python_api/simulations.md index ab300a966..a2b4a3f48 100644 --- a/docs/python_api/simulations.md +++ b/docs/python_api/simulations.md @@ -27,19 +27,25 @@ The `policyengine_core.simulations` module contains the definition of `Simulatio Country models can override `TaxBenefitSystem.preprocess_situation(situation, default_period)` to fill missing groups before an entity-form situation is built. The hook receives a deep copy after singular entity aliases have been -expanded, and returns a situation for the builder's normal validation. Both -`Simulation(situation=...)` and the YAML runner use this path. Dataset and -variable-only construction do not call the hook. The default hook returns its -input unchanged. +expanded, with any `axes` still present, and must return a dict for the +builder's normal validation. `default_period` is the builder's default period +as set, so it may be `None`, a string, an integer or a `Period`. Both +`Simulation(situation=...)` and the YAML runner use this path, and a `Reform` +delegates the hook to its baseline. Dataset and variable-only construction do +not call the hook. The default hook returns its input unchanged. `Simulation.input_group_entities` is an immutable set of singular group keys -supplied before preprocessing. For datasets, it identifies groups with person -membership columns, before loader defaults are inserted. An explicitly empty -group mapping counts as supplied. Synthesized groups and the singleton groups -created for variable-only input do not. Country formulas can use this provenance -to distinguish reported relationships from defaults without inspecting group -sizes or identifier values. Population-only construction does not assert input -provenance; simulation clones preserve their source's set. +supplied before preprocessing. An explicitly empty group mapping counts as +supplied. Synthesized groups, the singleton groups created for variable-only +input, and the placeholder and default groups `IndividualSim` adds do not. For +datasets, it records groups declared by person membership columns or, in flat +files, by the group's ID column on each person row. Core's loaders require +membership for every group, so a loaded dataset records all of them; a country +that overrides `Simulation.build_from_dataset` must set the attribute itself. +Country formulas can use this provenance to distinguish reported relationships +from defaults without inspecting group sizes or identifier values. +Population-only construction does not assert input provenance; simulation +clones preserve their source's set. ```{eval-rst} .. autoclass:: policyengine_core.simulations.simulation_builder.SimulationBuilder diff --git a/policyengine_core/reforms/reform.py b/policyengine_core/reforms/reform.py index afcee0358..82f352046 100644 --- a/policyengine_core/reforms/reform.py +++ b/policyengine_core/reforms/reform.py @@ -147,6 +147,11 @@ def __init__(self, baseline: TaxBenefitSystem): def __getattr__(self, attribute): return getattr(self.baseline, attribute) + def preprocess_situation(self, situation: dict, default_period=None) -> dict: + # TaxBenefitSystem defines a no-op hook, so ``__getattr__`` never + # reaches the baseline's override; delegate to it explicitly. + return self.baseline.preprocess_situation(situation, default_period) + @property def full_key(self) -> str: key = self.key diff --git a/policyengine_core/simulations/individual_sim.py b/policyengine_core/simulations/individual_sim.py index c00933c87..4d896343f 100644 --- a/policyengine_core/simulations/individual_sim.py +++ b/policyengine_core/simulations/individual_sim.py @@ -54,6 +54,13 @@ def __init__( setattr(self, f"add_{entity}", partial(self.add_data, entity=entity)) def build(self): + # The empty mappings seeded in __init__ and the required groups + # filled in below are not groups the caller supplied. + supplied_groups = frozenset( + entity.key + for entity in self.system.group_entities + if self.situation_data.get(entity.plural) + ) if self.required_entities is not None: # Check for missing entities entities = {entity.key: entity for entity in self.system.entities} @@ -80,6 +87,9 @@ def build(self): situation=self.situation_data, reform=self.reform, ) + for simulation in (self.simulation, getattr(self.simulation, "baseline", None)): + if simulation is not None: + simulation.input_group_entities = supplied_groups self.simulation.trace = True self.sim = self.simulation diff --git a/policyengine_core/simulations/simulation.py b/policyengine_core/simulations/simulation.py index d2b869073..7490c1d35 100644 --- a/policyengine_core/simulations/simulation.py +++ b/policyengine_core/simulations/simulation.py @@ -728,10 +728,13 @@ class Simulation: input_group_entities: frozenset[str] = frozenset() """Singular keys of group entities explicitly supplied by the caller. - Entity-form situations record groups before country preprocessing; - datasets record groups with person membership columns before defaults - are inserted. Variable-only and population-only construction do not - declare input groups. This immutable provenance survives cloning. + Entity-form situations record groups before country preprocessing. + Datasets record groups declared by person membership columns or, in + flat files, by the group's ID column on each person row; Core's loaders + require membership for every group, so a loaded dataset records all of + them, and a country that overrides ``build_from_dataset`` must set this + itself. Variable-only and population-only construction do not declare + input groups. This immutable provenance survives cloning. """ macro_cache_read: bool = False @@ -1143,11 +1146,17 @@ def get_eternity_array(name): entity_ids = get_eternity_array(entity_id_field) builder.declare_person_entity(person_entity.key, entity_ids) + # A dataset declares a group through its person membership column + # or, in a flat file, the group's ID column on each person row. membership_columns = {column.split("__")[0] for column in data} self.input_group_entities = frozenset( group_entity.key for group_entity in self.tax_benefit_system.group_entities if f"{person_entity.key}_{group_entity.key}_id" in membership_columns + or ( + self.dataset.data_format == Dataset.FLAT_FILE + and f"{group_entity.key}_id" in membership_columns + ) ) for group_entity in self.tax_benefit_system.group_entities: entity_id_field = f"{group_entity.key}_id" diff --git a/policyengine_core/simulations/simulation_builder.py b/policyengine_core/simulations/simulation_builder.py index f98b2e3dd..c58acfa6c 100644 --- a/policyengine_core/simulations/simulation_builder.py +++ b/policyengine_core/simulations/simulation_builder.py @@ -234,7 +234,11 @@ def build_from_entities( input_dict = tax_benefit_system.preprocess_situation( input_dict, self.default_period ) - check_type(input_dict, dict, ["error"]) + if not isinstance(input_dict, dict): + raise TypeError( + f"{type(tax_benefit_system).__name__}.preprocess_situation must " + f"return a dict, not {type(input_dict).__name__}." + ) axes = input_dict.pop("axes", None) unexpected_entities = [ diff --git a/policyengine_core/taxbenefitsystems/tax_benefit_system.py b/policyengine_core/taxbenefitsystems/tax_benefit_system.py index cc840a374..b6be2543f 100644 --- a/policyengine_core/taxbenefitsystems/tax_benefit_system.py +++ b/policyengine_core/taxbenefitsystems/tax_benefit_system.py @@ -155,9 +155,11 @@ def preprocess_situation(self, situation: dict, default_period=None) -> dict: Country models can override this to fill missing entity definitions. The builder passes a deep copy of the caller's situation, with singular - entity aliases expanded and the default input period (possibly None). - Return an entity-form situation for the builder's usual validation. - Dataset and variable-only inputs do not use this hook. + entity aliases expanded and any ``axes`` still present, and its default + period as set (None, a string, an integer or a ``Period``). Return a + dict holding an entity-form situation for the builder's usual + validation. A ``Reform`` delegates this hook to its baseline. Dataset + and variable-only inputs do not use this hook. ``Simulation.input_group_entities`` records the groups supplied by the caller before this hook runs, separately from inferred groups. diff --git a/tests/core/test_individual_sim.py b/tests/core/test_individual_sim.py new file mode 100644 index 000000000..ec5fe8b2d --- /dev/null +++ b/tests/core/test_individual_sim.py @@ -0,0 +1,26 @@ +from policyengine_core.country_template import CountryTaxBenefitSystem +from policyengine_core.simulations.individual_sim import IndividualSim + + +class TemplateIndividualSim(IndividualSim): + tax_benefit_system = CountryTaxBenefitSystem + required_entities = ["household"] + default_roles = {"household": "parent"} + + +def test_placeholder_and_default_groups_are_not_supplied(): + # __init__ seeds an empty mapping for every group and build() fills the + # required household with every person; neither was supplied by the + # caller. + simulation = TemplateIndividualSim(year=2024) + simulation.add_person(name="Alicia") + simulation.build() + assert simulation.simulation.input_group_entities == frozenset() + + +def test_added_groups_are_supplied(): + simulation = TemplateIndividualSim(year=2024) + simulation.add_person(name="Alicia") + simulation.add_household(name="home", parents=["Alicia"]) + simulation.build() + assert simulation.simulation.input_group_entities == frozenset({"household"}) diff --git a/tests/core/test_join_with_persons_identity.py b/tests/core/test_join_with_persons_identity.py index 6ff739a88..d75c3e119 100644 --- a/tests/core/test_join_with_persons_identity.py +++ b/tests/core/test_join_with_persons_identity.py @@ -99,6 +99,7 @@ class InteriorEmptyHousehold(Dataset): household = simulation.populations["household"] assert household.members_entity_id.tolist() == [0, 2] + assert simulation.input_group_entities == frozenset({"household"}) assert simulation.calculate("rent", "2024-01", map_to="person").tolist() == [ 100.0, 300.0, @@ -189,6 +190,8 @@ def test_flat_file_with_only_a_household_id_column_reads_it_as_membership( household = simulation.populations["household"] assert household.count == 2 assert household.members_entity_id.tolist() == [0, 0, 1] + # The ID column declares the households, so they count as supplied. + assert simulation.input_group_entities == frozenset({"household"}) def test_flat_file_default_roles_cover_every_person(): diff --git a/tests/core/test_simulation_builder.py b/tests/core/test_simulation_builder.py index f866ec300..bfeb7e177 100644 --- a/tests/core/test_simulation_builder.py +++ b/tests/core/test_simulation_builder.py @@ -8,12 +8,20 @@ import pytest from policyengine_core import periods, tools -from policyengine_core.country_template import entities, situation_examples +from policyengine_core.country_template import ( + CountryTaxBenefitSystem, + entities, + situation_examples, +) +from policyengine_core.entities import build_entity from policyengine_core.enums import Enum from policyengine_core.errors import SituationParsingError from policyengine_core.populations import Population +from policyengine_core.reforms import Reform from policyengine_core.simulations import Simulation, SimulationBuilder +from policyengine_core.taxbenefitsystems import TaxBenefitSystem from policyengine_core.tools import test_runner +from policyengine_core.tools.test_runner import _get_tax_benefit_system from policyengine_core.variables import Variable @@ -191,6 +199,59 @@ def test_explicit_empty_group_mapping_is_recorded(tax_benefit_system): assert simulation.input_group_entities == frozenset({"household"}) +def test_only_supplied_groups_are_recorded(): + person = build_entity("person", "persons", "", is_person=True) + household = build_entity( + "household", "households", "", roles=[{"key": "member", "plural": "members"}] + ) + family = build_entity( + "family", "families", "", roles=[{"key": "member", "plural": "members"}] + ) + simulation = SimulationBuilder().build_from_dict( + TaxBenefitSystem([person, household, family]), + {"persons": {"Alicia": {}}, "households": {"home": {"members": ["Alicia"]}}}, + ) + assert simulation.input_group_entities == frozenset({"household"}) + + +class _PreprocessingTaxBenefitSystem(CountryTaxBenefitSystem): + def preprocess_situation(self, situation, default_period=None): + situation.setdefault("households", {"inferred": {"parents": ["Alicia"]}}) + return situation + + +class _NoOpReform(Reform): + def apply(self): + pass + + +def test_reform_delegates_preprocessing_to_its_baseline(): + # Reform inherits TaxBenefitSystem's no-op hook, so a reform, including + # the one the YAML runner builds for inline parameters, must delegate. + baseline = _PreprocessingTaxBenefitSystem() + for system in ( + _NoOpReform(baseline), + _get_tax_benefit_system(baseline, [_NoOpReform], []), + ): + simulation = SimulationBuilder().build_from_dict( + system, {"persons": {"Alicia": {}}} + ) + assert list(simulation.household.ids) == ["inferred"] + assert simulation.input_group_entities == frozenset() + + +def test_country_preprocessing_must_return_a_dict(tax_benefit_system, monkeypatch): + monkeypatch.setattr( + tax_benefit_system, + "preprocess_situation", + lambda situation, default_period: None, + ) + with pytest.raises(TypeError, match="preprocess_situation must return a dict"): + SimulationBuilder().build_from_dict( + tax_benefit_system, {"persons": {"Alicia": {}}} + ) + + def test_add_person_entity(persons): persons_json = {"Alicia": {"salary": {}}, "Javier": {}} simulation_builder = SimulationBuilder() From 477ea8c10cf807c667f2d16918fee30e49f7fbf5 Mon Sep 17 00:00:00 2001 From: ZimingHua Date: Thu, 12 Nov 2026 12:38:39 -0500 Subject: [PATCH 3/5] Keep only the situation preprocessing hook Remove Simulation.input_group_entities and its recording in the builder, build_from_dataset and IndividualSim. The country model now builds marital units that agree with its tax roles, so no formula needs to know which groups the caller supplied. Datasets still bypass the hook. Co-Authored-By: Claude Opus 5.5 --- changelog.d/situation-preprocessing.added.md | 2 +- docs/python_api/simulations.md | 13 ------ .../simulations/individual_sim.py | 10 ----- policyengine_core/simulations/simulation.py | 24 ----------- .../simulations/simulation_builder.py | 6 --- .../taxbenefitsystems/tax_benefit_system.py | 3 -- .../test_dataset_unknown_column_warning.py | 1 - tests/core/test_individual_sim.py | 26 ------------ tests/core/test_join_with_persons_identity.py | 3 -- tests/core/test_simulation_builder.py | 42 +++---------------- 10 files changed, 6 insertions(+), 124 deletions(-) delete mode 100644 tests/core/test_individual_sim.py diff --git a/changelog.d/situation-preprocessing.added.md b/changelog.d/situation-preprocessing.added.md index a8dfee87d..2a53d6456 100644 --- a/changelog.d/situation-preprocessing.added.md +++ b/changelog.d/situation-preprocessing.added.md @@ -1 +1 @@ -Add country-specific entity-situation preprocessing and retain which group memberships were explicitly supplied by a situation or dataset. +Add a `TaxBenefitSystem.preprocess_situation` hook so country models can fill missing groups in an entity-form situation before it is built. diff --git a/docs/python_api/simulations.md b/docs/python_api/simulations.md index a2b4a3f48..2b4c5b40d 100644 --- a/docs/python_api/simulations.md +++ b/docs/python_api/simulations.md @@ -34,19 +34,6 @@ as set, so it may be `None`, a string, an integer or a `Period`. Both delegates the hook to its baseline. Dataset and variable-only construction do not call the hook. The default hook returns its input unchanged. -`Simulation.input_group_entities` is an immutable set of singular group keys -supplied before preprocessing. An explicitly empty group mapping counts as -supplied. Synthesized groups, the singleton groups created for variable-only -input, and the placeholder and default groups `IndividualSim` adds do not. For -datasets, it records groups declared by person membership columns or, in flat -files, by the group's ID column on each person row. Core's loaders require -membership for every group, so a loaded dataset records all of them; a country -that overrides `Simulation.build_from_dataset` must set the attribute itself. -Country formulas can use this provenance to distinguish reported relationships -from defaults without inspecting group sizes or identifier values. -Population-only construction does not assert input provenance; simulation -clones preserve their source's set. - ```{eval-rst} .. autoclass:: policyengine_core.simulations.simulation_builder.SimulationBuilder :members: diff --git a/policyengine_core/simulations/individual_sim.py b/policyengine_core/simulations/individual_sim.py index 4d896343f..c00933c87 100644 --- a/policyengine_core/simulations/individual_sim.py +++ b/policyengine_core/simulations/individual_sim.py @@ -54,13 +54,6 @@ def __init__( setattr(self, f"add_{entity}", partial(self.add_data, entity=entity)) def build(self): - # The empty mappings seeded in __init__ and the required groups - # filled in below are not groups the caller supplied. - supplied_groups = frozenset( - entity.key - for entity in self.system.group_entities - if self.situation_data.get(entity.plural) - ) if self.required_entities is not None: # Check for missing entities entities = {entity.key: entity for entity in self.system.entities} @@ -87,9 +80,6 @@ def build(self): situation=self.situation_data, reform=self.reform, ) - for simulation in (self.simulation, getattr(self.simulation, "baseline", None)): - if simulation is not None: - simulation.input_group_entities = supplied_groups self.simulation.trace = True self.sim = self.simulation diff --git a/policyengine_core/simulations/simulation.py b/policyengine_core/simulations/simulation.py index 7490c1d35..de5824c59 100644 --- a/policyengine_core/simulations/simulation.py +++ b/policyengine_core/simulations/simulation.py @@ -725,18 +725,6 @@ class Simulation: is_over_dataset: bool = False """Whether this simulation is built over a dataset.""" - input_group_entities: frozenset[str] = frozenset() - """Singular keys of group entities explicitly supplied by the caller. - - Entity-form situations record groups before country preprocessing. - Datasets record groups declared by person membership columns or, in - flat files, by the group's ID column on each person row; Core's loaders - require membership for every group, so a loaded dataset records all of - them, and a country that overrides ``build_from_dataset`` must set this - itself. Variable-only and population-only construction do not declare - input groups. This immutable provenance survives cloning. - """ - macro_cache_read: bool = False """Whether to read from the macro cache.""" @@ -1146,18 +1134,6 @@ def get_eternity_array(name): entity_ids = get_eternity_array(entity_id_field) builder.declare_person_entity(person_entity.key, entity_ids) - # A dataset declares a group through its person membership column - # or, in a flat file, the group's ID column on each person row. - membership_columns = {column.split("__")[0] for column in data} - self.input_group_entities = frozenset( - group_entity.key - for group_entity in self.tax_benefit_system.group_entities - if f"{person_entity.key}_{group_entity.key}_id" in membership_columns - or ( - self.dataset.data_format == Dataset.FLAT_FILE - and f"{group_entity.key}_id" in membership_columns - ) - ) for group_entity in self.tax_benefit_system.group_entities: entity_id_field = f"{group_entity.key}_id" person_membership_id_field = f"{person_entity.key}_{group_entity.key}_id" diff --git a/policyengine_core/simulations/simulation_builder.py b/policyengine_core/simulations/simulation_builder.py index c58acfa6c..dbda2e86f 100644 --- a/policyengine_core/simulations/simulation_builder.py +++ b/policyengine_core/simulations/simulation_builder.py @@ -226,11 +226,6 @@ def build_from_entities( ) check_type(input_dict, dict, ["error"]) - simulation.input_group_entities = frozenset( - entity.key - for entity in tax_benefit_system.group_entities - if input_dict.get(entity.plural) is not None - ) input_dict = tax_benefit_system.preprocess_situation( input_dict, self.default_period ) @@ -363,7 +358,6 @@ def build_default_simulation( tax_benefit_system=tax_benefit_system, populations=tax_benefit_system.instantiate_entities(), ) - simulation.input_group_entities = frozenset() for population in simulation.populations.values(): population.count = count population.ids = np.arange(id_count, dtype=np.int64) diff --git a/policyengine_core/taxbenefitsystems/tax_benefit_system.py b/policyengine_core/taxbenefitsystems/tax_benefit_system.py index b6be2543f..4fe0a9395 100644 --- a/policyengine_core/taxbenefitsystems/tax_benefit_system.py +++ b/policyengine_core/taxbenefitsystems/tax_benefit_system.py @@ -160,9 +160,6 @@ def preprocess_situation(self, situation: dict, default_period=None) -> dict: dict holding an entity-form situation for the builder's usual validation. A ``Reform`` delegates this hook to its baseline. Dataset and variable-only inputs do not use this hook. - - ``Simulation.input_group_entities`` records the groups supplied by - the caller before this hook runs, separately from inferred groups. """ return situation diff --git a/tests/core/test_dataset_unknown_column_warning.py b/tests/core/test_dataset_unknown_column_warning.py index d621cfa70..7450b92dc 100644 --- a/tests/core/test_dataset_unknown_column_warning.py +++ b/tests/core/test_dataset_unknown_column_warning.py @@ -38,7 +38,6 @@ def test__given_unknown_dataset_column__then_warns_and_still_loads(caplog): # And the known columns still load normally np.testing.assert_array_equal(salary.values, np.array([100.0, 200.0, 300.0])) - assert simulation.input_group_entities == frozenset({"household"}) def test__given_only_known_columns__then_no_unknown_column_warning(caplog): diff --git a/tests/core/test_individual_sim.py b/tests/core/test_individual_sim.py deleted file mode 100644 index ec5fe8b2d..000000000 --- a/tests/core/test_individual_sim.py +++ /dev/null @@ -1,26 +0,0 @@ -from policyengine_core.country_template import CountryTaxBenefitSystem -from policyengine_core.simulations.individual_sim import IndividualSim - - -class TemplateIndividualSim(IndividualSim): - tax_benefit_system = CountryTaxBenefitSystem - required_entities = ["household"] - default_roles = {"household": "parent"} - - -def test_placeholder_and_default_groups_are_not_supplied(): - # __init__ seeds an empty mapping for every group and build() fills the - # required household with every person; neither was supplied by the - # caller. - simulation = TemplateIndividualSim(year=2024) - simulation.add_person(name="Alicia") - simulation.build() - assert simulation.simulation.input_group_entities == frozenset() - - -def test_added_groups_are_supplied(): - simulation = TemplateIndividualSim(year=2024) - simulation.add_person(name="Alicia") - simulation.add_household(name="home", parents=["Alicia"]) - simulation.build() - assert simulation.simulation.input_group_entities == frozenset({"household"}) diff --git a/tests/core/test_join_with_persons_identity.py b/tests/core/test_join_with_persons_identity.py index d75c3e119..6ff739a88 100644 --- a/tests/core/test_join_with_persons_identity.py +++ b/tests/core/test_join_with_persons_identity.py @@ -99,7 +99,6 @@ class InteriorEmptyHousehold(Dataset): household = simulation.populations["household"] assert household.members_entity_id.tolist() == [0, 2] - assert simulation.input_group_entities == frozenset({"household"}) assert simulation.calculate("rent", "2024-01", map_to="person").tolist() == [ 100.0, 300.0, @@ -190,8 +189,6 @@ def test_flat_file_with_only_a_household_id_column_reads_it_as_membership( household = simulation.populations["household"] assert household.count == 2 assert household.members_entity_id.tolist() == [0, 0, 1] - # The ID column declares the households, so they count as supplied. - assert simulation.input_group_entities == frozenset({"household"}) def test_flat_file_default_roles_cover_every_person(): diff --git a/tests/core/test_simulation_builder.py b/tests/core/test_simulation_builder.py index bfeb7e177..c0accaf9a 100644 --- a/tests/core/test_simulation_builder.py +++ b/tests/core/test_simulation_builder.py @@ -13,13 +13,11 @@ entities, situation_examples, ) -from policyengine_core.entities import build_entity from policyengine_core.enums import Enum from policyengine_core.errors import SituationParsingError from policyengine_core.populations import Population from policyengine_core.reforms import Reform from policyengine_core.simulations import Simulation, SimulationBuilder -from policyengine_core.taxbenefitsystems import TaxBenefitSystem from policyengine_core.tools import test_runner from policyengine_core.tools.test_runner import _get_tax_benefit_system from policyengine_core.variables import Variable @@ -134,9 +132,7 @@ def test_explicit_singular_entities(tax_benefit_system): } -def test_country_preprocessing_preserves_caller_input_and_group_provenance( - tax_benefit_system, monkeypatch -): +def test_country_preprocessing_preserves_caller_input(tax_benefit_system, monkeypatch): seen_periods = [] def preprocess(situation, default_period): @@ -153,24 +149,20 @@ def preprocess(situation, default_period): builder.set_default_period("2024-01") inferred = builder.build_from_dict(tax_benefit_system, situation) assert list(inferred.household.ids) == ["inferred"] - assert inferred.input_group_entities == frozenset() assert situation == original assert seen_periods == ["2024-01"] - # Singular aliases are normalized before the hook; explicit membership - # remains distinguishable even if it has the same shape as the default. + # Singular aliases are expanded before the hook, so it sees the caller's + # household and keeps it. explicit = builder.build_from_dict( tax_benefit_system, {**situation, "household": {"parents": ["Alicia", "Javier"]}}, ) - assert explicit.input_group_entities == frozenset({"household"}) - assert explicit.clone().input_group_entities == explicit.input_group_entities + assert list(explicit.household.ids) == ["household"] - # Reusing a simulation for variable-only construction resets provenance - # and does not run entity preprocessing. + # Variable-only construction does not run entity preprocessing. calls = len(seen_periods) builder.build_default_simulation(tax_benefit_system, 2, explicit) - assert explicit.input_group_entities == frozenset() assert len(seen_periods) == calls @@ -189,29 +181,6 @@ def preprocess(situation, default_period): situation={"persons": {"Alicia": {}, "Javier": {}}}, ) assert simulation.household.members_entity_id.tolist() == [0, 1] - assert simulation.input_group_entities == frozenset() - - -def test_explicit_empty_group_mapping_is_recorded(tax_benefit_system): - simulation = SimulationBuilder().build_from_dict( - tax_benefit_system, {"persons": {"Alicia": {}}, "households": {}} - ) - assert simulation.input_group_entities == frozenset({"household"}) - - -def test_only_supplied_groups_are_recorded(): - person = build_entity("person", "persons", "", is_person=True) - household = build_entity( - "household", "households", "", roles=[{"key": "member", "plural": "members"}] - ) - family = build_entity( - "family", "families", "", roles=[{"key": "member", "plural": "members"}] - ) - simulation = SimulationBuilder().build_from_dict( - TaxBenefitSystem([person, household, family]), - {"persons": {"Alicia": {}}, "households": {"home": {"members": ["Alicia"]}}}, - ) - assert simulation.input_group_entities == frozenset({"household"}) class _PreprocessingTaxBenefitSystem(CountryTaxBenefitSystem): @@ -237,7 +206,6 @@ def test_reform_delegates_preprocessing_to_its_baseline(): system, {"persons": {"Alicia": {}}} ) assert list(simulation.household.ids) == ["inferred"] - assert simulation.input_group_entities == frozenset() def test_country_preprocessing_must_return_a_dict(tax_benefit_system, monkeypatch): From 4aaae3af96207ac7f64b4772dc081c2a423d798a Mon Sep 17 00:00:00 2001 From: ZimingHua Date: Thu, 12 Nov 2026 15:47:39 -0500 Subject: [PATCH 4/5] Run the situation hook with the reformed system A Reform now calls its country system's hook with the reform as the system, so the hook reads the reformed parameters and variables; a hook set on the country system instance itself stays bound to it. Document where singular aliases are expanded and that the hook runs before entity validation. Test datasets and variable-only builds skipping the hook, the hook receiving axes, nested reforms and a reformed parameter. Co-Authored-By: Claude Opus 5.5 --- docs/python_api/simulations.md | 17 ++-- policyengine_core/reforms/reform.py | 15 +++- .../taxbenefitsystems/tax_benefit_system.py | 17 ++-- tests/core/test_simulation_builder.py | 82 ++++++++++++++++++- 4 files changed, 116 insertions(+), 15 deletions(-) diff --git a/docs/python_api/simulations.md b/docs/python_api/simulations.md index 2b4c5b40d..e0c984ac1 100644 --- a/docs/python_api/simulations.md +++ b/docs/python_api/simulations.md @@ -26,13 +26,18 @@ The `policyengine_core.simulations` module contains the definition of `Simulatio Country models can override `TaxBenefitSystem.preprocess_situation(situation, default_period)` to fill missing groups before an entity-form situation is -built. The hook receives a deep copy after singular entity aliases have been -expanded, with any `axes` still present, and must return a dict for the +built. `SimulationBuilder.build_from_entities` passes it a deep copy of the +situation, with any `axes` still present, and it must return a dict for the builder's normal validation. `default_period` is the builder's default period -as set, so it may be `None`, a string, an integer or a `Period`. Both -`Simulation(situation=...)` and the YAML runner use this path, and a `Reform` -delegates the hook to its baseline. Dataset and variable-only construction do -not call the hook. The default hook returns its input unchanged. +as set, so it may be `None`, a string, an integer or a `Period`. +`Simulation(situation=...)` and the YAML runner reach it through +`build_from_dict`, which expands singular entity aliases first; a direct +`build_from_entities` call passes the caller's keys as given. The hook runs +before the builder validates entities, so it may receive a situation with no +persons. A `Reform` runs its country system's hook with the reform as the +system, so the hook sees reformed parameters and variables. Dataset and +variable-only construction do not call the hook. The default hook returns its +input unchanged. ```{eval-rst} .. autoclass:: policyengine_core.simulations.simulation_builder.SimulationBuilder diff --git a/policyengine_core/reforms/reform.py b/policyengine_core/reforms/reform.py index 82f352046..d584fd1a1 100644 --- a/policyengine_core/reforms/reform.py +++ b/policyengine_core/reforms/reform.py @@ -149,8 +149,19 @@ def __getattr__(self, attribute): def preprocess_situation(self, situation: dict, default_period=None) -> dict: # TaxBenefitSystem defines a no-op hook, so ``__getattr__`` never - # reaches the baseline's override; delegate to it explicitly. - return self.baseline.preprocess_situation(situation, default_period) + # reaches the country's override. Run the country system's hook with + # this reform as the system, so the hook reads the reformed parameters + # and variables. A hook set on the country system instance itself + # stays bound to that instance. + country_system = self.baseline + while isinstance(country_system, Reform): + country_system = country_system.baseline + instance_hook = vars(country_system).get("preprocess_situation") + if instance_hook is not None: + return instance_hook(situation, default_period) + return type(country_system).preprocess_situation( + self, situation, default_period + ) @property def full_key(self) -> str: diff --git a/policyengine_core/taxbenefitsystems/tax_benefit_system.py b/policyengine_core/taxbenefitsystems/tax_benefit_system.py index 4fe0a9395..8756d4e8f 100644 --- a/policyengine_core/taxbenefitsystems/tax_benefit_system.py +++ b/policyengine_core/taxbenefitsystems/tax_benefit_system.py @@ -154,12 +154,17 @@ def preprocess_situation(self, situation: dict, default_period=None) -> dict: """Normalize entity-form input before the builder creates populations. Country models can override this to fill missing entity definitions. - The builder passes a deep copy of the caller's situation, with singular - entity aliases expanded and any ``axes`` still present, and its default - period as set (None, a string, an integer or a ``Period``). Return a - dict holding an entity-form situation for the builder's usual - validation. A ``Reform`` delegates this hook to its baseline. Dataset - and variable-only inputs do not use this hook. + ``SimulationBuilder.build_from_entities`` passes a deep copy of the + caller's situation, with any ``axes`` still present, and its default + period as set (None, a string, an integer or a ``Period``). Through + ``build_from_dict`` and ``Simulation(situation=...)``, singular entity + aliases are already expanded; a direct ``build_from_entities`` call + passes the caller's keys as given. The hook runs before the builder + validates entities, so it may receive a situation with no persons. + Return a dict holding an entity-form situation for the builder's usual + validation. A ``Reform`` runs its country system's hook with the + reform as the system, so the hook sees reformed parameters and + variables. Dataset and variable-only inputs do not use this hook. """ return situation diff --git a/tests/core/test_simulation_builder.py b/tests/core/test_simulation_builder.py index c0accaf9a..1e32f30e7 100644 --- a/tests/core/test_simulation_builder.py +++ b/tests/core/test_simulation_builder.py @@ -5,6 +5,7 @@ from typing import Iterable import numpy as np +import pandas as pd import pytest from policyengine_core import periods, tools @@ -13,6 +14,7 @@ entities, situation_examples, ) +from policyengine_core.data import Dataset from policyengine_core.enums import Enum from policyengine_core.errors import SituationParsingError from policyengine_core.populations import Population @@ -162,10 +164,54 @@ def preprocess(situation, default_period): # Variable-only construction does not run entity preprocessing. calls = len(seen_periods) - builder.build_default_simulation(tax_benefit_system, 2, explicit) + builder.build_from_dict(tax_benefit_system, {"salary": {"2024-01": 100}}) assert len(seen_periods) == calls +def test_country_preprocessing_receives_axes(tax_benefit_system, monkeypatch): + seen = [] + + def preprocess(situation, default_period): + seen.append(situation.get("axes")) + return situation + + monkeypatch.setattr(tax_benefit_system, "preprocess_situation", preprocess) + axes = [ + [{"name": "salary", "min": 0, "max": 2_000, "count": 3, "period": "2024-01"}] + ] + simulation = SimulationBuilder().build_from_dict( + tax_benefit_system, + { + "persons": {"Alicia": {}}, + "households": {"home": {"parents": ["Alicia"]}}, + "axes": axes, + }, + ) + assert seen == [axes] + assert simulation.persons.count == 3 + + +def test_datasets_do_not_run_country_preprocessing(tax_benefit_system, monkeypatch): + calls = [] + + def preprocess(situation, default_period): + calls.append(situation) + return situation + + monkeypatch.setattr(tax_benefit_system, "preprocess_situation", preprocess) + dataframe = pd.DataFrame({"person_id": [0, 1, 2], "household_id": [7, 7, 9]}) + simulation = Simulation( + tax_benefit_system=tax_benefit_system, + dataset=Dataset.from_dataframe(dataframe, "2024"), + ) + assert simulation.populations["household"].members_entity_id.tolist() == [ + 0, + 0, + 1, + ] + assert calls == [] + + def test_country_preprocessing_applies_to_simulation_api( tax_benefit_system, monkeypatch ): @@ -200,6 +246,7 @@ def test_reform_delegates_preprocessing_to_its_baseline(): baseline = _PreprocessingTaxBenefitSystem() for system in ( _NoOpReform(baseline), + _NoOpReform(_NoOpReform(baseline)), _get_tax_benefit_system(baseline, [_NoOpReform], []), ): simulation = SimulationBuilder().build_from_dict( @@ -208,6 +255,39 @@ def test_reform_delegates_preprocessing_to_its_baseline(): assert list(simulation.household.ids) == ["inferred"] +class _RateReadingTaxBenefitSystem(CountryTaxBenefitSystem): + def preprocess_situation(self, situation, default_period=None): + rate = self.parameters("2024-01-01").taxes.income_tax_rate + situation.setdefault("households", {f"rate {rate}": {"parents": ["Alicia"]}}) + return situation + + +class _RateReform(Reform): + def apply(self): + def modify(parameters): + parameters.taxes.income_tax_rate.update( + period=periods.period("year:2015:20"), value=0.99 + ) + return parameters + + self.modify_parameters(modify) + + +def test_reform_runs_the_country_hook_with_the_reformed_system(): + # The hook reads parameters through the system it is given, so under a + # reform it sees the reform's value, not the baseline's 0.15. + baseline = _RateReadingTaxBenefitSystem() + for system, expected in ( + (baseline, "rate 0.15"), + (_RateReform(baseline), "rate 0.99"), + (_get_tax_benefit_system(baseline, [_RateReform], []), "rate 0.99"), + ): + simulation = SimulationBuilder().build_from_dict( + system, {"persons": {"Alicia": {}}} + ) + assert list(simulation.household.ids) == [expected] + + def test_country_preprocessing_must_return_a_dict(tax_benefit_system, monkeypatch): monkeypatch.setattr( tax_benefit_system, From cc9785d066aaf981045d98815d2ab0398ff4e82a Mon Sep 17 00:00:00 2001 From: ZimingHua Date: Thu, 12 Nov 2026 20:41:20 -0500 Subject: [PATCH 5/5] Say which hook a reform runs, and rename the reform test Only the country system's hook runs: a Reform subclass that overrides it is skipped when another reform wraps it, and a hook set on the country system instance reads the baseline's values. Rename the reform test to match what it checks. Co-Authored-By: Claude Opus 5.5 --- docs/python_api/simulations.md | 5 ++++- policyengine_core/taxbenefitsystems/tax_benefit_system.py | 6 +++++- tests/core/test_simulation_builder.py | 5 +++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/python_api/simulations.md b/docs/python_api/simulations.md index e0c984ac1..176c4be80 100644 --- a/docs/python_api/simulations.md +++ b/docs/python_api/simulations.md @@ -35,7 +35,10 @@ as set, so it may be `None`, a string, an integer or a `Period`. `build_from_entities` call passes the caller's keys as given. The hook runs before the builder validates entities, so it may receive a situation with no persons. A `Reform` runs its country system's hook with the reform as the -system, so the hook sees reformed parameters and variables. Dataset and +system, so the hook sees reformed parameters and variables. Only the country +system's hook runs: a `Reform` subclass that overrides it is skipped when +another reform wraps it, and a hook set on the country system instance itself +stays bound to that instance and reads the baseline's values. Dataset and variable-only construction do not call the hook. The default hook returns its input unchanged. diff --git a/policyengine_core/taxbenefitsystems/tax_benefit_system.py b/policyengine_core/taxbenefitsystems/tax_benefit_system.py index 8756d4e8f..6137efe28 100644 --- a/policyengine_core/taxbenefitsystems/tax_benefit_system.py +++ b/policyengine_core/taxbenefitsystems/tax_benefit_system.py @@ -164,7 +164,11 @@ def preprocess_situation(self, situation: dict, default_period=None) -> dict: Return a dict holding an entity-form situation for the builder's usual validation. A ``Reform`` runs its country system's hook with the reform as the system, so the hook sees reformed parameters and - variables. Dataset and variable-only inputs do not use this hook. + variables. Only the country system's hook runs: a ``Reform`` subclass + that overrides this method is skipped when another reform wraps it, + and a hook set on the country system instance itself stays bound to + that instance, so it reads the baseline's values. Dataset and + variable-only inputs do not use this hook. """ return situation diff --git a/tests/core/test_simulation_builder.py b/tests/core/test_simulation_builder.py index 1e32f30e7..b0acdb23d 100644 --- a/tests/core/test_simulation_builder.py +++ b/tests/core/test_simulation_builder.py @@ -240,9 +240,10 @@ def apply(self): pass -def test_reform_delegates_preprocessing_to_its_baseline(): +def test_reforms_run_the_country_systems_hook(): # Reform inherits TaxBenefitSystem's no-op hook, so a reform, including - # the one the YAML runner builds for inline parameters, must delegate. + # the one the YAML runner builds for inline parameters, must run the + # country system's hook. baseline = _PreprocessingTaxBenefitSystem() for system in ( _NoOpReform(baseline),