diff --git a/changelog.d/situation-preprocessing.added.md b/changelog.d/situation-preprocessing.added.md new file mode 100644 index 000000000..2a53d6456 --- /dev/null +++ b/changelog.d/situation-preprocessing.added.md @@ -0,0 +1 @@ +Add a `TaxBenefitSystem.preprocess_situation` hook so country models can fill missing groups in an entity-form situation before it is built. diff --git a/docs/python_api/simulations.md b/docs/python_api/simulations.md index 0b65c1dd9..176c4be80 100644 --- a/docs/python_api/simulations.md +++ b/docs/python_api/simulations.md @@ -24,6 +24,24 @@ The `policyengine_core.simulations` module contains the definition of `Simulatio ## SimulationBuilder +Country models can override `TaxBenefitSystem.preprocess_situation(situation, +default_period)` to fill missing groups before an entity-form situation is +built. `SimulationBuilder.build_from_entities` passes it a deep copy of the +situation, with any `axes` still present, and it must return a dict for the +builder's normal validation. `default_period` is the builder's default period +as set, so it may be `None`, a string, an integer or a `Period`. +`Simulation(situation=...)` and the YAML runner reach it through +`build_from_dict`, which expands singular entity aliases first; a direct +`build_from_entities` call passes the caller's keys as given. The hook runs +before the builder validates entities, so it may receive a situation with no +persons. A `Reform` runs its country system's hook with the reform as the +system, so the hook sees reformed parameters and variables. Only the country +system's hook runs: a `Reform` subclass that overrides it is skipped when +another reform wraps it, and a hook set on the country system instance itself +stays bound to that instance and reads the baseline's values. Dataset and +variable-only construction do not call the hook. The default hook returns its +input unchanged. + ```{eval-rst} .. autoclass:: policyengine_core.simulations.simulation_builder.SimulationBuilder :members: @@ -31,4 +49,3 @@ The `policyengine_core.simulations` module contains the definition of `Simulatio :inherited-members: :show-inheritance: ``` - diff --git a/policyengine_core/reforms/reform.py b/policyengine_core/reforms/reform.py index afcee0358..d584fd1a1 100644 --- a/policyengine_core/reforms/reform.py +++ b/policyengine_core/reforms/reform.py @@ -147,6 +147,22 @@ def __init__(self, baseline: TaxBenefitSystem): def __getattr__(self, attribute): return getattr(self.baseline, attribute) + def preprocess_situation(self, situation: dict, default_period=None) -> dict: + # TaxBenefitSystem defines a no-op hook, so ``__getattr__`` never + # reaches the country's override. Run the country system's hook with + # this reform as the system, so the hook reads the reformed parameters + # and variables. A hook set on the country system instance itself + # stays bound to that instance. + country_system = self.baseline + while isinstance(country_system, Reform): + country_system = country_system.baseline + instance_hook = vars(country_system).get("preprocess_situation") + if instance_hook is not None: + return instance_hook(situation, default_period) + return type(country_system).preprocess_situation( + self, situation, default_period + ) + @property def full_key(self) -> str: key = self.key diff --git a/policyengine_core/simulations/simulation_builder.py b/policyengine_core/simulations/simulation_builder.py index c16a767d3..dbda2e86f 100644 --- a/policyengine_core/simulations/simulation_builder.py +++ b/policyengine_core/simulations/simulation_builder.py @@ -226,6 +226,14 @@ def build_from_entities( ) check_type(input_dict, dict, ["error"]) + input_dict = tax_benefit_system.preprocess_situation( + input_dict, self.default_period + ) + if not isinstance(input_dict, dict): + raise TypeError( + f"{type(tax_benefit_system).__name__}.preprocess_situation must " + f"return a dict, not {type(input_dict).__name__}." + ) axes = input_dict.pop("axes", None) unexpected_entities = [ diff --git a/policyengine_core/taxbenefitsystems/tax_benefit_system.py b/policyengine_core/taxbenefitsystems/tax_benefit_system.py index ccadd2a39..6137efe28 100644 --- a/policyengine_core/taxbenefitsystems/tax_benefit_system.py +++ b/policyengine_core/taxbenefitsystems/tax_benefit_system.py @@ -150,6 +150,28 @@ class TaxBenefitSystem: _defined_for_checks_deferred: bool = False """Whether ``load_variable`` leaves ``defined_for`` checks to a later pass over every variable.""" + def preprocess_situation(self, situation: dict, default_period=None) -> dict: + """Normalize entity-form input before the builder creates populations. + + Country models can override this to fill missing entity definitions. + ``SimulationBuilder.build_from_entities`` passes a deep copy of the + caller's situation, with any ``axes`` still present, and its default + period as set (None, a string, an integer or a ``Period``). Through + ``build_from_dict`` and ``Simulation(situation=...)``, singular entity + aliases are already expanded; a direct ``build_from_entities`` call + passes the caller's keys as given. The hook runs before the builder + validates entities, so it may receive a situation with no persons. + Return a dict holding an entity-form situation for the builder's usual + validation. A ``Reform`` runs its country system's hook with the + reform as the system, so the hook sees reformed parameters and + variables. Only the country system's hook runs: a ``Reform`` subclass + that overrides this method is skipped when another reform wraps it, + and a hook set on the country system instance itself stays bound to + that instance, so it reads the baseline's values. Dataset and + variable-only inputs do not use this hook. + """ + return situation + def __init__(self, entities: Sequence[Entity] = None, reform=None) -> None: if entities is None: entities = self.entities diff --git a/tests/core/test_simulation_builder.py b/tests/core/test_simulation_builder.py index 049bedf78..b0acdb23d 100644 --- a/tests/core/test_simulation_builder.py +++ b/tests/core/test_simulation_builder.py @@ -1,18 +1,27 @@ import datetime +import copy from decimal import Decimal from fractions import Fraction from typing import Iterable import numpy as np +import pandas as pd import pytest from policyengine_core import periods, tools -from policyengine_core.country_template import entities, situation_examples +from policyengine_core.country_template import ( + CountryTaxBenefitSystem, + entities, + situation_examples, +) +from policyengine_core.data import Dataset from policyengine_core.enums import Enum from policyengine_core.errors import SituationParsingError from policyengine_core.populations import Population +from policyengine_core.reforms import Reform from policyengine_core.simulations import Simulation, SimulationBuilder from policyengine_core.tools import test_runner +from policyengine_core.tools.test_runner import _get_tax_benefit_system from policyengine_core.variables import Variable @@ -125,6 +134,173 @@ def test_explicit_singular_entities(tax_benefit_system): } +def test_country_preprocessing_preserves_caller_input(tax_benefit_system, monkeypatch): + seen_periods = [] + + def preprocess(situation, default_period): + seen_periods.append(default_period) + situation.setdefault( + "households", {"inferred": {"parents": ["Alicia", "Javier"]}} + ) + return situation + + monkeypatch.setattr(tax_benefit_system, "preprocess_situation", preprocess) + situation = {"persons": {"Alicia": {}, "Javier": {}}} + original = copy.deepcopy(situation) + builder = SimulationBuilder() + builder.set_default_period("2024-01") + inferred = builder.build_from_dict(tax_benefit_system, situation) + assert list(inferred.household.ids) == ["inferred"] + assert situation == original + assert seen_periods == ["2024-01"] + + # Singular aliases are expanded before the hook, so it sees the caller's + # household and keeps it. + explicit = builder.build_from_dict( + tax_benefit_system, + {**situation, "household": {"parents": ["Alicia", "Javier"]}}, + ) + assert list(explicit.household.ids) == ["household"] + + # Variable-only construction does not run entity preprocessing. + calls = len(seen_periods) + builder.build_from_dict(tax_benefit_system, {"salary": {"2024-01": 100}}) + assert len(seen_periods) == calls + + +def test_country_preprocessing_receives_axes(tax_benefit_system, monkeypatch): + seen = [] + + def preprocess(situation, default_period): + seen.append(situation.get("axes")) + return situation + + monkeypatch.setattr(tax_benefit_system, "preprocess_situation", preprocess) + axes = [ + [{"name": "salary", "min": 0, "max": 2_000, "count": 3, "period": "2024-01"}] + ] + simulation = SimulationBuilder().build_from_dict( + tax_benefit_system, + { + "persons": {"Alicia": {}}, + "households": {"home": {"parents": ["Alicia"]}}, + "axes": axes, + }, + ) + assert seen == [axes] + assert simulation.persons.count == 3 + + +def test_datasets_do_not_run_country_preprocessing(tax_benefit_system, monkeypatch): + calls = [] + + def preprocess(situation, default_period): + calls.append(situation) + return situation + + monkeypatch.setattr(tax_benefit_system, "preprocess_situation", preprocess) + dataframe = pd.DataFrame({"person_id": [0, 1, 2], "household_id": [7, 7, 9]}) + simulation = Simulation( + tax_benefit_system=tax_benefit_system, + dataset=Dataset.from_dataframe(dataframe, "2024"), + ) + assert simulation.populations["household"].members_entity_id.tolist() == [ + 0, + 0, + 1, + ] + assert calls == [] + + +def test_country_preprocessing_applies_to_simulation_api( + tax_benefit_system, monkeypatch +): + def preprocess(situation, default_period): + situation["households"] = { + name: {"parents": [name]} for name in situation["persons"] + } + return situation + + monkeypatch.setattr(tax_benefit_system, "preprocess_situation", preprocess) + simulation = Simulation( + tax_benefit_system=tax_benefit_system, + situation={"persons": {"Alicia": {}, "Javier": {}}}, + ) + assert simulation.household.members_entity_id.tolist() == [0, 1] + + +class _PreprocessingTaxBenefitSystem(CountryTaxBenefitSystem): + def preprocess_situation(self, situation, default_period=None): + situation.setdefault("households", {"inferred": {"parents": ["Alicia"]}}) + return situation + + +class _NoOpReform(Reform): + def apply(self): + pass + + +def test_reforms_run_the_country_systems_hook(): + # Reform inherits TaxBenefitSystem's no-op hook, so a reform, including + # the one the YAML runner builds for inline parameters, must run the + # country system's hook. + baseline = _PreprocessingTaxBenefitSystem() + for system in ( + _NoOpReform(baseline), + _NoOpReform(_NoOpReform(baseline)), + _get_tax_benefit_system(baseline, [_NoOpReform], []), + ): + simulation = SimulationBuilder().build_from_dict( + system, {"persons": {"Alicia": {}}} + ) + assert list(simulation.household.ids) == ["inferred"] + + +class _RateReadingTaxBenefitSystem(CountryTaxBenefitSystem): + def preprocess_situation(self, situation, default_period=None): + rate = self.parameters("2024-01-01").taxes.income_tax_rate + situation.setdefault("households", {f"rate {rate}": {"parents": ["Alicia"]}}) + return situation + + +class _RateReform(Reform): + def apply(self): + def modify(parameters): + parameters.taxes.income_tax_rate.update( + period=periods.period("year:2015:20"), value=0.99 + ) + return parameters + + self.modify_parameters(modify) + + +def test_reform_runs_the_country_hook_with_the_reformed_system(): + # The hook reads parameters through the system it is given, so under a + # reform it sees the reform's value, not the baseline's 0.15. + baseline = _RateReadingTaxBenefitSystem() + for system, expected in ( + (baseline, "rate 0.15"), + (_RateReform(baseline), "rate 0.99"), + (_get_tax_benefit_system(baseline, [_RateReform], []), "rate 0.99"), + ): + simulation = SimulationBuilder().build_from_dict( + system, {"persons": {"Alicia": {}}} + ) + assert list(simulation.household.ids) == [expected] + + +def test_country_preprocessing_must_return_a_dict(tax_benefit_system, monkeypatch): + monkeypatch.setattr( + tax_benefit_system, + "preprocess_situation", + lambda situation, default_period: None, + ) + with pytest.raises(TypeError, match="preprocess_situation must return a dict"): + SimulationBuilder().build_from_dict( + tax_benefit_system, {"persons": {"Alicia": {}}} + ) + + def test_add_person_entity(persons): persons_json = {"Alicia": {"salary": {}}, "Javier": {}} simulation_builder = SimulationBuilder()