Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/situation-preprocessing.added.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Add a `TaxBenefitSystem.preprocess_situation` hook so country models can fill missing groups in an entity-form situation before it is built.
19 changes: 18 additions & 1 deletion docs/python_api/simulations.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,28 @@ The `policyengine_core.simulations` module contains the definition of `Simulatio

## SimulationBuilder

Country models can override `TaxBenefitSystem.preprocess_situation(situation,
default_period)` to fill missing groups before an entity-form situation is
built. `SimulationBuilder.build_from_entities` passes it a deep copy of the
situation, with any `axes` still present, and it must return a dict for the
builder's normal validation. `default_period` is the builder's default period
as set, so it may be `None`, a string, an integer or a `Period`.
`Simulation(situation=...)` and the YAML runner reach it through
`build_from_dict`, which expands singular entity aliases first; a direct
`build_from_entities` call passes the caller's keys as given. The hook runs
before the builder validates entities, so it may receive a situation with no
persons. A `Reform` runs its country system's hook with the reform as the
system, so the hook sees reformed parameters and variables. Only the country
system's hook runs: a `Reform` subclass that overrides it is skipped when
another reform wraps it, and a hook set on the country system instance itself
stays bound to that instance and reads the baseline's values. Dataset and
variable-only construction do not call the hook. The default hook returns its
input unchanged.

```{eval-rst}
.. autoclass:: policyengine_core.simulations.simulation_builder.SimulationBuilder
:members:
:undoc-members:
:inherited-members:
:show-inheritance:
```

16 changes: 16 additions & 0 deletions policyengine_core/reforms/reform.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,22 @@ def __init__(self, baseline: TaxBenefitSystem):
def __getattr__(self, attribute):
return getattr(self.baseline, attribute)

def preprocess_situation(self, situation: dict, default_period=None) -> dict:
# TaxBenefitSystem defines a no-op hook, so ``__getattr__`` never
# reaches the country's override. Run the country system's hook with
# this reform as the system, so the hook reads the reformed parameters
# and variables. A hook set on the country system instance itself
# stays bound to that instance.
country_system = self.baseline
while isinstance(country_system, Reform):
country_system = country_system.baseline
instance_hook = vars(country_system).get("preprocess_situation")
if instance_hook is not None:
return instance_hook(situation, default_period)
return type(country_system).preprocess_situation(
self, situation, default_period
)

@property
def full_key(self) -> str:
key = self.key
Expand Down
8 changes: 8 additions & 0 deletions policyengine_core/simulations/simulation_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,14 @@ def build_from_entities(
)

check_type(input_dict, dict, ["error"])
input_dict = tax_benefit_system.preprocess_situation(
input_dict, self.default_period
)
if not isinstance(input_dict, dict):
raise TypeError(
f"{type(tax_benefit_system).__name__}.preprocess_situation must "
f"return a dict, not {type(input_dict).__name__}."
)
axes = input_dict.pop("axes", None)

unexpected_entities = [
Expand Down
22 changes: 22 additions & 0 deletions policyengine_core/taxbenefitsystems/tax_benefit_system.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,28 @@ class TaxBenefitSystem:
_defined_for_checks_deferred: bool = False
"""Whether ``load_variable`` leaves ``defined_for`` checks to a later pass over every variable."""

def preprocess_situation(self, situation: dict, default_period=None) -> dict:
"""Normalize entity-form input before the builder creates populations.

Country models can override this to fill missing entity definitions.
``SimulationBuilder.build_from_entities`` passes a deep copy of the
caller's situation, with any ``axes`` still present, and its default
period as set (None, a string, an integer or a ``Period``). Through
``build_from_dict`` and ``Simulation(situation=...)``, singular entity
aliases are already expanded; a direct ``build_from_entities`` call
passes the caller's keys as given. The hook runs before the builder
validates entities, so it may receive a situation with no persons.
Return a dict holding an entity-form situation for the builder's usual
validation. A ``Reform`` runs its country system's hook with the
reform as the system, so the hook sees reformed parameters and
variables. Only the country system's hook runs: a ``Reform`` subclass
that overrides this method is skipped when another reform wraps it,
and a hook set on the country system instance itself stays bound to
that instance, so it reads the baseline's values. Dataset and
variable-only inputs do not use this hook.
"""
return situation

def __init__(self, entities: Sequence[Entity] = None, reform=None) -> None:
if entities is None:
entities = self.entities
Expand Down
178 changes: 177 additions & 1 deletion tests/core/test_simulation_builder.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,27 @@
import datetime
import copy
from decimal import Decimal
from fractions import Fraction
from typing import Iterable

import numpy as np
import pandas as pd
import pytest

from policyengine_core import periods, tools
from policyengine_core.country_template import entities, situation_examples
from policyengine_core.country_template import (
CountryTaxBenefitSystem,
entities,
situation_examples,
)
from policyengine_core.data import Dataset
from policyengine_core.enums import Enum
from policyengine_core.errors import SituationParsingError
from policyengine_core.populations import Population
from policyengine_core.reforms import Reform
from policyengine_core.simulations import Simulation, SimulationBuilder
from policyengine_core.tools import test_runner
from policyengine_core.tools.test_runner import _get_tax_benefit_system
from policyengine_core.variables import Variable


Expand Down Expand Up @@ -125,6 +134,173 @@ def test_explicit_singular_entities(tax_benefit_system):
}


def test_country_preprocessing_preserves_caller_input(tax_benefit_system, monkeypatch):
seen_periods = []

def preprocess(situation, default_period):
seen_periods.append(default_period)
situation.setdefault(
"households", {"inferred": {"parents": ["Alicia", "Javier"]}}
)
return situation

monkeypatch.setattr(tax_benefit_system, "preprocess_situation", preprocess)
situation = {"persons": {"Alicia": {}, "Javier": {}}}
original = copy.deepcopy(situation)
builder = SimulationBuilder()
builder.set_default_period("2024-01")
inferred = builder.build_from_dict(tax_benefit_system, situation)
assert list(inferred.household.ids) == ["inferred"]
assert situation == original
assert seen_periods == ["2024-01"]

# Singular aliases are expanded before the hook, so it sees the caller's
# household and keeps it.
explicit = builder.build_from_dict(
tax_benefit_system,
{**situation, "household": {"parents": ["Alicia", "Javier"]}},
)
assert list(explicit.household.ids) == ["household"]

# Variable-only construction does not run entity preprocessing.
calls = len(seen_periods)
builder.build_from_dict(tax_benefit_system, {"salary": {"2024-01": 100}})
assert len(seen_periods) == calls


def test_country_preprocessing_receives_axes(tax_benefit_system, monkeypatch):
seen = []

def preprocess(situation, default_period):
seen.append(situation.get("axes"))
return situation

monkeypatch.setattr(tax_benefit_system, "preprocess_situation", preprocess)
axes = [
[{"name": "salary", "min": 0, "max": 2_000, "count": 3, "period": "2024-01"}]
]
simulation = SimulationBuilder().build_from_dict(
tax_benefit_system,
{
"persons": {"Alicia": {}},
"households": {"home": {"parents": ["Alicia"]}},
"axes": axes,
},
)
assert seen == [axes]
assert simulation.persons.count == 3


def test_datasets_do_not_run_country_preprocessing(tax_benefit_system, monkeypatch):
calls = []

def preprocess(situation, default_period):
calls.append(situation)
return situation

monkeypatch.setattr(tax_benefit_system, "preprocess_situation", preprocess)
dataframe = pd.DataFrame({"person_id": [0, 1, 2], "household_id": [7, 7, 9]})
simulation = Simulation(
tax_benefit_system=tax_benefit_system,
dataset=Dataset.from_dataframe(dataframe, "2024"),
)
assert simulation.populations["household"].members_entity_id.tolist() == [
0,
0,
1,
]
assert calls == []


def test_country_preprocessing_applies_to_simulation_api(
tax_benefit_system, monkeypatch
):
def preprocess(situation, default_period):
situation["households"] = {
name: {"parents": [name]} for name in situation["persons"]
}
return situation

monkeypatch.setattr(tax_benefit_system, "preprocess_situation", preprocess)
simulation = Simulation(
tax_benefit_system=tax_benefit_system,
situation={"persons": {"Alicia": {}, "Javier": {}}},
)
assert simulation.household.members_entity_id.tolist() == [0, 1]


class _PreprocessingTaxBenefitSystem(CountryTaxBenefitSystem):
def preprocess_situation(self, situation, default_period=None):
situation.setdefault("households", {"inferred": {"parents": ["Alicia"]}})
return situation


class _NoOpReform(Reform):
def apply(self):
pass


def test_reforms_run_the_country_systems_hook():
# Reform inherits TaxBenefitSystem's no-op hook, so a reform, including
# the one the YAML runner builds for inline parameters, must run the
# country system's hook.
baseline = _PreprocessingTaxBenefitSystem()
for system in (
_NoOpReform(baseline),
_NoOpReform(_NoOpReform(baseline)),
_get_tax_benefit_system(baseline, [_NoOpReform], []),
):
simulation = SimulationBuilder().build_from_dict(
system, {"persons": {"Alicia": {}}}
)
assert list(simulation.household.ids) == ["inferred"]


class _RateReadingTaxBenefitSystem(CountryTaxBenefitSystem):
def preprocess_situation(self, situation, default_period=None):
rate = self.parameters("2024-01-01").taxes.income_tax_rate
situation.setdefault("households", {f"rate {rate}": {"parents": ["Alicia"]}})
return situation


class _RateReform(Reform):
def apply(self):
def modify(parameters):
parameters.taxes.income_tax_rate.update(
period=periods.period("year:2015:20"), value=0.99
)
return parameters

self.modify_parameters(modify)


def test_reform_runs_the_country_hook_with_the_reformed_system():
# The hook reads parameters through the system it is given, so under a
# reform it sees the reform's value, not the baseline's 0.15.
baseline = _RateReadingTaxBenefitSystem()
for system, expected in (
(baseline, "rate 0.15"),
(_RateReform(baseline), "rate 0.99"),
(_get_tax_benefit_system(baseline, [_RateReform], []), "rate 0.99"),
):
simulation = SimulationBuilder().build_from_dict(
system, {"persons": {"Alicia": {}}}
)
assert list(simulation.household.ids) == [expected]


def test_country_preprocessing_must_return_a_dict(tax_benefit_system, monkeypatch):
monkeypatch.setattr(
tax_benefit_system,
"preprocess_situation",
lambda situation, default_period: None,
)
with pytest.raises(TypeError, match="preprocess_situation must return a dict"):
SimulationBuilder().build_from_dict(
tax_benefit_system, {"persons": {"Alicia": {}}}
)


def test_add_person_entity(persons):
persons_json = {"Alicia": {"salary": {}}, "Javier": {}}
simulation_builder = SimulationBuilder()
Expand Down
Loading