From f7cef1e2f4f35e6e6acf845d9753e967829ca6fa Mon Sep 17 00:00:00 2001 From: Deepak Jindal Date: Tue, 17 Nov 2026 17:04:44 -0800 Subject: [PATCH 1/3] Avoid eager Holder allocation in input_variables discovery --- .../avoid-eager-holder-allocation.fixed.md | 1 + policyengine_core/simulations/simulation.py | 35 +++++++++-- .../simulations/simulation_builder.py | 6 +- .../test_apply_reform_holders_property.py | 4 +- tests/core/test_reform_baseline_system.py | 9 ++- tests/core/test_simulations.py | 58 +++++++++++++++++++ tests/core/test_storage_input_index.py | 4 +- tests/core/test_storage_shared_index.py | 6 +- tests/fixtures/set_input_helper_order.py | 6 +- 9 files changed, 110 insertions(+), 19 deletions(-) create mode 100644 changelog.d/avoid-eager-holder-allocation.fixed.md diff --git a/changelog.d/avoid-eager-holder-allocation.fixed.md b/changelog.d/avoid-eager-holder-allocation.fixed.md new file mode 100644 index 000000000..f93c4c9a7 --- /dev/null +++ b/changelog.d/avoid-eager-holder-allocation.fixed.md @@ -0,0 +1 @@ +Avoid eagerly allocating empty ``Holder`` objects for every variable during ``Simulation.__init__`` and ``SimulationBuilder.build_from_dict`` when discovering ``input_variables``. diff --git a/policyengine_core/simulations/simulation.py b/policyengine_core/simulations/simulation.py index 297e7a778..d149b10bd 100644 --- a/policyengine_core/simulations/simulation.py +++ b/policyengine_core/simulations/simulation.py @@ -892,11 +892,16 @@ def __init__( self.calc = self.calculate self.df = self.calculate_dataframe - self.input_variables = [ - variable.name - for variable in self.tax_benefit_system.variables.values() - if len(self.get_holder(variable.name).get_known_periods()) > 0 - ] + # ``build_from_dict`` already populated ``input_variables`` when building + # from ``situation``; only recompute if later initialization steps + # (``populations``, ``dataset``, or ``reform``) may have changed holders. + if ( + situation is None + or populations is not None + or dataset is not None + or self.reform is not None + ): + self.input_variables = self._collect_input_variables() self.situation_input = situation if self.situation_input is not None: @@ -3290,6 +3295,26 @@ def check_macro_cache(self, variable_name: str, period: str) -> bool: return True + def _collect_input_variables(self) -> List[str]: + """Return active system variables that already have stored periods. + + Iterate only over holders that already exist on each population so + untouched variables do not eagerly allocate empty ``Holder`` objects. + """ + variables = self.tax_benefit_system.variables + input_variables = [] + for population in self.populations.values(): + entity_key = population.entity.key + for name, holder in population._holders.items(): + variable = variables.get(name) + if ( + variable is not None + and variable.entity.key == entity_key + and len(holder.get_known_periods()) > 0 + ): + input_variables.append(name) + return input_variables + def get_input_variables(self, include_computed_variables: bool = True) -> List[str]: """Return variable names stored as inputs on this simulation. diff --git a/policyengine_core/simulations/simulation_builder.py b/policyengine_core/simulations/simulation_builder.py index dbda2e86f..69dd6be95 100644 --- a/policyengine_core/simulations/simulation_builder.py +++ b/policyengine_core/simulations/simulation_builder.py @@ -186,11 +186,7 @@ def build_from_dict( tax_benefit_system, input_dict, simulation ) - simulation.input_variables = [ - variable.name - for variable in simulation.tax_benefit_system.variables.values() - if len(simulation.get_holder(variable.name).get_known_periods()) > 0 - ] + simulation.input_variables = simulation._collect_input_variables() return simulation diff --git a/tests/core/test_apply_reform_holders_property.py b/tests/core/test_apply_reform_holders_property.py index d787346dc..7f2a6878d 100644 --- a/tests/core/test_apply_reform_holders_property.py +++ b/tests/core/test_apply_reform_holders_property.py @@ -72,7 +72,7 @@ def test_every_existing_holder_uses_its_system_variable( simulation = build(mask) for name in REPLACED: population = simulation.get_variable_population(name) - del population._holders[name] + population._holders.pop(name, None) for name in order: simulation.get_holder(name) if warm: @@ -132,7 +132,7 @@ def test_reform_before_or_after_holder_creation_returns_same_values( mask, default, warm ): before_holder = build(mask) - del before_holder.person._holders["award"] + before_holder.person._holders.pop("award", None) after_holder = build(mask) if warm: after_holder.calculate("award", YEAR) diff --git a/tests/core/test_reform_baseline_system.py b/tests/core/test_reform_baseline_system.py index 519223374..c1c337280 100644 --- a/tests/core/test_reform_baseline_system.py +++ b/tests/core/test_reform_baseline_system.py @@ -193,7 +193,14 @@ def test_the_baseline_drops_a_holder_built_for_another_entity(): "households": {"h": {"parents": ["a", "b"]}}, } reform = Simulation(situation=situation, reform=move_salary_to_households) - baseline = reform.baseline + # Create the reform's household holder before cloning the baseline branch so + # ``_bind_to_tax_benefit_system`` must drop the cross-entity holder. + reform.calculate("salary", PERIOD) + del reform.branches["baseline"] + baseline = reform.get_branch("baseline") + baseline.tax_benefit_system = reform.default_tax_benefit_system_instance + baseline._bind_to_tax_benefit_system() + baseline.input_variables = list(baseline.input_variables) assert "salary" in reform.household._holders assert "salary" not in baseline.household._holders diff --git a/tests/core/test_simulations.py b/tests/core/test_simulations.py index 5118021b4..4f2b7173d 100644 --- a/tests/core/test_simulations.py +++ b/tests/core/test_simulations.py @@ -242,3 +242,61 @@ def test__given_branch_inherits_dataset_inputs__then_safe_exports_include_them( assert "salary" in exported_data assert "pseudo_input_for_safe_export" not in exported_data assert "salary" in branch.true_input_variables + + +def test__given_situation_simulation__then_untouched_variables_have_no_holders( + isolated_tax_benefit_system, +): + situation = { + "persons": {"Alicia": {"salary": {"2022-01": 1000}}}, + "households": {"_": {"parents": ["Alicia"]}}, + } + + built = SimulationBuilder().build_from_dict(isolated_tax_benefit_system, situation) + simulation = CountryTemplateSimulation( + tax_benefit_system=isolated_tax_benefit_system, + situation=situation, + ) + cloned = simulation.clone() + + for sim in (built, simulation, cloned): + assert sim.input_variables == ["salary"] + assert set(sim.persons._holders) == {"salary"} + assert sim.household._holders == {} + assert "income_tax" not in sim.persons._holders + assert "rent" not in sim.household._holders + + +def test__given_dataset_simulation__then_untouched_variables_have_no_holders( + isolated_tax_benefit_system, +): + simulation = _safe_export_simulation(isolated_tax_benefit_system) + + assert "salary" in simulation.input_variables + assert "income_tax" not in simulation.persons._holders + assert "rent" not in simulation.household._holders + + +def test__given_reform_removing_input_variable__then_init_excludes_it_without_allocating_holders( + isolated_tax_benefit_system, +): + from policyengine_core.reforms import Reform + + class DropSalaryReform(Reform): + def apply(self): + del self.variables["salary"] + + situation = { + "persons": {"Alicia": {"salary": {"2022-01": 1000}, "age": {"2022-01": 30}}}, + "households": {"_": {"parents": ["Alicia"]}}, + } + + simulation = CountryTemplateSimulation( + tax_benefit_system=isolated_tax_benefit_system, + situation=situation, + reform=DropSalaryReform, + ) + + assert set(simulation.input_variables) == {"age"} + assert "income_tax" not in simulation.persons._holders + assert "rent" not in simulation.household._holders diff --git a/tests/core/test_storage_input_index.py b/tests/core/test_storage_input_index.py index ffbd75bcd..fa95a6d9c 100644 --- a/tests/core/test_storage_input_index.py +++ b/tests/core/test_storage_input_index.py @@ -237,9 +237,11 @@ def test_a_pickle_records_that_the_inputs_were_recorded(): def test_only_storages_holding_an_input_have_a_set(tax_benefit_system): simulation = build_simulation(tax_benefit_system) simulation.calculate("disposable_income", JANUARY) + # Ensure at least one empty storage exists alongside populated storages. + simulation.get_holder("postal_code") storages = _storages(simulation) - assert len(storages) == len(tax_benefit_system.variables) + assert set(storages) < set(tax_benefit_system.variables) with_inputs = {name for name, storage in storages.items() if storage._inputs} with_derived = { name diff --git a/tests/core/test_storage_shared_index.py b/tests/core/test_storage_shared_index.py index c919eb959..0dcc80ac8 100644 --- a/tests/core/test_storage_shared_index.py +++ b/tests/core/test_storage_shared_index.py @@ -234,10 +234,10 @@ def test_storage_from_a_pickle_with_an_empty_set_still_works(): def test_simulation_storages_have_no_sets_of_their_own(tax_benefit_system): simulation = build_simulation(tax_benefit_system) simulation.calculate("disposable_income", JANUARY) + simulation.get_holder("postal_code") storages = _storages(simulation) - # One storage per variable, as in every simulation. - assert len(storages) == len(tax_benefit_system.variables) + assert storages and set(storages) < set(tax_benefit_system.variables) assert all(storage._shared is _NOTHING_SHARED for storage in storages.values()) copy = simulation.clone() @@ -251,6 +251,8 @@ def test_branch_storages_have_a_set_only_for_the_arrays_they_share( ): simulation = build_simulation(tax_benefit_system) simulation.calculate("disposable_income", JANUARY) + # Include an empty storage so the branch checks unpopulated storages too. + simulation.get_holder("postal_code") with_arrays = { name for name, storage in _storages(simulation).items() if storage._arrays } diff --git a/tests/fixtures/set_input_helper_order.py b/tests/fixtures/set_input_helper_order.py index e0a184647..7ce0bb160 100644 --- a/tests/fixtures/set_input_helper_order.py +++ b/tests/fixtures/set_input_helper_order.py @@ -205,10 +205,10 @@ def build_simulation(system=None, on_disk: bool = False): warnings.simplefilter("ignore") simulation.memory_config = MemoryConfig(max_memory_occupation=0) # A holder takes its storage from the configuration it is created - # under, and building the simulation creates them all: drop the - # (still empty) ones of the fixture's variables. + # under, so drop any already-created empty holders for the fixture's + # variables before continuing with on-disk storage. for name in NAMES: - del simulation.get_variable_population(name)._holders[name] + simulation.get_variable_population(name)._holders.pop(name, None) return simulation From 7c9e3ad3d557660df19b2ea87c484b39720ca405 Mon Sep 17 00:00:00 2001 From: Deepak Jindal Date: Wed, 18 Nov 2026 14:12:00 -0800 Subject: [PATCH 2/3] Build input_variables from _user_input_keys --- policyengine_core/simulations/simulation.py | 50 ++++------ .../simulations/simulation_builder.py | 9 +- tests/core/test_simulations.py | 95 ++++++++++++++++++- 3 files changed, 122 insertions(+), 32 deletions(-) diff --git a/policyengine_core/simulations/simulation.py b/policyengine_core/simulations/simulation.py index d149b10bd..0768e4e6d 100644 --- a/policyengine_core/simulations/simulation.py +++ b/policyengine_core/simulations/simulation.py @@ -892,16 +892,26 @@ def __init__( self.calc = self.calculate self.df = self.calculate_dataframe - # ``build_from_dict`` already populated ``input_variables`` when building - # from ``situation``; only recompute if later initialization steps - # (``populations``, ``dataset``, or ``reform``) may have changed holders. - if ( - situation is None - or populations is not None - or dataset is not None - or self.reform is not None - ): - self.input_variables = self._collect_input_variables() + recorded_input_names = { + name for name, _branch_name, _period in self._user_input_keys + } + if populations is not None: + # Populations passed in may already hold values this simulation + # did not record. + recorded_input_names.update( + name + for population in self.populations.values() + for name, holder in population._holders.items() + if len(holder.get_known_periods()) > 0 + ) + # A reform may have moved a recorded variable to another entity, leaving + # its value in a holder this simulation no longer reads. + self.input_variables = [ + name + for name, variable in self.tax_benefit_system.variables.items() + if name in recorded_input_names + and name in self.populations[variable.entity.key]._holders + ] self.situation_input = situation if self.situation_input is not None: @@ -3295,26 +3305,6 @@ def check_macro_cache(self, variable_name: str, period: str) -> bool: return True - def _collect_input_variables(self) -> List[str]: - """Return active system variables that already have stored periods. - - Iterate only over holders that already exist on each population so - untouched variables do not eagerly allocate empty ``Holder`` objects. - """ - variables = self.tax_benefit_system.variables - input_variables = [] - for population in self.populations.values(): - entity_key = population.entity.key - for name, holder in population._holders.items(): - variable = variables.get(name) - if ( - variable is not None - and variable.entity.key == entity_key - and len(holder.get_known_periods()) > 0 - ): - input_variables.append(name) - return input_variables - def get_input_variables(self, include_computed_variables: bool = True) -> List[str]: """Return variable names stored as inputs on this simulation. diff --git a/policyengine_core/simulations/simulation_builder.py b/policyengine_core/simulations/simulation_builder.py index 69dd6be95..b88641166 100644 --- a/policyengine_core/simulations/simulation_builder.py +++ b/policyengine_core/simulations/simulation_builder.py @@ -186,7 +186,14 @@ def build_from_dict( tax_benefit_system, input_dict, simulation ) - simulation.input_variables = simulation._collect_input_variables() + recorded_input_names = { + name for name, _branch_name, _period in simulation._user_input_keys + } + simulation.input_variables = [ + name + for name in simulation.tax_benefit_system.variables + if name in recorded_input_names + ] return simulation diff --git a/tests/core/test_simulations.py b/tests/core/test_simulations.py index 4f2b7173d..a90cffc1e 100644 --- a/tests/core/test_simulations.py +++ b/tests/core/test_simulations.py @@ -297,6 +297,99 @@ def apply(self): reform=DropSalaryReform, ) - assert set(simulation.input_variables) == {"age"} + assert simulation.input_variables == ["age"] + assert "income_tax" not in simulation.persons._holders + assert "rent" not in simulation.household._holders + + +def test__given_dataset_simulation__then_input_variables_preserve_system_order( + isolated_tax_benefit_system, +): + system_variables = list(isolated_tax_benefit_system.variables) + dataframe = pd.DataFrame( + { + "household_weight__2022": [1.0], + "rent__2022-01": [1200.0], + "salary__2022-01": [500.0], + "age__2022-01": [30], + "person_household_role__2022": ["parent"], + "person_household_id__2022": [0], + "household_id__2022": [0], + "person_id__2022": [0], + } + ) + simulation = CountryTemplateSimulation( + tax_benefit_system=isolated_tax_benefit_system, + dataset=_safe_export_dataset(dataframe), + ) + + expected_names = { + "person_id", + "household_id", + "person_household_id", + "person_household_role", + "household_weight", + "salary", + "age", + "rent", + } + expected_order = [name for name in system_variables if name in expected_names] + assert simulation.input_variables == expected_order + assert "income_tax" not in simulation.persons._holders + assert "accommodation_size" not in simulation.household._holders + + +def test__given_prepopulated_populations__then_input_variables_include_their_inputs( + isolated_tax_benefit_system, +): + situation = { + "persons": {"Alicia": {"salary": {"2022-01": 1000}, "age": {"2022-01": 30}}}, + "households": {"_": {"parents": ["Alicia"]}}, + } + built = SimulationBuilder().build_from_dict(isolated_tax_benefit_system, situation) + + # Fresh Simulation given pre-filled populations has an empty _user_input_keys + # of its own, so __init__ must still discover existing holder inputs. + simulation = CountryTemplateSimulation( + tax_benefit_system=isolated_tax_benefit_system, + populations=built.populations, + ) + + expected_order = [ + name + for name in isolated_tax_benefit_system.variables + if name in {"salary", "age"} + ] + assert simulation.input_variables == expected_order + assert "income_tax" not in simulation.persons._holders + assert "rent" not in simulation.household._holders + + +def test__given_reform_moving_input_variable_entity__then_init_excludes_it_without_allocating_holders( + isolated_tax_benefit_system, +): + from policyengine_core.country_template.entities import Household + from policyengine_core.reforms import Reform + + class MoveSalaryToHousehold(Reform): + def apply(self): + class salary(Variable): + entity = Household + + self.update_variable(salary) + + situation = { + "persons": {"Alicia": {"salary": {"2022-01": 1000}, "age": {"2022-01": 30}}}, + "households": {"_": {"parents": ["Alicia"]}}, + } + + simulation = CountryTemplateSimulation( + tax_benefit_system=isolated_tax_benefit_system, + situation=situation, + reform=MoveSalaryToHousehold, + ) + + assert simulation.input_variables == ["age"] + assert "salary" not in simulation.household._holders assert "income_tax" not in simulation.persons._holders assert "rent" not in simulation.household._holders From 1579c8d8edb98b58ec27429903dcfda5c81b7f22 Mon Sep 17 00:00:00 2001 From: Deepak Jindal Date: Wed, 18 Nov 2026 16:19:36 -0800 Subject: [PATCH 3/3] Handle missing _user_input_keys in build_from_dict and clarify changelog --- .../avoid-eager-holder-allocation.fixed.md | 2 +- .../simulations/simulation_builder.py | 7 +++++- tests/core/test_simulations.py | 25 +++++++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/changelog.d/avoid-eager-holder-allocation.fixed.md b/changelog.d/avoid-eager-holder-allocation.fixed.md index f93c4c9a7..b8e59c467 100644 --- a/changelog.d/avoid-eager-holder-allocation.fixed.md +++ b/changelog.d/avoid-eager-holder-allocation.fixed.md @@ -1 +1 @@ -Avoid eagerly allocating empty ``Holder`` objects for every variable during ``Simulation.__init__`` and ``SimulationBuilder.build_from_dict`` when discovering ``input_variables``. +`Simulation.__init__` and `SimulationBuilder.build_from_dict` now build `input_variables` from recorded inputs without eagerly allocating empty `Holder` objects for every variable in the system, so `memory_config` set after setup applies to holders created afterwards (values can go to disk, and `variables_to_drop` variables are no longer stored). diff --git a/policyengine_core/simulations/simulation_builder.py b/policyengine_core/simulations/simulation_builder.py index b88641166..6b65cfe9b 100644 --- a/policyengine_core/simulations/simulation_builder.py +++ b/policyengine_core/simulations/simulation_builder.py @@ -186,8 +186,13 @@ def build_from_dict( tax_benefit_system, input_dict, simulation ) + # A subclass that replaces ``__init__`` has no record until an input is + # stored. recorded_input_names = { - name for name, _branch_name, _period in simulation._user_input_keys + name + for name, _branch_name, _period in getattr( + simulation, "_user_input_keys", () + ) } simulation.input_variables = [ name diff --git a/tests/core/test_simulations.py b/tests/core/test_simulations.py index a90cffc1e..8cf99c1a5 100644 --- a/tests/core/test_simulations.py +++ b/tests/core/test_simulations.py @@ -393,3 +393,28 @@ class salary(Variable): assert "salary" not in simulation.household._holders assert "income_tax" not in simulation.persons._holders assert "rent" not in simulation.household._holders + + +def test__given_simulation_without_user_input_keys__then_build_from_dict_leaves_input_variables_empty( + isolated_tax_benefit_system, +): + from policyengine_core.simulations import Simulation + + class CustomInitSimulation(Simulation): + def __init__(self, tax_benefit_system): + self.tax_benefit_system = tax_benefit_system + self.build_from_populations(tax_benefit_system.instantiate_entities()) + + simulation = CustomInitSimulation(isolated_tax_benefit_system) + situation = { + "persons": {"Alicia": {"income_tax": {"2022-01": None}}}, + "households": {"_": {"parents": ["Alicia"]}}, + } + + built = SimulationBuilder().build_from_dict( + isolated_tax_benefit_system, situation, simulation=simulation + ) + + assert built is simulation + assert not hasattr(built, "_user_input_keys") + assert built.input_variables == []