From da9892115d3da331727fa5940c86d1e6df5aec25 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:35:07 +0400 Subject: [PATCH 1/6] Return plain arrays to Microsimulation calls made inside a calculation Co-Authored-By: Claude Opus 5.5 --- ...-internal-microsimulation-calls.changed.md | 1 + .../simulations/microsimulation.py | 149 +++++- ...st_microsimulation_calculate_signatures.py | 10 + .../test_microsimulation_internal_calls.py | 481 ++++++++++++++++++ .../core/test_microsimulation_option_reads.py | 22 +- .../microsimulation_internal_calls.py | 310 +++++++++++ 6 files changed, 947 insertions(+), 26 deletions(-) create mode 100644 changelog.d/unweighted-internal-microsimulation-calls.changed.md create mode 100644 tests/core/test_microsimulation_internal_calls.py create mode 100644 tests/fixtures/microsimulation_internal_calls.py diff --git a/changelog.d/unweighted-internal-microsimulation-calls.changed.md b/changelog.d/unweighted-internal-microsimulation-calls.changed.md new file mode 100644 index 000000000..8d7d9c56c --- /dev/null +++ b/changelog.d/unweighted-internal-microsimulation-calls.changed.md @@ -0,0 +1 @@ +`Microsimulation.calculate`, `calculate_add` and `calculate_divide` now return plain arrays with enum codes to calls made inside a calculation (core's own reads and formula calls), so they no longer build weights that are then discarded, and enum reads with ADD or DIVIDE match a `Simulation` (#658). diff --git a/policyengine_core/simulations/microsimulation.py b/policyengine_core/simulations/microsimulation.py index 4ee5758a6..4364584e7 100644 --- a/policyengine_core/simulations/microsimulation.py +++ b/policyengine_core/simulations/microsimulation.py @@ -1,8 +1,10 @@ -from typing import Dict, Type +from contextvars import ContextVar, Token +from typing import Dict, Optional, Tuple, Type from microdf import MicroDataFrame, MicroSeries import numpy as np from policyengine_core.data.dataset import Dataset +from policyengine_core.enums import EnumArray from policyengine_core.periods import Period from policyengine_core.periods import period as get_period from policyengine_core.periods.config import MONTH, YEAR @@ -10,8 +12,72 @@ from policyengine_core.types import ArrayLike +class _RunningCall: + """The outermost ``Microsimulation`` calculate call running in a context.""" + + __slots__ = ("running",) + + def __init__(self): + self.running = True + + +# Set while a ``Microsimulation``'s ``calculate``, ``calculate_add`` or +# ``calculate_divide`` call runs, by the outermost such call in the context. +# Core reads variables for its own use through those same methods (the terms +# of ``adds``, a ``defined_for`` mask, each month of a monthly variable read +# over a year, the terms of ADD and DIVIDE), as do formulas. A call that +# arrives while this is set comes from inside a calculation, and its defaults +# give a plain array with enum codes. A thread started meanwhile has its own +# empty context, so its calls count as calls from outside; a context copied +# meanwhile sees the call until it ends, when ``running`` turns false. +_running_call: ContextVar[Optional[_RunningCall]] = ContextVar( + "_microsimulation_running_call", default=None +) + + +def _inside_a_calculation() -> bool: + call = _running_call.get() + return call is not None and call.running + + +def _start_call() -> Optional[Tuple[_RunningCall, Token]]: + """Record an outermost call; return what ``_end_call`` needs, or ``None``.""" + if _inside_a_calculation(): + return None + call = _RunningCall() + return call, _running_call.set(call) + + +def _end_call(started: Optional[Tuple[_RunningCall, Token]]) -> None: + if started is None: + return + call, token = started + call.running = False + _running_call.reset(token) + + +def _decode(values: ArrayLike) -> ArrayLike: + """Decode enum codes to member names. + + The twelfth of a code that ``calculate_divide`` returns over a month names + no member, so it is returned as it is, as a ``Simulation`` returns it. + """ + if isinstance(values, EnumArray) and np.issubdtype(values.dtype, np.integer): + return values.decode_to_str() + return values + + class Microsimulation(Simulation): - """A `Simulation` whose entities use weights to represent larger populations.""" + """A `Simulation` whose entities use weights to represent larger populations. + + ``calculate``, ``calculate_add`` and ``calculate_divide`` return a + weighted ``MicroSeries`` to a caller outside a calculation, and + ``calculate`` decodes enums to member names for it. A call made while + another of these calls runs (by core, or by a formula) returns a plain + array, with enums left as codes, as a ``Simulation`` does: core and + formulas use the values, not the weights. Passing ``use_weights`` or + ``decode_enums`` overrides either default. + """ def get_weights( self, variable_name: str, period: Period, map_to: str = None @@ -42,6 +108,7 @@ def get_weights( "household_weight", weight_period, use_weights=False, + decode_enums=False, ) return self._project_household_weights(household_weights, entity_key) @@ -68,14 +135,30 @@ def calculate( variable_name: str, period: Period = None, map_to: str = None, - use_weights: bool = True, - decode_enums: bool = True, + use_weights: Optional[bool] = None, + decode_enums: Optional[bool] = None, ) -> MicroSeries: + """Calculate ``variable_name`` for ``period``. + + ``use_weights`` and ``decode_enums`` default to ``True`` for a call + from outside a calculation and to ``False`` for a call made inside one + (see ``Microsimulation``). + """ + if use_weights is None or decode_enums is None: + outside = not _inside_a_calculation() + if use_weights is None: + use_weights = outside + if decode_enums is None: + decode_enums = outside if period is not None and not isinstance(period, Period): period = get_period(period) elif period is None and self.default_calculation_period is not None: period = get_period(self.default_calculation_period) - values = super().calculate(variable_name, period, map_to, decode_enums) + started = _start_call() + try: + values = super().calculate(variable_name, period, map_to, decode_enums) + finally: + _end_call(started) if not use_weights: return values weights = self.get_weights(variable_name, period, map_to) @@ -86,12 +169,31 @@ def calculate_add( variable_name: str, period: Period = None, map_to: str = None, - use_weights: bool = True, - decode_enums: bool = False, + use_weights: Optional[bool] = None, + decode_enums: Optional[bool] = None, ) -> MicroSeries: + """Sum ``variable_name`` over the subperiods of ``period``. + + ``use_weights`` defaults to ``True`` for a call from outside a + calculation and to ``False`` for a call made inside one (see + ``Microsimulation``). ``decode_enums`` defaults to ``False``. + """ + if use_weights is None: + use_weights = not _inside_a_calculation() + if decode_enums is None: + decode_enums = False # ``Population.__call__`` passes ``decode_enums`` with every ADD or - # DIVIDE read in a formula. - values = super().calculate_add(variable_name, period, decode_enums=decode_enums) + # DIVIDE read in a formula. The terms are read inside this call, so + # they are enum codes; the sum is decoded once, here. + started = _start_call() + try: + values = super().calculate_add( + variable_name, period, decode_enums=decode_enums + ) + finally: + _end_call(started) + if decode_enums: + values = _decode(values) if map_to is not None: source_entity = self.tax_benefit_system.get_variable( variable_name, check_existence=True @@ -107,12 +209,31 @@ def calculate_divide( variable_name: str, period: Period = None, map_to: str = None, - use_weights: bool = True, - decode_enums: bool = False, + use_weights: Optional[bool] = None, + decode_enums: Optional[bool] = None, ) -> MicroSeries: - values = super().calculate_divide( - variable_name, period, decode_enums=decode_enums - ) + """Divide the yearly value of ``variable_name`` to match ``period``. + + ``use_weights`` defaults to ``True`` for a call from outside a + calculation and to ``False`` for a call made inside one (see + ``Microsimulation``). ``decode_enums`` defaults to ``False``. + """ + if use_weights is None: + use_weights = not _inside_a_calculation() + if decode_enums is None: + decode_enums = False + # Over a year, ``Simulation.calculate_divide`` reads the variable with + # ``self.calculate``, inside this call, so it gets enum codes; the + # result is decoded once, here. + started = _start_call() + try: + values = super().calculate_divide( + variable_name, period, decode_enums=decode_enums + ) + finally: + _end_call(started) + if decode_enums: + values = _decode(values) if map_to is not None: source_entity = self.tax_benefit_system.get_variable( variable_name, check_existence=True diff --git a/tests/core/test_microsimulation_calculate_signatures.py b/tests/core/test_microsimulation_calculate_signatures.py index 21cc98289..e5fff091f 100644 --- a/tests/core/test_microsimulation_calculate_signatures.py +++ b/tests/core/test_microsimulation_calculate_signatures.py @@ -84,3 +84,13 @@ def test_keyword_call_to_a_simulation_method_binds_to_the_override(name, argumen f"takes: {error}", pytrace=False, ) + + +@pytest.mark.parametrize("name", ["calculate", "calculate_add", "calculate_divide"]) +def test_weights_and_enum_decoding_default_to_none(name): + # ``None`` lets the method choose by caller: a plain array with enum codes + # for a call made inside a calculation, weighted values otherwise. + parameters = inspect.signature(getattr(Microsimulation, name)).parameters + + assert parameters["use_weights"].default is None + assert parameters["decode_enums"].default is None diff --git a/tests/core/test_microsimulation_internal_calls.py b/tests/core/test_microsimulation_internal_calls.py new file mode 100644 index 000000000..f1d1a58fe --- /dev/null +++ b/tests/core/test_microsimulation_internal_calls.py @@ -0,0 +1,481 @@ +"""Calls made inside a calculation get plain arrays in a Microsimulation. + +``Microsimulation.calculate``, ``calculate_add`` and ``calculate_divide`` +return a weighted ``MicroSeries`` with enums decoded, for a caller that wants +weighted results. Core also calls these methods for itself while it +calculates: for each term of ``adds`` and ``subtracts``, for a +``defined_for`` mask, for each month of a monthly variable read over a year, +and so on. Each such call built weights from ``household_weight`` and decoded +enums, and core then threw the weights away before storing the value. +Decoding broke enum reads with ADD and DIVIDE (core issue #658). + +Now a call that arrives while another of these three methods is running, in +the same thread or a context copied from it, is a call from inside a +calculation. Left at their defaults, its ``use_weights`` and ``decode_enums`` +mean a plain array with enums left as codes, as a ``Simulation`` returns. A +call from outside a calculation gets weighted values, as before. An explicit +argument wins in both cases. +""" + +from contextlib import contextmanager +from unittest import mock + +import numpy as np +import pytest +from microdf import MicroDataFrame, MicroSeries + +from policyengine_core.enums import EnumArray +from policyengine_core.model_api import ADD, DIVIDE, MONTH +from policyengine_core.simulations import Microsimulation, Simulation +from tests.fixtures.microsimulation_internal_calls import ( + RECEIVED, + NoChange, + ProbeInterrupt, + microsimulation, + simulation, +) +from tests.fixtures.microsimulation_option_reads import DATASET_YEAR, READERS + +PERSON_WEIGHTS = [10.0, 10.0, 20.0] +GRANT = [120.0, 240.0, 0.0] +GRANT_A_MONTH = [10.0, 20.0, 0.0] +PAY = [78.0, 156.0, 390.0] +TENURE_CODES = [0, 1, 0] +TENURE_NAMES = ["OWNER", "RENTER", "OWNER"] + + +@pytest.fixture(autouse=True) +def clear_received(): + RECEIVED.clear() + yield + RECEIVED.clear() + + +def assert_array(value, expected): + """``value`` is a plain NumPy array (not an ``EnumArray``) of ``expected``.""" + assert type(value) is np.ndarray, type(value).__name__ + np.testing.assert_array_equal(value, expected) + + +def assert_codes(value, expected=TENURE_CODES): + assert type(value) is EnumArray, type(value).__name__ + np.testing.assert_array_equal(np.asarray(value), expected) + + +def assert_weighted(value, expected, weights=PERSON_WEIGHTS): + assert isinstance(value, MicroSeries), type(value).__name__ + assert value.tolist() == list(expected) + np.testing.assert_array_equal(value.weights, weights) + + +@contextmanager +def weight_reads(): + """Record the variable of each ``Microsimulation.get_weights`` call.""" + variables = [] + get_weights = Microsimulation.get_weights + + def record(self, variable_name, *arguments, **keywords): + variables.append(variable_name) + return get_weights(self, variable_name, *arguments, **keywords) + + with mock.patch.object(Microsimulation, "get_weights", record): + yield variables + + +@contextmanager +def household_weight_calculations(): + """Record the period of each ``_calculate`` of ``household_weight``.""" + periods = [] + calculate = Simulation._calculate + + def record(self, variable_name, period=None): + if variable_name == "household_weight": + periods.append(str(period)) + return calculate(self, variable_name, period) + + with mock.patch.object(Simulation, "_calculate", record): + yield periods + + +# Calls from inside a calculation. + +READS_OF_ITS_OWN_SIMULATION = { + "population read": (assert_array, GRANT), + "population enum read": (assert_codes, TENURE_CODES), + "calculate": (assert_array, GRANT), + "calculate enum": (assert_codes, TENURE_CODES), + "calculate again": (assert_array, GRANT), + "calculate_add": (assert_array, PAY), + "calculate_divide": (assert_array, GRANT), + "calculate_divide month": (assert_array, GRANT_A_MONTH), +} + + +@pytest.mark.parametrize("trace", [False, True], ids=["trace off", "trace on"]) +@pytest.mark.parametrize("label", list(READS_OF_ITS_OWN_SIMULATION)) +def test_formula_call_on_its_own_simulation_gets_a_plain_array(label, trace): + microsimulation(trace=trace).calculate("probe_reads_own_simulation", DATASET_YEAR) + + check, expected = READS_OF_ITS_OWN_SIMULATION[label] + check(RECEIVED[label], expected) + + +@pytest.mark.parametrize( + "label,expected", + [ + ("calculate use_weights=True", GRANT), + ("calculate_add use_weights=True", PAY), + ("calculate_divide use_weights=True", GRANT), + ], +) +def test_explicit_use_weights_inside_a_formula_gets_weights(label, expected): + microsimulation().calculate("probe_reads_own_simulation", DATASET_YEAR) + + assert_weighted(RECEIVED[label], expected) + + +def test_explicit_decode_enums_inside_a_formula_gets_names(): + microsimulation().calculate("probe_reads_own_simulation", DATASET_YEAR) + + names = RECEIVED["calculate decode_enums=True"] + assert type(names) is np.ndarray + assert names.tolist() == TENURE_NAMES + + +@pytest.mark.parametrize("trace", [False, True], ids=["trace off", "trace on"]) +def test_formula_call_on_a_branch_gets_a_plain_array(trace): + microsimulation(trace=trace).calculate("probe_reads_a_branch", DATASET_YEAR) + + assert_array(RECEIVED["branch calculate"], GRANT) + assert_codes(RECEIVED["branch calculate enum"]) + + +def test_reform_formula_call_on_its_baseline_gets_a_plain_array(): + microsimulation(reform=NoChange).calculate("probe_reads_its_baseline", DATASET_YEAR) + + assert_array(RECEIVED["baseline calculate"], GRANT) + + +def test_microsimulation_built_inside_a_microsimulation_formula_gets_arrays(): + microsimulation().calculate("probe_builds_a_microsimulation", DATASET_YEAR) + + assert_array(RECEIVED["new microsimulation calculate"], GRANT) + + +def test_microsimulation_built_inside_a_simulation_formula_gets_weights(): + # Only a running Microsimulation call makes a call internal. A plain + # Simulation's calculation does not, so this one is weighted, as before. + simulation().calculate("probe_builds_a_microsimulation", DATASET_YEAR) + + assert_weighted(RECEIVED["new microsimulation calculate"], GRANT) + + +def test_worker_thread_gets_weights_and_a_copied_context_gets_an_array(): + # A new thread starts with an empty context, so its call is a call from + # outside a calculation. A context copied while the formula runs carries + # the formula's calculation with it. + microsimulation().calculate("probe_reads_from_worker_threads", DATASET_YEAR) + + assert_weighted(RECEIVED["thread calculate"], GRANT) + assert_array(RECEIVED["thread calculate, copied context"], GRANT) + + +def test_context_copied_in_a_formula_gets_weights_after_the_calculation_ends(): + simulation_under_test = microsimulation() + simulation_under_test.calculate("probe_keeps_its_context", DATASET_YEAR) + + result = RECEIVED["context"].run( + simulation_under_test.calculate, "yearly_grant", DATASET_YEAR + ) + + assert_weighted(result, GRANT) + + +# Calls from outside a calculation: weighted, as before. + + +def test_top_level_calculate_is_weighted_and_decoded(): + simulation_under_test = microsimulation() + + assert_weighted(simulation_under_test.calculate("yearly_grant"), GRANT) + # The second read comes from Simulation.calculate's fast cache. + assert_weighted(simulation_under_test.calculate("yearly_grant"), GRANT) + assert_weighted(simulation_under_test.calculate("yearly_tenure"), TENURE_NAMES) + + +@pytest.mark.parametrize( + "method,variable,period,expected", + [ + ("calculate", "pay_and_grant", DATASET_YEAR, [139.5, 279.0, 97.5]), + ("calculate", "pay_if_granted", DATASET_YEAR, [78.0, 156.0, 0.0]), + ("calculate", "monthly_pay", DATASET_YEAR, PAY), + ("calculate", "yearly_grant", "2022-06", GRANT_A_MONTH), + ("calculate", "pay_added", DATASET_YEAR, PAY), + ("calculate_add", "monthly_pay", DATASET_YEAR, PAY), + ("calculate_divide", "yearly_grant", DATASET_YEAR, GRANT), + ("calculate_divide", "yearly_grant", "2022-06", GRANT_A_MONTH), + ("calculate_output", "pay_and_grant", DATASET_YEAR, [139.5, 279.0, 97.5]), + ], + ids=lambda argument: str(argument) if isinstance(argument, str) else None, +) +def test_top_level_call_builds_one_weighted_series(method, variable, period, expected): + simulation_under_test = microsimulation() + + with weight_reads() as variables: + result = getattr(simulation_under_test, method)(variable, period) + + assert_weighted(result, expected) + # Core's own reads (the terms of ``adds`` and ``subtracts``, the + # ``defined_for`` mask, each month of a monthly variable read over a year, + # the year of a yearly variable read over a month) build no weights. + assert variables == [variable] + + +def test_top_level_calculate_dataframe_is_weighted(): + result = microsimulation().calculate_dataframe( + ["yearly_grant", "pay_and_grant", "yearly_tenure"] + ) + + assert isinstance(result, MicroDataFrame) + assert result["yearly_grant"].tolist() == GRANT + assert result["pay_and_grant"].tolist() == [139.5, 279.0, 97.5] + assert result["yearly_tenure"].tolist() == TENURE_NAMES + np.testing.assert_array_equal(result.weights, PERSON_WEIGHTS) + + +def test_top_level_explicit_arguments_win(): + simulation_under_test = microsimulation() + + assert_array( + simulation_under_test.calculate("pay_and_grant", use_weights=False), + [139.5, 279.0, 97.5], + ) + assert_weighted( + simulation_under_test.calculate("yearly_tenure", decode_enums=False), + TENURE_CODES, + ) + assert_array( + simulation_under_test.calculate_add( + "monthly_pay", DATASET_YEAR, use_weights=False + ), + PAY, + ) + assert_array( + simulation_under_test.calculate_divide( + "yearly_grant", DATASET_YEAR, use_weights=False + ), + GRANT, + ) + + +def test_household_weight_is_calculated_once_per_top_level_call(): + simulation_under_test = microsimulation() + + with household_weight_calculations() as periods: + simulation_under_test.calculate("pay_and_grant") + + assert periods == [DATASET_YEAR] + + +# After errors, calls from outside a calculation are weighted again. + + +@pytest.mark.parametrize( + "method,variable,period,error,options", + [ + ("calculate", "probe_raises", DATASET_YEAR, RuntimeError, {}), + ("calculate", "probe_raises", DATASET_YEAR, RuntimeError, {"trace": True}), + ("calculate", "probe_raises_base_exception", DATASET_YEAR, ProbeInterrupt, {}), + ("calculate_add", "probe_monthly_raises", DATASET_YEAR, RuntimeError, {}), + ("calculate_divide", "probe_raises", "2022-06", RuntimeError, {}), + ("calculate_divide", "probe_raises", DATASET_YEAR, RuntimeError, {}), + ], + ids=[ + "formula raises", + "formula raises, trace on", + "formula raises a BaseException", + "ADD term raises", + "month DIVIDE term raises", + "year DIVIDE term raises", + ], +) +def test_call_after_an_error_is_weighted(method, variable, period, error, options): + simulation_under_test = microsimulation(**options) + + with pytest.raises(error): + getattr(simulation_under_test, method)(variable, period) + + assert_weighted(simulation_under_test.calculate("yearly_tenure"), TENURE_NAMES) + assert_weighted(microsimulation().calculate("yearly_grant"), GRANT) + + +def test_formula_that_catches_a_branch_error_still_gets_an_array(): + simulation_under_test = microsimulation() + + simulation_under_test.calculate("probe_catches_a_branch_error") + + assert_array(RECEIVED["calculate after a caught branch error"], GRANT) + assert_weighted(simulation_under_test.calculate("yearly_grant"), GRANT) + + +def test_calculation_that_runs_again_after_an_input_change_gets_arrays(): + # The probe sets an input while it runs, which can make core run the + # outermost calculation again. Whatever core does then, a Microsimulation + # does as a Simulation does, and its formula's reads are plain arrays. + plain = simulation() + plain.calculate("probe_sets_an_input") + plain_read = RECEIVED.pop("calculate after setting an input") + expected = plain.calculate("probe_sets_an_input") + simulation_under_test = microsimulation() + + result = simulation_under_test.calculate("probe_sets_an_input") + + assert_array(RECEIVED["calculate after setting an input"], plain_read) + assert_weighted(result, expected) + assert_weighted( + simulation_under_test.calculate("yearly_grant"), plain.calculate("yearly_grant") + ) + + +# Values: the same as a Simulation's. + +REQUESTS = [ + *[ + (reader.__name__, "2022-06" if reader.definition_period == MONTH else "2022") + for reader in READERS + ], + ("pay_and_grant", DATASET_YEAR), + ("pay_if_granted", DATASET_YEAR), + ("monthly_pay", DATASET_YEAR), + ("yearly_grant", "2022-06"), + ("owner_by_add", DATASET_YEAR), + ("owner_by_divide", DATASET_YEAR), + ("owner_twelfth", "2022-06"), +] + + +def stored_arrays(simulation_under_test) -> dict: + """Every array the simulation stores, by variable and period.""" + arrays = {} + for name in simulation_under_test.tax_benefit_system.variables: + for period in simulation_under_test.get_known_periods(name): + arrays[(name, str(period))] = simulation_under_test.get_array(name, period) + return arrays + + +def describe(array) -> tuple: + """The type, shape, contents and writability of an array.""" + array = np.asarray(array) + # An object array's bytes are the addresses of its items. + contents = array.tolist() if array.dtype == object else array.tobytes() + return array.dtype.str, array.shape, contents, array.flags.writeable + + +def test_microsimulation_stores_and_returns_what_a_simulation_does(): + plain, micro = simulation(), microsimulation() + + for variable, period in REQUESTS: + plain_result = plain.calculate(variable, period) + micro_result = micro.calculate(variable, period, use_weights=False) + assert type(micro_result) is type(plain_result), variable + assert describe(micro_result) == describe(plain_result), variable + # Top-level weighted reads store the same values too. + micro.calculate(variable, period) + + plain_arrays, micro_arrays = stored_arrays(plain), stored_arrays(micro) + assert sorted(micro_arrays) == sorted(plain_arrays) + for key, plain_array in plain_arrays.items(): + assert describe(micro_arrays[key]) == describe(plain_array), key + + +# Enum reads with ADD and DIVIDE (core issue #658). + +ENUM_READS = [ + ("yearly_tenure", DATASET_YEAR, ADD), + ("yearly_tenure", DATASET_YEAR, DIVIDE), + ("yearly_tenure", "2022-06", DIVIDE), +] + + +@pytest.mark.parametrize( + "variable,period,option", ENUM_READS, ids=lambda argument: str(argument) +) +def test_enum_read_with_an_option_matches_a_simulation(variable, period, option): + plain_read, micro_read = ( + each.populations["person"](variable, period, options=[option]) + for each in (simulation(), microsimulation()) + ) + + assert type(micro_read) is type(plain_read) is EnumArray + assert describe(micro_read) == describe(plain_read) + + +@pytest.mark.parametrize( + "variable,period,expected", + [ + ("owner_by_add", DATASET_YEAR, [True, False, True]), + ("owner_by_divide", DATASET_YEAR, [True, False, True]), + # Stored as the variable's float32. + ("owner_twelfth", "2022-06", [0.0, np.float32(1 / 12), 0.0]), + ], +) +def test_formula_reading_an_enum_with_an_option_matches_a_simulation( + variable, period, expected +): + micro_result = microsimulation().calculate(variable, period, use_weights=False) + + np.testing.assert_array_equal(micro_result, expected) + assert describe(micro_result) == describe(simulation().calculate(variable, period)) + + +def test_top_level_enum_divide_over_a_year_returns_weighted_codes(): + simulation_under_test = microsimulation() + + with weight_reads() as variables: + default = simulation_under_test.calculate_divide("yearly_tenure", DATASET_YEAR) + explicit = simulation_under_test.calculate_divide( + "yearly_tenure", DATASET_YEAR, decode_enums=False + ) + decoded = simulation_under_test.calculate_divide( + "yearly_tenure", DATASET_YEAR, decode_enums=True + ) + + assert_weighted(default, TENURE_CODES) + assert_weighted(explicit, TENURE_CODES) + assert_weighted(decoded, TENURE_NAMES) + assert variables == ["yearly_tenure"] + + +def test_top_level_enum_add_over_a_year_decodes_when_asked(): + simulation_under_test = microsimulation() + + assert_weighted( + simulation_under_test.calculate_add("yearly_tenure", DATASET_YEAR), + TENURE_CODES, + ) + assert_weighted( + simulation_under_test.calculate_add( + "yearly_tenure", DATASET_YEAR, decode_enums=True + ), + TENURE_NAMES, + ) + + +def test_top_level_enum_divide_over_a_month_is_not_decoded(): + # A twelfth of a code names no member. A Simulation ignores + # ``decode_enums`` here, and so does a Microsimulation. + result = microsimulation().calculate_divide( + "yearly_tenure", "2022-06", decode_enums=True, use_weights=False + ) + + assert type(result) is EnumArray + np.testing.assert_array_equal(np.asarray(result), [0.0, 1 / 12, 0.0]) + + +def test_unweighted_top_level_add_returns_a_plain_array(): + result = microsimulation().calculate_add( + "monthly_pay", DATASET_YEAR, use_weights=False + ) + + assert_array(result, PAY) + assert result.sum() == 624.0 diff --git a/tests/core/test_microsimulation_option_reads.py b/tests/core/test_microsimulation_option_reads.py index f6bde27bd..568a57317 100644 --- a/tests/core/test_microsimulation_option_reads.py +++ b/tests/core/test_microsimulation_option_reads.py @@ -201,21 +201,19 @@ def test_microsimulation_add_and_divide_take_decode_enums( ) -def test_enum_read_with_divide_over_a_year_gives_member_names_in_a_microsimulation(): - # This records a difference that remains. Over a year, - # Simulation.calculate_divide reads the variable with self.calculate, - # which in a Microsimulation decodes enums by default. So a - # Microsimulation returns member names where a Simulation returns an - # EnumArray, and comparing the names with a member gives False for - # everyone. No policyengine-us formula reads an Enum with ADD or DIVIDE. +def test_enum_read_with_divide_over_a_year_gives_codes_in_a_microsimulation(): + # Over a year, Simulation.calculate_divide reads the variable with + # self.calculate. In a Microsimulation that read once decoded enums, so + # the formula got member names, and comparing them with a member gave + # False for everyone (core issue #658). The read is made inside the + # calculate_divide call, so it now keeps the codes, as a Simulation does. inputs = dataclasses.replace(EXAMPLE, yearly_tenure=("OWNER", "RENTER", "OWNER")) simulation_read, microsimulation_read = ( each.populations["person"]("yearly_tenure", DATASET_YEAR, options=[DIVIDE]) for each in simulations(inputs) ) - assert isinstance(simulation_read, EnumArray) - assert simulation_read.decode_to_str().tolist() == ["OWNER", "RENTER", "OWNER"] - assert (simulation_read == Tenure.OWNER).tolist() == [True, False, True] - assert microsimulation_read.tolist() == ["OWNER", "RENTER", "OWNER"] - assert (microsimulation_read == Tenure.OWNER).tolist() == [False, False, False] + for read in (simulation_read, microsimulation_read): + assert isinstance(read, EnumArray) + assert read.decode_to_str().tolist() == ["OWNER", "RENTER", "OWNER"] + assert (read == Tenure.OWNER).tolist() == [True, False, True] diff --git a/tests/fixtures/microsimulation_internal_calls.py b/tests/fixtures/microsimulation_internal_calls.py new file mode 100644 index 000000000..a4b6dc4a9 --- /dev/null +++ b/tests/fixtures/microsimulation_internal_calls.py @@ -0,0 +1,310 @@ +"""A small system for checking what calls made inside a calculation return. + +Used by ``tests/core/test_microsimulation_internal_calls.py``. It adds to the +ADD and DIVIDE fixture (``microsimulation_option_reads``): + +- variables that core reads for itself: one with ``adds`` and ``subtracts``, + one with ``defined_for``; +- probe formulas. Each makes one kind of call from inside a calculation and + stores what the call returns in ``RECEIVED``, under a label. +""" + +import contextvars +from concurrent.futures import ThreadPoolExecutor +from functools import lru_cache +from typing import Dict + +import numpy as np + +from policyengine_core.country_template import CountryTaxBenefitSystem +from policyengine_core.country_template.entities import Person +from policyengine_core.data import Dataset +from policyengine_core.model_api import ADD, DIVIDE, MONTH, YEAR, Variable +from policyengine_core.reforms import Reform +from policyengine_core.simulations import Microsimulation, Simulation +from tests.fixtures.microsimulation_option_reads import ( + DATASET_YEAR, + READERS, + SOURCES, + Inputs, + Tenure, + dataframe, + yearly_tenure, +) + +EXAMPLE = Inputs( + household_of_person=(0, 0, 1), + household_weight=(10.0, 20.0), + monthly_pay=( + tuple(1.0 * month for month in range(1, 13)), + tuple(2.0 * month for month in range(1, 13)), + tuple(5.0 * month for month in range(1, 13)), + ), + yearly_grant=(120.0, 240.0, 0.0), + household_monthly_fee=((5.0,) * 12, (0.0,) * 12), + household_yearly_grant=(12.0, 36.0), + yearly_tenure=("OWNER", "RENTER", "OWNER"), +) + +# What the probe formulas received, by label. Tests clear it before each run. +RECEIVED: Dict[str, object] = {} + + +# Variables core reads for itself, with no formula of their own. + + +class pay_and_grant(Variable): + value_type = float + entity = Person + definition_period = YEAR + label = "Pay summed over the year, plus the grant, less the take-home pay" + # ``monthly_pay`` and ``monthly_take_home`` are monthly, so each is read + # over the year by summing its months. + adds = ["monthly_pay", "yearly_grant"] + subtracts = ["monthly_take_home"] + + +class pay_if_granted(Variable): + value_type = float + entity = Person + definition_period = YEAR + label = "Pay summed over the year, for people with a grant" + defined_for = "yearly_grant" + + def formula(person, period, parameters): + return person("monthly_pay", period, options=[ADD]) + + +# Formulas that read an Enum with ADD or DIVIDE (core issue #658). + + +class owner_by_add(Variable): + value_type = bool + entity = Person + definition_period = YEAR + label = "Owner, from a read of the tenure with ADD" + + def formula(person, period, parameters): + return person("yearly_tenure", period, options=[ADD]) == Tenure.OWNER + + +class owner_by_divide(Variable): + value_type = bool + entity = Person + definition_period = YEAR + label = "Owner, from a read of the tenure with DIVIDE" + + def formula(person, period, parameters): + return person("yearly_tenure", period, options=[DIVIDE]) == Tenure.OWNER + + +class owner_twelfth(Variable): + value_type = float + entity = Person + definition_period = MONTH + label = "A twelfth of the tenure code, from a read with DIVIDE" + + def formula(person, period, parameters): + return np.asarray(person("yearly_tenure", period, options=[DIVIDE])) + + +# Probe formulas. + + +def _zeros(person): + return np.zeros(person.count) + + +def _reads_own_simulation(person, period, parameters): + simulation = person.simulation + RECEIVED["population read"] = person("yearly_grant", period) + RECEIVED["population enum read"] = person("yearly_tenure", period) + RECEIVED["calculate"] = simulation.calculate("yearly_grant", period) + RECEIVED["calculate enum"] = simulation.calculate("yearly_tenure", period) + # Already stored by the read above: Simulation.calculate returns it from + # its fast cache. + RECEIVED["calculate again"] = simulation.calculate("yearly_grant", period) + RECEIVED["calculate_add"] = simulation.calculate_add("monthly_pay", period) + RECEIVED["calculate_divide"] = simulation.calculate_divide("yearly_grant", period) + RECEIVED["calculate_divide month"] = simulation.calculate_divide( + "yearly_grant", period.first_month + ) + RECEIVED["calculate use_weights=True"] = simulation.calculate( + "yearly_grant", period, use_weights=True + ) + RECEIVED["calculate decode_enums=True"] = simulation.calculate( + "yearly_tenure", period, decode_enums=True + ) + RECEIVED["calculate_add use_weights=True"] = simulation.calculate_add( + "monthly_pay", period, use_weights=True + ) + RECEIVED["calculate_divide use_weights=True"] = simulation.calculate_divide( + "yearly_grant", period, use_weights=True + ) + return _zeros(person) + + +def _reads_a_branch(person, period, parameters): + branch = person.simulation.get_branch("probe_branch") + RECEIVED["branch calculate"] = branch.calculate("yearly_grant", period) + RECEIVED["branch calculate enum"] = branch.calculate("yearly_tenure", period) + return _zeros(person) + + +def _reads_its_baseline(person, period, parameters): + RECEIVED["baseline calculate"] = person.simulation.baseline.calculate( + "yearly_grant", period + ) + return _zeros(person) + + +def _builds_a_microsimulation(person, period, parameters): + inner = ProbeMicrosimulation(dataset=example_dataset()) + RECEIVED["new microsimulation calculate"] = inner.calculate("yearly_grant", period) + return _zeros(person) + + +def _reads_from_worker_threads(person, period, parameters): + simulation = person.simulation + with ThreadPoolExecutor(max_workers=1) as pool: + RECEIVED["thread calculate"] = pool.submit( + simulation.calculate, "yearly_grant", period + ).result() + context = contextvars.copy_context() + RECEIVED["thread calculate, copied context"] = pool.submit( + context.run, simulation.calculate, "yearly_grant", period + ).result() + return _zeros(person) + + +def _keeps_its_context(person, period, parameters): + # A test runs a calculation in this context after the formula returns. + RECEIVED["context"] = contextvars.copy_context() + return _zeros(person) + + +def _raises(person, period, parameters): + person.simulation.calculate("yearly_grant", period.this_year) + raise RuntimeError("probe error") + + +class ProbeInterrupt(BaseException): + """Raised by a probe formula: not an ``Exception``, as ``KeyboardInterrupt``.""" + + +def _raises_base_exception(person, period, parameters): + person.simulation.calculate("yearly_grant", period) + raise ProbeInterrupt() + + +def _catches_a_branch_error(person, period, parameters): + branch = person.simulation.get_branch("probe_error_branch") + try: + branch.calculate("probe_raises", period) + except RuntimeError: + pass + RECEIVED["calculate after a caught branch error"] = person.simulation.calculate( + "yearly_grant", period + ) + return _zeros(person) + + +def _sets_an_input(person, period, parameters): + # Setting an input drops values computed from it, so the outermost + # calculation runs again. + simulation = person.simulation + simulation.calculate("pay_and_grant", period) + simulation.set_input("yearly_grant", period, np.full(person.count, 1234.0)) + RECEIVED["calculate after setting an input"] = simulation.calculate( + "pay_and_grant", period + ) + return simulation.calculate("pay_and_grant", period) + + +def _probe(name, formula, definition_period=YEAR): + return type( + name, + (Variable,), + { + "value_type": float, + "entity": Person, + "definition_period": definition_period, + "label": name, + "formula": formula, + }, + ) + + +PROBES = [ + _probe("probe_reads_own_simulation", _reads_own_simulation), + _probe("probe_reads_a_branch", _reads_a_branch), + _probe("probe_reads_its_baseline", _reads_its_baseline), + _probe("probe_builds_a_microsimulation", _builds_a_microsimulation), + _probe("probe_reads_from_worker_threads", _reads_from_worker_threads), + _probe("probe_keeps_its_context", _keeps_its_context), + _probe("probe_raises", _raises), + _probe("probe_monthly_raises", _raises, definition_period=MONTH), + _probe("probe_raises_base_exception", _raises_base_exception), + _probe("probe_catches_a_branch_error", _catches_a_branch_error), + _probe("probe_sets_an_input", _sets_an_input), +] + + +class ProbeSystem(CountryTaxBenefitSystem): + """The country template's system with this module's variables added. + + A simulation with a reform builds the reform's system, and its baseline's, + from its ``default_tax_benefit_system`` class, so the variables are added + in ``__init__``. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + for variable in [ + *SOURCES, + *READERS, + yearly_tenure, + pay_and_grant, + pay_if_granted, + owner_by_add, + owner_by_divide, + owner_twelfth, + *PROBES, + ]: + self.add_variable(variable) + + +@lru_cache(maxsize=None) +def tax_benefit_system() -> ProbeSystem: + return ProbeSystem() + + +class ProbeMicrosimulation(Microsimulation): + default_tax_benefit_system = ProbeSystem + default_tax_benefit_system_instance = tax_benefit_system() + default_calculation_period = DATASET_YEAR + + +class ProbeSimulation(Simulation): + default_tax_benefit_system = ProbeSystem + default_tax_benefit_system_instance = tax_benefit_system() + default_calculation_period = DATASET_YEAR + + +class NoChange(Reform): + """A reform that changes nothing, so the simulation has a baseline.""" + + def apply(self): + pass + + +def example_dataset(inputs: Inputs = EXAMPLE) -> Dataset: + return Dataset.from_dataframe(dataframe(inputs), DATASET_YEAR) + + +def microsimulation(**options) -> ProbeMicrosimulation: + return ProbeMicrosimulation(dataset=example_dataset(), **options) + + +def simulation(**options) -> ProbeSimulation: + return ProbeSimulation(dataset=example_dataset(), **options) From 52ea14ab21160f58f3a83fc49750d40c5146d75f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:54:14 +0400 Subject: [PATCH 2/6] Record every Microsimulation call so a copied context stays inside while its own call runs Co-Authored-By: Claude Opus 5.5 --- ...-internal-microsimulation-calls.changed.md | 2 +- ...ed-internal-microsimulation-calls.fixed.md | 1 + .../simulations/microsimulation.py | 36 ++++++++++--------- .../test_microsimulation_internal_calls.py | 30 +++++++++++++--- ...t_microsimulation_option_reads_property.py | 5 ++- .../microsimulation_internal_calls.py | 28 +++++++++++++++ 6 files changed, 76 insertions(+), 26 deletions(-) create mode 100644 changelog.d/unweighted-internal-microsimulation-calls.fixed.md diff --git a/changelog.d/unweighted-internal-microsimulation-calls.changed.md b/changelog.d/unweighted-internal-microsimulation-calls.changed.md index 8d7d9c56c..e5eac8f37 100644 --- a/changelog.d/unweighted-internal-microsimulation-calls.changed.md +++ b/changelog.d/unweighted-internal-microsimulation-calls.changed.md @@ -1 +1 @@ -`Microsimulation.calculate`, `calculate_add` and `calculate_divide` now return plain arrays with enum codes to calls made inside a calculation (core's own reads and formula calls), so they no longer build weights that are then discarded, and enum reads with ADD or DIVIDE match a `Simulation` (#658). +`Microsimulation.calculate`, `calculate_add` and `calculate_divide` now default to `use_weights=None` and `decode_enums=None`, which return a plain array with enum codes to a call made while another of these calls runs (core's own reads and formula calls) and weighted values to any other call, so internal reads no longer build weights that are then discarded. diff --git a/changelog.d/unweighted-internal-microsimulation-calls.fixed.md b/changelog.d/unweighted-internal-microsimulation-calls.fixed.md new file mode 100644 index 000000000..cea39e74f --- /dev/null +++ b/changelog.d/unweighted-internal-microsimulation-calls.fixed.md @@ -0,0 +1 @@ +In a `Microsimulation`, enum reads with ADD or DIVIDE now match a `Simulation` (#658), and `use_weights=False` now returns a plain array from `calculate_add`, `calculate_divide` and a `calculate` over a period of another length, which returned a weighted `MicroSeries` before. diff --git a/policyengine_core/simulations/microsimulation.py b/policyengine_core/simulations/microsimulation.py index 4364584e7..808b9a2d9 100644 --- a/policyengine_core/simulations/microsimulation.py +++ b/policyengine_core/simulations/microsimulation.py @@ -13,7 +13,7 @@ class _RunningCall: - """The outermost ``Microsimulation`` calculate call running in a context.""" + """A ``Microsimulation`` calculate call, running until ``running`` is false.""" __slots__ = ("running",) @@ -21,15 +21,21 @@ def __init__(self): self.running = True -# Set while a ``Microsimulation``'s ``calculate``, ``calculate_add`` or -# ``calculate_divide`` call runs, by the outermost such call in the context. -# Core reads variables for its own use through those same methods (the terms -# of ``adds``, a ``defined_for`` mask, each month of a monthly variable read -# over a year, the terms of ADD and DIVIDE), as do formulas. A call that -# arrives while this is set comes from inside a calculation, and its defaults -# give a plain array with enum codes. A thread started meanwhile has its own -# empty context, so its calls count as calls from outside; a context copied -# meanwhile sees the call until it ends, when ``running`` turns false. +# The innermost ``Microsimulation`` ``calculate``, ``calculate_add`` or +# ``calculate_divide`` call running in this context. Core reads variables for +# its own use through those same methods (the terms of ``adds``, a +# ``defined_for`` mask, each month of a monthly variable read over a year, the +# terms of ADD and DIVIDE), as do formulas. A call that arrives while one is +# running comes from inside a calculation, and its defaults give a plain array +# with enum codes. +# +# A context copied while a call runs (for a worker thread, say) holds that +# call's record, and sees the call end when ``running`` turns false. A call +# made in the copy records itself there, so what it reads stays inside while +# it runs. A thread starts with an empty context, so its calls come from +# outside, unless Python gives new threads a copy of the starting thread's +# context (``-X thread_inherit_context=1``, the default on free-threaded +# builds). _running_call: ContextVar[Optional[_RunningCall]] = ContextVar( "_microsimulation_running_call", default=None ) @@ -40,17 +46,13 @@ def _inside_a_calculation() -> bool: return call is not None and call.running -def _start_call() -> Optional[Tuple[_RunningCall, Token]]: - """Record an outermost call; return what ``_end_call`` needs, or ``None``.""" - if _inside_a_calculation(): - return None +def _start_call() -> Tuple[_RunningCall, Token]: + """Record a call as running; return what ``_end_call`` needs.""" call = _RunningCall() return call, _running_call.set(call) -def _end_call(started: Optional[Tuple[_RunningCall, Token]]) -> None: - if started is None: - return +def _end_call(started: Tuple[_RunningCall, Token]) -> None: call, token = started call.running = False _running_call.reset(token) diff --git a/tests/core/test_microsimulation_internal_calls.py b/tests/core/test_microsimulation_internal_calls.py index f1d1a58fe..20c0cf3a2 100644 --- a/tests/core/test_microsimulation_internal_calls.py +++ b/tests/core/test_microsimulation_internal_calls.py @@ -17,6 +17,7 @@ argument wins in both cases. """ +import sys from contextlib import contextmanager from unittest import mock @@ -170,13 +171,19 @@ def test_microsimulation_built_inside_a_simulation_formula_gets_weights(): assert_weighted(RECEIVED["new microsimulation calculate"], GRANT) -def test_worker_thread_gets_weights_and_a_copied_context_gets_an_array(): - # A new thread starts with an empty context, so its call is a call from - # outside a calculation. A context copied while the formula runs carries - # the formula's calculation with it. +def test_worker_thread_follows_its_context(): + # A thread started inside a formula runs in an empty context, unless + # Python is set to give new threads a copy of the starting thread's + # context (``-X thread_inherit_context=1``, the default on free-threaded + # builds). In an empty context the thread's call is a call from outside a + # calculation. A context copied while the formula runs carries the + # formula's calculation with it. microsimulation().calculate("probe_reads_from_worker_threads", DATASET_YEAR) - assert_weighted(RECEIVED["thread calculate"], GRANT) + if getattr(sys.flags, "thread_inherit_context", 0): + assert_array(RECEIVED["thread calculate"], GRANT) + else: + assert_weighted(RECEIVED["thread calculate"], GRANT) assert_array(RECEIVED["thread calculate, copied context"], GRANT) @@ -191,6 +198,19 @@ def test_context_copied_in_a_formula_gets_weights_after_the_calculation_ends(): assert_weighted(result, GRANT) +def test_call_in_a_copied_context_stays_inside_after_the_first_call_ends(): + # The worker's own calculate call is still running when it reads, so its + # reads are inside a calculation, though the call whose context it copied + # has ended. + microsimulation().calculate("probe_starts_a_worker", DATASET_YEAR) + RECEIVED["release worker"].set() + RECEIVED["worker"].join(timeout=30) + + assert not RECEIVED["worker"].is_alive() + assert_array(RECEIVED["worker calculate"], GRANT) + assert_codes(RECEIVED["worker calculate enum"]) + + # Calls from outside a calculation: weighted, as before. diff --git a/tests/core/test_microsimulation_option_reads_property.py b/tests/core/test_microsimulation_option_reads_property.py index 61bc0324e..4f6371e52 100644 --- a/tests/core/test_microsimulation_option_reads_property.py +++ b/tests/core/test_microsimulation_option_reads_property.py @@ -20,9 +20,8 @@ results also equal the same arithmetic done in NumPy, so the two simulations cannot agree on a wrong value. -Enum variables are left out: -``test_enum_read_with_divide_over_a_year_gives_member_names_in_a_microsimulation`` -records how their reads differ. +Enum variables are left out: ``test_microsimulation_internal_calls.py`` checks +their ADD and DIVIDE reads against a Simulation (core issue #658). """ import numpy as np diff --git a/tests/fixtures/microsimulation_internal_calls.py b/tests/fixtures/microsimulation_internal_calls.py index a4b6dc4a9..597c8b0d0 100644 --- a/tests/fixtures/microsimulation_internal_calls.py +++ b/tests/fixtures/microsimulation_internal_calls.py @@ -10,6 +10,7 @@ """ import contextvars +import threading from concurrent.futures import ThreadPoolExecutor from functools import lru_cache from typing import Dict @@ -183,6 +184,31 @@ def _keeps_its_context(person, period, parameters): return _zeros(person) +def _starts_a_worker(person, period, parameters): + # The worker runs a calculation in a context copied here, and keeps + # running after this formula and its calculation return. A test releases + # it then. + release = threading.Event() + context = contextvars.copy_context() + inner = ProbeMicrosimulation(dataset=example_dataset()) + worker = threading.Thread( + target=context.run, + args=(inner.calculate, "probe_waits_then_reads", period), + ) + RECEIVED["release worker"] = release + RECEIVED["worker"] = worker + worker.start() + return _zeros(person) + + +def _waits_then_reads(person, period, parameters): + RECEIVED["release worker"].wait(timeout=30) + simulation = person.simulation + RECEIVED["worker calculate"] = simulation.calculate("yearly_grant", period) + RECEIVED["worker calculate enum"] = simulation.calculate("yearly_tenure", period) + return _zeros(person) + + def _raises(person, period, parameters): person.simulation.calculate("yearly_grant", period.this_year) raise RuntimeError("probe error") @@ -242,6 +268,8 @@ def _probe(name, formula, definition_period=YEAR): _probe("probe_builds_a_microsimulation", _builds_a_microsimulation), _probe("probe_reads_from_worker_threads", _reads_from_worker_threads), _probe("probe_keeps_its_context", _keeps_its_context), + _probe("probe_starts_a_worker", _starts_a_worker), + _probe("probe_waits_then_reads", _waits_then_reads), _probe("probe_raises", _raises), _probe("probe_monthly_raises", _raises, definition_period=MONTH), _probe("probe_raises_base_exception", _raises_base_exception), From c47cdbb9ca7719d576bfd5ce06970dcdb04cfa0d Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 14:21:54 +0400 Subject: [PATCH 3/6] Keep a context copied in a nested call inside while an enclosing call runs Fix tests that did not reach the fast cache or the rerun after an input change, wait for the worker thread's call to start, and narrow the changelog fragments to what changed. Co-Authored-By: Claude Opus 5.5 --- ...-internal-microsimulation-calls.changed.md | 2 +- ...ed-internal-microsimulation-calls.fixed.md | 2 +- .../simulations/microsimulation.py | 31 +++++++---- .../test_microsimulation_internal_calls.py | 45 ++++++++++------ .../microsimulation_internal_calls.py | 53 ++++++++++++++----- 5 files changed, 91 insertions(+), 42 deletions(-) diff --git a/changelog.d/unweighted-internal-microsimulation-calls.changed.md b/changelog.d/unweighted-internal-microsimulation-calls.changed.md index e5eac8f37..9ad471256 100644 --- a/changelog.d/unweighted-internal-microsimulation-calls.changed.md +++ b/changelog.d/unweighted-internal-microsimulation-calls.changed.md @@ -1 +1 @@ -`Microsimulation.calculate`, `calculate_add` and `calculate_divide` now default to `use_weights=None` and `decode_enums=None`, which return a plain array with enum codes to a call made while another of these calls runs (core's own reads and formula calls) and weighted values to any other call, so internal reads no longer build weights that are then discarded. +`Microsimulation.calculate`, `calculate_add` and `calculate_divide` now default to `use_weights=None` and `decode_enums=None`, which return a plain array with enum codes to a call made while another of these calls runs in the same thread or a context copied from it (core's own reads and formula calls), and weighted values to any other call, so internal reads no longer build weights that are then discarded. diff --git a/changelog.d/unweighted-internal-microsimulation-calls.fixed.md b/changelog.d/unweighted-internal-microsimulation-calls.fixed.md index cea39e74f..bd7b21b03 100644 --- a/changelog.d/unweighted-internal-microsimulation-calls.fixed.md +++ b/changelog.d/unweighted-internal-microsimulation-calls.fixed.md @@ -1 +1 @@ -In a `Microsimulation`, enum reads with ADD or DIVIDE now match a `Simulation` (#658), and `use_weights=False` now returns a plain array from `calculate_add`, `calculate_divide` and a `calculate` over a period of another length, which returned a weighted `MicroSeries` before. +In a `Microsimulation`, enum reads with ADD or DIVIDE now match a `Simulation`, so a top-level `calculate_divide` of an Enum over a year returns codes unless `decode_enums=True` (#658), and `use_weights=False` without `map_to` now returns a plain array from `calculate_add`, `calculate_divide` and a `calculate` of a FLOW variable over a period of another length, which returned a weighted `MicroSeries` before. diff --git a/policyengine_core/simulations/microsimulation.py b/policyengine_core/simulations/microsimulation.py index 808b9a2d9..641a725e8 100644 --- a/policyengine_core/simulations/microsimulation.py +++ b/policyengine_core/simulations/microsimulation.py @@ -13,12 +13,17 @@ class _RunningCall: - """A ``Microsimulation`` calculate call, running until ``running`` is false.""" + """A ``Microsimulation`` calculate call, running until ``running`` is false. - __slots__ = ("running",) + ``enclosing`` is the record of the call that was running in the context + when this one started, if any. + """ + + __slots__ = ("running", "enclosing") - def __init__(self): + def __init__(self, enclosing: Optional["_RunningCall"]): self.running = True + self.enclosing = enclosing # The innermost ``Microsimulation`` ``calculate``, ``calculate_add`` or @@ -30,12 +35,12 @@ def __init__(self): # with enum codes. # # A context copied while a call runs (for a worker thread, say) holds that -# call's record, and sees the call end when ``running`` turns false. A call -# made in the copy records itself there, so what it reads stays inside while -# it runs. A thread starts with an empty context, so its calls come from -# outside, unless Python gives new threads a copy of the starting thread's -# context (``-X thread_inherit_context=1``, the default on free-threaded -# builds). +# call's record, and sees each call end when its ``running`` turns false; it +# is inside while that call or one enclosing it runs. A call made in the copy +# records itself there, so what it reads stays inside while it runs. A thread +# starts with an empty context, so its calls come from outside, unless Python +# gives new threads a copy of the starting thread's context +# (``-X thread_inherit_context=1``, the default on free-threaded builds). _running_call: ContextVar[Optional[_RunningCall]] = ContextVar( "_microsimulation_running_call", default=None ) @@ -43,12 +48,16 @@ def __init__(self): def _inside_a_calculation() -> bool: call = _running_call.get() - return call is not None and call.running + # In the context where the calls run, the innermost is running. Only a + # copied context can hold a call that has ended. + while call is not None and not call.running: + call = call.enclosing + return call is not None def _start_call() -> Tuple[_RunningCall, Token]: """Record a call as running; return what ``_end_call`` needs.""" - call = _RunningCall() + call = _RunningCall(_running_call.get()) return call, _running_call.set(call) diff --git a/tests/core/test_microsimulation_internal_calls.py b/tests/core/test_microsimulation_internal_calls.py index 20c0cf3a2..172802e7b 100644 --- a/tests/core/test_microsimulation_internal_calls.py +++ b/tests/core/test_microsimulation_internal_calls.py @@ -27,6 +27,7 @@ from policyengine_core.enums import EnumArray from policyengine_core.model_api import ADD, DIVIDE, MONTH +from policyengine_core.periods import period as make_period from policyengine_core.simulations import Microsimulation, Simulation from tests.fixtures.microsimulation_internal_calls import ( RECEIVED, @@ -105,7 +106,7 @@ def record(self, variable_name, period=None): "population enum read": (assert_codes, TENURE_CODES), "calculate": (assert_array, GRANT), "calculate enum": (assert_codes, TENURE_CODES), - "calculate again": (assert_array, GRANT), + "calculate again": (assert_array, PAY), "calculate_add": (assert_array, PAY), "calculate_divide": (assert_array, GRANT), "calculate_divide month": (assert_array, GRANT_A_MONTH), @@ -119,6 +120,10 @@ def test_formula_call_on_its_own_simulation_gets_a_plain_array(label, trace): check, expected = READS_OF_ITS_OWN_SIMULATION[label] check(RECEIVED[label], expected) + # The fast cache holds the first read of ``pay_added``, so with tracing + # off the second read comes from it; with tracing on, Simulation.calculate + # skips the fast cache. + assert RECEIVED["fast cache holds the first read"] @pytest.mark.parametrize( @@ -198,13 +203,22 @@ def test_context_copied_in_a_formula_gets_weights_after_the_calculation_ends(): assert_weighted(result, GRANT) +def test_context_copied_in_a_nested_call_stays_inside_while_an_outer_call_runs(): + microsimulation().calculate("probe_reads_in_a_nested_context", DATASET_YEAR) + + assert_array(RECEIVED["nested context calculate"], GRANT) + + def test_call_in_a_copied_context_stays_inside_after_the_first_call_ends(): # The worker's own calculate call is still running when it reads, so its # reads are inside a calculation, though the call whose context it copied # has ended. - microsimulation().calculate("probe_starts_a_worker", DATASET_YEAR) - RECEIVED["release worker"].set() - RECEIVED["worker"].join(timeout=30) + try: + microsimulation().calculate("probe_starts_a_worker", DATASET_YEAR) + finally: + if "release worker" in RECEIVED: + RECEIVED["release worker"].set() + RECEIVED["worker"].join(timeout=30) assert not RECEIVED["worker"].is_alive() assert_array(RECEIVED["worker calculate"], GRANT) @@ -217,10 +231,12 @@ def test_call_in_a_copied_context_stays_inside_after_the_first_call_ends(): def test_top_level_calculate_is_weighted_and_decoded(): simulation_under_test = microsimulation() - assert_weighted(simulation_under_test.calculate("yearly_grant"), GRANT) - # The second read comes from Simulation.calculate's fast cache. assert_weighted(simulation_under_test.calculate("yearly_grant"), GRANT) assert_weighted(simulation_under_test.calculate("yearly_tenure"), TENURE_NAMES) + assert_weighted(simulation_under_test.calculate("pay_added"), PAY) + # The second read comes from Simulation.calculate's fast cache. + assert ("pay_added", make_period(DATASET_YEAR)) in simulation_under_test._fast_cache + assert_weighted(simulation_under_test.calculate("pay_added"), PAY) @pytest.mark.parametrize( @@ -339,22 +355,21 @@ def test_formula_that_catches_a_branch_error_still_gets_an_array(): def test_calculation_that_runs_again_after_an_input_change_gets_arrays(): - # The probe sets an input while it runs, which can make core run the - # outermost calculation again. Whatever core does then, a Microsimulation - # does as a Simulation does, and its formula's reads are plain arrays. + # The probe reads a branch, then sets an input there that drops what it + # read, so core runs its calculation again, in both classes. plain = simulation() - plain.calculate("probe_sets_an_input") - plain_read = RECEIVED.pop("calculate after setting an input") expected = plain.calculate("probe_sets_an_input") + assert RECEIVED["runs"] == 2 + RECEIVED.clear() simulation_under_test = microsimulation() result = simulation_under_test.calculate("probe_sets_an_input") - assert_array(RECEIVED["calculate after setting an input"], plain_read) + assert RECEIVED["runs"] == 2 + assert_array(RECEIVED["branch calculate before the input"], expected) + assert_array(RECEIVED["branch calculate after the input"], expected) assert_weighted(result, expected) - assert_weighted( - simulation_under_test.calculate("yearly_grant"), plain.calculate("yearly_grant") - ) + assert_weighted(simulation_under_test.calculate("yearly_grant"), GRANT) # Values: the same as a Simulation's. diff --git a/tests/fixtures/microsimulation_internal_calls.py b/tests/fixtures/microsimulation_internal_calls.py index 597c8b0d0..2b5673e64 100644 --- a/tests/fixtures/microsimulation_internal_calls.py +++ b/tests/fixtures/microsimulation_internal_calls.py @@ -122,9 +122,14 @@ def _reads_own_simulation(person, period, parameters): RECEIVED["population enum read"] = person("yearly_tenure", period) RECEIVED["calculate"] = simulation.calculate("yearly_grant", period) RECEIVED["calculate enum"] = simulation.calculate("yearly_tenure", period) - # Already stored by the read above: Simulation.calculate returns it from - # its fast cache. - RECEIVED["calculate again"] = simulation.calculate("yearly_grant", period) + # ``pay_added`` has a formula, so Simulation.calculate stores it in its + # fast cache, and with tracing off the second read returns it from there. + simulation.calculate("pay_added", period) + RECEIVED["fast cache holds the first read"] = ( + "pay_added", + period, + ) in simulation._fast_cache + RECEIVED["calculate again"] = simulation.calculate("pay_added", period) RECEIVED["calculate_add"] = simulation.calculate_add("monthly_pay", period) RECEIVED["calculate_divide"] = simulation.calculate_divide("yearly_grant", period) RECEIVED["calculate_divide month"] = simulation.calculate_divide( @@ -184,24 +189,39 @@ def _keeps_its_context(person, period, parameters): return _zeros(person) +def _reads_in_a_nested_context(person, period, parameters): + # ``probe_keeps_its_context`` copies the context during its own, nested + # calculation. That calculation has ended when this formula runs a call in + # the copy, but this formula's calculation has not. + person("probe_keeps_its_context", period) + RECEIVED["nested context calculate"] = RECEIVED["context"].run( + person.simulation.calculate, "yearly_grant", period + ) + return _zeros(person) + + def _starts_a_worker(person, period, parameters): - # The worker runs a calculation in a context copied here, and keeps - # running after this formula and its calculation return. A test releases - # it then. - release = threading.Event() + # The worker runs a calculation in a context copied here. This formula + # returns once the worker's calculation has started, and the worker keeps + # running after this formula's calculation ends, until a test releases it. + started, release = threading.Event(), threading.Event() context = contextvars.copy_context() inner = ProbeMicrosimulation(dataset=example_dataset()) worker = threading.Thread( target=context.run, args=(inner.calculate, "probe_waits_then_reads", period), + daemon=True, ) + RECEIVED["worker started"] = started RECEIVED["release worker"] = release RECEIVED["worker"] = worker worker.start() + started.wait(timeout=30) return _zeros(person) def _waits_then_reads(person, period, parameters): + RECEIVED["worker started"].set() RECEIVED["release worker"].wait(timeout=30) simulation = person.simulation RECEIVED["worker calculate"] = simulation.calculate("yearly_grant", period) @@ -236,15 +256,19 @@ def _catches_a_branch_error(person, period, parameters): def _sets_an_input(person, period, parameters): - # Setting an input drops values computed from it, so the outermost - # calculation runs again. - simulation = person.simulation - simulation.calculate("pay_and_grant", period) - simulation.set_input("yearly_grant", period, np.full(person.count, 1234.0)) - RECEIVED["calculate after setting an input"] = simulation.calculate( + # An input set on a branch drops the branch's values calculated from the + # value it replaces. This formula read one of them, so core does not keep + # its result and runs the calculation again. + RECEIVED["runs"] = RECEIVED.get("runs", 0) + 1 + branch = person.simulation.get_branch("probe_input_branch") + RECEIVED["branch calculate before the input"] = branch.calculate( + "pay_and_grant", period + ) + branch.set_input("yearly_grant", period, np.full(person.count, 1234.0)) + RECEIVED["branch calculate after the input"] = branch.calculate( "pay_and_grant", period ) - return simulation.calculate("pay_and_grant", period) + return RECEIVED["branch calculate after the input"] def _probe(name, formula, definition_period=YEAR): @@ -268,6 +292,7 @@ def _probe(name, formula, definition_period=YEAR): _probe("probe_builds_a_microsimulation", _builds_a_microsimulation), _probe("probe_reads_from_worker_threads", _reads_from_worker_threads), _probe("probe_keeps_its_context", _keeps_its_context), + _probe("probe_reads_in_a_nested_context", _reads_in_a_nested_context), _probe("probe_starts_a_worker", _starts_a_worker), _probe("probe_waits_then_reads", _waits_then_reads), _probe("probe_raises", _raises), From 7a41e254947faa394c835cb9db0fe1f928f2591a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 14:39:16 +0400 Subject: [PATCH 4/6] Describe the change for formulas that call Microsimulation methods directly Co-Authored-By: Claude Opus 5.5 --- .../unweighted-internal-microsimulation-calls.changed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/unweighted-internal-microsimulation-calls.changed.md b/changelog.d/unweighted-internal-microsimulation-calls.changed.md index 9ad471256..24b5b4bdd 100644 --- a/changelog.d/unweighted-internal-microsimulation-calls.changed.md +++ b/changelog.d/unweighted-internal-microsimulation-calls.changed.md @@ -1 +1 @@ -`Microsimulation.calculate`, `calculate_add` and `calculate_divide` now default to `use_weights=None` and `decode_enums=None`, which return a plain array with enum codes to a call made while another of these calls runs in the same thread or a context copied from it (core's own reads and formula calls), and weighted values to any other call, so internal reads no longer build weights that are then discarded. +`Microsimulation.calculate`, `calculate_add` and `calculate_divide` now default to `use_weights=None` and `decode_enums=None`: a call made while another of these calls runs in the same `contextvars` context, or a context copied from it (core's own reads, and formula code that calls these methods or `calculate_dataframe` directly), gets a plain NumPy array with enum codes instead of a weighted `MicroSeries` with member names, so internal reads no longer build weights, while other calls are weighted as before. From 986a83b874ce7001072dcf3f17f2e3679aeff325 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:19:39 +0400 Subject: [PATCH 5/6] Keep enum names in calculate_dataframe inside a calculation Read each column unweighted and decoded, so a frame built in a formula matches one built at the top level, and weights are built once per frame. Co-Authored-By: Claude Opus 5.5 --- ...-internal-microsimulation-calls.changed.md | 2 +- .../simulations/microsimulation.py | 36 ++++++++++++- .../test_microsimulation_internal_calls.py | 51 +++++++++++++++++++ .../microsimulation_internal_calls.py | 11 ++++ 4 files changed, 97 insertions(+), 3 deletions(-) diff --git a/changelog.d/unweighted-internal-microsimulation-calls.changed.md b/changelog.d/unweighted-internal-microsimulation-calls.changed.md index 24b5b4bdd..5f2c07257 100644 --- a/changelog.d/unweighted-internal-microsimulation-calls.changed.md +++ b/changelog.d/unweighted-internal-microsimulation-calls.changed.md @@ -1 +1 @@ -`Microsimulation.calculate`, `calculate_add` and `calculate_divide` now default to `use_weights=None` and `decode_enums=None`: a call made while another of these calls runs in the same `contextvars` context, or a context copied from it (core's own reads, and formula code that calls these methods or `calculate_dataframe` directly), gets a plain NumPy array with enum codes instead of a weighted `MicroSeries` with member names, so internal reads no longer build weights, while other calls are weighted as before. +`Microsimulation.calculate`, `calculate_add` and `calculate_divide` no longer build weights for core's internal reads. Formula code that calls these methods directly now gets a NumPy array with enum codes instead of a weighted `MicroSeries`; calls from outside a calculation stay weighted, and `calculate_dataframe` returns the same frame everywhere, building weights once. diff --git a/policyengine_core/simulations/microsimulation.py b/policyengine_core/simulations/microsimulation.py index 641a725e8..ea3606c15 100644 --- a/policyengine_core/simulations/microsimulation.py +++ b/policyengine_core/simulations/microsimulation.py @@ -3,6 +3,7 @@ from microdf import MicroDataFrame, MicroSeries import numpy as np +import pandas as pd from policyengine_core.data.dataset import Dataset from policyengine_core.enums import EnumArray from policyengine_core.periods import Period @@ -87,7 +88,8 @@ class Microsimulation(Simulation): another of these calls runs (by core, or by a formula) returns a plain array, with enums left as codes, as a ``Simulation`` does: core and formulas use the values, not the weights. Passing ``use_weights`` or - ``decode_enums`` overrides either default. + ``decode_enums`` overrides either default. ``calculate_dataframe`` returns + the same frame wherever it is called from. """ def get_weights( @@ -262,11 +264,41 @@ def calculate_dataframe( map_to: str = None, use_weights: bool = True, ) -> MicroDataFrame: + """Calculate ``variable_names`` for ``period`` as a data frame. + + Inside a calculation or not, columns hold enum member names, and a + frame with ``use_weights`` carries the weights of the first variable's + entity (or of ``map_to``). Columns of other entities are mapped to it. + """ if period is not None and not isinstance(period, Period): period = get_period(period) elif period is None and self.default_calculation_period is not None: period = get_period(self.default_calculation_period) - values = super().calculate_dataframe(variable_names, period, map_to) + + for variable_name in variable_names: + if variable_name not in self.tax_benefit_system.variables: + raise ValueError(f"Variable {variable_name} does not exist.") + entities = [ + self.tax_benefit_system.get_variable(variable_name).entity.key + for variable_name in variable_names + ] + frame_entity = map_to or entities[0] + column_map_to = map_to + if any(entity != frame_entity for entity in entities): + column_map_to = frame_entity + + # ``Simulation.calculate_dataframe`` reads each column with the default + # arguments, which would build weights for every column here and give + # enum codes inside a calculation. + values = pd.DataFrame() + for variable_name in variable_names: + values[variable_name] = self.calculate( + variable_name, + period, + column_map_to, + use_weights=False, + decode_enums=True, + ) if not use_weights: return values weights = self.get_weights(variable_names[0], period, map_to) diff --git a/tests/core/test_microsimulation_internal_calls.py b/tests/core/test_microsimulation_internal_calls.py index 172802e7b..5569658e7 100644 --- a/tests/core/test_microsimulation_internal_calls.py +++ b/tests/core/test_microsimulation_internal_calls.py @@ -22,6 +22,7 @@ from unittest import mock import numpy as np +import pandas as pd import pytest from microdf import MicroDataFrame, MicroSeries @@ -279,6 +280,56 @@ def test_top_level_calculate_dataframe_is_weighted(): np.testing.assert_array_equal(result.weights, PERSON_WEIGHTS) +def test_calculate_dataframe_in_a_formula_has_names_and_weights(): + # A frame is for reading by column, inside a formula or not: a filter such + # as ``frame[frame.yearly_tenure == "OWNER"]`` needs member names. + microsimulation().calculate("probe_reads_a_dataframe", DATASET_YEAR) + + weighted = RECEIVED["calculate_dataframe"] + assert isinstance(weighted, MicroDataFrame) + assert weighted["yearly_grant"].tolist() == GRANT + assert weighted["yearly_tenure"].tolist() == TENURE_NAMES + np.testing.assert_array_equal(weighted.weights, PERSON_WEIGHTS) + + unweighted = RECEIVED["calculate_dataframe use_weights=False"] + assert type(unweighted) is pd.DataFrame + assert unweighted["yearly_grant"].tolist() == GRANT + assert unweighted["yearly_tenure"].tolist() == TENURE_NAMES + + +@pytest.mark.parametrize( + "use_weights,expected_weight_reads", + [(True, ["yearly_grant"]), (False, [])], +) +def test_calculate_dataframe_builds_weights_only_for_the_frame( + use_weights, expected_weight_reads +): + simulation_under_test = microsimulation() + + with weight_reads() as variables: + simulation_under_test.calculate_dataframe( + ["yearly_grant", "pay_and_grant", "yearly_tenure"], + use_weights=use_weights, + ) + + assert variables == expected_weight_reads + + +def test_calculate_dataframe_maps_columns_to_the_first_variables_entity(): + result = microsimulation().calculate_dataframe( + ["yearly_grant", "household_yearly_grant"] + ) + + assert result["yearly_grant"].tolist() == GRANT + assert result["household_yearly_grant"].tolist() == [12.0, 12.0, 36.0] + np.testing.assert_array_equal(result.weights, PERSON_WEIGHTS) + + +def test_calculate_dataframe_rejects_an_unknown_variable(): + with pytest.raises(ValueError, match="no_such_variable does not exist"): + microsimulation().calculate_dataframe(["yearly_grant", "no_such_variable"]) + + def test_top_level_explicit_arguments_win(): simulation_under_test = microsimulation() diff --git a/tests/fixtures/microsimulation_internal_calls.py b/tests/fixtures/microsimulation_internal_calls.py index 2b5673e64..adf68f577 100644 --- a/tests/fixtures/microsimulation_internal_calls.py +++ b/tests/fixtures/microsimulation_internal_calls.py @@ -150,6 +150,16 @@ def _reads_own_simulation(person, period, parameters): return _zeros(person) +def _reads_a_dataframe(person, period, parameters): + columns = ["yearly_grant", "yearly_tenure"] + simulation = person.simulation + RECEIVED["calculate_dataframe"] = simulation.calculate_dataframe(columns, period) + RECEIVED["calculate_dataframe use_weights=False"] = simulation.calculate_dataframe( + columns, period, use_weights=False + ) + return _zeros(person) + + def _reads_a_branch(person, period, parameters): branch = person.simulation.get_branch("probe_branch") RECEIVED["branch calculate"] = branch.calculate("yearly_grant", period) @@ -287,6 +297,7 @@ def _probe(name, formula, definition_period=YEAR): PROBES = [ _probe("probe_reads_own_simulation", _reads_own_simulation), + _probe("probe_reads_a_dataframe", _reads_a_dataframe), _probe("probe_reads_a_branch", _reads_a_branch), _probe("probe_reads_its_baseline", _reads_its_baseline), _probe("probe_builds_a_microsimulation", _builds_a_microsimulation), From afcc7fd9fdae88a56b28a64ff16a9d8828b07339 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 19 Nov 2026 21:16:06 +0400 Subject: [PATCH 6/6] Add an added fragment so the release is a minor version The None defaults of use_weights and decode_enums are a new way to call these methods, and the change in what formulas get from direct calls warrants more than a patch. Co-Authored-By: Claude Opus 5.5 --- changelog.d/unweighted-internal-microsimulation-calls.added.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/unweighted-internal-microsimulation-calls.added.md diff --git a/changelog.d/unweighted-internal-microsimulation-calls.added.md b/changelog.d/unweighted-internal-microsimulation-calls.added.md new file mode 100644 index 000000000..1f46438e5 --- /dev/null +++ b/changelog.d/unweighted-internal-microsimulation-calls.added.md @@ -0,0 +1 @@ +`use_weights=None` and `decode_enums=None`, now the defaults of `Microsimulation.calculate`, `calculate_add` and `calculate_divide`, choose by caller instead of meaning `False`: weights (and, in `calculate`, enum names) outside a calculation, neither inside one.