diff --git a/changelog.d/lazy-parameter-node-at-instant.changed.md b/changelog.d/lazy-parameter-node-at-instant.changed.md new file mode 100644 index 000000000..5b9983d2b --- /dev/null +++ b/changelog.d/lazy-parameter-node-at-instant.changed.md @@ -0,0 +1 @@ +Resolve `ParameterNodeAtInstant` children on first read instead of building the whole parameter tree for every instant, which cost policyengine-us 16 to 33 MB and several seconds per instant on every tax-benefit system, so a node at instant held across an in-place parameter change, such as one returned by `TaxBenefitSystem.get_parameters_at_instant`, now reads the changed value for each child it has not read yet, while the children it has read keep their values, where before every child kept the value from when the node was built. diff --git a/changelog.d/lazy-parameter-node-at-instant.fixed.md b/changelog.d/lazy-parameter-node-at-instant.fixed.md new file mode 100644 index 000000000..708939bc0 --- /dev/null +++ b/changelog.d/lazy-parameter-node-at-instant.fixed.md @@ -0,0 +1 @@ +Clear the nodes at instant cached by a parameter node and every node above it when `ParameterNode.add_child` adds a child, so that a node at instant requested afterwards includes the new child. diff --git a/policyengine_core/parameters/parameter_node.py b/policyengine_core/parameters/parameter_node.py index 57497804f..f1c2f84ed 100644 --- a/policyengine_core/parameters/parameter_node.py +++ b/policyengine_core/parameters/parameter_node.py @@ -74,6 +74,50 @@ def _check_child_type(child) -> None: ) +def _clear_at_instant_caches_upward(node) -> None: + """Clear the nodes at instant cached by ``node`` and every node above it. + + ``add_child`` uses this instead of ``clear_parent_cache`` so that it keeps + accepting every tree it accepted before it cleared caches. It walks up + without recursing and stops at a node it has already seen, so a loop in + the ``parent`` links ends. It skips a node with no cache of its own, such + as one built without ``ParameterNode.__init__``, and a parent that is not + a parameter node. + """ + seen = set() + while node is not None and id(node) not in seen: + seen.add(id(node)) + cache = node.__dict__.get("_at_instant_cache") + if isinstance(cache, dict): + cache.clear() + node = getattr(node, "parent", None) + + +class _AtInstantCache(dict): + """A parameter node's nodes at instant, by instant. + + Reads add to it: a node at instant that resolves a child fills that + child's cache. So a copy or a pickle takes its items in one step first, + instead of going through the dictionary while another thread adds to it. + It stays one object, so parameter nodes that share it, such as shallow + copies, share its copy too. A pickle holds a plain ``dict``, which earlier + versions of core can load, and which ``_get_at_instant`` replaces by one + of these before it adds to it. + """ + + __slots__ = () + + def __reduce_ex__(self, protocol): + return (dict, (dict.copy(self),)) + + def __deepcopy__(self, memo: dict) -> "_AtInstantCache": + copied = type(self)() + memo[id(self)] = copied + for instant, node_at_instant in dict.copy(self).items(): + copied[copy.deepcopy(instant, memo)] = copy.deepcopy(node_at_instant, memo) + return copied + + class ParameterNode(AtInstantLike): """ A node in the legislation `parameter tree `_. @@ -135,7 +179,9 @@ def __init__( self.trace: bool = False self.tracer = None self.branch_name = None - self._at_instant_cache: typing.Dict[Instant, ParameterNodeAtInstant] = {} + self._at_instant_cache: typing.Dict[Instant, ParameterNodeAtInstant] = ( + _AtInstantCache() + ) self.parent = None if directory_path: @@ -217,10 +263,15 @@ def add_child(self, name: str, child: Union["ParameterNode", Parameter]): :param name: Name of the child that must be used to access that child. Should not contain anything that could interfere with the operator `.` (dot). The child is also set as an attribute of the node, unless the name matches one of the node's own attributes (such as ``parent`` or ``children``) or a method or class attribute of its class; it then stays reachable through ``children`` and at-instant lookups. :param child: The new child, an instance of :class:`.ParameterScale` or :class:`.Parameter` or :class:`.ParameterNode`. + + Nodes at instant cached by this node and by every node above it were + built without the new child, so they are cleared. """ if name in self.children: raise ValueError("{} has already a child named {}".format(self.name, name)) _check_child_type(child) + # Nodes at instant cached here and above were built without the child. + _clear_at_instant_caches_upward(self) self._link_child(name, child) def _replace_child( @@ -303,13 +354,14 @@ def clone(self) -> "ParameterNode": if _can_expose_child_as_attribute(clone, child_key): setattr(clone, child_key, child) child.parent = clone - clone._at_instant_cache = {} + clone._at_instant_cache = _AtInstantCache() return clone def _get_at_instant(self, instant: Instant) -> ParameterNodeAtInstant: - if instant in self._at_instant_cache: - return self._at_instant_cache[instant] + cache = self._at_instant_cache + if instant in cache: + return cache[instant] node_at_instant = ParameterNodeAtInstant(self.name, self, instant) if self.trace: at_instant = TracingParameterNodeAtInstant( @@ -317,7 +369,16 @@ def _get_at_instant(self, instant: Instant) -> ParameterNodeAtInstant: ) else: at_instant = node_at_instant - self._at_instant_cache[instant] = at_instant + # Building the node can read code that replaces this node's cache, so + # store into the cache the node holds now, as before. + cache = self._at_instant_cache + if type(cache) is dict: + # A plain ``dict`` set from outside core (policyengine-us's path + # copies set one) changes as this node is read, so copying it + # while another thread reads could fail. It is replaced, with its + # entries, rather than written to. + cache = self._at_instant_cache = _AtInstantCache(cache) + cache[instant] = at_instant return at_instant def attach_to_parent(self, parent: "ParameterNode"): diff --git a/policyengine_core/parameters/parameter_node_at_instant.py b/policyengine_core/parameters/parameter_node_at_instant.py index b55889eb6..4e49e7a7a 100644 --- a/policyengine_core/parameters/parameter_node_at_instant.py +++ b/policyengine_core/parameters/parameter_node_at_instant.py @@ -1,10 +1,11 @@ import os import sys +import threading from typing import TYPE_CHECKING, Iterable, Union import numpy -from policyengine_core import parameters, tools +from policyengine_core import parameters, periods, tools from policyengine_core.errors import ParameterNotFoundError from policyengine_core.parameters import helpers @@ -16,9 +17,136 @@ ) +# The one instance attribute a node keeps for itself; a parameter of this +# name is rejected. +_STATE = "_ParameterNodeAtInstant__state" +# Attributes a node sets on itself, which a child of the same name shadows. +_OWN_NAMES = frozenset({"_name", "_instant_str", "_children", "_vectorial_node"}) +_MISSING = object() + +# Held while a child is resolved for the first time, so that concurrent +# readers never see a half-resolved node or resolve one child twice. Reads of +# a resolved child never take it. Reentrant because resolving a child can +# build that child's own node. +_RESOLUTION_LOCK = threading.RLock() + + +def _replace_resolution_lock() -> None: + """Give a forked process a lock that no thread holds. A thread that held + the lock when another thread forked does not exist in the forked process, + so nothing there would release it, and the first resolution would wait for + ever.""" + global _RESOLUTION_LOCK + _RESOLUTION_LOCK = threading.RLock() + + +if hasattr(os, "register_at_fork"): + os.register_at_fork(after_in_child=_replace_resolution_lock) + + +def _copied_state(instance): + """What ``object.__getstate__`` gives, with the instance dictionary + copied: a ``dict``, ``None`` when it is empty, or a ``(dict, slots)`` pair + for an instance with ``__slots__``. + + Reads change the dictionaries of parameter nodes and nodes at instant, so + copying or pickling a dictionary itself would fail if another thread read + meanwhile. Copying a ``dict`` is a single step for other threads. + """ + state = object.__getstate__(instance) + if isinstance(state, tuple): + attributes, slots = state + return (None if attributes is None else dict(attributes), slots) + return None if state is None else dict(state) + + +# Instant strings that ``periods.instant`` has accepted. +_DATES = set() + + +def _is_date(instant_str) -> bool: + if not isinstance(instant_str, str): + return False + if instant_str in _DATES: + return True + try: + # As a scale's build reads it: ``str`` checks that the day exists. + str(periods.instant(instant_str)) + except ValueError: + return False + _DATES.add(instant_str) + return True + + +class _LazyChildren: + """What a node at instant knows about its children. A shallow copy of the + node shares it, as copies of a node that built every child up front shared + their ``_children`` mapping.""" + + __slots__ = ("instant", "source", "resolved", "absent", "added", "children") + + def __init__(self, instant: str, source: dict): + self.instant = instant + # Each child's parameter node as at construction, in order; None once + # every child is resolved. + self.source = source + # Children resolved from ``source`` so far. + self.resolved = {} + # Children of ``source`` with no value at the instant. + self.absent = set() + # Children given with ``add_child``, in call order. + self.added = {} + # Every child in order, once all of ``source`` is resolved. + self.children = None + + def __getstate__(self) -> dict: + # Only a deep copy or a pickle copies this state; a shallow copy of + # the node shares it. Either is a snapshot, as when every child was + # built up front: resolving every child first lets go of ``source``, + # whose parameter nodes would bring the whole tree, through their + # parents, into the copy. Once every child is resolved, reads no + # longer change these containers (only ``add_child`` does), so they + # are given as they are and keep their identities. Pickle protocols 0 + # and 1 also refuse a class with ``__slots__`` that does not define + # this. + with _RESOLUTION_LOCK: + _resolve_remaining(self) + return {name: getattr(self, name) for name in self.__slots__} + + def __setstate__(self, state) -> None: + if isinstance(state, tuple): + # ``(None, slots)``, as ``object.__getstate__`` gives for a class + # with ``__slots__``. + state = state[1] + for name in self.__slots__: + setattr(self, name, state[name]) + + class ParameterNodeAtInstant: """ Parameter node of the legislation, at a given instant. + + Children resolve on first read and are kept from then on. Building every + descendant up front made each instant a model is asked about cost a copy of + the whole parameter tree (16 to 33 MB and several seconds per instant for + policyengine-us), which every tax-benefit system then kept for its + lifetime. Which children a node has, and in what order, is fixed when it is + built. A child's value is taken when the child is first read, so a node + held across an in-place change to its parameters reads the changed value + for a child it has not read yet; once read (including found to have no + value), a child keeps what that read found. + + A child that fails to build at the instant (a scale that refuses its + brackets, say) fails when it is read, not when the node is built, so the + node's other children can be read. A deep copy or pickle of the node + builds every child, so it fails then too. + + Several threads can read a node, and copy or pickle it while others read. + A deep copy or pickle of a node resolves every child first, so it holds + the same values as when every child was built up front, and no parameter + nodes. Its pickle holds this version's lazy state, which earlier versions + of core can't load. At an instant that is not a date, and in a subclass, + every child is built up front, as before. """ def __init__(self, name: str, node: "ParameterNode", instant_str: str): @@ -31,23 +159,97 @@ def __init__(self, name: str, node: "ParameterNode", instant_str: str): # The "technical" attributes are hidden, so that the node children can be easily browsed with auto-completion without pollution self._name = name self._instant_str = instant_str - self._children = {} - - for child_name, child in node.children.items(): - child_at_instant = child._get_at_instant(instant_str) - if child_at_instant is not None: - self.add_child(child_name, child_at_instant) + source = dict(node.children.items()) + if _STATE in source: + raise ValueError(f"{_STATE!r} cannot name a parameter.") + state = _LazyChildren(instant_str, source) + self.__dict__[_STATE] = state + cls = type(self) + if cls is not ParameterNodeAtInstant: + # A subclass gets the up-front build, as before: its add_child + # sees every child, every child is an attribute before its own + # ``__getattr__`` or ``__getitem__`` runs, and ``_children`` is + # also the mapping items are read from. + children = {} + with _RESOLUTION_LOCK: + state.source = None + state.children = children + self.__dict__["_children"] = children + for child_name, child in source.items(): + child_at_instant = child._get_at_instant(instant_str) + if child_at_instant is not None: + self.add_child(child_name, child_at_instant) + return + for child_name in source: + # A child whose name the node itself answers to shadows it, as + # when every child was set here. + if child_name in _OWN_NAMES or hasattr(cls, child_name): + child_at_instant = _resolve(state, child_name) + if child_at_instant is not _MISSING: + setattr(self, child_name, child_at_instant) + if not _is_date(instant_str): + # Built up front, a node at an instant that is not a date (such as + # ``parameters(None)``, which reads "None") failed on the first + # scale below it, and nothing was cached. Building it up front + # still does that, and otherwise holds what that build held. + _resolve_remaining(state) def add_child(self, child_name: str, child_at_instant: "ParameterNodeAtInstant"): - self._children[child_name] = child_at_instant + if child_name == _STATE: + raise ValueError(f"{_STATE!r} cannot name a parameter.") + state = self.__dict__[_STATE] + with _RESOLUTION_LOCK: + state.added[child_name] = child_at_instant + if state.children is not None: + state.children[child_name] = child_at_instant setattr(self, child_name, child_at_instant) def __getattr__(self, key: str): - if key == "_name": + # Reached only for a name that is not an instance attribute: a child + # not read yet by this instance, ``_children`` before it is set, or a + # name this node does not have. + state = self.__dict__.get(_STATE) + if state is None or key == "_name": + # An instance still being rebuilt by copy or pickle. raise AttributeError(key) + if key == "_children": + return _materialise(self, state) + child_at_instant = _resolve(state, key) + if child_at_instant is not _MISSING: + # Attributes hold what the parameters give; ``add_child`` on + # another copy changes only that copy's attribute, as before. + self.__dict__.setdefault(key, child_at_instant) + return child_at_instant param_name = helpers._compose_name(self._name, item_name=key) raise ParameterNotFoundError(param_name, self._instant_str) + def __dir__(self): + children = self._children + names = set(super().__dir__()) | set(map(str, children)) + names.discard(_STATE) + return list(names) + + def __getstate__(self): + # Reads resolve children into ``__dict__``: see ``_copied_state``. + return _copied_state(self) + + def __setstate__(self, state): + if isinstance(state, tuple): + state, slots = state + for slot_name, value in slots.items(): + setattr(self, slot_name, value) + state = state or {} + self.__dict__.update(state) + if _STATE not in state: + # Pickled when every child was built up front, all in ``_children``. + children = state.get("_children") + if children is None: + children = self.__dict__["_children"] = {} + lazy = _LazyChildren(state.get("_instant_str"), None) + lazy.children = children + lazy.resolved = dict(children) + self.__dict__[_STATE] = lazy + def __getitem__( self, key: str ) -> Union["ParameterNodeAtInstant", VectorialParameterNodeAtInstant]: @@ -68,7 +270,19 @@ def __getitem__( ) self._vectorial_node = vectorial return vectorial[key] - return self._children[key] + state = self.__dict__[_STATE] + children = state.children + if children is not None: + # Every child is resolved: the items are ``_children``'s. + return children[key] + child_at_instant = state.added.get(key, _MISSING) + if child_at_instant is _MISSING: + child_at_instant = state.resolved.get(key, _MISSING) + if child_at_instant is _MISSING: + child_at_instant = _resolve(state, key) + if child_at_instant is _MISSING: + raise KeyError(key) + return child_at_instant def __iter__(self) -> Iterable: return iter(self._children) @@ -81,3 +295,78 @@ def __repr__(self) -> str: ] ) return result + + +def _resolve(state: _LazyChildren, child_name): + """A child as the parameters give it, resolving it if no read has yet; + ``_MISSING`` if the node has no such child or it has no value at the + instant.""" + with _RESOLUTION_LOCK: + child_at_instant = state.resolved.get(child_name, _MISSING) + if child_at_instant is not _MISSING or child_name in state.absent: + return child_at_instant + source = state.source + child = None if source is None else source.get(child_name) + if child is None: + return _MISSING + child_at_instant = child._get_at_instant(state.instant) + # A read made while resolving (the same thread re-entering) may have + # recorded this child already; the first recorded result stands. + if child_name in state.absent: + return _MISSING + recorded = state.resolved.get(child_name, _MISSING) + if recorded is not _MISSING: + child_at_instant = recorded + elif child_at_instant is None: + state.absent.add(child_name) + else: + state.resolved[child_name] = child_at_instant + if state.source is not None and len(state.resolved) + len(state.absent) == len( + state.source + ): + _complete(state) + return _MISSING if child_at_instant is None else child_at_instant + + +def _complete(state: _LazyChildren): + """Order every child once all of ``source`` is resolved; let go of the + parameter nodes. The caller holds ``_RESOLUTION_LOCK``.""" + resolved, added = state.resolved, state.added + children = { + name: added.get(name, resolved[name]) + for name in state.source + if name in resolved + } + for name, child_at_instant in added.items(): + # Added children the parameters do not have (or have no value for) + # follow, in the order they were added. + children.setdefault(name, child_at_instant) + # ``resolved`` stays apart from ``children``: attribute reads return what + # the parameters give, whatever ``add_child`` or a direct write does to + # the items. + state.children = children + state.source = None + state.absent = set() + + +def _resolve_remaining(state: _LazyChildren): + """Resolve every child not read yet, and order every child.""" + with _RESOLUTION_LOCK: + if state.children is None: + for child_name in list(state.source): + if state.children is not None: + break + _resolve(state, child_name) + if state.children is None: + _complete(state) + + +def _materialise(node_at_instant: ParameterNodeAtInstant, state: _LazyChildren): + """Resolve every child and set ``_children``.""" + with _RESOLUTION_LOCK: + _resolve_remaining(state) + children = state.children + if "_children" not in node_at_instant.__dict__: + # Unless a child named ``_children`` took the name. + node_at_instant.__dict__["_children"] = children + return children diff --git a/tests/core/parameters/test_parameter_node_add_child_caches.py b/tests/core/parameters/test_parameter_node_add_child_caches.py new file mode 100644 index 000000000..76020201a --- /dev/null +++ b/tests/core/parameters/test_parameter_node_add_child_caches.py @@ -0,0 +1,226 @@ +"""``ParameterNode.add_child`` clears the nodes at instant cached by the node it +changes and by every node above it. + +A node at instant lists the children its parameter node had when it was +built. Without clearing, a node at instant cached before ``add_child`` was +served again afterwards, without the new child. +""" + +import types + +import pytest + +from policyengine_core.errors import ParameterNotFoundError +from policyengine_core.parameters import Parameter, ParameterNode + +INSTANT = "2020-01-01" + + +def leaf(name: str, value) -> Parameter: + return Parameter(name, data={"values": {"2010-01-01": value}}) + + +def three_level_tree() -> ParameterNode: + return ParameterNode( + "", data={"gov": {"tax": {"rate": {"values": {"2010-01-01": 0.1}}}}} + ) + + +def read_the_root(tree: ParameterNode): + tree(INSTANT) + + +def read_down_to_the_leaf(tree: ParameterNode): + tree(INSTANT).gov.tax.rate + + +def read_every_level(tree: ParameterNode): + tree(INSTANT).gov.tax.rate + tree.gov(INSTANT).tax.rate + tree.gov.tax(INSTANT).rate + + +@pytest.mark.parametrize( + "read_first", [read_the_root, read_down_to_the_leaf, read_every_level] +) +def test_nodes_requested_after_add_child_have_the_new_child(read_first): + tree = three_level_tree() + read_first(tree) + + tree.gov.tax.add_child("allowance", leaf("gov.tax.allowance", 500)) + + assert tree(INSTANT).gov.tax.allowance == 500 + assert tree.gov(INSTANT).tax.allowance == 500 + assert tree.gov.tax(INSTANT).allowance == 500 + assert list(tree(INSTANT).gov.tax) == ["rate", "allowance"] + + +def test_merged_children_are_in_nodes_requested_afterwards(): + tree = three_level_tree() + read_down_to_the_leaf(tree) + + tree.gov.tax.merge( + ParameterNode("other", data={"allowance": {"values": {"2010-01-01": 500}}}) + ) + + assert tree(INSTANT).gov.tax.allowance == 500 + + +def test_add_child_leaves_other_branches_cached(): + tree = ParameterNode( + "", + data={ + "gov": {"tax": {"rate": {"values": {"2010-01-01": 0.1}}}}, + "other": {"amount": {"values": {"2010-01-01": 3}}}, + }, + ) + other_at_instant = tree(INSTANT).other + + tree.gov.tax.add_child("allowance", leaf("gov.tax.allowance", 500)) + + assert tree(INSTANT).other is other_at_instant + + +def test_a_node_held_from_before_add_child_keeps_the_children_it_had(): + tree = three_level_tree() + held = tree(INSTANT) + tax_at_instant = held.gov.tax + + tree.gov.tax.add_child("allowance", leaf("gov.tax.allowance", 500)) + + with pytest.raises(ParameterNotFoundError): + tax_at_instant.allowance + assert list(held.gov.tax) == ["rate"] + + +@pytest.mark.parametrize( + "parent_child", + [ + {"values": {"2010-01-01": True}}, + {"nested": {"values": {"2010-01-01": True}}}, + ], + ids=["parameter", "node"], +) +def test_a_node_with_a_child_named_parent_loads_and_reads(parent_child): + # policyengine-us has ``gov.hhs.medicaid.eligibility.categories.parent``. + # While a node loads, ``add_child`` sets each child as an attribute, so + # until the node is attached, ``categories.parent`` is that child, whose + # own parent is ``categories``. Children added after it must not walk + # that loop for ever. + tree = ParameterNode( + "", + data={ + "categories": { + "adult": {"values": {"2010-01-01": True}}, + "parent": parent_child, + "senior": {"values": {"2010-01-01": False}}, + } + }, + ) + + categories = tree(INSTANT).categories + assert categories.adult is True + assert categories.senior is False + assert categories["parent"] is not None + assert tree.categories.parent is tree + + +def test_a_child_named_like_the_cache_can_be_added(): + # A reform can add a child of any name. policyengine-us adds one named + # after the cache and reads the tree through a clone. + tree = three_level_tree() + read_down_to_the_leaf(tree) + + tree.gov.add_child("_at_instant_cache", leaf("gov._at_instant_cache", 1)) + + assert tree.clone()(INSTANT).gov.tax.rate == 0.1 + assert tree.clone()(INSTANT).gov["_at_instant_cache"] == 1 + + +SCALE_TREE = { + "scale": { + "brackets": [ + { + "threshold": {"2010-01-01": {"value": 0}}, + "rate": {"2010-01-01": {"value": 0.1}}, + }, + { + "threshold": {"2010-01-01": {"value": 100}}, + "rate": {"2010-01-01": {"value": 0.2}}, + }, + ] + } +} + + +def scale_description(scale): + return (type(scale).__name__, list(scale.thresholds), list(scale.rates)) + + +@pytest.mark.parametrize("cloned", [False, True], ids=["loaded", "cloned"]) +def test_nodes_requested_after_add_child_on_a_bracket_have_the_new_child(cloned): + # A bracket's ``parent`` links it to its scale, as a clone's does, so + # clearing reaches the nodes above the scale. + tree = ParameterNode("", data=SCALE_TREE) + if cloned: + tree = tree.clone() + tree(INSTANT).scale + + tree.scale.brackets[0].add_child("base", leaf("scale.brackets[0].base", 2)) + + assert scale_description(tree(INSTANT).scale) == scale_description( + tree.scale._get_at_instant(INSTANT) + ) + + +# ``add_child`` accepts every tree it accepted before it cleared caches. These +# trees aren't ones core builds, but a node or holder can come from code that +# skips ``ParameterNode.__init__`` or links nodes by hand. + + +def holder_without_a_cache() -> ParameterNode: + # Built without ``ParameterNode.__init__``, so it has no cache or parent. + holder = object.__new__(ParameterNode) + holder.name = "holder" + holder.children = {} + return holder + + +def test_add_child_accepts_a_node_with_no_cache_of_its_own(): + holder = holder_without_a_cache() + + holder.add_child("amount", leaf("holder.amount", 5)) + + assert holder.amount is holder.children["amount"] + + +def test_add_child_clears_past_a_node_above_with_no_cache_of_its_own(): + tree = three_level_tree() + tree.attach_to_parent(holder_without_a_cache()) + read_down_to_the_leaf(tree) + + tree.gov.tax.add_child("allowance", leaf("gov.tax.allowance", 500)) + + assert tree(INSTANT).gov.tax.allowance == 500 + + +def test_add_child_accepts_a_parent_that_is_not_a_parameter_node(): + tree = three_level_tree() + tree.attach_to_parent(types.SimpleNamespace(name="holder")) + read_down_to_the_leaf(tree) + + tree.gov.tax.add_child("allowance", leaf("gov.tax.allowance", 500)) + + assert tree(INSTANT).gov.tax.allowance == 500 + + +def test_add_child_stops_when_parent_links_form_a_loop(): + tree = three_level_tree() + read_down_to_the_leaf(tree) + # The root's parent is a node below it. + tree.attach_to_parent(tree.gov.tax) + + tree.gov.tax.add_child("allowance", leaf("gov.tax.allowance", 500)) + + assert tree.gov.tax(INSTANT).allowance == 500 + assert tree(INSTANT).gov.tax.allowance == 500 diff --git a/tests/core/parameters/test_parameter_node_at_instant_copies.py b/tests/core/parameters/test_parameter_node_at_instant_copies.py new file mode 100644 index 000000000..2e143d274 --- /dev/null +++ b/tests/core/parameters/test_parameter_node_at_instant_copies.py @@ -0,0 +1,504 @@ +"""Copies, pickles and forks of parameter trees whose nodes at instant are +partly read. + +A node at instant resolves a child on first read, so reads change the node, +its shared state and the at-instant caches of the parameter nodes below it. +Copying or pickling must not fail when such a read happens part-way through +(here, a read made by a value that is being copied, as another thread could +make it at that moment), and a copy must read the parameters it was copied +with. +""" + +import copy +import os +import pickle +import signal +import threading + +import pytest + +from policyengine_core.errors import ParameterNotFoundError +from policyengine_core.parameters import ( + Parameter, + ParameterNode, + ParameterNodeAtInstant, +) +from tests.fixtures.parameter_nodes_at_instant import ( + TREE, + build_tree, + materialise, + ordered, + snapshot, +) + +INSTANT = "2017-03-01" +OTHER_INSTANT = "2018-01-01" +NEW_INSTANT = "2019-01-01" +PROTOCOLS = list(range(pickle.HIGHEST_PROTOCOL + 1)) + +DUPLICATES = { + "deepcopy": copy.deepcopy, + **{ + f"pickle-{protocol}": ( + lambda value, protocol=protocol: pickle.loads( + pickle.dumps(value, protocol=protocol) + ) + ) + for protocol in PROTOCOLS + }, +} + + +def partly_read_tree(): + tree = build_tree() + held = tree(INSTANT) + held.group.x + held.by_zone.z1.amount + return tree, held + + +@pytest.mark.parametrize("duplicate", DUPLICATES.values(), ids=DUPLICATES.keys()) +def test_a_partly_read_node_copies_with_every_value(duplicate): + tree, held = partly_read_tree() + + copied = duplicate(held) + + assert ordered(materialise(copied)) == ordered(snapshot(tree, INSTANT)) + + +@pytest.mark.parametrize("duplicate", DUPLICATES.values(), ids=DUPLICATES.keys()) +def test_a_tree_holding_partly_read_nodes_copies_with_its_cache(duplicate): + tree, _ = partly_read_tree() + + copied = duplicate(tree) + + assert INSTANT in copied._at_instant_cache + assert copied(INSTANT) is copied._at_instant_cache[INSTANT] + assert ordered(materialise(copied(INSTANT))) == ordered(snapshot(tree, INSTANT)) + + +@pytest.mark.parametrize("duplicate", DUPLICATES.values(), ids=DUPLICATES.keys()) +def test_a_copied_tree_reads_its_own_parameters(duplicate): + tree, held = partly_read_tree() + copied = duplicate(tree) + copied_held = copied._at_instant_cache[INSTANT] + + # Changes to either tree after the copy stay in that tree, for children + # read before the copy and for children not read yet. + tree.flat_rate.update(period="2017", value=0.5) + tree.group.x.update(period="2017", value=0.6) + copied.by_zone.z2.amount.update(period="2017", value=7) + + assert copied_held.flat_rate == 0.1 + assert copied_held.group.x == 0.1 + assert copied(INSTANT).flat_rate == 0.1 + assert copied(INSTANT).by_zone.z2.amount == 7 + assert held.by_zone.z2.amount == 200 + assert tree(INSTANT).flat_rate == 0.5 + + +class ReadsWhenCopied: + """A parameter value that reads parameters when it is copied or pickled, + as another thread could while a copy is under way.""" + + def __init__(self, read): + self.read = read + + def __deepcopy__(self, memo): + self.read() + return "copied" + + def __reduce__(self): + self.read() + return (str, ("copied",)) + + +def give_plain_caches(tree): + """Give every parameter node a plain ``dict`` cache, as policyengine-us's + ``copy_parameter_nodes`` does, and as a tree pickled by an earlier core + has.""" + for node in (tree, *tree.get_descendants()): + if isinstance(node, ParameterNode): + node._at_instant_cache = {} + + +def tree_with_a_value_that_reads(read_for, caches="core"): + """A tree whose nodes at instant hold a value that reads parameters when + it is copied. + + Only nodes at instant hold that value: the root's node at ``INSTANT`` as + ``first``, and the group's node at ``OTHER_INSTANT`` as ``x``. The + parameters hold plain values again. + """ + tree = ParameterNode( + "", + data={ + "first": {"values": {"2010-01-01": 0}}, + "second": {"values": {"2010-01-01": 2}}, + "group": {"x": {"values": {"2010-01-01": 3}}}, + }, + ) + if caches == "plain": + give_plain_caches(tree) + held = tree(INSTANT) + reader = ReadsWhenCopied(lambda: read_for(tree, held)) + first = tree.children["first"] + x = tree.group.children["x"] + first.values_list[0].value = x.values_list[0].value = reader + assert held.first is reader + assert tree.group(OTHER_INSTANT).x is reader + first.values_list[0].value = 0 + x.values_list[0].value = 3 + return tree, held + + +READS_DURING_A_COPY = { + # Resolves a child of the node being copied. + "a child of the node": lambda tree, held: held.second, + # Resolves a node child, which fills that parameter node's cache. + "a node child": lambda tree, held: held.group.x, + # A new instant fills the root's cache and the group's. + "a new instant": lambda tree, held: tree(NEW_INSTANT).group.x, +} + + +@pytest.mark.parametrize("duplicate", DUPLICATES.values(), ids=DUPLICATES.keys()) +@pytest.mark.parametrize( + "read", READS_DURING_A_COPY.values(), ids=READS_DURING_A_COPY.keys() +) +def test_a_read_during_a_copy_of_a_node_does_not_break_it(read, duplicate): + _, held = tree_with_a_value_that_reads(read) + + copied = duplicate(held) + + assert copied.first == "copied" + assert copied.second == 2 + assert copied.group.x == 3 + + +@pytest.mark.parametrize("caches", ["core", "plain"]) +@pytest.mark.parametrize("duplicate", DUPLICATES.values(), ids=DUPLICATES.keys()) +@pytest.mark.parametrize( + "read", READS_DURING_A_COPY.values(), ids=READS_DURING_A_COPY.keys() +) +def test_a_read_during_a_copy_of_a_tree_does_not_break_it(read, duplicate, caches): + tree, _ = tree_with_a_value_that_reads(read, caches) + + copied = duplicate(tree) + + assert copied(INSTANT).first == "copied" + assert copied(INSTANT).second == 2 + assert copied(INSTANT).group.x == 3 + assert copied.group(OTHER_INSTANT).x == "copied" + assert copied(NEW_INSTANT).group.x == 3 + + +def test_a_shallow_copy_of_a_tree_shares_its_cache_as_before(): + tree, held = partly_read_tree() + + shallow = copy.copy(tree) + + assert shallow._at_instant_cache is tree._at_instant_cache + assert shallow(INSTANT) is held + + +class BlockingChild: + """A parameter child whose first resolution waits until released.""" + + def __init__(self): + self.entered = threading.Event() + self.release = threading.Event() + + def _get_at_instant(self, instant): + self.entered.set() + assert self.release.wait(10), "never released" + return 1 + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="needs os.fork") +@pytest.mark.filterwarnings("ignore::DeprecationWarning") +def test_a_process_forked_during_a_resolution_can_resolve_children(): + tree = ParameterNode("", data={"other": {"values": {"2010-01-01": 2}}}) + slow = BlockingChild() + tree.children = {"slow": slow, **tree.children} + held = tree(INSTANT) + reader = threading.Thread(target=lambda: held.slow) + reader.start() + assert slow.entered.wait(10) + # The reader now holds the resolution lock, and the forked process gets a + # copy of the lock but not the reader. + try: + pid = os.fork() + if pid == 0: # pragma: no cover - runs in the forked process + signal.alarm(10) + try: + os._exit(0 if held.other == 2 else 1) + except BaseException: + os._exit(2) + _, status = os.waitpid(pid, 0) + finally: + slow.release.set() + reader.join(10) + + assert os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0, status + + +class SlottedNodeAtInstant(ParameterNodeAtInstant): + __slots__ = ("marker",) + + +class SlottedParameterNode(ParameterNode): + __slots__ = ("marker",) + + +@pytest.mark.parametrize( + "duplicate", + [copy.copy, *DUPLICATES.values()], + ids=["copy", *DUPLICATES.keys()], +) +def test_a_slotted_node_at_instant_keeps_its_slots(duplicate): + tree, held = partly_read_tree() + node = SlottedNodeAtInstant.__new__(SlottedNodeAtInstant) + node.__dict__.update(held.__dict__) + node.marker = "node at instant" + + copied = duplicate(node) + + assert copied.marker == "node at instant" + assert ordered(materialise(copied)) == ordered(snapshot(tree, INSTANT)) + + +SLOTTED_PARAMETER_NODE_DUPLICATES = { + "copy": copy.copy, + **{ + name: duplicate + for name, duplicate in DUPLICATES.items() + # Protocols 0 and 1 refuse a class with ``__slots__`` and no + # ``__getstate__``, as before. + if name not in ("pickle-0", "pickle-1") + }, +} + + +@pytest.mark.parametrize( + "duplicate", + SLOTTED_PARAMETER_NODE_DUPLICATES.values(), + ids=SLOTTED_PARAMETER_NODE_DUPLICATES.keys(), +) +def test_a_slotted_parameter_node_keeps_its_slots(duplicate): + parameter_node = SlottedParameterNode("", data=TREE) + parameter_node.marker = "parameter node" + parameter_node(INSTANT).group.x + + copied = duplicate(parameter_node) + + assert copied.marker == "parameter node" + assert ordered(materialise(copied(INSTANT))) == ordered( + snapshot(parameter_node, INSTANT) + ) + + +# A deep copy or pickle of a node at instant is a snapshot, as when every +# child was built up front: it holds no parameter nodes. + + +class RecordsDeepCopies: + copies = 0 + + def __deepcopy__(self, memo): + RecordsDeepCopies.copies += 1 + return self + + +def tree_with_an_unrelated_branch(): + tree = build_tree() + tree.add_child( + "unrelated_branch_marker", + ParameterNode( + "unrelated_branch_marker", + data={"amount": {"values": {"2010-01-01": 1}}}, + ), + ) + tree.unrelated_branch_marker.amount.values_list[0].value = RecordsDeepCopies() + return tree + + +@pytest.mark.parametrize("protocol", PROTOCOLS) +def test_a_pickled_node_leaves_the_parameter_tree_behind(protocol): + tree = tree_with_an_unrelated_branch() + group = tree(INSTANT).group + group.x + + data = pickle.dumps(group, protocol=protocol) + + assert b"unrelated_branch_marker" not in data + assert ordered(materialise(pickle.loads(data))) == ordered( + snapshot(tree.group, INSTANT) + ) + + +def test_a_deep_copied_node_leaves_the_parameter_tree_behind(): + tree = tree_with_an_unrelated_branch() + group = tree(INSTANT).group + group.x + RecordsDeepCopies.copies = 0 + + copied = copy.deepcopy(group) + + assert RecordsDeepCopies.copies == 0 + assert ordered(materialise(copied)) == ordered(snapshot(tree.group, INSTANT)) + + +@pytest.mark.parametrize("duplicate", DUPLICATES.values(), ids=DUPLICATES.keys()) +def test_add_child_on_a_copy_of_a_listed_node_is_listed(duplicate): + tree = ParameterNode( + "", + data={"x": {"values": {"2010-01-01": 10}}, "y": {"values": {"2010-01-01": 20}}}, + ) + held = tree(INSTANT) + list(held) + + copied = duplicate(held) + copied.add_child("z", 30) + + assert list(copied) == ["x", "y", "z"] + assert "z" in copied + assert copied["z"] == 30 and copied.z == 30 + assert list(copied._children) == ["x", "y", "z"] + + +@pytest.mark.parametrize("read_in_full", [list, lambda node: "x" in node]) +def test_add_child_on_a_copy_of_a_fully_read_node_changes_only_shared_items( + read_in_full, +): + tree = ParameterNode( + "", + data={"x": {"values": {"2010-01-01": 10}}, "y": {"values": {"2010-01-01": 20}}}, + ) + held = tree(INSTANT) + read_in_full(held) + duplicate = copy.copy(held) + + duplicate.add_child("x", 99) + duplicate.add_child("extra", 77) + + # Items are one mapping shared by copies; attributes belong to each copy. + assert held["x"] == 99 and held.x == 10 + assert held["extra"] == 77 + with pytest.raises(ParameterNotFoundError): + held.extra + assert duplicate.x == 99 and duplicate.extra == 77 + assert list(held._children) == ["x", "y", "extra"] + + +def test_children_written_directly_are_items(): + tree = ParameterNode( + "", + data={"x": {"values": {"2010-01-01": 10}}, "y": {"values": {"2010-01-01": 20}}}, + ) + held = tree(INSTANT) + held.add_child("added", 1) + list(held) + + held._children["written"] = 2 + + assert held["written"] == 2 + assert held["x"] == 10 + + +@pytest.mark.parametrize("duplicate", DUPLICATES.values(), ids=DUPLICATES.keys()) +def test_shallow_copies_sharing_a_cache_still_share_it_once_copied(duplicate): + tree, _ = partly_read_tree() + alias = copy.copy(tree) + + copied_tree, copied_alias = duplicate((tree, alias)) + copied_tree.flat_rate.update(period="2017", value=0.5) + + assert copied_alias._at_instant_cache is copied_tree._at_instant_cache + assert copied_alias(INSTANT).flat_rate == 0.5 + + +def test_a_plain_cache_is_replaced_by_a_copyable_one_when_first_filled(): + tree, held = partly_read_tree() + give_plain_caches(tree) + plain = tree._at_instant_cache + plain[INSTANT] = held + + tree(OTHER_INSTANT) + + assert type(tree._at_instant_cache) is not dict + assert isinstance(tree._at_instant_cache, dict) + assert tree._at_instant_cache[INSTANT] is held + assert list(plain) == [INSTANT] # Not written to. + assert tree(INSTANT) is held + + +@pytest.mark.parametrize( + "replacement", + [dict, lambda: type(ParameterNode("", data={})._at_instant_cache)()], + ids=["plain", "core"], +) +@pytest.mark.parametrize("starting_cache", ["plain", "core"]) +def test_a_cache_replaced_while_a_node_is_built_is_the_one_filled( + replacement, starting_cache +): + # policyengine-us's isolate_parameter_tracing replaces a root's cache to + # drop an earlier request's nodes. If that happens while the root is read, + # the node built goes into the cache the root holds afterwards. + tree = ParameterNode("", data={"a": {"values": {"2010-01-01": 1}}}) + + class ReplacesTheRootCache(Parameter): + replaced = False + + def _get_at_instant(self, instant): + if not ReplacesTheRootCache.replaced: + ReplacesTheRootCache.replaced = True + tree._at_instant_cache = replacement() + return super()._get_at_instant(instant) + + # Named like a node's own attribute, so it resolves while the node is + # built. + tree.add_child( + "_vectorial_node", + ReplacesTheRootCache("_vectorial_node", {"values": {"2010-01-01": 5}}), + ) + if starting_cache == "plain": + tree._at_instant_cache = {} + tree._at_instant_cache[OTHER_INSTANT] = "a node from an earlier request" + + node = tree(INSTANT) + + assert list(tree._at_instant_cache) == [INSTANT] + assert tree(INSTANT) is node + + +def test_children_written_directly_are_items_and_not_attributes(): + tree = ParameterNode( + "", + data={"x": {"values": {"2010-01-01": 10}}, "y": {"values": {"2010-01-01": 20}}}, + ) + held = tree(INSTANT) + children = held._children + + children["written"] = 2 + children["x"] = 99 + del children["y"] + + assert held["written"] == 2 and held["x"] == 99 + with pytest.raises(KeyError): + held["y"] + with pytest.raises(ParameterNotFoundError): + held.written + assert held.x == 10 and held.y == 20 + + +@pytest.mark.parametrize("protocol", PROTOCOLS) +def test_a_pickled_cache_is_a_plain_dict_that_earlier_cores_load(protocol): + tree = build_tree() + + data = pickle.dumps(tree, protocol=protocol) + restored = pickle.loads(data) + + assert b"_AtInstantCache" not in data + assert type(restored._at_instant_cache) is dict + assert ordered(materialise(restored(INSTANT))) == ordered(snapshot(tree, INSTANT)) diff --git a/tests/core/parameters/test_parameter_node_at_instant_lazy.py b/tests/core/parameters/test_parameter_node_at_instant_lazy.py new file mode 100644 index 000000000..959b30074 --- /dev/null +++ b/tests/core/parameters/test_parameter_node_at_instant_lazy.py @@ -0,0 +1,613 @@ +"""``ParameterNodeAtInstant`` resolves its children on first read. + +It used to build every descendant in its constructor, so each instant a +system was asked about cost a copy of the whole parameter tree (about 34,000 +node objects and 16-33 MB per instant for policyengine-us) that the system +then kept. These tests pin that nothing is built before it is read, and that +everything read is what the up-front build held. +""" + +import copy +import gc +import pickle +import threading +import weakref + +import numpy as np +import pytest + +from policyengine_core.country_template import CountryTaxBenefitSystem +from policyengine_core.errors import ParameterNotFoundError +from policyengine_core.parameters import ParameterNode, ParameterNodeAtInstant +from policyengine_core.parameters.parameter_node_at_instant import _STATE +from policyengine_core.tracers import FullTracer +from tests.fixtures.parameter_nodes_at_instant import ( + INSTANTS, + build_tree, + materialise, + ordered, + snapshot, +) + + +@pytest.mark.parametrize("instant", INSTANTS) +def test_fully_read_node_matches_the_up_front_build(instant): + tree = build_tree() + assert ordered(materialise(tree.get_at_instant(instant))) == ordered( + snapshot(tree, instant) + ) + + +@pytest.mark.parametrize( + "instant", ["2000-01-01", "2012-06-01", "2015-01-01", "2017-01-01", "2030-01-01"] +) +def test_country_template_matches_the_up_front_build(instant): + parameters = CountryTaxBenefitSystem().parameters + assert ordered(materialise(parameters.get_at_instant(instant))) == ordered( + snapshot(parameters, instant) + ) + + +def resolved(node_at_instant): + return list(node_at_instant.__dict__[_STATE].resolved) + + +def test_nothing_is_resolved_until_it_is_read(): + at_instant = build_tree().get_at_instant("2017-03-01") + assert resolved(at_instant) == [] + + assert at_instant.group.x == 0.1 + # Only the path that was read: the root's ``group`` and the group's ``x``. + assert resolved(at_instant) == ["group"] + assert resolved(at_instant.group) == ["x"] + + +def test_asking_for_many_instants_builds_one_node_each(): + tree = build_tree() + gc.collect() + before = sum(isinstance(o, ParameterNodeAtInstant) for o in gc.get_objects()) + for year in range(2000, 2100): + tree.get_at_instant(f"{year}-01-01") + after = sum(isinstance(o, ParameterNodeAtInstant) for o in gc.get_objects()) + # The up-front build made 9 nodes per instant for this tree (900 here). + assert after - before == 100 + + +def test_a_read_child_is_resolved_once(): + at_instant = build_tree().get_at_instant("2017-03-01") + # A scale is rebuilt by every ``_get_at_instant`` call, so identity shows + # that the node kept what it resolved. + assert at_instant.scale is at_instant.scale + assert at_instant["scale"] is at_instant.scale + assert at_instant._children["scale"] is at_instant.scale + assert at_instant.group is at_instant["group"] + + +def test_children_are_in_parameter_order_whatever_was_read_first(): + tree = build_tree() + at_instant = tree.get_at_instant("2030-01-01") + at_instant.by_zone + at_instant["late"] + at_instant.group + assert list(at_instant) == list(snapshot(tree, "2030-01-01")) + assert list(at_instant._children) == list(snapshot(tree, "2030-01-01")) + + +def test_child_without_a_value_yet_is_absent(): + at_instant = build_tree().get_at_instant("2017-03-01") + with pytest.raises(ParameterNotFoundError): + at_instant.late + with pytest.raises(ParameterNotFoundError) as error: + at_instant.group.y + assert str(error.value) == ( + "The parameter 'group[y]' was not found in the 2017-03-01 " + "tax and benefit system." + ) + with pytest.raises(KeyError): + at_instant["late"] + assert not hasattr(at_instant, "late") + assert "late" not in at_instant + assert "late" not in at_instant._children + assert at_instant.group.inner._children == {} + + +def test_unknown_names_raise_as_before(): + at_instant = build_tree().get_at_instant("2017-03-01") + for read_first in (False, True): + if read_first: + at_instant._children + with pytest.raises(ParameterNotFoundError) as error: + at_instant.group.nope + assert str(error.value) == ( + "The parameter 'group[nope]' was not found in the 2017-03-01 " + "tax and benefit system." + ) + with pytest.raises(KeyError): + at_instant.group["nope"] + assert not hasattr(at_instant, "nope") + + +def test_vectorial_indexing_reads_the_whole_subtree(): + at_instant = build_tree().get_at_instant("2020-01-01") + zones = np.array(["z1", "z3", "z2"]) + np.testing.assert_array_equal(at_instant.by_zone[zones].amount, [200, 600, 400]) + np.testing.assert_allclose(at_instant.by_zone[zones].rate, [0.1, 0.3, 0.2]) + # Nothing outside the indexed subtree was resolved for it. + assert resolved(at_instant) == ["by_zone"] + + +def test_dir_lists_children_for_completion(): + at_instant = build_tree().get_at_instant("2017-03-01") + names = dir(at_instant) + assert {"flat_rate", "group", "scale", "by_zone"} <= set(names) + assert "late" not in names + + +def test_repr_shows_every_child(): + text = repr(build_tree().get_at_instant("2017-03-01")) + for name in ("flat_rate", "group", "scale", "by_zone", "z3"): + assert name in text + + +def test_add_child_before_and_after_a_full_read(): + at_instant = build_tree().get_at_instant("2017-03-01") + at_instant.add_child("extra", 1) + assert at_instant.extra == 1 and at_instant["extra"] == 1 + assert list(at_instant._children)[-1] == "extra" + at_instant.add_child("later", 2) + assert at_instant.later == 2 and at_instant._children["later"] == 2 + + +def test_a_node_asked_for_after_an_update_reads_the_update(): + tree = build_tree() + assert tree.get_at_instant("2017-03-01").flat_rate == 0.1 + tree.flat_rate.update(value=0.5, period="year:2017:1") + assert tree.get_at_instant("2017-03-01").flat_rate == 0.5 + + +def test_a_child_read_before_an_update_keeps_what_was_read(): + tree = build_tree() + held = tree.get_at_instant("2017-03-01") + assert held.flat_rate == 0.1 + tree.flat_rate.update(value=0.5, period="year:2017:1") + assert held.flat_rate == 0.1 + + +def test_a_child_not_read_before_an_update_reads_the_update(): + """Intended difference from the up-front build, which held 0.1 here: a + node held across a change to its parameters reads, for each child, the + value at the time that child is first read.""" + tree = build_tree() + held = tree.get_at_instant("2017-03-01") + tree.flat_rate.update(value=0.5, period="year:2017:1") + assert held.flat_rate == 0.5 + + +def test_deep_copied_tree_keeps_its_own_nodes_at_instant(): + tree = build_tree() + tree.get_at_instant("2017-03-01").group.x # cache a partly read node + clone = copy.deepcopy(tree) + cached = clone._at_instant_cache["2017-03-01"] + clone.group.y.update(value=0.9, period="year:2010:30") + clone.flat_rate.update(value=0.7, period="year:2010:30") + # A copy is a snapshot, as when every child was built up front: its cached + # node keeps what the tree held when copied. Nodes requested from the copy + # afterwards read the copy, and the original keeps its values. + assert cached.flat_rate == 0.1 + assert clone.get_at_instant("2017-03-01").flat_rate == 0.7 + assert clone.get_at_instant("2017-03-01").group.y == 0.9 + assert tree.get_at_instant("2017-03-01").flat_rate == 0.1 + assert not hasattr(tree.get_at_instant("2017-03-01").group, "y") + + +def test_copy_of_a_node_at_instant_reads_the_same_values(): + at_instant = build_tree().get_at_instant("2017-03-01") + at_instant.group + for duplicate in (copy.copy(at_instant), copy.deepcopy(at_instant)): + assert materialise(duplicate) == materialise(at_instant) + + +@pytest.mark.parametrize( + "child_name", + # Not ``add_child``: ParameterNode itself cannot hold a child of that name. + ["_name", "_instant_str", "_vectorial_node", "_node", "_resolved"] + + ["_resolve", "_resolve_all", "_store", "_materialise"], +) +def test_children_named_like_the_nodes_own_attributes(child_name): + """Such a child reads as its value whatever the read order, as when every + child was set up front, and leaves its siblings readable.""" + tree = ParameterNode( + "", + data={ + child_name: {"values": {"2010-01-01": 42}}, + "ok": {"values": {"2010-01-01": 7}}, + }, + ) + for read_name_first in (True, False): + tree._at_instant_cache.clear() + at_instant = tree.get_at_instant("2017-03-01") + if read_name_first: + assert getattr(at_instant, child_name) == 42 + assert at_instant.ok == 7 and at_instant["ok"] == 7 + else: + assert at_instant["ok"] == 7 and at_instant.ok == 7 + assert getattr(at_instant, child_name) == 42 + assert at_instant[child_name] == 42 + assert set(at_instant._children) == {child_name, "ok"} + + +class BlockingChild: + """A parameter child whose first resolution waits until released.""" + + def __init__(self): + self.calls = 0 + self.entered = threading.Event() + self.release = threading.Event() + + def _get_at_instant(self, instant): + self.calls += 1 + if self.calls == 1: + self.entered.set() + assert self.release.wait(10), "never released" + return object() + + +@pytest.mark.parametrize( + "second_read", + [ + lambda node: node.slow, + lambda node: node["slow"], + lambda node: node._children["slow"], + lambda node: [node[name] for name in node][0], + ], +) +def test_concurrent_first_reads_resolve_a_child_once(second_read): + tree = ParameterNode("", data={"other": {"values": {"2010-01-01": 1}}}) + slow = BlockingChild() + tree.children = {"slow": slow, **tree.children} + at_instant = tree.get_at_instant("2017-03-01") + results, errors = [], [] + + def read(reader): + try: + results.append(reader(at_instant)) + except Exception as error: # noqa: BLE001 - recorded for the assertion + errors.append(error) + + first = threading.Thread(target=read, args=(lambda node: node.slow,)) + first.start() + assert slow.entered.wait(10) + second = threading.Thread(target=read, args=(second_read,)) + second.start() + second.join(0.2) # let the second reader reach the child too + slow.release.set() + first.join(10) + second.join(10) + + assert not errors, errors + assert len(results) == 2 and results[0] is results[1] + assert slow.calls == 1 + + +def test_shallow_copies_read_the_same_children(): + held = build_tree().get_at_instant("2017-03-01") + duplicate = copy.copy(held) + original_scale = held.scale + # A scale is rebuilt by each resolution, so identity shows one resolution. + assert duplicate.scale is original_scale + assert duplicate["scale"] is original_scale + assert held["scale"] is original_scale + duplicate._children + assert held._children["scale"] is original_scale + assert held.by_zone is duplicate["by_zone"] + + +@pytest.mark.parametrize( + "duplicate", [copy.deepcopy, lambda n: pickle.loads(pickle.dumps(n))] +) +def test_traced_nodes_copy_and_pickle(duplicate): + tree = build_tree() + tree.trace = True + tree.tracer = FullTracer() + tree.branch_name = "default" + traced = tree.get_at_instant("2017-03-01") + traced.group.x # partly resolved, through the tracing wrapper + raw = traced.parameter_node_at_instant + for node in (raw, traced): + assert duplicate(node).flat_rate == 0.1 + assert duplicate(node).group.x == 0.1 + + +def test_a_child_without_a_value_is_resolved_once(): + tree = ParameterNode("", data={"late": {"values": {"2030-01-01": 1}}}) + held = tree.get_at_instant("2020-01-01") + calls = [] + original = tree.children["late"]._get_at_instant + + def counting(instant): + calls.append(instant) + return original(instant) + + tree.children["late"]._get_at_instant = counting + for _ in range(3): + with pytest.raises(ParameterNotFoundError): + held.late + with pytest.raises(KeyError): + held["late"] + assert list(held) == [] + assert len(calls) == 1 + + # A child already read keeps what that read found, value or none. + tree.children["late"].update(value=2, period="year:2020:1") + with pytest.raises(KeyError): + held["late"] + assert tree.get_at_instant("2020-01-01").late == 2 + + +def test_a_node_lets_go_of_its_parameter_node_once_every_child_is_read(): + tree = ParameterNode( + "", + data={ + "group": { + "x": {"values": {"2010-01-01": 1}}, + "late": {"values": {"2030-01-01": 2}}, + }, + "unused": {"values": {"2010-01-01": 3}}, + }, + ) + group = tree.get_at_instant("2017-03-01").group + tree_ref = weakref.ref(tree) + del tree + gc.collect() + # Partly read: the node still needs the tree to resolve the rest. + assert tree_ref() is not None + assert group.x == 1 + with pytest.raises(ParameterNotFoundError): + group.late + gc.collect() + assert tree_ref() is None + assert group.x == 1 and list(group) == ["x"] + + +def plain_tree(**values): + return ParameterNode( + "", data={name: {"values": {"2010-01-01": v}} for name, v in values.items()} + ) + + +def test_the_reserved_state_name_is_rejected(): + with pytest.raises(ValueError): + plain_tree(_ParameterNodeAtInstant__state=1).get_at_instant("2017-01-01") + at_instant = plain_tree(x=1).get_at_instant("2017-01-01") + with pytest.raises(ValueError): + at_instant.add_child("_ParameterNodeAtInstant__state", 2) + + +@pytest.mark.parametrize( + "child_name", + [ + "_ParameterNodeAtInstant__resolved", + "_ParameterNodeAtInstant__node", + "_ParameterNodeAtInstant__business", + "__custom__", + "__iter__", + "__dir__", + ], +) +def test_other_unusual_names_are_plain_children(child_name): + at_instant = plain_tree(**{child_name: 42, "ok": 7}).get_at_instant("2017-01-01") + assert getattr(at_instant, child_name) == 42 + assert at_instant[child_name] == 42 + assert at_instant.ok == 7 and at_instant["ok"] == 7 + assert child_name in dir(at_instant) + + +def test_many_added_children_do_not_break_the_first_read(): + names = {f"p{i}": i for i in range(2000)} + at_instant = plain_tree(**names).get_at_instant("2017-01-01") + for i in range(2000): + at_instant.add_child(f"extra{i}", i) + assert at_instant.p0 == 0 + assert len(at_instant._children) == 4000 + assert list(at_instant._children)[:2] == ["p0", "p1"] + + +def test_add_child_on_a_copy_changes_only_shared_items_as_before(): + tree = ParameterNode( + "", + data={ + "x": {"values": {"2010-01-01": 10}}, + "late": {"values": {"2030-01-01": 5}}, + }, + ) + held = tree.get_at_instant("2017-01-01") + with pytest.raises(ParameterNotFoundError): + held.late + duplicate = copy.copy(held) + duplicate.add_child("x", 99) + duplicate.add_child("late", 88) + duplicate.add_child("extra", 77) + # Items are one mapping shared by copies; attributes belong to each copy. + assert held["x"] == 99 and held.x == 10 + assert held["late"] == 88 + with pytest.raises(ParameterNotFoundError): + held.late + assert held["extra"] == 77 + with pytest.raises(ParameterNotFoundError): + held.extra + assert duplicate.x == 99 and duplicate.late == 88 and duplicate.extra == 77 + assert list(held._children) == ["x", "late", "extra"] + + +def test_a_copy_installs_what_it_reads_as_attributes(): + held = build_tree().get_at_instant("2017-03-01") + duplicate = copy.copy(held) + scale = held.scale + assert duplicate.scale is scale + assert duplicate.__dict__["scale"] is scale + + +class _MethodCollision(ParameterNodeAtInstant): + def policy(self): + return "method" + + +class _DoublingAddChild(ParameterNodeAtInstant): + def add_child(self, child_name, child_at_instant): + super().add_child(child_name, child_at_instant * 2) + + +def test_subclasses_keep_their_collisions_and_add_child_hooks(): + tree = plain_tree(policy=42, x=5) + collided = _MethodCollision("", tree, "2017-01-01") + assert collided.policy == 42 and collided["policy"] == 42 + doubled = _DoublingAddChild("", tree, "2017-01-01") + assert doubled.x == 10 and doubled["x"] == 10 + assert dict(doubled._children) == {"policy": 84, "x": 10} + + +class ReenteringChild: + """A child whose first resolution reads the same child again.""" + + def __init__(self): + self.node_at_instant = None + self.calls = 0 + + def _get_at_instant(self, instant): + self.calls += 1 + result = object() + if self.calls == 1: + self.inner = self.node_at_instant["again"] + return result + + +def test_a_reentrant_read_keeps_the_first_recorded_child(): + tree = plain_tree(other=1) + child = ReenteringChild() + tree.children = {"again": child, **tree.children} + at_instant = tree.get_at_instant("2017-01-01") + child.node_at_instant = at_instant + outer = at_instant["again"] + assert outer is child.inner + assert at_instant["again"] is outer and at_instant.again is outer + + +def test_children_and_their_order_are_fixed_when_the_node_is_built(): + tree = plain_tree(x=10, y=20, z=30) + held = tree.get_at_instant("2017-01-01") + assert held.x == 10 + del tree.children["x"] + tree.children["w"] = ParameterNode("", data={"v": {"values": {"2010-01-01": 1}}}) + assert list(held) == ["x", "y", "z"] + assert held.x == 10 + assert "w" not in held._children + + +def test_a_node_pickled_when_every_child_was_built_up_front_loads(): + """State as the up-front build pickled it: everything in ``_children``.""" + restored = ParameterNodeAtInstant.__new__(ParameterNodeAtInstant) + restored.__setstate__( + {"_name": "", "_instant_str": "2017-01-01", "_children": {"x": 10}, "x": 10} + ) + assert restored.x == 10 and restored["x"] == 10 and list(restored) == ["x"] + restored.add_child("y", 2) + assert restored["y"] == 2 and list(restored) == ["x", "y"] + with pytest.raises(KeyError): + restored["nope"] + + +class _AddChildLikeTheUpFrontBuild(ParameterNodeAtInstant): + """Writes ``_children`` and sets the attribute, as the up-front build's + ``add_child`` did, without calling it.""" + + def add_child(self, child_name, child_at_instant): + self._children[child_name] = child_at_instant + setattr(self, child_name, child_at_instant) + + +class _AddChildThatChecksChildren(ParameterNodeAtInstant): + def add_child(self, child_name, child_at_instant): + assert child_name not in self._children + super().add_child(child_name, child_at_instant) + + +@pytest.mark.parametrize( + "node_class", [_AddChildLikeTheUpFrontBuild, _AddChildThatChecksChildren] +) +def test_subclass_add_child_builds_items_and_iteration_as_before(node_class): + tree = ParameterNode( + "", + data={"x": {"values": {"2010-01-01": 10}}, "y": {"values": {"2010-01-01": 20}}}, + ) + + node = node_class("", tree, "2017-01-01") + node.add_child("later", 3) + + assert node["x"] == 10 and node.x == 10 + assert node["later"] == 3 + assert list(node) == ["x", "y", "later"] + with pytest.raises(ParameterNotFoundError): + node.absent + + +@pytest.mark.parametrize("instant", ["2015-02-29", "2015-02-30", "0000-01-01"]) +def test_a_node_at_a_date_that_does_not_exist_fails_as_before(instant): + # ``periods.instant`` reads these, but a scale's build cannot. + tree = build_tree() + + with pytest.raises(ValueError): + tree._get_at_instant(instant) + + assert instant not in tree._at_instant_cache + + +class _ChildrenHidingOne(dict): + """A ``children`` mapping whose ``items`` leaves an entry out.""" + + def items(self): + return [(name, child) for name, child in super().items() if name != "hidden"] + + +def test_children_are_the_ones_children_items_gives_as_before(): + tree = ParameterNode( + "", + data={ + "shown": {"values": {"2010-01-01": 1}}, + "hidden": {"values": {"2010-01-01": 2}}, + }, + ) + tree.children = _ChildrenHidingOne(tree.children) + + node = tree("2017-01-01") + + assert list(node) == ["shown"] + with pytest.raises(ParameterNotFoundError): + node.hidden + + +class _FallbackAttributes(ParameterNodeAtInstant): + """Answers names that are not children, as the up-front build let a + subclass do: ``__getattr__`` ran only for names that were not children.""" + + def __getattr__(self, key): + if key.startswith("_"): + return super().__getattr__(key) + return 0 + + +class _FallbackItems(ParameterNodeAtInstant): + def __getitem__(self, key): + try: + return self._children[key] + except KeyError: + return 0 + + +def test_subclass_attribute_and_item_fallbacks_see_every_child_as_before(): + tree = plain_tree(x=10, y=20) + + attributes = _FallbackAttributes("", tree, "2017-01-01") + items = _FallbackItems("", tree, "2017-01-01") + + assert attributes.x == 10 and attributes.absent == 0 + assert items["x"] == 10 and items["absent"] == 0 diff --git a/tests/core/parameters/test_parameter_node_at_instant_property.py b/tests/core/parameters/test_parameter_node_at_instant_property.py new file mode 100644 index 000000000..5eeec9372 --- /dev/null +++ b/tests/core/parameters/test_parameter_node_at_instant_property.py @@ -0,0 +1,91 @@ +"""Reads of a lazily resolved ``ParameterNodeAtInstant`` in any order. + +For any instant and any sequence of reads (attribute, item, iteration, +``_children``, ``dir``, ``repr``, vectorial indexing, names that do not +exist), every read returns what the up-front build held, and the node read in +full afterwards equals that build. ``test_parameter_node_at_instant_lazy.py`` +pins the same behaviour with examples. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +# The country-package smoke job installs no dev dependencies. +hypothesis = pytest.importorskip("hypothesis") +st = hypothesis.strategies + +from policyengine_core.errors import ParameterNotFoundError +from policyengine_core.parameters import ParameterNodeAtInstant +from tests.fixtures.parameter_nodes_at_instant import ( + INSTANTS, + build_tree, + describe, + materialise, + ordered, + snapshot, +) + +# Node paths in the fixture tree, and names to ask each for (present at some +# instants, absent at others, or never there). +PATHS = [(), ("group",), ("group", "inner"), ("by_zone",), ("by_zone", "z2")] +NAMES = ["flat_rate", "late", "group", "scale", "by_zone", "x", "y", "inner"] +NAMES += ["1", "2", "z1", "z2", "z3", "amount", "rate", "nope"] +KINDS = ["getattr", "getitem", "iter", "children", "dir", "repr", "vector"] + +reads = st.tuples( + st.sampled_from(PATHS), st.sampled_from(KINDS), st.sampled_from(NAMES) +) + + +def _comparable(value): + if isinstance(value, ParameterNodeAtInstant): + return materialise(value) + return describe(value) + + +@hypothesis.given( + instant=st.sampled_from(INSTANTS), sequence=st.lists(reads, max_size=30) +) +@hypothesis.settings(max_examples=400, deadline=None) +def test_reads_in_any_order_match_the_up_front_build(instant, sequence): + tree = build_tree() + expected_root = snapshot(tree, instant) + root = tree.get_at_instant(instant) + + for path, kind, name in sequence: + node, expected = root, expected_root + for step in path: + node, expected = node[step], expected[step] + + if kind == "getattr": + if name in expected: + assert _comparable(getattr(node, name)) == expected[name] + else: + with pytest.raises(ParameterNotFoundError): + getattr(node, name) + assert not hasattr(node, name) + elif kind == "getitem": + if name in expected: + assert _comparable(node[name]) == expected[name] + else: + with pytest.raises(KeyError): + node[name] + elif kind == "iter": + assert list(node) == list(expected) + assert (name in node) == (name in expected) + elif kind == "children": + assert ordered(materialise(node)) == ordered(expected) + elif kind == "dir": + assert set(expected) <= set(dir(node)) + elif kind == "repr": + assert all(child in repr(node) for child in expected) + elif kind == "vector" and path == ("by_zone",) and expected["z1"]: + zones = np.array(["z3", "z1"]) + np.testing.assert_array_equal( + node[zones].amount, + [expected["z3"]["amount"], expected["z1"]["amount"]], + ) + + assert ordered(materialise(root)) == ordered(expected_root) diff --git a/tests/core/parameters/test_parameter_node_at_instant_systems.py b/tests/core/parameters/test_parameter_node_at_instant_systems.py new file mode 100644 index 000000000..8557eb10e --- /dev/null +++ b/tests/core/parameters/test_parameter_node_at_instant_systems.py @@ -0,0 +1,383 @@ +"""Nodes at instant that resolve children on first read, as reforms and +country packages use them. + +The sharing tests follow how policyengine-us's ``spm.py`` handles parameter +trees and their caches: a shallow copy of the root with its own cache for a +traced request, systems that share one ``_parameters_at_instant_cache``, path +copies that share parameter values, and resetting every cache after editing +values in place. +""" + +import copy + +import pytest + +from policyengine_core.parameters import ( + Parameter, + ParameterNode, + ParameterNodeAtInstant, + ParameterScale, +) +from policyengine_core.reforms import Reform +from policyengine_core.tracers import TracingParameterNodeAtInstant +from tests.fixtures.parameter_nodes_at_instant import ( + build_tree, + materialise, + ordered, + snapshot, +) + +INSTANT = "2017-03-01" +SYSTEM_INSTANT = "2017-01-01" + + +# Reforms, on the country template. + + +def test_a_reform_reads_its_update_and_the_baseline_keeps_its_values( + isolated_tax_benefit_system, +): + system = isolated_tax_benefit_system + held = system.parameters(SYSTEM_INSTANT) + assert held.taxes.income_tax_rate == 0.15 + + class RaiseRate(Reform): + def apply(self): + self.parameters.taxes.income_tax_rate.update(period="2017", value=0.3) + + reformed = RaiseRate(system) + + assert reformed.parameters(SYSTEM_INSTANT).taxes.income_tax_rate == 0.3 + assert reformed.get_parameters_at_instant(SYSTEM_INSTANT).taxes.income_tax_rate == ( + 0.3 + ) + assert system.parameters(SYSTEM_INSTANT).taxes.income_tax_rate == 0.15 + # A child of the held baseline node read only now reads the baseline. + assert held.benefits.basic_income == 600 + + +def test_a_reform_that_reads_before_it_updates_reads_the_update( + isolated_tax_benefit_system, +): + class ReadThenRaise(Reform): + def apply(self): + assert self.parameters(SYSTEM_INSTANT).taxes.income_tax_rate == 0.15 + self.parameters.taxes.income_tax_rate.update(period="2017", value=0.3) + assert self.parameters(SYSTEM_INSTANT).taxes.income_tax_rate == 0.3 + + reformed = ReadThenRaise(isolated_tax_benefit_system) + + assert reformed.parameters(SYSTEM_INSTANT).taxes.income_tax_rate == 0.3 + + +def test_a_reform_that_adds_a_parameter_after_reading_reads_it( + isolated_tax_benefit_system, +): + class AddAllowance(Reform): + def apply(self): + self.parameters(SYSTEM_INSTANT).taxes.income_tax_rate + self.parameters.taxes.add_child( + "allowance", + Parameter("taxes.allowance", data={"values": {"2010-01-01": 100}}), + ) + + reformed = AddAllowance(isolated_tax_benefit_system) + + assert reformed.parameters(SYSTEM_INSTANT).taxes.allowance == 100 + assert reformed.get_parameters_at_instant(SYSTEM_INSTANT).taxes.allowance == 100 + + +def test_modify_parameters_copies_partly_read_nodes_onto_the_reform_tree( + isolated_tax_benefit_system, +): + system = isolated_tax_benefit_system + # The baseline caches a node for ``benefits`` whose children are not read. + system.parameters(SYSTEM_INSTANT).benefits + + def raise_rate(parameters): + parameters.taxes.income_tax_rate.update(period="2017", value=0.3) + return parameters + + class RaiseRate(Reform): + def apply(self): + self.modify_parameters(raise_rate) + + reformed = RaiseRate(system) + # A change to the baseline after the reform was built stays in the + # baseline, including for the copied nodes' children not read yet. + system.parameters.benefits.basic_income.update(period="2017", value=700) + + reform_node = reformed.get_parameters_at_instant(SYSTEM_INSTANT) + assert reform_node.taxes.income_tax_rate == 0.3 + assert reform_node.benefits.basic_income == 600 + assert system.parameters(SYSTEM_INSTANT).benefits.basic_income == 700 + assert system.parameters(SYSTEM_INSTANT).taxes.income_tax_rate == 0.15 + + +# Sharing trees and caches, as policyengine-us's spm.py does. + + +class RecordingTracer: + def __init__(self): + self.accesses = [] + + def record_parameter_access(self, name, period, branch_name, value): + self.accesses.append((name, period, branch_name, value)) + + +def test_a_shallow_copy_of_the_root_with_its_own_cache_reads_like_the_original(): + tree = build_tree() + original = tree(INSTANT) + original.group.x + # As ``isolate_parameter_tracing``: the copy shares the children and gets + # its own cache and tracer. + private = copy.copy(tree) + private._at_instant_cache = {} + private.trace = True + private.tracer = RecordingTracer() + private.branch_name = "request" + + traced = private(INSTANT) + rate = traced.group.x + + assert isinstance(traced, TracingParameterNodeAtInstant) + node = traced.parameter_node_at_instant + assert ordered(materialise(node)) == ordered(snapshot(tree, INSTANT)) + # The children's caches are shared, as when every child was built up front. + assert node.group is original.group + assert rate == 0.1 + assert private.tracer.accesses == [("group.x", INSTANT, "request", 0.1)] + assert tree._at_instant_cache[INSTANT] is original + assert not isinstance(tree(INSTANT), TracingParameterNodeAtInstant) + + +def test_systems_sharing_one_cache_read_the_same_nodes( + isolated_tax_benefit_system, +): + system = isolated_tax_benefit_system + policy = system.clone() + policy.parameters = system.parameters + policy._parameters_at_instant_cache = system._parameters_at_instant_cache + + from_system = system.get_parameters_at_instant(SYSTEM_INSTANT) + from_policy = policy.get_parameters_at_instant(SYSTEM_INSTANT) + + assert from_policy is from_system + assert from_policy.taxes.income_tax_rate == 0.15 + assert from_system.taxes is from_policy.taxes + assert ordered(materialise(from_policy)) == ordered( + snapshot(system.parameters, SYSTEM_INSTANT) + ) + + +def path_copy(node, parent=None): + """Copy the parameter nodes and share the dated values, as policyengine-us's + ``copy_parameter_nodes`` does.""" + copied = object.__new__(type(node)) + copied.__dict__.update(node.__dict__) + copied.metadata = dict(node.metadata) + if isinstance(node, Parameter): + copied.values_list = list(node.values_list) + elif isinstance(node, ParameterScale): + copied.brackets = [path_copy(bracket, copied) for bracket in node.brackets] + else: + copied.children = { + name: path_copy(child, copied) for name, child in node.children.items() + } + for name, child in copied.children.items(): + setattr(copied, name, child) + copied._at_instant_cache = {} + copied.parent = parent + return copied + + +def test_a_path_copy_reads_like_the_original_and_keeps_its_updates(): + tree = build_tree() + held = tree(INSTANT) + held.group.x + + copied = path_copy(tree) + + assert ordered(materialise(copied(INSTANT))) == ordered(snapshot(tree, INSTANT)) + copied.flat_rate.update(period="2017", value=0.5) + copied.group.x.update(period="2017", value=0.6) + assert copied(INSTANT).flat_rate == 0.5 + assert copied(INSTANT).group.x == 0.6 + assert held.flat_rate == 0.1 + assert tree(INSTANT).flat_rate == 0.1 + assert tree(INSTANT).group.x == 0.1 + + +def clear_every_cache(system_or_tree): + root = getattr(system_or_tree, "parameters", system_or_tree) + if root is not system_or_tree: + system_or_tree._parameters_at_instant_cache.clear() + for node in (root, *root.get_descendants()): + cache = getattr(node, "_at_instant_cache", None) + if cache is not None: + cache.clear() + + +def test_values_edited_in_place_are_read_once_every_cache_is_cleared( + isolated_tax_benefit_system, +): + system = isolated_tax_benefit_system + system.get_parameters_at_instant(SYSTEM_INSTANT).taxes.income_tax_rate + system.parameters(SYSTEM_INSTANT).benefits + + for value in system.parameters.taxes.income_tax_rate.values_list: + value.value = 0.5 + system.parameters.benefits.basic_income.values_list[0].value = 900 + clear_every_cache(system) + + for node in ( + system.get_parameters_at_instant(SYSTEM_INSTANT), + system.parameters(SYSTEM_INSTANT), + ): + assert node.taxes.income_tax_rate == 0.5 + assert node.benefits.basic_income == 900 + + +def test_values_edited_in_place_are_read_once_the_parents_cache_is_cleared(): + # As policyengine-us's per-capita uprating: edit, then clear the caches + # above the edited parameter. + tree = build_tree() + tree(INSTANT).group.x + tree(INSTANT).flat_rate + + tree.group.x.values_list[0].value = 0.7 + tree.group.x.parent.clear_parent_cache() + + assert tree(INSTANT).group.x == 0.7 + assert tree.group(INSTANT).x == 0.7 + + +# Instants that are not dates. + + +def test_a_tree_with_a_scale_cannot_be_read_at_none_as_before(): + tree = build_tree() + + with pytest.raises(ValueError): + tree(None) + + assert set(tree._at_instant_cache) == set() + + +def test_a_tree_without_a_scale_reads_at_none_as_before(): + tree = ParameterNode( + "", + data={ + "a": {"values": {"2010-01-01": 1, "2020-01-01": 2}}, + "b": {"c": {"values": {"2015-01-01": 3}}}, + }, + ) + + node = tree(None) + + assert isinstance(node, ParameterNodeAtInstant) + assert ordered(materialise(node)) == ordered(snapshot(tree, "None")) + + +# Updates below a scale bracket. + +SCALE_REFORM = { + "taxes.social_security_contribution[1].rate": {"2017-01-01.2017-12-31": 0.5} +} +SALARY_SITUATION = { + "persons": {"bill": {"salary": {"2017-01": 20000}}}, + "households": {"h": {"parents": ["bill"]}}, +} + + +def contribution_after_reform(read_the_scale_first): + from policyengine_core.country_template import CountryTaxBenefitSystem + from policyengine_core.simulations import SimulationBuilder + + simulation = SimulationBuilder().build_from_entities( + CountryTaxBenefitSystem(), SALARY_SITUATION + ) + simulation.calculate("income_tax", "2017-01") + if read_the_scale_first: + simulation.calculate("social_security_contribution", "2017-01") + simulation.apply_reform(SCALE_REFORM) + return simulation.calculate("social_security_contribution", "2017-01")[0] + + +def test_a_reform_of_a_bracket_reads_the_same_whatever_was_calculated_first(): + # ``Parameter.update`` on a bracket's rate clears the caches above the + # scale, because the bracket's ``parent`` is its scale. + assert contribution_after_reform(False) == contribution_after_reform(True) + + +@pytest.mark.parametrize("read_the_scale_first", [False, True]) +def test_nodes_requested_after_a_bracket_update_read_it(read_the_scale_first): + tree = build_tree() + tree(INSTANT).flat_rate + if read_the_scale_first: + tree(INSTANT).scale + + tree.scale.brackets[1].rate.update(period="2017", value=0.9) + + assert list(tree(INSTANT).scale.rates) == [0.1, 0.9] + assert list(tree.scale(INSTANT).rates) == [0.1, 0.9] + + +# Updates below a node that homogenization put in place of a missing +# breakdown value. + + +def homogenized_tree(): + from policyengine_core.parameters.operations.homogenize_parameters import ( + homogenize_parameter_structures, + ) + + tree = ParameterNode( + "", + data={ + "rates": { + "metadata": {"breakdown": [["A", "B"], ["X", "Y"]]}, + "A": { + "X": {"values": {"2010-01-01": 1}}, + "Y": {"values": {"2010-01-01": 2}}, + }, + }, + }, + ) + homogenize_parameter_structures(tree, {}) + return tree + + +def test_homogenization_links_the_nodes_it_creates_to_their_parents(): + tree = homogenized_tree() + + created = tree.rates.children["B"] + + assert created.parent is tree.rates + assert created.children["X"].parent is created + + +@pytest.mark.parametrize("read_the_leaf_first", [False, True]) +def test_nodes_requested_after_an_update_below_a_homogenized_node_read_it( + read_the_leaf_first, +): + tree = homogenized_tree() + node = tree(INSTANT) + node.rates.A.X + if read_the_leaf_first: + node.rates["B"]["X"] + + tree.rates.children["B"].children["X"].update(period="2017", value=50) + + assert tree(INSTANT).rates["B"]["X"] == 50 + + +def test_a_bracket_update_marks_the_scale_and_the_nodes_above_it_modified(): + # As in a cloned tree, where brackets were already linked to their scale. + tree = build_tree() + + tree.scale.brackets[1].rate.update(period="2017", value=0.9) + + assert tree.scale.brackets[1].modified + assert tree.scale.modified + assert tree.modified diff --git a/tests/fixtures/parameter_nodes_at_instant.py b/tests/fixtures/parameter_nodes_at_instant.py new file mode 100644 index 000000000..7b991041b --- /dev/null +++ b/tests/fixtures/parameter_nodes_at_instant.py @@ -0,0 +1,131 @@ +"""A parameter tree and an up-front reference for the at-instant tests. + +``snapshot`` computes what a ``ParameterNodeAtInstant`` held when it built +every descendant in its constructor, straight from the parameter tree and +without using ``ParameterNodeAtInstant``. ``materialise`` reads the same +thing out of a ``ParameterNodeAtInstant``. +""" + +import numpy as np + +from policyengine_core.parameters import ( + Parameter, + ParameterNode, + ParameterNodeAtInstant, + ParameterScale, +) + +INSTANTS = [ + "2009-12-31", + "2010-01-01", + "2014-06-30", + "2015-01-01", + "2017-03-01", + "2018-01-01", + "2020-01-01", + "2029-12-31", + "2030-01-01", + "2035-07-01", +] + + +def _zone(amount, rate): + return { + "amount": {"values": {"2010-01-01": amount, "2020-01-01": amount * 2}}, + "rate": {"values": {"2010-01-01": rate}}, + } + + +TREE = { + "flat_rate": {"values": {"2015-01-01": 0.1, "2020-01-01": 0.2}}, + # Not defined before 2030, so absent from earlier instants. + "late": {"values": {"2030-01-01": 5}}, + "group": { + "x": {"values": {"2010-01-01": 0.1}}, + "y": {"values": {"2018-01-01": 0.2}}, + # Every child undefined before 2018: an empty node, not an absent one. + "inner": { + "1": {"values": {"2018-01-01": 10}}, + "2": {"values": {"2018-01-01": 20, "2020-01-01": 30}}, + }, + }, + "scale": { + "brackets": [ + { + "threshold": {"2010-01-01": {"value": 0}}, + "rate": {"2010-01-01": {"value": 0.1}}, + }, + { + "threshold": {"2010-01-01": {"value": 1000}}, + "rate": { + "2010-01-01": {"value": 0.2}, + "2020-01-01": {"value": 0.3}, + }, + }, + ] + }, + # Homogeneous children: vectorisable with ``node[array]``. + "by_zone": {"z1": _zone(100, 0.1), "z2": _zone(200, 0.2), "z3": _zone(300, 0.3)}, +} + + +def build_tree() -> ParameterNode: + return ParameterNode("", data=TREE) + + +def describe(value): + """A comparable form of one at-instant value.""" + if isinstance(value, (int, float, bool, str)) or value is None: + return value + if isinstance(value, (list, tuple)): + return tuple(describe(item) for item in value) + if isinstance(value, dict): + return {key: describe(item) for key, item in value.items()} + if hasattr(value, "thresholds"): + # Tax scales refuse ``==``. + return ( + type(value).__name__, + tuple(value.thresholds), + tuple(getattr(value, "rates", getattr(value, "amounts", ()))), + ) + if isinstance(value, np.ndarray): + return ("ndarray", str(value.dtype), value.tolist()) + if isinstance(value, np.generic): + return value.item() + return (type(value).__name__, repr(value)) + + +def snapshot(node: ParameterNode, instant_str: str) -> dict: + """Every descendant's value at the instant, in the node's own order, + leaving out children that have no value yet.""" + result = {} + for name, child in node.children.items(): + if isinstance(child, ParameterScale): + value = describe(child._get_at_instant(instant_str)) + elif isinstance(child, Parameter): + value = child._get_at_instant(instant_str) + else: + value = snapshot(child, instant_str) + if value is not None: + result[name] = value + return result + + +def materialise(node_at_instant: ParameterNodeAtInstant) -> dict: + """The same nested mapping, read from a ``ParameterNodeAtInstant``.""" + return { + name: ( + materialise(child) + if isinstance(child, ParameterNodeAtInstant) + else describe(child) + ) + for name, child in node_at_instant._children.items() + } + + +def ordered(mapping: dict): + """A form that compares key order as well as content.""" + return [ + (key, ordered(value) if isinstance(value, dict) else value) + for key, value in mapping.items() + ]