diff --git a/changelog.d/link-parameter-parents.fixed.md b/changelog.d/link-parameter-parents.fixed.md new file mode 100644 index 000000000..ec4b78ad9 --- /dev/null +++ b/changelog.d/link-parameter-parents.fixed.md @@ -0,0 +1 @@ +Loaded parameter trees now link every scale bracket and every node built by breakdown homogenization to the node that holds it, as a cloned tree already did, so updating one clears cached ancestor values and marks ancestors as modified. The bracket that the country template reform appends is linked to its scale too. A child whose name matches a node's own attribute or method (such as `parent`, `children` or `modified`) is no longer set as an attribute, so it can no longer replace the node's own state. Reading a homogenized breakdown position as an attribute, such as `amount.B`, now returns the `ParameterNode` that homogenization built there instead of the placeholder `Parameter` it replaced, so calling it returns a node at that instant and calling `update` on it raises, while lookups at an instant, such as `amount("2021-01-01").B`, are unchanged. diff --git a/policyengine_core/country_template/reforms/modify_social_security_taxation.py b/policyengine_core/country_template/reforms/modify_social_security_taxation.py index 78a6be226..194235a32 100644 --- a/policyengine_core/country_template/reforms/modify_social_security_taxation.py +++ b/policyengine_core/country_template/reforms/modify_social_security_taxation.py @@ -35,7 +35,8 @@ def modify_brackets(parameters): of the YAML parameters. It can be modified and must be returned. """ # Access the right parameter node: - brackets = parameters.taxes.social_security_contribution.brackets + scale = parameters.taxes.social_security_contribution + brackets = scale.brackets # Add 0.1 to the rates of the second bracket, keeping the same thresholds: for rate in brackets[1].rate.values_list: @@ -52,6 +53,9 @@ def modify_brackets(parameters): "threshold": {"2017-01-01": {"value": 40000}}, }, ) + # Link the bracket to its scale, so that updates to it clear cached + # values and mark the scale and its ancestors as modified. + new_bracket.parent = scale brackets.append(new_bracket) return parameters diff --git a/policyengine_core/parameters/operations/homogenize_parameters.py b/policyengine_core/parameters/operations/homogenize_parameters.py index 39164b721..dca165976 100644 --- a/policyengine_core/parameters/operations/homogenize_parameters.py +++ b/policyengine_core/parameters/operations/homogenize_parameters.py @@ -80,7 +80,7 @@ def homogenize_parameter_node( possible_values = [True, False] else: possible_values = evaluate_dynamic_breakdown(first_breakdown) - if not hasattr(node, "children"): + if not isinstance(node, ParameterNode): node = ParameterNode( node.name, data={ @@ -120,11 +120,15 @@ def homogenize_parameter_node( f"metadata references the correct variable and that all " f"parameter keys are valid enum values." ) - for child in node.children: - if further_breakdown: - node.children[child] = homogenize_parameter_node( + if further_breakdown: + for child in list(node.children): + homogenized_child = homogenize_parameter_node( node.children[child], breakdown[1:], variables, default_value ) + if homogenized_child is not node.children[child]: + # A placeholder Parameter became a ParameterNode: link the + # replacement as add_child would. + node._replace_child(child, homogenized_child) return node diff --git a/policyengine_core/parameters/parameter_node.py b/policyengine_core/parameters/parameter_node.py index 8338716f2..57497804f 100644 --- a/policyengine_core/parameters/parameter_node.py +++ b/policyengine_core/parameters/parameter_node.py @@ -1,4 +1,5 @@ import copy +import functools import os import typing from typing import Iterable, List, Type, Union @@ -22,6 +23,56 @@ EXCLUDED_PARAMETER_CHILD_NAMES = ["reference", "__pycache__"] +# Attributes every ParameterNode sets on itself. A child with one of these +# names stays reachable through ``children`` and at-instant lookups, but is not +# set as an attribute, so it cannot replace the node's own state (for example a +# child named ``parent`` replacing the link to the holding node). +NODE_OWN_ATTRIBUTE_NAMES = frozenset( + { + "name", + "children", + "description", + "documentation", + "file_path", + "metadata", + "trace", + "tracer", + "branch_name", + "_at_instant_cache", + "parent", + "modified", + } +) + + +@functools.lru_cache(maxsize=None) +def _names_blocked_as_child_attributes(node_class: type) -> frozenset: + # The node's own attributes, plus the methods and class attributes of its + # classes (including subclasses such as ParameterScaleBracket). Metaclass + # attributes such as ABCMeta.register or type.mro are not included: they + # are not reachable from an instance. Computed once per class. + names = set(NODE_OWN_ATTRIBUTE_NAMES) + for klass in node_class.__mro__: + names.update(klass.__dict__) + return frozenset(names) + + +def _can_expose_child_as_attribute(node: "ParameterNode", child_name: str) -> bool: + return child_name not in _names_blocked_as_child_attributes(type(node)) + + +def _check_child_type(child) -> None: + if not ( + isinstance(child, ParameterNode) + or isinstance(child, Parameter) + or isinstance(child, parameters.ParameterScale) + ): + raise TypeError( + "child must be of type ParameterNode, Parameter, or Scale. Instead got {}".format( + type(child) + ) + ) + class ParameterNode(AtInstantLike): """ @@ -164,23 +215,64 @@ def add_child(self, name: str, child: Union["ParameterNode", Parameter]): """ Add a new child to the node. - :param name: Name of the child that must be used to access that child. Should not contain anything that could interfere with the operator `.` (dot). + :param name: Name of the child that must be used to access that child. Should not contain anything that could interfere with the operator `.` (dot). The child is also set as an attribute of the node, unless the name matches one of the node's own attributes (such as ``parent`` or ``children``) or a method or class attribute of its class; it then stays reachable through ``children`` and at-instant lookups. :param child: The new child, an instance of :class:`.ParameterScale` or :class:`.Parameter` or :class:`.ParameterNode`. """ if name in self.children: raise ValueError("{} has already a child named {}".format(self.name, name)) - if not ( - isinstance(child, ParameterNode) - or isinstance(child, Parameter) - or isinstance(child, parameters.ParameterScale) - ): + _check_child_type(child) + self._link_child(name, child) + + def _replace_child( + self, + name: str, + child: Union["ParameterNode", Parameter, "parameters.ParameterScale"], + ) -> None: + """ + Replace an existing child and link the replacement to this node. + + Internal: homogenization uses it to replace a placeholder Parameter + with the ParameterNode it builds for a further breakdown level. + + Cached values of this node and its ancestors are cleared. The previous + child is left as it is, and nodes are not marked as modified. The tree + is left unchanged when an argument is rejected. + + :param name: Name of the existing child. + :param child: The replacement, an instance of :class:`.ParameterScale`, :class:`.Parameter` or :class:`.ParameterNode`. It must not be this node or one of its ancestors. + """ + # Check every argument before changing anything, so a rejected call + # leaves the tree as it was. + if name not in self.children: + raise KeyError( + "{} has no child named {} to replace".format(self.name, name) + ) + if not isinstance(name, str): raise TypeError( - "child must be of type ParameterNode, Parameter, or Scale. Instead got {}".format( - type(child) - ) + "Child names must be strings. {} got {!r}".format(self.name, name) ) + _check_child_type(child) + node_or_ancestor = self + while node_or_ancestor is not None: + if node_or_ancestor is child: + raise ValueError( + "{} cannot hold itself or one of its ancestors ({}) as a " + "child".format(self.name, child.name) + ) + node_or_ancestor = getattr(node_or_ancestor, "parent", None) + + self._link_child(name, child) + # Cached at-instant nodes of this node and its ancestors still hold + # the previous child's values. + self.clear_parent_cache() + + def _link_child(self, name: str, child) -> None: + # Store the child, expose it as an attribute when its name does not + # collide with one of the node's own attributes or methods, and link it + # upward so Parameter.update can clear caches and mark ancestors. self.children[name] = child - setattr(self, name, child) + if _can_expose_child_as_attribute(self, name): + setattr(self, name, child) child.parent = self def __repr__(self) -> str: @@ -208,7 +300,8 @@ def clone(self) -> "ParameterNode": clone.metadata = copy.copy(self.metadata) clone.children = {key: child.clone() for key, child in self.children.items()} for child_key, child in clone.children.items(): - setattr(clone, child_key, child) + if _can_expose_child_as_attribute(clone, child_key): + setattr(clone, child_key, child) child.parent = clone clone._at_instant_cache = {} diff --git a/policyengine_core/parameters/parameter_scale.py b/policyengine_core/parameters/parameter_scale.py index 9c501765a..507d01511 100644 --- a/policyengine_core/parameters/parameter_scale.py +++ b/policyengine_core/parameters/parameter_scale.py @@ -53,6 +53,7 @@ def __init__(self, name: str, data: dict, file_path: str): bracket = parameters.ParameterScaleBracket( name=bracket_name, data=bracket_data, file_path=file_path ) + bracket.parent = self brackets.append(bracket) self.brackets: typing.List[parameters.ParameterScaleBracket] = brackets self.propagate_uprating() diff --git a/tests/core/parameters/operations/test_homogenization_parent_links.py b/tests/core/parameters/operations/test_homogenization_parent_links.py new file mode 100644 index 000000000..1ee887b1f --- /dev/null +++ b/tests/core/parameters/operations/test_homogenization_parent_links.py @@ -0,0 +1,99 @@ +"""Nodes that homogenization builds for further breakdown levels are linked. + +``homogenize_parameter_node`` replaces a placeholder ``Parameter`` with a +``ParameterNode`` when a breakdown has more than one level. The replacement +must link to the node holding it, and that node's attribute must point at the +replacement rather than the placeholder. Placeholders that homogenization adds +for missing breakdown values must stay linked too. +""" + +from policyengine_core.parameters import Parameter, ParameterNode +from policyengine_core.parameters.operations.homogenize_parameters import ( + homogenize_parameter_structures, +) + + +def build_two_level_tree(first_level_values) -> ParameterNode: + return ParameterNode( + data={ + "program": { + "amount": { + str(first_level_values[0]): { + "1": {"2020-01-01": 10}, + }, + "metadata": { + "breakdown": [first_level_values, [1, 2]], + }, + } + } + } + ) + + +def test_homogenized_replacements_link_to_their_holder(): + root = build_two_level_tree(["A", "B"]) + homogenize_parameter_structures(root, variables={}) + amount = root.program.amount + + for key in ("A", "B"): + replacement = amount.children[key] + assert isinstance(replacement, ParameterNode) + assert replacement.parent is amount + assert getattr(amount, key) is replacement + for leaf in replacement.children.values(): + assert leaf.parent is replacement + + +def test_update_under_homogenized_node_marks_every_ancestor_and_refreshes_cached_value(): + root = build_two_level_tree(["A", "B"]) + homogenize_parameter_structures(root, variables={}) + + before = root("2021-01-01").program.amount.B["2"] + assert before == 0 + + leaf = root.program.amount.children["B"].children["2"] + leaf.update(period="year:2021:1", value=7) + + assert root.program.amount.children["B"].modified + assert root.program.amount.modified + assert root.program.modified + assert root.modified + assert root("2021-01-01").program.amount.B["2"] == 7 + + +def test_missing_breakdown_value_named_parent_keeps_the_parent_link(): + # "parent" is a missing breakdown value, so homogenization adds it as a + # placeholder Parameter and then replaces it with a ParameterNode. + root = build_two_level_tree(["child", "parent"]) + homogenize_parameter_structures(root, variables={}) + program = root.children["program"] + amount = program.children["amount"] + + assert amount.parent is program + replacement = amount.children["parent"] + assert isinstance(replacement, ParameterNode) + assert replacement.parent is amount + + replacement.children["2"].update(period="year:2021:1", value=4) + assert amount.modified + assert program.modified + assert root.modified + assert root("2021-01-01").program.amount["parent"]["2"] == 4 + + +def test_single_level_homogenization_links_added_placeholders(): + root = ParameterNode( + data={ + "program": { + "A": {"2020-01-01": 1}, + "metadata": {"breakdown": [["A", "B"]]}, + } + } + ) + homogenize_parameter_structures(root, variables={}) + program = root.program + + placeholder = program.children["B"] + assert isinstance(placeholder, Parameter) + assert placeholder.parent is program + assert program.B is placeholder diff --git a/tests/core/parameters/test_parameter_tree_parent_links.py b/tests/core/parameters/test_parameter_tree_parent_links.py new file mode 100644 index 000000000..65e04493a --- /dev/null +++ b/tests/core/parameters/test_parameter_tree_parent_links.py @@ -0,0 +1,316 @@ +"""Every node in a loaded parameter tree links to the node that holds it. + +``Parameter.update`` walks upward through ``parent`` to clear cached +at-instant nodes and to mark ancestors as modified, so a missing or wrong +``parent`` link leaves stale values and unmarked ancestors. A cloned tree links +every node, so a loaded tree and its clone must behave the same. +""" + +import numpy +import pytest + +from policyengine_core.parameters import ( + Parameter, + ParameterNode, + ParameterScale, +) + + +def build_tree_with_scale() -> ParameterNode: + return ParameterNode( + data={ + "tax": { + "credit": { + "base": { + "brackets": [ + { + "threshold": {"values": {"2020-01-01": 0}}, + "amount": {"values": {"2020-01-01": 100}}, + }, + { + "threshold": {"values": {"2020-01-01": 10}}, + "amount": {"values": {"2020-01-01": 50}}, + }, + ] + }, + }, + "rate": {"values": {"2020-01-01": 0.2}}, + } + } + ) + + +def iter_tree_with_holders(root): + """Yield (holding node, child) for every link in the tree, brackets included.""" + if isinstance(root, ParameterScale): + for bracket in root.brackets: + yield root, bracket + yield from iter_tree_with_holders(bracket) + elif isinstance(root, ParameterNode): + for child in root.children.values(): + yield root, child + yield from iter_tree_with_holders(child) + + +def modified_names(root) -> set: + return { + child.name + for _, child in iter_tree_with_holders(root) + if getattr(child, "modified", False) + } | ({root.name} if getattr(root, "modified", False) else set()) + + +def test_scale_brackets_link_to_their_scale(): + root = build_tree_with_scale() + scale = root.tax.credit.base + + assert len(scale.brackets) == 2 + for bracket in scale.brackets: + assert bracket.parent is scale + + +def test_every_loaded_node_links_to_its_holder(): + root = build_tree_with_scale() + + for holder, child in iter_tree_with_holders(root): + assert child.parent is holder, child.name + + +def test_bracket_update_marks_same_nodes_on_loaded_tree_and_clone(): + loaded_root = build_tree_with_scale() + cloned_root = build_tree_with_scale().clone() + + for root in (loaded_root, cloned_root): + root.tax.credit.base.brackets[0].amount.update(period="year:2021:1", value=200) + + loaded_modified = modified_names(loaded_root) + assert loaded_modified == modified_names(cloned_root) + assert { + "tax.credit.base[0].amount", + "tax.credit.base[0]", + "tax.credit.base", + "tax.credit", + "tax", + } <= loaded_modified + assert "tax.rate" not in loaded_modified + + +def test_bracket_update_clears_cached_ancestor_values(): + root = build_tree_with_scale() + income = numpy.array([5]) + before = root("2021-01-01").tax.credit.base.calc(income) + assert before[0] == 100 + + root.tax.credit.base.brackets[0].amount.update(period="year:2021:1", value=200) + + assert root("2021-01-01").tax.credit.base.calc(income)[0] == 200 + + +def test_child_named_parent_does_not_replace_the_parent_link(): + root = ParameterNode( + data={ + "categories": { + "parent": {"values": {"2020-01-01": 1}}, + "child": {"values": {"2020-01-01": 2}}, + } + } + ) + categories = root.children["categories"] + parent_category = categories.children["parent"] + + assert categories.parent is root + assert parent_category.parent is categories + + standalone = ParameterNode(data={"parent": {"values": {"2020-01-01": 1}}}) + assert standalone.parent is None + assert standalone.children["parent"].parent is standalone + + parent_category.update(period="year:2021:1", value=3) + assert root.modified + assert root("2021-01-01").categories.parent == 3 + + +@pytest.mark.parametrize("reserved_name", ["children", "name", "modified", "parent"]) +def test_child_name_cannot_overwrite_node_attributes(reserved_name): + # A root node and its clone. No holder re-assigns a root's ``parent`` + # afterwards, so an overwrite of ``parent``, ``name`` or ``children`` stays + # visible on the root. ``__init__`` resets ``modified`` at its end, so + # only the clone shows an overwrite of that one. + root = ParameterNode("root", data={reserved_name: {"values": {"2020-01-01": 1}}}) + + for node in (root, root.clone()): + assert isinstance(node.children, dict) + assert node.name == "root" + assert node.parent is None + assert node.modified is False + assert node.children[reserved_name].parent is node + assert reserved_name not in vars(node) or not isinstance( + vars(node)[reserved_name], (Parameter, ParameterNode) + ) + assert node("2021-01-01")[reserved_name] == 1 + + +def test_child_name_cannot_shadow_node_methods(): + root = ParameterNode(data={"clone": {"values": {"2020-01-01": 1}}}) + + assert "clone" not in vars(root) + cloned = root.clone() + assert "clone" not in vars(cloned) + assert cloned.children["clone"].parent is cloned + assert cloned("2021-01-01").clone == 1 + + +def test_replace_child_links_the_replacement(): + root = ParameterNode(data={"node": {"value": {"values": {"2020-01-01": 1}}}}) + node = root.children["node"] + replacement = ParameterNode("node.value", data={"a": {"values": {"2020-01-01": 5}}}) + + node._replace_child("value", replacement) + + assert node.children["value"] is replacement + assert node.value is replacement + assert replacement.parent is node + assert root("2021-01-01").node.value.a == 5 + + +def test_replace_child_requires_an_existing_child(): + node = ParameterNode(data={"value": {"values": {"2020-01-01": 1}}}) + + with pytest.raises(KeyError): + node._replace_child("missing", Parameter("missing", {"2020-01-01": 1})) + + +def test_bracket_appended_by_template_reform_links_to_its_scale(): + from policyengine_core.country_template import CountryTaxBenefitSystem + from policyengine_core.country_template.reforms.modify_social_security_taxation import ( + modify_social_security_taxation, + ) + + reformed = modify_social_security_taxation(CountryTaxBenefitSystem()) + scale = reformed.parameters.taxes.social_security_contribution + + for bracket in scale.brackets: + assert bracket.parent is scale, bracket.name + + +def test_replace_child_clears_cached_values_and_marks_nothing_modified(): + root = ParameterNode(data={"node": {"value": {"values": {"2020-01-01": 1}}}}) + node = root.children["node"] + assert root("2021-01-01").node.value == 1 + old_child = node.children["value"] + + node._replace_child("value", Parameter("node.value", {"2020-01-01": 9})) + + assert root("2021-01-01").node.value == 9 + # The previous child is left as it is. + assert old_child.parent is node + assert not node.modified + assert not root.modified + + +def test_clone_keeps_modified_flag_when_a_child_is_named_modified(): + root = ParameterNode(data={"node": {"modified": {"values": {"2020-01-01": 1}}}}) + + cloned_node = root.clone().children["node"] + + assert cloned_node.modified is False + assert cloned_node.children["modified"].parent is cloned_node + + +def test_replace_child_keeps_link_of_previous_child_moved_elsewhere(): + root = ParameterNode(data={"node": {"value": {"values": {"2020-01-01": 1}}}}) + node = root.children["node"] + previous_child = node.children["value"] + wrapper = ParameterNode("node.value", data={}) + wrapper.add_child("default", previous_child) + + node._replace_child("value", wrapper) + + assert previous_child.parent is wrapper + assert wrapper.parent is node + previous_child.update(period="year:2021:1", value=5) + assert root("2021-01-01").node.value.default == 5 + + +@pytest.mark.parametrize("metaclass_attribute_name", ["register", "mro"]) +def test_child_named_after_metaclass_attribute_is_an_attribute( + metaclass_attribute_name, +): + root = ParameterNode(data={metaclass_attribute_name: {"values": {"2020-01-01": 1}}}) + child = root.children[metaclass_attribute_name] + + assert getattr(root, metaclass_attribute_name) is child + assert getattr(root.clone(), metaclass_attribute_name).name == child.name + + +@pytest.mark.parametrize("previous_child_kind", ["parameter", "scale"]) +def test_replace_child_accepts_previous_child_without_parent_attribute( + previous_child_kind, +): + # A Parameter or ParameterScale stored in ``children`` directly, without + # add_child, has no ``parent`` attribute at all. + root = ParameterNode(data={"node": {"other": {"values": {"2020-01-01": 0}}}}) + node = root.children["node"] + if previous_child_kind == "parameter": + previous_child = Parameter("node.x", {"2020-01-01": 1}) + else: + previous_child = ParameterScale( + "node.x", + {"brackets": [{"threshold": {"values": {"2020-01-01": 0}}}]}, + None, + ) + node.children["x"] = previous_child + assert not hasattr(previous_child, "parent") + root("2021-01-01") + + node._replace_child("x", Parameter("node.x", {"2020-01-01": 2})) + + assert node.children["x"].parent is node + assert root("2021-01-01").node.x == 2 + + +def test_replace_child_keeps_link_of_previous_child_still_held_under_another_name(): + root = ParameterNode(data={"node": {"a": {"values": {"2020-01-01": 1}}}}) + node = root.children["node"] + shared_child = node.children["a"] + node.children["b"] = shared_child + + node._replace_child("a", Parameter("node.a", {"2020-01-01": 2})) + + assert shared_child.parent is node + shared_child.update(period="year:2021:1", value=3) + assert root("2021-01-01").node["b"] == 3 + + +@pytest.mark.parametrize("replacement_kind", ["holder", "ancestor"]) +def test_replace_child_rejects_holder_or_ancestor_and_leaves_tree_unchanged( + replacement_kind, +): + root = ParameterNode(data={"node": {"a": {"values": {"2020-01-01": 1}}}}) + node = root.children["node"] + previous_child = node.children["a"] + replacement = node if replacement_kind == "holder" else root + + with pytest.raises(ValueError): + node._replace_child("a", replacement) + + assert node.children["a"] is previous_child + assert previous_child.parent is node + assert node.parent is root + assert root.parent is None + + +def test_replace_child_rejects_non_string_name_and_leaves_tree_unchanged(): + root = ParameterNode(data={"node": {"a": {"values": {"2020-01-01": 1}}}}) + node = root.children["node"] + previous_child = Parameter("node.1", {"2020-01-01": 1}) + node.children[1] = previous_child + previous_child.parent = node + replacement = Parameter("node.1", {"2020-01-01": 2}) + + with pytest.raises(TypeError): + node._replace_child(1, replacement) + + assert node.children[1] is previous_child + assert previous_child.parent is node + assert not hasattr(replacement, "parent")