Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/link-parameter-parents.fixed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Loaded parameter trees now link every scale bracket and every node built by breakdown homogenization to the node that holds it, as a cloned tree already did, so updating one clears cached ancestor values and marks ancestors as modified. The bracket that the country template reform appends is linked to its scale too. A child whose name matches a node's own attribute or method (such as `parent`, `children` or `modified`) is no longer set as an attribute, so it can no longer replace the node's own state. Reading a homogenized breakdown position as an attribute, such as `amount.B`, now returns the `ParameterNode` that homogenization built there instead of the placeholder `Parameter` it replaced, so calling it returns a node at that instant and calling `update` on it raises, while lookups at an instant, such as `amount("2021-01-01").B`, are unchanged.
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ def modify_brackets(parameters):
of the YAML parameters. It can be modified and must be returned.
"""
# Access the right parameter node:
brackets = parameters.taxes.social_security_contribution.brackets
scale = parameters.taxes.social_security_contribution
brackets = scale.brackets

# Add 0.1 to the rates of the second bracket, keeping the same thresholds:
for rate in brackets[1].rate.values_list:
Expand All @@ -52,6 +53,9 @@ def modify_brackets(parameters):
"threshold": {"2017-01-01": {"value": 40000}},
},
)
# Link the bracket to its scale, so that updates to it clear cached
# values and mark the scale and its ancestors as modified.
new_bracket.parent = scale
brackets.append(new_bracket)

return parameters
12 changes: 8 additions & 4 deletions policyengine_core/parameters/operations/homogenize_parameters.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ def homogenize_parameter_node(
possible_values = [True, False]
else:
possible_values = evaluate_dynamic_breakdown(first_breakdown)
if not hasattr(node, "children"):
if not isinstance(node, ParameterNode):
node = ParameterNode(
node.name,
data={
Expand Down Expand Up @@ -120,11 +120,15 @@ def homogenize_parameter_node(
f"metadata references the correct variable and that all "
f"parameter keys are valid enum values."
)
for child in node.children:
if further_breakdown:
node.children[child] = homogenize_parameter_node(
if further_breakdown:
for child in list(node.children):
homogenized_child = homogenize_parameter_node(
node.children[child], breakdown[1:], variables, default_value
)
if homogenized_child is not node.children[child]:
# A placeholder Parameter became a ParameterNode: link the
# replacement as add_child would.
node._replace_child(child, homogenized_child)
return node


Expand Down
115 changes: 104 additions & 11 deletions policyengine_core/parameters/parameter_node.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import copy
import functools
import os
import typing
from typing import Iterable, List, Type, Union
Expand All @@ -22,6 +23,56 @@

EXCLUDED_PARAMETER_CHILD_NAMES = ["reference", "__pycache__"]

# Attributes every ParameterNode sets on itself. A child with one of these
# names stays reachable through ``children`` and at-instant lookups, but is not
# set as an attribute, so it cannot replace the node's own state (for example a
# child named ``parent`` replacing the link to the holding node).
NODE_OWN_ATTRIBUTE_NAMES = frozenset(
{
"name",
"children",
"description",
"documentation",
"file_path",
"metadata",
"trace",
"tracer",
"branch_name",
"_at_instant_cache",
"parent",
"modified",
}
)


@functools.lru_cache(maxsize=None)
def _names_blocked_as_child_attributes(node_class: type) -> frozenset:
# The node's own attributes, plus the methods and class attributes of its
# classes (including subclasses such as ParameterScaleBracket). Metaclass
# attributes such as ABCMeta.register or type.mro are not included: they
# are not reachable from an instance. Computed once per class.
names = set(NODE_OWN_ATTRIBUTE_NAMES)
for klass in node_class.__mro__:
names.update(klass.__dict__)
return frozenset(names)


def _can_expose_child_as_attribute(node: "ParameterNode", child_name: str) -> bool:
return child_name not in _names_blocked_as_child_attributes(type(node))


def _check_child_type(child) -> None:
if not (
isinstance(child, ParameterNode)
or isinstance(child, Parameter)
or isinstance(child, parameters.ParameterScale)
):
raise TypeError(
"child must be of type ParameterNode, Parameter, or Scale. Instead got {}".format(
type(child)
)
)


class ParameterNode(AtInstantLike):
"""
Expand Down Expand Up @@ -164,23 +215,64 @@ def add_child(self, name: str, child: Union["ParameterNode", Parameter]):
"""
Add a new child to the node.

:param name: Name of the child that must be used to access that child. Should not contain anything that could interfere with the operator `.` (dot).
:param name: Name of the child that must be used to access that child. Should not contain anything that could interfere with the operator `.` (dot). The child is also set as an attribute of the node, unless the name matches one of the node's own attributes (such as ``parent`` or ``children``) or a method or class attribute of its class; it then stays reachable through ``children`` and at-instant lookups.
:param child: The new child, an instance of :class:`.ParameterScale` or :class:`.Parameter` or :class:`.ParameterNode`.
"""
if name in self.children:
raise ValueError("{} has already a child named {}".format(self.name, name))
if not (
isinstance(child, ParameterNode)
or isinstance(child, Parameter)
or isinstance(child, parameters.ParameterScale)
):
_check_child_type(child)
self._link_child(name, child)

def _replace_child(
self,
name: str,
child: Union["ParameterNode", Parameter, "parameters.ParameterScale"],
) -> None:
"""
Replace an existing child and link the replacement to this node.

Internal: homogenization uses it to replace a placeholder Parameter
with the ParameterNode it builds for a further breakdown level.

Cached values of this node and its ancestors are cleared. The previous
child is left as it is, and nodes are not marked as modified. The tree
is left unchanged when an argument is rejected.

:param name: Name of the existing child.
:param child: The replacement, an instance of :class:`.ParameterScale`, :class:`.Parameter` or :class:`.ParameterNode`. It must not be this node or one of its ancestors.
"""
# Check every argument before changing anything, so a rejected call
# leaves the tree as it was.
if name not in self.children:
raise KeyError(
"{} has no child named {} to replace".format(self.name, name)
)
if not isinstance(name, str):
raise TypeError(
"child must be of type ParameterNode, Parameter, or Scale. Instead got {}".format(
type(child)
)
"Child names must be strings. {} got {!r}".format(self.name, name)
)
_check_child_type(child)
node_or_ancestor = self
while node_or_ancestor is not None:
if node_or_ancestor is child:
raise ValueError(
"{} cannot hold itself or one of its ancestors ({}) as a "
"child".format(self.name, child.name)
)
node_or_ancestor = getattr(node_or_ancestor, "parent", None)

self._link_child(name, child)
# Cached at-instant nodes of this node and its ancestors still hold
# the previous child's values.
self.clear_parent_cache()

def _link_child(self, name: str, child) -> None:
# Store the child, expose it as an attribute when its name does not
# collide with one of the node's own attributes or methods, and link it
# upward so Parameter.update can clear caches and mark ancestors.
self.children[name] = child
setattr(self, name, child)
if _can_expose_child_as_attribute(self, name):
setattr(self, name, child)
child.parent = self

def __repr__(self) -> str:
Expand Down Expand Up @@ -208,7 +300,8 @@ def clone(self) -> "ParameterNode":
clone.metadata = copy.copy(self.metadata)
clone.children = {key: child.clone() for key, child in self.children.items()}
for child_key, child in clone.children.items():
setattr(clone, child_key, child)
if _can_expose_child_as_attribute(clone, child_key):
setattr(clone, child_key, child)
child.parent = clone
clone._at_instant_cache = {}

Expand Down
1 change: 1 addition & 0 deletions policyengine_core/parameters/parameter_scale.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ def __init__(self, name: str, data: dict, file_path: str):
bracket = parameters.ParameterScaleBracket(
name=bracket_name, data=bracket_data, file_path=file_path
)
bracket.parent = self
brackets.append(bracket)
self.brackets: typing.List[parameters.ParameterScaleBracket] = brackets
self.propagate_uprating()
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
"""Nodes that homogenization builds for further breakdown levels are linked.

``homogenize_parameter_node`` replaces a placeholder ``Parameter`` with a
``ParameterNode`` when a breakdown has more than one level. The replacement
must link to the node holding it, and that node's attribute must point at the
replacement rather than the placeholder. Placeholders that homogenization adds
for missing breakdown values must stay linked too.
"""

from policyengine_core.parameters import Parameter, ParameterNode
from policyengine_core.parameters.operations.homogenize_parameters import (
homogenize_parameter_structures,
)


def build_two_level_tree(first_level_values) -> ParameterNode:
return ParameterNode(
data={
"program": {
"amount": {
str(first_level_values[0]): {
"1": {"2020-01-01": 10},
},
"metadata": {
"breakdown": [first_level_values, [1, 2]],
},
}
}
}
)


def test_homogenized_replacements_link_to_their_holder():
root = build_two_level_tree(["A", "B"])
homogenize_parameter_structures(root, variables={})
amount = root.program.amount

for key in ("A", "B"):
replacement = amount.children[key]
assert isinstance(replacement, ParameterNode)
assert replacement.parent is amount
assert getattr(amount, key) is replacement
for leaf in replacement.children.values():
assert leaf.parent is replacement


def test_update_under_homogenized_node_marks_every_ancestor_and_refreshes_cached_value():
root = build_two_level_tree(["A", "B"])
homogenize_parameter_structures(root, variables={})

before = root("2021-01-01").program.amount.B["2"]
assert before == 0

leaf = root.program.amount.children["B"].children["2"]
leaf.update(period="year:2021:1", value=7)

assert root.program.amount.children["B"].modified
assert root.program.amount.modified
assert root.program.modified
assert root.modified
assert root("2021-01-01").program.amount.B["2"] == 7


def test_missing_breakdown_value_named_parent_keeps_the_parent_link():
# "parent" is a missing breakdown value, so homogenization adds it as a
# placeholder Parameter and then replaces it with a ParameterNode.
root = build_two_level_tree(["child", "parent"])
homogenize_parameter_structures(root, variables={})
program = root.children["program"]
amount = program.children["amount"]

assert amount.parent is program
replacement = amount.children["parent"]
assert isinstance(replacement, ParameterNode)
assert replacement.parent is amount

replacement.children["2"].update(period="year:2021:1", value=4)
assert amount.modified
assert program.modified
assert root.modified
assert root("2021-01-01").program.amount["parent"]["2"] == 4


def test_single_level_homogenization_links_added_placeholders():
root = ParameterNode(
data={
"program": {
"A": {"2020-01-01": 1},
"metadata": {"breakdown": [["A", "B"]]},
}
}
)
homogenize_parameter_structures(root, variables={})
program = root.program

placeholder = program.children["B"]
assert isinstance(placeholder, Parameter)
assert placeholder.parent is program
assert program.B is placeholder
Loading
Loading