Skip to content
Open
1 change: 1 addition & 0 deletions changelog.d/dependency-map.added.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
`policyengine-core dependency-map -c <country package>` traces which variables read each parameter and which variables feed which, over the package's YAML tests and optionally its microdata, and writes the map with the package's version and a fingerprint of the model and traced tests.
1 change: 1 addition & 0 deletions changelog.d/tracer-parameter-reads.fixed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
`FullTracer` now records tax-scale reads with their schedule, reads of a parameter node's children and look-ups by name that find no child on a node at an instant, parameters read before an instant is chosen (`parameters.gov.rate(period)`), and the parameters core reads for `adds`, `subtracts` and uprating; each trace node also records whether its formula ran. Core's abolition check no longer appears in a trace when a country package traces its own parameter root.
7 changes: 7 additions & 0 deletions docs/python_api/tools.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,10 @@ The `policyengine_core.tools` module contains miscellaneous utility functions, i
```{eval-rst}
.. autofunction:: policyengine_core.tools.simulation_dumper.restore_simulation
```

## dependency_map

```{eval-rst}
.. automodule:: policyengine_core.tools.dependency_map
:members: build_dependency_map, trace_yaml_tests, trace_microdata, model_fingerprint
```
8 changes: 8 additions & 0 deletions docs/python_api/tracers.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,3 +73,11 @@ The `policyengine_core.tracers` module contains classes used to represent tracer
:inherited-members:
:show-inheritance:
```

## TracingChildren

```{eval-rst}
.. autoclass:: policyengine_core.tracers.tracing_parameter_node_at_instant.TracingChildren
:members:
:show-inheritance:
```
36 changes: 36 additions & 0 deletions policyengine_core/scripts/policyengine_command.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,12 +105,43 @@ def build_data_parser(parser):

return parser

def build_dependency_map_parser(parser):
parser = add_tax_benefit_system_arguments(parser, country_only=True)
parser.add_argument(
"--population",
choices=["tests", "microdata", "both"],
default="tests",
help="what to trace: the package's YAML tests (default), a microdata subsample, or both",
)
parser.add_argument(
"--tests-root",
default=None,
help="directory of YAML tests to trace (default: <package>/tests)",
)
parser.add_argument(
"--every-test",
action="store_true",
help="trace every test, including tests whose output formulas have already run earlier in their file",
)
parser.add_argument(
"--households", type=int, default=2000, help="microdata subsample size"
)
parser.add_argument("--year", type=int, default=2026, help="microdata year")
parser.add_argument("--output", default="dependency-map.json")
return parser

parser_test = subparsers.add_parser("test", help="Run OpenFisca YAML tests")
parser_test = build_test_parser(parser_test)

parser_data = subparsers.add_parser("data", help="Manage OpenFisca data")
parser_data = build_data_parser(parser_data)

parser_dependency_map = subparsers.add_parser(
"dependency-map",
help="Trace which variables read each parameter and which variables feed which",
)
parser_dependency_map = build_dependency_map_parser(parser_dependency_map)

return parser


Expand All @@ -129,6 +160,11 @@ def main():

return sys.exit(main(parser))

if args.command == "dependency-map":
from policyengine_core.tools.dependency_map import main

return sys.exit(main(parser))


if __name__ == "__main__":
sys.exit(main())
58 changes: 54 additions & 4 deletions policyengine_core/simulations/simulation.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,14 @@
from policyengine_core.enums import Enum, EnumArray
from policyengine_core.errors import CycleError, SpiralError
from policyengine_core.holders.holder import Holder
from policyengine_core.periods import Period
from policyengine_core.periods import Instant, Period
from policyengine_core.periods.config import ETERNITY, MONTH, YEAR
from policyengine_core.periods.helpers import period
from policyengine_core.tracers import (
FullTracer,
SimpleTracer,
TracingParameterNode,
TracingParameterNodeAtInstant,
)
import random
from policyengine_core.tools.hugging_face import *
Expand Down Expand Up @@ -686,6 +687,18 @@ class PreservedUserInput:
disk_enum: object = None


def _untraced(parameters_at_instant: object) -> object:
"""The parameter tree at an instant, without a tracing view around it.

A country package can trace its own parameter root, so reading it gives a
tracing view. Core's own checks read the tree through this, so that they
are not recorded as reads of the variable's formula.
"""
if isinstance(parameters_at_instant, TracingParameterNodeAtInstant):
return parameters_at_instant.parameter_node_at_instant
return parameters_at_instant


class Simulation:
"""
Represents a simulation, and handles the calculation logic
Expand Down Expand Up @@ -1705,7 +1718,7 @@ def _calculate(self, variable_name: str, period: Period = None) -> ArrayLike:
try:
if (
variable.is_neutralized
or self.tax_benefit_system.parameters(period).gov.abolitions[
or _untraced(self.tax_benefit_system.parameters(period)).gov.abolitions[
variable.name
]
):
Expand Down Expand Up @@ -1925,6 +1938,14 @@ def _calculate(self, variable_name: str, period: Period = None) -> ArrayLike:
value_in_this_period = _uprating_index_value(
uprating_parameter, period.start
)
self._record_parameter_read(
uprating_parameter.name,
latest_known_period.start,
value_in_last_period,
)
self._record_parameter_read(
uprating_parameter.name, period.start, value_in_this_period
)
if (
value_in_last_period is None
or value_in_this_period is None
Expand Down Expand Up @@ -2365,6 +2386,19 @@ def calculate_output(self, variable_name: str, period: Period = None) -> ArrayLi

return variable.calculate_output(self, variable_name, period)

def _record_parameter_read(
self, name: str, instant: Instant, value: object
) -> None:
"""Record a parameter core read for the variable being calculated.

``adds``, ``subtracts`` and uprating read the parameter tree directly,
not through the view a formula receives.
"""
if self.trace:
self.tracer.record_parameter_access(
name, str(instant), self.branch_name, value
)

def _run_formula(
self, variable: str, population: Population, period: Period
) -> ArrayLike:
Expand All @@ -2373,6 +2407,8 @@ def _run_formula(
"""

formula = variable.get_formula(period)
if self.trace and (formula is not None or variable.adds or variable.subtracts):
self.tracer.record_formula_run()
if formula is None:
values = None
if variable.adds is not None and len(variable.adds) > 0:
Expand All @@ -2387,6 +2423,9 @@ def _run_formula(
f"In the variable '{variable.name}', the 'adds' attribute is a string '{variable.adds}' that does not match any parameter."
)
adds_list = adds_parameter(period.start)
self._record_parameter_read(
adds_parameter.name, period.start, adds_list
)
else:
adds_list = variable.adds
values = 0
Expand All @@ -2401,7 +2440,11 @@ def _run_formula(
self.tax_benefit_system.parameters,
added_variable,
)
values = values + parameter(period.start)
added_value = parameter(period.start)
self._record_parameter_read(
parameter.name, period.start, added_value
)
values = values + added_value
except:
raise ValueError(
f"In the variable '{variable.name}', the 'adds' attribute is a list that contains a string '{added_variable}' that does not match any variable or parameter."
Expand All @@ -2418,6 +2461,9 @@ def _run_formula(
f"In the variable '{variable.name}', the 'subtracts' attribute is a string '{variable.subtracts}' that does not match any parameter."
)
subtracts_list = subtracts_parameter(period.start)
self._record_parameter_read(
subtracts_parameter.name, period.start, subtracts_list
)
else:
subtracts_list = variable.subtracts
if values is None:
Expand All @@ -2435,7 +2481,11 @@ def _run_formula(
self.tax_benefit_system.parameters,
subtracted_variable,
)
values = values - parameter(period.start)
subtracted_value = parameter(period.start)
self._record_parameter_read(
parameter.name, period.start, subtracted_value
)
values = values - subtracted_value
except:
raise ValueError(
f"In the variable '{variable.name}', the 'subtracts' attribute is a list that contains a string '{subtracted_variable}' that does not match any variable or parameter."
Expand Down
Loading