diff --git a/changelog.d/dependency-map.added.md b/changelog.d/dependency-map.added.md new file mode 100644 index 000000000..a98a283e0 --- /dev/null +++ b/changelog.d/dependency-map.added.md @@ -0,0 +1 @@ +`policyengine-core dependency-map -c ` traces which variables read each parameter and which variables feed which, over the package's YAML tests and optionally its microdata, and writes the map with the package's version and a fingerprint of the model and traced tests. diff --git a/changelog.d/tracer-parameter-reads.fixed.md b/changelog.d/tracer-parameter-reads.fixed.md new file mode 100644 index 000000000..8875b026a --- /dev/null +++ b/changelog.d/tracer-parameter-reads.fixed.md @@ -0,0 +1 @@ +`FullTracer` now records tax-scale reads with their schedule, reads of a parameter node's children and look-ups by name that find no child on a node at an instant, parameters read before an instant is chosen (`parameters.gov.rate(period)`), and the parameters core reads for `adds`, `subtracts` and uprating; each trace node also records whether its formula ran. Core's abolition check no longer appears in a trace when a country package traces its own parameter root. diff --git a/docs/python_api/tools.md b/docs/python_api/tools.md index 4c8473ab3..c5e092dc8 100644 --- a/docs/python_api/tools.md +++ b/docs/python_api/tools.md @@ -19,3 +19,10 @@ The `policyengine_core.tools` module contains miscellaneous utility functions, i ```{eval-rst} .. autofunction:: policyengine_core.tools.simulation_dumper.restore_simulation ``` + +## dependency_map + +```{eval-rst} +.. automodule:: policyengine_core.tools.dependency_map + :members: build_dependency_map, trace_yaml_tests, trace_microdata, model_fingerprint +``` diff --git a/docs/python_api/tracers.md b/docs/python_api/tracers.md index 6c72e330a..1457ee4ba 100644 --- a/docs/python_api/tracers.md +++ b/docs/python_api/tracers.md @@ -73,3 +73,11 @@ The `policyengine_core.tracers` module contains classes used to represent tracer :inherited-members: :show-inheritance: ``` + +## TracingChildren + +```{eval-rst} +.. autoclass:: policyengine_core.tracers.tracing_parameter_node_at_instant.TracingChildren + :members: + :show-inheritance: +``` diff --git a/policyengine_core/scripts/policyengine_command.py b/policyengine_core/scripts/policyengine_command.py index 7aa85c888..70134c1fb 100644 --- a/policyengine_core/scripts/policyengine_command.py +++ b/policyengine_core/scripts/policyengine_command.py @@ -105,12 +105,43 @@ def build_data_parser(parser): return parser + def build_dependency_map_parser(parser): + parser = add_tax_benefit_system_arguments(parser, country_only=True) + parser.add_argument( + "--population", + choices=["tests", "microdata", "both"], + default="tests", + help="what to trace: the package's YAML tests (default), a microdata subsample, or both", + ) + parser.add_argument( + "--tests-root", + default=None, + help="directory of YAML tests to trace (default: /tests)", + ) + parser.add_argument( + "--every-test", + action="store_true", + help="trace every test, including tests whose output formulas have already run earlier in their file", + ) + parser.add_argument( + "--households", type=int, default=2000, help="microdata subsample size" + ) + parser.add_argument("--year", type=int, default=2026, help="microdata year") + parser.add_argument("--output", default="dependency-map.json") + return parser + parser_test = subparsers.add_parser("test", help="Run OpenFisca YAML tests") parser_test = build_test_parser(parser_test) parser_data = subparsers.add_parser("data", help="Manage OpenFisca data") parser_data = build_data_parser(parser_data) + parser_dependency_map = subparsers.add_parser( + "dependency-map", + help="Trace which variables read each parameter and which variables feed which", + ) + parser_dependency_map = build_dependency_map_parser(parser_dependency_map) + return parser @@ -129,6 +160,11 @@ def main(): return sys.exit(main(parser)) + if args.command == "dependency-map": + from policyengine_core.tools.dependency_map import main + + return sys.exit(main(parser)) + if __name__ == "__main__": sys.exit(main()) diff --git a/policyengine_core/simulations/simulation.py b/policyengine_core/simulations/simulation.py index 297e7a778..f3e0e72f7 100644 --- a/policyengine_core/simulations/simulation.py +++ b/policyengine_core/simulations/simulation.py @@ -27,13 +27,14 @@ from policyengine_core.enums import Enum, EnumArray from policyengine_core.errors import CycleError, SpiralError from policyengine_core.holders.holder import Holder -from policyengine_core.periods import Period +from policyengine_core.periods import Instant, Period from policyengine_core.periods.config import ETERNITY, MONTH, YEAR from policyengine_core.periods.helpers import period from policyengine_core.tracers import ( FullTracer, SimpleTracer, TracingParameterNode, + TracingParameterNodeAtInstant, ) import random from policyengine_core.tools.hugging_face import * @@ -686,6 +687,18 @@ class PreservedUserInput: disk_enum: object = None +def _untraced(parameters_at_instant: object) -> object: + """The parameter tree at an instant, without a tracing view around it. + + A country package can trace its own parameter root, so reading it gives a + tracing view. Core's own checks read the tree through this, so that they + are not recorded as reads of the variable's formula. + """ + if isinstance(parameters_at_instant, TracingParameterNodeAtInstant): + return parameters_at_instant.parameter_node_at_instant + return parameters_at_instant + + class Simulation: """ Represents a simulation, and handles the calculation logic @@ -1705,7 +1718,7 @@ def _calculate(self, variable_name: str, period: Period = None) -> ArrayLike: try: if ( variable.is_neutralized - or self.tax_benefit_system.parameters(period).gov.abolitions[ + or _untraced(self.tax_benefit_system.parameters(period)).gov.abolitions[ variable.name ] ): @@ -1925,6 +1938,14 @@ def _calculate(self, variable_name: str, period: Period = None) -> ArrayLike: value_in_this_period = _uprating_index_value( uprating_parameter, period.start ) + self._record_parameter_read( + uprating_parameter.name, + latest_known_period.start, + value_in_last_period, + ) + self._record_parameter_read( + uprating_parameter.name, period.start, value_in_this_period + ) if ( value_in_last_period is None or value_in_this_period is None @@ -2365,6 +2386,19 @@ def calculate_output(self, variable_name: str, period: Period = None) -> ArrayLi return variable.calculate_output(self, variable_name, period) + def _record_parameter_read( + self, name: str, instant: Instant, value: object + ) -> None: + """Record a parameter core read for the variable being calculated. + + ``adds``, ``subtracts`` and uprating read the parameter tree directly, + not through the view a formula receives. + """ + if self.trace: + self.tracer.record_parameter_access( + name, str(instant), self.branch_name, value + ) + def _run_formula( self, variable: str, population: Population, period: Period ) -> ArrayLike: @@ -2373,6 +2407,8 @@ def _run_formula( """ formula = variable.get_formula(period) + if self.trace and (formula is not None or variable.adds or variable.subtracts): + self.tracer.record_formula_run() if formula is None: values = None if variable.adds is not None and len(variable.adds) > 0: @@ -2387,6 +2423,9 @@ def _run_formula( f"In the variable '{variable.name}', the 'adds' attribute is a string '{variable.adds}' that does not match any parameter." ) adds_list = adds_parameter(period.start) + self._record_parameter_read( + adds_parameter.name, period.start, adds_list + ) else: adds_list = variable.adds values = 0 @@ -2401,7 +2440,11 @@ def _run_formula( self.tax_benefit_system.parameters, added_variable, ) - values = values + parameter(period.start) + added_value = parameter(period.start) + self._record_parameter_read( + parameter.name, period.start, added_value + ) + values = values + added_value except: raise ValueError( f"In the variable '{variable.name}', the 'adds' attribute is a list that contains a string '{added_variable}' that does not match any variable or parameter." @@ -2418,6 +2461,9 @@ def _run_formula( f"In the variable '{variable.name}', the 'subtracts' attribute is a string '{variable.subtracts}' that does not match any parameter." ) subtracts_list = subtracts_parameter(period.start) + self._record_parameter_read( + subtracts_parameter.name, period.start, subtracts_list + ) else: subtracts_list = variable.subtracts if values is None: @@ -2435,7 +2481,11 @@ def _run_formula( self.tax_benefit_system.parameters, subtracted_variable, ) - values = values - parameter(period.start) + subtracted_value = parameter(period.start) + self._record_parameter_read( + parameter.name, period.start, subtracted_value + ) + values = values - subtracted_value except: raise ValueError( f"In the variable '{variable.name}', the 'subtracts' attribute is a list that contains a string '{subtracted_variable}' that does not match any variable or parameter." diff --git a/policyengine_core/tools/dependency_map.py b/policyengine_core/tools/dependency_map.py new file mode 100644 index 000000000..14808a2f5 --- /dev/null +++ b/policyengine_core/tools/dependency_map.py @@ -0,0 +1,543 @@ +"""Trace the parameter → variable dependency map of a country package. + +Downstream tools need to know which variables a parameter path moves and +which variables feed which: validation matching against calibration targets +and scorecards, model-coverage audits, reform classifiers. Static scans of +formula source miss bracket, scale, and vectorised reads; the only exact +record is what the model reads at run time. This module runs simulations +under ``FullTracer`` and folds the trace trees into two edge sets: + + readers[parameter_path] -> variables whose formula read that parameter + consumers[variable] -> variables whose formula read that variable + +Parameter paths are recorded at the node the formula indexed, so a bracket +read ``p.base.calc(age)`` is recorded as ``gov.irs.credits.ctc.amount.base``. +A read of which children a node has, or of a parameter or node outside a dated +call (its value history, for example), is recorded at that node's own path and +stands for everything below it; the empty path is the whole tree. A look-up +by name that finds no child on a node at an instant is recorded at the missing +child's path. + +Core's check of each variable's own abolition switch, +``gov.abolitions.``, is not recorded, so it is not in the map. A +formula's read of a switch is, like any other read. + +Populations +----------- +``tests`` builds a simulation for every YAML test under the package's + tests directory (skipping tests that apply reforms, + extensions, or inline parameter changes) and calculates the + test's outputs. Deterministic, no data download, and country + test suites deliberately exercise every program. +``microdata`` calculates every variable over a subsample of the package's + default microdata. Broad, but a formula behind ``defined_for`` + only runs when someone in the sample qualifies. +``both`` the union. + +Tests in one file mostly vary inputs for the same outputs and record the same +edges. By default a test is skipped when, earlier in its file, the formula +each of its outputs uses at the test's period has already run. A formula that +did not run, because ``defined_for`` was false for everyone or the value was +an input, does not count, so a later test of that output is still traced. +``every_test`` traces them all; it can still find more edges, because a +skipped test can reach formulas and conditional reads below its outputs that +earlier tests did not. + +A test's ``output`` block is read the way the YAML runner reads it: a key is a +variable, an entity's singular key holding variables, or an entity's plural +key holding instances of variables, and a dict value keyed by period checks +the variable at each period. + +Output +------ +The map is one JSON object:: + + generatedAt ISO-8601 UTC timestamp + model package, version, fingerprint, coreVersion. The + fingerprint is a sha256 over the package's files and, when + tests were traced, the traced test files. Microdata is + not hashed. + populations per population traced, its selection and counts: + tests: root, everyTest, tests (situations built), skipped + (tests not traced because their outputs were covered), + excluded (tests with reforms, extensions or inline + parameter changes), failed (situations that could not be + built, and test files or entries that could not be read), + outputs + (variables calculated), outputErrors (output + calculations that raised; their edges are whatever + ran before the error); microdata: households, year, + variables (calculated), failed (raised at every period) + tracingSeconds wall time + readers parameter path -> sorted variables that read it + consumers variable -> sorted variables whose formula read it + +The map records what ran. A test that could not be built contributes no +edges, and an output that raised contributes only the edges recorded before +it raised, so ``failed`` and ``outputErrors`` bound how incomplete a map may +be; the command prints both, and a map with non-zero counts should be read +as a lower bound on the model's dependencies. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib +import json +import sys +import time +from collections import defaultdict +from collections.abc import Callable, Iterable, Iterator +from datetime import datetime, timezone +from importlib import metadata +from pathlib import Path +from typing import TYPE_CHECKING, Any, Optional + +from policyengine_core.periods import ETERNITY, MONTH +from policyengine_core.simulations import Simulation, SimulationBuilder +from policyengine_core.tools.test_runner import import_yaml +from policyengine_core.tracers import FullTracer, TraceNode + +if TYPE_CHECKING: + from policyengine_core.taxbenefitsystems import TaxBenefitSystem + +yaml, YamlLoader = import_yaml() + +Edges = tuple[dict[str, set[str]], dict[str, set[str]]] +Progress = Callable[[str], None] +YamlTest = dict[str, Any] +# A period as a YAML test writes it: a string, a year or a date. +YamlPeriod = Any +# A variable and the formula it uses at some period (None if it has none). +FormulaKey = tuple[str, Optional[Callable]] + +DEFAULT_OUTPUT = Path("dependency-map.json") +YAML_SUFFIXES = (".yaml", ".yml") +COMPILED_SUFFIXES = (".pyc", ".pyo") + + +class _QuietFullTracer(FullTracer): + """Keeps the trace's structure and drops the values, which nothing reads.""" + + def record_calculation_result(self, value: object) -> None: + pass + + def record_parameter_access( + self, parameter: str, period: str, branch_name: str, value: object + ) -> None: + super().record_parameter_access(parameter, period, branch_name, None) + + +def _enable_tracing(simulation: Simulation) -> None: + simulation.trace = True + # A country package can hand the new tracer to its own parameter root when + # tracing is switched on. Replacing the tracer would leave the root + # recording into the old one, so the same tracer drops values instead. + if type(simulation.tracer) is FullTracer: + simulation.tracer.__class__ = _QuietFullTracer + + +def _trace_nodes(simulation: Simulation) -> Iterator[TraceNode]: + pending_nodes = list(simulation.tracer.trees) + while pending_nodes: + node = pending_nodes.pop() + yield node + pending_nodes.extend(node.children) + + +def collect_edges( + simulation: Simulation, + readers: Optional[dict[str, set[str]]] = None, + consumers: Optional[dict[str, set[str]]] = None, +) -> Edges: + """Fold a traced simulation's trees into the two edge sets.""" + readers = {} if readers is None else readers + consumers = {} if consumers is None else consumers + for node in _trace_nodes(simulation): + for parameter in node.parameters: + readers.setdefault(parameter.name, set()).add(node.name) + for child in node.children: + if child.name != node.name: + consumers.setdefault(child.name, set()).add(node.name) + return readers, consumers + + +def iter_output_variables( + output: object, system: Optional[TaxBenefitSystem] = None +) -> Iterator[tuple[str, Optional[YamlPeriod]]]: + """Yield ``(variable, period)`` for each variable a test's output checks. + + Mirrors ``YamlItem.check_output``: a key is a variable, an entity's + singular key holding variables, or an entity's plural key holding + instances of variables. A dict value keyed by period checks the variable + at each of those periods; ``period`` is ``None`` for the test's own + period. Without a ``system`` every key is taken to be a variable. + """ + if not isinstance(output, dict): + return + singular_keys = set() if system is None else {e.key for e in system.entities} + plural_keys = set() if system is None else {e.plural for e in system.entities} + + def variable_periods(name: str, expected: object, period: object = None): + if isinstance(expected, dict): + for nested_period, nested_expected in expected.items(): + yield from variable_periods(name, nested_expected, nested_period) + else: + yield name, period + + for key, value in output.items(): + if system is None or system.get_variable(key) is not None: + yield from variable_periods(key, value) + elif key in singular_keys and isinstance(value, dict): + for name, expected in value.items(): + yield from variable_periods(name, expected) + elif key in plural_keys and isinstance(value, dict): + for instance in value.values(): + if isinstance(instance, dict): + for name, expected in instance.items(): + yield from variable_periods(name, expected) + # Any other key fails the YAML runner as an unknown variable. + + +def _yaml_files(path: Path) -> list[Path]: + if not path.is_dir(): + return [path] + return sorted( + file + for file in path.rglob("*") + if file.suffix in YAML_SUFFIXES and file.is_file() + ) + + +def _load_yaml_tests(file: Path) -> list[object]: + """The tests of a YAML file; like the runner, a lone mapping is one test.""" + tests = yaml.load(file.read_text(), Loader=YamlLoader) + if tests is None: + return [] + return tests if isinstance(tests, list) else [tests] + + +def _runs_on_baseline(test: YamlTest) -> bool: + if test.get("reforms") or test.get("extensions"): + return False + inputs = test.get("input") + if not isinstance(inputs, dict): + return True # building the situation will report an invalid input + # A dotted key is an inline parameter change. + return not any(isinstance(key, str) and "." in key for key in inputs) + + +def iter_yaml_tests( + paths: Iterable[Path], + system: Optional[TaxBenefitSystem] = None, + stats: Optional[dict[str, int]] = None, +) -> Iterator[tuple[Path, YamlTest]]: + """Yield (file, test) for the tests that run on the baseline system. + + With ``stats``, adds to its ``excluded`` count the tests that change the + system, and to ``failed`` the files and entries that cannot be read. + """ + counts = {"excluded": 0, "failed": 0} if stats is None else stats + for path in paths: + for file in _yaml_files(path): + try: + tests = _load_yaml_tests(file) + except yaml.YAMLError: + counts["failed"] += 1 + continue + for test in tests: + if not isinstance(test, dict): + counts["failed"] += 1 + elif test.get("output") and not _runs_on_baseline(test): + counts["excluded"] += 1 + elif any( + True for _ in iter_output_variables(test.get("output"), system) + ): + yield file, test + + +def _formula_key( + system: TaxBenefitSystem, variable_name: str, period: object +) -> FormulaKey: + variable = system.get_variable(variable_name) + if variable is None: + return variable_name, None + try: + return variable_name, variable.get_formula(period) + except ValueError: # not a period: calculating this output will report it + return variable_name, None + + +def _output_formulas(system: TaxBenefitSystem, test: YamlTest) -> set[FormulaKey]: + return { + _formula_key(system, name, output_period or test.get("period")) + for name, output_period in iter_output_variables(test["output"], system) + } + + +def _formulas_run(system: TaxBenefitSystem, simulation: Simulation) -> set[FormulaKey]: + return { + _formula_key(system, node.name, node.period) + for node in _trace_nodes(simulation) + if node.formula_ran + } + + +def trace_yaml_tests( + system: TaxBenefitSystem, + paths: Iterable[Path], + progress: Optional[Progress] = None, + every_test: bool = False, +) -> tuple[Edges, dict[str, int]]: + """Trace the baseline YAML tests under ``paths`` into edge sets. + + Unless ``every_test`` is set, a test is skipped once the formulas of all + its outputs have run in an earlier test of the same file. + """ + readers: dict[str, set[str]] = {} + consumers: dict[str, set[str]] = {} + stats = { + "tests": 0, + "skipped": 0, + "excluded": 0, + "failed": 0, + "outputs": 0, + "outputErrors": 0, + } + current_file: Optional[Path] = None + covered_formulas: set[FormulaKey] = set() + for index, (file, test) in enumerate(iter_yaml_tests(paths, system, stats)): + if file != current_file: + current_file = file + covered_formulas = set() + output_formulas = _output_formulas(system, test) + if not every_test and output_formulas <= covered_formulas: + stats["skipped"] += 1 + continue + period = test.get("period") + try: + builder = SimulationBuilder() + builder.set_default_period(period) + simulation = builder.build_from_dict(system, test.get("input") or {}) + simulation.default_calculation_period = builder.default_period + _enable_tracing(simulation) + calculated_formulas: set[FormulaKey] = set() + for name, output_period in iter_output_variables(test["output"], system): + calculation_period = output_period or period + try: + simulation.calculate(name, calculation_period) + stats["outputs"] += 1 + calculated_formulas.add( + _formula_key(system, name, calculation_period) + ) + except Exception: # what ran before the error is still traced + stats["outputErrors"] += 1 + collect_edges(simulation, readers, consumers) + covered_formulas |= calculated_formulas & _formulas_run(system, simulation) + stats["tests"] += 1 + except Exception: # unbuildable situation: skip it + stats["failed"] += 1 + if progress and index % 500 == 0: + progress(f" {index} tests traced ({file.name})") + return (readers, consumers), stats + + +def trace_microdata( + microsimulation_class: Callable[[], Simulation], + households: int = 2000, + year: int = 2026, + progress: Optional[Progress] = None, +) -> tuple[Edges, dict[str, int]]: + simulation = microsimulation_class() + simulation = simulation.subsample(n=households, seed=0) or simulation + _enable_tracing(simulation) + variables = simulation.tax_benefit_system.variables + stats = {"variables": 0, "failed": 0} + for index, name in enumerate(sorted(variables)): + variable = variables[name] + if variable.definition_period == ETERNITY: + periods = [ETERNITY] + elif variable.definition_period == MONTH: + periods = [f"{year}-01"] + else: + periods = [year, f"{year}-01"] + for period in periods: + try: + simulation.calculate(name, period) + stats["variables"] += 1 + break + except Exception: # any formula failure just skips the variable + continue + else: + stats["failed"] += 1 + if progress and index % 500 == 0: + progress(f" {index}/{len(variables)} variables") + return collect_edges(simulation), stats + + +def merge_edges(*edge_sets: Edges) -> Edges: + readers: dict[str, set[str]] = defaultdict(set) + consumers: dict[str, set[str]] = defaultdict(set) + for edge_readers, edge_consumers in edge_sets: + for path, names in edge_readers.items(): + readers[path] |= names + for name, users in edge_consumers.items(): + consumers[name] |= users + return readers, consumers + + +def _hashable_files(root: Path) -> list[Path]: + return sorted( + file + for file in root.rglob("*") + if file.is_file() + and "__pycache__" not in file.parts + and file.suffix not in COMPILED_SUFFIXES + and not any(part.startswith(".") for part in file.relative_to(root).parts) + ) + + +def _hash_files(digest: Any, root: Path, files: Iterable[Path], label: str) -> None: + for file in files: + digest.update(f"{label}/{file.relative_to(root).as_posix()}".encode()) + digest.update(b"\0") + digest.update(file.read_bytes()) + digest.update(b"\0") + + +def model_fingerprint(package_root: Path, tests_root: Optional[Path] = None) -> str: + """sha256 over what the map depends on, so a consumer can tell whether a + map still matches the model it runs against. + + It covers every file of the package except its ``tests`` directory, since + formulas import helpers from anywhere in the package, and the files under + ``tests_root``, which decide what the map traced. + """ + package_tests = package_root / "tests" + digest = hashlib.sha256() + _hash_files( + digest, + package_root, + [ + file + for file in _hashable_files(package_root) + if not file.is_relative_to(package_tests) + ], + "package", + ) + if tests_root is not None: + files = _hashable_files(tests_root) if tests_root.is_dir() else [tests_root] + root = tests_root if tests_root.is_dir() else tests_root.parent + _hash_files(digest, root, files, "tests") + return f"sha256:{digest.hexdigest()}" + + +def _package_version(package_name: str) -> Optional[str]: + for distribution in (package_name, package_name.replace("_", "-")): + try: + return metadata.version(distribution) + except metadata.PackageNotFoundError: + continue + return None + + +def build_dependency_map( + country_package_name: str, + population: str = "tests", + tests_root: Optional[Path] = None, + every_test: bool = False, + households: int = 2000, + year: int = 2026, + progress: Optional[Progress] = None, +) -> dict: + package = importlib.import_module(country_package_name) + package_root = Path(package.__file__).resolve().parent + tests_root = tests_root or package_root / "tests" + + edge_sets: list[Edges] = [] + populations: dict[str, dict] = {} + started = time.time() + if population in ("tests", "both"): + edges, stats = trace_yaml_tests( + package.CountryTaxBenefitSystem(), [tests_root], progress, every_test + ) + edge_sets.append(edges) + populations["tests"] = { + "root": str(tests_root), + "everyTest": every_test, + **stats, + } + if population in ("microdata", "both"): + edges, stats = trace_microdata( + package.Microsimulation, households, year, progress + ) + edge_sets.append(edges) + populations["microdata"] = {"households": households, "year": year, **stats} + if not edge_sets: + raise ValueError(f"unknown population {population!r}") + + readers, consumers = merge_edges(*edge_sets) + return { + "generatedAt": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "model": { + "package": country_package_name, + "version": _package_version(country_package_name), + "fingerprint": model_fingerprint( + package_root, tests_root if "tests" in populations else None + ), + "coreVersion": metadata.version("policyengine-core"), + }, + "populations": populations, + "tracingSeconds": round(time.time() - started), + "readers": {path: sorted(names) for path, names in sorted(readers.items())}, + "consumers": {name: sorted(users) for name, users in sorted(consumers.items())}, + } + + +def write_dependency_map(payload: dict, output: Path = DEFAULT_OUTPUT) -> Path: + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(payload, separators=(",", ":")) + "\n") + return output + + +def main(parser: argparse.ArgumentParser) -> int: + args = parser.parse_args() + if args.tests_root and not Path(args.tests_root).exists(): + parser.error(f"--tests-root does not exist: {args.tests_root}") + + def progress(message: str) -> None: + print(message, file=sys.stderr, flush=True) + + from policyengine_core.scripts import detect_country_package + + country_package_name = args.country_package or detect_country_package() + payload = build_dependency_map( + country_package_name, + population=args.population, + tests_root=Path(args.tests_root) if args.tests_root else None, + every_test=args.every_test, + households=args.households, + year=args.year, + progress=progress, + ) + output = write_dependency_map(payload, Path(args.output)) + for name, counts in payload["populations"].items(): + summary = ", ".join( + f"{key}: {value}" + for key, value in counts.items() + if key not in ("root", "everyTest") + ) + progress(f"{name}: {summary}") + if counts.get("skipped"): + progress( + f"{counts['skipped']} tests were skipped because their outputs' " + "formulas had already run in the same file. They can reach other " + "formulas, so use --every-test for the complete map." + ) + progress( + f"wrote {output}: {len(payload['readers'])} parameter paths, " + f"{len(payload['consumers'])} consumed variables, " + f"{payload['tracingSeconds']}s" + ) + return 0 diff --git a/policyengine_core/tracers/__init__.py b/policyengine_core/tracers/__init__.py index c3df2b66a..f23c0bbcd 100644 --- a/policyengine_core/tracers/__init__.py +++ b/policyengine_core/tracers/__init__.py @@ -6,6 +6,8 @@ from .simple_tracer import SimpleTracer from .trace_node import TraceNode from .tracing_parameter_node_at_instant import ( + TracingChildren, TracingParameterNode, TracingParameterNodeAtInstant, + describe_tax_scale, ) diff --git a/policyengine_core/tracers/full_tracer.py b/policyengine_core/tracers/full_tracer.py index 3cab802ec..f2d93e7f4 100644 --- a/policyengine_core/tracers/full_tracer.py +++ b/policyengine_core/tracers/full_tracer.py @@ -72,6 +72,10 @@ def record_parameter_access( ), ) + def record_formula_run(self) -> None: + if self._current_node is not None: + self._current_node.formula_ran = True + def _record_start_time( self, time_in_s: Optional[float] = None, diff --git a/policyengine_core/tracers/simple_tracer.py b/policyengine_core/tracers/simple_tracer.py index d503ec5b7..9838491be 100644 --- a/policyengine_core/tracers/simple_tracer.py +++ b/policyengine_core/tracers/simple_tracer.py @@ -30,6 +30,9 @@ def record_calculation_result(self, value: ArrayLike) -> None: def record_parameter_access(self, parameter: str, period, branch_name: str, value): pass + def record_formula_run(self) -> None: + pass + def record_calculation_end(self) -> None: self.stack.pop() diff --git a/policyengine_core/tracers/trace_node.py b/policyengine_core/tracers/trace_node.py index 501648774..53679efab 100644 --- a/policyengine_core/tracers/trace_node.py +++ b/policyengine_core/tracers/trace_node.py @@ -24,6 +24,10 @@ class TraceNode: value: typing.Optional[Array] = None start: float = 0 end: float = 0 + # Whether this calculation ran the variable's formula, or its ``adds`` and + # ``subtracts``. It stays False for a value that was stored, cached or + # defaulted, including one masked by ``defined_for``. + formula_ran: bool = False def calculation_time(self, round_: bool = True) -> Time: result = self.end - self.start diff --git a/policyengine_core/tracers/tracing_parameter_node_at_instant.py b/policyengine_core/tracers/tracing_parameter_node_at_instant.py index 7b2e46db1..b44acc3bf 100644 --- a/policyengine_core/tracers/tracing_parameter_node_at_instant.py +++ b/policyengine_core/tracers/tracing_parameter_node_at_instant.py @@ -1,11 +1,13 @@ from __future__ import annotations import typing -from typing import Iterator, Union +from collections.abc import Iterator, Mapping +from typing import Optional, Union import numpy -from policyengine_core import parameters +from policyengine_core import parameters, periods +from policyengine_core.taxscales import TaxScaleLike from .. import tracers @@ -64,12 +66,61 @@ def _wrapped(wrapper: object, attribute: str, key: str) -> object: raise AttributeError(key) from None +# Attributes of an undated parameter that describe it and hold no values. +_UNDATED_LABELS = frozenset({"name", "description", "documentation", "file_path"}) + + +def describe_tax_scale(scale: TaxScaleLike) -> dict[str, object]: + """Describe a tax scale in plain values a trace can serialize as JSON. + + The description holds the scale's class name and its thresholds, and its + rates or amounts when it has them. + """ + description: dict[str, object] = {"type": type(scale).__name__} + for attribute in ("thresholds", "rates", "amounts"): + values = getattr(scale, attribute, None) + if values is not None: + description[attribute] = [_plain_number(value) for value in values] + return description + + +def _plain_number(value: object) -> object: + return value.item() if isinstance(value, numpy.generic) else value + + +def _record_value_read( + tracer: tracers.FullTracer, + name: str, + instant_str: str, + branch_name: str, + value: object, +) -> None: + """Record a read of a parameter value or a tax scale. + + Any other object, such as a method or a node's label, is not a parameter + value and is not recorded. + """ + if isinstance(value, TaxScaleLike): + value = describe_tax_scale(value) + elif not isinstance(value, (numpy.ndarray,) + parameters.ALLOWED_PARAM_TYPES): + return + tracer.record_parameter_access(name, instant_str, branch_name, value) + + class TracingParameterNode: """The parameter tree as the formulas of one traced simulation see it. - Calling it at an instant returns a :class:`TracingParameterNodeAtInstant` - that records every parameter a formula reads in ``tracer``, under - ``branch_name``. Any other attribute is read from the wrapped node. + Calling it at an instant records the read in ``tracer``, under + ``branch_name``. A parameter or a tax scale returns its value; a node + returns a :class:`TracingParameterNodeAtInstant`, which records the reads + made through it. + + Children read before an instant is chosen, as in + ``parameters.gov.rate(period)``, are returned as tracing views too. Any + other attribute, such as ``values_list``, ``children`` or ``clone``, is + read from the wrapped node. What the formula does with it cannot be + followed, so the read is recorded at this node's path for all instants + (period ``eternity``, no value); it stands for everything below the node. The wrapped node is never modified, so tracing one simulation does not trace the tax-benefit system it shares with other simulations, branches @@ -79,7 +130,7 @@ class TracingParameterNode: def __init__( self, - parameter_node: parameters.ParameterNode, + parameter_node: parameters.AtInstantLike, tracer: tracers.FullTracer, branch_name: str, ) -> None: @@ -87,27 +138,104 @@ def __init__( self.tracer = tracer self.branch_name = branch_name - def __call__(self, instant) -> TracingParameterNodeAtInstant: + def __call__(self, instant) -> object: return self.get_at_instant(instant) - def get_at_instant(self, instant) -> TracingParameterNodeAtInstant: - node_at_instant = self.parameter_node.get_at_instant(instant) - if isinstance(node_at_instant, TracingParameterNodeAtInstant): + def get_at_instant(self, instant) -> object: + value_at_instant = self.parameter_node.get_at_instant(instant) + if isinstance(value_at_instant, TracingParameterNodeAtInstant): # The node traces by itself (its ``trace`` flag is set): record # in this simulation's tracer, not in the one the node holds. - node_at_instant = node_at_instant.parameter_node_at_instant - return TracingParameterNodeAtInstant( - node_at_instant, self.tracer, self.branch_name + value_at_instant = value_at_instant.parameter_node_at_instant + if isinstance(value_at_instant, _node_at_instant_types()): + return TracingParameterNodeAtInstant( + value_at_instant, self.tracer, self.branch_name + ) + _record_value_read( + self.tracer, + self.parameter_node.name, + str(periods.instant(instant)), + self.branch_name, + value_at_instant, + ) + return value_at_instant + + def _traced_child(self, child: object) -> object: + if isinstance(child, parameters.AtInstantLike): + return TracingParameterNode(child, self.tracer, self.branch_name) + return child + + def _record_read_for_all_instants(self, key: str) -> None: + if key.startswith("__") or key in _UNDATED_LABELS: + return + self.tracer.record_parameter_access( + self.parameter_node.name, periods.ETERNITY, self.branch_name, None ) def __getattr__(self, key: str): - return getattr(_wrapped(self, "parameter_node", key), key) + parameter_node = _wrapped(self, "parameter_node", key) + try: + attribute = getattr(parameter_node, key) + except AttributeError: + self._record_read_for_all_instants(key) + raise + if isinstance(attribute, parameters.AtInstantLike): + return self._traced_child(attribute) + self._record_read_for_all_instants(key) + return attribute + + def __getitem__(self, key: object) -> object: + # A tax scale gives its brackets by position. + return self._traced_child(self.parameter_node[key]) + + def get_child(self, path: str) -> object: + return self._traced_child(self.parameter_node.get_child(path)) def __repr__(self) -> str: return repr(self.parameter_node) +def _node_at_instant_types() -> tuple[type, ...]: + # Read when called: ``parameters`` imports this module while it loads. + return ( + parameters.ParameterNodeAtInstant, + parameters.VectorialParameterNodeAtInstant, + ) + + +def _child_path(parent_path: str, key: str) -> str: + """The path of the child ``key`` of the node at ``parent_path``. + + The root of the tree has the empty path, so its children's paths are their + keys. Joining uses the key's string value, which ``str()`` of a ``str`` + based enum member is not. + """ + return ".".join([parent_path, key]) if parent_path else str.__str__(key) + + class TracingParameterNodeAtInstant: + """A parameter node at one instant, as the formulas of a traced simulation + see it. + + Each parameter value or tax scale read through it is recorded in + ``tracer``, under ``branch_name``, at its own path. A child node is + returned as another tracing view. + + A look-up by name that finds no child is recorded too, because a formula + can fall back on a default and a value added there would change its + result: + + - an attribute look-up (``node.rate``, ``getattr``, ``hasattr``) records the + node's own path, with the names of its children as the value; + - a key look-up (``node["rate"]``) records the missing child's path, with + no value. + + These are not recorded when they find nothing: a key look-up on a + vectorial node, a look-up by an array that names a missing child, and, + on the undated tree, ``get_child`` and a bracket position past the end of + a scale. + """ + def __init__( self, parameter_node_at_instant: ParameterNode, @@ -122,23 +250,85 @@ def __getattr__( self, key: str, ) -> Union[TracingParameterNodeAtInstant, Child]: - child = getattr(_wrapped(self, "parameter_node_at_instant", key), key) + node_at_instant = _wrapped(self, "parameter_node_at_instant", key) + if key == "_children" and isinstance( + node_at_instant, parameters.ParameterNodeAtInstant + ): + return TracingChildren(self) + try: + child = getattr(node_at_instant, key) + except AttributeError: + self._record_absent_child_read(key) + raise return self.get_traced_child(child, key) def __getitem__( self, key: str, ) -> Union[TracingParameterNodeAtInstant, Child]: - child = self.parameter_node_at_instant[key] + try: + child = self.parameter_node_at_instant[key] + except KeyError: + self._record_missing_key_read(key) + raise return self.get_traced_child(child, key) def __iter__(self) -> Iterator: # Without it, ``iter`` and ``in`` fall back to ``__getitem__(0)``. + if isinstance( + self.parameter_node_at_instant, parameters.ParameterNodeAtInstant + ): + return iter(self._record_children_read()) return iter(self.parameter_node_at_instant) def __repr__(self) -> str: return repr(self.parameter_node_at_instant) + def _record_absent_child_read(self, key: str) -> None: + """Record an attribute look-up that found no child. + + The read is recorded at the node's own path, with the names of its + children. Special names such as ``__array__`` are not children and are + not recorded. + """ + if not key.startswith("__") and isinstance( + self.parameter_node_at_instant, parameters.ParameterNodeAtInstant + ): + self._record_children_read() + + def _record_missing_key_read(self, key: object) -> None: + """Record a key look-up that found no child, at that child's path. + + The read holds no value. A key that is not a string, such as an array + for a vectorial read, names no single child and is not recorded. + """ + node_at_instant = self.parameter_node_at_instant + if isinstance(key, str) and isinstance( + node_at_instant, parameters.ParameterNodeAtInstant + ): + self.tracer.record_parameter_access( + _child_path(node_at_instant._name, key), + node_at_instant._instant_str, + self.branch_name, + None, + ) + + def _record_children_read(self) -> list[str]: + """Record that the formula read which children this node has. + + The read is recorded at the node's own path, with the child names as + its value, and the names are returned. + """ + node_at_instant = self.parameter_node_at_instant + child_names = list(node_at_instant._children) + self.tracer.record_parameter_access( + node_at_instant._name, + node_at_instant._instant_str, + self.branch_name, + child_names, + ) + return child_names + def get_traced_child( self, child: Child, @@ -146,13 +336,7 @@ def get_traced_child( ) -> Union[TracingParameterNodeAtInstant, Child]: period: str = self.parameter_node_at_instant._instant_str - if isinstance( - child, - ( - parameters.ParameterNodeAtInstant, - parameters.VectorialParameterNodeAtInstant, - ), - ): + if isinstance(child, _node_at_instant_types()): return TracingParameterNodeAtInstant(child, self.tracer, self.branch_name) if not isinstance(key, str) or isinstance( @@ -167,7 +351,42 @@ def get_traced_child( else: name = ".".join([self.parameter_node_at_instant._name, key]) - if isinstance(child, (numpy.ndarray,) + parameters.ALLOWED_PARAM_TYPES): - self.tracer.record_parameter_access(name, period, self.branch_name, child) + _record_value_read(self.tracer, name, period, self.branch_name, child) return child + + +class TracingChildren(Mapping): + """The ``_children`` mapping of a traced node. + + Reading which children exist (iterating, counting, testing membership or + asking for a missing child) records one read at the node's own path, for + this mapping. Taking a child returns it through the node's tracing, so + the child's value is recorded at its own path and a child node stays + traced. + """ + + def __init__(self, parent: TracingParameterNodeAtInstant) -> None: + self._parent = parent + self._child_names: Optional[list[str]] = None + + def _names_read(self) -> list[str]: + if self._child_names is None: + self._child_names = self._parent._record_children_read() + return self._child_names + + def __getitem__(self, key: str) -> object: + children = self._parent.parameter_node_at_instant._children + if key not in children: + self._names_read() + raise KeyError(key) + return self._parent.get_traced_child(children[key], key) + + def __iter__(self) -> Iterator[str]: + return iter(self._names_read()) + + def __len__(self) -> int: + return len(self._names_read()) + + def __contains__(self, key: object) -> bool: + return key in self._names_read() diff --git a/tests/core/test_tracer_parameter_reads.py b/tests/core/test_tracer_parameter_reads.py new file mode 100644 index 000000000..8412e6687 --- /dev/null +++ b/tests/core/test_tracer_parameter_reads.py @@ -0,0 +1,823 @@ +"""What a traced simulation records about the parameters it reads. + +Every test here runs a calculation and looks at the reads recorded on the +variable's node of the trace, so none depends on how reads reach the tracer. +""" + +import enum +import json + +import numpy as np +import pytest + +from policyengine_core.country_template.entities import Person +from policyengine_core.parameters import Parameter, ParameterNode +from policyengine_core.parameters.operations.get_parameter import get_parameter +from policyengine_core.periods import ETERNITY, MONTH, YEAR +from policyengine_core.tracers import SimpleTracer +from policyengine_core.variables import Variable +from tests.fixtures.tracing import assert_untraced + +JANUARY = "2017-01" +INSTANT = "2017-01-01" +SOCIAL_SECURITY_SCALE = "taxes.social_security_contribution" +SOCIAL_SECURITY_DESCRIPTION = { + "type": "MarginalRateTaxScale", + "thresholds": [0.0, 6000.0, 12400.0], + "rates": [0.02, 0.06, 0.12], +} + + +def trace_node(simulation, variable_name): + """The first node calculating ``variable_name``, anywhere in the trace.""" + pending_nodes = list(simulation.tracer.trees) + while pending_nodes: + node = pending_nodes.pop(0) + if node.name == variable_name: + return node + pending_nodes.extend(node.children) + raise AssertionError(f"{variable_name} is not in the trace") + + +def recorded_reads(simulation, variable_name): + """The reads on the variable's node, as (name, period, branch, value).""" + return [ + (read.name, read.period, read.branch_name, read.value) + for read in trace_node(simulation, variable_name).parameters + ] + + +def recorded_names(simulation, variable_name): + return [read[0] for read in recorded_reads(simulation, variable_name)] + + +def define_variable(system, name, formula, definition_period=MONTH, **attributes): + """Add a float person variable to ``system``.""" + class_attributes = { + "value_type": float, + "entity": Person, + "definition_period": definition_period, + "label": name, + **attributes, + } + if formula is not None: + class_attributes["formula"] = formula + system.add_variable(type(name, (Variable,), class_attributes)) + + +@pytest.fixture +def system(isolated_tax_benefit_system): + return isolated_tax_benefit_system + + +@pytest.fixture +def traced_simulation(system, make_simulation): + def build(variables=None, period=JANUARY): + simulation = make_simulation(system, variables or {"salary": 2000}, period) + simulation.trace = True + return simulation + + return build + + +# ----- Reads at an instant ----- # + + +def test_yearly_formula_read_is_recorded_when_the_instant_was_read_before( + system, traced_simulation +): + def formula(person, period, parameters): + return person.filled_array(parameters(period).taxes.income_tax_rate) + + define_variable(system, "yearly_rate", formula, definition_period=YEAR) + rate = system.parameters(INSTANT).taxes.income_tax_rate + simulation = traced_simulation() + + simulation.calculate("yearly_rate", "2017") + + assert recorded_reads(simulation, "yearly_rate") == [ + ("taxes.income_tax_rate", INSTANT, "default", rate) + ] + + +def test_scale_read_is_recorded_with_its_schedule(system, traced_simulation): + simulation = traced_simulation() + + simulation.calculate("social_security_contribution", JANUARY) + + assert recorded_reads(simulation, "social_security_contribution") == [ + (SOCIAL_SECURITY_SCALE, INSTANT, "default", SOCIAL_SECURITY_DESCRIPTION) + ] + json.dumps(simulation.tracer.get_serialized_flat_trace()) + + +def test_scale_read_by_key_is_recorded(system, traced_simulation): + def formula(person, period, parameters): + scale = parameters(period).taxes["social_security_contribution"] + return scale.calc(person("salary", period)) + + define_variable(system, "scale_by_key", formula) + simulation = traced_simulation() + + simulation.calculate("scale_by_key", JANUARY) + + assert recorded_names(simulation, "scale_by_key") == [SOCIAL_SECURITY_SCALE] + + +def test_reading_private_attributes_of_a_node_records_nothing( + system, traced_simulation +): + def formula(person, period, parameters): + taxes = parameters(period).taxes + return person.filled_array(len(taxes._name) + len(taxes._instant_str)) + + define_variable(system, "node_labels", formula) + simulation = traced_simulation() + + simulation.calculate("node_labels", JANUARY) + + assert recorded_reads(simulation, "node_labels") == [] + + +# ----- Reads of a node's children ----- # + + +def test_listing_a_nodes_children_records_the_node_with_their_names( + system, traced_simulation +): + def formula(person, period, parameters): + return person.filled_array(len(list(parameters(period).taxes._children))) + + define_variable(system, "tax_count", formula) + child_names = list(system.parameters(INSTANT).taxes._children) + simulation = traced_simulation() + + simulation.calculate("tax_count", JANUARY) + + assert recorded_reads(simulation, "tax_count") == [ + ("taxes", INSTANT, "default", child_names) + ] + + +@pytest.mark.parametrize( + "read_children", + [ + lambda children: len(children), + lambda children: "income_tax_rate" in children, + lambda children: "no_such_child" in children, + lambda children: list(children.keys()), + lambda children: children.get("no_such_child", 0), + ], + ids=["len", "in", "not-in", "keys", "get-missing"], +) +def test_reading_which_children_exist_records_the_node( + system, traced_simulation, read_children +): + def formula(person, period, parameters): + read_children(parameters(period).taxes._children) + return person.filled_array(0) + + define_variable(system, "children_shape", formula) + simulation = traced_simulation() + + simulation.calculate("children_shape", JANUARY) + + assert recorded_names(simulation, "children_shape") == ["taxes"] + + +def test_child_taken_from_the_mapping_is_recorded_at_its_own_path( + system, traced_simulation +): + def formula(person, period, parameters): + return person.filled_array( + parameters(period).taxes._children["income_tax_rate"] + ) + + define_variable(system, "rate_from_mapping", formula) + rate = system.parameters(INSTANT).taxes.income_tax_rate + simulation = traced_simulation() + + simulation.calculate("rate_from_mapping", JANUARY) + + assert recorded_reads(simulation, "rate_from_mapping") == [ + ("taxes.income_tax_rate", INSTANT, "default", rate) + ] + + +def test_reads_below_a_node_taken_from_the_mapping_are_recorded( + system, traced_simulation +): + def formula(person, period, parameters): + housing_tax = parameters(period).taxes._children["housing_tax"] + return person.filled_array(housing_tax.rate) + + define_variable(system, "nested_rate", formula) + simulation = traced_simulation() + + simulation.calculate("nested_rate", JANUARY) + + assert recorded_names(simulation, "nested_rate") == ["taxes.housing_tax.rate"] + + +def test_values_of_the_mapping_are_traced(system, traced_simulation): + def formula(person, period, parameters): + child_nodes = [ + child + for child in parameters(period).benefits._children.values() + if not isinstance(child, (int, float)) + ] + return person.filled_array(len(child_nodes[0]._children)) + + define_variable(system, "first_benefit_size", formula) + simulation = traced_simulation() + + simulation.calculate("first_benefit_size", JANUARY) + + names = recorded_names(simulation, "first_benefit_size") + assert names[0] == "benefits" + assert "benefits.parenting_allowance" in names + assert not [name for name in names if name.endswith("_children")] + + +def test_iterating_a_node_records_it_and_yields_child_names(system, traced_simulation): + def formula(person, period, parameters): + return person.filled_array(len(sorted(parameters(period).taxes))) + + define_variable(system, "iterated_taxes", formula) + child_names = list(system.parameters(INSTANT).taxes) + simulation = traced_simulation() + + assert simulation.calculate("iterated_taxes", JANUARY)[0] == len(child_names) + + assert recorded_reads(simulation, "iterated_taxes") == [ + ("taxes", INSTANT, "default", child_names) + ] + + +def test_membership_test_on_a_node_records_it(system, traced_simulation): + def formula(person, period, parameters): + return person.filled_array("income_tax_rate" in parameters(period).taxes) + + define_variable(system, "has_income_tax", formula) + simulation = traced_simulation() + + assert simulation.calculate("has_income_tax", JANUARY)[0] == 1 + + assert recorded_names(simulation, "has_income_tax") == ["taxes"] + + +# ----- Reads through the tree a formula receives, before choosing an instant ----- # + + +def test_parameter_called_from_the_undated_tree_is_recorded(system, traced_simulation): + def formula(person, period, parameters): + return person.filled_array(parameters.taxes.income_tax_rate(period)) + + define_variable(system, "undated_rate", formula) + rate = system.parameters(INSTANT).taxes.income_tax_rate + simulation = traced_simulation() + + assert simulation.calculate("undated_rate", JANUARY)[0] == pytest.approx(rate) + + assert recorded_reads(simulation, "undated_rate") == [ + ("taxes.income_tax_rate", INSTANT, "default", rate) + ] + + +def test_parameter_called_at_another_instant_records_that_instant( + system, traced_simulation +): + def formula(person, period, parameters): + return person.filled_array(parameters.taxes.income_tax_rate("2015-06-01")) + + define_variable(system, "frozen_rate", formula) + rate = system.parameters("2015-06-01").taxes.income_tax_rate + simulation = traced_simulation() + + simulation.calculate("frozen_rate", JANUARY) + + assert recorded_reads(simulation, "frozen_rate") == [ + ("taxes.income_tax_rate", "2015-06-01", "default", rate) + ] + + +def test_node_called_from_the_undated_tree_records_later_reads( + system, traced_simulation +): + def formula(person, period, parameters): + return person.filled_array(parameters.taxes.housing_tax(period).rate) + + define_variable(system, "undated_node_rate", formula) + simulation = traced_simulation() + + simulation.calculate("undated_node_rate", JANUARY) + + assert recorded_names(simulation, "undated_node_rate") == ["taxes.housing_tax.rate"] + + +def test_scale_called_from_the_undated_tree_is_recorded(system, traced_simulation): + def formula(person, period, parameters): + scale = parameters.taxes.social_security_contribution(period) + return scale.calc(person("salary", period)) + + define_variable(system, "undated_scale", formula) + simulation = traced_simulation() + + simulation.calculate("undated_scale", JANUARY) + + assert recorded_reads(simulation, "undated_scale") == [ + (SOCIAL_SECURITY_SCALE, INSTANT, "default", SOCIAL_SECURITY_DESCRIPTION) + ] + + +def test_bracket_of_a_scale_on_the_undated_tree_is_recorded(system, traced_simulation): + def formula(person, period, parameters): + bracket = parameters.taxes.social_security_contribution[1] + return person.filled_array(bracket.rate(period)) + + define_variable(system, "undated_bracket", formula) + simulation = traced_simulation() + + assert simulation.calculate("undated_bracket", JANUARY)[0] == pytest.approx(0.06) + + assert recorded_names(simulation, "undated_bracket") == [ + f"{SOCIAL_SECURITY_SCALE}[1].rate" + ] + + +def test_undated_reads_leave_the_system_untraced(system, traced_simulation): + def formula(person, period, parameters): + return person.filled_array(parameters.taxes.income_tax_rate(period)) + + define_variable(system, "undated_rate", formula) + + traced_simulation().calculate("undated_rate", JANUARY) + + assert_untraced(system.parameters) + + +def test_traced_and_untraced_undated_reads_agree( + system, traced_simulation, make_simulation +): + def formula(person, period, parameters): + return parameters.taxes.social_security_contribution(period).calc( + person("salary", period) + ) + parameters.taxes.income_tax_rate(period) + + define_variable(system, "undated_total", formula) + + traced = traced_simulation().calculate("undated_total", JANUARY) + untraced = make_simulation(system, {"salary": 2000}, JANUARY).calculate( + "undated_total", JANUARY + ) + + np.testing.assert_array_equal(traced, untraced) + + +def read_clone(rate): + return rate.clone()("2017-01-01") + + +def read_values_list(rate): + return rate.values_list[0].value + + +@pytest.mark.parametrize( + "read_rate", [read_clone, read_values_list], ids=["clone", "values-list"] +) +def test_parameter_used_outside_a_dated_call_is_recorded_for_all_instants( + system, traced_simulation, read_rate +): + def formula(person, period, parameters): + return person.filled_array(read_rate(parameters.taxes.income_tax_rate)) + + define_variable(system, "rate_history", formula) + simulation = traced_simulation() + + simulation.calculate("rate_history", JANUARY) + + assert recorded_reads(simulation, "rate_history") == [ + ("taxes.income_tax_rate", ETERNITY, "default", None) + ] + + +def test_node_children_read_before_an_instant_are_recorded_at_the_node( + system, traced_simulation +): + def formula(person, period, parameters): + rate = parameters.taxes.children["income_tax_rate"] + return person.filled_array(rate(period)) + + define_variable(system, "rate_from_children", formula) + simulation = traced_simulation() + + simulation.calculate("rate_from_children", JANUARY) + + assert recorded_names(simulation, "rate_from_children") == ["taxes"] + + +def test_scale_brackets_read_before_an_instant_are_recorded_at_the_scale( + system, traced_simulation +): + def formula(person, period, parameters): + bracket = parameters.taxes.social_security_contribution.brackets[1] + return person.filled_array(bracket.rate(period)) + + define_variable(system, "rate_from_brackets", formula) + simulation = traced_simulation() + + simulation.calculate("rate_from_brackets", JANUARY) + + assert recorded_names(simulation, "rate_from_brackets") == [SOCIAL_SECURITY_SCALE] + + +def test_child_found_by_path_on_the_undated_tree_is_recorded(system, traced_simulation): + def formula(person, period, parameters): + rate = parameters.get_child("taxes.income_tax_rate") + return person.filled_array(rate(period)) + + define_variable(system, "rate_by_path", formula) + simulation = traced_simulation() + + simulation.calculate("rate_by_path", JANUARY) + + assert recorded_names(simulation, "rate_by_path") == ["taxes.income_tax_rate"] + + +def test_get_parameter_on_the_undated_tree_is_recorded(system, traced_simulation): + def formula(person, period, parameters): + rate = get_parameter(parameters.taxes, "income_tax_rate") + return person.filled_array(rate(period)) + + define_variable(system, "rate_from_helper", formula) + simulation = traced_simulation() + + simulation.calculate("rate_from_helper", JANUARY) + + assert recorded_names(simulation, "rate_from_helper") == ["taxes"] + + +def test_labels_of_undated_parameters_are_not_reads(system, traced_simulation): + def formula(person, period, parameters): + rate = parameters.taxes.income_tax_rate + return person.filled_array(len(rate.name) + len(rate.description)) + + define_variable(system, "rate_label_size", formula) + simulation = traced_simulation() + + simulation.calculate("rate_label_size", JANUARY) + + assert recorded_reads(simulation, "rate_label_size") == [] + + +# ----- Reads of a child that is absent ----- # + + +def read_absent_with_default(taxes): + return getattr(taxes, "no_such_rate", 0.0) + + +def read_absent_with_hasattr(taxes): + return float(hasattr(taxes, "no_such_rate")) + + +@pytest.mark.parametrize( + "read_absent", + [read_absent_with_default, read_absent_with_hasattr], + ids=["getattr-default", "hasattr"], +) +def test_looking_for_an_absent_child_records_the_node( + system, traced_simulation, read_absent +): + def formula(person, period, parameters): + return person.filled_array(read_absent(parameters(period).taxes)) + + define_variable(system, "absent_rate", formula) + child_names = list(system.parameters(INSTANT).taxes) + simulation = traced_simulation() + + assert simulation.calculate("absent_rate", JANUARY)[0] == 0 + + assert recorded_reads(simulation, "absent_rate") == [ + ("taxes", INSTANT, "default", child_names) + ] + + +def read_rate_by_key_with_default(node, key): + try: + return node[key] + except KeyError: + return 0.0 + + +def taxes_node(parameters_at_instant): + return parameters_at_instant.taxes + + +def root_node(parameters_at_instant): + return parameters_at_instant + + +@pytest.mark.parametrize( + "select_node, missing_path", + [(taxes_node, "taxes.no_such_rate"), (root_node, "no_such_rate")], + ids=["below-a-node", "below-the-root"], +) +def test_key_look_up_that_finds_no_child_records_the_missing_path( + system, traced_simulation, select_node, missing_path +): + def formula(person, period, parameters): + node = select_node(parameters(period)) + return person.filled_array(read_rate_by_key_with_default(node, "no_such_rate")) + + define_variable(system, "rate_by_key", formula) + simulation = traced_simulation() + + assert simulation.calculate("rate_by_key", JANUARY)[0] == 0 + + assert recorded_reads(simulation, "rate_by_key") == [ + (missing_path, INSTANT, "default", None) + ] + + +def test_key_look_up_before_a_parameter_starts_records_that_parameter( + system, traced_simulation +): + """A value given to the parameter at the instant read would change the + result, so a reform of it must find this formula among its readers.""" + future_rate = Parameter( + "taxes.future_rate", {"values": {"2030-01-01": {"value": 0.3}}} + ) + system.parameters.taxes.add_child("future_rate", future_rate) + + def formula(person, period, parameters): + taxes = parameters(period).taxes + return person.filled_array(read_rate_by_key_with_default(taxes, "future_rate")) + + define_variable(system, "rate_from_2030", formula) + simulation = traced_simulation() + + assert simulation.calculate("rate_from_2030", JANUARY)[0] == 0 + + assert recorded_reads(simulation, "rate_from_2030") == [ + (future_rate.name, INSTANT, "default", None) + ] + + +class Zone(str, enum.Enum): + NORTH = "north" + + +def test_key_look_up_by_a_string_enum_member_records_the_member_as_a_path( + system, traced_simulation +): + """``str()`` of such a member is ``Zone.NORTH`` from Python 3.12, but the + look-up uses its string value, so that is the path a value would go to.""" + + def formula(person, period, parameters): + taxes = parameters(period).taxes + return person.filled_array(read_rate_by_key_with_default(taxes, Zone.NORTH)) + + define_variable(system, "rate_by_zone", formula) + simulation = traced_simulation() + + assert simulation.calculate("rate_by_zone", JANUARY)[0] == 0 + + assert recorded_reads(simulation, "rate_by_zone") == [ + ("taxes.north", INSTANT, "default", None) + ] + + +ABOLITION_TREES = { + "switch-off": {"abolitions": {"income_tax": {"2015-01-01": {"value": False}}}}, + "no-switch": {"abolitions": {"other_variable": {"2015-01-01": {"value": False}}}}, + "no-gov-node": None, +} + + +@pytest.mark.parametrize("gov_data", ABOLITION_TREES.values(), ids=ABOLITION_TREES) +def test_abolition_check_on_a_tracing_root_records_nothing( + system, traced_simulation, gov_data +): + """A country package can switch tracing on in its own parameter root, so + core's abolition check, a look-up per variable, would reach the trace. + Core reads the untraced tree instead: the check is not the formula's.""" + if gov_data is not None: + system.parameters.add_child("gov", ParameterNode("gov", data=gov_data)) + simulation = traced_simulation() + system.parameters.trace = True + system.parameters.tracer = simulation.tracer + system.parameters.branch_name = "default" + + simulation.calculate("income_tax", JANUARY) + + assert recorded_names(simulation, "income_tax") == ["taxes.income_tax_rate"] + + +def test_absent_child_on_the_undated_tree_records_the_node(system, traced_simulation): + def formula(person, period, parameters): + return person.filled_array(float(hasattr(parameters.taxes, "no_such_rate"))) + + define_variable(system, "undated_absent", formula) + simulation = traced_simulation() + + assert simulation.calculate("undated_absent", JANUARY)[0] == 0 + + assert recorded_reads(simulation, "undated_absent") == [ + ("taxes", ETERNITY, "default", None) + ] + + +# ----- Reads core makes for a variable ----- # + + +def add_list_parameter(system, name, value): + system.parameters.taxes.add_child( + name, + Parameter(f"taxes.{name}", {"values": {"2015-01-01": {"value": value}}}), + ) + + +def test_parameter_in_an_adds_list_is_recorded(system, traced_simulation): + define_variable( + system, + "salary_plus_basic_income", + None, + adds=["salary", "benefits.basic_income"], + ) + basic_income = system.parameters(INSTANT).benefits.basic_income + simulation = traced_simulation() + + assert simulation.calculate("salary_plus_basic_income", JANUARY)[ + 0 + ] == pytest.approx(2000 + basic_income) + + assert recorded_reads(simulation, "salary_plus_basic_income") == [ + ("benefits.basic_income", INSTANT, "default", basic_income) + ] + + +def test_parameter_in_a_subtracts_list_is_recorded(system, traced_simulation): + define_variable( + system, "minus_basic_income", None, subtracts=["benefits.basic_income"] + ) + basic_income = system.parameters(INSTANT).benefits.basic_income + simulation = traced_simulation() + + simulation.calculate("minus_basic_income", JANUARY) + + assert recorded_reads(simulation, "minus_basic_income") == [ + ("benefits.basic_income", INSTANT, "default", basic_income) + ] + + +def test_parameter_holding_the_adds_list_is_recorded(system, traced_simulation): + add_list_parameter(system, "income_components", ["salary"]) + define_variable(system, "income_total", None, adds="taxes.income_components") + simulation = traced_simulation() + + assert simulation.calculate("income_total", JANUARY)[0] == 2000 + + assert recorded_reads(simulation, "income_total") == [ + ("taxes.income_components", INSTANT, "default", ["salary"]) + ] + + +def test_parameter_holding_the_subtracts_list_is_recorded(system, traced_simulation): + add_list_parameter(system, "income_deductions", ["salary"]) + define_variable(system, "minus_income", None, subtracts="taxes.income_deductions") + simulation = traced_simulation() + + assert simulation.calculate("minus_income", JANUARY)[0] == -2000 + + assert recorded_reads(simulation, "minus_income") == [ + ("taxes.income_deductions", INSTANT, "default", ["salary"]) + ] + + +def test_uprating_records_the_index_at_both_instants(system, traced_simulation): + define_variable(system, "uprated_amount", None, uprating="taxes.income_tax_rate") + rate = system.parameters.taxes.income_tax_rate + simulation = traced_simulation() + simulation.set_input("uprated_amount", "2015-01", [100]) + + uprated = simulation.calculate("uprated_amount", JANUARY) + + assert uprated[0] == pytest.approx(100 * rate(INSTANT) / rate("2015-01-01")) + assert recorded_reads(simulation, "uprated_amount") == [ + ("taxes.income_tax_rate", "2015-01-01", "default", rate("2015-01-01")), + ("taxes.income_tax_rate", INSTANT, "default", rate(INSTANT)), + ] + + +def test_uprating_records_the_value_held_before_the_index_starts( + system, traced_simulation +): + system.parameters.taxes.add_child( + "late_index", + Parameter( + "taxes.late_index", + {"values": {"2016-01-01": {"value": 2.0}, "2017-01-01": {"value": 3.0}}}, + ), + ) + define_variable(system, "late_uprated", None, uprating="taxes.late_index") + simulation = traced_simulation() + simulation.set_input("late_uprated", "2015-01", [100]) + + assert simulation.calculate("late_uprated", JANUARY)[0] == pytest.approx(150) + + assert recorded_reads(simulation, "late_uprated") == [ + ("taxes.late_index", "2015-01-01", "default", 2.0), + ("taxes.late_index", INSTANT, "default", 3.0), + ] + + +def test_untraced_simulation_records_no_core_reads(system, make_simulation): + define_variable(system, "added_income", None, adds=["benefits.basic_income"]) + simulation = make_simulation(system, {"salary": 2000}, JANUARY) + + simulation.calculate("added_income", JANUARY) + + assert not hasattr(simulation.tracer, "trees") + + +# ----- Branches ----- # + + +def test_nested_branch_keeps_each_formulas_reads_under_its_own_branch( + system, traced_simulation +): + def formula(person, period, parameters): + rate_before = parameters(period).taxes.income_tax_rate + person.simulation.get_branch("inner").calculate("income_tax", period) + rate_after = parameters(period).taxes.income_tax_rate + return person.filled_array(rate_before + rate_after) + + define_variable(system, "reads_around_a_branch", formula) + simulation = traced_simulation() + + simulation.calculate("reads_around_a_branch", JANUARY) + + outer_node = trace_node(simulation, "reads_around_a_branch") + assert [(read.name, read.branch_name) for read in outer_node.parameters] == [ + ("taxes.income_tax_rate", "default"), + ("taxes.income_tax_rate", "default"), + ] + inner_node = next( + child for child in outer_node.children if child.name == "income_tax" + ) + assert [(read.name, read.branch_name) for read in inner_node.parameters] == [ + ("taxes.income_tax_rate", "inner") + ] + + +def test_core_reads_in_a_branch_carry_the_branch_name(system, traced_simulation): + define_variable(system, "added_income", None, adds=["benefits.basic_income"]) + simulation = traced_simulation() + + simulation.get_branch("inner").calculate("added_income", JANUARY) + + assert [read[2] for read in recorded_reads(simulation, "added_income")] == ["inner"] + + +# ----- Which calculations ran a formula ----- # + + +def test_trace_marks_a_calculation_that_ran_its_formula(system, traced_simulation): + simulation = traced_simulation() + + simulation.calculate("income_tax", JANUARY) + + assert trace_node(simulation, "income_tax").formula_ran is True + assert trace_node(simulation, "salary").formula_ran is False + + +def test_trace_marks_adds_as_having_run(system, traced_simulation): + define_variable(system, "added_salary", None, adds=["salary"]) + simulation = traced_simulation() + + simulation.calculate("added_salary", JANUARY) + + assert trace_node(simulation, "added_salary").formula_ran is True + + +def test_trace_marks_a_formula_skipped_by_defined_for(system, traced_simulation): + def formula(person, period, parameters): + return person.filled_array(parameters(period).taxes.income_tax_rate) + + define_variable(system, "never_eligible", None, default_value=0) + define_variable(system, "masked_rate", formula, defined_for="never_eligible") + simulation = traced_simulation() + + simulation.calculate("masked_rate", JANUARY) + + assert trace_node(simulation, "masked_rate").formula_ran is False + assert recorded_reads(simulation, "masked_rate") == [] + + +def test_simple_tracer_accepts_a_formula_run(system, make_simulation): + simulation = make_simulation(system, {"salary": 2000}, JANUARY) + simulation.trace = True + simulation.tracer = SimpleTracer() + + assert simulation.calculate("income_tax", JANUARY)[0] > 0 diff --git a/tests/core/test_tracing_parameter_isolation.py b/tests/core/test_tracing_parameter_isolation.py index 8c4f5e95a..c6b77e3f0 100644 --- a/tests/core/test_tracing_parameter_isolation.py +++ b/tests/core/test_tracing_parameter_isolation.py @@ -250,7 +250,10 @@ def test_tracing_parameter_node_reads_through_to_the_node(isolated_tax_benefit_s tracer = FullTracer() view = TracingParameterNode(parameters, tracer, "policy") - assert view.taxes is parameters.taxes + assert type(view.taxes) is TracingParameterNode + assert view.taxes.parameter_node is parameters.taxes + assert view.taxes.tracer is tracer + assert view.taxes.branch_name == "policy" assert view.children is parameters.children at_instant = view(INSTANT) diff --git a/tests/core/tools/test_dependency_map.py b/tests/core/tools/test_dependency_map.py new file mode 100644 index 000000000..af9f608bc --- /dev/null +++ b/tests/core/tools/test_dependency_map.py @@ -0,0 +1,798 @@ +import json +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +import policyengine_core.country_template as country_template +from policyengine_core.country_template import ( + CountryTaxBenefitSystem, + Microsimulation, +) +from policyengine_core.country_template.entities import Person +from policyengine_core.parameters import ParameterNode +from policyengine_core.periods import MONTH +from policyengine_core.scripts import policyengine_command +from policyengine_core.tools.dependency_map import ( + build_dependency_map, + collect_edges, + iter_yaml_tests, + merge_edges, + model_fingerprint, + trace_microdata, + trace_yaml_tests, + write_dependency_map, +) +from policyengine_core.tracers import FullTracer +from policyengine_core.variables import Variable + +TESTS_ROOT = Path(country_template.__file__).resolve().parent / "tests" +TEMPLATE_PACKAGE = "policyengine_core.country_template" + + +class surtax_eligible(Variable): + value_type = float + entity = Person + definition_period = MONTH + label = "Whether the surtax applies" + + +class surtax(Variable): + value_type = float + entity = Person + definition_period = MONTH + label = "A tax that only runs for eligible people" + defined_for = "surtax_eligible" + + def formula(person, period, parameters): + return person("salary", period) * parameters(period).taxes.income_tax_rate + + +class counts_taxes(Variable): + value_type = float + entity = Person + definition_period = MONTH + label = "Reads which children the taxes node has" + + def formula(person, period, parameters): + return person.filled_array(len(parameters(period).taxes._children)) + + +class undated_rate(Variable): + value_type = float + entity = Person + definition_period = MONTH + label = "Reads a parameter before choosing an instant" + + def formula(person, period, parameters): + return person.filled_array(parameters.taxes.income_tax_rate(period)) + + +class raises_without_salary(Variable): + value_type = float + entity = Person + definition_period = MONTH + label = "Raises before reading its parameter when nobody has a salary" + + def formula(person, period, parameters): + salary = person("salary", period) + if not salary.any(): + raise ValueError("no salary") + return salary * parameters(period).taxes.income_tax_rate + + +@pytest.fixture +def system(isolated_tax_benefit_system): + for variable in ( + surtax_eligible, + surtax, + counts_taxes, + undated_rate, + raises_without_salary, + ): + isolated_tax_benefit_system.add_variable(variable) + return isolated_tax_benefit_system + + +def write_tests(directory: Path, content: str, name: str = "cases.yaml") -> Path: + file = directory / name + file.write_text(content) + return file + + +INELIGIBLE_THEN_ELIGIBLE = """ +- name: not yet eligible + period: 2017-01 + input: {salary: 1000} + output: {surtax: 0} +- name: eligible + period: 2017-01 + input: {salary: 1000, surtax_eligible: 1} + output: {surtax: 150} +- name: eligible again + period: 2017-01 + input: {salary: 2000, surtax_eligible: 1} + output: {surtax: 300} +""" + + +# ----- Edges ----- # + + +def test_trace_yaml_tests_records_parameter_and_variable_edges(): + (readers, consumers), stats = trace_yaml_tests( + CountryTaxBenefitSystem(), [TESTS_ROOT / "income_tax.yaml"] + ) + + assert stats == { + "tests": 1, + "skipped": 0, + "excluded": 0, + "failed": 0, + "outputs": 1, + "outputErrors": 0, + } + assert readers["taxes.income_tax_rate"] == {"income_tax"} + assert consumers["salary"] == {"income_tax"} + + +def test_trace_yaml_tests_records_scale_reads(): + (readers, _), _ = trace_yaml_tests( + CountryTaxBenefitSystem(), [TESTS_ROOT / "social_security_contribution.yaml"] + ) + + assert readers["taxes.social_security_contribution"] == { + "social_security_contribution" + } + + +def test_map_holds_only_paths_that_exist(system, tmp_path: Path): + write_tests( + tmp_path, + """ +- period: 2017-01 + input: {salary: 1000} + output: {counts_taxes: 3, undated_rate: 0.15} +""", + ) + + (readers, _), _ = trace_yaml_tests(system, [tmp_path]) + + assert readers["taxes"] == {"counts_taxes"} + assert readers["taxes.income_tax_rate"] == {"undated_rate"} + assert not [path for path in readers if path.endswith("_children")] + + +class replacement_payment(Variable): + value_type = float + entity = Person + definition_period = MONTH + label = "Paid only while the basic income is abolished" + + def formula(person, period, parameters): + basic_income_abolished = parameters(period).gov.abolitions.basic_income + return person.filled_array(100.0 * basic_income_abolished) + + +def test_formula_reading_another_variables_abolition_switch_keeps_its_edge( + system, make_simulation +): + """A formula that reads another variable's switch changes with it. Core's + check of each variable's own switch is not the formula's read, and is not + recorded.""" + system.parameters.add_child( + "gov", + ParameterNode( + "gov", + data={"abolitions": {"basic_income": {"2015-01-01": {"value": False}}}}, + ), + ) + system.add_variable(replacement_payment) + simulation = make_simulation(system, {"salary": 1000}, "2017-01") + simulation.trace = True + # A country package can trace its own parameter root, which sends core's + # abolition check for every variable through this simulation's tracer. + system.parameters.trace = True + system.parameters.tracer = simulation.tracer + system.parameters.branch_name = "default" + + simulation.calculate("replacement_payment", "2017-01") + simulation.calculate("basic_income", "2017-01") + readers, _ = collect_edges(simulation) + + assert readers["gov.abolitions.basic_income"] == {"replacement_payment"} + assert "gov.abolitions.replacement_payment" not in readers + + +def test_trace_yaml_tests_follows_entity_scoped_and_period_keyed_outputs( + tmp_path: Path, +): + write_tests( + tmp_path, + """ +- name: plural entity outputs + period: 2017-01 + input: + persons: + alice: {salary: 1000} + households: + home: {parents: [alice]} + output: + persons: + alice: {income_tax: 150} +- name: singular entity output keyed by period + period: 2017-01 + input: {salary: 1000} + output: + person: + social_security_contribution: {2017-01: 20} +- name: same variable again under the plural key + period: 2017-01 + input: + persons: + bob: {salary: 3000} + households: + home: {parents: [bob]} + output: + persons: + bob: {income_tax: 450} +""", + ) + + (readers, consumers), stats = trace_yaml_tests( + CountryTaxBenefitSystem(), [tmp_path] + ) + + assert stats == { + "tests": 2, + "skipped": 1, + "excluded": 0, + "failed": 0, + "outputs": 2, + "outputErrors": 0, + } + assert readers["taxes.income_tax_rate"] == {"income_tax"} + assert readers["taxes.social_security_contribution"] == { + "social_security_contribution" + } + assert consumers["salary"] == {"income_tax", "social_security_contribution"} + + +def test_trace_yaml_tests_counts_outputs_that_fail_to_calculate(tmp_path: Path): + write_tests( + tmp_path, + """ +- name: an output the model does not define + period: 2017-01 + input: {salary: 1000} + output: + person: + no_such_variable: 1 + social_security_contribution: 20 +""", + ) + + (readers, _), stats = trace_yaml_tests(CountryTaxBenefitSystem(), [tmp_path]) + + assert stats["outputs"] == 1 + assert stats["outputErrors"] == 1 + assert readers["taxes.social_security_contribution"] == { + "social_security_contribution" + } + + +# ----- Which tests are traced ----- # + + +def test_iter_yaml_tests_keeps_tests_of_the_baseline_system(tmp_path: Path): + write_tests( + tmp_path, + """ +- name: plain + period: 2017-01 + input: {salary: 1000} + output: {income_tax: 150} +- name: with reform + period: 2017-01 + reforms: policyengine_core.country_template.reforms.some_reform + input: {salary: 1000} + output: {income_tax: 150} +- name: inline parameter change + period: 2017-01 + input: {salary: 1000, taxes.income_tax_rate: 0.5} + output: {income_tax: 500} +- name: no output + period: 2017-01 + input: {salary: 1000} +- name: same output again + period: 2017-01 + input: {salary: 2000} + output: {income_tax: 300} +""", + ) + + names = [test["name"] for _, test in iter_yaml_tests([tmp_path])] + + assert names == ["plain", "same output again"] + + +def test_a_file_holding_one_mapping_is_one_test(tmp_path: Path): + write_tests( + tmp_path, + """ +name: lone test +period: 2017-01 +input: {salary: 1000} +output: {income_tax: 150} +""", + ) + + (readers, _), stats = trace_yaml_tests(CountryTaxBenefitSystem(), [tmp_path]) + + assert stats["tests"] == 1 + assert readers["taxes.income_tax_rate"] == {"income_tax"} + + +def test_yml_files_are_traced_and_empty_files_ignored(tmp_path: Path): + write_tests( + tmp_path, + """ +- period: 2017-01 + input: {salary: 1000} + output: {income_tax: 150} +""", + "cases.yml", + ) + write_tests(tmp_path, "", "empty.yaml") + + (readers, _), stats = trace_yaml_tests(CountryTaxBenefitSystem(), [tmp_path]) + + assert stats["tests"] == 1 + assert readers["taxes.income_tax_rate"] == {"income_tax"} + + +def test_repeated_output_is_skipped_and_keeps_every_edge(tmp_path: Path): + write_tests( + tmp_path, + """ +- period: 2017-01 + input: {salary: 1000} + output: {income_tax: 150} +- period: 2017-01 + input: {salary: 2000} + output: {income_tax: 300} +- period: 2017-01 + input: {salary: 2000} + output: {income_tax: 300, social_security_contribution: 40} +""", + ) + system = CountryTaxBenefitSystem() + + selected_edges, selected_stats = trace_yaml_tests(system, [tmp_path]) + every_edges, every_stats = trace_yaml_tests(system, [tmp_path], every_test=True) + + assert (selected_stats["tests"], selected_stats["skipped"]) == (2, 1) + assert (every_stats["tests"], every_stats["skipped"]) == (3, 0) + assert selected_edges == every_edges + + +def test_output_masked_by_defined_for_stays_uncovered(system, tmp_path: Path): + write_tests(tmp_path, INELIGIBLE_THEN_ELIGIBLE) + + selected_edges, stats = trace_yaml_tests(system, [tmp_path]) + every_edges, _ = trace_yaml_tests(system, [tmp_path], every_test=True) + + assert selected_edges[0]["taxes.income_tax_rate"] == {"surtax"} + assert selected_edges[1]["salary"] == {"surtax"} + assert selected_edges == every_edges + assert (stats["tests"], stats["skipped"]) == (2, 1) + + +def test_output_given_as_an_input_stays_uncovered(tmp_path: Path): + write_tests( + tmp_path, + """ +- period: 2017-01 + input: {income_tax: 5} + output: {income_tax: 5} +- period: 2017-01 + input: {salary: 1000} + output: {income_tax: 150} +""", + ) + + (readers, _), stats = trace_yaml_tests(CountryTaxBenefitSystem(), [tmp_path]) + + assert readers["taxes.income_tax_rate"] == {"income_tax"} + assert stats["tests"] == 2 + + +def test_output_that_raised_stays_uncovered(system, tmp_path: Path): + write_tests( + tmp_path, + """ +- name: the formula starts, then raises before its later reads + period: 2017-01 + input: {salary: 0} + output: {raises_without_salary: 0} +- name: the formula finishes + period: 2017-01 + input: {salary: 1000} + output: {raises_without_salary: 150} +""", + ) + + (readers, _), stats = trace_yaml_tests(system, [tmp_path]) + + assert stats["outputErrors"] == 1 + assert stats["tests"] == 2 + assert readers["taxes.income_tax_rate"] == {"raises_without_salary"} + + +def test_each_dated_formula_of_an_output_is_traced(tmp_path: Path): + write_tests( + tmp_path, + """ +- name: formula from December 2016, which does not read salary + period: 2017-01 + input: {age: 30, salary: 0} + output: {basic_income: 600} +- name: formula from December 2015, which reads salary + period: 2016-01 + input: {age: 30, salary: 0} + output: {basic_income: 600} +""", + ) + + (_, consumers), stats = trace_yaml_tests(CountryTaxBenefitSystem(), [tmp_path]) + + assert consumers["salary"] == {"basic_income"} + assert stats["tests"] == 2 + + +def test_coverage_does_not_carry_between_files(tmp_path: Path): + test_text = """ +- period: 2017-01 + input: {salary: 1000} + output: {income_tax: 150} +""" + write_tests(tmp_path, test_text, "first.yaml") + write_tests(tmp_path, test_text, "second.yaml") + + _, stats = trace_yaml_tests(CountryTaxBenefitSystem(), [tmp_path]) + + assert (stats["tests"], stats["skipped"]) == (2, 0) + + +def test_tests_outside_the_baseline_system_are_counted(tmp_path: Path): + write_tests( + tmp_path, + """ +- name: with reform + period: 2017-01 + reforms: policyengine_core.country_template.reforms.some_reform + output: {income_tax: 150} +- name: with extension + period: 2017-01 + extensions: some_extension + output: {income_tax: 150} +- name: inline parameter change + period: 2017-01 + input: {salary: 1000, taxes.income_tax_rate: 0.5} + output: {income_tax: 500} +- name: plain + period: 2017-01 + input: {salary: 1000} + output: {income_tax: 150} +""", + ) + + _, stats = trace_yaml_tests(CountryTaxBenefitSystem(), [tmp_path]) + + assert (stats["excluded"], stats["tests"]) == (3, 1) + + +def test_unreadable_file_is_counted_and_later_files_are_traced(tmp_path: Path): + write_tests(tmp_path, "- {unclosed", "first.yaml") + write_tests( + tmp_path, + """ +- period: 2017-01 + input: {salary: 1000} + output: {income_tax: 150} +""", + "second.yaml", + ) + + (readers, _), stats = trace_yaml_tests(CountryTaxBenefitSystem(), [tmp_path]) + + assert (stats["failed"], stats["tests"]) == (1, 1) + assert readers["taxes.income_tax_rate"] == {"income_tax"} + + +def test_test_with_invalid_input_is_counted_and_later_tests_are_traced(tmp_path: Path): + write_tests( + tmp_path, + """ +- period: 2017-01 + input: 3 + output: {income_tax: 0} +- period: 2017-01 + input: {1: 2} + output: {income_tax: 0} +- period: 2017-01 + input: {salary: 1000} + output: {income_tax: 150} +""", + ) + + (readers, _), stats = trace_yaml_tests(CountryTaxBenefitSystem(), [tmp_path]) + + assert (stats["failed"], stats["tests"]) == (2, 1) + assert readers["taxes.income_tax_rate"] == {"income_tax"} + + +def test_nested_period_mappings_are_followed_to_each_period(tmp_path: Path): + write_tests( + tmp_path, + """ +- period: 2017-01 + input: {salary: 1000} + output: + income_tax: + 2017: + 2017-01: 150 + 2017-02: 150 +""", + ) + + _, stats = trace_yaml_tests(CountryTaxBenefitSystem(), [tmp_path]) + + assert (stats["outputs"], stats["outputErrors"]) == (2, 0) + + +# ----- Microdata ----- # + + +class rate_read_through_the_root(Variable): + value_type = float + entity = Person + definition_period = MONTH + label = "Reads a parameter through the system's root, not its argument" + + def formula(person, period, parameters): + root = person.simulation.tax_benefit_system.parameters + return person.filled_array(root(period).taxes.income_tax_rate) + + +class RootTracingMicrosimulation(Microsimulation): + """Hands its tracer to its own parameter root when tracing is switched on, + as policyengine-us's simulations do.""" + + @property + def trace(self) -> bool: + return Microsimulation.trace.fget(self) + + @trace.setter + def trace(self, trace: bool) -> None: + Microsimulation.trace.fset(self, trace) + root = self.tax_benefit_system.parameters + root.trace = bool(trace) + root.tracer = self.tracer + root.branch_name = self.branch_name + root.clear_parent_cache() + + +def build_root_tracing_microsimulation() -> RootTracingMicrosimulation: + system = CountryTaxBenefitSystem() + system.add_variable(rate_read_through_the_root) + return RootTracingMicrosimulation(tax_benefit_system=system) + + +def test_trace_microdata_records_reads_through_a_root_the_package_traces(): + (readers, _), _ = trace_microdata( + build_root_tracing_microsimulation, households=2, year=2022 + ) + + assert "rate_read_through_the_root" in readers["taxes.income_tax_rate"] + + +def test_trace_microdata_records_edges_of_every_calculated_variable(): + (readers, consumers), stats = trace_microdata( + Microsimulation, households=2, year=2022 + ) + + assert stats["variables"] > 0 + assert stats["failed"] == 0 + assert "income_tax" in readers["taxes.income_tax_rate"] + assert "income_tax" in consumers["salary"] + + +# ----- Merging and fingerprints ----- # + + +def test_merge_edges_unions_both_sides(): + readers, consumers = merge_edges( + ({"gov.a": {"x"}}, {"x": {"y"}}), + ({"gov.a": {"z"}, "gov.b": {"w"}}, {"x": {"q"}}), + ) + + assert readers == {"gov.a": {"x", "z"}, "gov.b": {"w"}} + assert consumers == {"x": {"y", "q"}} + + +@pytest.fixture +def package_root(tmp_path: Path) -> Path: + root = tmp_path / "country" + for relative_path, content in { + "parameters/rate.yaml": "values: {2017-01-01: 0.1}", + "variables/tax.py": "RATE = 1", + "model_api.py": "HELPER = 1", + "tools/helpers.py": "HELPER = 1", + "tests/tax.yaml": "[]", + }.items(): + file = root / relative_path + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text(content) + return root + + +@pytest.mark.parametrize( + "changed_file", + ["parameters/rate.yaml", "variables/tax.py", "model_api.py", "tools/helpers.py"], +) +def test_fingerprint_changes_with_any_model_file(package_root: Path, changed_file: str): + before = model_fingerprint(package_root) + + (package_root / changed_file).write_text("CHANGED = 2") + + assert before.startswith("sha256:") + assert model_fingerprint(package_root) != before + + +def test_fingerprint_changes_with_the_traced_tests(package_root: Path): + tests_root = package_root / "tests" + before = model_fingerprint(package_root, tests_root) + + (tests_root / "tax.yaml").write_text("- {}") + + assert model_fingerprint(package_root, tests_root) != before + + +def test_fingerprint_ignores_tests_that_were_not_traced(package_root: Path): + before = model_fingerprint(package_root) + + (package_root / "tests" / "tax.yaml").write_text("- {}") + + assert model_fingerprint(package_root) == before + + +def test_fingerprint_ignores_compiled_files(package_root: Path): + before = model_fingerprint(package_root) + cache_directory = package_root / "variables" / "__pycache__" + cache_directory.mkdir() + + (cache_directory / "tax.cpython-314.pyc").write_bytes(b"compiled") + + assert model_fingerprint(package_root) == before + + +def test_fingerprint_ignores_hidden_files(package_root: Path): + before = model_fingerprint(package_root) + + (package_root / "variables" / ".DS_Store").write_bytes(b"folder settings") + + assert model_fingerprint(package_root) == before + + +# ----- The map and the command ----- # + + +def test_build_and_write_dependency_map_for_the_country_template(tmp_path: Path): + payload = build_dependency_map(TEMPLATE_PACKAGE, tests_root=TESTS_ROOT) + output = write_dependency_map(payload, tmp_path / "map.json") + + written = json.loads(output.read_text()) + assert written["model"]["package"] == TEMPLATE_PACKAGE + assert written["model"]["fingerprint"].startswith("sha256:") + assert written["model"]["coreVersion"] + assert written["populations"]["tests"]["tests"] > 0 + assert "income_tax" in written["readers"]["taxes.income_tax_rate"] + assert "income_tax" in written["consumers"]["salary"] + + +def test_command_writes_the_map(tmp_path: Path, monkeypatch, capsys): + tests_directory = tmp_path / "tests" + tests_directory.mkdir() + write_tests( + tests_directory, + """ +- period: 2017-01 + input: {salary: 1000} + output: {income_tax: 150} +- period: 2017-01 + input: {salary: 2000} + output: {income_tax: 300} +""", + ) + output = tmp_path / "nested" / "map.json" + monkeypatch.setattr( + sys, + "argv", + [ + "policyengine-core", + "dependency-map", + "--country-package", + TEMPLATE_PACKAGE, + "--tests-root", + str(tests_directory), + "--every-test", + "--output", + str(output), + ], + ) + + with pytest.raises(SystemExit) as command_exit: + policyengine_command.main() + + assert command_exit.value.code == 0 + written = json.loads(output.read_text()) + assert written["populations"]["tests"]["everyTest"] is True + assert written["populations"]["tests"]["tests"] == 2 + assert written["readers"]["taxes.income_tax_rate"] == ["income_tax"] + assert "tests: 2" in capsys.readouterr().err + + +def run_command(monkeypatch, *arguments: str) -> int: + monkeypatch.setattr(sys, "argv", ["policyengine-core", *arguments]) + with pytest.raises(SystemExit) as command_exit: + policyengine_command.main() + return command_exit.value.code + + +def test_command_says_when_tests_were_skipped(tmp_path: Path, monkeypatch, capsys): + tests_directory = tmp_path / "tests" + tests_directory.mkdir() + write_tests( + tests_directory, + """ +- period: 2017-01 + input: {salary: 1000} + output: {income_tax: 150} +- period: 2017-01 + input: {salary: 2000} + output: {income_tax: 300} +""", + ) + + exit_code = run_command( + monkeypatch, + "dependency-map", + "--country-package", + TEMPLATE_PACKAGE, + "--tests-root", + str(tests_directory), + "--output", + str(tmp_path / "map.json"), + ) + + assert exit_code == 0 + assert "--every-test" in capsys.readouterr().err + + +def test_command_rejects_a_missing_tests_root(tmp_path: Path, monkeypatch, capsys): + exit_code = run_command( + monkeypatch, + "dependency-map", + "--country-package", + TEMPLATE_PACKAGE, + "--tests-root", + str(tmp_path / "missing"), + "--output", + str(tmp_path / "map.json"), + ) + + assert exit_code == 2 + assert "missing" in capsys.readouterr().err + assert not (tmp_path / "map.json").exists()