diff --git a/README.md b/README.md index 2fe9d1258..0a68b5c4a 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,7 @@ Full guide: `diff_diff.get_llm_guide("practitioner")`. - [DifferenceInDifferences](https://diff-diff.readthedocs.io/en/stable/api/estimators.html) - basic 2x2 DiD with robust/cluster-robust SEs, wild bootstrap, formula interface, and fixed effects - [TwoWayFixedEffects](https://diff-diff.readthedocs.io/en/stable/api/estimators.html) - panel data DiD with unit and time fixed effects via within-transformation or dummies; event-study mode (`event_study=True`) for period-specific treatment effects - [CallawaySantAnna](https://diff-diff.readthedocs.io/en/stable/api/staggered.html) - Callaway & Sant'Anna (2021) group-time ATT estimator for staggered adoption +- [ChainedDiD](https://diff-diff.readthedocs.io/en/stable/api/chained_did.html) - Bellégo, Benatia & Dortet-Bernadet (2025) chained IPW for overlapping incomplete panels; experimental, identity GMM and PT-post - [ChaisemartinDHaultfoeuille](https://diff-diff.readthedocs.io/en/stable/api/chaisemartin_dhaultfoeuille.html) - de Chaisemartin & D'Haultfœuille (2020/2022) for **reversible (non-absorbing) treatments** with multi-horizon event study, normalized effects, cost-benefit delta, sup-t bands, and dynamic placebos. The most general option for treatments that switch on AND off (see also `LPDiD`/`TROP` `non_absorbing`). Alias `DCDH`. - [SunAbraham](https://diff-diff.readthedocs.io/en/stable/api/staggered.html) - Sun & Abraham (2021) interaction-weighted estimator for heterogeneity-robust event studies - [ImputationDiD](https://diff-diff.readthedocs.io/en/stable/api/imputation.html) - Borusyak, Jaravel & Spiess (2024) imputation estimator, most efficient under homogeneous effects diff --git a/benchmarks/R/benchmark_chained_did.R b/benchmarks/R/benchmark_chained_did.R new file mode 100644 index 000000000..41b968160 --- /dev/null +++ b/benchmarks/R/benchmark_chained_did.R @@ -0,0 +1,66 @@ +#!/usr/bin/env Rscript +# External reference only; refuses unpinned environments and never installs packages. +# Usage: Rscript benchmarks/R/benchmark_chained_did.R input.csv output.json +# Input columns: id, t, g, y and optionally static x / cluster. + +args <- commandArgs(trailingOnly = TRUE) +if (length(args) != 2L) stop("Expected input.csv and output.json") +if (getRversion() != "4.5.2") stop("Reference requires R 4.5.2") +versions <- c(cdid = "1.1.0.0", did = "2.5.1", DRDID = "1.3.0") +for (package in names(versions)) { + if (!requireNamespace(package, quietly = TRUE)) stop(paste("Missing reference package", package)) + if (as.character(packageVersion(package)) != versions[[package]]) { + stop(paste("Unmatched reference version for", package)) + } +} +if (!requireNamespace("jsonlite", quietly = TRUE)) stop("Missing jsonlite") +sha <- packageDescription("cdid")[["RemoteSha"]] +if (is.null(sha) || sha != "1039120668cceabd8111cad82d32ab3927c2b94f") { + stop("cdid must record the pinned GitHub RemoteSha") +} +data <- read.csv(args[[1L]], na.strings = c("", "NA")) +stopifnot(all(c("id", "t", "g", "y") %in% names(data))) +data <- data[order(data$id, data$t), ] +frame_columns <- intersect(c("id", "g", "x", "cluster"), names(data)) +frame <- data[!duplicated(data$id), frame_columns, drop = FALSE] +fit <- cdid::att_gt( + yname = "y", tname = "t", idname = "id", gname = "g", data = data, frame = frame, + xformla = if ("x" %in% names(data)) ~ x else ~ 1, + control_group = "nevertreated", anticipation = 0, est_method = "ipw", + base_period = "universal", mode = "chained", selection = "none", + gmm_weighting = "identity", pt_assumption = "post", + bstrap = FALSE, cband = FALSE, + clustervars = if ("cluster" %in% names(data)) "cluster" else NULL +) +simple <- cdid::aggte(fit, type = "simple", bstrap = FALSE, cband = FALSE) +dynamic <- cdid::aggte(fit, type = "dynamic", bstrap = FALSE, cband = FALSE) +event_if <- dynamic$inf.function$dynamic.inf.func.e +stopifnot(is.matrix(event_if), nrow(event_if) == fit$n) +if ("cluster" %in% names(data)) { + positions <- match(rownames(event_if), as.character(frame$id)) + stopifnot(!anyNA(positions)) + event_if <- rowsum(event_if, frame$cluster[positions], reorder = FALSE) +} +event_vcov <- crossprod(event_if) / fit$n^2 +finite_se <- is.finite(dynamic$se.egt) +stopifnot(all(abs(sqrt(diag(event_vcov))[finite_se] - dynamic$se.egt[finite_se]) < 1e-8)) +calendar <- sort(unique(data$t)) +groups <- sort(unique(data$g[data$g > 0])) +references <- calendar[match(groups, calendar) - 1L] +event_reference <- vapply(dynamic$egt, function(e) { + included <- (fit$t - fit$group) == e + any(included) && all(fit$t[included] == references[match(fit$group[included], groups)]) +}, logical(1)) +answer <- list( + metadata = list(R = as.character(getRversion()), packages = as.list(versions), + cdid_sha = sha, n = fit$n, covariance_scale = "estimator", + cluster_correction = "none"), + cells = data.frame(group = fit$group, time = fit$t, att = fit$att, + available = is.finite(fit$att)), + vcov = fit$V_analytical / fit$n, + overall = list(att = simple$overall.att, se = simple$overall.se), + events = list(event_time = dynamic$egt, att = dynamic$att.egt, se = dynamic$se.egt, + is_reference = event_reference, vcov = event_vcov) +) +jsonlite::write_json(answer, args[[2L]], auto_unbox = TRUE, pretty = TRUE, + na = "null", digits = 17) diff --git a/benchmarks/R/inspect_chained_did_weights.R b/benchmarks/R/inspect_chained_did_weights.R new file mode 100644 index 000000000..26c86e396 --- /dev/null +++ b/benchmarks/R/inspect_chained_did_weights.R @@ -0,0 +1,45 @@ +#!/usr/bin/env Rscript +# Inspect public estimator-callback inputs for the first pair; never estimates ATT. +# Usage: Rscript benchmarks/R/inspect_chained_did_weights.R input.csv output.json +args <- commandArgs(trailingOnly = TRUE) +stopifnot(length(args) == 2L, getRversion() == "4.5.2", + as.character(packageVersion("cdid")) == "1.1.0.0", + packageDescription("cdid")[["RemoteSha"]] == "1039120668cceabd8111cad82d32ab3927c2b94f") +data <- read.csv(args[[1L]], na.strings = c("", "NA")) +stopifnot(all(c("id", "t", "g", "y", "x") %in% names(data)), + identical(sort(unique(data$t)), 0:3), + identical(sort(unique(data$g)), c(0L, 2L))) +observed_counts <- tapply(is.finite(data$y), data$id, sum) +frame <- data[!duplicated(data$id), c("id", "g", "x")] +eligible <- frame[observed_counts[as.character(frame$id)] >= 2, ] +eligible_counts <- table(factor(eligible$g > 0, levels = c(FALSE, TRUE))) +pair <- data[data$t %in% 0:1 & is.finite(data$y), ] +pair_ids <- names(which(table(pair$id) == 2L)) +pair_counts <- table(factor(frame$g[as.character(frame$id) %in% pair_ids] > 0, + levels = c(FALSE, TRUE))) +expected <- as.numeric(eligible_counts / pair_counts) +stopifnot(all(pair_counts > 0), all(is.finite(expected))) +inspected <- FALSE +probe <- function(y1, y0, D, covariates, i.weights, inffunc = TRUE, ...) { + counts <- table(factor(D, levels = 0:1)) + stopifnot(identical(as.integer(counts), as.integer(pair_counts)), + length(i.weights) == length(D), + all(abs(i.weights - expected[D + 1L]) < 1e-12)) + jsonlite::write_json(list( + status = "public_inputs_verified_no_estimate", pair = c(0L, 1L), + cohorts = c(0L, 2L), eligible_counts = as.integer(eligible_counts), + pair_counts = as.integer(counts), weights = expected, + max_abs_weight_error = max(abs(i.weights - expected[D + 1L])) + ), args[[2L]], auto_unbox = TRUE, pretty = TRUE, digits = 17) + inspected <<- TRUE + stop("PUBLIC_INPUT_INSPECTION_ONLY", call. = FALSE) +} +tryCatch(cdid::att_gt( + yname = "y", tname = "t", idname = "id", gname = "g", data = data, frame = frame, + xformla = ~ x, est_method = probe, control_group = "nevertreated", + base_period = "universal", mode = "chained", selection = "none", + gmm_weighting = "identity", pt_assumption = "post", bstrap = FALSE, cband = FALSE +), error = function(e) { + if (!identical(conditionMessage(e), "PUBLIC_INPUT_INSPECTION_ONLY")) stop(e) +}) +stopifnot(inspected) diff --git a/benchmarks/data/chained_did/calibration-diagnostics.json b/benchmarks/data/chained_did/calibration-diagnostics.json new file mode 100644 index 000000000..659b5676d --- /dev/null +++ b/benchmarks/data/chained_did/calibration-diagnostics.json @@ -0,0 +1,2928 @@ +{ + "status": "complete", + "mode": "diagnostic", + "replications_per_case": 2000, + "seed": 20261104, + "seed_derivation": "SeedSequence([seed,case_index,replication]).generate_state(1,dtype=uint64)[0]", + "python": "3.12.11", + "numpy": "2.3.5", + "pandas": "2.3.3", + "scipy": "1.15.3", + "baseline_sha256": "93b99edee3eedca4d4d421742d350d98eae2e07a1a7c03c4741f18bbb8edeeaf", + "source_hashes": { + "diff_diff/chained_did.py": "88de46fcee3edf3ff8710ae89fdd356c7cf726e0d21eec569cf6a736e6163e61", + "diff_diff/chained_did_results.py": "e88ee7515d8d1b27432e3e2681c055df386aedace3ed0d811cbaaf218534067a", + "benchmarks/python/validate_chained_did.py": "4b6466659badc98efc56e01bea482fa34c2b4ac3bac783a35a228bc727d097cd", + "benchmarks/python/diagnose_chained_did.py": "4813859170b69e778077cd897ce567c02e2861a33596f9492337232669b3bbfd" + }, + "cases": [ + { + "n": 1000, + "observation": "rotating", + "clusters": 0, + "null": false, + "violation": "none" + }, + { + "n": 1000, + "observation": "intermittent", + "clusters": 0, + "null": false, + "violation": "none" + }, + { + "n": 1000, + "observation": "intermittent", + "clusters": 50, + "null": false, + "violation": "none" + }, + { + "n": 1000, + "observation": "intermittent", + "clusters": 100, + "null": false, + "violation": "none" + }, + { + "n": 4000, + "observation": "rotating", + "clusters": 0, + "null": false, + "violation": "none" + }, + { + "n": 4000, + "observation": "intermittent", + "clusters": 0, + "null": false, + "violation": "none" + }, + { + "n": 4000, + "observation": "intermittent", + "clusters": 50, + "null": false, + "violation": "none" + }, + { + "n": 4000, + "observation": "intermittent", + "clusters": 100, + "null": false, + "violation": "none" + } + ], + "results": { + "0": { + "design": { + "n": 1000, + "observation": "rotating", + "clusters": 0, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 32.96549663600126, + "variants": { + "fitted_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.0029146760628591117, + "bias_mcse": 0.003537739601943199, + "rmse": 0.15819981895493654, + "empirical_sd": 0.15821252473276082, + "mean_variance": 0.02548044137686198, + "variance_ratio": 1.0179471356156926, + "coverage": 0.9485, + "coverage_mcse": 0.004942051699446293, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9485, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.6250760745353869, + "meets_prespecified_thresholds": true, + "z_mean": 0.01619129680362326, + "z_sd": 0.9955478340419069, + "z_quantiles": [ + -1.9970926364580177, + 0.03566884806839593, + 1.917419757575977 + ], + "abs_z_95": 1.9686659068010763 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": 0.0037359088176946997, + "bias_mcse": 0.004729036609900388, + "rmse": 0.21146907041691143, + "empirical_sd": 0.21148894655644845, + "mean_variance": 0.044805041952726436, + "variance_ratio": 1.0017319838602734, + "coverage": 0.9465, + "coverage_mcse": 0.005031786462082825, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9465, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.8286336834706661, + "meets_prespecified_thresholds": true, + "z_mean": 0.014597827751620295, + "z_sd": 1.0038786016690227, + "z_quantiles": [ + -1.9946585447756424, + 0.05948297915372776, + 1.937136449855952 + ], + "abs_z_95": 1.986481000881638 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.002543435478552958, + "bias_mcse": 0.005584718211096229, + "rmse": 0.24970669791024017, + "empirical_sd": 0.24975619110384378, + "mean_variance": 0.064431714999425, + "variance_ratio": 1.032921140500224, + "coverage": 0.954, + "coverage_mcse": 0.0046842288586276416, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.954, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.9930918302850396, + "meets_prespecified_thresholds": true, + "z_mean": 0.013229495534208342, + "z_sd": 0.9864870256458865, + "z_quantiles": [ + -1.8752377774187143, + -0.002554643120750151, + 1.9481004749357131 + ], + "abs_z_95": 1.9131670794122775 + } + } + }, + "fitted_known_mass": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.0030259573973330234, + "bias_mcse": 0.003541582023545102, + "rmse": 0.15837367228784155, + "empirical_sd": 0.1583843630507622, + "mean_variance": 0.025474940072678247, + "variance_ratio": 1.0155201993748486, + "coverage": 0.9485, + "coverage_mcse": 0.004942051699446293, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9485, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.6250098623161456, + "meets_prespecified_thresholds": true, + "z_mean": 0.017089181167654958, + "z_sd": 0.9965940630411788, + "z_quantiles": [ + -1.998533482038652, + 0.03464275767010549, + 1.9227859556125 + ], + "abs_z_95": 1.9720016698391352 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": 0.0037359088176946997, + "bias_mcse": 0.004729036609900388, + "rmse": 0.21146907041691143, + "empirical_sd": 0.21148894655644845, + "mean_variance": 0.044805041952726436, + "variance_ratio": 1.0017319838602734, + "coverage": 0.9465, + "coverage_mcse": 0.005031786462082825, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9465, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.8286336834706661, + "meets_prespecified_thresholds": true, + "z_mean": 0.014597827751620295, + "z_sd": 1.0038786016690227, + "z_quantiles": [ + -1.9946585447756424, + 0.05948297915372776, + 1.937136449855952 + ], + "abs_z_95": 1.986481000881638 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.002543435478552958, + "bias_mcse": 0.005584718211096229, + "rmse": 0.24970669791024017, + "empirical_sd": 0.24975619110384378, + "mean_variance": 0.064431714999425, + "variance_ratio": 1.032921140500224, + "coverage": 0.954, + "coverage_mcse": 0.0046842288586276416, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.954, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.9930918302850396, + "meets_prespecified_thresholds": true, + "z_mean": 0.013229495534208342, + "z_sd": 0.9864870256458865, + "z_quantiles": [ + -1.8752377774187143, + -0.002554643120750151, + 1.9481004749357131 + ], + "abs_z_95": 1.9131670794122775 + } + } + }, + "oracle_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.003183376529274806, + "bias_mcse": 0.0036378422951690408, + "rmse": 0.16267972559174096, + "empirical_sd": 0.1626892532684366, + "mean_variance": 0.026997996153495196, + "variance_ratio": 1.0200320072727151, + "coverage": 0.954, + "coverage_mcse": 0.0046842288586276416, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.954, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.6435613228442633, + "meets_prespecified_thresholds": true, + "z_mean": 0.02103919275586552, + "z_sd": 0.9951696136894894, + "z_quantiles": [ + -1.9414463730001215, + 0.043914787447553345, + 1.9151428734585485 + ], + "abs_z_95": 1.9206004711884523 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": 0.004947001690276842, + "bias_mcse": 0.0048803355266480615, + "rmse": 0.21825674068902745, + "empirical_sd": 0.21825523981184605, + "mean_variance": 0.047534316868503096, + "variance_ratio": 0.9978790365254306, + "coverage": 0.95, + "coverage_mcse": 0.004873397172404485, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.95, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.8537640907327703, + "meets_prespecified_thresholds": true, + "z_mean": 0.02488583537347737, + "z_sd": 1.006066653364704, + "z_quantiles": [ + -1.962496317832081, + 0.021413339466298552, + 1.9437052740016447 + ], + "abs_z_95": 1.9599915255227338 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.00019999564220762878, + "bias_mcse": 0.005768105080076158, + "rmse": 0.2578930813179037, + "empirical_sd": 0.2579575012082432, + "mean_variance": 0.06919474824261422, + "variance_ratio": 1.0398646407627277, + "coverage": 0.955, + "coverage_mcse": 0.004635461142108735, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.955, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 1.029264710158617, + "meets_prespecified_thresholds": true, + "z_mean": 0.0009347532281608935, + "z_sd": 0.9845760207473208, + "z_quantiles": [ + -1.9314984125734693, + 0.0012466891259965615, + 1.873597165737457 + ], + "abs_z_95": 1.904735162402783 + } + } + }, + "oracle_known_mass": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.0033059206088492665, + "bias_mcse": 0.0036412321999318327, + "rmse": 0.16283370169218359, + "empirical_sd": 0.16284085441817367, + "mean_variance": 0.026992182530492143, + "variance_ratio": 1.017914397765553, + "coverage": 0.9545, + "coverage_mcse": 0.004659922209651143, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9545, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.6434892325296591, + "meets_prespecified_thresholds": true, + "z_mean": 0.02196735773914431, + "z_sd": 0.9961083835628586, + "z_quantiles": [ + -1.945322550466994, + 0.039714362029927494, + 1.9096762012455997 + ], + "abs_z_95": 1.9236968131721002 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": 0.004947001690276842, + "bias_mcse": 0.0048803355266480615, + "rmse": 0.21825674068902745, + "empirical_sd": 0.21825523981184605, + "mean_variance": 0.047534316868503096, + "variance_ratio": 0.9978790365254306, + "coverage": 0.95, + "coverage_mcse": 0.004873397172404485, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.95, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.8537640907327703, + "meets_prespecified_thresholds": true, + "z_mean": 0.02488583537347737, + "z_sd": 1.006066653364704, + "z_quantiles": [ + -1.962496317832081, + 0.021413339466298552, + 1.9437052740016447 + ], + "abs_z_95": 1.9599915255227338 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.00019999564220762878, + "bias_mcse": 0.005768105080076158, + "rmse": 0.2578930813179037, + "empirical_sd": 0.2579575012082432, + "mean_variance": 0.06919474824261422, + "variance_ratio": 1.0398646407627277, + "coverage": 0.955, + "coverage_mcse": 0.004635461142108735, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.955, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 1.029264710158617, + "meets_prespecified_thresholds": true, + "z_mean": 0.0009347532281608935, + "z_sd": 0.9845760207473208, + "z_quantiles": [ + -1.9314984125734693, + 0.0012466891259965615, + 1.873597165737457 + ], + "abs_z_95": 1.904735162402783 + } + } + } + } + }, + "3": { + "design": { + "n": 1000, + "observation": "intermittent", + "clusters": 100, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 53.02578066800015, + "variants": { + "fitted_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": -0.0025149293317148214, + "bias_mcse": 0.0026685640992813796, + "rmse": 0.11933847803260451, + "empirical_sd": 0.11934181456617325, + "mean_variance": 0.01480608628669308, + "variance_ratio": 1.0395730258891147, + "coverage": 0.9525, + "coverage_mcse": 0.00475624589355933, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9525, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.4749430567522107, + "meets_prespecified_thresholds": true, + "z_mean": -0.020731835262152217, + "z_sd": 0.9918557380653712, + "z_quantiles": [ + -1.9776449765835338, + -0.029092336987483258, + 1.874920564906888 + ], + "abs_z_95": 1.918842383700929 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.004869671236213513, + "bias_mcse": 0.004025130934708902, + "rmse": 0.18003019219566588, + "empirical_sd": 0.1800093277669275, + "mean_variance": 0.03166544925750358, + "variance_ratio": 0.9772273965030068, + "coverage": 0.9445, + "coverage_mcse": 0.005119558086397692, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9445, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.6942004772201725, + "meets_prespecified_thresholds": true, + "z_mean": -0.026335262425984073, + "z_sd": 1.025218129933998, + "z_quantiles": [ + -2.001213874653426, + -0.028180140566554483, + 1.980162160015353 + ], + "abs_z_95": 1.9987050630846217 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.0019139948998709855, + "bias_mcse": 0.004673756551351729, + "rmse": 0.20897325182611734, + "empirical_sd": 0.20901674718214908, + "mean_variance": 0.046387550147674934, + "variance_ratio": 1.0617915562129767, + "coverage": 0.9555, + "coverage_mcse": 0.004610843198374891, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9555, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.8401316937630906, + "meets_prespecified_thresholds": true, + "z_mean": 0.008432956352084756, + "z_sd": 0.9744038825554272, + "z_quantiles": [ + -1.8730787370395503, + 0.005287199761039089, + 1.9531524847889858 + ], + "abs_z_95": 1.9143968978194517 + } + } + }, + "fitted_known_mass": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": -0.00256061406054231, + "bias_mcse": 0.0026677851910438502, + "rmse": 0.11930463242325189, + "empirical_sd": 0.11930698073082625, + "mean_variance": 0.014790652320954238, + "variance_ratio": 1.0390958679593518, + "coverage": 0.9525, + "coverage_mcse": 0.00475624589355933, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9525, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.47470650745346554, + "meets_prespecified_thresholds": true, + "z_mean": -0.020886683520945026, + "z_sd": 0.9919650564764679, + "z_quantiles": [ + -1.9848939936706353, + -0.030530910410796856, + 1.892772126793461 + ], + "abs_z_95": 1.9318051189242593 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.004869671236213513, + "bias_mcse": 0.004025130934708902, + "rmse": 0.18003019219566588, + "empirical_sd": 0.1800093277669275, + "mean_variance": 0.03166544925750358, + "variance_ratio": 0.9772273965030068, + "coverage": 0.9445, + "coverage_mcse": 0.005119558086397692, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9445, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.6942004772201725, + "meets_prespecified_thresholds": true, + "z_mean": -0.026335262425984073, + "z_sd": 1.025218129933998, + "z_quantiles": [ + -2.001213874653426, + -0.028180140566554483, + 1.980162160015353 + ], + "abs_z_95": 1.9987050630846217 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.0019139948998709855, + "bias_mcse": 0.004673756551351729, + "rmse": 0.20897325182611734, + "empirical_sd": 0.20901674718214908, + "mean_variance": 0.046387550147674934, + "variance_ratio": 1.0617915562129767, + "coverage": 0.9555, + "coverage_mcse": 0.004610843198374891, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9555, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.8401316937630906, + "meets_prespecified_thresholds": true, + "z_mean": 0.008432956352084756, + "z_sd": 0.9744038825554272, + "z_quantiles": [ + -1.8730787370395503, + 0.005287199761039089, + 1.9531524847889858 + ], + "abs_z_95": 1.9143968978194517 + } + } + }, + "oracle_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": -0.0017030692279513815, + "bias_mcse": 0.0027987980418265335, + "rmse": 0.12514634685910866, + "empirical_sd": 0.12516605353634858, + "mean_variance": 0.01642515557141092, + "variance_ratio": 1.0484225979166155, + "coverage": 0.952, + "coverage_mcse": 0.004779958158812692, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.952, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.5003638218450758, + "meets_prespecified_thresholds": true, + "z_mean": -0.007996311048417649, + "z_sd": 0.9900247206886272, + "z_quantiles": [ + -1.8986002582012564, + -0.017144424405890746, + 1.9546678658663372 + ], + "abs_z_95": 1.936212051840079 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.003896880343776221, + "bias_mcse": 0.004145078403936092, + "rmse": 0.18536815786696945, + "empirical_sd": 0.1853735416653487, + "mean_variance": 0.03402540397349233, + "variance_ratio": 0.9901655113206791, + "coverage": 0.944, + "coverage_mcse": 0.005141206084179083, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.944, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.7198555162954151, + "meets_prespecified_thresholds": true, + "z_mean": -0.013774959920486031, + "z_sd": 1.019496963695303, + "z_quantiles": [ + -1.994555834939413, + -0.027275626005121168, + 2.0246033585298786 + ], + "abs_z_95": 2.0073725431610856 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.0026681322009447506, + "bias_mcse": 0.004891138934561238, + "rmse": 0.21869996763540284, + "empirical_sd": 0.2187383829014965, + "mean_variance": 0.05090337499844652, + "variance_ratio": 1.0638896494417711, + "coverage": 0.9565, + "coverage_mcse": 0.004561126505590477, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9565, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.8802856745829886, + "meets_prespecified_thresholds": true, + "z_mean": 0.008713438003032738, + "z_sd": 0.9733038412679058, + "z_quantiles": [ + -1.8524560782041561, + 0.02538020388208782, + 1.945643468942334 + ], + "abs_z_95": 1.9082603196478667 + } + } + }, + "oracle_known_mass": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": -0.0017433798054193872, + "bias_mcse": 0.0027982091314744223, + "rmse": 0.12512057419040307, + "empirical_sd": 0.1251397166647491, + "mean_variance": 0.016409479375313254, + "variance_ratio": 1.0478629083251698, + "coverage": 0.9535, + "coverage_mcse": 0.004708383480558906, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9535, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.5001376273946685, + "meets_prespecified_thresholds": true, + "z_mean": -0.008135109583997074, + "z_sd": 0.9900587147564457, + "z_quantiles": [ + -1.9079961623641792, + -0.010247336893443088, + 1.9507529930912666 + ], + "abs_z_95": 1.9361491765322427 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.003896880343776221, + "bias_mcse": 0.004145078403936092, + "rmse": 0.18536815786696945, + "empirical_sd": 0.1853735416653487, + "mean_variance": 0.03402540397349233, + "variance_ratio": 0.9901655113206791, + "coverage": 0.944, + "coverage_mcse": 0.005141206084179083, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.944, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.7198555162954151, + "meets_prespecified_thresholds": true, + "z_mean": -0.013774959920486031, + "z_sd": 1.019496963695303, + "z_quantiles": [ + -1.994555834939413, + -0.027275626005121168, + 2.0246033585298786 + ], + "abs_z_95": 2.0073725431610856 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.0026681322009447506, + "bias_mcse": 0.004891138934561238, + "rmse": 0.21869996763540284, + "empirical_sd": 0.2187383829014965, + "mean_variance": 0.05090337499844652, + "variance_ratio": 1.0638896494417711, + "coverage": 0.9565, + "coverage_mcse": 0.004561126505590477, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9565, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.8802856745829886, + "meets_prespecified_thresholds": true, + "z_mean": 0.008713438003032738, + "z_sd": 0.9733038412679058, + "z_quantiles": [ + -1.8524560782041561, + 0.02538020388208782, + 1.945643468942334 + ], + "abs_z_95": 1.9082603196478667 + } + } + } + } + }, + "1": { + "design": { + "n": 1000, + "observation": "intermittent", + "clusters": 0, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 56.56108488599966, + "variants": { + "fitted_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.0038502899836587723, + "bias_mcse": 0.0018087742321090262, + "rmse": 0.08096222275986943, + "empirical_sd": 0.08089084277891531, + "mean_variance": 0.006821387644678014, + "variance_ratio": 1.0424950698274607, + "coverage": 0.9535, + "coverage_mcse": 0.004708383480558906, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9535, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.3234980295396818, + "meets_prespecified_thresholds": true, + "z_mean": 0.04264632084674333, + "z_sd": 0.9807226822101789, + "z_quantiles": [ + -1.791614448491489, + 0.0415234558460061, + 2.0480601922549546 + ], + "abs_z_95": 1.8960737457440524 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": 0.0037132837110687156, + "bias_mcse": 0.00254964171329706, + "rmse": 0.11405539678341152, + "empirical_sd": 0.11402344378402511, + "mean_variance": 0.013849139066155933, + "variance_ratio": 1.0652081216236327, + "coverage": 0.9595, + "coverage_mcse": 0.004407933189148855, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9595, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.46086409563765246, + "meets_prespecified_thresholds": true, + "z_mean": 0.0262964781696574, + "z_sd": 0.9722298154003931, + "z_quantiles": [ + -1.8490836749853938, + 0.044089938299558123, + 1.9078321098262603 + ], + "abs_z_95": 1.885665260849771 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.005760587029643105, + "bias_mcse": 0.0032524048804903555, + "rmse": 0.14552965759148348, + "empirical_sd": 0.1454519680625703, + "mean_variance": 0.02073993117833356, + "variance_ratio": 0.9803205510100352, + "coverage": 0.9455, + "coverage_mcse": 0.005075911248239078, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9455, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.5636893053122438, + "meets_prespecified_thresholds": true, + "z_mean": 0.03976631150705458, + "z_sd": 1.016861057402839, + "z_quantiles": [ + -1.918077609476687, + 0.026450912552794605, + 2.0724258736253325 + ], + "abs_z_95": 2.015434123607845 + } + } + }, + "fitted_known_mass": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.0038474909800245705, + "bias_mcse": 0.0018096816425669131, + "rmse": 0.0810026143657184, + "empirical_sd": 0.0809314234082619, + "mean_variance": 0.006814827870043217, + "variance_ratio": 1.0404483679857681, + "coverage": 0.9535, + "coverage_mcse": 0.004708383480558906, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9535, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.32334617146437394, + "meets_prespecified_thresholds": true, + "z_mean": 0.04300284329177084, + "z_sd": 0.9818667109410084, + "z_quantiles": [ + -1.7910894443607082, + 0.046862104360125764, + 2.0556267612925256 + ], + "abs_z_95": 1.884365780620705 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": 0.0037132837110687156, + "bias_mcse": 0.00254964171329706, + "rmse": 0.11405539678341152, + "empirical_sd": 0.11402344378402511, + "mean_variance": 0.013849139066155933, + "variance_ratio": 1.0652081216236327, + "coverage": 0.9595, + "coverage_mcse": 0.004407933189148855, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9595, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.46086409563765246, + "meets_prespecified_thresholds": true, + "z_mean": 0.0262964781696574, + "z_sd": 0.9722298154003931, + "z_quantiles": [ + -1.8490836749853938, + 0.044089938299558123, + 1.9078321098262603 + ], + "abs_z_95": 1.885665260849771 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.005760587029643105, + "bias_mcse": 0.0032524048804903555, + "rmse": 0.14552965759148348, + "empirical_sd": 0.1454519680625703, + "mean_variance": 0.02073993117833356, + "variance_ratio": 0.9803205510100352, + "coverage": 0.9455, + "coverage_mcse": 0.005075911248239078, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9455, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.5636893053122438, + "meets_prespecified_thresholds": true, + "z_mean": 0.03976631150705458, + "z_sd": 1.016861057402839, + "z_quantiles": [ + -1.918077609476687, + 0.026450912552794605, + 2.0724258736253325 + ], + "abs_z_95": 2.015434123607845 + } + } + }, + "oracle_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.0036069392506119513, + "bias_mcse": 0.00200616990387939, + "rmse": 0.08976870646019833, + "empirical_sd": 0.0897186455897707, + "mean_variance": 0.008449592561246685, + "variance_ratio": 1.0497124551680999, + "coverage": 0.952, + "coverage_mcse": 0.004779958158812692, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.952, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.36011738237006574, + "meets_prespecified_thresholds": true, + "z_mean": 0.041176198545197724, + "z_sd": 0.9774823020040947, + "z_quantiles": [ + -1.7989822294361333, + 0.03185424138790459, + 2.032603032430075 + ], + "abs_z_95": 1.931473010700731 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": 0.003837675177040367, + "bias_mcse": 0.0027582177539337356, + "rmse": 0.12338010512970428, + "empirical_sd": 0.12335124789085242, + "mean_variance": 0.016252727981082807, + "variance_ratio": 1.0681670372684424, + "coverage": 0.9555, + "coverage_mcse": 0.004610843198374891, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9555, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.49939041083844316, + "meets_prespecified_thresholds": true, + "z_mean": 0.031742006385064546, + "z_sd": 0.9712016989767963, + "z_quantiles": [ + -1.8611379471402747, + 0.021991442837164085, + 1.9702866137541017 + ], + "abs_z_95": 1.929629628418509 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.003718003971571182, + "bias_mcse": 0.003631419665291511, + "rmse": 0.16240398368167874, + "empirical_sd": 0.16240202452842706, + "mean_variance": 0.025341924132980115, + "variance_ratio": 0.960852464886669, + "coverage": 0.9425, + "coverage_mcse": 0.005205465877325487, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9425, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.6231070476777781, + "meets_prespecified_thresholds": true, + "z_mean": 0.017620968466916933, + "z_sd": 1.0264070575266115, + "z_quantiles": [ + -1.9699360023683254, + 0.023593378953064376, + 2.030156537791508 + ], + "abs_z_95": 2.0281618247913062 + } + } + }, + "oracle_known_mass": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.0035980841939058372, + "bias_mcse": 0.0020072517605680563, + "rmse": 0.0898166822524817, + "empirical_sd": 0.08976702769172612, + "mean_variance": 0.008442850180622734, + "variance_ratio": 1.0477445055816441, + "coverage": 0.952, + "coverage_mcse": 0.004779958158812692, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.952, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.359978058136079, + "meets_prespecified_thresholds": true, + "z_mean": 0.041429644484179164, + "z_sd": 0.978583875546786, + "z_quantiles": [ + -1.7812065875521323, + 0.034175088622831376, + 2.031099709428121 + ], + "abs_z_95": 1.9369178321238663 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": 0.003837675177040367, + "bias_mcse": 0.0027582177539337356, + "rmse": 0.12338010512970428, + "empirical_sd": 0.12335124789085242, + "mean_variance": 0.016252727981082807, + "variance_ratio": 1.0681670372684424, + "coverage": 0.9555, + "coverage_mcse": 0.004610843198374891, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9555, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.49939041083844316, + "meets_prespecified_thresholds": true, + "z_mean": 0.031742006385064546, + "z_sd": 0.9712016989767963, + "z_quantiles": [ + -1.8611379471402747, + 0.021991442837164085, + 1.9702866137541017 + ], + "abs_z_95": 1.929629628418509 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.003718003971571182, + "bias_mcse": 0.003631419665291511, + "rmse": 0.16240398368167874, + "empirical_sd": 0.16240202452842706, + "mean_variance": 0.025341924132980115, + "variance_ratio": 0.960852464886669, + "coverage": 0.9425, + "coverage_mcse": 0.005205465877325487, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9425, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.6231070476777781, + "meets_prespecified_thresholds": true, + "z_mean": 0.017620968466916933, + "z_sd": 1.0264070575266115, + "z_quantiles": [ + -1.9699360023683254, + 0.023593378953064376, + 2.030156537791508 + ], + "abs_z_95": 2.0281618247913062 + } + } + } + } + }, + "4": { + "design": { + "n": 4000, + "observation": "rotating", + "clusters": 0, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 47.5333055709998, + "variants": { + "fitted_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": -0.002922585114471854, + "bias_mcse": 0.001800744243586566, + "rmse": 0.08056462303977369, + "empirical_sd": 0.08053173077502003, + "mean_variance": 0.006337215309984757, + "variance_ratio": 0.9771571108828004, + "coverage": 0.946, + "coverage_mcse": 0.005053909377897473, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.946, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.31197664799092795, + "meets_prespecified_thresholds": true, + "z_mean": -0.03859939155178525, + "z_sd": 1.0116936205885396, + "z_quantiles": [ + -2.0678334977697332, + -0.07512956655712202, + 1.9484815331660101 + ], + "abs_z_95": 2.003998804683684 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.00411163416687943, + "bias_mcse": 0.002370283187836084, + "rmse": 0.10605551411505433, + "empirical_sd": 0.10600228667852774, + "mean_variance": 0.011120793789648375, + "variance_ratio": 0.9897039871737077, + "coverage": 0.949, + "coverage_mcse": 0.004919298730510276, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.949, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.41324729654874515, + "meets_prespecified_thresholds": true, + "z_mean": -0.04141734438401964, + "z_sd": 1.0057200251254073, + "z_quantiles": [ + -2.019264665035074, + -0.08293233711523289, + 1.9532855916427214 + ], + "abs_z_95": 1.9667736306131802 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": -0.001940947030648399, + "bias_mcse": 0.002810722904829157, + "rmse": 0.12568290897170767, + "empirical_sd": 0.12569934962227333, + "mean_variance": 0.015924448796133396, + "variance_ratio": 1.0078556794827331, + "coverage": 0.9475, + "coverage_mcse": 0.004987171041783106, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9475, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.494445544759917, + "meets_prespecified_thresholds": true, + "z_mean": -0.014273009418878255, + "z_sd": 0.9968023415121349, + "z_quantiles": [ + -1.979962173389047, + -0.011221635570841427, + 1.9452754947895499 + ], + "abs_z_95": 1.9784820745006455 + } + } + }, + "fitted_known_mass": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": -0.0029318058264209412, + "bias_mcse": 0.0017996853196846037, + "rmse": 0.0805176447614465, + "empirical_sd": 0.08048437425846429, + "mean_variance": 0.006336039660221045, + "variance_ratio": 0.978125864905332, + "coverage": 0.9455, + "coverage_mcse": 0.005075911248239078, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9455, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.31194896794768684, + "meets_prespecified_thresholds": true, + "z_mean": -0.0386118015017661, + "z_sd": 1.011327469676155, + "z_quantiles": [ + -2.067983910088427, + -0.0738564549079985, + 1.9530937477533692 + ], + "abs_z_95": 2.004088248561374 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.00411163416687943, + "bias_mcse": 0.002370283187836084, + "rmse": 0.10605551411505433, + "empirical_sd": 0.10600228667852774, + "mean_variance": 0.011120793789648375, + "variance_ratio": 0.9897039871737077, + "coverage": 0.949, + "coverage_mcse": 0.004919298730510276, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.949, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.41324729654874515, + "meets_prespecified_thresholds": true, + "z_mean": -0.04141734438401964, + "z_sd": 1.0057200251254073, + "z_quantiles": [ + -2.019264665035074, + -0.08293233711523289, + 1.9532855916427214 + ], + "abs_z_95": 1.9667736306131802 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": -0.001940947030648399, + "bias_mcse": 0.002810722904829157, + "rmse": 0.12568290897170767, + "empirical_sd": 0.12569934962227333, + "mean_variance": 0.015924448796133396, + "variance_ratio": 1.0078556794827331, + "coverage": 0.9475, + "coverage_mcse": 0.004987171041783106, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9475, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.494445544759917, + "meets_prespecified_thresholds": true, + "z_mean": -0.014273009418878255, + "z_sd": 0.9968023415121349, + "z_quantiles": [ + -1.979962173389047, + -0.011221635570841427, + 1.9452754947895499 + ], + "abs_z_95": 1.9784820745006455 + } + } + }, + "oracle_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": -0.0026639867537183726, + "bias_mcse": 0.0018600763629368113, + "rmse": 0.08320700146778817, + "empirical_sd": 0.08318514381734561, + "mean_variance": 0.006769313516533537, + "variance_ratio": 0.9782572721981512, + "coverage": 0.9425, + "coverage_mcse": 0.005205465877325487, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9425, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.32245133231040707, + "meets_prespecified_thresholds": true, + "z_mean": -0.03244507065614135, + "z_sd": 1.0111613697035786, + "z_quantiles": [ + -2.045293374335892, + -0.0708907899840203, + 1.9776211096746124 + ], + "abs_z_95": 2.004557341043065 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.0038279950702624355, + "bias_mcse": 0.002461574658726723, + "rmse": 0.11012399283275483, + "empirical_sd": 0.11008496537207599, + "mean_variance": 0.011913155055040505, + "variance_ratio": 0.9830390592473972, + "coverage": 0.946, + "coverage_mcse": 0.005053909377897473, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.946, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.42774298033413694, + "meets_prespecified_thresholds": true, + "z_mean": -0.03486132636012149, + "z_sd": 1.009491201599124, + "z_quantiles": [ + -1.9651408395148664, + -0.06494145950536734, + 2.0225349998962514 + ], + "abs_z_95": 1.9835693853354612 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": -0.002110522973433504, + "bias_mcse": 0.0029228477800883083, + "rmse": 0.13069808552236004, + "empirical_sd": 0.1307137264832363, + "mean_variance": 0.01729487249519137, + "variance_ratio": 1.0122201362126149, + "coverage": 0.9535, + "coverage_mcse": 0.004708383480558906, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9535, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.5152869292058432, + "meets_prespecified_thresholds": true, + "z_mean": -0.016438291252537658, + "z_sd": 0.9946828976300535, + "z_quantiles": [ + -1.9591054656644178, + -0.021527921275002505, + 1.9215774855069243 + ], + "abs_z_95": 1.9436821388471777 + } + } + }, + "oracle_known_mass": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": -0.0026719758201689814, + "bias_mcse": 0.0018587813281646306, + "rmse": 0.08314938627930198, + "empirical_sd": 0.08312722810166917, + "mean_variance": 0.006768099186449548, + "variance_ratio": 0.979445141983997, + "coverage": 0.941, + "coverage_mcse": 0.005268728499363013, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.941, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.32242366034466363, + "meets_prespecified_thresholds": true, + "z_mean": -0.03245543709229485, + "z_sd": 1.0107039572512453, + "z_quantiles": [ + -2.0444723469530452, + -0.06784445660940504, + 1.977987276373411 + ], + "abs_z_95": 2.0035755387129885 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.0038279950702624355, + "bias_mcse": 0.002461574658726723, + "rmse": 0.11012399283275483, + "empirical_sd": 0.11008496537207599, + "mean_variance": 0.011913155055040505, + "variance_ratio": 0.9830390592473972, + "coverage": 0.946, + "coverage_mcse": 0.005053909377897473, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.946, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.42774298033413694, + "meets_prespecified_thresholds": true, + "z_mean": -0.03486132636012149, + "z_sd": 1.009491201599124, + "z_quantiles": [ + -1.9651408395148664, + -0.06494145950536734, + 2.0225349998962514 + ], + "abs_z_95": 1.9835693853354612 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": -0.002110522973433504, + "bias_mcse": 0.0029228477800883083, + "rmse": 0.13069808552236004, + "empirical_sd": 0.1307137264832363, + "mean_variance": 0.01729487249519137, + "variance_ratio": 1.0122201362126149, + "coverage": 0.9535, + "coverage_mcse": 0.004708383480558906, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9535, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.5152869292058432, + "meets_prespecified_thresholds": true, + "z_mean": -0.016438291252537658, + "z_sd": 0.9946828976300535, + "z_quantiles": [ + -1.9591054656644178, + -0.021527921275002505, + 1.9215774855069243 + ], + "abs_z_95": 1.9436821388471777 + } + } + } + } + }, + "2": { + "design": { + "n": 1000, + "observation": "intermittent", + "clusters": 50, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 81.91162499600068, + "variants": { + "fitted_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": -0.00012124921433587232, + "bias_mcse": 0.002703173419589671, + "rmse": 0.12085942506725667, + "empirical_sd": 0.12088959042346131, + "mean_variance": 0.014562458389544742, + "variance_ratio": 0.9964531515175283, + "coverage": 0.948, + "coverage_mcse": 0.0049646752159632785, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.948, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.4693512337737541, + "meets_prespecified_thresholds": true, + "z_mean": -0.0017930613862680777, + "z_sd": 1.023517162420456, + "z_quantiles": [ + -1.9708025403458247, + -0.02971026037732693, + 1.9892935975673482 + ], + "abs_z_95": 1.9839708902429192 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.0042802093320677415, + "bias_mcse": 0.004000025093515098, + "rmse": 0.17889304483036367, + "empirical_sd": 0.17888656041609427, + "mean_variance": 0.031230652197068862, + "variance_ratio": 0.975945636166715, + "coverage": 0.9455, + "coverage_mcse": 0.005075911248239078, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9455, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.6866593105342641, + "meets_prespecified_thresholds": true, + "z_mean": -0.022084086075359947, + "z_sd": 1.0270311190059236, + "z_quantiles": [ + -2.026144779957991, + -0.045192504648564454, + 1.960100981665705 + ], + "abs_z_95": 2.0010704428432344 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.003343022568029319, + "bias_mcse": 0.00465329178369762, + "rmse": 0.2080763598391746, + "empirical_sd": 0.20810153494978254, + "mean_variance": 0.04503104086608863, + "variance_ratio": 1.0398277861393341, + "coverage": 0.9495, + "coverage_mcse": 0.0048964145045124596, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9495, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.8244351111876297, + "meets_prespecified_thresholds": true, + "z_mean": 0.01628711182053067, + "z_sd": 1.000211266557684, + "z_quantiles": [ + -1.937452997850081, + 0.005795482145052417, + 1.9825342247508821 + ], + "abs_z_95": 1.9631623754872107 + } + } + }, + "fitted_known_mass": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": -7.067812108862981e-05, + "bias_mcse": 0.0027028435115192054, + "rmse": 0.12084463466729493, + "empirical_sd": 0.1208748364860236, + "mean_variance": 0.01455993937404678, + "variance_ratio": 0.9965240111173036, + "coverage": 0.9455, + "coverage_mcse": 0.005075911248239078, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9455, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.4693060826736608, + "meets_prespecified_thresholds": true, + "z_mean": -0.0013933620255721926, + "z_sd": 1.024268328051068, + "z_quantiles": [ + -1.9773073170193567, + -0.030386817270340448, + 1.9888032091084926 + ], + "abs_z_95": 1.9877399995263818 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.0042802093320677415, + "bias_mcse": 0.004000025093515098, + "rmse": 0.17889304483036367, + "empirical_sd": 0.17888656041609427, + "mean_variance": 0.031230652197068862, + "variance_ratio": 0.975945636166715, + "coverage": 0.9455, + "coverage_mcse": 0.005075911248239078, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9455, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.6866593105342641, + "meets_prespecified_thresholds": true, + "z_mean": -0.022084086075359947, + "z_sd": 1.0270311190059236, + "z_quantiles": [ + -2.026144779957991, + -0.045192504648564454, + 1.960100981665705 + ], + "abs_z_95": 2.0010704428432344 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.003343022568029319, + "bias_mcse": 0.00465329178369762, + "rmse": 0.2080763598391746, + "empirical_sd": 0.20810153494978254, + "mean_variance": 0.04503104086608863, + "variance_ratio": 1.0398277861393341, + "coverage": 0.9495, + "coverage_mcse": 0.0048964145045124596, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9495, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.8244351111876297, + "meets_prespecified_thresholds": true, + "z_mean": 0.01628711182053067, + "z_sd": 1.000211266557684, + "z_quantiles": [ + -1.937452997850081, + 0.005795482145052417, + 1.9825342247508821 + ], + "abs_z_95": 1.9631623754872107 + } + } + }, + "oracle_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.00019204280881268687, + "bias_mcse": 0.0028549915774942237, + "rmse": 0.12764732554448122, + "empirical_sd": 0.12767910484932887, + "mean_variance": 0.01618241240325009, + "variance_ratio": 0.9926670500216417, + "coverage": 0.95, + "coverage_mcse": 0.004873397172404485, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.95, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.49499488203666286, + "meets_prespecified_thresholds": true, + "z_mean": 0.004538568503581486, + "z_sd": 1.0275843360471713, + "z_quantiles": [ + -1.9419401067702977, + -0.008660774358968788, + 2.007768497494872 + ], + "abs_z_95": 1.9575428652483167 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.004157445794874981, + "bias_mcse": 0.004135027481493961, + "rmse": 0.1849245533959328, + "empirical_sd": 0.18492405074900503, + "mean_variance": 0.033570452966704564, + "variance_ratio": 0.9816810443213079, + "coverage": 0.945, + "coverage_mcse": 0.005097793640389931, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.945, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.7123624214446113, + "meets_prespecified_thresholds": true, + "z_mean": -0.016583025284785682, + "z_sd": 1.0234300907351794, + "z_quantiles": [ + -1.9788807207675922, + -0.04518325403701322, + 2.0101811302122914 + ], + "abs_z_95": 1.9979365434380332 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.005036011089840198, + "bias_mcse": 0.004907553987861658, + "rmse": 0.21947539636393631, + "empirical_sd": 0.21947248640217693, + "mean_variance": 0.049474526229312324, + "variance_ratio": 1.0271206873692433, + "coverage": 0.9495, + "coverage_mcse": 0.0048964145045124596, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9495, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.8645706082751747, + "meets_prespecified_thresholds": true, + "z_mean": 0.020099867633920538, + "z_sd": 1.0067933104505238, + "z_quantiles": [ + -1.927319052356681, + 0.012705592071491604, + 2.030805514893061 + ], + "abs_z_95": 1.9633793692623944 + } + } + }, + "oracle_known_mass": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.00025624426133408207, + "bias_mcse": 0.0028547902037089116, + "rmse": 0.12763843483997686, + "empirical_sd": 0.12767009913987198, + "mean_variance": 0.016181183563851995, + "variance_ratio": 0.9927317077421068, + "coverage": 0.9475, + "coverage_mcse": 0.004987171041783106, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9475, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.4949754921849046, + "meets_prespecified_thresholds": true, + "z_mean": 0.00506136119177252, + "z_sd": 1.0283070957397389, + "z_quantiles": [ + -1.9486023457360477, + -0.010953520605389627, + 2.0104610969765315 + ], + "abs_z_95": 2.0037671528544654 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.004157445794874981, + "bias_mcse": 0.004135027481493961, + "rmse": 0.1849245533959328, + "empirical_sd": 0.18492405074900503, + "mean_variance": 0.033570452966704564, + "variance_ratio": 0.9816810443213079, + "coverage": 0.945, + "coverage_mcse": 0.005097793640389931, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.945, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.7123624214446113, + "meets_prespecified_thresholds": true, + "z_mean": -0.016583025284785682, + "z_sd": 1.0234300907351794, + "z_quantiles": [ + -1.9788807207675922, + -0.04518325403701322, + 2.0101811302122914 + ], + "abs_z_95": 1.9979365434380332 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.005036011089840198, + "bias_mcse": 0.004907553987861658, + "rmse": 0.21947539636393631, + "empirical_sd": 0.21947248640217693, + "mean_variance": 0.049474526229312324, + "variance_ratio": 1.0271206873692433, + "coverage": 0.9495, + "coverage_mcse": 0.0048964145045124596, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9495, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.8645706082751747, + "meets_prespecified_thresholds": true, + "z_mean": 0.020099867633920538, + "z_sd": 1.0067933104505238, + "z_quantiles": [ + -1.927319052356681, + 0.012705592071491604, + 2.030805514893061 + ], + "abs_z_95": 1.9633793692623944 + } + } + } + } + }, + "5": { + "design": { + "n": 4000, + "observation": "intermittent", + "clusters": 0, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 76.64435981099996, + "variants": { + "fitted_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": -0.0002213731529505658, + "bias_mcse": 0.0009330124160114228, + "rmse": 0.041715738406114304, + "empirical_sd": 0.04172558372105709, + "mean_variance": 0.0017105024437493407, + "variance_ratio": 0.9824690026053284, + "coverage": 0.956, + "coverage_mcse": 0.004586065852122058, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.956, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.1620865616374868, + "meets_prespecified_thresholds": true, + "z_mean": -0.006661524562352927, + "z_sd": 1.0095609133876569, + "z_quantiles": [ + -1.929252881218562, + 0.016963889928061297, + 1.8980345115718018 + ], + "abs_z_95": 1.9167043265013204 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": 0.0003581010273694396, + "bias_mcse": 0.0013217772390806245, + "rmse": 0.05909798034433367, + "empirical_sd": 0.05911167515392537, + "mean_variance": 0.0034721211538900148, + "variance_ratio": 0.9936840913825254, + "coverage": 0.9495, + "coverage_mcse": 0.0048964145045124596, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9495, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.23092168162493987, + "meets_prespecified_thresholds": true, + "z_mean": 0.004394094016094513, + "z_sd": 1.003732078382579, + "z_quantiles": [ + -2.0036316915636982, + 0.017831587538138904, + 1.9349559753221484 + ], + "abs_z_95": 1.9601985456843893 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.0011525477847267318, + "bias_mcse": 0.001630443668841427, + "rmse": 0.07290653697967693, + "empirical_sd": 0.07291565754027174, + "mean_variance": 0.0051700590845501935, + "variance_ratio": 0.9724200688621182, + "coverage": 0.949, + "coverage_mcse": 0.004919298730510276, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.949, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.2817469466796198, + "meets_prespecified_thresholds": true, + "z_mean": 0.017162463357015507, + "z_sd": 1.0149390770546611, + "z_quantiles": [ + -1.993820075365317, + 0.0046983764265315486, + 1.9451304449399052 + ], + "abs_z_95": 1.962874015675631 + } + } + }, + "fitted_known_mass": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": -0.00021545438693593155, + "bias_mcse": 0.000932666258576734, + "rmse": 0.041700230862886226, + "empirical_sd": 0.04171010308995947, + "mean_variance": 0.0017095029936234165, + "variance_ratio": 0.9826239363348527, + "coverage": 0.9555, + "coverage_mcse": 0.004610843198374891, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9555, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.1620400640896885, + "meets_prespecified_thresholds": true, + "z_mean": -0.006322140508336625, + "z_sd": 1.0093485246798948, + "z_quantiles": [ + -1.9312281103218445, + 0.02529678779551625, + 1.8981935519795123 + ], + "abs_z_95": 1.92136600429808 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": 0.0003581010273694396, + "bias_mcse": 0.0013217772390806245, + "rmse": 0.05909798034433367, + "empirical_sd": 0.05911167515392537, + "mean_variance": 0.0034721211538900148, + "variance_ratio": 0.9936840913825254, + "coverage": 0.9495, + "coverage_mcse": 0.0048964145045124596, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9495, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.23092168162493987, + "meets_prespecified_thresholds": true, + "z_mean": 0.004394094016094513, + "z_sd": 1.003732078382579, + "z_quantiles": [ + -2.0036316915636982, + 0.017831587538138904, + 1.9349559753221484 + ], + "abs_z_95": 1.9601985456843893 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.0011525477847267318, + "bias_mcse": 0.001630443668841427, + "rmse": 0.07290653697967693, + "empirical_sd": 0.07291565754027174, + "mean_variance": 0.0051700590845501935, + "variance_ratio": 0.9724200688621182, + "coverage": 0.949, + "coverage_mcse": 0.004919298730510276, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.949, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.2817469466796198, + "meets_prespecified_thresholds": true, + "z_mean": 0.017162463357015507, + "z_sd": 1.0149390770546611, + "z_quantiles": [ + -1.993820075365317, + 0.0046983764265315486, + 1.9451304449399052 + ], + "abs_z_95": 1.962874015675631 + } + } + }, + "oracle_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": -0.0004641140485324158, + "bias_mcse": 0.0010406734592242964, + "rmse": 0.046531010069297205, + "empirical_sd": 0.046540331944107644, + "mean_variance": 0.0021271676769727098, + "variance_ratio": 0.9820707406660802, + "coverage": 0.953, + "coverage_mcse": 0.004732388403332932, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.953, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.18076424266382657, + "meets_prespecified_thresholds": true, + "z_mean": -0.008300073252410764, + "z_sd": 1.0095141744318443, + "z_quantiles": [ + -2.011687181225445, + -0.0050148722899276445, + 1.8843489629866321 + ], + "abs_z_95": 1.9355134684009176 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": 0.0003150496223911503, + "bias_mcse": 0.001435705974970718, + "rmse": 0.06419144255857819, + "empirical_sd": 0.06420672311474275, + "mean_variance": 0.0040932650526356975, + "variance_ratio": 0.9929076489650729, + "coverage": 0.9495, + "coverage_mcse": 0.0048964145045124596, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9495, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.2507426882224669, + "meets_prespecified_thresholds": true, + "z_mean": 0.006889655317884701, + "z_sd": 1.0040493661474441, + "z_quantiles": [ + -2.0240919296978017, + 0.01709520991371951, + 1.8977894333747103 + ], + "abs_z_95": 1.9706311452571288 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.0003919171889337214, + "bias_mcse": 0.001794204440339499, + "rmse": 0.08022015692441145, + "empirical_sd": 0.08023926188262172, + "mean_variance": 0.006348028518006113, + "variance_ratio": 0.9859729928179762, + "coverage": 0.95, + "coverage_mcse": 0.004873397172404485, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.95, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.31220691127392497, + "meets_prespecified_thresholds": true, + "z_mean": 0.0031732646602695633, + "z_sd": 1.0080461769397973, + "z_quantiles": [ + -1.9864390272331114, + 0.016920921011748597, + 1.9225828120121298 + ], + "abs_z_95": 1.9548158707252898 + } + } + }, + "oracle_known_mass": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": -0.00046459524283948284, + "bias_mcse": 0.0010404914229849135, + "rmse": 0.046522876404503456, + "empirical_sd": 0.046532191035995076, + "mean_variance": 0.002126172478729981, + "variance_ratio": 0.9819547776615118, + "coverage": 0.953, + "coverage_mcse": 0.004732388403332932, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.953, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.18072199321954222, + "meets_prespecified_thresholds": true, + "z_mean": -0.00810392254004001, + "z_sd": 1.0093503688864813, + "z_quantiles": [ + -2.0145580405472643, + -0.006319262783343017, + 1.8942203830893074 + ], + "abs_z_95": 1.9364519712734052 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": 0.0003150496223911503, + "bias_mcse": 0.001435705974970718, + "rmse": 0.06419144255857819, + "empirical_sd": 0.06420672311474275, + "mean_variance": 0.0040932650526356975, + "variance_ratio": 0.9929076489650729, + "coverage": 0.9495, + "coverage_mcse": 0.0048964145045124596, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9495, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.2507426882224669, + "meets_prespecified_thresholds": true, + "z_mean": 0.006889655317884701, + "z_sd": 1.0040493661474441, + "z_quantiles": [ + -2.0240919296978017, + 0.01709520991371951, + 1.8977894333747103 + ], + "abs_z_95": 1.9706311452571288 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.0003919171889337214, + "bias_mcse": 0.001794204440339499, + "rmse": 0.08022015692441145, + "empirical_sd": 0.08023926188262172, + "mean_variance": 0.006348028518006113, + "variance_ratio": 0.9859729928179762, + "coverage": 0.95, + "coverage_mcse": 0.004873397172404485, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.95, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.31220691127392497, + "meets_prespecified_thresholds": true, + "z_mean": 0.0031732646602695633, + "z_sd": 1.0080461769397973, + "z_quantiles": [ + -1.9864390272331114, + 0.016920921011748597, + 1.9225828120121298 + ], + "abs_z_95": 1.9548158707252898 + } + } + } + } + }, + "6": { + "design": { + "n": 4000, + "observation": "intermittent", + "clusters": 50, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 82.11885297100162, + "variants": { + "fitted_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.00018192713146536477, + "bias_mcse": 0.0013617505592732509, + "rmse": 0.06088438144760531, + "empirical_sd": 0.060899336378666916, + "mean_variance": 0.0036737815071382364, + "variance_ratio": 0.9905769166177878, + "coverage": 0.9385, + "coverage_mcse": 0.005372045699731156, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9385, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.23566265393704838, + "meets_prespecified_thresholds": true, + "z_mean": 0.007127870142645677, + "z_sd": 1.0249962577043739, + "z_quantiles": [ + -1.9971856853580425, + -0.0053203675749210425, + 2.090630036010773 + ], + "abs_z_95": 2.062470425233165 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.0013319738391825986, + "bias_mcse": 0.0019715702343719135, + "rmse": 0.08815931852754905, + "empirical_sd": 0.08817130132941584, + "mean_variance": 0.007885006058343082, + "variance_ratio": 1.014255870502318, + "coverage": 0.944, + "coverage_mcse": 0.005141206084179083, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.944, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.3450658092630556, + "meets_prespecified_thresholds": true, + "z_mean": -0.015131448827056707, + "z_sd": 1.0148595386484116, + "z_quantiles": [ + -2.008019945695321, + -0.03647449225192627, + 2.035507797736078 + ], + "abs_z_95": 2.0355225405478268 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.002838268883863737, + "bias_mcse": 0.002470418395682394, + "rmse": 0.11048930673444977, + "empirical_sd": 0.11048046931223612, + "mean_variance": 0.01130798509959159, + "variance_ratio": 0.9264334058709462, + "coverage": 0.935, + "coverage_mcse": 0.005512485827646179, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.935, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.41311638063323186, + "meets_prespecified_thresholds": false, + "z_mean": 0.03236004991896569, + "z_sd": 1.0632161929252744, + "z_quantiles": [ + -2.000647390168091, + 0.014586002079121977, + 2.169150597097301 + ], + "abs_z_95": 2.078378362182321 + } + } + }, + "fitted_known_mass": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.00016036306159783464, + "bias_mcse": 0.0013603617851728428, + "rmse": 0.0608222287099236, + "empirical_sd": 0.060837228512788846, + "mean_variance": 0.0036711253474601855, + "variance_ratio": 0.9918828265486881, + "coverage": 0.9395, + "coverage_mcse": 0.005331029450303197, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9395, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.23558564741242874, + "meets_prespecified_thresholds": true, + "z_mean": 0.00696568125367279, + "z_sd": 1.0244327284724641, + "z_quantiles": [ + -2.003267311564591, + -0.0011759417560417728, + 2.104392828158784 + ], + "abs_z_95": 2.0596725512663103 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.0013319738391825986, + "bias_mcse": 0.0019715702343719135, + "rmse": 0.08815931852754905, + "empirical_sd": 0.08817130132941584, + "mean_variance": 0.007885006058343082, + "variance_ratio": 1.014255870502318, + "coverage": 0.944, + "coverage_mcse": 0.005141206084179083, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.944, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.3450658092630556, + "meets_prespecified_thresholds": true, + "z_mean": -0.015131448827056707, + "z_sd": 1.0148595386484116, + "z_quantiles": [ + -2.008019945695321, + -0.03647449225192627, + 2.035507797736078 + ], + "abs_z_95": 2.0355225405478268 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.002838268883863737, + "bias_mcse": 0.002470418395682394, + "rmse": 0.11048930673444977, + "empirical_sd": 0.11048046931223612, + "mean_variance": 0.01130798509959159, + "variance_ratio": 0.9264334058709462, + "coverage": 0.935, + "coverage_mcse": 0.005512485827646179, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.935, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.41311638063323186, + "meets_prespecified_thresholds": false, + "z_mean": 0.03236004991896569, + "z_sd": 1.0632161929252744, + "z_quantiles": [ + -2.000647390168091, + 0.014586002079121977, + 2.169150597097301 + ], + "abs_z_95": 2.078378362182321 + } + } + }, + "oracle_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.0008904373114595465, + "bias_mcse": 0.001433966848940566, + "rmse": 0.06411909594966893, + "empirical_sd": 0.06412894703424556, + "mean_variance": 0.0040973707411079, + "variance_ratio": 0.996315859909275, + "coverage": 0.947, + "coverage_mcse": 0.005009540897128201, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.947, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.24899711985755513, + "meets_prespecified_thresholds": true, + "z_mean": 0.02049923616253911, + "z_sd": 1.0187660004855887, + "z_quantiles": [ + -1.9470649779351943, + -0.012331907033251857, + 2.0219413430005972 + ], + "abs_z_95": 1.9868611166905108 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.000297915466755061, + "bias_mcse": 0.0020517409587033487, + "rmse": 0.09173418684386744, + "empirical_sd": 0.09175664511762553, + "mean_variance": 0.00850870364955285, + "variance_ratio": 1.0106210633075634, + "coverage": 0.9435, + "coverage_mcse": 0.005162739098579358, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9435, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.3585971314909699, + "meets_prespecified_thresholds": true, + "z_mean": -0.0010180504625501765, + "z_sd": 1.015895935328483, + "z_quantiles": [ + -1.9430609842832656, + -0.019769431654416865, + 2.03717170710839 + ], + "abs_z_95": 1.999481245796861 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.0027559669567192024, + "bias_mcse": 0.002586754845740877, + "rmse": 0.11568710091087243, + "empirical_sd": 0.11568319352407168, + "mean_variance": 0.012490904809133502, + "variance_ratio": 0.9333689738483172, + "coverage": 0.9395, + "coverage_mcse": 0.005331029450303197, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9395, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.43444342834767524, + "meets_prespecified_thresholds": true, + "z_mean": 0.02883556261586344, + "z_sd": 1.0569563668774526, + "z_quantiles": [ + -2.023526565558136, + 0.03269503431015777, + 2.061829994333442 + ], + "abs_z_95": 2.0492688741032046 + } + } + }, + "oracle_known_mass": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.000866933220617927, + "bias_mcse": 0.001432608548886126, + "rmse": 0.06405804956941705, + "empirical_sd": 0.06406820200913417, + "mean_variance": 0.0040944512544176526, + "variance_ratio": 0.9974947821244424, + "coverage": 0.9465, + "coverage_mcse": 0.005031786462082825, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9465, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.2489172134391213, + "meets_prespecified_thresholds": true, + "z_mean": 0.020335599618993314, + "z_sd": 1.0182593816748293, + "z_quantiles": [ + -1.9334091179348598, + -0.0123494275199014, + 2.0100644362809907 + ], + "abs_z_95": 1.9841012505692175 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.000297915466755061, + "bias_mcse": 0.0020517409587033487, + "rmse": 0.09173418684386744, + "empirical_sd": 0.09175664511762553, + "mean_variance": 0.00850870364955285, + "variance_ratio": 1.0106210633075634, + "coverage": 0.9435, + "coverage_mcse": 0.005162739098579358, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9435, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.3585971314909699, + "meets_prespecified_thresholds": true, + "z_mean": -0.0010180504625501765, + "z_sd": 1.015895935328483, + "z_quantiles": [ + -1.9430609842832656, + -0.019769431654416865, + 2.03717170710839 + ], + "abs_z_95": 1.999481245796861 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.0027559669567192024, + "bias_mcse": 0.002586754845740877, + "rmse": 0.11568710091087243, + "empirical_sd": 0.11568319352407168, + "mean_variance": 0.012490904809133502, + "variance_ratio": 0.9333689738483172, + "coverage": 0.9395, + "coverage_mcse": 0.005331029450303197, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9395, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.43444342834767524, + "meets_prespecified_thresholds": true, + "z_mean": 0.02883556261586344, + "z_sd": 1.0569563668774526, + "z_quantiles": [ + -2.023526565558136, + 0.03269503431015777, + 2.061829994333442 + ], + "abs_z_95": 2.0492688741032046 + } + } + } + } + }, + "7": { + "design": { + "n": 4000, + "observation": "intermittent", + "clusters": 100, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 73.97967386599885, + "variants": { + "fitted_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": -0.001469540346653364, + "bias_mcse": 0.0013673712014518945, + "rmse": 0.06115306901495524, + "empirical_sd": 0.06115069913843991, + "mean_variance": 0.003671319436711743, + "variance_ratio": 0.981791618268179, + "coverage": 0.948, + "coverage_mcse": 0.0049646752159632785, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.948, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.23659376896069872, + "meets_prespecified_thresholds": true, + "z_mean": -0.02483619798044315, + "z_sd": 1.0189919522366797, + "z_quantiles": [ + -1.9857186271991465, + -0.025927159204843425, + 1.9721302585263962 + ], + "abs_z_95": 1.9857103512008656 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.00043435935861957375, + "bias_mcse": 0.001943335381757669, + "rmse": 0.08688795617131802, + "empirical_sd": 0.08690860033381305, + "mean_variance": 0.007899755614413955, + "variance_ratio": 1.0458951399537801, + "coverage": 0.9515, + "coverage_mcse": 0.004803527349771207, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9515, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.3468713466265063, + "meets_prespecified_thresholds": true, + "z_mean": -0.008774361457307249, + "z_sd": 0.9861174059744037, + "z_quantiles": [ + -1.9607332659394545, + 0.009795186969391455, + 1.939233764248359 + ], + "abs_z_95": 1.9449608006491637 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": -0.001323242684776016, + "bias_mcse": 0.0024756999420163703, + "rmse": 0.11069689375406898, + "empirical_sd": 0.11071666724481785, + "mean_variance": 0.011521845278845028, + "variance_ratio": 0.9399311233332549, + "coverage": 0.9335, + "coverage_mcse": 0.005571254347092763, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9335, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.41890669742173664, + "meets_prespecified_thresholds": false, + "z_mean": -0.01305740498670007, + "z_sd": 1.038617658989081, + "z_quantiles": [ + -2.132982799334231, + -0.017059061859617475, + 2.0310653103726115 + ], + "abs_z_95": 2.0833041331428745 + } + } + }, + "fitted_known_mass": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": -0.0014307850170696135, + "bias_mcse": 0.0013675398399958635, + "rmse": 0.06115968777309871, + "empirical_sd": 0.06115824088339874, + "mean_variance": 0.003670297792935488, + "variance_ratio": 0.9812763507488598, + "coverage": 0.9465, + "coverage_mcse": 0.005031786462082825, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9465, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.2365592215035422, + "meets_prespecified_thresholds": true, + "z_mean": -0.024204993197790303, + "z_sd": 1.0192655170288794, + "z_quantiles": [ + -1.986245763000655, + -0.023949647802102854, + 1.974929234111895 + ], + "abs_z_95": 1.9862666093806802 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.00043435935861957375, + "bias_mcse": 0.001943335381757669, + "rmse": 0.08688795617131802, + "empirical_sd": 0.08690860033381305, + "mean_variance": 0.007899755614413955, + "variance_ratio": 1.0458951399537801, + "coverage": 0.9515, + "coverage_mcse": 0.004803527349771207, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9515, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.3468713466265063, + "meets_prespecified_thresholds": true, + "z_mean": -0.008774361457307249, + "z_sd": 0.9861174059744037, + "z_quantiles": [ + -1.9607332659394545, + 0.009795186969391455, + 1.939233764248359 + ], + "abs_z_95": 1.9449608006491637 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": -0.001323242684776016, + "bias_mcse": 0.0024756999420163703, + "rmse": 0.11069689375406898, + "empirical_sd": 0.11071666724481785, + "mean_variance": 0.011521845278845028, + "variance_ratio": 0.9399311233332549, + "coverage": 0.9335, + "coverage_mcse": 0.005571254347092763, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9335, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.41890669742173664, + "meets_prespecified_thresholds": false, + "z_mean": -0.01305740498670007, + "z_sd": 1.038617658989081, + "z_quantiles": [ + -2.132982799334231, + -0.017059061859617475, + 2.0310653103726115 + ], + "abs_z_95": 2.0833041331428745 + } + } + }, + "oracle_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": -0.0017824454920902788, + "bias_mcse": 0.0014546662302085697, + "rmse": 0.06506280612384764, + "empirical_sd": 0.0650546515063944, + "mean_variance": 0.004091956089672656, + "variance_ratio": 0.966883736554866, + "coverage": 0.942, + "coverage_mcse": 0.005226662414964259, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.942, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.249844062188883, + "meets_prespecified_thresholds": true, + "z_mean": -0.026585492765094414, + "z_sd": 1.028185641108731, + "z_quantiles": [ + -1.9868578001538886, + -0.033097858981691444, + 2.028174699138671 + ], + "abs_z_95": 2.001959774401261 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.000885099942075751, + "bias_mcse": 0.002024590822620607, + "rmse": 0.09052414281277453, + "empirical_sd": 0.09054245412003793, + "mean_variance": 0.008520950428816189, + "variance_ratio": 1.0394019215095969, + "coverage": 0.9475, + "coverage_mcse": 0.004987171041783106, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9475, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.3603360383275399, + "meets_prespecified_thresholds": true, + "z_mean": -0.011311023123201366, + "z_sd": 0.9906386350409169, + "z_quantiles": [ + -1.9832494548818687, + -0.00866598237431854, + 1.9478193829145114 + ], + "abs_z_95": 1.9827965147705784 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": -0.0015868420500573244, + "bias_mcse": 0.002586210964437145, + "rmse": 0.11564084004230112, + "empirical_sd": 0.11565887041273495, + "mean_variance": 0.012684856757376503, + "variance_ratio": 0.9482605309702311, + "coverage": 0.9365, + "coverage_mcse": 0.005452877680637995, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9365, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.43967289600370807, + "meets_prespecified_thresholds": true, + "z_mean": -0.015271100249943324, + "z_sd": 1.0357236141953605, + "z_quantiles": [ + -2.156827834500411, + -0.0035819833131153483, + 2.0250535712237885 + ], + "abs_z_95": 2.0986089411809195 + } + } + }, + "oracle_known_mass": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": -0.0017444164798834372, + "bias_mcse": 0.0014550888767894345, + "rmse": 0.06508066520809855, + "empirical_sd": 0.06507355283609983, + "mean_variance": 0.004090907029888498, + "variance_ratio": 0.9660743968937833, + "coverage": 0.9435, + "coverage_mcse": 0.005162739098579358, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9435, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.24981179744392576, + "meets_prespecified_thresholds": true, + "z_mean": -0.02599462245068922, + "z_sd": 1.0286965846017497, + "z_quantiles": [ + -1.986540843679838, + -0.026672032660459685, + 2.0326680106878845 + ], + "abs_z_95": 2.0078004492809503 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.000885099942075751, + "bias_mcse": 0.002024590822620607, + "rmse": 0.09052414281277453, + "empirical_sd": 0.09054245412003793, + "mean_variance": 0.008520950428816189, + "variance_ratio": 1.0394019215095969, + "coverage": 0.9475, + "coverage_mcse": 0.004987171041783106, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9475, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.3603360383275399, + "meets_prespecified_thresholds": true, + "z_mean": -0.011311023123201366, + "z_sd": 0.9906386350409169, + "z_quantiles": [ + -1.9832494548818687, + -0.00866598237431854, + 1.9478193829145114 + ], + "abs_z_95": 1.9827965147705784 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": -0.0015868420500573244, + "bias_mcse": 0.002586210964437145, + "rmse": 0.11564084004230112, + "empirical_sd": 0.11565887041273495, + "mean_variance": 0.012684856757376503, + "variance_ratio": 0.9482605309702311, + "coverage": 0.9365, + "coverage_mcse": 0.005452877680637995, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9365, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.43967289600370807, + "meets_prespecified_thresholds": true, + "z_mean": -0.015271100249943324, + "z_sd": 1.0357236141953605, + "z_quantiles": [ + -2.156827834500411, + -0.0035819833131153483, + 2.0250535712237885 + ], + "abs_z_95": 2.0986089411809195 + } + } + } + } + } + }, + "acceptance_rule": "zero failures; |bias|<.1 empirical SD; variance ratio [.9,1.1]; 95% coverage and null 5% rejection within 3 binomial MCSE", + "interpretation": "Oracle variants are diagnostics, not public estimator options. Reference agreement is a separate report." +} diff --git a/benchmarks/data/chained_did/qualification-followup.json b/benchmarks/data/chained_did/qualification-followup.json new file mode 100644 index 000000000..f4e82b08e --- /dev/null +++ b/benchmarks/data/chained_did/qualification-followup.json @@ -0,0 +1,1392 @@ +{ + "status": "complete", + "mode": "qualification", + "replications_per_case": 10000, + "seed": 20261103, + "seed_derivation": "SeedSequence([seed,case_index,replication]).generate_state(1,dtype=uint64)[0]", + "python": "3.12.11", + "numpy": "2.3.5", + "pandas": "2.3.3", + "scipy": "1.15.3", + "baseline_sha256": "93b99edee3eedca4d4d421742d350d98eae2e07a1a7c03c4741f18bbb8edeeaf", + "source_hashes": { + "diff_diff/chained_did.py": "88de46fcee3edf3ff8710ae89fdd356c7cf726e0d21eec569cf6a736e6163e61", + "diff_diff/chained_did_results.py": "e88ee7515d8d1b27432e3e2681c055df386aedace3ed0d811cbaaf218534067a", + "benchmarks/python/validate_chained_did.py": "4b6466659badc98efc56e01bea482fa34c2b4ac3bac783a35a228bc727d097cd", + "benchmarks/python/diagnose_chained_did.py": "4813859170b69e778077cd897ce567c02e2861a33596f9492337232669b3bbfd" + }, + "cases": [ + { + "n": 1000, + "observation": "balanced", + "clusters": 0, + "null": false, + "violation": "none" + }, + { + "n": 1000, + "observation": "rotating", + "clusters": 0, + "null": false, + "violation": "none" + }, + { + "n": 1000, + "observation": "intermittent", + "clusters": 50, + "null": false, + "violation": "none" + }, + { + "n": 1000, + "observation": "cohort_dependent", + "clusters": 0, + "null": false, + "violation": "none" + }, + { + "n": 1000, + "observation": "rotating", + "clusters": 0, + "null": true, + "violation": "none" + }, + { + "n": 4000, + "observation": "balanced", + "clusters": 0, + "null": false, + "violation": "none" + }, + { + "n": 4000, + "observation": "rotating", + "clusters": 0, + "null": false, + "violation": "none" + }, + { + "n": 4000, + "observation": "intermittent", + "clusters": 100, + "null": false, + "violation": "none" + }, + { + "n": 4000, + "observation": "cohort_dependent", + "clusters": 0, + "null": false, + "violation": "none" + }, + { + "n": 4000, + "observation": "rotating", + "clusters": 0, + "null": true, + "violation": "none" + }, + { + "n": 1000, + "observation": "intermittent", + "clusters": 0, + "null": false, + "violation": "selection" + }, + { + "n": 1000, + "observation": "intermittent", + "clusters": 0, + "null": false, + "violation": "parallel_trends" + }, + { + "n": 1000, + "observation": "intermittent", + "clusters": 0, + "null": false, + "violation": "propensity" + } + ], + "results": { + "3": { + "design": { + "n": 1000, + "observation": "cohort_dependent", + "clusters": 0, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 131.27786515799926, + "variants": { + "fitted_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 10000, + "failures": 0, + "bias": 0.004217254834744999, + "bias_mcse": 0.001745545520803128, + "rmse": 0.1745967638907853, + "empirical_sd": 0.1745545520803128, + "mean_variance": 0.029079516938461086, + "variance_ratio": 0.9543876920614853, + "coverage": 0.9454, + "coverage_mcse": 0.002271977992851163, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9454, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.6676352734042855, + "meets_prespecified_thresholds": true, + "z_mean": 0.02209782280897322, + "z_sd": 1.0273884100455624, + "z_quantiles": [ + -1.986126903111886, + 0.028931993501330126, + 2.0210605676074924 + ], + "abs_z_95": 2.0032940903149226 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 10000, + "failures": 0, + "bias": 0.0032704880534337732, + "bias_mcse": 0.0022051287522764727, + "rmse": 0.22052610194817635, + "empirical_sd": 0.22051287522764726, + "mean_variance": 0.04668849540750063, + "variance_ratio": 0.9601563855390314, + "coverage": 0.941, + "coverage_mcse": 0.0023562470159132305, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.941, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.8456498407186579, + "meets_prespecified_thresholds": false, + "z_mean": 0.011333702000538036, + "z_sd": 1.0245803284812915, + "z_quantiles": [ + -2.0223754935463965, + 0.010442176888611876, + 2.0521386021621484 + ], + "abs_z_95": 2.042569120147097 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 10000, + "failures": 0, + "bias": 0.004943153549368695, + "bias_mcse": 0.0029441246611644372, + "rmse": 0.2944392417830176, + "empirical_sd": 0.2944124661164437, + "mean_variance": 0.08498386690819941, + "variance_ratio": 0.9804469461059877, + "coverage": 0.9492, + "coverage_mcse": 0.0021958907076628374, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9492, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 1.1395155204752805, + "meets_prespecified_thresholds": true, + "z_mean": 0.019682535941848107, + "z_sd": 1.0158717959631522, + "z_quantiles": [ + -1.9268240063517061, + 0.009972758474586644, + 2.000759693478983 + ], + "abs_z_95": 1.9677695357735312 + } + } + } + } + }, + "1": { + "design": { + "n": 1000, + "observation": "rotating", + "clusters": 0, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 140.00030349999906, + "variants": { + "fitted_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 10000, + "failures": 0, + "bias": -0.001323499260753369, + "bias_mcse": 0.001620844583904231, + "rmse": 0.16208175765595845, + "empirical_sd": 0.1620844583904231, + "mean_variance": 0.025458163078809923, + "variance_ratio": 0.9690458273862632, + "coverage": 0.947, + "coverage_mcse": 0.0022403347964087875, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.947, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.624807583586931, + "meets_prespecified_thresholds": true, + "z_mean": -0.010894583108481197, + "z_sd": 1.0180993535357588, + "z_quantiles": [ + -1.993208797189272, + -0.013347859393126224, + 1.9720070364102893 + ], + "abs_z_95": 1.9894837158337104 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 10000, + "failures": 0, + "bias": -0.00025078982128379224, + "bias_mcse": 0.002144539432324312, + "rmse": 0.21444336691559499, + "empirical_sd": 0.21445394323243122, + "mean_variance": 0.044746184443885964, + "variance_ratio": 0.972944206028066, + "coverage": 0.9456, + "coverage_mcse": 0.0022680529094357567, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9456, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.8280869064583192, + "meets_prespecified_thresholds": true, + "z_mean": -0.0048702029687784035, + "z_sd": 1.017196308315831, + "z_quantiles": [ + -2.000114149540391, + -0.002980728664950169, + 2.0155492378922517 + ], + "abs_z_95": 2.0077499032341724 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 10000, + "failures": 0, + "bias": -0.003082104431259669, + "bias_mcse": 0.00255637814690246, + "rmse": 0.25564361247271516, + "empirical_sd": 0.255637814690246, + "mean_variance": 0.0642804683179382, + "variance_ratio": 0.9836233719336921, + "coverage": 0.9474, + "coverage_mcse": 0.0022323359962156232, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9474, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.9919701571612728, + "meets_prespecified_thresholds": true, + "z_mean": -0.010047504946419853, + "z_sd": 1.0141812636118788, + "z_quantiles": [ + -2.040061187483886, + -0.01644035411304571, + 1.9379583737238693 + ], + "abs_z_95": 1.9855505837981042 + } + } + } + } + }, + "2": { + "design": { + "n": 1000, + "observation": "intermittent", + "clusters": 50, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 194.71834558699993, + "variants": { + "fitted_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 10000, + "failures": 0, + "bias": 0.0005264341135986506, + "bias_mcse": 0.0012214187439244836, + "rmse": 0.12213690166869545, + "empirical_sd": 0.12214187439244835, + "mean_variance": 0.014653363380809688, + "variance_ratio": 0.9822186108051235, + "coverage": 0.9425, + "coverage_mcse": 0.002327955111251074, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9425, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.4706646388773877, + "meets_prespecified_thresholds": false, + "z_mean": 0.0017917739094081753, + "z_sd": 1.0275380804430727, + "z_quantiles": [ + -2.0310746956361627, + 0.019137275742775496, + 2.011454875445718 + ], + "abs_z_95": 2.020078150619346 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 10000, + "failures": 0, + "bias": 0.0005472209231043301, + "bias_mcse": 0.0017954690925307854, + "rmse": 0.1795387656296257, + "empirical_sd": 0.17954690925307853, + "mean_variance": 0.03142110763441152, + "variance_ratio": 0.9746880093226397, + "coverage": 0.939, + "coverage_mcse": 0.0023933031567271216, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.939, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.688502160960106, + "meets_prespecified_thresholds": false, + "z_mean": 0.0007407049553786692, + "z_sd": 1.0359450287848009, + "z_quantiles": [ + -2.0826234403932826, + 0.010582331794590802, + 2.0322927940295514 + ], + "abs_z_95": 2.057125312037281 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 10000, + "failures": 0, + "bias": 0.0015722690443644716, + "bias_mcse": 0.002123396008428074, + "rmse": 0.21233480474340996, + "empirical_sd": 0.2123396008428074, + "mean_variance": 0.045540686302278276, + "variance_ratio": 1.0100376852230484, + "coverage": 0.9471, + "coverage_mcse": 0.002238338446258741, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9471, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.8287403822030684, + "meets_prespecified_thresholds": true, + "z_mean": 0.009089813126256485, + "z_sd": 1.0172800710704153, + "z_quantiles": [ + -1.987875853402501, + 0.018440712664434943, + 1.9945306085504944 + ], + "abs_z_95": 1.9927242733280717 + } + } + } + } + }, + "0": { + "design": { + "n": 1000, + "observation": "balanced", + "clusters": 0, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 212.30504135499905, + "variants": { + "fitted_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 10000, + "failures": 0, + "bias": -0.0003188183686200219, + "bias_mcse": 0.0006719698663028727, + "rmse": 0.06719438305276682, + "empirical_sd": 0.06719698663028727, + "mean_variance": 0.0043337829570821, + "variance_ratio": 0.9597708626924155, + "coverage": 0.9429, + "coverage_mcse": 0.002320335967053048, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9429, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.257938871881968, + "meets_prespecified_thresholds": false, + "z_mean": -0.008168687819807357, + "z_sd": 1.022115545177736, + "z_quantiles": [ + -2.049460396211594, + -0.0030078315923897114, + 1.9925834302323464 + ], + "abs_z_95": 2.033461884905313 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 10000, + "failures": 0, + "bias": -0.00012628080670751274, + "bias_mcse": 0.0009467839996726071, + "rmse": 0.09467375014892301, + "empirical_sd": 0.09467839996726071, + "mean_variance": 0.008673852115963426, + "variance_ratio": 0.9676319362830239, + "coverage": 0.9466, + "coverage_mcse": 0.0022482980229498045, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9466, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.364877370789074, + "meets_prespecified_thresholds": true, + "z_mean": -0.00501409054352129, + "z_sd": 1.018688903420637, + "z_quantiles": [ + -2.0142987737228406, + -0.0038610055869863847, + 1.9571033542902663 + ], + "abs_z_95": 1.986961839159858 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 10000, + "failures": 0, + "bias": -0.0011362418760114679, + "bias_mcse": 0.0011193654971189966, + "rmse": 0.11193671974776302, + "empirical_sd": 0.11193654971189965, + "mean_variance": 0.012641326790022582, + "variance_ratio": 1.008901635085632, + "coverage": 0.9517, + "coverage_mcse": 0.0021439941697681923, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9517, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.440350692768527, + "meets_prespecified_thresholds": true, + "z_mean": -0.008256984872346678, + "z_sd": 0.9976640471941446, + "z_quantiles": [ + -1.9699501431068833, + -0.002172773005770127, + 1.937381690959387 + ], + "abs_z_95": 1.9486413338897823 + } + } + } + } + }, + "4": { + "design": { + "n": 1000, + "observation": "rotating", + "clusters": 0, + "null": true, + "violation": "none" + }, + "elapsed_seconds": 99.66611969400037, + "variants": { + "fitted_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 0.0, + "replications": 10000, + "failures": 0, + "bias": -0.0012398916080528781, + "bias_mcse": 0.001576024729588171, + "rmse": 0.15759947004982638, + "empirical_sd": 0.1576024729588171, + "mean_variance": 0.024898224913089337, + "variance_ratio": 1.0024029363882747, + "coverage": 0.9468, + "coverage_mcse": 0.0022443208326796775, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9468, + "rejection_at_zero": 0.0532, + "rejection_mcse": 0.002244320832679677, + "mean_interval_width": 0.6178842348460777, + "meets_prespecified_thresholds": true, + "z_mean": -0.008258081261993046, + "z_sd": 1.0022246939686272, + "z_quantiles": [ + -2.0244534861424213, + -0.0009686074608361642, + 1.9622893885785406 + ], + "abs_z_95": 1.9847069878373989 + }, + "g2_t3": { + "truth": 0.0, + "replications": 10000, + "failures": 0, + "bias": -0.0018936694531897916, + "bias_mcse": 0.0020896605726734747, + "rmse": 0.20896418925870838, + "empirical_sd": 0.2089660572673475, + "mean_variance": 0.043613629851242515, + "variance_ratio": 0.998782067321735, + "coverage": 0.9476, + "coverage_mcse": 0.002228323136351638, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9476, + "rejection_at_zero": 0.0524, + "rejection_mcse": 0.002228323136351638, + "mean_interval_width": 0.8175240427069372, + "meets_prespecified_thresholds": true, + "z_mean": -0.009136627776334666, + "z_sd": 1.0045010524315756, + "z_quantiles": [ + -1.9813412736238698, + -0.0050153927984515085, + 1.9843711294021653 + ], + "abs_z_95": 1.9817122716393976 + }, + "g3_t4": { + "truth": 0.0, + "replications": 10000, + "failures": 0, + "bias": -1.295979725728138e-05, + "bias_mcse": 0.002473590264424336, + "rmse": 0.24734665852141333, + "empirical_sd": 0.24735902644243363, + "mean_variance": 0.06219633947669803, + "variance_ratio": 1.0165044856761114, + "coverage": 0.9497, + "coverage_mcse": 0.002185632860294702, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9497, + "rejection_at_zero": 0.0503, + "rejection_mcse": 0.0021856328602947016, + "mean_interval_width": 0.9758776241024957, + "meets_prespecified_thresholds": true, + "z_mean": -0.001225993645996478, + "z_sd": 0.9965649248388969, + "z_quantiles": [ + -1.9466383005949466, + 0.0072858182169957385, + 1.978539623387434 + ], + "abs_z_95": 1.9626585098475011 + } + } + } + } + }, + "6": { + "design": { + "n": 4000, + "observation": "rotating", + "clusters": 0, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 148.03879536900058, + "variants": { + "fitted_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 10000, + "failures": 0, + "bias": 0.00024007977169748784, + "bias_mcse": 0.0007846954949319544, + "rmse": 0.07846599320050732, + "empirical_sd": 0.07846954949319544, + "mean_variance": 0.006339494691970662, + "variance_ratio": 1.0295615713048254, + "coverage": 0.9535, + "coverage_mcse": 0.0021056531053333543, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9535, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.312033298507766, + "meets_prespecified_thresholds": true, + "z_mean": 0.0017093092296618384, + "z_sd": 0.9860883289756504, + "z_quantiles": [ + -1.91767887057109, + -0.0014157948727167903, + 1.9296306703252684 + ], + "abs_z_95": 1.926176465522221 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 10000, + "failures": 0, + "bias": -0.0003989759705795848, + "bias_mcse": 0.00103628560504758, + "rmse": 0.1036241470230668, + "empirical_sd": 0.10362856050475801, + "mean_variance": 0.011125483307114846, + "variance_ratio": 1.0360004774189548, + "coverage": 0.9548, + "coverage_mcse": 0.0020774253295846763, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9548, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.4133341694436416, + "meets_prespecified_thresholds": true, + "z_mean": -0.0054297156457058845, + "z_sd": 0.9831340729244586, + "z_quantiles": [ + -1.904801210403122, + -0.011800724728690305, + 1.9243295007573904 + ], + "abs_z_95": 1.9169034166853263 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 10000, + "failures": 0, + "bias": 0.003750950095361194, + "bias_mcse": 0.0012576222979062287, + "rmse": 0.125811869291745, + "empirical_sd": 0.12576222979062288, + "mean_variance": 0.015968210968285056, + "variance_ratio": 1.0096150224616562, + "coverage": 0.9526, + "coverage_mcse": 0.00212492917529032, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9526, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.4951230986122531, + "meets_prespecified_thresholds": true, + "z_mean": 0.030279163469914567, + "z_sd": 0.9958700365206283, + "z_quantiles": [ + -1.913504410142896, + 0.03209378132295074, + 1.9760515574499091 + ], + "abs_z_95": 1.9335936985603908 + } + } + } + } + }, + "8": { + "design": { + "n": 4000, + "observation": "cohort_dependent", + "clusters": 0, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 152.9021948640002, + "variants": { + "fitted_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 10000, + "failures": 0, + "bias": -0.0010421150298114412, + "bias_mcse": 0.0008473223262765445, + "rmse": 0.08473440443336876, + "empirical_sd": 0.08473223262765445, + "mean_variance": 0.007219270665769527, + "variance_ratio": 1.005532298376495, + "coverage": 0.9505, + "coverage_mcse": 0.0021690954335851615, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9505, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.3329724916707266, + "meets_prespecified_thresholds": true, + "z_mean": -0.013224646580046996, + "z_sd": 0.9979007632004807, + "z_quantiles": [ + -1.9848396725903723, + -0.005755619942266796, + 1.920911392940589 + ], + "abs_z_95": 1.9542183388003183 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 10000, + "failures": 0, + "bias": -0.0012374436412052603, + "bias_mcse": 0.0010783728581403459, + "rmse": 0.10783899383099683, + "empirical_sd": 0.1078372858140346, + "mean_variance": 0.011578850836459608, + "variance_ratio": 0.9956978338096836, + "coverage": 0.9503, + "coverage_mcse": 0.0021732443488940672, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9503, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.42165293714533986, + "meets_prespecified_thresholds": true, + "z_mean": -0.01309631732194421, + "z_sd": 1.002772548947394, + "z_quantiles": [ + -1.9571671600570812, + -0.02207635938339421, + 1.9504946971898518 + ], + "abs_z_95": 1.956108577418235 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 10000, + "failures": 0, + "bias": -0.0015098496404060135, + "bias_mcse": 0.0014530149115586852, + "rmse": 0.1453020706174635, + "empirical_sd": 0.14530149115586852, + "mean_variance": 0.020980082542759227, + "variance_ratio": 0.9937269085615065, + "coverage": 0.9468, + "coverage_mcse": 0.0022443208326796775, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9468, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.5674354143928517, + "meets_prespecified_thresholds": true, + "z_mean": -0.008653408914133005, + "z_sd": 1.0054754095250538, + "z_quantiles": [ + -1.9939603296911421, + -0.0016421425336125583, + 1.9713412870073914 + ], + "abs_z_95": 1.9856366071541764 + } + } + } + } + }, + "5": { + "design": { + "n": 4000, + "observation": "balanced", + "clusters": 0, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 290.5276722509989, + "variants": { + "fitted_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 10000, + "failures": 0, + "bias": 0.00016703584389032855, + "bias_mcse": 0.00032967978300034353, + "rmse": 0.03296675303097594, + "empirical_sd": 0.032967978300034353, + "mean_variance": 0.0010836006400644623, + "variance_ratio": 0.9969758113463951, + "coverage": 0.9489, + "coverage_mcse": 0.0022020170299068996, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9489, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.12902240242958884, + "meets_prespecified_thresholds": true, + "z_mean": 0.0035929332509566764, + "z_sd": 1.0017835804147048, + "z_quantiles": [ + -1.9679610071273481, + 0.019117788971210796, + 1.9630023460248316 + ], + "abs_z_95": 1.9649398226290329 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 10000, + "failures": 0, + "bias": 0.0001328674623848336, + "bias_mcse": 0.000464079220777053, + "rmse": 0.04640579183474591, + "empirical_sd": 0.0464079220777053, + "mean_variance": 0.002167955904894891, + "variance_ratio": 1.006621490875534, + "coverage": 0.9499, + "coverage_mcse": 0.002181513007066427, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9499, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.18249233763385087, + "meets_prespecified_thresholds": true, + "z_mean": 0.0011688734815904908, + "z_sd": 0.9970345769133263, + "z_quantiles": [ + -1.9645085670765754, + 0.013807855841549487, + 1.9539882694094666 + ], + "abs_z_95": 1.960172500908972 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 10000, + "failures": 0, + "bias": 0.0005398262575155854, + "bias_mcse": 0.000563968561892882, + "rmse": 0.05639661993315518, + "empirical_sd": 0.056396856189288205, + "mean_variance": 0.0031625012021522544, + "variance_ratio": 0.9943079434025028, + "coverage": 0.9492, + "coverage_mcse": 0.0021958907076628374, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9492, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.22039472406764868, + "meets_prespecified_thresholds": true, + "z_mean": 0.010486654730707866, + "z_sd": 1.0036548948577808, + "z_quantiles": [ + -1.9790221034301445, + 0.015951468678190618, + 1.958898240618538 + ], + "abs_z_95": 1.966770231239412 + } + } + } + } + }, + "7": { + "design": { + "n": 4000, + "observation": "intermittent", + "clusters": 100, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 260.6998880020001, + "variants": { + "fitted_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 10000, + "failures": 0, + "bias": -0.00026878887926207184, + "bias_mcse": 0.0006087798588529814, + "rmse": 0.060875535315885555, + "empirical_sd": 0.060877985885298136, + "mean_variance": 0.003694090604613123, + "variance_ratio": 0.9967517157929459, + "coverage": 0.945, + "coverage_mcse": 0.0022798026230356006, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.945, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.23729309282637956, + "meets_prespecified_thresholds": true, + "z_mean": -0.005880406864338889, + "z_sd": 1.0150255918193025, + "z_quantiles": [ + -2.0455289993225305, + -0.011276272023742668, + 1.9877303550039966 + ], + "abs_z_95": 2.0170344275182273 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 10000, + "failures": 0, + "bias": -0.0005151236263158943, + "bias_mcse": 0.0008935537244744143, + "rmse": 0.08935238944377952, + "empirical_sd": 0.08935537244744143, + "mean_variance": 0.00794091197131758, + "variance_ratio": 0.9945555447225363, + "coverage": 0.9445, + "coverage_mcse": 0.002289535979188796, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9445, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.34777180918031264, + "meets_prespecified_thresholds": true, + "z_mean": -0.007068846030442143, + "z_sd": 1.0177232001432281, + "z_quantiles": [ + -2.0184060044081886, + -0.012152676177432504, + 2.003142336721603 + ], + "abs_z_95": 2.0112627673786045 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 10000, + "failures": 0, + "bias": -0.00046818713628571373, + "bias_mcse": 0.001079725627270353, + "rmse": 0.10796817907887475, + "empirical_sd": 0.10797256272703529, + "mean_variance": 0.011461892055639283, + "variance_ratio": 0.9831719852588979, + "coverage": 0.9445, + "coverage_mcse": 0.002289535979188796, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9445, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.41781177503485883, + "meets_prespecified_thresholds": true, + "z_mean": -0.004481785754366645, + "z_sd": 1.01809314062556, + "z_quantiles": [ + -2.0166714218930544, + -0.0004258039164022858, + 1.9837474325850057 + ], + "abs_z_95": 2.006565997547963 + } + } + } + } + }, + "9": { + "design": { + "n": 4000, + "observation": "rotating", + "clusters": 0, + "null": true, + "violation": "none" + }, + "elapsed_seconds": 140.2502569570006, + "variants": { + "fitted_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 0.0, + "replications": 10000, + "failures": 0, + "bias": 0.0004111797309783051, + "bias_mcse": 0.000786991501875898, + "rmse": 0.07869628932418579, + "empirical_sd": 0.0786991501875898, + "mean_variance": 0.006198077545048631, + "variance_ratio": 1.0007300014118599, + "coverage": 0.9501, + "coverage_mcse": 0.0021773835215689505, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9501, + "rejection_at_zero": 0.0499, + "rejection_mcse": 0.0021773835215689492, + "mean_interval_width": 0.30853378508250684, + "meets_prespecified_thresholds": true, + "z_mean": 0.005035293658274068, + "z_sd": 1.0008129376844974, + "z_quantiles": [ + -1.9408169293598319, + 0.012959471208656969, + 1.9784113254173599 + ], + "abs_z_95": 1.9581221714812949 + }, + "g2_t3": { + "truth": 0.0, + "replications": 10000, + "failures": 0, + "bias": 0.001050418745617564, + "bias_mcse": 0.001041716973074626, + "rmse": 0.10417178468859388, + "empirical_sd": 0.10417169730746259, + "mean_variance": 0.010854592372599248, + "variance_ratio": 1.0002626170567914, + "coverage": 0.9485, + "coverage_mcse": 0.0022101527096560544, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9485, + "rejection_at_zero": 0.0515, + "rejection_mcse": 0.0022101527096560544, + "mean_interval_width": 0.40827194418734636, + "meets_prespecified_thresholds": true, + "z_mean": 0.00973086959239889, + "z_sd": 1.0010687921758095, + "z_quantiles": [ + -1.9642575475072774, + 0.008321804609112957, + 1.9861922185684566 + ], + "abs_z_95": 1.9696482006508376 + }, + "g3_t4": { + "truth": 0.0, + "replications": 10000, + "failures": 0, + "bias": -0.001762310409929624, + "bias_mcse": 0.0012576328550949233, + "rmse": 0.1257693447535003, + "empirical_sd": 0.12576328550949234, + "mean_variance": 0.015434351400635694, + "variance_ratio": 0.9758445357150868, + "coverage": 0.9467, + "coverage_mcse": 0.0022463105306257196, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9467, + "rejection_at_zero": 0.0533, + "rejection_mcse": 0.002246310530625719, + "mean_interval_width": 0.48678704888635876, + "meets_prespecified_thresholds": true, + "z_mean": -0.014051974213968333, + "z_sd": 1.0137598922410809, + "z_quantiles": [ + -1.9784834087021765, + -0.0028404263086091075, + 1.9806813627156354 + ], + "abs_z_95": 1.9794518244833976 + } + } + } + } + }, + "10": { + "design": { + "n": 1000, + "observation": "intermittent", + "clusters": 0, + "null": false, + "violation": "selection" + }, + "elapsed_seconds": 149.98925528499967, + "variants": { + "fitted_empirical": { + "failures": { + "IPW propensity fit failed: Logistic regression did not converge in 25 iterations. Propensity score estimates may be unreliable.": 301 + }, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 10000, + "failures": 301, + "bias": 0.30971948374344205, + "bias_mcse": 0.00123780613648487, + "rmse": 0.332843935483255, + "empirical_sd": 0.12190348207376443, + "mean_variance": 0.014640508145282958, + "variance_ratio": 0.9851989230421198, + "coverage": 0.2632, + "coverage_mcse": 0.004403700262279439, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.2713681822868337, + "rejection_at_zero": 0.9699, + "rejection_mcse": 0.0017086248856902446, + "mean_interval_width": 0.47334412851716723, + "meets_prespecified_thresholds": false, + "z_mean": 2.573211960687419, + "z_sd": 1.02123349388991, + "z_quantiles": [ + 0.5786320479286295, + 2.5590156544429536, + 4.59124568109343 + ], + "abs_z_95": 4.245454868857546 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 10000, + "failures": 301, + "bias": 0.30181728534156305, + "bias_mcse": 0.0017339623869158656, + "rmse": 0.3467735981940523, + "empirical_sd": 0.17076668673678344, + "mean_variance": 0.029240928599637485, + "variance_ratio": 1.002731956610578, + "coverage": 0.5594, + "coverage_mcse": 0.004964591020416485, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.5767604907722446, + "rejection_at_zero": 0.9699, + "rejection_mcse": 0.0017086248856902446, + "mean_interval_width": 0.6687596704925153, + "meets_prespecified_thresholds": false, + "z_mean": 1.774245791155438, + "z_sd": 1.0079673739093187, + "z_quantiles": [ + -0.1912462679460911, + 1.7624193327975808, + 3.818655910174921 + ], + "abs_z_95": 3.4502158773144793 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 10000, + "failures": 301, + "bias": 0.44379139730027806, + "bias_mcse": 0.0024194090144431537, + "rmse": 0.5037047101583547, + "empirical_sd": 0.2382718704714388, + "mean_variance": 0.0550227089244787, + "variance_ratio": 0.9691620946580533, + "coverage": 0.5079, + "coverage_mcse": 0.004999375861045057, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.5236622332199196, + "rejection_at_zero": 0.9699, + "rejection_mcse": 0.0017086248856902446, + "mean_interval_width": 0.9150320323480879, + "meets_prespecified_thresholds": false, + "z_mean": 1.9215870030836832, + "z_sd": 1.0549037064948277, + "z_quantiles": [ + -0.08269565981636287, + 1.909002003035749, + 4.045332316236637 + ], + "abs_z_95": 3.691381696916717 + } + } + } + } + }, + "11": { + "design": { + "n": 1000, + "observation": "intermittent", + "clusters": 0, + "null": false, + "violation": "parallel_trends" + }, + "elapsed_seconds": 149.1788728269985, + "variants": { + "fitted_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 10000, + "failures": 0, + "bias": 0.7521974789414307, + "bias_mcse": 0.0008330121150799783, + "rmse": 0.7567955108583567, + "empirical_sd": 0.08330121150799782, + "mean_variance": 0.0068569637524843186, + "variance_ratio": 0.9881644330230893, + "coverage": 0.0, + "coverage_mcse": 0.0, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.0, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.3243340289126125, + "meets_prespecified_thresholds": false, + "z_mean": 9.102078439861934, + "z_sd": 1.0403879397644527, + "z_quantiles": [ + 7.113816253745072, + 9.102477547876504, + 11.167365104398582 + ], + "abs_z_95": 10.787841272336342 + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 10000, + "failures": 0, + "bias": 0.8019365329085422, + "bias_mcse": 0.001183711803843327, + "rmse": 0.810624782488988, + "empirical_sd": 0.1183711803843327, + "mean_variance": 0.013892298751219553, + "variance_ratio": 0.9914758891107488, + "coverage": 0.0, + "coverage_mcse": 0.0, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.0, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.4615742152775251, + "meets_prespecified_thresholds": false, + "z_mean": 6.820353393974694, + "z_sd": 1.0291090538092698, + "z_quantiles": [ + 4.824687252368925, + 6.809293875178487, + 8.849218640078265 + ], + "abs_z_95": 8.51479748936089 + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 10000, + "failures": 0, + "bias": 0.7974843547231698, + "bias_mcse": 0.0014536186675311383, + "rmse": 0.810622757715912, + "empirical_sd": 0.14536186675311383, + "mean_variance": 0.020669826213904438, + "variance_ratio": 0.9782184326971763, + "coverage": 0.0002, + "coverage_mcse": 0.00014140721339450828, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.0002, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.562706738350993, + "meets_prespecified_thresholds": false, + "z_mean": 5.574840634622723, + "z_sd": 1.0731711738306486, + "z_quantiles": [ + 3.580832419850364, + 5.568368576653017, + 7.72095115024115 + ], + "abs_z_95": 7.380711320861592 + } + } + } + } + }, + "12": { + "design": { + "n": 1000, + "observation": "intermittent", + "clusters": 0, + "null": false, + "violation": "propensity" + }, + "elapsed_seconds": 144.01954894200026, + "variants": { + "fitted_empirical": { + "failures": {}, + "targets": { + "overall": { + "truth": 3.2242670754154847, + "replications": 10000, + "failures": 0, + "bias": 0.10041252024972414, + "bias_mcse": 0.0012260674471804133, + "rmse": 0.15847266272727012, + "empirical_sd": 0.12260674471804132, + "mean_variance": 0.014629748246063338, + "variance_ratio": 0.9732135099325975, + "coverage": 0.8557, + "coverage_mcse": 0.0035139366812735823, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.8557, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.47121778505119516, + "meets_prespecified_thresholds": false, + "z_mean": 0.8436742261384025, + "z_sd": 1.0345993059032845, + "z_quantiles": [ + -1.1392315736272403, + 0.828819267507573, + 2.8934464848287984 + ], + "abs_z_95": 2.5869684852844097 + }, + "g2_t3": { + "truth": 3.332688995347808, + "replications": 10000, + "failures": 0, + "bias": 0.11584823821102397, + "bias_mcse": 0.0016301484512075833, + "rmse": 0.19997999060498536, + "empirical_sd": 0.16301484512075834, + "mean_variance": 0.02575928156474173, + "variance_ratio": 0.9693473666851636, + "coverage": 0.8769, + "coverage_mcse": 0.003285519593610727, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.8769, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.6245603800717073, + "meets_prespecified_thresholds": false, + "z_mean": 0.7376802436599997, + "z_sd": 1.0405238106751202, + "z_quantiles": [ + -1.2493157844882798, + 0.730864488769541, + 2.8151823824068356 + ], + "abs_z_95": 2.4917556553067737 + }, + "g3_t4": { + "truth": 3.3861450053055644, + "replications": 10000, + "failures": 0, + "bias": 0.007500032568309223, + "bias_mcse": 0.0017691277992061165, + "rmse": 0.1770628486571775, + "empirical_sd": 0.17691277992061166, + "mean_variance": 0.03078407261801188, + "variance_ratio": 0.9835754067953072, + "coverage": 0.9446, + "coverage_mcse": 0.002287593495357075, + "nominal_binomial_mcse": 0.0021794494717703367, + "coverage_given_valid": 0.9446, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.6861356188513233, + "meets_prespecified_thresholds": true, + "z_mean": 0.046638484775978074, + "z_sd": 1.0147957472649543, + "z_quantiles": [ + -1.9554278201037094, + 0.04638717196466406, + 2.0476993012243527 + ], + "abs_z_95": 2.0041661217589066 + } + } + } + } + } + }, + "acceptance_rule": "zero failures; |bias|<.1 empirical SD; variance ratio [.9,1.1]; 95% coverage and null 5% rejection within 3 binomial MCSE", + "interpretation": "Oracle variants are diagnostics, not public estimator options. Reference agreement is a separate report.", + "core_thresholds_pass": false +} diff --git a/benchmarks/data/chained_did/qualification-pre-review.json b/benchmarks/data/chained_did/qualification-pre-review.json new file mode 100644 index 000000000..c19712bb8 --- /dev/null +++ b/benchmarks/data/chained_did/qualification-pre-review.json @@ -0,0 +1,990 @@ +{ + "status": "complete", + "scope": "qualification", + "replications_per_case": 2000, + "seed": 20261102, + "seed_derivation": "SeedSequence([seed, case_index, replication]).generate_state(1)[0]", + "python": "3.12.11", + "numpy": "2.3.5", + "pandas": "2.3.3", + "scipy": "1.15.3", + "backend": "native Python estimator", + "source_commit": "ba746948b7fc7d06a5e8cdbf9601e21c3eaac6f0", + "source_uncommitted": true, + "source_hashes": { + "diff_diff/chained_did.py": "71ebff4b4ad18c0418043cb818b3ae13d895f907f91a3c31514dc16c93de27c1", + "diff_diff/chained_did_results.py": "1ffc707e417364fa3ea4788f96402e518497bed2a3972743587669e7a84da3bd", + "benchmarks/python/validate_chained_did.py": "92df057f449969754b78c1ec6942ce689b61daf88bbf69833d23237715103f3a" + }, + "cases": [ + { + "n": 1000, + "observation": "balanced", + "clusters": 0, + "null": false, + "violation": "none" + }, + { + "n": 1000, + "observation": "rotating", + "clusters": 0, + "null": false, + "violation": "none" + }, + { + "n": 1000, + "observation": "intermittent", + "clusters": 50, + "null": false, + "violation": "none" + }, + { + "n": 1000, + "observation": "cohort_dependent", + "clusters": 0, + "null": false, + "violation": "none" + }, + { + "n": 1000, + "observation": "rotating", + "clusters": 0, + "null": true, + "violation": "none" + }, + { + "n": 4000, + "observation": "balanced", + "clusters": 0, + "null": false, + "violation": "none" + }, + { + "n": 4000, + "observation": "rotating", + "clusters": 0, + "null": false, + "violation": "none" + }, + { + "n": 4000, + "observation": "intermittent", + "clusters": 100, + "null": false, + "violation": "none" + }, + { + "n": 4000, + "observation": "cohort_dependent", + "clusters": 0, + "null": false, + "violation": "none" + }, + { + "n": 4000, + "observation": "rotating", + "clusters": 0, + "null": true, + "violation": "none" + }, + { + "n": 1000, + "observation": "intermittent", + "clusters": 0, + "null": false, + "violation": "selection" + }, + { + "n": 1000, + "observation": "intermittent", + "clusters": 0, + "null": false, + "violation": "parallel_trends" + }, + { + "n": 1000, + "observation": "intermittent", + "clusters": 0, + "null": false, + "violation": "propensity" + } + ], + "results": { + "1": { + "design": { + "n": 1000, + "observation": "rotating", + "clusters": 0, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 14.828669385000467, + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.0013256551872644629, + "bias_mcse": 0.0036641673176459576, + "rmse": 0.16383093573239874, + "empirical_sd": 0.16386654406378853, + "mean_variance": 0.025548537664320397, + "variance_ratio": 0.9514488775574838, + "coverage": 0.9435, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9435, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.6258755434912127, + "meets_prespecified_thresholds": true + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": 0.002774992889462702, + "bias_mcse": 0.004937892250870673, + "rmse": 0.22079147983577777, + "empirical_sd": 0.22082925477032542, + "mean_variance": 0.04493583687007351, + "variance_ratio": 0.9214666475479428, + "coverage": 0.9365, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9365, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.8297870183649659, + "meets_prespecified_thresholds": true + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": -0.0020128670697885376, + "bias_mcse": 0.005820917079220396, + "rmse": 0.26026202151777145, + "empirical_sd": 0.2603193256105267, + "mean_variance": 0.06421370709386517, + "variance_ratio": 0.9475778965601165, + "coverage": 0.934, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.934, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.9914464513011779, + "meets_prespecified_thresholds": false + } + } + }, + "3": { + "design": { + "n": 1000, + "observation": "cohort_dependent", + "clusters": 0, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 15.25175211900023, + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.0015496974511757244, + "bias_mcse": 0.003907656749463025, + "rmse": 0.17471890087842942, + "empirical_sd": 0.1747557224907038, + "mean_variance": 0.028893612297839814, + "variance_ratio": 0.946104328014614, + "coverage": 0.9395, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9395, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.6655179269263534, + "meets_prespecified_thresholds": true + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": 0.005460381896478683, + "bias_mcse": 0.0049154542412676175, + "rmse": 0.21983865646434667, + "empirical_sd": 0.21982579647528092, + "mean_variance": 0.04639954055058544, + "variance_ratio": 0.9601882108884698, + "coverage": 0.9415, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9415, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.8430647808954038, + "meets_prespecified_thresholds": true + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": -0.005129287755606615, + "bias_mcse": 0.006536372142383926, + "rmse": 0.2922873705782742, + "empirical_sd": 0.2923154487321279, + "mean_variance": 0.08509969792997686, + "variance_ratio": 0.9959200645361632, + "coverage": 0.9465, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9465, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 1.1403671360999714, + "meets_prespecified_thresholds": true + } + } + }, + "2": { + "design": { + "n": 1000, + "observation": "intermittent", + "clusters": 50, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 25.00555120700119, + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": -0.0018675186735697125, + "bias_mcse": 0.0028023808763087943, + "rmse": 0.12530886416859233, + "empirical_sd": 0.1253262827654379, + "mean_variance": 0.014655582320497395, + "variance_ratio": 0.933079745566396, + "coverage": 0.9315, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9315, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.47085526795173815, + "meets_prespecified_thresholds": false + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.0015688192941309964, + "bias_mcse": 0.004165970509492481, + "rmse": 0.18626788895462837, + "empirical_sd": 0.18630786502969243, + "mean_variance": 0.031114103336694995, + "variance_ratio": 0.8963856832298651, + "coverage": 0.93, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.93, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.685192690602935, + "meets_prespecified_thresholds": false + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.0023287799291238365, + "bias_mcse": 0.0048261643612166685, + "rmse": 0.21579123299261396, + "empirical_sd": 0.2158326316453464, + "mean_variance": 0.04540021239756534, + "variance_ratio": 0.9745938632351944, + "coverage": 0.9445, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9445, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.8279313992030578, + "meets_prespecified_thresholds": true + } + } + }, + "0": { + "design": { + "n": 1000, + "observation": "balanced", + "clusters": 0, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 25.72468188899984, + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.00036592089574050135, + "bias_mcse": 0.0015200420591149693, + "rmse": 0.06796233584271082, + "empirical_sd": 0.06797834745679651, + "mean_variance": 0.004347523430277386, + "variance_ratio": 0.9408074022304718, + "coverage": 0.945, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.945, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.25834438987067615, + "meets_prespecified_thresholds": true + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": 0.0012659412842261908, + "bias_mcse": 0.0021369304737643173, + "rmse": 0.09555092796457071, + "empirical_sd": 0.09556643605055688, + "mean_variance": 0.008682896360888506, + "variance_ratio": 0.9507226417539414, + "coverage": 0.948, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.948, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.3650639344195169, + "meets_prespecified_thresholds": true + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": -0.003015294603054386, + "bias_mcse": 0.0025523995811574315, + "rmse": 0.11415806805349578, + "empirical_sd": 0.11414677938420016, + "mean_variance": 0.01268427857177705, + "variance_ratio": 0.9735055827179331, + "coverage": 0.9455, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9455, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.4410747171206066, + "meets_prespecified_thresholds": true + } + } + }, + "4": { + "design": { + "n": 1000, + "observation": "rotating", + "clusters": 0, + "null": true, + "violation": "none" + }, + "elapsed_seconds": 15.004269212000509, + "failures": {}, + "targets": { + "overall": { + "truth": 0.0, + "replications": 2000, + "failures": 0, + "bias": 0.0003460631825616036, + "bias_mcse": 0.0035201479344368226, + "rmse": 0.15738682053744418, + "empirical_sd": 0.15742580144512416, + "mean_variance": 0.024893088143610506, + "variance_ratio": 1.0044468265918003, + "coverage": 0.9455, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9455, + "rejection_at_zero": 0.0545, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.6178502057376661, + "meets_prespecified_thresholds": true + }, + "g2_t3": { + "truth": 0.0, + "replications": 2000, + "failures": 0, + "bias": 0.0002146792937239449, + "bias_mcse": 0.004660468772842476, + "rmse": 0.2083704981116478, + "empirical_sd": 0.20842249966181603, + "mean_variance": 0.04360960481309825, + "variance_ratio": 1.0039057709150472, + "coverage": 0.948, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.948, + "rejection_at_zero": 0.052, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.8175040603276089, + "meets_prespecified_thresholds": true + }, + "g3_t4": { + "truth": 0.0, + "replications": 2000, + "failures": 0, + "bias": 0.0015798301089758496, + "bias_mcse": 0.005585720691054373, + "rmse": 0.24974356220827923, + "empirical_sd": 0.2498010233704936, + "mean_variance": 0.06221043518752747, + "variance_ratio": 0.996953294714102, + "coverage": 0.9505, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9505, + "rejection_at_zero": 0.0495, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.9758904417243588, + "meets_prespecified_thresholds": true + } + } + }, + "6": { + "design": { + "n": 4000, + "observation": "rotating", + "clusters": 0, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 21.25967499699982, + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": -0.0015232495500052678, + "bias_mcse": 0.0018107538784726393, + "rmse": 0.08097345661629247, + "empirical_sd": 0.0809793752557243, + "mean_variance": 0.00633021194173059, + "variance_ratio": 0.9653157830321725, + "coverage": 0.949, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.949, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.3118086571549198, + "meets_prespecified_thresholds": true + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.0015608290043358508, + "bias_mcse": 0.0024028837591007777, + "rmse": 0.10744469766499228, + "empirical_sd": 0.10746022854759137, + "mean_variance": 0.011112285449619876, + "variance_ratio": 0.9622942020704252, + "coverage": 0.9515, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9515, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.41309391985051713, + "meets_prespecified_thresholds": true + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": -0.0024411481268087343, + "bias_mcse": 0.0028390873884072344, + "rmse": 0.12695957303411204, + "empirical_sd": 0.1269678478908185, + "mean_variance": 0.015941234494283155, + "variance_ratio": 0.9888591434346066, + "coverage": 0.9505, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9505, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.4947158090946397, + "meets_prespecified_thresholds": true + } + } + }, + "8": { + "design": { + "n": 4000, + "observation": "cohort_dependent", + "clusters": 0, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 22.640558365999823, + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.0012596399858137985, + "bias_mcse": 0.001954632812992143, + "rmse": 0.08740105817844195, + "empirical_sd": 0.08741383681804134, + "mean_variance": 0.007238073932202719, + "variance_ratio": 0.9472457140381726, + "coverage": 0.9425, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9425, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.3333978693676465, + "meets_prespecified_thresholds": true + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": 0.002928423947319292, + "bias_mcse": 0.002457049552410634, + "rmse": 0.10989414716132441, + "empirical_sd": 0.1098825964655122, + "mean_variance": 0.011595976227857263, + "variance_ratio": 0.9603941153868694, + "coverage": 0.943, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.943, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.42195395789633877, + "meets_prespecified_thresholds": true + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": -0.0013388912616920656, + "bias_mcse": 0.003320245609996749, + "rmse": 0.14845480935390842, + "empirical_sd": 0.14848589771895973, + "mean_variance": 0.02105901807725185, + "variance_ratio": 0.9551414653965952, + "coverage": 0.944, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.944, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.568494209479711, + "meets_prespecified_thresholds": true + } + } + }, + "5": { + "design": { + "n": 4000, + "observation": "balanced", + "clusters": 0, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 44.89723243200024, + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.00034006704997979753, + "bias_mcse": 0.0007426381671007187, + "rmse": 0.03320522592334943, + "empirical_sd": 0.0332117884864611, + "mean_variance": 0.0010836992455515396, + "variance_ratio": 0.9824811896337062, + "coverage": 0.9515, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9515, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.1290273475877689, + "meets_prespecified_thresholds": true + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": 0.0005103874893133131, + "bias_mcse": 0.001056910258995275, + "rmse": 0.047257401824821015, + "empirical_sd": 0.04726646370460687, + "mean_variance": 0.0021667216847834556, + "variance_ratio": 0.9698328877335446, + "coverage": 0.9465, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9465, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.18243979213314837, + "meets_prespecified_thresholds": true + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": -0.000714268432508991, + "bias_mcse": 0.0012534425743530296, + "rmse": 0.0560461919804368, + "empirical_sd": 0.056055656042914175, + "mean_variance": 0.00316172597138394, + "variance_ratio": 1.0062023964526452, + "coverage": 0.951, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.951, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.22036790844110946, + "meets_prespecified_thresholds": true + } + } + }, + "7": { + "design": { + "n": 4000, + "observation": "intermittent", + "clusters": 100, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 39.789969179000764, + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 2.2281944117352737e-05, + "bias_mcse": 0.0013752188381324988, + "rmse": 0.06148628282097788, + "empirical_sd": 0.06150165612005095, + "mean_variance": 0.003674991084307411, + "variance_ratio": 0.9715891774048486, + "coverage": 0.949, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.949, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.23663459304380588, + "meets_prespecified_thresholds": true + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.0012636103883512106, + "bias_mcse": 0.0020087886369256837, + "rmse": 0.08982218576225587, + "empirical_sd": 0.08983575889189946, + "mean_variance": 0.007899589939243222, + "variance_ratio": 0.9788272836079557, + "coverage": 0.9455, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9455, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.3467875796767015, + "meets_prespecified_thresholds": true + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.002298883696887799, + "bias_mcse": 0.0023604496653877468, + "rmse": 0.1055611594730147, + "empirical_sd": 0.10556251818547269, + "mean_variance": 0.011534624672091199, + "variance_ratio": 1.0351039932273511, + "coverage": 0.954, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.954, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.41901890460500046, + "meets_prespecified_thresholds": true + } + } + }, + "9": { + "design": { + "n": 4000, + "observation": "rotating", + "clusters": 0, + "null": true, + "violation": "none" + }, + "elapsed_seconds": 20.906472161001147, + "failures": {}, + "targets": { + "overall": { + "truth": 0.0, + "replications": 2000, + "failures": 0, + "bias": 0.0025359520068616442, + "bias_mcse": 0.0017237037932036816, + "rmse": 0.07710881552211733, + "empirical_sd": 0.07708637709355344, + "mean_variance": 0.006205114087085194, + "variance_ratio": 1.0442259953302844, + "coverage": 0.955, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.955, + "rejection_at_zero": 0.045, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.30870518752867315, + "meets_prespecified_thresholds": true + }, + "g2_t3": { + "truth": 0.0, + "replications": 2000, + "failures": 0, + "bias": 0.002619145423286187, + "bias_mcse": 0.0022777592812033044, + "rmse": 0.10187269713530119, + "empirical_sd": 0.10186449178303296, + "mean_variance": 0.010866740929719494, + "variance_ratio": 1.0472579545682104, + "coverage": 0.9535, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9535, + "rejection_at_zero": 0.0465, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.40849545750369926, + "meets_prespecified_thresholds": true + }, + "g3_t4": { + "truth": 0.0, + "replications": 2000, + "failures": 0, + "bias": 0.0014901115079163894, + "bias_mcse": 0.002787046225326044, + "rmse": 0.12461824156138707, + "empirical_sd": 0.12464049632526461, + "mean_variance": 0.015409858887253089, + "variance_ratio": 0.9919283944498148, + "coverage": 0.95, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.95, + "rejection_at_zero": 0.05, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.4863972132520547, + "meets_prespecified_thresholds": true + } + } + }, + "10": { + "design": { + "n": 1000, + "observation": "intermittent", + "clusters": 0, + "null": false, + "violation": "selection" + }, + "elapsed_seconds": 22.462370172999726, + "failures": { + "IPW propensity fit failed: Logistic regression did not converge in 25 iterations. Propensity score estimates may be unreliable.": 63 + }, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 63, + "bias": 0.31057869606726385, + "bias_mcse": 0.002773883452392924, + "rmse": 0.33369977791973876, + "empirical_sd": 0.12208238923877858, + "mean_variance": 0.014676948234010409, + "variance_ratio": 0.9847584638155564, + "coverage": 0.263, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.2715539494062984, + "rejection_at_zero": 0.9685, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.47387103422059695, + "meets_prespecified_thresholds": false + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 63, + "bias": 0.30634690127900877, + "bias_mcse": 0.0038460054703377284, + "rmse": 0.349978953027182, + "empirical_sd": 0.1692679396602611, + "mean_variance": 0.029333879962731096, + "variance_ratio": 1.0238117146352068, + "coverage": 0.54, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.5575632421270005, + "rejection_at_zero": 0.9685, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.6697519151450257, + "meets_prespecified_thresholds": false + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 63, + "bias": 0.4414022724954214, + "bias_mcse": 0.0055403497908995914, + "rmse": 0.5042443924562886, + "empirical_sd": 0.24383834119205794, + "mean_variance": 0.054556254976283414, + "variance_ratio": 0.9175728611171998, + "coverage": 0.5045, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.5209086215797625, + "rejection_at_zero": 0.9685, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.9112860177942907, + "meets_prespecified_thresholds": false + } + } + }, + "11": { + "design": { + "n": 1000, + "observation": "intermittent", + "clusters": 0, + "null": false, + "violation": "parallel_trends" + }, + "elapsed_seconds": 23.67295190599907, + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.7520209634666437, + "bias_mcse": 0.0018945118998704136, + "rmse": 0.7567762489636294, + "empirical_sd": 0.0847251478458504, + "mean_variance": 0.006865271942500445, + "variance_ratio": 0.9563857006899881, + "coverage": 0.0, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.0, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.3245309652142032, + "meets_prespecified_thresholds": false + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": 0.8010906122729633, + "bias_mcse": 0.002626351828109169, + "rmse": 0.809650986041499, + "empirical_sd": 0.11745402440965891, + "mean_variance": 0.013920454765729322, + "variance_ratio": 1.0090614612199305, + "coverage": 0.0, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.0, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.4620349869723595, + "meets_prespecified_thresholds": false + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.8005011108278408, + "bias_mcse": 0.003231716681667597, + "rmse": 0.8134368874504234, + "empirical_sd": 0.1445267636845759, + "mean_variance": 0.02066334773574415, + "variance_ratio": 0.9892456031127933, + "coverage": 0.0, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.0, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.5626509379588227, + "meets_prespecified_thresholds": false + } + } + }, + "12": { + "design": { + "n": 1000, + "observation": "intermittent", + "clusters": 0, + "null": false, + "violation": "propensity" + }, + "elapsed_seconds": 24.57715074099906, + "failures": {}, + "targets": { + "overall": { + "truth": 3.2242670754154847, + "replications": 2000, + "failures": 0, + "bias": 0.1017035416834382, + "bias_mcse": 0.0026887928829709036, + "rmse": 0.15746617134742733, + "empirical_sd": 0.12024647327481154, + "mean_variance": 0.014521820569018594, + "variance_ratio": 1.004329850331975, + "coverage": 0.862, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.862, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.46985846101580137, + "meets_prespecified_thresholds": false + }, + "g2_t3": { + "truth": 3.332688995347808, + "replications": 2000, + "failures": 0, + "bias": 0.11776642068682008, + "bias_mcse": 0.0034599678375010364, + "rmse": 0.19442148373323306, + "empirical_sd": 0.15473446569230528, + "mean_variance": 0.025604488950655948, + "variance_ratio": 1.0694044643726814, + "coverage": 0.8775, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.8775, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.6232256242218566, + "meets_prespecified_thresholds": false + }, + "g3_t4": { + "truth": 3.3861450053055644, + "replications": 2000, + "failures": 0, + "bias": 0.006593203788076263, + "bias_mcse": 0.003967119476032169, + "rmse": 0.17749311584814073, + "empirical_sd": 0.17741497646542556, + "mean_variance": 0.030694548556901756, + "variance_ratio": 0.9751708132199531, + "coverage": 0.947, + "coverage_mcse": 0.004873397172404482, + "coverage_given_valid": 0.947, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.6851786160389953, + "meets_prespecified_thresholds": true + } + } + } + }, + "external_reference": "not run: required R 4.5.2/cdid/did/DRDID environment unavailable", + "thresholds": "zero failures; |bias|<.1 empirical SD; variance ratio [.9,1.1]; 95% coverage and null 5% rejection within 3 binomial MCSE", + "core_thresholds_pass": false +} diff --git a/benchmarks/data/chained_did/qualification-sparse.json b/benchmarks/data/chained_did/qualification-sparse.json new file mode 100644 index 000000000..cdf90c342 --- /dev/null +++ b/benchmarks/data/chained_did/qualification-sparse.json @@ -0,0 +1,273 @@ +{ + "status": "complete", + "scope": "pilot", + "replications_per_case": 300, + "seed": 20261105, + "design": "sparse", + "seed_derivation": "SeedSequence([seed, case_index, replication]).generate_state(1)[0]", + "python": "3.14.0", + "numpy": "2.5.4", + "pandas": "3.1.0", + "scipy": "1.18.1", + "backend": "native Python estimator", + "source_commit": "ba746948b7fc7d06a5e8cdbf9601e21c3eaac6f0", + "source_uncommitted": true, + "source_hashes": { + "diff_diff/chained_did.py": "870355980e3eb8d255db709daefaadefc7c5249925734d1d6bcc92cc79770934", + "diff_diff/chained_did_results.py": "df3e0b4b1853b9f9c98ddfda10b43255d311c38f5dd993ee45b2d4921e54cade", + "benchmarks/python/validate_chained_did.py": "cf3c5219738fc0ec7ff69cd6b0676df65320b800c255e5c3ecc81c06e08cd799" + }, + "cases": [ + { + "n": 400, + "sparse_treated": 0, + "sparse_controls": 19, + "violation": "none" + }, + { + "n": 400, + "sparse_treated": 1, + "sparse_controls": 19, + "violation": "none" + }, + { + "n": 400, + "sparse_treated": 2, + "sparse_controls": 19, + "violation": "none" + }, + { + "n": 400, + "sparse_treated": 5, + "sparse_controls": 19, + "violation": "none" + }, + { + "n": 400, + "sparse_treated": 20, + "sparse_controls": 19, + "violation": "none" + }, + { + "n": 400, + "sparse_treated": 50, + "sparse_controls": 19, + "violation": "none" + } + ], + "results": { + "0": { + "design": { + "n": 400, + "sparse_treated": 0, + "sparse_controls": 19, + "violation": "none" + }, + "elapsed_seconds": 1.9591164880002907, + "failures": {}, + "pair_diagnostics": { + "replications_with_singleton_exclusions": 0, + "replications_with_low_support": 0 + }, + "targets": { + "g3_t3": { + "truth": 1.0, + "replications": 300, + "failures": 0, + "bias": -0.005405635840533218, + "bias_mcse": 0.007457991041311265, + "rmse": 0.12907396476498237, + "empirical_sd": 0.1291761940594463, + "mean_variance": 0.019983789680413297, + "variance_ratio": 1.1976030156290232, + "coverage": 0.9666666666666667, + "coverage_mcse": 0.010363754503432016, + "nominal_binomial_mcse": 0.012583057392117916, + "coverage_given_valid": 0.9666666666666667, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.5537666779969338, + "meets_prespecified_thresholds": false + } + } + }, + "1": { + "design": { + "n": 400, + "sparse_treated": 1, + "sparse_controls": 19, + "violation": "none" + }, + "elapsed_seconds": 2.899518722999346, + "failures": {}, + "pair_diagnostics": { + "replications_with_singleton_exclusions": 300, + "replications_with_low_support": 0 + }, + "targets": { + "g3_t3": { + "truth": 1.0, + "replications": 300, + "failures": 0, + "bias": 0.010626881115757492, + "bias_mcse": 0.008114611347894781, + "rmse": 0.14071659064636227, + "empirical_sd": 0.14054919138228733, + "mean_variance": 0.01996979312537547, + "variance_ratio": 1.0109201734323727, + "coverage": 0.9433333333333334, + "coverage_mcse": 0.013348602368232605, + "nominal_binomial_mcse": 0.012583057392117916, + "coverage_given_valid": 0.9433333333333334, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.5535572707826139, + "meets_prespecified_thresholds": true + } + } + }, + "2": { + "design": { + "n": 400, + "sparse_treated": 2, + "sparse_controls": 19, + "violation": "none" + }, + "elapsed_seconds": 3.143980550001288, + "failures": {}, + "pair_diagnostics": { + "replications_with_singleton_exclusions": 0, + "replications_with_low_support": 300 + }, + "targets": { + "g3_t3": { + "truth": 1.0, + "replications": 300, + "failures": 0, + "bias": 0.022178409128551513, + "bias_mcse": 0.016647853705580414, + "rmse": 0.28872138940368325, + "empirical_sd": 0.2883492845503909, + "mean_variance": 0.05015609284456146, + "variance_ratio": 0.6032341800691966, + "coverage": 0.7866666666666666, + "coverage_mcse": 0.023651795014489014, + "nominal_binomial_mcse": 0.012583057392117916, + "coverage_given_valid": 0.7866666666666666, + "rejection_at_zero": 0.9633333333333334, + "rejection_mcse": 0.010850840554571826, + "mean_interval_width": 0.8219970986327887, + "meets_prespecified_thresholds": false + } + } + }, + "3": { + "design": { + "n": 400, + "sparse_treated": 5, + "sparse_controls": 19, + "violation": "none" + }, + "elapsed_seconds": 2.0853491269990627, + "failures": {}, + "pair_diagnostics": { + "replications_with_singleton_exclusions": 0, + "replications_with_low_support": 300 + }, + "targets": { + "g3_t3": { + "truth": 1.0, + "replications": 300, + "failures": 0, + "bias": -0.003902042021098895, + "bias_mcse": 0.01252493944971736, + "rmse": 0.21661159777223657, + "empirical_sd": 0.21693831488634244, + "mean_variance": 0.0418617034723171, + "variance_ratio": 0.8894967637330812, + "coverage": 0.91, + "coverage_mcse": 0.016522711641858305, + "nominal_binomial_mcse": 0.012583057392117916, + "coverage_given_valid": 0.91, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.7887476692898945, + "meets_prespecified_thresholds": false + } + } + }, + "4": { + "design": { + "n": 400, + "sparse_treated": 20, + "sparse_controls": 19, + "violation": "none" + }, + "elapsed_seconds": 2.3441334369999822, + "failures": {}, + "pair_diagnostics": { + "replications_with_singleton_exclusions": 0, + "replications_with_low_support": 300 + }, + "targets": { + "g3_t3": { + "truth": 1.0, + "replications": 300, + "failures": 0, + "bias": -0.008874114863725469, + "bias_mcse": 0.009422716105198497, + "rmse": 0.1631754760237137, + "empirical_sd": 0.16320623039501325, + "mean_variance": 0.02845665195134923, + "variance_ratio": 1.0683420787839661, + "coverage": 0.94, + "coverage_mcse": 0.013711309200802093, + "nominal_binomial_mcse": 0.012583057392117916, + "coverage_given_valid": 0.94, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.6593439564640485, + "meets_prespecified_thresholds": true + } + } + }, + "5": { + "design": { + "n": 400, + "sparse_treated": 50, + "sparse_controls": 19, + "violation": "none" + }, + "elapsed_seconds": 2.7721103180010687, + "failures": {}, + "pair_diagnostics": { + "replications_with_singleton_exclusions": 0, + "replications_with_low_support": 300 + }, + "targets": { + "g3_t3": { + "truth": 1.0, + "replications": 300, + "failures": 0, + "bias": -0.018339304061281814, + "bias_mcse": 0.010013964872112745, + "rmse": 0.17412609890238928, + "empirical_sd": 0.1734469594370925, + "mean_variance": 0.025448154811245828, + "variance_ratio": 0.8459075791400404, + "coverage": 0.9233333333333333, + "coverage_mcse": 0.015361085995559135, + "nominal_binomial_mcse": 0.012583057392117916, + "coverage_given_valid": 0.9233333333333333, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.623935584349509, + "meets_prespecified_thresholds": false + } + } + } + }, + "external_reference": "not run in this invocation; offline regression checks use frozen public R outputs", + "thresholds": "zero failures; |bias|<.1 empirical SD; variance ratio [.9,1.1]; 95% coverage and null 5% rejection within 3 binomial MCSE", + "core_thresholds_pass": false +} diff --git a/benchmarks/data/chained_did/qualification.json b/benchmarks/data/chained_did/qualification.json new file mode 100644 index 000000000..81ce15bf0 --- /dev/null +++ b/benchmarks/data/chained_did/qualification.json @@ -0,0 +1,1029 @@ +{ + "status": "complete", + "scope": "qualification", + "replications_per_case": 2000, + "seed": 20261102, + "seed_derivation": "SeedSequence([seed, case_index, replication]).generate_state(1)[0]", + "python": "3.12.11", + "numpy": "2.3.5", + "pandas": "2.3.3", + "scipy": "1.15.3", + "backend": "native Python estimator", + "source_commit": "ba746948b7fc7d06a5e8cdbf9601e21c3eaac6f0", + "source_uncommitted": true, + "source_hashes": { + "diff_diff/chained_did.py": "88de46fcee3edf3ff8710ae89fdd356c7cf726e0d21eec569cf6a736e6163e61", + "diff_diff/chained_did_results.py": "e88ee7515d8d1b27432e3e2681c055df386aedace3ed0d811cbaaf218534067a", + "benchmarks/python/validate_chained_did.py": "4b6466659badc98efc56e01bea482fa34c2b4ac3bac783a35a228bc727d097cd" + }, + "cases": [ + { + "n": 1000, + "observation": "balanced", + "clusters": 0, + "null": false, + "violation": "none" + }, + { + "n": 1000, + "observation": "rotating", + "clusters": 0, + "null": false, + "violation": "none" + }, + { + "n": 1000, + "observation": "intermittent", + "clusters": 50, + "null": false, + "violation": "none" + }, + { + "n": 1000, + "observation": "cohort_dependent", + "clusters": 0, + "null": false, + "violation": "none" + }, + { + "n": 1000, + "observation": "rotating", + "clusters": 0, + "null": true, + "violation": "none" + }, + { + "n": 4000, + "observation": "balanced", + "clusters": 0, + "null": false, + "violation": "none" + }, + { + "n": 4000, + "observation": "rotating", + "clusters": 0, + "null": false, + "violation": "none" + }, + { + "n": 4000, + "observation": "intermittent", + "clusters": 100, + "null": false, + "violation": "none" + }, + { + "n": 4000, + "observation": "cohort_dependent", + "clusters": 0, + "null": false, + "violation": "none" + }, + { + "n": 4000, + "observation": "rotating", + "clusters": 0, + "null": true, + "violation": "none" + }, + { + "n": 1000, + "observation": "intermittent", + "clusters": 0, + "null": false, + "violation": "selection" + }, + { + "n": 1000, + "observation": "intermittent", + "clusters": 0, + "null": false, + "violation": "parallel_trends" + }, + { + "n": 1000, + "observation": "intermittent", + "clusters": 0, + "null": false, + "violation": "propensity" + } + ], + "results": { + "1": { + "design": { + "n": 1000, + "observation": "rotating", + "clusters": 0, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 18.858903174999796, + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.0013256551872644629, + "bias_mcse": 0.0036641673176459576, + "rmse": 0.16383093573239874, + "empirical_sd": 0.16386654406378853, + "mean_variance": 0.025548537664320397, + "variance_ratio": 0.9514488775574838, + "coverage": 0.9435, + "coverage_mcse": 0.005162739098579358, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9435, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.6258755434912127, + "meets_prespecified_thresholds": true + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": 0.002774992889462702, + "bias_mcse": 0.004937892250870673, + "rmse": 0.22079147983577777, + "empirical_sd": 0.22082925477032542, + "mean_variance": 0.04493583687007351, + "variance_ratio": 0.9214666475479428, + "coverage": 0.9365, + "coverage_mcse": 0.005452877680637995, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9365, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.8297870183649659, + "meets_prespecified_thresholds": true + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": -0.0020128670697885376, + "bias_mcse": 0.005820917079220396, + "rmse": 0.26026202151777145, + "empirical_sd": 0.2603193256105267, + "mean_variance": 0.06421370709386517, + "variance_ratio": 0.9475778965601165, + "coverage": 0.934, + "coverage_mcse": 0.005551756478809205, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.934, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.9914464513011779, + "meets_prespecified_thresholds": false + } + } + }, + "3": { + "design": { + "n": 1000, + "observation": "cohort_dependent", + "clusters": 0, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 19.416340782001498, + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.0015496974511757244, + "bias_mcse": 0.003907656749463025, + "rmse": 0.17471890087842942, + "empirical_sd": 0.1747557224907038, + "mean_variance": 0.028893612297839814, + "variance_ratio": 0.946104328014614, + "coverage": 0.9395, + "coverage_mcse": 0.005331029450303197, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9395, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.6655179269263534, + "meets_prespecified_thresholds": true + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": 0.005460381896478683, + "bias_mcse": 0.0049154542412676175, + "rmse": 0.21983865646434667, + "empirical_sd": 0.21982579647528092, + "mean_variance": 0.04639954055058544, + "variance_ratio": 0.9601882108884698, + "coverage": 0.9415, + "coverage_mcse": 0.0052477495176503994, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9415, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.8430647808954038, + "meets_prespecified_thresholds": true + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": -0.005129287755606615, + "bias_mcse": 0.006536372142383926, + "rmse": 0.2922873705782742, + "empirical_sd": 0.2923154487321279, + "mean_variance": 0.08509969792997686, + "variance_ratio": 0.9959200645361632, + "coverage": 0.9465, + "coverage_mcse": 0.005031786462082825, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9465, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 1.1403671360999714, + "meets_prespecified_thresholds": true + } + } + }, + "2": { + "design": { + "n": 1000, + "observation": "intermittent", + "clusters": 50, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 34.227761035001095, + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": -0.0018675186735697125, + "bias_mcse": 0.0028023808763087943, + "rmse": 0.12530886416859233, + "empirical_sd": 0.1253262827654379, + "mean_variance": 0.014655582320497395, + "variance_ratio": 0.933079745566396, + "coverage": 0.9315, + "coverage_mcse": 0.0056483515294287415, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9315, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.47085526795173815, + "meets_prespecified_thresholds": false + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.0015688192941309964, + "bias_mcse": 0.004165970509492481, + "rmse": 0.18626788895462837, + "empirical_sd": 0.18630786502969243, + "mean_variance": 0.031114103336694995, + "variance_ratio": 0.8963856832298651, + "coverage": 0.93, + "coverage_mcse": 0.005705260730238364, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.93, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.685192690602935, + "meets_prespecified_thresholds": false + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.0023287799291238365, + "bias_mcse": 0.0048261643612166685, + "rmse": 0.21579123299261396, + "empirical_sd": 0.2158326316453464, + "mean_variance": 0.04540021239756534, + "variance_ratio": 0.9745938632351944, + "coverage": 0.9445, + "coverage_mcse": 0.005119558086397692, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9445, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.8279313992030578, + "meets_prespecified_thresholds": true + } + } + }, + "0": { + "design": { + "n": 1000, + "observation": "balanced", + "clusters": 0, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 35.40982071500002, + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.00036592089574050135, + "bias_mcse": 0.0015200420591149693, + "rmse": 0.06796233584271082, + "empirical_sd": 0.06797834745679651, + "mean_variance": 0.004347523430277386, + "variance_ratio": 0.9408074022304718, + "coverage": 0.945, + "coverage_mcse": 0.005097793640389931, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.945, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.25834438987067615, + "meets_prespecified_thresholds": true + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": 0.0012659412842261908, + "bias_mcse": 0.0021369304737643173, + "rmse": 0.09555092796457071, + "empirical_sd": 0.09556643605055688, + "mean_variance": 0.008682896360888506, + "variance_ratio": 0.9507226417539414, + "coverage": 0.948, + "coverage_mcse": 0.0049646752159632785, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.948, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.3650639344195169, + "meets_prespecified_thresholds": true + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": -0.003015294603054386, + "bias_mcse": 0.0025523995811574315, + "rmse": 0.11415806805349578, + "empirical_sd": 0.11414677938420016, + "mean_variance": 0.01268427857177705, + "variance_ratio": 0.9735055827179331, + "coverage": 0.9455, + "coverage_mcse": 0.005075911248239078, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9455, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.4410747171206066, + "meets_prespecified_thresholds": true + } + } + }, + "4": { + "design": { + "n": 1000, + "observation": "rotating", + "clusters": 0, + "null": true, + "violation": "none" + }, + "elapsed_seconds": 18.296180182998796, + "failures": {}, + "targets": { + "overall": { + "truth": 0.0, + "replications": 2000, + "failures": 0, + "bias": 0.0003460631825616036, + "bias_mcse": 0.0035201479344368226, + "rmse": 0.15738682053744418, + "empirical_sd": 0.15742580144512416, + "mean_variance": 0.024893088143610506, + "variance_ratio": 1.0044468265918003, + "coverage": 0.9455, + "coverage_mcse": 0.005075911248239078, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9455, + "rejection_at_zero": 0.0545, + "rejection_mcse": 0.005075911248239078, + "mean_interval_width": 0.6178502057376661, + "meets_prespecified_thresholds": true + }, + "g2_t3": { + "truth": 0.0, + "replications": 2000, + "failures": 0, + "bias": 0.0002146792937239449, + "bias_mcse": 0.004660468772842476, + "rmse": 0.2083704981116478, + "empirical_sd": 0.20842249966181603, + "mean_variance": 0.04360960481309825, + "variance_ratio": 1.0039057709150472, + "coverage": 0.948, + "coverage_mcse": 0.0049646752159632785, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.948, + "rejection_at_zero": 0.052, + "rejection_mcse": 0.004964675215963275, + "mean_interval_width": 0.8175040603276089, + "meets_prespecified_thresholds": true + }, + "g3_t4": { + "truth": 0.0, + "replications": 2000, + "failures": 0, + "bias": 0.0015798301089758496, + "bias_mcse": 0.005585720691054373, + "rmse": 0.24974356220827923, + "empirical_sd": 0.2498010233704936, + "mean_variance": 0.06221043518752747, + "variance_ratio": 0.996953294714102, + "coverage": 0.9505, + "coverage_mcse": 0.004850244839180801, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9505, + "rejection_at_zero": 0.0495, + "rejection_mcse": 0.004850244839180801, + "mean_interval_width": 0.9758904417243588, + "meets_prespecified_thresholds": true + } + } + }, + "6": { + "design": { + "n": 4000, + "observation": "rotating", + "clusters": 0, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 28.870721263001542, + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": -0.0015232495500052678, + "bias_mcse": 0.0018107538784726393, + "rmse": 0.08097345661629247, + "empirical_sd": 0.0809793752557243, + "mean_variance": 0.00633021194173059, + "variance_ratio": 0.9653157830321725, + "coverage": 0.949, + "coverage_mcse": 0.004919298730510276, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.949, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.3118086571549198, + "meets_prespecified_thresholds": true + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.0015608290043358508, + "bias_mcse": 0.0024028837591007777, + "rmse": 0.10744469766499228, + "empirical_sd": 0.10746022854759137, + "mean_variance": 0.011112285449619876, + "variance_ratio": 0.9622942020704252, + "coverage": 0.9515, + "coverage_mcse": 0.004803527349771207, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9515, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.41309391985051713, + "meets_prespecified_thresholds": true + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": -0.0024411481268087343, + "bias_mcse": 0.0028390873884072344, + "rmse": 0.12695957303411204, + "empirical_sd": 0.1269678478908185, + "mean_variance": 0.015941234494283155, + "variance_ratio": 0.9888591434346066, + "coverage": 0.9505, + "coverage_mcse": 0.004850244839180801, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9505, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.4947158090946397, + "meets_prespecified_thresholds": true + } + } + }, + "8": { + "design": { + "n": 4000, + "observation": "cohort_dependent", + "clusters": 0, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 29.87254005000068, + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.0012596399858137985, + "bias_mcse": 0.001954632812992143, + "rmse": 0.08740105817844195, + "empirical_sd": 0.08741383681804134, + "mean_variance": 0.007238073932202719, + "variance_ratio": 0.9472457140381726, + "coverage": 0.9425, + "coverage_mcse": 0.005205465877325487, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9425, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.3333978693676465, + "meets_prespecified_thresholds": true + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": 0.002928423947319292, + "bias_mcse": 0.002457049552410634, + "rmse": 0.10989414716132441, + "empirical_sd": 0.1098825964655122, + "mean_variance": 0.011595976227857263, + "variance_ratio": 0.9603941153868694, + "coverage": 0.943, + "coverage_mcse": 0.005184158562389852, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.943, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.42195395789633877, + "meets_prespecified_thresholds": true + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": -0.0013388912616920656, + "bias_mcse": 0.003320245609996749, + "rmse": 0.14845480935390842, + "empirical_sd": 0.14848589771895973, + "mean_variance": 0.02105901807725185, + "variance_ratio": 0.9551414653965952, + "coverage": 0.944, + "coverage_mcse": 0.005141206084179083, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.944, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.568494209479711, + "meets_prespecified_thresholds": true + } + } + }, + "5": { + "design": { + "n": 4000, + "observation": "balanced", + "clusters": 0, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 60.03162533499926, + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.00034006704997979753, + "bias_mcse": 0.0007426381671007187, + "rmse": 0.03320522592334943, + "empirical_sd": 0.0332117884864611, + "mean_variance": 0.0010836992455515396, + "variance_ratio": 0.9824811896337062, + "coverage": 0.9515, + "coverage_mcse": 0.004803527349771207, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9515, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.1290273475877689, + "meets_prespecified_thresholds": true + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": 0.0005103874893133131, + "bias_mcse": 0.001056910258995275, + "rmse": 0.047257401824821015, + "empirical_sd": 0.04726646370460687, + "mean_variance": 0.0021667216847834556, + "variance_ratio": 0.9698328877335446, + "coverage": 0.9465, + "coverage_mcse": 0.005031786462082825, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9465, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.18243979213314837, + "meets_prespecified_thresholds": true + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": -0.000714268432508991, + "bias_mcse": 0.0012534425743530296, + "rmse": 0.0560461919804368, + "empirical_sd": 0.056055656042914175, + "mean_variance": 0.00316172597138394, + "variance_ratio": 1.0062023964526452, + "coverage": 0.951, + "coverage_mcse": 0.004826955562256609, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.951, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.22036790844110946, + "meets_prespecified_thresholds": true + } + } + }, + "7": { + "design": { + "n": 4000, + "observation": "intermittent", + "clusters": 100, + "null": false, + "violation": "none" + }, + "elapsed_seconds": 53.15926129699983, + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 2.2281944117352737e-05, + "bias_mcse": 0.0013752188381324988, + "rmse": 0.06148628282097788, + "empirical_sd": 0.06150165612005095, + "mean_variance": 0.003674991084307411, + "variance_ratio": 0.9715891774048486, + "coverage": 0.949, + "coverage_mcse": 0.004919298730510276, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.949, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.23663459304380588, + "meets_prespecified_thresholds": true + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": -0.0012636103883512106, + "bias_mcse": 0.0020087886369256837, + "rmse": 0.08982218576225587, + "empirical_sd": 0.08983575889189946, + "mean_variance": 0.007899589939243222, + "variance_ratio": 0.9788272836079557, + "coverage": 0.9455, + "coverage_mcse": 0.005075911248239078, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9455, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.3467875796767015, + "meets_prespecified_thresholds": true + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.002298883696887799, + "bias_mcse": 0.0023604496653877468, + "rmse": 0.1055611594730147, + "empirical_sd": 0.10556251818547269, + "mean_variance": 0.011534624672091199, + "variance_ratio": 1.0351039932273511, + "coverage": 0.954, + "coverage_mcse": 0.0046842288586276416, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.954, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.41901890460500046, + "meets_prespecified_thresholds": true + } + } + }, + "9": { + "design": { + "n": 4000, + "observation": "rotating", + "clusters": 0, + "null": true, + "violation": "none" + }, + "elapsed_seconds": 28.748735406999913, + "failures": {}, + "targets": { + "overall": { + "truth": 0.0, + "replications": 2000, + "failures": 0, + "bias": 0.0025359520068616442, + "bias_mcse": 0.0017237037932036816, + "rmse": 0.07710881552211733, + "empirical_sd": 0.07708637709355344, + "mean_variance": 0.006205114087085194, + "variance_ratio": 1.0442259953302844, + "coverage": 0.955, + "coverage_mcse": 0.004635461142108735, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.955, + "rejection_at_zero": 0.045, + "rejection_mcse": 0.004635461142108733, + "mean_interval_width": 0.30870518752867315, + "meets_prespecified_thresholds": true + }, + "g2_t3": { + "truth": 0.0, + "replications": 2000, + "failures": 0, + "bias": 0.002619145423286187, + "bias_mcse": 0.0022777592812033044, + "rmse": 0.10187269713530119, + "empirical_sd": 0.10186449178303296, + "mean_variance": 0.010866740929719494, + "variance_ratio": 1.0472579545682104, + "coverage": 0.9535, + "coverage_mcse": 0.004708383480558906, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9535, + "rejection_at_zero": 0.0465, + "rejection_mcse": 0.004708383480558906, + "mean_interval_width": 0.40849545750369926, + "meets_prespecified_thresholds": true + }, + "g3_t4": { + "truth": 0.0, + "replications": 2000, + "failures": 0, + "bias": 0.0014901115079163894, + "bias_mcse": 0.002787046225326044, + "rmse": 0.12461824156138707, + "empirical_sd": 0.12464049632526461, + "mean_variance": 0.015409858887253089, + "variance_ratio": 0.9919283944498148, + "coverage": 0.95, + "coverage_mcse": 0.004873397172404485, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.95, + "rejection_at_zero": 0.05, + "rejection_mcse": 0.004873397172404482, + "mean_interval_width": 0.4863972132520547, + "meets_prespecified_thresholds": true + } + } + }, + "10": { + "design": { + "n": 1000, + "observation": "intermittent", + "clusters": 0, + "null": false, + "violation": "selection" + }, + "elapsed_seconds": 30.562752775998888, + "failures": { + "IPW propensity fit failed: Logistic regression did not converge in 25 iterations. Propensity score estimates may be unreliable.": 63 + }, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 63, + "bias": 0.31057869606726385, + "bias_mcse": 0.002773883452392924, + "rmse": 0.33369977791973876, + "empirical_sd": 0.12208238923877858, + "mean_variance": 0.014676948234010409, + "variance_ratio": 0.9847584638155564, + "coverage": 0.263, + "coverage_mcse": 0.009844567029585404, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.2715539494062984, + "rejection_at_zero": 0.9685, + "rejection_mcse": 0.0039056209493497944, + "mean_interval_width": 0.47387103422059695, + "meets_prespecified_thresholds": false + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 63, + "bias": 0.30634690127900877, + "bias_mcse": 0.0038460054703377284, + "rmse": 0.349978953027182, + "empirical_sd": 0.1692679396602611, + "mean_variance": 0.029333879962731096, + "variance_ratio": 1.0238117146352068, + "coverage": 0.54, + "coverage_mcse": 0.011144505372604026, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.5575632421270005, + "rejection_at_zero": 0.9685, + "rejection_mcse": 0.0039056209493497944, + "mean_interval_width": 0.6697519151450257, + "meets_prespecified_thresholds": false + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 63, + "bias": 0.4414022724954214, + "bias_mcse": 0.0055403497908995914, + "rmse": 0.5042443924562886, + "empirical_sd": 0.24383834119205794, + "mean_variance": 0.054556254976283414, + "variance_ratio": 0.9175728611171998, + "coverage": 0.5045, + "coverage_mcse": 0.011179887074563856, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.5209086215797625, + "rejection_at_zero": 0.9685, + "rejection_mcse": 0.0039056209493497944, + "mean_interval_width": 0.9112860177942907, + "meets_prespecified_thresholds": false + } + } + }, + "11": { + "design": { + "n": 1000, + "observation": "intermittent", + "clusters": 0, + "null": false, + "violation": "parallel_trends" + }, + "elapsed_seconds": 29.838820496999688, + "failures": {}, + "targets": { + "overall": { + "truth": 2.5923947479165568, + "replications": 2000, + "failures": 0, + "bias": 0.7520209634666437, + "bias_mcse": 0.0018945118998704136, + "rmse": 0.7567762489636294, + "empirical_sd": 0.0847251478458504, + "mean_variance": 0.006865271942500445, + "variance_ratio": 0.9563857006899881, + "coverage": 0.0, + "coverage_mcse": 0.0, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.0, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.3245309652142032, + "meets_prespecified_thresholds": false + }, + "g2_t3": { + "truth": 2.6180143123990307, + "replications": 2000, + "failures": 0, + "bias": 0.8010906122729633, + "bias_mcse": 0.002626351828109169, + "rmse": 0.809650986041499, + "empirical_sd": 0.11745402440965891, + "mean_variance": 0.013920454765729322, + "variance_ratio": 1.0090614612199305, + "coverage": 0.0, + "coverage_mcse": 0.0, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.0, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.4620349869723595, + "meets_prespecified_thresholds": false + }, + "g3_t4": { + "truth": 3.350038149052618, + "replications": 2000, + "failures": 0, + "bias": 0.8005011108278408, + "bias_mcse": 0.003231716681667597, + "rmse": 0.8134368874504234, + "empirical_sd": 0.1445267636845759, + "mean_variance": 0.02066334773574415, + "variance_ratio": 0.9892456031127933, + "coverage": 0.0, + "coverage_mcse": 0.0, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.0, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.5626509379588227, + "meets_prespecified_thresholds": false + } + } + }, + "12": { + "design": { + "n": 1000, + "observation": "intermittent", + "clusters": 0, + "null": false, + "violation": "propensity" + }, + "elapsed_seconds": 28.44114170300054, + "failures": {}, + "targets": { + "overall": { + "truth": 3.2242670754154847, + "replications": 2000, + "failures": 0, + "bias": 0.1017035416834382, + "bias_mcse": 0.0026887928829709036, + "rmse": 0.15746617134742733, + "empirical_sd": 0.12024647327481154, + "mean_variance": 0.014521820569018594, + "variance_ratio": 1.004329850331975, + "coverage": 0.862, + "coverage_mcse": 0.007712198130234985, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.862, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.46985846101580137, + "meets_prespecified_thresholds": false + }, + "g2_t3": { + "truth": 3.332688995347808, + "replications": 2000, + "failures": 0, + "bias": 0.11776642068682008, + "bias_mcse": 0.0034599678375010364, + "rmse": 0.19442148373323306, + "empirical_sd": 0.15473446569230528, + "mean_variance": 0.025604488950655948, + "variance_ratio": 1.0694044643726814, + "coverage": 0.8775, + "coverage_mcse": 0.007331226022978695, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.8775, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.6232256242218566, + "meets_prespecified_thresholds": false + }, + "g3_t4": { + "truth": 3.3861450053055644, + "replications": 2000, + "failures": 0, + "bias": 0.006593203788076263, + "bias_mcse": 0.003967119476032169, + "rmse": 0.17749311584814073, + "empirical_sd": 0.17741497646542556, + "mean_variance": 0.030694548556901756, + "variance_ratio": 0.9751708132199531, + "coverage": 0.947, + "coverage_mcse": 0.005009540897128201, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.947, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.6851786160389953, + "meets_prespecified_thresholds": true + } + } + } + }, + "external_reference": "not run: required R 4.5.2/cdid/did/DRDID environment unavailable", + "thresholds": "zero failures; |bias|<.1 empirical SD; variance ratio [.9,1.1]; 95% coverage and null 5% rejection within 3 binomial MCSE", + "core_thresholds_pass": false +} diff --git a/benchmarks/data/chained_did/reference-validation.json b/benchmarks/data/chained_did/reference-validation.json new file mode 100644 index 000000000..bf854c9ff --- /dev/null +++ b/benchmarks/data/chained_did/reference-validation.json @@ -0,0 +1,5287 @@ +{ + "status": "complete", + "image": "diff-diff-cdid-reference:1.1.0.0", + "image_id": "sha256:2fadc7192742ae0002562962e48d1b8cf0a37d5f11c737d1709250f4e884fa15", + "environment_sha256": "a5c8d4b30e241795a019514e0d7295728d7bb0b1275177a5920543a340e68937", + "environment_verified": true, + "installed_environment": { + "R": "R version 4.5.2 (2025-10-31)", + "platform": "x86_64-pc-linux-gnu", + "packages": [ + { + "Package": "BH", + "Version": "1.90.0-1", + "Built": "4.5.2", + "_row": "BH" + }, + { + "Package": "bigmemory", + "Version": "4.6.6", + "Built": "4.5.2", + "_row": "bigmemory" + }, + { + "Package": "bigmemory.sri", + "Version": "0.1.8", + "Built": "4.5.2", + "_row": "bigmemory.sri" + }, + { + "Package": "BMisc", + "Version": "1.4.10", + "Built": "4.5.2", + "_row": "BMisc" + }, + { + "Package": "cdid", + "Version": "1.1.0.0", + "Built": "4.5.2", + "_row": "cdid" + }, + { + "Package": "cli", + "Version": "3.6.6", + "Built": "4.5.2", + "_row": "cli" + }, + { + "Package": "cpp11", + "Version": "0.5.5", + "Built": "4.5.2", + "_row": "cpp11" + }, + { + "Package": "data.table", + "Version": "1.18.6.1", + "Built": "4.5.2", + "_row": "data.table" + }, + { + "Package": "did", + "Version": "2.5.1", + "Built": "4.5.2", + "_row": "did" + }, + { + "Package": "docopt", + "Version": "0.7.2", + "Built": "4.5.2", + "_row": "docopt" + }, + { + "Package": "dplyr", + "Version": "1.2.1", + "Built": "4.5.2", + "_row": "dplyr" + }, + { + "Package": "DRDID", + "Version": "1.3.0", + "Built": "4.5.2", + "_row": "DRDID" + }, + { + "Package": "dreamerr", + "Version": "1.5.0", + "Built": "4.5.2", + "_row": "dreamerr" + }, + { + "Package": "farver", + "Version": "2.1.2", + "Built": "4.5.2", + "_row": "farver" + }, + { + "Package": "fastglm", + "Version": "0.1.2", + "Built": "4.5.2", + "_row": "fastglm" + }, + { + "Package": "Formula", + "Version": "1.2-6", + "Built": "4.5.2", + "_row": "Formula" + }, + { + "Package": "generics", + "Version": "0.1.4", + "Built": "4.5.2", + "_row": "generics" + }, + { + "Package": "ggplot2", + "Version": "4.0.3", + "Built": "4.5.2", + "_row": "ggplot2" + }, + { + "Package": "glue", + "Version": "1.8.1", + "Built": "4.5.2", + "_row": "glue" + }, + { + "Package": "gtable", + "Version": "0.3.6", + "Built": "4.5.2", + "_row": "gtable" + }, + { + "Package": "isoband", + "Version": "0.3.0", + "Built": "4.5.2", + "_row": "isoband" + }, + { + "Package": "jsonlite", + "Version": "2.0.0", + "Built": "4.5.2", + "_row": "jsonlite" + }, + { + "Package": "labeling", + "Version": "0.4.3", + "Built": "4.5.2", + "_row": "labeling" + }, + { + "Package": "lifecycle", + "Version": "1.0.5", + "Built": "4.5.2", + "_row": "lifecycle" + }, + { + "Package": "littler", + "Version": "0.3.22", + "Built": "4.5.2", + "_row": "littler" + }, + { + "Package": "magrittr", + "Version": "2.0.5", + "Built": "4.5.2", + "_row": "magrittr" + }, + { + "Package": "pbapply", + "Version": "1.7-5", + "Built": "4.5.2", + "_row": "pbapply" + }, + { + "Package": "pillar", + "Version": "1.11.1", + "Built": "4.5.2", + "_row": "pillar" + }, + { + "Package": "pkgconfig", + "Version": "2.0.3", + "Built": "4.5.2", + "_row": "pkgconfig" + }, + { + "Package": "purrr", + "Version": "1.2.2", + "Built": "4.5.2", + "_row": "purrr" + }, + { + "Package": "R6", + "Version": "2.6.1", + "Built": "4.5.2", + "_row": "R6" + }, + { + "Package": "RColorBrewer", + "Version": "1.1-3", + "Built": "4.5.2", + "_row": "RColorBrewer" + }, + { + "Package": "Rcpp", + "Version": "1.1.2", + "Built": "4.5.2", + "_row": "Rcpp" + }, + { + "Package": "RcppArmadillo", + "Version": "15.6.2-1", + "Built": "4.5.2", + "_row": "RcppArmadillo" + }, + { + "Package": "RcppEigen", + "Version": "0.3.4.0.2", + "Built": "4.5.2", + "_row": "RcppEigen" + }, + { + "Package": "remotes", + "Version": "2.5.0", + "Built": "4.5.2", + "_row": "remotes" + }, + { + "Package": "rlang", + "Version": "1.3.0", + "Built": "4.5.2", + "_row": "rlang" + }, + { + "Package": "S7", + "Version": "0.2.2", + "Built": "4.5.2", + "_row": "S7" + }, + { + "Package": "scales", + "Version": "1.4.0", + "Built": "4.5.2", + "_row": "scales" + }, + { + "Package": "stringi", + "Version": "1.8.9", + "Built": "4.5.2", + "_row": "stringi" + }, + { + "Package": "stringmagic", + "Version": "1.3.0", + "Built": "4.5.2", + "_row": "stringmagic" + }, + { + "Package": "stringr", + "Version": "1.6.0", + "Built": "4.5.2", + "_row": "stringr" + }, + { + "Package": "tibble", + "Version": "3.3.1", + "Built": "4.5.2", + "_row": "tibble" + }, + { + "Package": "tidyr", + "Version": "1.3.2", + "Built": "4.5.2", + "_row": "tidyr" + }, + { + "Package": "tidyselect", + "Version": "1.2.1", + "Built": "4.5.2", + "_row": "tidyselect" + }, + { + "Package": "trust", + "Version": "0.1-9", + "Built": "4.5.2", + "_row": "trust" + }, + { + "Package": "utf8", + "Version": "1.2.6", + "Built": "4.5.2", + "_row": "utf8" + }, + { + "Package": "uuid", + "Version": "1.2-2", + "Built": "4.5.2", + "_row": "uuid" + }, + { + "Package": "vctrs", + "Version": "0.7.3", + "Built": "4.5.2", + "_row": "vctrs" + }, + { + "Package": "viridisLite", + "Version": "0.4.3", + "Built": "4.5.2", + "_row": "viridisLite" + }, + { + "Package": "withr", + "Version": "3.0.3", + "Built": "4.5.2", + "_row": "withr" + }, + { + "Package": "base", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "base" + }, + { + "Package": "boot", + "Version": "1.3-32", + "Built": "4.5.2", + "_row": "boot" + }, + { + "Package": "class", + "Version": "7.3-23", + "Built": "4.5.2", + "_row": "class" + }, + { + "Package": "cluster", + "Version": "2.1.8.1", + "Built": "4.5.2", + "_row": "cluster" + }, + { + "Package": "codetools", + "Version": "0.2-20", + "Built": "4.5.2", + "_row": "codetools" + }, + { + "Package": "compiler", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "compiler" + }, + { + "Package": "datasets", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "datasets" + }, + { + "Package": "foreign", + "Version": "0.8-90", + "Built": "4.5.2", + "_row": "foreign" + }, + { + "Package": "graphics", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "graphics" + }, + { + "Package": "grDevices", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "grDevices" + }, + { + "Package": "grid", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "grid" + }, + { + "Package": "KernSmooth", + "Version": "2.23-26", + "Built": "4.5.2", + "_row": "KernSmooth" + }, + { + "Package": "lattice", + "Version": "0.22-7", + "Built": "4.5.2", + "_row": "lattice" + }, + { + "Package": "MASS", + "Version": "7.3-65", + "Built": "4.5.2", + "_row": "MASS" + }, + { + "Package": "Matrix", + "Version": "1.7-4", + "Built": "4.5.2", + "_row": "Matrix" + }, + { + "Package": "methods", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "methods" + }, + { + "Package": "mgcv", + "Version": "1.9-3", + "Built": "4.5.2", + "_row": "mgcv" + }, + { + "Package": "nlme", + "Version": "3.1-168", + "Built": "4.5.2", + "_row": "nlme" + }, + { + "Package": "nnet", + "Version": "7.3-20", + "Built": "4.5.2", + "_row": "nnet" + }, + { + "Package": "parallel", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "parallel" + }, + { + "Package": "rpart", + "Version": "4.1.24", + "Built": "4.5.2", + "_row": "rpart" + }, + { + "Package": "spatial", + "Version": "7.3-18", + "Built": "4.5.2", + "_row": "spatial" + }, + { + "Package": "splines", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "splines" + }, + { + "Package": "stats", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "stats" + }, + { + "Package": "stats4", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "stats4" + }, + { + "Package": "survival", + "Version": "3.8-3", + "Built": "4.5.2", + "_row": "survival" + }, + { + "Package": "tcltk", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "tcltk" + }, + { + "Package": "tools", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "tools" + }, + { + "Package": "utils", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "utils" + } + ] + }, + "source_hashes": { + "benchmarks/R/benchmark_chained_did.R": "cb220128e55379e4d836991804c525b953cec12b119053d4819ec969f6799096", + "benchmarks/R/inspect_chained_did_weights.R": "1e0304191d4d88d6238ef74594c2fb1cd921b77364927f55b114252d1c7bbfb0", + "diff_diff/chained_did.py": "88de46fcee3edf3ff8710ae89fdd356c7cf726e0d21eec569cf6a736e6163e61", + "diff_diff/chained_did_results.py": "e88ee7515d8d1b27432e3e2681c055df386aedace3ed0d811cbaaf218534067a", + "benchmarks/python/compare_chained_did.py": "2ffc856f391ab0ef22f5db05ce9f69c6d49b6c058d505bd40dd42a1cc2377665" + }, + "cases": { + "rotating": { + "passed": true, + "checks": { + "registry": true, + "labels": true, + "support": true, + "estimates": true, + "covariance": true, + "overall_estimate": true, + "overall_se": true, + "event_labels": true, + "event_estimates": true, + "event_reference_roles": true, + "event_reference_se": true, + "event_se": true, + "event_covariance": true + }, + "covariance_max_abs_error": 1.3322676295501878e-15, + "cluster_cr1_factor_applied": 1.0, + "tolerances": { + "estimate_rtol": 1e-06, + "estimate_atol": 8.60685855491986e-07, + "covariance_rtol": 1e-05, + "covariance_atol": 7.407801418439719e-06 + }, + "reference": { + "metadata": { + "R": "4.5.2", + "packages": { + "cdid": "1.1.0.0", + "did": "2.5.1", + "DRDID": "1.3.0" + }, + "cdid_sha": "1039120668cceabd8111cad82d32ab3927c2b94f", + "n": 24, + "covariance_scale": "estimator", + "cluster_correction": "none" + }, + "cells": [ + { + "group": 2, + "time": 0, + "att": -2, + "available": true + }, + { + "group": 2, + "time": 1, + "att": 0, + "available": true + }, + { + "group": 2, + "time": 2, + "att": 2.9999999999999996, + "available": true + }, + { + "group": 2, + "time": 3, + "att": 7.000000000000001, + "available": true + } + ], + "vcov": [ + [ + 0.625, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0.6249999999999997, + 0.6249999999999999 + ], + [ + 0, + 0, + 0.6249999999999999, + 1.25 + ] + ], + "overall": { + "att": 5, + "se": 0.8838834764831844 + }, + "events": { + "event_time": [ + -2, + -1, + 0, + 1 + ], + "att": [ + -2, + 0, + 2.9999999999999996, + 7.000000000000001 + ], + "se": [ + 0.7905694150420949, + 0, + 0.7905694150420947, + 1.118033988749895 + ], + "is_reference": [ + false, + true, + false, + false + ], + "vcov": [ + [ + 0.625, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0.6249999999999997, + 0.6249999999999999 + ], + [ + 0, + 0, + 0.6249999999999999, + 1.25 + ] + ] + } + }, + "native": { + "att": 5.000000000000002, + "se": 0.8838834764831848, + "t_stat": 5.65685424949238, + "p_value": 1.541725790028002e-08, + "conf_int": [ + 3.267620219562904, + 6.7323797804371 + ], + "alpha": 0.05, + "n_units": 24, + "group_time_effects": [ + { + "group": 2, + "time": 0, + "att": -2.0, + "se": 0.7905694150420949, + "t_stat": -2.5298221281347035, + "p_value": 0.011412036386001651, + "conf_int_lower": -3.549487580761404, + "conf_int_upper": -0.4505124192385961, + "n_treated": 12, + "n_control": 12, + "available": true, + "status": "ok" + }, + { + "group": 2, + "time": 1, + "att": 0.0, + "se": 0.0, + "t_stat": null, + "p_value": null, + "conf_int_lower": null, + "conf_int_upper": null, + "n_treated": 12, + "n_control": 12, + "available": true, + "status": "ok" + }, + { + "group": 2, + "time": 2, + "att": 3.000000000000001, + "se": 0.7905694150420949, + "t_stat": 3.794733192202056, + "p_value": 0.0001478023103344534, + "conf_int_lower": 1.450512419238597, + "conf_int_upper": 4.549487580761404, + "n_treated": 12, + "n_control": 12, + "available": true, + "status": "ok" + }, + { + "group": 2, + "time": 3, + "att": 7.0000000000000036, + "se": 1.1180339887498953, + "t_stat": 6.260990336999411, + "p_value": 3.8254023259386257e-10, + "conf_int_lower": 4.808693648558549, + "conf_int_upper": 9.191306351441458, + "n_treated": 12, + "n_control": 12, + "available": true, + "status": "ok" + } + ], + "vcov_index": [ + [ + 2, + 0 + ], + [ + 2, + 1 + ], + [ + 2, + 2 + ], + [ + 2, + 3 + ] + ], + "vcov": [ + [ + 0.625, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.625, + 0.6250000000000006 + ], + [ + 0.0, + 0.0, + 0.6250000000000006, + 1.2500000000000013 + ] + ], + "diagnostics": { + "catalogue": [ + [ + 2, + 0, + 1 + ], + [ + 2, + 1, + 2 + ], + [ + 2, + 2, + 3 + ] + ], + "links": [ + { + "group": 2, + "s": 0, + "t": 1, + "att": 2.0, + "n_treated": 4, + "n_control": 4, + "propensity_min": 0.5, + "propensity_max": 0.5, + "control_ess": 4.0 + }, + { + "group": 2, + "s": 1, + "t": 2, + "att": 3.0, + "n_treated": 4, + "n_control": 4, + "propensity_min": 0.5, + "propensity_max": 0.5, + "control_ess": 4.0 + }, + { + "group": 2, + "s": 2, + "t": 3, + "att": 4.0, + "n_treated": 4, + "n_control": 4, + "propensity_min": 0.5, + "propensity_max": 0.5, + "control_ess": 4.0 + } + ], + "rank": { + "2": 3 + }, + "gmm_weighting": "identity", + "selection": "none" + } + }, + "input_sha256": "cd7268d27011ceb51f8d31c2064721c4f03370b0b34655901c6d5414cc861924" + }, + "earlier_bridge": { + "passed": true, + "checks": { + "registry": true, + "labels": true, + "support": true, + "estimates": true, + "covariance": true, + "overall_estimate": true, + "overall_se": true, + "event_labels": true, + "event_estimates": true, + "event_reference_roles": true, + "event_reference_se": true, + "event_se": true, + "event_covariance": true + }, + "covariance_max_abs_error": 4.440892098500626e-15, + "cluster_cr1_factor_applied": 1.0, + "tolerances": { + "estimate_rtol": 1e-06, + "estimate_atol": 8.950581166930248e-07, + "covariance_rtol": 1e-05, + "covariance_atol": 8.011290322580645e-06 + }, + "reference": { + "metadata": { + "R": "4.5.2", + "packages": { + "cdid": "1.1.0.0", + "did": "2.5.1", + "DRDID": "1.3.0" + }, + "cdid_sha": "1039120668cceabd8111cad82d32ab3927c2b94f", + "n": 16, + "covariance_scale": "estimator", + "cluster_correction": "none" + }, + "cells": [ + { + "group": 2, + "time": 0, + "att": -2.0000000000000004, + "available": true + }, + { + "group": 2, + "time": 1, + "att": 0, + "available": true + }, + { + "group": 2, + "time": 2, + "att": null, + "available": false + }, + { + "group": 2, + "time": 3, + "att": 7.000000000000001, + "available": true + } + ], + "vcov": [ + [ + 0.625, + 0, + null, + 0.625 + ], + [ + 0, + 0, + null, + 0 + ], + [ + null, + null, + null, + null + ], + [ + 0.625, + 0, + null, + 6.25 + ] + ], + "overall": { + "att": null, + "se": null + }, + "events": { + "event_time": [ + -2, + -1, + 0, + 1 + ], + "att": [ + -2.0000000000000004, + 0, + null, + 7.000000000000001 + ], + "se": [ + 0.7905694150420949, + 0, + null, + 2.5 + ], + "is_reference": [ + false, + true, + false, + false + ], + "vcov": [ + [ + 0.625, + 0, + null, + 0.625 + ], + [ + 0, + 0, + null, + 0 + ], + [ + null, + null, + null, + null + ], + [ + 0.625, + 0, + null, + 6.25 + ] + ] + } + }, + "native": { + "att": null, + "se": null, + "t_stat": null, + "p_value": null, + "conf_int": [ + null, + null + ], + "alpha": 0.05, + "n_units": 16, + "group_time_effects": [ + { + "group": 2, + "time": 0, + "att": -2.0, + "se": 0.7905694150420949, + "t_stat": -2.5298221281347035, + "p_value": 0.011412036386001651, + "conf_int_lower": -3.549487580761404, + "conf_int_upper": -0.4505124192385961, + "n_treated": 8, + "n_control": 8, + "available": true, + "status": "ok" + }, + { + "group": 2, + "time": 1, + "att": 0.0, + "se": 0.0, + "t_stat": null, + "p_value": null, + "conf_int_lower": null, + "conf_int_upper": null, + "n_treated": 8, + "n_control": 8, + "available": true, + "status": "ok" + }, + { + "group": 2, + "time": 2, + "att": null, + "se": null, + "t_stat": null, + "p_value": null, + "conf_int_lower": null, + "conf_int_upper": null, + "n_treated": 8, + "n_control": 8, + "available": false, + "status": "unsupported_target" + }, + { + "group": 2, + "time": 3, + "att": 7.0, + "se": 2.500000000000001, + "t_stat": 2.799999999999999, + "p_value": 0.005110260660855878, + "conf_int_lower": 2.1000900386498635, + "conf_int_upper": 11.899909961350136, + "n_treated": 8, + "n_control": 8, + "available": true, + "status": "ok" + } + ], + "vcov_index": [ + [ + 2, + 0 + ], + [ + 2, + 1 + ], + [ + 2, + 2 + ], + [ + 2, + 3 + ] + ], + "vcov": [ + [ + 0.625, + 0.0, + null, + 0.6250000000000004 + ], + [ + 0.0, + 0.0, + null, + 0.0 + ], + [ + null, + null, + null, + null + ], + [ + 0.6250000000000004, + 0.0, + null, + 6.250000000000004 + ] + ], + "diagnostics": { + "catalogue": [ + [ + 2, + 0, + 1 + ], + [ + 2, + 0, + 3 + ] + ], + "links": [ + { + "group": 2, + "s": 0, + "t": 1, + "att": 2.0, + "n_treated": 4, + "n_control": 4, + "propensity_min": 0.5, + "propensity_max": 0.5, + "control_ess": 4.0 + }, + { + "group": 2, + "s": 0, + "t": 3, + "att": 9.0, + "n_treated": 4, + "n_control": 4, + "propensity_min": 0.5, + "propensity_max": 0.5, + "control_ess": 4.0 + } + ], + "rank": { + "2": 2 + }, + "gmm_weighting": "identity", + "selection": "none" + } + }, + "input_sha256": "951631414ba002391942af4a285c648dd0c77388927026a6b3e8d084d416d53a" + }, + "nonadjacent": { + "passed": true, + "checks": { + "registry": true, + "labels": true, + "support": true, + "estimates": true, + "covariance": true, + "overall_estimate": true, + "overall_se": true, + "event_labels": true, + "event_estimates": true, + "event_reference_roles": true, + "event_reference_se": true, + "event_se": true, + "event_covariance": true + }, + "covariance_max_abs_error": 0.0, + "cluster_cr1_factor_applied": 1.0, + "tolerances": { + "estimate_rtol": 1e-06, + "estimate_atol": 9.902861539306035e-07, + "covariance_rtol": 1e-05, + "covariance_atol": 9.806666666666667e-06 + }, + "reference": { + "metadata": { + "R": "4.5.2", + "packages": { + "cdid": "1.1.0.0", + "did": "2.5.1", + "DRDID": "1.3.0" + }, + "cdid_sha": "1039120668cceabd8111cad82d32ab3927c2b94f", + "n": 8, + "covariance_scale": "estimator", + "cluster_correction": "none" + }, + "cells": [ + { + "group": 2, + "time": 0, + "att": null, + "available": false + }, + { + "group": 2, + "time": 1, + "att": 0, + "available": true + }, + { + "group": 2, + "time": 2, + "att": null, + "available": false + }, + { + "group": 2, + "time": 3, + "att": 7, + "available": true + } + ], + "vcov": [ + [ + null, + null, + null, + null + ], + [ + null, + 0, + null, + 0 + ], + [ + null, + null, + null, + null + ], + [ + null, + 0, + null, + 2.5 + ] + ], + "overall": { + "att": null, + "se": null + }, + "events": { + "event_time": [ + -2, + -1, + 0, + 1 + ], + "att": [ + null, + 0, + null, + 7 + ], + "se": [ + null, + 0, + null, + 1.5811388300841898 + ], + "is_reference": [ + false, + true, + false, + false + ], + "vcov": [ + [ + null, + null, + null, + null + ], + [ + null, + 0, + null, + 0 + ], + [ + null, + null, + null, + null + ], + [ + null, + 0, + null, + 2.5 + ] + ] + } + }, + "native": { + "att": null, + "se": null, + "t_stat": null, + "p_value": null, + "conf_int": [ + null, + null + ], + "alpha": 0.05, + "n_units": 8, + "group_time_effects": [ + { + "group": 2, + "time": 0, + "att": null, + "se": null, + "t_stat": null, + "p_value": null, + "conf_int_lower": null, + "conf_int_upper": null, + "n_treated": 4, + "n_control": 4, + "available": false, + "status": "unsupported_direct_pre_comparison" + }, + { + "group": 2, + "time": 1, + "att": 0.0, + "se": 0.0, + "t_stat": null, + "p_value": null, + "conf_int_lower": null, + "conf_int_upper": null, + "n_treated": 4, + "n_control": 4, + "available": true, + "status": "ok" + }, + { + "group": 2, + "time": 2, + "att": null, + "se": null, + "t_stat": null, + "p_value": null, + "conf_int_lower": null, + "conf_int_upper": null, + "n_treated": 4, + "n_control": 4, + "available": false, + "status": "unsupported_target" + }, + { + "group": 2, + "time": 3, + "att": 7.0, + "se": 1.5811388300841898, + "t_stat": 4.427188724235731, + "p_value": 9.546919845238127e-06, + "conf_int_lower": 3.901024838477192, + "conf_int_upper": 10.098975161522809, + "n_treated": 4, + "n_control": 4, + "available": true, + "status": "ok" + } + ], + "vcov_index": [ + [ + 2, + 0 + ], + [ + 2, + 1 + ], + [ + 2, + 2 + ], + [ + 2, + 3 + ] + ], + "vcov": [ + [ + null, + null, + null, + null + ], + [ + null, + 0.0, + null, + 0.0 + ], + [ + null, + null, + null, + null + ], + [ + null, + 0.0, + null, + 2.5 + ] + ], + "diagnostics": { + "catalogue": [ + [ + 2, + 1, + 3 + ] + ], + "links": [ + { + "group": 2, + "s": 1, + "t": 3, + "att": 7.0, + "n_treated": 4, + "n_control": 4, + "propensity_min": 0.5, + "propensity_max": 0.5, + "control_ess": 4.0 + } + ], + "rank": { + "2": 1 + }, + "gmm_weighting": "identity", + "selection": "none" + } + }, + "input_sha256": "cb3717f06656dd95e118cbd100ea1d737d401cb566cb1e3032b5e071021d92c4" + }, + "balanced_binary_x": { + "passed": true, + "checks": { + "registry": true, + "labels": true, + "support": true, + "estimates": true, + "covariance": true, + "overall_estimate": true, + "overall_se": true, + "event_labels": true, + "event_estimates": true, + "event_reference_roles": true, + "event_reference_se": true, + "event_se": true, + "event_covariance": true + }, + "covariance_max_abs_error": 8.673617379884035e-18, + "cluster_cr1_factor_applied": 1.0, + "tolerances": { + "estimate_rtol": 1e-06, + "estimate_atol": 3.749087875965563e-07, + "covariance_rtol": 1e-05, + "covariance_atol": 1.4055659901711977e-06 + }, + "reference": { + "metadata": { + "R": "4.5.2", + "packages": { + "cdid": "1.1.0.0", + "did": "2.5.1", + "DRDID": "1.3.0" + }, + "cdid_sha": "1039120668cceabd8111cad82d32ab3927c2b94f", + "n": 800, + "covariance_scale": "estimator", + "cluster_correction": "none" + }, + "cells": [ + { + "group": 2, + "time": 0, + "att": -1.9954170665707653, + "available": true + }, + { + "group": 2, + "time": 1, + "att": 0, + "available": true + }, + { + "group": 2, + "time": 2, + "att": 1.4124819485128892, + "available": true + }, + { + "group": 2, + "time": 3, + "att": 2.683666256488769, + "available": true + }, + { + "group": 2, + "time": 4, + "att": 3.9121373906057273, + "available": true + }, + { + "group": 3, + "time": 0, + "att": -1.8471945035233281, + "available": true + }, + { + "group": 3, + "time": 1, + "att": -1.9862888261438718, + "available": true + }, + { + "group": 3, + "time": 2, + "att": 0, + "available": true + }, + { + "group": 3, + "time": 3, + "att": 1.8378469205696195, + "available": true + }, + { + "group": 3, + "time": 4, + "att": 3.4727492155260564, + "available": true + } + ], + "vcov": [ + [ + 0.006880144434107427, + 0, + 0.00202010186311363, + 0.0032976193293707203, + 0.0034567904267746243, + 0.0022671213520037707, + -0.000767131330440558, + 0, + 0.0004265362575645783, + 0.00038035566845477013 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0.00202010186311363, + 0, + 0.007156791201169449, + 0.005800474627120123, + 0.005298370770536371, + -0.00205685466731668, + -0.0028239859977572377, + 0, + -0.0006900266705056777, + -0.0007970883219180522 + ], + [ + 0.0032976193293707203, + 0, + 0.005800474627120123, + 0.011489078495067123, + 0.009528371979955118, + -0.0009402917392464243, + -0.0021339593272515604, + 0, + 0.001964893612678801, + 0.0010977636535178066 + ], + [ + 0.0034567904267746243, + 0, + 0.005298370770536371, + 0.009528371979955118, + 0.014145948041000299, + -0.0008794106769438586, + -0.002026897675839186, + 0, + 0.0012048253049301801, + 0.0032649343206617833 + ], + [ + 0.0022671213520037707, + 0, + -0.00205685466731668, + -0.0009402917392464243, + -0.0008794106769438586, + 0.013722092148201586, + 0.006625741613141686, + 0, + 0.0033686211993197057, + 0.004710473679418281 + ], + [ + -0.000767131330440558, + 0, + -0.0028239859977572377, + -0.0021339593272515604, + -0.002026897675839186, + 0.006625741613141686, + 0.008577305302131622, + 0, + 0.0025332298927948273, + 0.0036433856083358178 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0.0004265362575645783, + 0, + -0.0006900266705056777, + 0.001964893612678801, + 0.0012048253049301801, + 0.0033686211993197057, + 0.0025332298927948273, + 0, + 0.010186624025646051, + 0.007725385801404198 + ], + [ + 0.00038035566845477013, + 0, + -0.0007970883219180522, + 0.0010977636535178066, + 0.0032649343206617833, + 0.004710473679418281, + 0.0036433856083358178, + 0, + 0.007725385801404198, + 0.014763765074850648 + ] + ], + "overall": { + "att": 2.6659707242516832, + "se": 0.07530456318476528 + }, + "events": { + "event_time": [ + -3, + -2, + -1, + 0, + 1, + 2 + ], + "att": [ + -1.8471945035233281, + -1.9924317300789514, + 0, + 1.5515950211352163, + 2.9417311235953036, + 3.9121373906057273 + ], + "se": [ + 0.11714133407214375, + 0.060791342029571885, + 0, + 0.06411228649243655, + 0.08690598749427648, + 0.11893673966020886 + ], + "is_reference": [ + false, + false, + true, + false, + false, + false + ], + "vcov": [ + [ + 0.013722092148201588, + 0.0036925820663381826, + 0, + -0.00028248520149849076, + 0.0009077573284929771, + -0.0008794106769438585 + ], + [ + 0.0036925820663381826, + 0.0036955872657563928, + 0, + 0.0006599375457164884, + 0.0015004599033584807, + 0.0016633829969889768 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + -0.00028248520149849076, + 0.0006599375457164884, + 0, + 0.004110385279288262, + 0.0038650345494709187, + 0.003959601184300384 + ], + [ + 0.0009077573284929771, + 0.0015004599033584807, + 0, + 0.0038650345494709187, + 0.00755265066235534, + 0.007479952116538433 + ], + [ + -0.0008794106769438585, + 0.0016633829969889768, + 0, + 0.003959601184300384, + 0.007479952116538433, + 0.014145948041000299 + ] + ] + } + }, + "native": { + "att": 2.6659707242516837, + "se": 0.07530456318476529, + "t_stat": 35.40251229809976, + "p_value": 1.5620616956447664e-274, + "conf_int": [ + 2.5183764925380228, + 2.8135649559653446 + ], + "alpha": 0.05, + "n_units": 800, + "group_time_effects": [ + { + "group": 2, + "time": 0, + "att": -1.9954170665707651, + "se": 0.08294663606263625, + "t_stat": -24.056636426629137, + "p_value": 7.114187454343413e-128, + "conf_int_lower": -2.1579894858922835, + "conf_int_upper": -1.8328446472492468, + "n_treated": 321, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 2, + "time": 1, + "att": 0.0, + "se": 0.0, + "t_stat": null, + "p_value": null, + "conf_int_lower": null, + "conf_int_upper": null, + "n_treated": 321, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 2, + "time": 2, + "att": 1.4124819485128897, + "se": 0.08459782030980145, + "t_stat": 16.696434297483197, + "p_value": 1.391350065694723e-62, + "conf_int_lower": 1.2466732675350878, + "conf_int_upper": 1.5782906294906915, + "n_treated": 321, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 2, + "time": 3, + "att": 2.6836662564887686, + "se": 0.10718711907252249, + "t_stat": 25.037208572356608, + "p_value": 2.4063063477630528e-138, + "conf_int_lower": 2.473583363500018, + "conf_int_upper": 2.893749149477519, + "n_treated": 321, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 2, + "time": 4, + "att": 3.9121373906057264, + "se": 0.11893673966020886, + "t_stat": 32.892589806836284, + "p_value": 2.805197743810744e-237, + "conf_int_lower": 3.6790256644331003, + "conf_int_upper": 4.145249116778352, + "n_treated": 321, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 3, + "time": 0, + "att": -1.8471945035233277, + "se": 0.11714133407214378, + "t_stat": -15.768938591613589, + "p_value": 5.0897957218091575e-56, + "conf_int_lower": -2.076787299405704, + "conf_int_upper": -1.6176017076409512, + "n_treated": 156, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 3, + "time": 1, + "att": -1.9862888261438725, + "se": 0.09261374251228391, + "t_stat": -21.447020412552913, + "p_value": 4.868557908333007e-102, + "conf_int_lower": -2.167808425941415, + "conf_int_upper": -1.80476922634633, + "n_treated": 156, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 3, + "time": 2, + "att": 0.0, + "se": 0.0, + "t_stat": null, + "p_value": null, + "conf_int_lower": null, + "conf_int_upper": null, + "n_treated": 156, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 3, + "time": 3, + "att": 1.8378469205696206, + "se": 0.10092880671862739, + "t_stat": 18.209339635741756, + "p_value": 4.351638579608418e-74, + "conf_int_lower": 1.6400300943985067, + "conf_int_upper": 2.0356637467407346, + "n_treated": 156, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 3, + "time": 4, + "att": 3.472749215526058, + "se": 0.12150623471596281, + "t_stat": 28.58083145810647, + "p_value": 1.1630200818345391e-179, + "conf_int_lower": 3.2346013715857005, + "conf_int_upper": 3.710897059466416, + "n_treated": 156, + "n_control": 323, + "available": true, + "status": "ok" + } + ], + "vcov_index": [ + [ + 2, + 0 + ], + [ + 2, + 1 + ], + [ + 2, + 2 + ], + [ + 2, + 3 + ], + [ + 2, + 4 + ], + [ + 3, + 0 + ], + [ + 3, + 1 + ], + [ + 3, + 2 + ], + [ + 3, + 3 + ], + [ + 3, + 4 + ] + ], + "vcov": [ + [ + 0.006880144434107429, + 0.0, + 0.0020201018631136325, + 0.003297619329370722, + 0.0034567904267746286, + 0.0022671213520037703, + -0.0007671313304405577, + 0.0, + 0.00042653625756457837, + 0.0003803556684547705 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0020201018631136325, + 0.0, + 0.007156791201169456, + 0.005800474627120128, + 0.005298370770536376, + -0.0020568546673166792, + -0.0028239859977572373, + 0.0, + -0.0006900266705056773, + -0.0007970883219180505 + ], + [ + 0.003297619329370722, + 0.0, + 0.005800474627120128, + 0.011489078495067114, + 0.009528371979955118, + -0.0009402917392464232, + -0.002133959327251559, + 0.0, + 0.0019648936126788015, + 0.001097763653517808 + ], + [ + 0.0034567904267746286, + 0.0, + 0.005298370770536376, + 0.009528371979955118, + 0.014145948041000299, + -0.0008794106769438568, + -0.0020268976758391847, + 0.0, + 0.001204825304930181, + 0.003264934320661784 + ], + [ + 0.0022671213520037703, + 0.0, + -0.0020568546673166792, + -0.0009402917392464232, + -0.0008794106769438568, + 0.013722092148201592, + 0.006625741613141684, + 0.0, + 0.003368621199319707, + 0.004710473679418282 + ], + [ + -0.0007671313304405577, + 0.0, + -0.0028239859977572373, + -0.002133959327251559, + -0.0020268976758391847, + 0.006625741613141684, + 0.008577305302131624, + 0.0, + 0.002533229892794828, + 0.0036433856083358156 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.00042653625756457837, + 0.0, + -0.0006900266705056773, + 0.0019648936126788015, + 0.001204825304930181, + 0.003368621199319707, + 0.002533229892794828, + 0.0, + 0.010186624025646044, + 0.0077253858014042 + ], + [ + 0.0003803556684547705, + 0.0, + -0.0007970883219180505, + 0.001097763653517808, + 0.003264934320661784, + 0.004710473679418282, + 0.0036433856083358156, + 0.0, + 0.0077253858014042, + 0.014763765074850646 + ] + ], + "diagnostics": { + "catalogue": [ + [ + 2, + 0, + 1 + ], + [ + 2, + 0, + 2 + ], + [ + 2, + 0, + 3 + ], + [ + 2, + 0, + 4 + ], + [ + 2, + 1, + 2 + ], + [ + 2, + 1, + 3 + ], + [ + 2, + 1, + 4 + ], + [ + 2, + 2, + 3 + ], + [ + 2, + 2, + 4 + ], + [ + 2, + 3, + 4 + ], + [ + 3, + 0, + 1 + ], + [ + 3, + 0, + 2 + ], + [ + 3, + 0, + 3 + ], + [ + 3, + 0, + 4 + ], + [ + 3, + 1, + 2 + ], + [ + 3, + 1, + 3 + ], + [ + 3, + 1, + 4 + ], + [ + 3, + 2, + 3 + ], + [ + 3, + 2, + 4 + ], + [ + 3, + 3, + 4 + ] + ], + "links": [ + { + "group": 2, + "s": 0, + "t": 1, + "att": 1.9954170665707651, + "n_treated": 321, + "n_control": 323, + "propensity_min": 0.394648829431438, + "propensity_max": 0.5884057971014494, + "control_ess": 280.6644126130492 + }, + { + "group": 2, + "s": 0, + "t": 2, + "att": 3.407899015083653, + "n_treated": 321, + "n_control": 323, + "propensity_min": 0.394648829431438, + "propensity_max": 0.5884057971014494, + "control_ess": 280.6644126130492 + }, + { + "group": 2, + "s": 0, + "t": 3, + "att": 4.679083323059533, + "n_treated": 321, + "n_control": 323, + "propensity_min": 0.394648829431438, + "propensity_max": 0.5884057971014494, + "control_ess": 280.6644126130492 + }, + { + "group": 2, + "s": 0, + "t": 4, + "att": 5.907554457176492, + "n_treated": 321, + "n_control": 323, + "propensity_min": 0.394648829431438, + "propensity_max": 0.5884057971014494, + "control_ess": 280.6644126130492 + }, + { + "group": 2, + "s": 1, + "t": 2, + "att": 1.4124819485128888, + "n_treated": 321, + "n_control": 323, + "propensity_min": 0.394648829431438, + "propensity_max": 0.5884057971014494, + "control_ess": 280.6644126130492 + }, + { + "group": 2, + "s": 1, + "t": 3, + "att": 2.6836662564887677, + "n_treated": 321, + "n_control": 323, + "propensity_min": 0.394648829431438, + "propensity_max": 0.5884057971014494, + "control_ess": 280.6644126130492 + }, + { + "group": 2, + "s": 1, + "t": 4, + "att": 3.9121373906057264, + "n_treated": 321, + "n_control": 323, + "propensity_min": 0.394648829431438, + "propensity_max": 0.5884057971014494, + "control_ess": 280.6644126130492 + }, + { + "group": 2, + "s": 2, + "t": 3, + "att": 1.271184307975879, + "n_treated": 321, + "n_control": 323, + "propensity_min": 0.394648829431438, + "propensity_max": 0.5884057971014494, + "control_ess": 280.6644126130492 + }, + { + "group": 2, + "s": 2, + "t": 4, + "att": 2.499655442092836, + "n_treated": 321, + "n_control": 323, + "propensity_min": 0.394648829431438, + "propensity_max": 0.5884057971014494, + "control_ess": 280.6644126130492 + }, + { + "group": 2, + "s": 3, + "t": 4, + "att": 1.2284711341169583, + "n_treated": 321, + "n_control": 323, + "propensity_min": 0.394648829431438, + "propensity_max": 0.5884057971014494, + "control_ess": 280.6644126130492 + }, + { + "group": 3, + "s": 0, + "t": 1, + "att": -0.13909432262054366, + "n_treated": 156, + "n_control": 323, + "propensity_min": 0.29000000000000026, + "propensity_max": 0.3512544802867387, + "control_ess": 317.07765586631615 + }, + { + "group": 3, + "s": 0, + "t": 2, + "att": 1.8471945035233277, + "n_treated": 156, + "n_control": 323, + "propensity_min": 0.29000000000000026, + "propensity_max": 0.3512544802867387, + "control_ess": 317.07765586631615 + }, + { + "group": 3, + "s": 0, + "t": 3, + "att": 3.6850414240929483, + "n_treated": 156, + "n_control": 323, + "propensity_min": 0.29000000000000026, + "propensity_max": 0.3512544802867387, + "control_ess": 317.07765586631615 + }, + { + "group": 3, + "s": 0, + "t": 4, + "att": 5.319943719049386, + "n_treated": 156, + "n_control": 323, + "propensity_min": 0.29000000000000026, + "propensity_max": 0.3512544802867387, + "control_ess": 317.07765586631615 + }, + { + "group": 3, + "s": 1, + "t": 2, + "att": 1.9862888261438725, + "n_treated": 156, + "n_control": 323, + "propensity_min": 0.29000000000000026, + "propensity_max": 0.3512544802867387, + "control_ess": 317.07765586631615 + }, + { + "group": 3, + "s": 1, + "t": 3, + "att": 3.824135746713493, + "n_treated": 156, + "n_control": 323, + "propensity_min": 0.29000000000000026, + "propensity_max": 0.3512544802867387, + "control_ess": 317.07765586631615 + }, + { + "group": 3, + "s": 1, + "t": 4, + "att": 5.459038041669929, + "n_treated": 156, + "n_control": 323, + "propensity_min": 0.29000000000000026, + "propensity_max": 0.3512544802867387, + "control_ess": 317.07765586631615 + }, + { + "group": 3, + "s": 2, + "t": 3, + "att": 1.8378469205696197, + "n_treated": 156, + "n_control": 323, + "propensity_min": 0.29000000000000026, + "propensity_max": 0.3512544802867387, + "control_ess": 317.07765586631615 + }, + { + "group": 3, + "s": 2, + "t": 4, + "att": 3.472749215526057, + "n_treated": 156, + "n_control": 323, + "propensity_min": 0.29000000000000026, + "propensity_max": 0.3512544802867387, + "control_ess": 317.07765586631615 + }, + { + "group": 3, + "s": 3, + "t": 4, + "att": 1.6349022949564367, + "n_treated": 156, + "n_control": 323, + "propensity_min": 0.29000000000000026, + "propensity_max": 0.3512544802867387, + "control_ess": 317.07765586631615 + } + ], + "rank": { + "2": 4, + "3": 4 + }, + "gmm_weighting": "identity", + "selection": "none" + } + }, + "input_sha256": "6fd044a2e08c7fea52959e33f4c5a67ff5517c7286e61c450344bdff024e8820" + }, + "intermittent_shared_cluster": { + "passed": true, + "checks": { + "registry": true, + "labels": true, + "support": true, + "estimates": true, + "covariance": true, + "overall_estimate": true, + "overall_se": true, + "event_labels": true, + "event_estimates": true, + "event_reference_roles": true, + "event_reference_se": true, + "event_se": true, + "event_covariance": true + }, + "covariance_max_abs_error": 6.63358257213531e-15, + "cluster_cr1_factor_applied": 1.0256410256410255, + "tolerances": { + "estimate_rtol": 1e-06, + "estimate_atol": 3.925035020510428e-07, + "covariance_rtol": 1e-05, + "covariance_atol": 1.5405899912233297e-06 + }, + "reference": { + "metadata": { + "R": "4.5.2", + "packages": { + "cdid": "1.1.0.0", + "did": "2.5.1", + "DRDID": "1.3.0" + }, + "cdid_sha": "1039120668cceabd8111cad82d32ab3927c2b94f", + "n": 800, + "covariance_scale": "estimator", + "cluster_correction": "none" + }, + "cells": [ + { + "group": 2, + "time": 0, + "att": -2.162550430669498, + "available": true + }, + { + "group": 2, + "time": 1, + "att": 0, + "available": true + }, + { + "group": 2, + "time": 2, + "att": 1.3388588578850822, + "available": true + }, + { + "group": 2, + "time": 3, + "att": 2.4444087696787884, + "available": true + }, + { + "group": 2, + "time": 4, + "att": 3.7675836836828824, + "available": true + }, + { + "group": 3, + "time": 0, + "att": -1.3315066530674116, + "available": true + }, + { + "group": 3, + "time": 1, + "att": -1.904798463152893, + "available": true + }, + { + "group": 3, + "time": 2, + "att": 0, + "available": true + }, + { + "group": 3, + "time": 3, + "att": 2.02634444500214, + "available": true + }, + { + "group": 3, + "time": 4, + "att": 3.578693607976721, + "available": true + } + ], + "vcov": [ + [ + 0.03644174561402671, + 0, + 0.005886136024113046, + 0.017754606306487192, + 0.014413994638846484, + 0.005226326822602505, + -0.0008802807032479025, + 0, + 0.00821800938879967, + -0.0038958009039510287 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0.005886136024113046, + 0, + 0.03799341221781766, + 0.018820892294348867, + 0.019318046022643785, + -0.022660027970793636, + -0.03173187514103051, + 0, + -0.02053518427683898, + -0.013826692270208352 + ], + [ + 0.017754606306487192, + 0, + 0.018820892294348867, + 0.03921235846578972, + 0.02061744216109519, + -0.004179036673625508, + -0.0011738097488903834, + 0, + 0.0029745619312769663, + -0.001895955840463429 + ], + [ + 0.014413994638846484, + 0, + 0.019318046022643785, + 0.02061744216109519, + 0.04193021148032617, + -0.0012351684507866758, + -0.013857847056465184, + 0, + -0.0031368545607170033, + 0.00571838225469124 + ], + [ + 0.005226326822602505, + 0, + -0.022660027970793636, + -0.004179036673625508, + -0.0012351684507866758, + 0.08154456535799777, + 0.045409546325096084, + 0, + 0.009881026746601887, + 0.0094032155462698 + ], + [ + -0.0008802807032479025, + 0, + -0.03173187514103051, + -0.0011738097488903834, + -0.013857847056465184, + 0.045409546325096084, + 0.07414646968492698, + 0, + 0.0171790354586609, + 0.009939264247333997 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0.00821800938879967, + 0, + -0.02053518427683898, + 0.0029745619312769663, + -0.0031368545607170033, + 0.009881026746601887, + 0.0171790354586609, + 0, + 0.052853183478889634, + 0.016160449728581106 + ], + [ + -0.0038958009039510287, + 0, + -0.013826692270208352, + -0.001895955840463429, + 0.00571838225469124, + 0.0094032155462698, + 0.009939264247333997, + 0, + 0.016160449728581106, + 0.0331597037844302 + ] + ], + "overall": { + "att": 2.5868307507254196, + "se": 0.12172537490563684 + }, + "events": { + "event_time": [ + -3, + -2, + -1, + 0, + 1, + 2 + ], + "att": [ + -1.3315066530674116, + -2.0782541897206714, + 0, + 1.5636969115334283, + 2.8153698488705654, + 3.7675836836828824 + ], + "se": [ + 0.2855600906254195, + 0.15541832937578925, + 0, + 0.11885343410288553, + 0.14431585244151546, + 0.20476867797670173 + ], + "is_reference": [ + false, + false, + true, + false, + false, + false + ], + "vcov": [ + [ + 0.08154456535799777, + 0.018313349503796605, + 0, + -0.012163459410779467, + 2.2420059700185525e-05, + -0.0012351684507866756 + ], + [ + 0.018313349503796605, + 0.02415485710596131, + 0, + -0.0004700258734759338, + 0.008266970524951732, + 0.005000672734749184 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + -0.012163459410779467, + -0.0004700258734759338, + 0, + 0.014126138798048953, + 0.008224095681419982, + 0.011528382938656852 + ], + [ + 2.2420059700185525e-05, + 0.008266970524951732, + 0, + 0.008224095681419982, + 0.020827065265921256, + 0.01500906537347042 + ], + [ + -0.0012351684507866756, + 0.005000672734749184, + 0, + 0.011528382938656852, + 0.01500906537347042, + 0.04193021148032617 + ] + ] + } + }, + "native": { + "att": 2.5868307507254196, + "se": 0.12327607913995597, + "t_stat": 20.984044664403847, + "p_value": 9.174811779546434e-98, + "conf_int": [ + 2.3452140754557966, + 2.8284474259950425 + ], + "alpha": 0.05, + "n_units": 800, + "group_time_effects": [ + { + "group": 2, + "time": 0, + "att": -2.162550430669498, + "se": 0.19332912183041565, + "t_stat": -11.185849344344733, + "p_value": 4.783036721508569e-29, + "conf_int_lower": -2.541468546619869, + "conf_int_upper": -1.7836323147191269, + "n_treated": 321, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 2, + "time": 1, + "att": 0.0, + "se": 0.0, + "t_stat": null, + "p_value": null, + "conf_int_lower": null, + "conf_int_upper": null, + "n_treated": 321, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 2, + "time": 2, + "att": 1.3388588578850817, + "se": 0.19740213340965893, + "t_stat": 6.782393050973841, + "p_value": 1.18201360836234e-11, + "conf_int_lower": 0.9519577859307793, + "conf_int_upper": 1.7257599298393842, + "n_treated": 321, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 2, + "time": 3, + "att": 2.444408769678787, + "se": 0.20054376967299714, + "t_stat": 12.18890406650176, + "p_value": 3.561790656757549e-34, + "conf_int_lower": 2.051350203795817, + "conf_int_upper": 2.8374673355617572, + "n_treated": 321, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 2, + "time": 4, + "att": 3.767583683682881, + "se": 0.20737730133268412, + "t_stat": 18.167772747889856, + "p_value": 9.289324668049035e-74, + "conf_int_lower": 3.36113164185971, + "conf_int_upper": 4.1740357255060525, + "n_treated": 321, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 3, + "time": 0, + "att": -1.3315066530674158, + "se": 0.28919794544435323, + "t_stat": -4.6041359354111355, + "p_value": 4.141818743499468e-06, + "conf_int_lower": -1.8983242105413276, + "conf_int_upper": -0.7646890955935041, + "n_treated": 156, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 3, + "time": 1, + "att": -1.9047984631528943, + "se": 0.2757674041929351, + "t_stat": -6.907264724514868, + "p_value": 4.940874273844249e-12, + "conf_int_lower": -2.445292643481147, + "conf_int_upper": -1.3643042828246417, + "n_treated": 156, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 3, + "time": 2, + "att": 0.0, + "se": 0.0, + "t_stat": null, + "p_value": null, + "conf_int_lower": null, + "conf_int_upper": null, + "n_treated": 156, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 3, + "time": 3, + "att": 2.0263444450021413, + "se": 0.23282696001898326, + "t_stat": 8.70322081616719, + "p_value": 3.2259536378482385e-18, + "conf_int_lower": 1.570011988734987, + "conf_int_upper": 2.4826769012692957, + "n_treated": 156, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 3, + "time": 4, + "att": 3.5786936079767253, + "se": 0.18441787494550257, + "t_stat": 19.405351075833984, + "p_value": 6.954189279627515e-84, + "conf_int_lower": 3.2172412149781286, + "conf_int_upper": 3.940146000975322, + "n_treated": 156, + "n_control": 323, + "available": true, + "status": "ok" + } + ], + "vcov_index": [ + [ + 2, + 0 + ], + [ + 2, + 1 + ], + [ + 2, + 2 + ], + [ + 2, + 3 + ], + [ + 2, + 4 + ], + [ + 3, + 0 + ], + [ + 3, + 1 + ], + [ + 3, + 2 + ], + [ + 3, + 3 + ], + [ + 3, + 4 + ] + ], + "vcov": [ + [ + 0.03737614934771969, + 0.0, + 0.0060370625888336756, + 0.01820985262203815, + 0.01478358424497075, + 0.005360335202667528, + -0.0009028520033314267, + 0.0, + 0.00842872757825562, + -0.003995693234822009 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0060370625888336756, + 0.0, + 0.038967602274684786, + 0.01930347927625501, + 0.019813380536044773, + -0.023241054329018775, + -0.03254551296515961, + 0.0, + -0.021061727463424693, + -0.014181222841239498 + ], + [ + 0.01820985262203815, + 0.0, + 0.01930347927625501, + 0.04021780355465612, + 0.021146094524200194, + -0.004286191460130587, + -0.0012039074347594465, + 0.0, + 0.0030508327500272012, + -0.00194457009278345 + ], + [ + 0.01478358424497075, + 0.0, + 0.019813380536044773, + 0.021146094524200194, + 0.04300534510802687, + -0.0012668394367054657, + -0.014213176468169423, + 0.0, + -0.003217286728940781, + 0.0058650074407086954 + ], + [ + 0.005360335202667528, + 0.0, + -0.023241054329018775, + -0.004286191460130587, + -0.0012668394367054657, + 0.08363545164923511, + 0.04657389366676458, + 0.0, + 0.010134386406771034, + 0.009644323637199345 + ], + [ + -0.0009028520033314267, + 0.0, + -0.03254551296515961, + -0.0012039074347594465, + -0.014213176468169423, + 0.04657389366676458, + 0.07604766121530963, + 0.0, + 0.017619523547344237, + 0.010194117176752452 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.00842872757825562, + 0.0, + -0.021061727463424693, + 0.0030508327500272012, + -0.003217286728940781, + 0.010134386406771034, + 0.017619523547344237, + 0.0, + 0.05420839331168122, + 0.01657482023444165 + ], + [ + -0.003995693234822009, + 0.0, + -0.014181222841239498, + -0.00194457009278345, + 0.0058650074407086954, + 0.009644323637199345, + 0.010194117176752452, + 0.0, + 0.01657482023444165, + 0.03400995259941503 + ] + ], + "diagnostics": { + "catalogue": [ + [ + 2, + 0, + 1 + ], + [ + 2, + 0, + 2 + ], + [ + 2, + 0, + 3 + ], + [ + 2, + 0, + 4 + ], + [ + 2, + 1, + 2 + ], + [ + 2, + 1, + 3 + ], + [ + 2, + 1, + 4 + ], + [ + 2, + 2, + 3 + ], + [ + 2, + 2, + 4 + ], + [ + 2, + 3, + 4 + ], + [ + 3, + 0, + 1 + ], + [ + 3, + 0, + 2 + ], + [ + 3, + 0, + 3 + ], + [ + 3, + 0, + 4 + ], + [ + 3, + 1, + 2 + ], + [ + 3, + 1, + 3 + ], + [ + 3, + 1, + 4 + ], + [ + 3, + 2, + 3 + ], + [ + 3, + 2, + 4 + ], + [ + 3, + 3, + 4 + ] + ], + "links": [ + { + "group": 2, + "s": 0, + "t": 1, + "att": 2.162550430669498, + "n_treated": 156, + "n_control": 156, + "propensity_min": 0.36805555555555575, + "propensity_max": 0.6130952380952379, + "control_ess": 125.38931920097846 + }, + { + "group": 2, + "s": 0, + "t": 2, + "att": 3.712297244361433, + "n_treated": 161, + "n_control": 152, + "propensity_min": 0.42253521126760585, + "propensity_max": 0.590643274853801, + "control_ess": 136.69177809095612 + }, + { + "group": 2, + "s": 0, + "t": 3, + "att": 4.5079261250515295, + "n_treated": 144, + "n_control": 151, + "propensity_min": 0.35251798561151093, + "propensity_max": 0.6089743589743589, + "control_ess": 118.74343780346742 + }, + { + "group": 2, + "s": 0, + "t": 4, + "att": 6.240059303412773, + "n_treated": 152, + "n_control": 151, + "propensity_min": 0.3695652173913041, + "propensity_max": 0.6121212121212123, + "control_ess": 122.05792489404148 + }, + { + "group": 2, + "s": 1, + "t": 2, + "att": 1.3064369813285475, + "n_treated": 164, + "n_control": 146, + "propensity_min": 0.4228187919463088, + "propensity_max": 0.6273291925465836, + "control_ess": 124.42184421019417 + }, + { + "group": 2, + "s": 1, + "t": 3, + "att": 2.375653676899849, + "n_treated": 151, + "n_control": 154, + "propensity_min": 0.3378378378378378, + "propensity_max": 0.643312101910828, + "control_ess": 109.79389978748756 + }, + { + "group": 2, + "s": 1, + "t": 4, + "att": 3.763315635625728, + "n_treated": 153, + "n_control": 148, + "propensity_min": 0.3533834586466167, + "propensity_max": 0.6309523809523809, + "control_ess": 113.13513297935022 + }, + { + "group": 2, + "s": 2, + "t": 3, + "att": 1.259061422909732, + "n_treated": 163, + "n_control": 142, + "propensity_min": 0.4125874125874125, + "propensity_max": 0.6419753086419754, + "control_ess": 116.56994962366335 + }, + { + "group": 2, + "s": 2, + "t": 4, + "att": 2.3482343765394695, + "n_treated": 162, + "n_control": 144, + "propensity_min": 0.3897058823529409, + "propensity_max": 0.6411764705882353, + "control_ess": 114.79618686240353 + }, + { + "group": 2, + "s": 3, + "t": 4, + "att": 1.2034532396518067, + "n_treated": 143, + "n_control": 149, + "propensity_min": 0.31617647058823534, + "propensity_max": 0.6410256410256412, + "control_ess": 103.04195273737736 + }, + { + "group": 3, + "s": 0, + "t": 1, + "att": 0.022144796877110462, + "n_treated": 87, + "n_control": 156, + "propensity_min": 0.3546099290780146, + "propensity_max": 0.362745098039216, + "control_ess": 155.95231626137746 + }, + { + "group": 3, + "s": 0, + "t": 2, + "att": 1.3315066530674158, + "n_treated": 75, + "n_control": 152, + "propensity_min": 0.25531914893617036, + "propensity_max": 0.3834586466165414, + "control_ess": 140.80775571080937 + }, + { + "group": 3, + "s": 0, + "t": 3, + "att": 3.722401605393881, + "n_treated": 86, + "n_control": 151, + "propensity_min": 0.3617021276595748, + "propensity_max": 0.3645833333333337, + "control_ess": 150.9943438535425 + }, + { + "group": 3, + "s": 0, + "t": 4, + "att": 5.297653253675346, + "n_treated": 78, + "n_control": 151, + "propensity_min": 0.3402061855670102, + "propensity_max": 0.3409090909090907, + "control_ess": 150.9996389457215 + }, + { + "group": 3, + "s": 1, + "t": 2, + "att": 1.9047984631528943, + "n_treated": 75, + "n_control": 146, + "propensity_min": 0.27710843373494043, + "propensity_max": 0.3768115942028994, + "control_ess": 139.7219521118353 + }, + { + "group": 3, + "s": 1, + "t": 3, + "att": 3.6806990920389326, + "n_treated": 83, + "n_control": 154, + "propensity_min": 0.3333333333333329, + "propensity_max": 0.37777777777777755, + "control_ess": 152.60443037974682 + }, + { + "group": 3, + "s": 1, + "t": 4, + "att": 5.302283922455791, + "n_treated": 85, + "n_control": 148, + "propensity_min": 0.354166666666667, + "propensity_max": 0.37226277372262817, + "control_ess": 147.7827050997783 + }, + { + "group": 3, + "s": 2, + "t": 3, + "att": 1.7762903683115239, + "n_treated": 68, + "n_control": 142, + "propensity_min": 0.2749999999999998, + "propensity_max": 0.3538461538461538, + "control_ess": 137.88454193801104 + }, + { + "group": 3, + "s": 2, + "t": 4, + "att": 3.7219638396121404, + "n_treated": 70, + "n_control": 144, + "propensity_min": 0.27380952380952367, + "propensity_max": 0.36153846153846153, + "control_ess": 138.86295450474648 + }, + { + "group": 3, + "s": 3, + "t": 4, + "att": 1.3096179324369874, + "n_treated": 75, + "n_control": 149, + "propensity_min": 0.3161764705882357, + "propensity_max": 0.36363636363636404, + "control_ess": 147.37694691512053 + } + ], + "rank": { + "2": 4, + "3": 4 + }, + "gmm_weighting": "identity", + "selection": "none" + } + }, + "input_sha256": "f249d480c136727aa4800d4b264bfc220757ddde1f5900fb906fb17111166d85" + }, + "two_date": { + "passed": true, + "checks": { + "registry": true, + "labels": true, + "support": true, + "estimates": true, + "covariance": true, + "overall_estimate": true, + "overall_se": true, + "event_labels": true, + "event_estimates": true, + "event_reference_roles": true, + "event_reference_se": true, + "event_se": true, + "event_covariance": true + }, + "covariance_max_abs_error": 0.0, + "cluster_cr1_factor_applied": 1.0, + "tolerances": { + "estimate_rtol": 1e-06, + "estimate_atol": 7.611394966320781e-07, + "covariance_rtol": 1e-05, + "covariance_atol": 5.793333333333333e-06 + }, + "reference": { + "metadata": { + "R": "4.5.2", + "packages": { + "cdid": "1.1.0.0", + "did": "2.5.1", + "DRDID": "1.3.0" + }, + "cdid_sha": "1039120668cceabd8111cad82d32ab3927c2b94f", + "n": 8, + "covariance_scale": "estimator", + "cluster_correction": "none" + }, + "cells": [ + { + "group": 2, + "time": 1, + "att": 0, + "available": true + }, + { + "group": 2, + "time": 2, + "att": 3, + "available": true + } + ], + "vcov": [ + [ + 0, + 0 + ], + [ + 0, + 0.625 + ] + ], + "overall": { + "att": 3, + "se": 0.7905694150420949 + }, + "events": { + "event_time": [ + -1, + 0 + ], + "att": [ + 0, + 3 + ], + "se": [ + 0, + 0.7905694150420949 + ], + "is_reference": [ + true, + false + ], + "vcov": [ + [ + 0, + 0 + ], + [ + 0, + 0.625 + ] + ] + } + }, + "native": { + "att": 3.0, + "se": 0.7905694150420949, + "t_stat": 3.794733192202055, + "p_value": 0.0001478023103344541, + "conf_int": [ + 1.450512419238596, + 4.549487580761404 + ], + "alpha": 0.05, + "n_units": 8, + "group_time_effects": [ + { + "group": 2, + "time": 1, + "att": 0.0, + "se": 0.0, + "t_stat": null, + "p_value": null, + "conf_int_lower": null, + "conf_int_upper": null, + "n_treated": 4, + "n_control": 4, + "available": true, + "status": "ok" + }, + { + "group": 2, + "time": 2, + "att": 3.0, + "se": 0.7905694150420949, + "t_stat": 3.794733192202055, + "p_value": 0.0001478023103344541, + "conf_int_lower": 1.450512419238596, + "conf_int_upper": 4.549487580761404, + "n_treated": 4, + "n_control": 4, + "available": true, + "status": "ok" + } + ], + "vcov_index": [ + [ + 2, + 1 + ], + [ + 2, + 2 + ] + ], + "vcov": [ + [ + 0.0, + 0.0 + ], + [ + 0.0, + 0.625 + ] + ], + "diagnostics": { + "catalogue": [ + [ + 2, + 1, + 2 + ] + ], + "links": [ + { + "group": 2, + "s": 1, + "t": 2, + "att": 3.0, + "n_treated": 4, + "n_control": 4, + "propensity_min": 0.5, + "propensity_max": 0.5, + "control_ess": 4.0 + } + ], + "rank": { + "2": 1 + }, + "gmm_weighting": "identity", + "selection": "none" + } + }, + "input_sha256": "fe89ef3559021bb5383f5086d8e5babf92c0931e794b2c1644a17e93bbe6f222" + }, + "registry_units": { + "passed": true, + "checks": { + "registry": true, + "labels": true, + "support": true, + "estimates": true, + "covariance": true, + "overall_estimate": true, + "overall_se": true, + "event_labels": true, + "event_estimates": true, + "event_reference_roles": true, + "event_reference_se": true, + "event_se": true, + "event_covariance": true + }, + "covariance_max_abs_error": 6.647460359943125e-15, + "cluster_cr1_factor_applied": 1.0256410256410255, + "tolerances": { + "estimate_rtol": 1e-06, + "estimate_atol": 3.9244494933578847e-07, + "covariance_rtol": 1e-05, + "covariance_atol": 1.5401303825916957e-06 + }, + "reference": { + "metadata": { + "R": "4.5.2", + "packages": { + "cdid": "1.1.0.0", + "did": "2.5.1", + "DRDID": "1.3.0" + }, + "cdid_sha": "1039120668cceabd8111cad82d32ab3927c2b94f", + "n": 804, + "covariance_scale": "estimator", + "cluster_correction": "none" + }, + "cells": [ + { + "group": 2, + "time": 0, + "att": -2.162550430669498, + "available": true + }, + { + "group": 2, + "time": 1, + "att": 0, + "available": true + }, + { + "group": 2, + "time": 2, + "att": 1.3388588578850822, + "available": true + }, + { + "group": 2, + "time": 3, + "att": 2.4444087696787884, + "available": true + }, + { + "group": 2, + "time": 4, + "att": 3.7675836836828824, + "available": true + }, + { + "group": 3, + "time": 0, + "att": -1.3315066530674116, + "available": true + }, + { + "group": 3, + "time": 1, + "att": -1.904798463152893, + "available": true + }, + { + "group": 3, + "time": 2, + "att": 0, + "available": true + }, + { + "group": 3, + "time": 3, + "att": 2.02634444500214, + "available": true + }, + { + "group": 3, + "time": 4, + "att": 3.578693607976721, + "available": true + } + ], + "vcov": [ + [ + 0.036441745614026716, + 0, + 0.005886136024113044, + 0.017754606306487185, + 0.014413994638846489, + 0.005226326822602511, + -0.0008802807032479012, + 0, + 0.00821800938879967, + -0.0038958009039510295 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0.005886136024113044, + 0, + 0.03799341221781766, + 0.018820892294348867, + 0.019318046022643788, + -0.022660027970793643, + -0.03173187514103052, + 0, + -0.02053518427683898, + -0.013826692270208349 + ], + [ + 0.017754606306487185, + 0, + 0.018820892294348867, + 0.03921235846578973, + 0.020617442161095192, + -0.004179036673625506, + -0.001173809748890383, + 0, + 0.002974561931276964, + -0.0018959558404634286 + ], + [ + 0.014413994638846489, + 0, + 0.019318046022643788, + 0.020617442161095192, + 0.04193021148032618, + -0.001235168450786677, + -0.013857847056465193, + 0, + -0.003136854560717001, + 0.00571838225469124 + ], + [ + 0.005226326822602511, + 0, + -0.022660027970793643, + -0.004179036673625506, + -0.001235168450786677, + 0.08154456535799776, + 0.045409546325096105, + 0, + 0.009881026746601887, + 0.009403215546269798 + ], + [ + -0.0008802807032479012, + 0, + -0.03173187514103052, + -0.001173809748890383, + -0.013857847056465193, + 0.045409546325096105, + 0.07414646968492698, + 0, + 0.0171790354586609, + 0.009939264247333992 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0.00821800938879967, + 0, + -0.02053518427683898, + 0.002974561931276964, + -0.003136854560717001, + 0.009881026746601887, + 0.0171790354586609, + 0, + 0.052853183478889634, + 0.016160449728581102 + ], + [ + -0.0038958009039510295, + 0, + -0.013826692270208349, + -0.0018959558404634286, + 0.00571838225469124, + 0.009403215546269798, + 0.009939264247333992, + 0, + 0.016160449728581102, + 0.033159703784430214 + ] + ], + "overall": { + "att": 2.588175650340472, + "se": 0.12120693887655047 + }, + "events": { + "event_time": [ + -3, + -2, + -1, + 0, + 1, + 2 + ], + "att": [ + -1.3315066530674116, + -2.076811730456074, + 0, + 1.5675442922691347, + 2.8217176555990986, + 3.7675836836828824 + ], + "se": [ + 0.2855600906254194, + 0.1556118528033074, + 0, + 0.11827327151200284, + 0.1443804178819619, + 0.20476867797670173 + ], + "is_reference": [ + false, + false, + true, + false, + false, + false + ], + "vcov": [ + [ + 0.08154456535799776, + 0.01855429826309639, + 0, + -0.0119384842105692, + 0.0001691543766540108, + -0.001235168450786678 + ], + [ + 0.01855429826309639, + 0.02421504873287821, + 0, + -0.0004055959042637364, + 0.00832224252716473, + 0.004839589355591384 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + -0.0119384842105692, + -0.0004055959042637364, + 0, + 0.013988566754151945, + 0.008206954594507663, + 0.011395075549748293 + ], + [ + 0.0001691543766540108, + 0.00832224252716473, + 0, + 0.008206954594507663, + 0.020845705067769945, + 0.014913075670760839 + ], + [ + -0.001235168450786678, + 0.004839589355591384, + 0, + 0.011395075549748293, + 0.014913075670760839, + 0.04193021148032617 + ] + ] + } + }, + "native": { + "att": 2.588175650340472, + "se": 0.12275103856398563, + "t_stat": 21.084755620958354, + "p_value": 1.097791898162843e-98, + "conf_int": [ + 2.3475880356901726, + 2.828763264990771 + ], + "alpha": 0.05, + "n_units": 804, + "group_time_effects": [ + { + "group": 2, + "time": 0, + "att": -2.162550430669498, + "se": 0.19332912183041565, + "t_stat": -11.185849344344733, + "p_value": 4.783036721508569e-29, + "conf_int_lower": -2.541468546619869, + "conf_int_upper": -1.7836323147191269, + "n_treated": 321, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 2, + "time": 1, + "att": 0.0, + "se": 0.0, + "t_stat": null, + "p_value": null, + "conf_int_lower": null, + "conf_int_upper": null, + "n_treated": 321, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 2, + "time": 2, + "att": 1.3388588578850817, + "se": 0.19740213340965893, + "t_stat": 6.782393050973841, + "p_value": 1.18201360836234e-11, + "conf_int_lower": 0.9519577859307793, + "conf_int_upper": 1.7257599298393842, + "n_treated": 321, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 2, + "time": 3, + "att": 2.444408769678787, + "se": 0.20054376967299714, + "t_stat": 12.18890406650176, + "p_value": 3.561790656757549e-34, + "conf_int_lower": 2.051350203795817, + "conf_int_upper": 2.8374673355617572, + "n_treated": 321, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 2, + "time": 4, + "att": 3.767583683682881, + "se": 0.20737730133268412, + "t_stat": 18.167772747889856, + "p_value": 9.289324668049035e-74, + "conf_int_lower": 3.36113164185971, + "conf_int_upper": 4.1740357255060525, + "n_treated": 321, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 3, + "time": 0, + "att": -1.3315066530674158, + "se": 0.28919794544435323, + "t_stat": -4.6041359354111355, + "p_value": 4.141818743499468e-06, + "conf_int_lower": -1.8983242105413276, + "conf_int_upper": -0.7646890955935041, + "n_treated": 160, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 3, + "time": 1, + "att": -1.9047984631528943, + "se": 0.2757674041929351, + "t_stat": -6.907264724514868, + "p_value": 4.940874273844249e-12, + "conf_int_lower": -2.445292643481147, + "conf_int_upper": -1.3643042828246417, + "n_treated": 160, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 3, + "time": 2, + "att": 0.0, + "se": 0.0, + "t_stat": null, + "p_value": null, + "conf_int_lower": null, + "conf_int_upper": null, + "n_treated": 160, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 3, + "time": 3, + "att": 2.0263444450021413, + "se": 0.23282696001898326, + "t_stat": 8.70322081616719, + "p_value": 3.2259536378482385e-18, + "conf_int_lower": 1.570011988734987, + "conf_int_upper": 2.4826769012692957, + "n_treated": 160, + "n_control": 323, + "available": true, + "status": "ok" + }, + { + "group": 3, + "time": 4, + "att": 3.5786936079767253, + "se": 0.18441787494550257, + "t_stat": 19.405351075833984, + "p_value": 6.954189279627515e-84, + "conf_int_lower": 3.2172412149781286, + "conf_int_upper": 3.940146000975322, + "n_treated": 160, + "n_control": 323, + "available": true, + "status": "ok" + } + ], + "vcov_index": [ + [ + 2, + 0 + ], + [ + 2, + 1 + ], + [ + 2, + 2 + ], + [ + 2, + 3 + ], + [ + 2, + 4 + ], + [ + 3, + 0 + ], + [ + 3, + 1 + ], + [ + 3, + 2 + ], + [ + 3, + 3 + ], + [ + 3, + 4 + ] + ], + "vcov": [ + [ + 0.03737614934771969, + 0.0, + 0.0060370625888336756, + 0.01820985262203815, + 0.01478358424497075, + 0.005360335202667528, + -0.0009028520033314267, + 0.0, + 0.00842872757825562, + -0.003995693234822009 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0060370625888336756, + 0.0, + 0.038967602274684786, + 0.01930347927625501, + 0.019813380536044773, + -0.023241054329018775, + -0.03254551296515961, + 0.0, + -0.021061727463424693, + -0.014181222841239498 + ], + [ + 0.01820985262203815, + 0.0, + 0.01930347927625501, + 0.04021780355465612, + 0.021146094524200194, + -0.004286191460130587, + -0.0012039074347594465, + 0.0, + 0.0030508327500272012, + -0.00194457009278345 + ], + [ + 0.01478358424497075, + 0.0, + 0.019813380536044773, + 0.021146094524200194, + 0.04300534510802687, + -0.0012668394367054657, + -0.014213176468169423, + 0.0, + -0.003217286728940781, + 0.0058650074407086954 + ], + [ + 0.005360335202667528, + 0.0, + -0.023241054329018775, + -0.004286191460130587, + -0.0012668394367054657, + 0.08363545164923511, + 0.04657389366676458, + 0.0, + 0.010134386406771034, + 0.009644323637199345 + ], + [ + -0.0009028520033314267, + 0.0, + -0.03254551296515961, + -0.0012039074347594465, + -0.014213176468169423, + 0.04657389366676458, + 0.07604766121530963, + 0.0, + 0.017619523547344237, + 0.010194117176752452 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.00842872757825562, + 0.0, + -0.021061727463424693, + 0.0030508327500272012, + -0.003217286728940781, + 0.010134386406771034, + 0.017619523547344237, + 0.0, + 0.05420839331168122, + 0.01657482023444165 + ], + [ + -0.003995693234822009, + 0.0, + -0.014181222841239498, + -0.00194457009278345, + 0.0058650074407086954, + 0.009644323637199345, + 0.010194117176752452, + 0.0, + 0.01657482023444165, + 0.03400995259941503 + ] + ], + "diagnostics": { + "catalogue": [ + [ + 2, + 0, + 1 + ], + [ + 2, + 0, + 2 + ], + [ + 2, + 0, + 3 + ], + [ + 2, + 0, + 4 + ], + [ + 2, + 1, + 2 + ], + [ + 2, + 1, + 3 + ], + [ + 2, + 1, + 4 + ], + [ + 2, + 2, + 3 + ], + [ + 2, + 2, + 4 + ], + [ + 2, + 3, + 4 + ], + [ + 3, + 0, + 1 + ], + [ + 3, + 0, + 2 + ], + [ + 3, + 0, + 3 + ], + [ + 3, + 0, + 4 + ], + [ + 3, + 1, + 2 + ], + [ + 3, + 1, + 3 + ], + [ + 3, + 1, + 4 + ], + [ + 3, + 2, + 3 + ], + [ + 3, + 2, + 4 + ], + [ + 3, + 3, + 4 + ] + ], + "links": [ + { + "group": 2, + "s": 0, + "t": 1, + "att": 2.162550430669498, + "n_treated": 156, + "n_control": 156, + "propensity_min": 0.36805555555555575, + "propensity_max": 0.6130952380952379, + "control_ess": 125.38931920097846 + }, + { + "group": 2, + "s": 0, + "t": 2, + "att": 3.712297244361433, + "n_treated": 161, + "n_control": 152, + "propensity_min": 0.42253521126760585, + "propensity_max": 0.590643274853801, + "control_ess": 136.69177809095612 + }, + { + "group": 2, + "s": 0, + "t": 3, + "att": 4.5079261250515295, + "n_treated": 144, + "n_control": 151, + "propensity_min": 0.35251798561151093, + "propensity_max": 0.6089743589743589, + "control_ess": 118.74343780346742 + }, + { + "group": 2, + "s": 0, + "t": 4, + "att": 6.240059303412773, + "n_treated": 152, + "n_control": 151, + "propensity_min": 0.3695652173913041, + "propensity_max": 0.6121212121212123, + "control_ess": 122.05792489404148 + }, + { + "group": 2, + "s": 1, + "t": 2, + "att": 1.3064369813285475, + "n_treated": 164, + "n_control": 146, + "propensity_min": 0.4228187919463088, + "propensity_max": 0.6273291925465836, + "control_ess": 124.42184421019417 + }, + { + "group": 2, + "s": 1, + "t": 3, + "att": 2.375653676899849, + "n_treated": 151, + "n_control": 154, + "propensity_min": 0.3378378378378378, + "propensity_max": 0.643312101910828, + "control_ess": 109.79389978748756 + }, + { + "group": 2, + "s": 1, + "t": 4, + "att": 3.763315635625728, + "n_treated": 153, + "n_control": 148, + "propensity_min": 0.3533834586466167, + "propensity_max": 0.6309523809523809, + "control_ess": 113.13513297935022 + }, + { + "group": 2, + "s": 2, + "t": 3, + "att": 1.259061422909732, + "n_treated": 163, + "n_control": 142, + "propensity_min": 0.4125874125874125, + "propensity_max": 0.6419753086419754, + "control_ess": 116.56994962366335 + }, + { + "group": 2, + "s": 2, + "t": 4, + "att": 2.3482343765394695, + "n_treated": 162, + "n_control": 144, + "propensity_min": 0.3897058823529409, + "propensity_max": 0.6411764705882353, + "control_ess": 114.79618686240353 + }, + { + "group": 2, + "s": 3, + "t": 4, + "att": 1.2034532396518067, + "n_treated": 143, + "n_control": 149, + "propensity_min": 0.31617647058823534, + "propensity_max": 0.6410256410256412, + "control_ess": 103.04195273737736 + }, + { + "group": 3, + "s": 0, + "t": 1, + "att": 0.022144796877110462, + "n_treated": 87, + "n_control": 156, + "propensity_min": 0.3546099290780146, + "propensity_max": 0.362745098039216, + "control_ess": 155.95231626137746 + }, + { + "group": 3, + "s": 0, + "t": 2, + "att": 1.3315066530674158, + "n_treated": 75, + "n_control": 152, + "propensity_min": 0.25531914893617036, + "propensity_max": 0.3834586466165414, + "control_ess": 140.80775571080937 + }, + { + "group": 3, + "s": 0, + "t": 3, + "att": 3.722401605393881, + "n_treated": 86, + "n_control": 151, + "propensity_min": 0.3617021276595748, + "propensity_max": 0.3645833333333337, + "control_ess": 150.9943438535425 + }, + { + "group": 3, + "s": 0, + "t": 4, + "att": 5.297653253675346, + "n_treated": 78, + "n_control": 151, + "propensity_min": 0.3402061855670102, + "propensity_max": 0.3409090909090907, + "control_ess": 150.9996389457215 + }, + { + "group": 3, + "s": 1, + "t": 2, + "att": 1.9047984631528943, + "n_treated": 75, + "n_control": 146, + "propensity_min": 0.27710843373494043, + "propensity_max": 0.3768115942028994, + "control_ess": 139.7219521118353 + }, + { + "group": 3, + "s": 1, + "t": 3, + "att": 3.6806990920389326, + "n_treated": 83, + "n_control": 154, + "propensity_min": 0.3333333333333329, + "propensity_max": 0.37777777777777755, + "control_ess": 152.60443037974682 + }, + { + "group": 3, + "s": 1, + "t": 4, + "att": 5.302283922455791, + "n_treated": 85, + "n_control": 148, + "propensity_min": 0.354166666666667, + "propensity_max": 0.37226277372262817, + "control_ess": 147.7827050997783 + }, + { + "group": 3, + "s": 2, + "t": 3, + "att": 1.7762903683115239, + "n_treated": 68, + "n_control": 142, + "propensity_min": 0.2749999999999998, + "propensity_max": 0.3538461538461538, + "control_ess": 137.88454193801104 + }, + { + "group": 3, + "s": 2, + "t": 4, + "att": 3.7219638396121404, + "n_treated": 70, + "n_control": 144, + "propensity_min": 0.27380952380952367, + "propensity_max": 0.36153846153846153, + "control_ess": 138.86295450474648 + }, + { + "group": 3, + "s": 3, + "t": 4, + "att": 1.3096179324369874, + "n_treated": 75, + "n_control": 149, + "propensity_min": 0.3161764705882357, + "propensity_max": 0.36363636363636404, + "control_ess": 147.37694691512053 + } + ], + "rank": { + "2": 4, + "3": 4 + }, + "gmm_weighting": "identity", + "selection": "none" + } + }, + "input_sha256": "0b3d6e6c8e0ff40639f1765bd410320a9927bc6b7cbeb6e1f3b6524de56b429e" + }, + "balanced_continuous_x": { + "passed": true, + "checks": { + "registry": true, + "labels": true, + "support": true, + "estimates": true, + "covariance": true, + "overall_estimate": true, + "overall_se": true, + "event_labels": true, + "event_estimates": true, + "event_reference_roles": true, + "event_reference_se": true, + "event_se": true, + "event_covariance": true + }, + "covariance_max_abs_error": 8.673617379884035e-18, + "cluster_cr1_factor_applied": 1.0, + "tolerances": { + "estimate_rtol": 1e-06, + "estimate_atol": 2.6755469957800023e-07, + "covariance_rtol": 1e-05, + "covariance_atol": 7.158551726627395e-07 + }, + "reference": { + "metadata": { + "R": "4.5.2", + "packages": { + "cdid": "1.1.0.0", + "did": "2.5.1", + "DRDID": "1.3.0" + }, + "cdid_sha": "1039120668cceabd8111cad82d32ab3927c2b94f", + "n": 800, + "covariance_scale": "estimator", + "cluster_correction": "none" + }, + "cells": [ + { + "group": 2, + "time": 0, + "att": -0.18920102019638196, + "available": true + }, + { + "group": 2, + "time": 1, + "att": 0, + "available": true + }, + { + "group": 2, + "time": 2, + "att": 3.0436857047552905, + "available": true + }, + { + "group": 2, + "time": 3, + "att": 2.9742978265876574, + "available": true + } + ], + "vcov": [ + [ + 0.010772739453489555, + 0, + 0.005957116932665778, + 0.005614341327816943 + ], + [ + 0, + 0, + 0, + 0 + ], + [ + 0.005957116932665778, + 0, + 0.013306780907498978, + 0.008590656092678641 + ], + [ + 0.005614341327816943, + 0, + 0.008590656092678641, + 0.01594336993232175 + ] + ], + "overall": { + "att": 3.0089917656714738, + "se": 0.10773980581147576 + }, + "events": { + "event_time": [ + -2, + -1, + 0, + 1 + ], + "att": [ + -0.18920102019638196, + 0, + 3.0436857047552905, + 2.9742978265876574 + ], + "se": [ + 0.1037918082195775, + 0, + 0.11535502116292545, + 0.1262670579855322 + ], + "is_reference": [ + false, + true, + false, + false + ], + "vcov": [ + [ + 0.010772739453489555, + 0, + 0.005957116932665778, + 0.005614341327816943 + ], + [ + 0, + 0, + 0, + 0 + ], + [ + 0.005957116932665778, + 0, + 0.013306780907498978, + 0.008590656092678641 + ], + [ + 0.005614341327816943, + 0, + 0.008590656092678641, + 0.01594336993232175 + ] + ] + } + }, + "native": { + "att": 3.0089917656714738, + "se": 0.10773980581147576, + "t_stat": 27.928319927888484, + "p_value": 1.209023905502939e-171, + "conf_int": [ + 2.797825626579642, + 3.2201579047633055 + ], + "alpha": 0.05, + "n_units": 800, + "group_time_effects": [ + { + "group": 2, + "time": 0, + "att": -0.18920102019638196, + "se": 0.1037918082195775, + "t_stat": -1.8228897197369989, + "p_value": 0.06832009524997323, + "conf_int_lower": -0.3926292261970422, + "conf_int_upper": 0.014227185804278286, + "n_treated": 434, + "n_control": 366, + "available": true, + "status": "ok" + }, + { + "group": 2, + "time": 1, + "att": 0.0, + "se": 0.0, + "t_stat": null, + "p_value": null, + "conf_int_lower": null, + "conf_int_upper": null, + "n_treated": 434, + "n_control": 366, + "available": true, + "status": "ok" + }, + { + "group": 2, + "time": 2, + "att": 3.0436857047552905, + "se": 0.11535502116292548, + "t_stat": 26.38537684854169, + "p_value": 2.016636318783234e-153, + "conf_int_lower": 2.8175940178401007, + "conf_int_upper": 3.2697773916704804, + "n_treated": 434, + "n_control": 366, + "available": true, + "status": "ok" + }, + { + "group": 2, + "time": 3, + "att": 2.9742978265876574, + "se": 0.12626705798553217, + "t_stat": 23.55561200236768, + "p_value": 1.0996624956744944e-122, + "conf_int_lower": 2.726818940502184, + "conf_int_upper": 3.221776712673131, + "n_treated": 434, + "n_control": 366, + "available": true, + "status": "ok" + } + ], + "vcov_index": [ + [ + 2, + 0 + ], + [ + 2, + 1 + ], + [ + 2, + 2 + ], + [ + 2, + 3 + ] + ], + "vcov": [ + [ + 0.010772739453489555, + 0.0, + 0.0059571169326657816, + 0.005614341327816942 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0059571169326657816, + 0.0, + 0.013306780907498986, + 0.008590656092678646 + ], + [ + 0.005614341327816942, + 0.0, + 0.008590656092678646, + 0.015943369932321746 + ] + ], + "diagnostics": { + "catalogue": [ + [ + 2, + 0, + 1 + ], + [ + 2, + 0, + 2 + ], + [ + 2, + 0, + 3 + ], + [ + 2, + 1, + 2 + ], + [ + 2, + 1, + 3 + ], + [ + 2, + 2, + 3 + ] + ], + "links": [ + { + "group": 2, + "s": 0, + "t": 1, + "att": 0.18920102019638196, + "n_treated": 434, + "n_control": 366, + "propensity_min": 0.20456597976195268, + "propensity_max": 0.8112946033419738, + "control_ess": 306.4683612815502 + }, + { + "group": 2, + "s": 0, + "t": 2, + "att": 3.2328867249516726, + "n_treated": 434, + "n_control": 366, + "propensity_min": 0.20456597976195268, + "propensity_max": 0.8112946033419738, + "control_ess": 306.4683612815502 + }, + { + "group": 2, + "s": 0, + "t": 3, + "att": 3.1634988467840386, + "n_treated": 434, + "n_control": 366, + "propensity_min": 0.20456597976195268, + "propensity_max": 0.8112946033419738, + "control_ess": 306.4683612815502 + }, + { + "group": 2, + "s": 1, + "t": 2, + "att": 3.04368570475529, + "n_treated": 434, + "n_control": 366, + "propensity_min": 0.20456597976195268, + "propensity_max": 0.8112946033419738, + "control_ess": 306.4683612815502 + }, + { + "group": 2, + "s": 1, + "t": 3, + "att": 2.974297826587657, + "n_treated": 434, + "n_control": 366, + "propensity_min": 0.20456597976195268, + "propensity_max": 0.8112946033419738, + "control_ess": 306.4683612815502 + }, + { + "group": 2, + "s": 2, + "t": 3, + "att": -0.06938787816763352, + "n_treated": 434, + "n_control": 366, + "propensity_min": 0.20456597976195268, + "propensity_max": 0.8112946033419738, + "control_ess": 306.4683612815502 + } + ], + "rank": { + "2": 3 + }, + "gmm_weighting": "identity", + "selection": "none" + } + }, + "input_sha256": "d9bc7b317d2357cfe6e60573188678440bf7403f23f13d9f02ba78c667322d8b" + }, + "incomplete_continuous_x": { + "passed": false, + "checks": { + "registry": true, + "labels": true, + "support": true, + "estimates": false, + "covariance": false, + "overall_estimate": false, + "overall_se": false, + "event_labels": true, + "event_estimates": false, + "event_reference_roles": true, + "event_reference_se": true, + "event_se": false, + "event_covariance": false + }, + "covariance_max_abs_error": 0.0008715199801816349, + "cluster_cr1_factor_applied": 1.0, + "tolerances": { + "estimate_rtol": 1e-06, + "estimate_atol": 2.638959011024559e-07, + "covariance_rtol": 1e-05, + "covariance_atol": 6.96410466186772e-07 + }, + "calibrated_score_point_diagnostic": { + "matches_reference": true, + "points": [ + { + "group": 2, + "time": 0, + "att": -0.20035463010124827 + }, + { + "group": 2, + "time": 1, + "att": 0.0 + }, + { + "group": 2, + "time": 2, + "att": 3.0406236096451655 + }, + { + "group": 2, + "time": 3, + "att": 2.890041876494755 + } + ], + "interpretation": "Weighted-score point diagnostic only; native point criterion and inference remain unchanged." + }, + "reference": { + "metadata": { + "R": "4.5.2", + "packages": { + "cdid": "1.1.0.0", + "did": "2.5.1", + "DRDID": "1.3.0" + }, + "cdid_sha": "1039120668cceabd8111cad82d32ab3927c2b94f", + "n": 800, + "covariance_scale": "estimator", + "cluster_correction": "none" + }, + "cells": [ + { + "group": 2, + "time": 0, + "att": -0.200354630100947, + "available": true + }, + { + "group": 2, + "time": 1, + "att": 0, + "available": true + }, + { + "group": 2, + "time": 2, + "att": 3.0406236096388795, + "available": true + }, + { + "group": 2, + "time": 3, + "att": 2.8900418764849487, + "available": true + } + ], + "vcov": [ + [ + 0.014755844862619837, + 0, + 0.0080073458208847, + 0.007247739853604729 + ], + [ + 0, + 0, + 0, + 0 + ], + [ + 0.0080073458208847, + 0, + 0.018090475683363257, + 0.01128119091864458 + ], + [ + 0.007247739853604729, + 0, + 0.01128119091864458, + 0.020339480300262128 + ] + ], + "overall": { + "att": 2.9653327430619143, + "se": 0.12348313429464221 + }, + "events": { + "event_time": [ + -2, + -1, + 0, + 1 + ], + "att": [ + -0.200354630100947, + 0, + 3.0406236096388795, + 2.8900418764849487 + ], + "se": [ + 0.12147363855018026, + 0, + 0.134500838968994, + 0.14261654988206007 + ], + "is_reference": [ + false, + true, + false, + false + ], + "vcov": [ + [ + 0.014755844862619839, + 0, + 0.0080073458208847, + 0.007247739853604729 + ], + [ + 0, + 0, + 0, + 0 + ], + [ + 0.0080073458208847, + 0, + 0.018090475683363257, + 0.011281190918644584 + ], + [ + 0.007247739853604729, + 0, + 0.011281190918644584, + 0.020339480300262128 + ] + ] + } + }, + "native": { + "att": 2.9640472521835135, + "se": 0.12141115156073534, + "t_stat": 24.41330317751548, + "p_value": 1.235370307460404e-131, + "conf_int": [ + 2.7260857678029384, + 3.2020087365640886 + ], + "alpha": 0.05, + "n_units": 800, + "group_time_effects": [ + { + "group": 2, + "time": 0, + "att": -0.1977729923297328, + "se": 0.12142725599379886, + "t_stat": -1.6287364044513435, + "p_value": 0.10336883261665542, + "conf_int_lower": -0.435766040819104, + "conf_int_upper": 0.04022005615963836, + "n_treated": 434, + "n_control": 366, + "available": true, + "status": "ok" + }, + { + "group": 2, + "time": 1, + "att": 0.0, + "se": 0.0, + "t_stat": null, + "p_value": null, + "conf_int_lower": null, + "conf_int_upper": null, + "n_treated": 434, + "n_control": 366, + "available": true, + "status": "ok" + }, + { + "group": 2, + "time": 2, + "att": 3.039605604632994, + "se": 0.1335621138033546, + "t_stat": 22.757992652828545, + "p_value": 1.1959687205056618e-114, + "conf_int_lower": 2.7778286718793788, + "conf_int_upper": 3.301382537386609, + "n_treated": 434, + "n_control": 366, + "available": true, + "status": "ok" + }, + { + "group": 2, + "time": 3, + "att": 2.888488899734033, + "se": 0.1395276328190244, + "t_stat": 20.70191288546101, + "p_value": 3.328920006126057e-95, + "conf_int_lower": 2.615019764560617, + "conf_int_upper": 3.1619580349074496, + "n_treated": 434, + "n_control": 366, + "available": true, + "status": "ok" + } + ], + "vcov_index": [ + [ + 2, + 0 + ], + [ + 2, + 1 + ], + [ + 2, + 2 + ], + [ + 2, + 3 + ] + ], + "vcov": [ + [ + 0.01474457849818356, + 0.0, + 0.008086003968759962, + 0.00736438526037249 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.008086003968759962, + 0.0, + 0.01783883824362024, + 0.010827936164757336 + ], + [ + 0.00736438526037249, + 0.0, + 0.010827936164757336, + 0.019467960320080493 + ] + ], + "diagnostics": { + "catalogue": [ + [ + 2, + 0, + 1 + ], + [ + 2, + 0, + 2 + ], + [ + 2, + 0, + 3 + ], + [ + 2, + 1, + 2 + ], + [ + 2, + 1, + 3 + ], + [ + 2, + 2, + 3 + ] + ], + "links": [ + { + "group": 2, + "s": 0, + "t": 1, + "att": 0.1977729923297328, + "n_treated": 257, + "n_control": 301, + "propensity_min": 0.1934652381866911, + "propensity_max": 0.7189718650976834, + "control_ess": 264.6278406658256 + }, + { + "group": 2, + "s": 0, + "t": 2, + "att": 3.2310289247234834, + "n_treated": 251, + "n_control": 300, + "propensity_min": 0.17910347622148665, + "propensity_max": 0.7283942311692685, + "control_ess": 259.67198095082966 + }, + { + "group": 2, + "s": 0, + "t": 3, + "att": 3.141640845357102, + "n_treated": 259, + "n_control": 307, + "propensity_min": 0.1776359620477242, + "propensity_max": 0.7318591190574613, + "control_ess": 264.2700145155265 + }, + { + "group": 2, + "s": 1, + "t": 2, + "att": 3.005534788419771, + "n_treated": 236, + "n_control": 295, + "propensity_min": 0.14323925971646276, + "propensity_max": 0.7516003324340844, + "control_ess": 244.3137022349039 + }, + { + "group": 2, + "s": 1, + "t": 3, + "att": 2.906216622262557, + "n_treated": 256, + "n_control": 301, + "propensity_min": 0.14383883923399937, + "propensity_max": 0.7714987556581142, + "control_ess": 244.39018292107403 + }, + { + "group": 2, + "s": 2, + "t": 3, + "att": -0.20788028703612446, + "n_treated": 247, + "n_control": 295, + "propensity_min": 0.1331014808470547, + "propensity_max": 0.7788240262359655, + "control_ess": 236.01591481917325 + } + ], + "rank": { + "2": 3 + }, + "gmm_weighting": "identity", + "selection": "none" + } + }, + "input_sha256": "411d0a8d4d4cd39d0a2d9a8587586a65f2ad8006697c9bf402e1f862b606e095" + } + }, + "public_input_probe": { + "status": "public_inputs_verified_no_estimate", + "pair": [ + 0, + 1 + ], + "cohorts": [ + 0, + 2 + ], + "eligible_counts": [ + 365, + 418 + ], + "pair_counts": [ + 301, + 257 + ], + "weights": [ + 1.212624584717608, + 1.6264591439688716 + ], + "max_abs_weight_error": 2.220446049250313e-16, + "input_sha256": "411d0a8d4d4cd39d0a2d9a8587586a65f2ad8006697c9bf402e1f862b606e095" + }, + "all_comparisons_pass": false +} diff --git a/benchmarks/data/chained_did/sparse-covariance-diagnostic.json b/benchmarks/data/chained_did/sparse-covariance-diagnostic.json new file mode 100644 index 000000000..1f40d769c --- /dev/null +++ b/benchmarks/data/chained_did/sparse-covariance-diagnostic.json @@ -0,0 +1,1002 @@ +{ + "status": "complete", + "scope": "conditional Gaussian covariance diagnostic", + "seed": 20261106, + "replications_per_case": 2000, + "seed_derivation": "SeedSequence([seed,case_index,replication]), uint64", + "python": "3.14.0", + "numpy": "2.5.4", + "pandas": "3.1.0", + "scipy": "1.18.1", + "backend": "native Python estimator; single-threaded BLAS requested externally", + "source_hashes": { + "diff_diff/chained_did.py": "870355980e3eb8d255db709daefaadefc7c5249925734d1d6bcc92cc79770934", + "diff_diff/chained_did_results.py": "df3e0b4b1853b9f9c98ddfda10b43255d311c38f5dd993ee45b2d4921e54cade", + "benchmarks/python/validate_chained_did.py": "cf3c5219738fc0ec7ff69cd6b0676df65320b800c255e5c3ecc81c06e08cd799", + "benchmarks/python/diagnose_chained_did.py": "4813859170b69e778077cd897ce567c02e2861a33596f9492337232669b3bbfd", + "benchmarks/python/diagnose_chained_did_sparse.py": "191f2b99ac51f43b2629372190eaeea4601cf0cf6c76880f2ba2631ccb68426e" + }, + "cases": [ + { + "n": 400, + "sparse_treated": 0, + "sparse_controls": 19, + "regime": "baseline" + }, + { + "n": 400, + "sparse_treated": 1, + "sparse_controls": 19, + "regime": "baseline" + }, + { + "n": 400, + "sparse_treated": 2, + "sparse_controls": 19, + "regime": "baseline" + }, + { + "n": 400, + "sparse_treated": 5, + "sparse_controls": 19, + "regime": "baseline" + }, + { + "n": 400, + "sparse_treated": 20, + "sparse_controls": 19, + "regime": "baseline" + }, + { + "n": 400, + "sparse_treated": 50, + "sparse_controls": 19, + "regime": "baseline" + }, + { + "n": 4000, + "sparse_treated": 2, + "sparse_controls": 19, + "regime": "fixed" + }, + { + "n": 4000, + "sparse_treated": 5, + "sparse_controls": 19, + "regime": "fixed" + }, + { + "n": 4000, + "sparse_treated": 20, + "sparse_controls": 190, + "regime": "growing" + }, + { + "n": 4000, + "sparse_treated": 50, + "sparse_controls": 190, + "regime": "growing" + } + ], + "results": { + "1": { + "design": { + "n": 400, + "sparse_treated": 1, + "sparse_controls": 19, + "regime": "baseline" + }, + "elapsed_seconds": 8.931469712002581, + "failures": {}, + "native_max_abs_error": 2.6645352591003757e-15, + "native_variance_max_abs_error": 5.551115123125783e-17, + "oracle_variance_ratio_design_to_identity": 0.99957794582385, + "variants": { + "identity_hc0": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": 0.0010672238088267338, + "bias_mcse": 0.003211203470901806, + "rmse": 0.1435774446143621, + "empirical_sd": 0.14360938501039414, + "mean_variance": 0.020011636863899538, + "variance_ratio": 0.9703244364094336, + "coverage": 0.947, + "coverage_mcse": 0.005009540897128201, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.947, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.5541568927686699, + "meets_prespecified_thresholds": true + }, + "identity_oracle": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": 0.0010672238088267338, + "bias_mcse": 0.003211203470901806, + "rmse": 0.1435774446143621, + "empirical_sd": 0.14360938501039414, + "mean_variance": 0.02012221944590971, + "variance_ratio": 0.9756863656856581, + "coverage": 0.947, + "coverage_mcse": 0.005009540897128201, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.947, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.5560527938332213, + "meets_prespecified_thresholds": true + }, + "design_precision_hc0": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": 0.0010726124455922026, + "bias_mcse": 0.003210587091570648, + "rmse": 0.14354992710937203, + "empirical_sd": 0.14358181968870623, + "mean_variance": 0.020003089901291713, + "variance_ratio": 0.9702824601408018, + "coverage": 0.9465, + "coverage_mcse": 0.005031786462082825, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9465, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.5540389055720568, + "meets_prespecified_thresholds": true + }, + "design_precision_oracle": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": 0.0010726124455922026, + "bias_mcse": 0.003210587091570648, + "rmse": 0.14354992710937203, + "empirical_sd": 0.14358181968870623, + "mean_variance": 0.02011372677915916, + "variance_ratio": 0.9756490821261676, + "coverage": 0.946, + "coverage_mcse": 0.005053909377897473, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.946, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.5559354392475175, + "meets_prespecified_thresholds": true + } + } + }, + "0": { + "design": { + "n": 400, + "sparse_treated": 0, + "sparse_controls": 19, + "regime": "baseline" + }, + "elapsed_seconds": 9.014635602001363, + "failures": {}, + "native_max_abs_error": 2.886579864025407e-15, + "native_variance_max_abs_error": 4.163336342344337e-17, + "oracle_variance_ratio_design_to_identity": 1.0, + "variants": { + "identity_hc0": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": 0.0008823058360141189, + "bias_mcse": 0.0031527257848197664, + "rmse": 0.14096169171888093, + "empirical_sd": 0.14099418338546746, + "mean_variance": 0.019894607923604465, + "variance_ratio": 1.0007670355213911, + "coverage": 0.9495, + "coverage_mcse": 0.0048964145045124596, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9495, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.5525572323204188, + "meets_prespecified_thresholds": true + }, + "identity_oracle": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": 0.0008823058360141189, + "bias_mcse": 0.0031527257848197664, + "rmse": 0.14096169171888093, + "empirical_sd": 0.14099418338546746, + "mean_variance": 0.019999999999999983, + "variance_ratio": 1.006068618556694, + "coverage": 0.9495, + "coverage_mcse": 0.0048964145045124596, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9495, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.5543615297398709, + "meets_prespecified_thresholds": true + }, + "design_precision_hc0": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": 0.000882305836014341, + "bias_mcse": 0.0031527257848197664, + "rmse": 0.14096169171888093, + "empirical_sd": 0.14099418338546746, + "mean_variance": 0.019894607923604465, + "variance_ratio": 1.0007670355213911, + "coverage": 0.9495, + "coverage_mcse": 0.0048964145045124596, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9495, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.5525572323204188, + "meets_prespecified_thresholds": true + }, + "design_precision_oracle": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": 0.000882305836014341, + "bias_mcse": 0.0031527257848197664, + "rmse": 0.14096169171888093, + "empirical_sd": 0.14099418338546746, + "mean_variance": 0.019999999999999983, + "variance_ratio": 1.006068618556694, + "coverage": 0.9495, + "coverage_mcse": 0.0048964145045124596, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9495, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.5543615297398709, + "meets_prespecified_thresholds": true + } + } + }, + "2": { + "design": { + "n": 400, + "sparse_treated": 2, + "sparse_controls": 19, + "regime": "baseline" + }, + "elapsed_seconds": 9.364532750998478, + "failures": {}, + "native_max_abs_error": 1.3322676295501878e-15, + "native_variance_max_abs_error": 4.996003610813204e-16, + "oracle_variance_ratio_design_to_identity": 0.2351226501682532, + "variants": { + "identity_hc0": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": -0.0057718552940976275, + "bias_mcse": 0.006460665449570842, + "rmse": 0.28891516093490416, + "empirical_sd": 0.28892974250249287, + "mean_variance": 0.05381077669392442, + "variance_ratio": 0.6445917749098499, + "coverage": 0.8005, + "coverage_mcse": 0.008935875726530668, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.8005, + "rejection_at_zero": 0.9505, + "rejection_mcse": 0.004850244839180801, + "mean_interval_width": 0.8483903720260483, + "meets_prespecified_thresholds": false + }, + "identity_oracle": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": -0.0057718552940976275, + "bias_mcse": 0.006460665449570842, + "rmse": 0.28891516093490416, + "empirical_sd": 0.28892974250249287, + "mean_variance": 0.08540086823454064, + "variance_ratio": 1.023005067316936, + "coverage": 0.956, + "coverage_mcse": 0.004586065852122058, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.956, + "rejection_at_zero": 0.925, + "rejection_mcse": 0.0058896094946948715, + "mean_interval_width": 1.145537286297592, + "meets_prespecified_thresholds": true + }, + "design_precision_hc0": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": -0.002770467793183129, + "bias_mcse": 0.003151026451989693, + "rmse": 0.14091019099543312, + "empirical_sd": 0.14091818691097863, + "mean_variance": 0.019930057491560932, + "variance_ratio": 1.003631902704002, + "coverage": 0.952, + "coverage_mcse": 0.004779958158812692, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.952, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.5530377192029834, + "meets_prespecified_thresholds": true + }, + "design_precision_oracle": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": -0.002770467793183129, + "bias_mcse": 0.003151026451989693, + "rmse": 0.14091019099543312, + "empirical_sd": 0.14091818691097863, + "mean_variance": 0.020079678465974995, + "variance_ratio": 1.011166471196802, + "coverage": 0.952, + "coverage_mcse": 0.004779958158812692, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.952, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.5554646990037135, + "meets_prespecified_thresholds": true + } + } + }, + "3": { + "design": { + "n": 400, + "sparse_treated": 5, + "sparse_controls": 19, + "regime": "baseline" + }, + "elapsed_seconds": 9.534807227999408, + "failures": {}, + "native_max_abs_error": 1.5543122344752192e-15, + "native_variance_max_abs_error": 1.6653345369377348e-16, + "oracle_variance_ratio_design_to_identity": 0.41844429584208026, + "variants": { + "identity_hc0": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": -0.000569753236328796, + "bias_mcse": 0.004951581102448969, + "rmse": 0.22138680469168814, + "empirical_sd": 0.2214414388235849, + "mean_variance": 0.041620892406508604, + "variance_ratio": 0.8487769920595778, + "coverage": 0.908, + "coverage_mcse": 0.006462816723379983, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.908, + "rejection_at_zero": 0.993, + "rejection_mcse": 0.0018642692938521525, + "mean_interval_width": 0.7870738767273193, + "meets_prespecified_thresholds": false + }, + "identity_oracle": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": -0.000569753236328796, + "bias_mcse": 0.004951581102448969, + "rmse": 0.22138680469168814, + "empirical_sd": 0.2214414388235849, + "mean_variance": 0.047910580744253146, + "variance_ratio": 0.9770429286993262, + "coverage": 0.949, + "coverage_mcse": 0.004919298730510276, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.949, + "rejection_at_zero": 0.996, + "rejection_mcse": 0.0014113823011501884, + "mean_interval_width": 0.8580128740399864, + "meets_prespecified_thresholds": true + }, + "design_precision_hc0": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": -6.289370639567249e-05, + "bias_mcse": 0.0032704930169844553, + "rmse": 0.14622433784884215, + "empirical_sd": 0.14626089411831233, + "mean_variance": 0.019861441013473938, + "variance_ratio": 0.9284409916786739, + "coverage": 0.9395, + "coverage_mcse": 0.005331029450303197, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9395, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.5520831294188047, + "meets_prespecified_thresholds": true + }, + "design_precision_oracle": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": -6.289370639567249e-05, + "bias_mcse": 0.0032704930169844553, + "rmse": 0.14622433784884215, + "empirical_sd": 0.14626089411831233, + "mean_variance": 0.020047909222914138, + "variance_ratio": 0.9371576164780406, + "coverage": 0.945, + "coverage_mcse": 0.005097793640389931, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.945, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.5550251083359691, + "meets_prespecified_thresholds": true + } + } + }, + "5": { + "design": { + "n": 400, + "sparse_treated": 50, + "sparse_controls": 19, + "regime": "baseline" + }, + "elapsed_seconds": 9.462070366000262, + "failures": {}, + "native_max_abs_error": 1.3322676295501878e-15, + "native_variance_max_abs_error": 5.551115123125783e-17, + "oracle_variance_ratio_design_to_identity": 0.7847605497548567, + "variants": { + "identity_hc0": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": 5.262112839354849e-05, + "bias_mcse": 0.0036304217412670725, + "rmse": 0.16231681011495075, + "empirical_sd": 0.16235739600932655, + "mean_variance": 0.024774149399441576, + "variance_ratio": 0.939841456403828, + "coverage": 0.94, + "coverage_mcse": 0.005310367218940704, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.94, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.6157154210240336, + "meets_prespecified_thresholds": true + }, + "identity_oracle": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": 5.262112839354849e-05, + "bias_mcse": 0.0036304217412670725, + "rmse": 0.16231681011495075, + "empirical_sd": 0.16235739600932655, + "mean_variance": 0.025602888436561003, + "variance_ratio": 0.971280812446566, + "coverage": 0.945, + "coverage_mcse": 0.005097793640389931, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.945, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.6272238567525151, + "meets_prespecified_thresholds": true + }, + "design_precision_hc0": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": 0.001412912718488979, + "bias_mcse": 0.003210854579843838, + "rmse": 0.1435648320481916, + "empirical_sd": 0.14359378212794696, + "mean_variance": 0.01986859881577424, + "variance_ratio": 0.9635981809766894, + "coverage": 0.9395, + "coverage_mcse": 0.005331029450303197, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9395, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.5521844413035506, + "meets_prespecified_thresholds": true + }, + "design_precision_oracle": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": 0.001412912718488979, + "bias_mcse": 0.003210854579843838, + "rmse": 0.1435648320481916, + "empirical_sd": 0.14359378212794696, + "mean_variance": 0.020092136804787872, + "variance_ratio": 0.9744394487273729, + "coverage": 0.94, + "coverage_mcse": 0.005310367218940704, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.94, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.5556369899686142, + "meets_prespecified_thresholds": true + } + } + }, + "4": { + "design": { + "n": 400, + "sparse_treated": 20, + "sparse_controls": 19, + "regime": "baseline" + }, + "elapsed_seconds": 9.756875956998556, + "failures": {}, + "native_max_abs_error": 1.3322676295501878e-15, + "native_variance_max_abs_error": 8.326672684688674e-17, + "oracle_variance_ratio_design_to_identity": 0.6846086857473604, + "variants": { + "identity_hc0": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": 0.0033184492925115716, + "bias_mcse": 0.003730113033123431, + "rmse": 0.16680702875545042, + "empirical_sd": 0.16681572611643833, + "mean_variance": 0.028417921365400042, + "variance_ratio": 1.0212176865515699, + "coverage": 0.953, + "coverage_mcse": 0.004732388403332932, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.953, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.6591069682083197, + "meets_prespecified_thresholds": true + }, + "identity_oracle": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": 0.0033184492925115716, + "bias_mcse": 0.003730113033123431, + "rmse": 0.16680702875545042, + "empirical_sd": 0.16681572611643833, + "mean_variance": 0.029213999547672106, + "variance_ratio": 1.049825307396206, + "coverage": 0.956, + "coverage_mcse": 0.004586065852122058, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.956, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.6699981380575891, + "meets_prespecified_thresholds": true + }, + "design_precision_hc0": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": 0.0024109146188133757, + "bias_mcse": 0.0031420977577252523, + "rmse": 0.14050443540751384, + "empirical_sd": 0.1405188835644666, + "mean_variance": 0.01984371825641308, + "variance_ratio": 1.0049713269677614, + "coverage": 0.9525, + "coverage_mcse": 0.00475624589355933, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9525, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.5518494447008844, + "meets_prespecified_thresholds": true + }, + "design_precision_oracle": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": 0.0024109146188133757, + "bias_mcse": 0.0031420977577252523, + "rmse": 0.14050443540751384, + "empirical_sd": 0.1405188835644666, + "mean_variance": 0.020000157835755778, + "variance_ratio": 1.0128941007952672, + "coverage": 0.9535, + "coverage_mcse": 0.004708383480558906, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9535, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.554363717187331, + "meets_prespecified_thresholds": true + } + } + }, + "6": { + "design": { + "n": 4000, + "sparse_treated": 2, + "sparse_controls": 19, + "regime": "fixed" + }, + "elapsed_seconds": 35.03738819099817, + "failures": {}, + "native_max_abs_error": 3.552713678800501e-15, + "native_variance_max_abs_error": 1.8318679906315083e-15, + "oracle_variance_ratio_design_to_identity": 0.02829755674839057, + "variants": { + "identity_hc0": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": -0.0037320516726253583, + "bias_mcse": 0.005808605726193376, + "rmse": 0.25973060904317324, + "empirical_sd": 0.2597687451652584, + "mean_variance": 0.03843397085966734, + "variance_ratio": 0.5695625996252439, + "coverage": 0.726, + "coverage_mcse": 0.00997306372184596, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.726, + "rejection_at_zero": 0.976, + "rejection_mcse": 0.0034222799417931915, + "mean_interval_width": 0.6802611713789856, + "meets_prespecified_thresholds": false + }, + "identity_oracle": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": -0.0037320516726253583, + "bias_mcse": 0.005808605726193376, + "rmse": 0.25973060904317324, + "empirical_sd": 0.2597687451652584, + "mean_variance": 0.07070460937570833, + "variance_ratio": 1.0477892401114337, + "coverage": 0.9535, + "coverage_mcse": 0.004708383480558906, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9535, + "rejection_at_zero": 0.9695, + "rejection_mcse": 0.0038451105315712303, + "mean_interval_width": 1.042322110194439, + "meets_prespecified_thresholds": true + }, + "design_precision_hc0": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": -0.001328901617275946, + "bias_mcse": 0.000970647584712019, + "rmse": 0.04341816774334499, + "empirical_sd": 0.0434086796322412, + "mean_variance": 0.002000073846252282, + "variance_ratio": 1.0614337162258696, + "coverage": 0.9585, + "coverage_mcse": 0.004459694496263168, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9585, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.1752964857122475, + "meets_prespecified_thresholds": true + }, + "design_precision_oracle": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": -0.001328901617275946, + "bias_mcse": 0.000970647584712019, + "rmse": 0.04341816774334499, + "empirical_sd": 0.0434086796322412, + "mean_variance": 0.002000767696181894, + "variance_ratio": 1.0618019404845243, + "coverage": 0.958, + "coverage_mcse": 0.00448530935387962, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.958, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.1753381500376664, + "meets_prespecified_thresholds": true + } + } + }, + "7": { + "design": { + "n": 4000, + "sparse_treated": 5, + "sparse_controls": 19, + "regime": "fixed" + }, + "elapsed_seconds": 35.446236861000216, + "failures": {}, + "native_max_abs_error": 3.552713678800501e-15, + "native_variance_max_abs_error": 6.38378239159465e-16, + "oracle_variance_ratio_design_to_identity": 0.060246167592851174, + "variants": { + "identity_hc0": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": -0.002943376300343692, + "bias_mcse": 0.004031471655775182, + "rmse": 0.1802718450645344, + "empirical_sd": 0.18029289343353883, + "mean_variance": 0.0275364688289446, + "variance_ratio": 0.8471318883990094, + "coverage": 0.888, + "coverage_mcse": 0.007051808278732484, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.888, + "rejection_at_zero": 0.9995, + "rejection_mcse": 0.000499874984371065, + "mean_interval_width": 0.6311161360345383, + "meets_prespecified_thresholds": false + }, + "identity_oracle": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": -0.002943376300343692, + "bias_mcse": 0.004031471655775182, + "rmse": 0.1802718450645344, + "empirical_sd": 0.18029289343353883, + "mean_variance": 0.03320470345474979, + "variance_ratio": 1.0215094504704292, + "coverage": 0.95, + "coverage_mcse": 0.004873397172404485, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.95, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.7142954598057619, + "meets_prespecified_thresholds": true + }, + "design_precision_hc0": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": -0.0011786471651707986, + "bias_mcse": 0.0009730100664072463, + "rmse": 0.043519416806872574, + "empirical_sd": 0.04351433302556375, + "mean_variance": 0.0020001912111331003, + "variance_ratio": 1.0563476057297096, + "coverage": 0.957, + "coverage_mcse": 0.004536022486716751, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.957, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.1753023653503434, + "meets_prespecified_thresholds": true + }, + "design_precision_oracle": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": -0.0011786471651707986, + "bias_mcse": 0.0009730100664072463, + "rmse": 0.043519416806872574, + "empirical_sd": 0.04351433302556375, + "mean_variance": 0.00200045612920578, + "variance_ratio": 1.0564875151394861, + "coverage": 0.957, + "coverage_mcse": 0.004536022486716751, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.957, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.17532449735218605, + "meets_prespecified_thresholds": true + } + } + }, + "8": { + "design": { + "n": 4000, + "sparse_treated": 20, + "sparse_controls": 190, + "regime": "growing" + }, + "elapsed_seconds": 35.19077399299931, + "failures": {}, + "native_max_abs_error": 4.218847493575595e-15, + "native_variance_max_abs_error": 9.540979117872439e-17, + "oracle_variance_ratio_design_to_identity": 0.23512265016825207, + "variants": { + "identity_hc0": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": -0.0010834860170835237, + "bias_mcse": 0.002102475179479249, + "rmse": 0.09400828314952618, + "empirical_sd": 0.09402554844643343, + "mean_variance": 0.008226522038423485, + "variance_ratio": 0.9305174349686309, + "coverage": 0.9345, + "coverage_mcse": 0.005532167296819575, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9345, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.35295429606938783, + "meets_prespecified_thresholds": false + }, + "identity_oracle": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": -0.0010834860170835237, + "bias_mcse": 0.002102475179479249, + "rmse": 0.09400828314952618, + "empirical_sd": 0.09402554844643343, + "mean_variance": 0.0085400868234541, + "variance_ratio": 0.9659853396433378, + "coverage": 0.946, + "coverage_mcse": 0.005053909377897473, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.946, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.36225069693487916, + "meets_prespecified_thresholds": true + }, + "design_precision_hc0": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": 0.0007949803361091412, + "bias_mcse": 0.0010029698838864697, + "rmse": 0.04485000803916643, + "empirical_sd": 0.044854176795104346, + "mean_variance": 0.002007460417218636, + "variance_ratio": 0.997794738816257, + "coverage": 0.95, + "coverage_mcse": 0.004873397172404485, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.95, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.17562013900111148, + "meets_prespecified_thresholds": true + }, + "design_precision_oracle": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": 0.0007949803361091412, + "bias_mcse": 0.0010029698838864697, + "rmse": 0.04485000803916643, + "empirical_sd": 0.044854176795104346, + "mean_variance": 0.002007967846597497, + "variance_ratio": 0.9980469531863165, + "coverage": 0.951, + "coverage_mcse": 0.004826955562256609, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.951, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.17565336086715963, + "meets_prespecified_thresholds": true + } + } + }, + "9": { + "design": { + "n": 4000, + "sparse_treated": 50, + "sparse_controls": 190, + "regime": "growing" + }, + "elapsed_seconds": 34.96935531199779, + "failures": {}, + "native_max_abs_error": 3.6637359812630166e-15, + "native_variance_max_abs_error": 5.117434254131581e-17, + "oracle_variance_ratio_design_to_identity": 0.4184442958420789, + "variants": { + "identity_hc0": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": 0.0012103728778840583, + "bias_mcse": 0.0014912796838763502, + "rmse": 0.06668636501184234, + "empirical_sd": 0.06669205493223833, + "mean_variance": 0.0047234143787862155, + "variance_ratio": 1.061959242116281, + "coverage": 0.956, + "coverage_mcse": 0.004586065852122058, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.956, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.2689643287717973, + "meets_prespecified_thresholds": true + }, + "identity_oracle": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": 0.0012103728778840583, + "bias_mcse": 0.0014912796838763502, + "rmse": 0.06668636501184234, + "empirical_sd": 0.06669205493223833, + "mean_variance": 0.0047910580744253296, + "variance_ratio": 1.0771674880998394, + "coverage": 0.962, + "coverage_mcse": 0.004275277768753747, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.962, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.27132749437135173, + "meets_prespecified_thresholds": true + }, + "design_precision_hc0": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": -0.000642713871541245, + "bias_mcse": 0.0009763903179197072, + "rmse": 0.043659315717523996, + "empirical_sd": 0.04366550246882193, + "mean_variance": 0.0020034218320402744, + "variance_ratio": 1.050740514284786, + "coverage": 0.9605, + "coverage_mcse": 0.004355441998236229, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9605, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.17544333001118065, + "meets_prespecified_thresholds": true + }, + "design_precision_oracle": { + "truth": 1.0, + "replications": 2000, + "failures": 0, + "bias": -0.000642713871541245, + "bias_mcse": 0.0009763903179197072, + "rmse": 0.043659315717523996, + "empirical_sd": 0.04366550246882193, + "mean_variance": 0.0020047909222914153, + "variance_ratio": 1.0514585650575088, + "coverage": 0.9595, + "coverage_mcse": 0.004407933189148855, + "nominal_binomial_mcse": 0.004873397172404482, + "coverage_given_valid": 0.9595, + "rejection_at_zero": 1.0, + "rejection_mcse": 0.0, + "mean_interval_width": 0.17551435009233707, + "meets_prespecified_thresholds": true + } + } + } + }, + "statistically_qualified": false, + "thresholds": "zero failures; |bias|<.1 empirical SD; variance ratio [.9,1.1]; coverage within 3 nominal binomial MCSE", + "interpretation": "Oracle covariance is unavailable in empirical applications; no general inference qualification.", + "variant_thresholds_pass": { + "identity_hc0": false, + "identity_oracle": true, + "design_precision_hc0": true, + "design_precision_oracle": true + } +} diff --git a/benchmarks/environments/chained_did/Dockerfile b/benchmarks/environments/chained_did/Dockerfile new file mode 100644 index 000000000..5223d8120 --- /dev/null +++ b/benchmarks/environments/chained_did/Dockerfile @@ -0,0 +1,5 @@ +FROM rocker/r-ver:4.5.2@sha256:fd4ccdd3a4a6f7ef805e2daeee2a0fe3bf126bc231f36351223baecf5a595a4c + +ENV MAKEFLAGS=-j2 +RUN Rscript -e 'options(repos=c(CRAN="https://cloud.r-project.org")); install.packages(c("remotes", "jsonlite")); remotes::install_version("DRDID", version="1.3.0", dependencies=NA, upgrade="never"); remotes::install_version("did", version="2.5.1", dependencies=NA, upgrade="never"); remotes::install_github("dvbn/cdid@1039120668cceabd8111cad82d32ab3927c2b94f", dependencies=FALSE, upgrade="never", build_vignettes=FALSE); stopifnot(getRversion()=="4.5.2", packageVersion("cdid")=="1.1.0.0", packageVersion("did")=="2.5.1", packageVersion("DRDID")=="1.3.0")' +WORKDIR /work diff --git a/benchmarks/environments/chained_did/README.md b/benchmarks/environments/chained_did/README.md new file mode 100644 index 000000000..b9ccf1b33 --- /dev/null +++ b/benchmarks/environments/chained_did/README.md @@ -0,0 +1,25 @@ +# External chained DiD reference environment + +Build with `DOCKER_BUILDKIT=0 docker build -t diff-diff-cdid-reference:1.1.0.0 .` +from this directory. The explicit legacy-builder setting works on the inspected +host, whose Docker installation has no buildx component. + +The base image is digest-pinned. R 4.5.2, cdid 1.1.0.0 at commit +1039120668cceabd8111cad82d32ab3927c2b94f, did 2.5.1 and DRDID 1.3.0 are +asserted during build and independently checked by the benchmark script. +`packages.json` records the full installed environment used for the comparison. +Transitive CRAN dependencies are resolved at build time; the recipe does not +promise a hermetic rebuild of every transitive package. The executed image ID +and this manifest's hash are recorded in the comparison output. Before fitting, +the comparator verifies the selected image's R/platform and every installed +package/version/build against the manifest; it refuses a changed inventory. +An intentional environment refresh therefore requires a separately reviewed +manifest update, rather than silently relabelling a rebuilt image. + +Run `benchmarks/python/compare_chained_did.py` from the repository root. It +mounts the source read-only, writes inputs/outputs into its declared workspace +under the caller's UID, and never publishes an image or installs R packages +into the host. Its nonzero exit retains failing comparisons rather than +turning a mismatch into reference agreement. +The report also retains the asserted public callback weight probe from +`benchmarks/R/inspect_chained_did_weights.R`; that probe stops before estimation. diff --git a/benchmarks/environments/chained_did/packages.json b/benchmarks/environments/chained_did/packages.json new file mode 100644 index 000000000..a34f1dfd3 --- /dev/null +++ b/benchmarks/environments/chained_did/packages.json @@ -0,0 +1,487 @@ +{ + "R": "R version 4.5.2 (2025-10-31)", + "platform": "x86_64-pc-linux-gnu", + "packages": [ + { + "Package": "BH", + "Version": "1.90.0-1", + "Built": "4.5.2", + "_row": "BH" + }, + { + "Package": "bigmemory", + "Version": "4.6.6", + "Built": "4.5.2", + "_row": "bigmemory" + }, + { + "Package": "bigmemory.sri", + "Version": "0.1.8", + "Built": "4.5.2", + "_row": "bigmemory.sri" + }, + { + "Package": "BMisc", + "Version": "1.4.10", + "Built": "4.5.2", + "_row": "BMisc" + }, + { + "Package": "cdid", + "Version": "1.1.0.0", + "Built": "4.5.2", + "_row": "cdid" + }, + { + "Package": "cli", + "Version": "3.6.6", + "Built": "4.5.2", + "_row": "cli" + }, + { + "Package": "cpp11", + "Version": "0.5.5", + "Built": "4.5.2", + "_row": "cpp11" + }, + { + "Package": "data.table", + "Version": "1.18.6.1", + "Built": "4.5.2", + "_row": "data.table" + }, + { + "Package": "did", + "Version": "2.5.1", + "Built": "4.5.2", + "_row": "did" + }, + { + "Package": "docopt", + "Version": "0.7.2", + "Built": "4.5.2", + "_row": "docopt" + }, + { + "Package": "dplyr", + "Version": "1.2.1", + "Built": "4.5.2", + "_row": "dplyr" + }, + { + "Package": "DRDID", + "Version": "1.3.0", + "Built": "4.5.2", + "_row": "DRDID" + }, + { + "Package": "dreamerr", + "Version": "1.5.0", + "Built": "4.5.2", + "_row": "dreamerr" + }, + { + "Package": "farver", + "Version": "2.1.2", + "Built": "4.5.2", + "_row": "farver" + }, + { + "Package": "fastglm", + "Version": "0.1.2", + "Built": "4.5.2", + "_row": "fastglm" + }, + { + "Package": "Formula", + "Version": "1.2-6", + "Built": "4.5.2", + "_row": "Formula" + }, + { + "Package": "generics", + "Version": "0.1.4", + "Built": "4.5.2", + "_row": "generics" + }, + { + "Package": "ggplot2", + "Version": "4.0.3", + "Built": "4.5.2", + "_row": "ggplot2" + }, + { + "Package": "glue", + "Version": "1.8.1", + "Built": "4.5.2", + "_row": "glue" + }, + { + "Package": "gtable", + "Version": "0.3.6", + "Built": "4.5.2", + "_row": "gtable" + }, + { + "Package": "isoband", + "Version": "0.3.0", + "Built": "4.5.2", + "_row": "isoband" + }, + { + "Package": "jsonlite", + "Version": "2.0.0", + "Built": "4.5.2", + "_row": "jsonlite" + }, + { + "Package": "labeling", + "Version": "0.4.3", + "Built": "4.5.2", + "_row": "labeling" + }, + { + "Package": "lifecycle", + "Version": "1.0.5", + "Built": "4.5.2", + "_row": "lifecycle" + }, + { + "Package": "littler", + "Version": "0.3.22", + "Built": "4.5.2", + "_row": "littler" + }, + { + "Package": "magrittr", + "Version": "2.0.5", + "Built": "4.5.2", + "_row": "magrittr" + }, + { + "Package": "pbapply", + "Version": "1.7-5", + "Built": "4.5.2", + "_row": "pbapply" + }, + { + "Package": "pillar", + "Version": "1.11.1", + "Built": "4.5.2", + "_row": "pillar" + }, + { + "Package": "pkgconfig", + "Version": "2.0.3", + "Built": "4.5.2", + "_row": "pkgconfig" + }, + { + "Package": "purrr", + "Version": "1.2.2", + "Built": "4.5.2", + "_row": "purrr" + }, + { + "Package": "R6", + "Version": "2.6.1", + "Built": "4.5.2", + "_row": "R6" + }, + { + "Package": "RColorBrewer", + "Version": "1.1-3", + "Built": "4.5.2", + "_row": "RColorBrewer" + }, + { + "Package": "Rcpp", + "Version": "1.1.2", + "Built": "4.5.2", + "_row": "Rcpp" + }, + { + "Package": "RcppArmadillo", + "Version": "15.6.2-1", + "Built": "4.5.2", + "_row": "RcppArmadillo" + }, + { + "Package": "RcppEigen", + "Version": "0.3.4.0.2", + "Built": "4.5.2", + "_row": "RcppEigen" + }, + { + "Package": "remotes", + "Version": "2.5.0", + "Built": "4.5.2", + "_row": "remotes" + }, + { + "Package": "rlang", + "Version": "1.3.0", + "Built": "4.5.2", + "_row": "rlang" + }, + { + "Package": "S7", + "Version": "0.2.2", + "Built": "4.5.2", + "_row": "S7" + }, + { + "Package": "scales", + "Version": "1.4.0", + "Built": "4.5.2", + "_row": "scales" + }, + { + "Package": "stringi", + "Version": "1.8.9", + "Built": "4.5.2", + "_row": "stringi" + }, + { + "Package": "stringmagic", + "Version": "1.3.0", + "Built": "4.5.2", + "_row": "stringmagic" + }, + { + "Package": "stringr", + "Version": "1.6.0", + "Built": "4.5.2", + "_row": "stringr" + }, + { + "Package": "tibble", + "Version": "3.3.1", + "Built": "4.5.2", + "_row": "tibble" + }, + { + "Package": "tidyr", + "Version": "1.3.2", + "Built": "4.5.2", + "_row": "tidyr" + }, + { + "Package": "tidyselect", + "Version": "1.2.1", + "Built": "4.5.2", + "_row": "tidyselect" + }, + { + "Package": "trust", + "Version": "0.1-9", + "Built": "4.5.2", + "_row": "trust" + }, + { + "Package": "utf8", + "Version": "1.2.6", + "Built": "4.5.2", + "_row": "utf8" + }, + { + "Package": "uuid", + "Version": "1.2-2", + "Built": "4.5.2", + "_row": "uuid" + }, + { + "Package": "vctrs", + "Version": "0.7.3", + "Built": "4.5.2", + "_row": "vctrs" + }, + { + "Package": "viridisLite", + "Version": "0.4.3", + "Built": "4.5.2", + "_row": "viridisLite" + }, + { + "Package": "withr", + "Version": "3.0.3", + "Built": "4.5.2", + "_row": "withr" + }, + { + "Package": "base", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "base" + }, + { + "Package": "boot", + "Version": "1.3-32", + "Built": "4.5.2", + "_row": "boot" + }, + { + "Package": "class", + "Version": "7.3-23", + "Built": "4.5.2", + "_row": "class" + }, + { + "Package": "cluster", + "Version": "2.1.8.1", + "Built": "4.5.2", + "_row": "cluster" + }, + { + "Package": "codetools", + "Version": "0.2-20", + "Built": "4.5.2", + "_row": "codetools" + }, + { + "Package": "compiler", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "compiler" + }, + { + "Package": "datasets", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "datasets" + }, + { + "Package": "foreign", + "Version": "0.8-90", + "Built": "4.5.2", + "_row": "foreign" + }, + { + "Package": "graphics", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "graphics" + }, + { + "Package": "grDevices", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "grDevices" + }, + { + "Package": "grid", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "grid" + }, + { + "Package": "KernSmooth", + "Version": "2.23-26", + "Built": "4.5.2", + "_row": "KernSmooth" + }, + { + "Package": "lattice", + "Version": "0.22-7", + "Built": "4.5.2", + "_row": "lattice" + }, + { + "Package": "MASS", + "Version": "7.3-65", + "Built": "4.5.2", + "_row": "MASS" + }, + { + "Package": "Matrix", + "Version": "1.7-4", + "Built": "4.5.2", + "_row": "Matrix" + }, + { + "Package": "methods", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "methods" + }, + { + "Package": "mgcv", + "Version": "1.9-3", + "Built": "4.5.2", + "_row": "mgcv" + }, + { + "Package": "nlme", + "Version": "3.1-168", + "Built": "4.5.2", + "_row": "nlme" + }, + { + "Package": "nnet", + "Version": "7.3-20", + "Built": "4.5.2", + "_row": "nnet" + }, + { + "Package": "parallel", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "parallel" + }, + { + "Package": "rpart", + "Version": "4.1.24", + "Built": "4.5.2", + "_row": "rpart" + }, + { + "Package": "spatial", + "Version": "7.3-18", + "Built": "4.5.2", + "_row": "spatial" + }, + { + "Package": "splines", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "splines" + }, + { + "Package": "stats", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "stats" + }, + { + "Package": "stats4", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "stats4" + }, + { + "Package": "survival", + "Version": "3.8-3", + "Built": "4.5.2", + "_row": "survival" + }, + { + "Package": "tcltk", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "tcltk" + }, + { + "Package": "tools", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "tools" + }, + { + "Package": "utils", + "Version": "4.5.2", + "Built": "4.5.2", + "_row": "utils" + } + ], + "session": ["R version 4.5.2 (2025-10-31)", "Platform: x86_64-pc-linux-gnu", "Running under: Ubuntu 24.04.4 LTS", "", "Matrix products: default", "BLAS: /usr/lib/x86_64-linux-gnu/openblas-pthread/libblas.so.3 ", "LAPACK: /usr/lib/x86_64-linux-gnu/openblas-pthread/libopenblasp-r0.3.26.so; LAPACK version 3.12.0", "", "locale:", " [1] LC_CTYPE=en_US.UTF-8 LC_NUMERIC=C ", " [3] LC_TIME=en_US.UTF-8 LC_COLLATE=en_US.UTF-8 ", " [5] LC_MONETARY=en_US.UTF-8 LC_MESSAGES=en_US.UTF-8 ", " [7] LC_PAPER=en_US.UTF-8 LC_NAME=C ", " [9] LC_ADDRESS=C LC_TELEPHONE=C ", "[11] LC_MEASUREMENT=en_US.UTF-8 LC_IDENTIFICATION=C ", "", "time zone: Etc/UTC", "tzcode source: system (glibc)", "", "attached base packages:", "[1] stats graphics grDevices utils datasets methods base ", "", "loaded via a namespace (and not attached):", "[1] compiler_4.5.2 tools_4.5.2 jsonlite_2.0.0"] +} diff --git a/benchmarks/python/compare_chained_did.py b/benchmarks/python/compare_chained_did.py new file mode 100644 index 000000000..6a3454ec8 --- /dev/null +++ b/benchmarks/python/compare_chained_did.py @@ -0,0 +1,379 @@ +"""Matched public-output comparison with pinned cdid; never reads its estimator source.""" + +import argparse +import hashlib +import json +import os +import subprocess +from pathlib import Path + +import numpy as np +import pandas as pd +from scipy.optimize import root +from scipy.special import expit + +from benchmarks.python.validate_chained_did import generate_data +from diff_diff import ChainedDiD + +IMAGE = "diff-diff-cdid-reference:1.1.0.0" + + +def validate_environment(expected, actual): + """Fail before estimation if the selected image differs from its inventory.""" + + def inventory(value): + return ( + value["R"], + value["platform"], + sorted((p["Package"], p["Version"], p["Built"]) for p in value["packages"]), + ) + + if inventory(expected) != inventory(actual): + raise ValueError("Reference image package inventory differs from packages.json") + + +def compare(native, reference, *, outcome_sd, clusters=None): + """Compare original labels/registry and predeclared estimator-scale inference.""" + meta = reference["metadata"] + if meta["covariance_scale"] != "estimator": + raise ValueError("Reference must declare estimator covariance scale") + if meta["cluster_correction"] != "none": + raise ValueError("Reference must declare its raw cluster correction") + correction = clusters / (clusters - 1) if clusters is not None else 1.0 + rows = reference["cells"] + keys = [(r["group"], r["time"]) for r in rows] + checks = { + "registry": meta["n"] == native.n_units, + "labels": len(set(keys)) == len(keys) and set(keys) == set(native.vcov_index), + } + atol = 1e-7 * max(1.0, outcome_sd) + cov_atol = 1e-7 * max(1.0, outcome_sd**2) + + def close(a, b, *, covariance=False): + a, b = np.asarray(a, dtype=float), np.asarray(b, dtype=float) + return bool( + a.shape == b.shape + and np.allclose( + a, + b, + rtol=1e-5 if covariance else 1e-6, + atol=cov_atol if covariance else atol, + equal_nan=True, + ) + ) + + if checks["labels"]: + order = [keys.index(k) for k in native.vcov_index] + checks["support"] = all( + rows[i]["available"] == (native.group_time_effects[k].skip_reason is None) + for k, i in zip(native.vcov_index, order) + ) + checks["estimates"] = close( + [native.group_time_effects[k].effect for k in native.vcov_index], + [rows[i]["att"] for i in order], + ) + covariance = np.asarray(reference["vcov"], dtype=float)[np.ix_(order, order)] * correction + checks["covariance"] = close(native.vcov, covariance, covariance=True) + finite = np.isfinite(covariance) & np.isfinite(native.vcov) + covariance_error = ( + float(np.max(np.abs(covariance[finite] - native.vcov[finite]))) + if finite.any() + else None + ) + else: + checks.update(support=False, estimates=False, covariance=False) + covariance_error = None + checks["overall_estimate"] = close(native.att, reference["overall"]["att"]) + ref_se = np.asarray(reference["overall"]["se"], dtype=float) * np.sqrt(correction) + checks["overall_se"] = close(native.se, ref_se) + if "events" in reference: + event = native.aggregate("event_study") + r = reference["events"] + event_keys = list(r["event_time"]) + checks["event_labels"] = set(event_keys) == set(event.event_time) and len( + event_keys + ) == len(event.event_time) + if checks["event_labels"]: + order = [event_keys.index(t) for t in event.event_time] + checks["event_estimates"] = close(event.att, np.asarray(r["att"], dtype=float)[order]) + rse = np.asarray(r["se"], dtype=float)[order] * np.sqrt(correction) + refs = np.asarray(r["is_reference"], dtype=bool)[order] + checks["event_reference_roles"] = bool(np.array_equal(refs, event.is_reference)) + # Canonical Python references carry NaN inference; R carries exact zero SE. + checks["event_reference_se"] = bool( + np.all(rse[refs] == 0) and np.isnan(event.se[event.is_reference]).all() + ) + checks["event_se"] = close(event.se[~event.is_reference], rse[~event.is_reference]) + checks["event_covariance"] = close( + event.vcov, + np.asarray(r["vcov"], dtype=float)[np.ix_(order, order)] * correction, + covariance=True, + ) + if "events" not in reference: + checks["event_schema"] = False + return dict( + passed=all(checks.values()), + checks=checks, + covariance_max_abs_error=covariance_error, + cluster_cr1_factor_applied=correction, + tolerances=dict( + estimate_rtol=1e-6, estimate_atol=atol, covariance_rtol=1e-5, covariance_atol=cov_atol + ), + ) + + +def rotating(windows=((0, 1), (1, 2), (2, 3))): + rows = [] + for cohort in (0, 2): + for w in windows: + for noise in (-1.5, -0.5, 0.5, 1.5): + uid = max([r[0] for r in rows], default=0) + 1 + for t in w: + y = 3 * t + noise * t + if cohort: + y += 10 + 2 * (t >= 1) + {2: 3, 3: 7}.get(t, 0) + rows.append((uid, t, cohort, y)) + data = pd.DataFrame(rows, columns=["id", "t", "g", "y"]) + for t in set(range(4)) - set(data.t): + data = pd.concat([data, data.iloc[[0]].assign(t=t, y=np.nan)], ignore_index=True) + return data + + +def fixtures(): + result = { + "rotating": rotating(), + "earlier_bridge": rotating(((0, 1), (0, 3))), + "nonadjacent": rotating(((1, 3),)), + } + for name, observation, clusters in ( + ("balanced_binary_x", "balanced", 0), + ("intermittent_shared_cluster", "intermittent", 40), + ): + data, _ = generate_data(800, observation, 110, clusters=clusters) + result[name] = data if clusters else data.drop(columns="cluster") + result["two_date"] = rotating(((1, 2),)).query("t >= 1 and t <= 2").copy() + data = result["intermittent_shared_cluster"] + extra = [ + dict(id=800 + j, t=0, g=3, x=j % 2, y=np.nan if j < 2 else 0.0, cluster=j) for j in range(4) + ] + result["registry_units"] = pd.concat([data, pd.DataFrame(extra)], ignore_index=True) + rng = np.random.default_rng(141) + n = 800 + x = rng.normal(size=n) + g = np.where(rng.random(n) < expit(0.2 + 0.5 * x), 2, 0) + y = ( + rng.normal(size=(n, 4)) + + np.arange(4) * x[:, None] + + 3 * ((g[:, None] > 0) & (np.arange(4) >= 2)) + ) + continuous = pd.DataFrame( + dict( + id=np.repeat(np.arange(n), 4), + t=np.tile(np.arange(4), n), + g=np.repeat(g, 4), + x=np.repeat(x, 4), + y=y.ravel(), + ) + ) + result["balanced_continuous_x"] = continuous + missing = rng.random((n, 4)) > np.where(g[:, None] > 0, 0.25, 0.1) + result["incomplete_continuous_x"] = continuous.assign(y=np.where(missing, y, np.nan).ravel()) + for frame in result.values(): + frame["id"] = pd.factorize(frame.id, sort=True)[0] + 1 + return result + + +def calibrated_points(data): + """Independent weighted-score diagnostic; returns points, never inference.""" + registry = data.drop_duplicates("id") + times = sorted(data.t.unique()) + y = ( + data.pivot(index="id", columns="t", values="y") + .reindex(index=registry.id, columns=times) + .to_numpy() + ) + membership, x = registry.g.to_numpy(), registry.x.to_numpy() + eligible = np.isfinite(y).sum(axis=1) >= 2 + points = {} + for g in sorted(set(membership) - {0}): + base = times.index(g) - 1 + cols = [j for j in range(len(times)) if j != base] + rows, values, direct = [], [], {} + for s in range(len(times)): + for t in range(s + 1, len(times)): + observed = np.isfinite(y[:, s]) & np.isfinite(y[:, t]) + selected = observed & ((membership == g) | (membership == 0)) + treated = membership[selected] == g + if not treated.any() or treated.all(): + continue + z = np.column_stack([np.ones(selected.sum()), x[selected]]) + rate_t = treated.sum() / ((membership == g) & eligible).sum() + rate_c = (~treated).sum() / ((membership == 0) & eligible).sum() + calibration = np.where(treated, 1 / rate_t, 1 / rate_c) + fitted = root( + lambda b: z.T @ (calibration * (treated - expit(z @ b))), np.zeros(z.shape[1]) + ) + if not fitted.success: + raise ValueError("Calibrated-score diagnostic did not converge") + odds = np.exp(z @ fitted.x) + delta = y[selected, t] - y[selected, s] + w = (~treated) * odds + contrast = delta[treated].mean() - w @ delta / w.sum() + row = np.zeros(len(times)) + row[s], row[t] = -1, 1 + rows.append(row[cols]) + values.append(contrast) + direct[s, t] = contrast + a = np.asarray(rows) + if np.linalg.matrix_rank(a) != len(cols): + raise ValueError("Calibrated-score diagnostic requires full rank") + recovered = np.linalg.lstsq(a, values, rcond=None)[0] + for t, label in enumerate(times): + points[g, label] = ( + 0.0 + if t == base + else (-direct[t, base] if t < base else float(recovered[cols.index(t)])) + ) + return points + + +def finite_json(value): + """Represent undefined inference as JSON null in external-reference records.""" + if isinstance(value, dict): + return {k: finite_json(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [finite_json(v) for v in value] + if isinstance(value, (float, np.floating)) and not np.isfinite(value): + return None + if isinstance(value, np.generic): + return value.item() + return value + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--image", default=IMAGE) + parser.add_argument( + "--output", type=Path, default=Path("benchmarks/data/chained_did/reference-validation.json") + ) + parser.add_argument("--workspace", type=Path, required=True) + args = parser.parse_args() + root = Path.cwd().resolve() + args.workspace.mkdir(parents=True, exist_ok=True) + work = args.workspace.resolve() + script = root / "benchmarks/R/benchmark_chained_did.R" + probe = root / "benchmarks/R/inspect_chained_did_weights.R" + environment = root / "benchmarks/environments/chained_did/packages.json" + installed = json.loads( + subprocess.check_output( + [ + "docker", + "run", + "--rm", + args.image, + "Rscript", + "-e", + "jsonlite::write_json(list(R=R.version.string, platform=R.version$platform, " + 'packages=as.data.frame(installed.packages()[,c("Package","Version","Built")])), ' + "stdout(), auto_unbox=TRUE)", + ], + text=True, + ) + ) + validate_environment(json.loads(environment.read_text()), installed) + report = dict( + status="running", + image=args.image, + image_id=subprocess.check_output( + ["docker", "image", "inspect", args.image, "--format", "{{.Id}}"], text=True + ).strip(), + environment_sha256=hashlib.sha256(environment.read_bytes()).hexdigest(), + environment_verified=True, + installed_environment=installed, + source_hashes={ + str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in [ + script, + probe, + root / "diff_diff/chained_did.py", + root / "diff_diff/chained_did_results.py", + Path(__file__).resolve(), + ] + }, + cases={}, + ) + for name, data in fixtures().items(): + data.to_csv(work / f"{name}.csv", index=False) + native = ChainedDiD(cluster="cluster" if "cluster" in data else None).fit( + data, "y", "id", "t", "g", covariates=["x"] if "x" in data else None + ) + command = [ + "docker", + "run", + "--rm", + "--user", + f"{os.getuid()}:{os.getgid()}", + "-v", + f"{root}:/repo:ro", + "-v", + f"{work}:/work", + args.image, + "Rscript", + "/repo/benchmarks/R/benchmark_chained_did.R", + f"/work/{name}.csv", + f"/work/{name}.json", + ] + run = subprocess.run(command, capture_output=True, text=True) + if run.returncode: + report["cases"][name] = dict( + passed=False, + error=run.stderr[-6000:], + input_sha256=hashlib.sha256((work / f"{name}.csv").read_bytes()).hexdigest(), + ) + else: + reference = json.loads((work / f"{name}.json").read_text()) + k = data.drop_duplicates("id").cluster.nunique() if "cluster" in data else None + outcome = compare(native, reference, outcome_sd=float(data.y.std()), clusters=k) + if not outcome["checks"].get("estimates", False) and "x" in data: + diagnostic = calibrated_points(data) + expected = [row["att"] for row in reference["cells"]] + actual = [diagnostic[row["group"], row["time"]] for row in reference["cells"]] + outcome["calibrated_score_point_diagnostic"] = dict( + matches_reference=bool( + np.allclose( + actual, + np.asarray(expected, dtype=float), + rtol=1e-6, + atol=1e-7 * max(1.0, float(data.y.std())), + equal_nan=True, + ) + ), + points=[dict(group=g, time=t, att=v) for (g, t), v in diagnostic.items()], + interpretation="Weighted-score point diagnostic only; native point criterion and inference remain unchanged.", + ) + outcome["reference"] = reference + outcome["native"] = native.to_dict() + report["cases"][name] = outcome + report["cases"][name]["input_sha256"] = hashlib.sha256( + (work / f"{name}.csv").read_bytes() + ).hexdigest() + print(f"Reference {name}: {report['cases'][name]['passed']}", flush=True) + probe_command = command[:-3] + [ + "/repo/benchmarks/R/inspect_chained_did_weights.R", + "/work/incomplete_continuous_x.csv", + "/work/public-input-probe.json", + ] + subprocess.run(probe_command, check=True, capture_output=True, text=True) + report["public_input_probe"] = json.loads((work / "public-input-probe.json").read_text()) + report["public_input_probe"]["input_sha256"] = report["cases"]["incomplete_continuous_x"][ + "input_sha256" + ] + report["status"] = "complete" + report["all_comparisons_pass"] = all(r["passed"] for r in report["cases"].values()) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(finite_json(report), indent=2, allow_nan=False) + "\n") + return 0 if report["all_comparisons_pass"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/python/diagnose_chained_did.py b/benchmarks/python/diagnose_chained_did.py new file mode 100644 index 000000000..81863f544 --- /dev/null +++ b/benchmarks/python/diagnose_chained_did.py @@ -0,0 +1,280 @@ +"""Frozen-contract follow-up and controlled calibration experiments for ChainedDiD. + +The oracle is validation-only: it knows population treatment odds and/or cohort +masses. It is not an estimator option. Original qualification files are read-only. +""" + +import argparse +import hashlib +import json +import platform +import time +from concurrent.futures import ProcessPoolExecutor, as_completed +from pathlib import Path + +import numpy as np +import pandas as pd +import scipy + +from benchmarks.python.validate_chained_did import generate_data +from benchmarks.python.validate_chained_did import summarize as base_summary +from diff_diff import ChainedDiD + +BASELINE = Path("benchmarks/data/chained_did/qualification.json") +TARGETS = ("overall", "g2_t3", "g3_t4") +VARIANTS = ("fitted_empirical", "fitted_known_mass", "oracle_empirical", "oracle_known_mass") + + +def replication_seed(seed, case_index, replication): + """Use a fresh 64-bit seed stream without the original 32-bit truncation.""" + return int( + np.random.SeedSequence([seed, case_index, replication]).generate_state(1, dtype=np.uint64)[ + 0 + ] + ) + + +def diagnostic_cases(): + """Cross n and K rather than attributing their joint change to either one.""" + return [ + dict(n=n, observation=observation, clusters=k, null=False, violation="none") + for n in (1000, 4000) + for observation, k in ( + ("rotating", 0), + ("intermittent", 0), + ("intermittent", 50), + ("intermittent", 100), + ) + ] + + +def population_law(): + x = np.array([0.0, 1.0]) + p = np.column_stack([np.ones(2), np.exp(-0.2 + 0.7 * x), np.exp(-0.4 - 0.4 * x)]) + p /= p.sum(axis=1)[:, None] + return {2: float(p[:, 1].mean()), 3: float(p[:, 2].mean())} + + +def oracle_cells(data, odds_by_cohort): + """Independently recover post cells using known odds and ratio-score IFs. + + Cohort-specific pair response rates multiply odds by a constant, which + cancels in the normalised control mean. These diagnostics use the valid + baseline sampling designs only. Full rank is required, not presumed. + """ + registry = data.drop_duplicates("id") + times = sorted(data.t.unique()) + y = ( + data.pivot(index="id", columns="t", values="y") + .reindex(index=registry.id, columns=times) + .to_numpy() + ) + groups = registry.g.to_numpy() + cells, influence = {}, {} + for g, odds in odds_by_cohort.items(): + ref = times.index(g) - 1 + columns = [j for j in range(len(times)) if j != ref] + rows, contrasts, scores = [], [], [] + for s in range(len(times)): + for t in range(s + 1, len(times)): + treated = (groups == g) & np.isfinite(y[:, s]) & np.isfinite(y[:, t]) + control = (groups == 0) & np.isfinite(y[:, s]) & np.isfinite(y[:, t]) + if not treated.any() or not control.any(): + continue + delta = np.nan_to_num(y[:, t] - y[:, s], nan=0.0) + w = control * odds + mt, mc = delta[treated].mean(), w @ delta / w.sum() + q = treated * (delta - mt) / treated.sum() - w * (delta - mc) / w.sum() + row = np.zeros(len(times)) + row[s], row[t] = -1, 1 + rows.append(row[columns]) + contrasts.append(mt - mc) + scores.append(q) + a = np.asarray(rows) + if not rows or np.linalg.matrix_rank(a) != len(columns): + raise ValueError("Oracle diagnostic requires a connected comparison system") + recovery = np.linalg.lstsq(a, np.eye(len(rows)), rcond=None)[0] + estimates = recovery @ contrasts + contributions = np.column_stack(scores) @ recovery.T + for j, t in enumerate(columns): + if times[t] >= g: + cells[g, times[t]] = float(estimates[j]) + influence[g, times[t]] = contributions[:, j] + return cells, influence + + +def combine(cells, influence, membership, masses=None): + """Combine cell contributions with either estimated or known population mass.""" + keys = list(cells) + effects = np.array([cells[k] for k in keys]) + indicators = np.column_stack([membership == g for g, _ in keys]).astype(float) + mass = indicators.sum(axis=0) if masses is None else np.array([masses[g] for g, _ in keys]) + weights = mass / mass.sum() + q = np.column_stack([influence[k] for k in keys]) @ weights + if masses is None: + # Ratio differentiation retains uncertainty in the supplied registry shares. + estimate = weights @ effects + q += indicators @ (effects - estimate) / mass.sum() + return float(weights @ effects), q + + +def standard_error(q, labels=None): + if labels is None: + return float(np.linalg.norm(q)) + codes, unique = pd.factorize(labels, sort=False) + if len(unique) < 2: + return np.nan + sums = np.zeros(len(unique)) + np.add.at(sums, codes, q) + return float(np.sqrt(len(unique) / (len(unique) - 1) * (sums @ sums))) + + +def summarize(estimates, ses, truth): + result = base_summary(estimates, ses, truth) + valid = np.isfinite(estimates) & np.isfinite(ses) & (ses > 0) + z = (estimates[valid] - truth) / ses[valid] + result.update( + z_mean=float(z.mean()) if len(z) else np.nan, + z_sd=float(z.std(ddof=1)) if len(z) > 1 else np.nan, + z_quantiles=np.quantile(z, [0.025, 0.5, 0.975]).tolist() if len(z) else [np.nan] * 3, + abs_z_95=float(np.quantile(np.abs(z), 0.95)) if len(z) else np.nan, + ) + return result + + +def run_case(case, replications, seed, index, diagnostic): + started = time.monotonic() + variants = VARIANTS if diagnostic else VARIANTS[:1] + estimates = np.full((replications, len(variants), 3), np.nan) + ses = np.full_like(estimates, np.nan) + failures = {name: {} for name in variants} + for j in range(replications): + data, truth = generate_data(seed=replication_seed(seed, index, j), **case) + registry = data.drop_duplicates("id") + membership = registry.g.to_numpy() + labels = registry.cluster.to_numpy() if case["clusters"] else None + fitted = None + try: + fitted = ChainedDiD(cluster="cluster" if labels is not None else None).fit( + data, "y", "id", "t", "g", covariates=["x"] + ) + except (ValueError, np.linalg.LinAlgError) as exc: + for name in variants[:2]: + failures[name][str(exc)] = failures[name].get(str(exc), 0) + 1 + oracle = None + if diagnostic: + try: + x = registry.x.to_numpy() + oracle = oracle_cells(data, {2: np.exp(-0.2 + 0.7 * x), 3: np.exp(-0.4 - 0.4 * x)}) + except (ValueError, np.linalg.LinAlgError) as exc: + for name in variants[2:]: + failures[name][str(exc)] = failures[name].get(str(exc), 0) + 1 + for v, name in enumerate(variants): + if name.startswith("fitted"): + if fitted is None: + continue + cells = { + k: cell.effect for k, cell in fitted.group_time_effects.items() if k[1] >= k[0] + } + q = dict(zip(fitted.vcov_index, fitted.influence_functions.T)) + else: + if oracle is None: + continue + cells, q = oracle + masses = population_law() if name.endswith("known_mass") else None + overall, overall_q = combine(cells, q, membership, masses) + estimates[j, v] = [overall, cells[2, 3], cells[3, 4]] + ses[j, v] = [ + standard_error(overall_q, labels), + standard_error(q[2, 3], labels), + standard_error(q[3, 4], labels), + ] + return dict( + design=case, + elapsed_seconds=time.monotonic() - started, + variants={ + name: dict( + failures=failures[name], + targets={ + t: summarize(estimates[:, v, k], ses[:, v, k], truth[t]) + for k, t in enumerate(TARGETS) + }, + ) + for v, name in enumerate(variants) + }, + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mode", choices=["diagnostic", "qualification"], required=True) + parser.add_argument("--replications", type=int, required=True) + parser.add_argument("--seed", type=int, required=True) + parser.add_argument("--workers", type=int, default=4) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if args.replications < 2 or args.workers < 1: + parser.error("Require at least two replications and one worker") + if args.output.resolve() in [ + p.resolve() + for p in BASELINE.parent.glob("qualification*.json") + if p.name in ("qualification.json", "qualification-pre-review.json") + ]: + parser.error("Frozen baseline records cannot be overwritten") + baseline = json.loads(BASELINE.read_text()) + cases = diagnostic_cases() if args.mode == "diagnostic" else baseline["cases"] + sources = [ + "diff_diff/chained_did.py", + "diff_diff/chained_did_results.py", + "benchmarks/python/validate_chained_did.py", + "benchmarks/python/diagnose_chained_did.py", + ] + report = dict( + status="running", + mode=args.mode, + replications_per_case=args.replications, + seed=args.seed, + seed_derivation="SeedSequence([seed,case_index,replication]).generate_state(1,dtype=uint64)[0]", + python=platform.python_version(), + numpy=np.__version__, + pandas=pd.__version__, + scipy=scipy.__version__, + baseline_sha256=hashlib.sha256(BASELINE.read_bytes()).hexdigest(), + source_hashes={p: hashlib.sha256(Path(p).read_bytes()).hexdigest() for p in sources}, + cases=cases, + results={}, + acceptance_rule=baseline["thresholds"], + interpretation="Oracle variants are diagnostics, not public estimator options. Reference agreement is a separate report.", + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + + def save(): + tmp = args.output.with_suffix(".tmp") + tmp.write_text(json.dumps(report, indent=2) + "\n") + tmp.replace(args.output) + + save() + with ProcessPoolExecutor(max_workers=args.workers) as pool: + jobs = { + pool.submit(run_case, c, args.replications, args.seed, i, args.mode == "diagnostic"): i + for i, c in enumerate(cases) + } + for job in as_completed(jobs): + i = jobs[job] + report["results"][str(i)] = job.result() + save() + print(f"Completed {args.mode} case {i+1}/{len(cases)}", flush=True) + report["status"] = "complete" + if args.mode == "qualification": + report["core_thresholds_pass"] = all( + t["meets_prespecified_thresholds"] + for r in report["results"].values() + if r["design"]["violation"] == "none" + for t in r["variants"]["fitted_empirical"]["targets"].values() + ) + save() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/python/diagnose_chained_did_sparse.py b/benchmarks/python/diagnose_chained_did_sparse.py new file mode 100644 index 000000000..e5693e319 --- /dev/null +++ b/benchmarks/python/diagnose_chained_did_sparse.py @@ -0,0 +1,272 @@ +"""Same-panel Gaussian covariance diagnostic; oracle quantities are benchmark-only.""" + +import argparse +import hashlib +import json +import platform +import time +import warnings +from concurrent.futures import ProcessPoolExecutor, as_completed +from itertools import combinations +from pathlib import Path + +import numpy as np +import pandas as pd +import scipy + +from benchmarks.python.diagnose_chained_did import replication_seed +from benchmarks.python.validate_chained_did import generate_sparse_data, summarize +from diff_diff import ChainedDiD + +VARIANTS = ("identity_hc0", "identity_oracle", "design_precision_hc0", "design_precision_oracle") + + +def cases(): + """Prespecified fixed-count and proportionally growing support cases.""" + return [ + dict(n=400, sparse_treated=k, sparse_controls=19, regime="baseline") + for k in (0, 1, 2, 5, 20, 50) + ] + [ + dict(n=4000, sparse_treated=k, sparse_controls=c, regime=regime) + for k, c, regime in ( + (2, 19, "fixed"), + (5, 19, "fixed"), + (20, 190, "growing"), + (50, 190, "growing"), + ) + ] + + +def sparse_system(data): + """Build exact conditional innovation covariance for the declared DGP only.""" + registry = data.drop_duplicates("id").set_index("id") + times = sorted(data.t.unique()) + if set(registry.g) != {0, 3} or 2 not in times or 3 not in times: + raise ValueError("Diagnostic requires cohorts 0/3 and reference/target dates 2/3") + y = ( + data.pivot(index="id", columns="t", values="y") + .reindex(index=registry.index, columns=times) + .to_numpy() + ) + treated = registry.g.to_numpy() == 3 + columns = [t for t in times if t != 2] + pairs, rows, coefficients, patterns, metrics = [], [], [], [], [] + for s, t in combinations(range(len(times)), 2): + observed = np.isfinite(y[:, s]) & np.isfinite(y[:, t]) + nt, nc = np.count_nonzero(observed & treated), np.count_nonzero(observed & ~treated) + if min(nt, nc) < 2: + continue + row = np.array([float(u == times[t]) - float(u == times[s]) for u in columns]) + weight = np.where(treated, 1 / nt, -1 / nc) * observed + pairs.append((times[s], times[t])) + rows.append(row) + coefficients.append(weight) + patterns.append(observed) + metrics.append(nt * nc / (nt + nc)) + a = np.asarray(rows).reshape(-1, len(columns)) + contrast = np.array([float(t == 3) for t in columns]) + inverse = np.linalg.pinv(a) + if not np.allclose(contrast @ inverse @ a, contrast, atol=1e-12, rtol=0): + raise ValueError("Diagnostic target is unsupported by the replicated pair catalogue") + metrics = np.array([np.ones(len(pairs)), metrics]) + loadings = np.array( + [contrast @ np.linalg.pinv(np.sqrt(h)[:, None] * a) * np.sqrt(h) for h in metrics] + ) + error_loadings = np.zeros((len(registry), len(times), len(pairs))) + for j, (s, t) in enumerate(pairs): + error_loadings[:, times.index(s), j] = -coefficients[j] + error_loadings[:, times.index(t), j] = coefficients[j] + b = error_loadings.reshape(-1, len(pairs)) + omega = b.T @ b + return dict( + units=registry.index, + times=times, + treated=treated, + pairs=pairs, + incidence=a, + coefficients=np.array(coefficients).T, + observed=np.array(patterns).T, + metrics=metrics, + loadings=loadings, + error_loadings=b, + omega=omega, + oracle_variances=np.einsum("ij,jk,ik->i", loadings, omega, loadings), + ) + + +def evaluate(data, system): + """Evaluate two point estimators and two covariance choices on one panel.""" + times = system["times"] + y = ( + data.pivot(index="id", columns="t", values="y") + .reindex(index=system["units"], columns=times) + .to_numpy() + ) + d = np.nan_to_num(y).ravel() @ system["error_loadings"] + q = np.zeros_like(system["coefficients"]) + for j, (s, t) in enumerate(system["pairs"]): + observed = system["observed"][:, j] + target, control = observed & system["treated"], observed & ~system["treated"] + delta = y[:, times.index(t)] - y[:, times.index(s)] + centre = np.where(system["treated"], delta[target].mean(), delta[control].mean()) + q[observed, j] = system["coefficients"][observed, j] * (delta[observed] - centre[observed]) + points = system["loadings"] @ d + hc0_variances = np.sum((q @ system["loadings"].T) ** 2, axis=0) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="ChainedDiD pair support:", category=UserWarning) + native = ChainedDiD().fit(data, "y", "id", "t", "g") + cell = native.group_time_effects[3, 3] + point_error = float(abs(points[0] - cell.effect)) + variance_error = float(abs(hc0_variances[0] - cell.se**2)) + if ( + not np.isfinite([point_error, variance_error]).all() + or point_error > 1e-10 + or variance_error > 1e-10 + ): + raise ValueError("Independent identity recovery differs from native estimate or covariance") + return dict( + estimates=np.repeat(points, 2), + ses=np.sqrt( + np.array( + [ + hc0_variances[0], + system["oracle_variances"][0], + hc0_variances[1], + system["oracle_variances"][1], + ] + ) + ), + native_discrepancy=point_error, + native_variance_discrepancy=variance_error, + ) + + +def run_case(case, replications, seed, index): + """Replay actual native fits; retain every failure in unconditional metrics.""" + started = time.monotonic() + estimates = np.full((replications, 4), np.nan) + ses = np.full_like(estimates, np.nan) + failures = {} + point_error = variance_error = 0.0 + system = None + for j in range(replications): + data, _ = generate_sparse_data( + case["n"], + case["sparse_treated"], + replication_seed(seed, index, j), + case["sparse_controls"], + ) + try: + if system is None: + system = sparse_system(data) + result = evaluate(data, system) + estimates[j], ses[j] = result["estimates"], result["ses"] + point_error = max(point_error, result["native_discrepancy"]) + variance_error = max(variance_error, result["native_variance_discrepancy"]) + except (ValueError, np.linalg.LinAlgError) as exc: + failures[str(exc)] = failures.get(str(exc), 0) + 1 + return dict( + design=case, + elapsed_seconds=time.monotonic() - started, + failures=failures, + native_max_abs_error=point_error, + native_variance_max_abs_error=variance_error, + oracle_variance_ratio_design_to_identity=( + float(system["oracle_variances"][1] / system["oracle_variances"][0]) + if system is not None + else None + ), + variants={ + name: summarize(estimates[:, k], ses[:, k], 1.0) for k, name in enumerate(VARIANTS) + }, + ) + + +def main(): + """Write new evidence without Git or R; refuse existing/frozen output paths.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--replications", type=int, default=2000) + parser.add_argument("--workers", type=int, default=4) + parser.add_argument("--seed", type=int, default=20261106) + parser.add_argument( + "--output", + type=Path, + default=Path("benchmarks/data/chained_did/sparse-covariance-diagnostic.json"), + ) + args = parser.parse_args() + root = Path(__file__).resolve().parents[2] + frozen = root / "benchmarks/data/chained_did" + protected = { + frozen / name + for name in ( + "qualification.json", + "qualification-pre-review.json", + "qualification-followup.json", + "calibration-diagnostics.json", + "reference-validation.json", + "qualification-sparse.json", + ) + } + if args.replications < 2 or args.workers < 1: + parser.error("replications must be >=2 and workers >=1") + if args.output.exists() or args.output.resolve() in protected: + parser.error( + "Choose a new --output path; existing and frozen evidence cannot be overwritten" + ) + sources = [ + "diff_diff/chained_did.py", + "diff_diff/chained_did_results.py", + "benchmarks/python/validate_chained_did.py", + "benchmarks/python/diagnose_chained_did.py", + "benchmarks/python/diagnose_chained_did_sparse.py", + ] + manifest = dict( + status="running", + scope="conditional Gaussian covariance diagnostic", + seed=args.seed, + replications_per_case=args.replications, + seed_derivation="SeedSequence([seed,case_index,replication]), uint64", + python=platform.python_version(), + numpy=np.__version__, + pandas=pd.__version__, + scipy=scipy.__version__, + backend="native Python estimator; single-threaded BLAS requested externally", + source_hashes={p: hashlib.sha256((root / p).read_bytes()).hexdigest() for p in sources}, + cases=cases(), + results={}, + statistically_qualified=False, + thresholds="zero failures; |bias|<.1 empirical SD; variance ratio [.9,1.1]; coverage within 3 nominal binomial MCSE", + interpretation="Oracle covariance is unavailable in empirical applications; no general inference qualification.", + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + + def save(): + temporary = args.output.with_suffix(".tmp") + temporary.write_text(json.dumps(manifest, indent=2) + "\n") + temporary.replace(args.output) + + save() + with ProcessPoolExecutor(max_workers=args.workers) as pool: + futures = { + pool.submit(run_case, case, args.replications, args.seed, i): i + for i, case in enumerate(cases()) + } + for future in as_completed(futures): + index = futures[future] + manifest["results"][str(index)] = future.result() + save() + print(f"Completed case {index + 1}/{len(cases())}", flush=True) + manifest["status"] = "complete" + manifest["variant_thresholds_pass"] = { + name: all( + r["variants"][name]["meets_prespecified_thresholds"] + for r in manifest["results"].values() + ) + for name in VARIANTS + } + save() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/python/validate_chained_did.py b/benchmarks/python/validate_chained_did.py new file mode 100644 index 000000000..6ea24dbb6 --- /dev/null +++ b/benchmarks/python/validate_chained_did.py @@ -0,0 +1,348 @@ +"""Prespecified native chained-DiD qualification study. + +Run from the repository root with PYTHONPATH=. and single-threaded BLAS: +python benchmarks/python/validate_chained_did.py --replications 2000 --workers 4 +This runs fits; it does not install or modify any environment. Negative designs +are sensitivity evidence, not calibration acceptance cases. +""" + +import argparse +import hashlib +import json +import platform +import subprocess +import time +import warnings +from concurrent.futures import ProcessPoolExecutor, as_completed +from pathlib import Path + +import numpy as np +import pandas as pd +import scipy +from scipy.stats import norm + +from diff_diff import ChainedDiD + + +def generate_data( + n: int, + observation: str, + seed: int, + *, + null: bool = False, + clusters: int = 0, + violation: str = "none", +) -> tuple: + """Generate two cohorts with analytic superpopulation ATT targets. + + X is equiprobable binary. Cohort probabilities at each X are proportional + to (1, exp(-.2+.7X), exp(-.4-.4X)). Untreated changes are linear in X, + with an unrestricted pre-reference cohort jump and AR(.5) errors. Cohort + 2 effects are (t-1)*(1+.5X), cohort 3 effects (t-2)*(1.5+.5X). + """ + if observation not in ("balanced", "rotating", "intermittent", "cohort_dependent"): + raise ValueError("Unknown observation design") + if violation not in ("none", "selection", "parallel_trends", "propensity"): + raise ValueError("Unknown assumption violation") + rng = np.random.default_rng(seed) + levels = np.array([0.0, 1.0, 2.0]) if violation == "propensity" else np.array([0.0, 1.0]) + x = rng.choice(levels, n) + + def probabilities(values): + z = values**2 if violation == "propensity" else values + utilities = np.column_stack( + [np.ones(len(values)), np.exp(-0.2 + 0.7 * z), np.exp(-0.4 - 0.4 * z)] + ) + return utilities / utilities.sum(axis=1)[:, None] + + population_prob = probabilities(levels) + prob = probabilities(x) + group_index = (rng.random(n)[:, None] > np.cumsum(prob, axis=1)).sum(axis=1) + cohort = np.array([0, 2, 3])[group_index] + cohort_mass = population_prob.mean(axis=0) + x_mean = (population_prob * levels[:, None]).mean(axis=0) / cohort_mass + truth = { + "g2_t3": 2 + x_mean[1], + "g3_t4": 3 + x_mean[2], + "overall": (cohort_mass[1] * (6 + 3 * x_mean[1]) + cohort_mass[2] * (4.5 + 1.5 * x_mean[2])) + / (3 * cohort_mass[1] + 2 * cohort_mass[2]), + } + if null: + truth = {k: 0.0 for k in truth} + t = np.arange(5) + error = rng.normal(size=(n, 5)) + for j in range(1, 5): + error[:, j] = 0.5 * error[:, j - 1] + np.sqrt(0.75) * error[:, j] + cluster_ids = np.arange(n) % clusters if clusters else np.arange(n) + if clusters: + error += rng.normal(size=(clusters, 5))[cluster_ids] + y = rng.normal(scale=2, size=(n, 1)) + 0.5 * t + 0.4 * x[:, None] * t + error + for g, base_effect in ((2, 1), (3, 1.5)): + member = cohort == g + y[member] += 2 * (t >= g - 1) + if not null: + y[member] += np.maximum(t - g + 1, 0) * (base_effect + 0.5 * x[member, None]) + if violation == "parallel_trends": + y[member] += 0.4 * np.maximum(t - (g - 1), 0) + observed = np.ones((n, 5), dtype=bool) + if observation in ("rotating", "cohort_dependent"): + if observation == "rotating": + window = rng.integers(0, 4, n) + else: + rates = np.array([[0.4, 0.3, 0.2, 0.1], [0.1, 0.2, 0.3, 0.4], [0.25, 0.25, 0.25, 0.25]]) + window = (rng.random(n)[:, None] > np.cumsum(rates[group_index], axis=1)).sum(axis=1) + observed = (t[None, :] == window[:, None]) | (t[None, :] == window[:, None] + 1) + elif observation == "intermittent": + observed = rng.random((n, 5)) < 0.7 + if violation == "selection": + observed &= rng.random((n, 5)) < (0.3 + 0.5 * (x[:, None] > 0)) + data = pd.DataFrame( + { + "id": np.repeat(np.arange(n), 5), + "t": np.tile(t, n), + "g": np.repeat(cohort, 5), + "x": np.repeat(x, 5), + "y": np.where(observed, y, np.nan).ravel(), + "cluster": np.repeat(cluster_ids, 5), + } + ) + return data, truth + + +def generate_sparse_data(n: int, sparse_treated: int, seed: int, sparse_controls: int = 19): + """Return an outcome-independent sparse-date panel with ATT(3,3)=1. + + Half the units are never treated, half first treated at date 3. All have + dates 2 and 3; the first specified units per arm also have date 1. When + sparse_treated is positive, these extra-date units omit date 4. Unit + intercepts and independent unit-date errors are standard normal. + """ + if n < 2 or n % 2 or not 0 <= sparse_treated <= n // 2 or not 0 <= sparse_controls <= n // 2: + raise ValueError("Sparse design counts require even n and arm counts between 0 and n/2") + rng = np.random.default_rng(seed) + rows = [] + for i in range(n): + g = 3 if i % 2 else 0 + intercept = rng.normal() + sparse = i // 2 < (sparse_treated if g else sparse_controls) + for t in (1, 2, 3, 4): + if (t == 1 and not sparse) or (t == 4 and sparse and sparse_treated > 0): + continue + y = intercept + t + rng.normal() + float(g > 0 and t >= g) + rows.append((i, t, g, y)) + return pd.DataFrame(rows, columns=["id", "t", "g", "y"]), {"g3_t3": 1.0} + + +def summarize(estimates: np.ndarray, ses: np.ndarray, truth: float) -> dict: + """Report unconditional coverage, failures and Monte Carlo uncertainty.""" + valid = np.isfinite(estimates) & np.isfinite(ses) & (ses > 0) + r = len(estimates) + e, s = estimates[valid], ses[valid] + lower, upper = estimates - norm.ppf(0.975) * ses, estimates + norm.ppf(0.975) * ses + covered = valid & (lower <= truth) & (upper >= truth) + rejected = valid & ((lower > 0) | (upper < 0)) + sd = float(np.std(e, ddof=1)) if len(e) > 1 else np.nan + bias = float(np.mean(e) - truth) if len(e) else np.nan + ratio = float(np.mean(s * s) / (sd * sd)) if sd > 0 else np.nan + mcse = float(np.sqrt(0.95 * 0.05 / r)) + coverage = float(covered.mean()) + rejection = float(rejected.mean()) + passed = bool( + valid.all() + and abs(bias) < 0.1 * sd + and 0.9 <= ratio <= 1.1 + and abs(coverage - 0.95) <= 3 * mcse + ) + if truth == 0: + passed = passed and abs(rejection - 0.05) <= 3 * mcse + return { + "truth": truth, + "replications": r, + "failures": int((~valid).sum()), + "bias": bias, + "bias_mcse": sd / np.sqrt(len(e)) if len(e) else np.nan, + "rmse": float(np.sqrt(np.mean((e - truth) ** 2))) if len(e) else np.nan, + "empirical_sd": sd, + "mean_variance": float(np.mean(s * s)) if len(s) else np.nan, + "variance_ratio": ratio, + "coverage": coverage, + "coverage_mcse": float(np.sqrt(coverage * (1 - coverage) / r)), + "nominal_binomial_mcse": mcse, + "coverage_given_valid": float(covered[valid].mean()) if valid.any() else np.nan, + "rejection_at_zero": rejection, + "rejection_mcse": float(np.sqrt(rejection * (1 - rejection) / r)), + "mean_interval_width": float(2 * norm.ppf(0.975) * np.mean(s)) if len(s) else np.nan, + "meets_prespecified_thresholds": passed, + } + + +def _run_case(case: dict, replications: int, seed: int, case_index: int) -> dict: + started = time.monotonic() + sparse = "sparse_treated" in case + names = ("g3_t3",) if sparse else ("overall", "g2_t3", "g3_t4") + estimates = np.full((replications, len(names)), np.nan) + ses = np.full_like(estimates, np.nan) + pair_diagnostics = dict( + replications_with_singleton_exclusions=0, replications_with_low_support=0 + ) + failures = {} + truth = None + for j in range(replications): + run_seed = int(np.random.SeedSequence([seed, case_index, j]).generate_state(1)[0]) + if sparse: + data, truth = generate_sparse_data( + case["n"], case["sparse_treated"], run_seed, case["sparse_controls"] + ) + else: + data, truth = generate_data(seed=run_seed, **case) + try: + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", message="ChainedDiD pair support:", category=UserWarning + ) + result = ChainedDiD(cluster="cluster" if case.get("clusters") else None).fit( + data, "y", "id", "t", "g", covariates=None if sparse else ["x"] + ) + if sparse: + cell = result.group_time_effects[3, 3] + estimates[j], ses[j] = [cell.effect], [cell.se] + else: + cells = [result.group_time_effects[2, 3], result.group_time_effects[3, 4]] + estimates[j] = [result.att, cells[0].effect, cells[1].effect] + ses[j] = [result.se, cells[0].se, cells[1].se] + pair_diagnostics["replications_with_singleton_exclusions"] += int( + any( + row["reason"] == "insufficient_pair_replication" + for row in result.diagnostics["excluded_links"] + ) + ) + pair_diagnostics["replications_with_low_support"] += int( + any(row["low_support"] for row in result.diagnostics["links"]) + ) + except (ValueError, np.linalg.LinAlgError) as exc: + key = str(exc) + failures[key] = failures.get(key, 0) + 1 + return { + "design": case, + "elapsed_seconds": time.monotonic() - started, + "failures": failures, + "pair_diagnostics": pair_diagnostics, + "targets": { + name: summarize(estimates[:, k], ses[:, k], truth[name]) for k, name in enumerate(names) + }, + } + + +def main() -> int: + """Run the frozen scenario grid and write an explicitly labelled manifest.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--replications", type=int, default=2000) + parser.add_argument("--workers", type=int, default=4) + parser.add_argument("--seed", type=int, default=20261102) + parser.add_argument("--design", choices=("full", "sparse"), default="full") + parser.add_argument( + "--output", type=Path, default=Path("benchmarks/data/chained_did/qualification.json") + ) + args = parser.parse_args() + if args.replications < 2 or args.workers < 1: + parser.error("replications must be >=2 and workers >=1") + frozen = Path(__file__).resolve().parents[1] / "data/chained_did" + protected = { + frozen / name + for name in ( + "qualification.json", + "qualification-pre-review.json", + "qualification-followup.json", + "calibration-diagnostics.json", + "reference-validation.json", + ) + } + if args.output.resolve() in protected or args.output.exists(): + parser.error( + "Choose a new --output path; existing and frozen evidence cannot be overwritten" + ) + cases = [] + for n in (1000, 4000): + for observation in ("balanced", "rotating", "intermittent", "cohort_dependent"): + cases.append( + dict( + n=n, + observation=observation, + clusters=(50 if n == 1000 else 100) if observation == "intermittent" else 0, + null=False, + violation="none", + ) + ) + cases.append(dict(n=n, observation="rotating", clusters=0, null=True, violation="none")) + for violation in ("selection", "parallel_trends", "propensity"): + cases.append( + dict(n=1000, observation="intermittent", clusters=0, null=False, violation=violation) + ) + if args.design == "sparse": + cases = [ + dict(n=400, sparse_treated=k, sparse_controls=19, violation="none") + for k in (0, 1, 2, 5, 20, 50) + ] + revision = subprocess.run(["git", "rev-parse", "HEAD"], capture_output=True, text=True) + source_commit = revision.stdout.strip() if revision.returncode == 0 else None + source_uncommitted = None + if source_commit is not None: + state = subprocess.run(["git", "status", "--porcelain"], capture_output=True, text=True) + source_uncommitted = bool(state.stdout.strip()) if state.returncode == 0 else None + manifest = { + "status": "running", + "scope": "qualification" if args.replications >= 2000 else "pilot", + "replications_per_case": args.replications, + "seed": args.seed, + "design": args.design, + "seed_derivation": "SeedSequence([seed, case_index, replication]).generate_state(1)[0]", + "python": platform.python_version(), + "numpy": np.__version__, + "pandas": pd.__version__, + "scipy": scipy.__version__, + "backend": "native Python estimator", + "source_commit": source_commit, + "source_uncommitted": source_uncommitted, + "source_hashes": { + name: hashlib.sha256(Path(name).read_bytes()).hexdigest() + for name in ( + "diff_diff/chained_did.py", + "diff_diff/chained_did_results.py", + "benchmarks/python/validate_chained_did.py", + ) + }, + "cases": cases, + "results": {}, + "external_reference": "not run in this invocation; offline regression checks use frozen public R outputs", + "thresholds": "zero failures; |bias|<.1 empirical SD; variance ratio [.9,1.1]; 95% coverage and null 5% rejection within 3 binomial MCSE", + } + args.output.parent.mkdir(parents=True, exist_ok=True) + + def save(): + temporary = args.output.with_suffix(".tmp") + temporary.write_text(json.dumps(manifest, indent=2) + "\n") + temporary.replace(args.output) + + save() + with ProcessPoolExecutor(max_workers=args.workers) as pool: + futures = { + pool.submit(_run_case, case, args.replications, args.seed, i): i + for i, case in enumerate(cases) + } + for future in as_completed(futures): + i = futures[future] + manifest["results"][str(i)] = future.result() + save() + print(f"Completed scenario {i+1}/{len(cases)}: {cases[i]}", flush=True) + core = [r for r in manifest["results"].values() if r["design"]["violation"] == "none"] + manifest["core_thresholds_pass"] = all( + t["meets_prespecified_thresholds"] for r in core for t in r["targets"].values() + ) + manifest["status"] = "complete" + save() + print(f"Complete: {args.output}; core thresholds pass: {manifest['core_thresholds_pass']}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/changelog.d/20261102-chained-did.md b/changelog.d/20261102-chained-did.md new file mode 100644 index 000000000..12829c3a4 --- /dev/null +++ b/changelog.d/20261102-chained-did.md @@ -0,0 +1,23 @@ +### Added + +- Add experimental native Python `ChainedDiD` for overlapping incomplete + panels: never-treated controls, PT-post, normalised IPW and fixed + identity-weighted GMM. Results include explicit support diagnostics, + original-unit population accounting, full joint covariance, analytical + unit/cluster inference and canonical post-fit aggregation. The pinned R + comparison agrees on eight fixtures, with a documented propensity-fitting + difference for incomplete continuous covariates. Fresh simulation coverage + qualification remains incomplete. This does not implement conditional + attrition correction or the complete `cdid` feature set. + +- Require two observed units per arm for chained comparisons, record excluded + candidates and warn on singleton exclusions or low retained support. Identity + weighting still ignores pair precision; these safeguards do not establish + calibrated intervals. Exclude reference identities from mixed event-study + diagnostic rows, expose constituent weights with `aggregation_weights(level)`, + and check frozen public R outputs offline without changing tolerances. + +- Add a reproducible same-panel Gaussian diagnostic comparing native HC0 with + known covariance and benchmark-only design precision weighting. Preserve the + production metric and failed qualification gates; the prototype's bounded + simulation agreement is not general inference qualification. diff --git a/diff_diff/__init__.py b/diff_diff/__init__.py index 566219cd3..594c48d19 100644 --- a/diff_diff/__init__.py +++ b/diff_diff/__init__.py @@ -44,6 +44,8 @@ BusinessContext, BusinessReport, ) +from diff_diff.chained_did import ChainedDiD +from diff_diff.chained_did_results import ChainedDiDResults from diff_diff.chaisemartin_dhaultfoeuille import ( ChaisemartinDHaultfoeuille, TWFEWeightsResult, @@ -331,6 +333,8 @@ __version__ = "4.0.0" __all__ = [ + "ChainedDiD", + "ChainedDiDResults", # Estimators "DifferenceInDifferences", "TwoWayFixedEffects", diff --git a/diff_diff/chained_did.py b/diff_diff/chained_did.py new file mode 100644 index 000000000..0dcbcf6dd --- /dev/null +++ b/diff_diff/chained_did.py @@ -0,0 +1,382 @@ +"""Chained difference-in-differences on overlapping incomplete panels.""" + +import warnings +from itertools import combinations +from typing import List, Optional + +import numpy as np +import pandas as pd +from pandas.api.types import is_bool_dtype, is_complex_dtype, is_numeric_dtype +from scipy.special import expit + +from diff_diff._base import BaseEstimator +from diff_diff.aggregation import AggregationKit +from diff_diff.chained_did_results import ChainedDiDResults +from diff_diff.linalg import solve_logit +from diff_diff.staggered_results import GroupTimeEffect +from diff_diff.utils import ClusterArg, safe_inference, validate_cluster + +_MIN_PAIR_COUNT = 2 +_LOW_PAIR_COUNT = 20 + + +def _pair_ipw(changes, treated, covariates=None): + """Return a normalised pair contrast and O(1/n) unit contributions.""" + nt = int(treated.sum()) + nc = len(treated) - nt + if min(nt, nc) < _MIN_PAIR_COUNT: + raise ValueError("Pair IPW requires at least two observed units in each arm") + info = {"n_treated": nt, "n_control": nc} + with np.errstate(over="ignore", invalid="ignore", divide="ignore"): + mt = changes[treated].mean() + if covariates is None or not covariates.shape[1]: + mc = changes[~treated].mean() + contributions = np.where(treated, (changes - mt) / nt, -(changes - mc) / nc) + info.update( + propensity_min=nt / len(treated), + propensity_max=nt / len(treated), + control_ess=float(nc), + ) + else: + # Reparameterise before forming the score Jacobian to preserve affine scale invariance. + scale = np.maximum(np.abs(covariates).max(axis=0), np.finfo(float).tiny) + x = covariates / scale + x = x - x.mean(axis=0) + spread = np.sqrt(np.mean(x * x, axis=0)) + if (spread == 0).any(): + raise ValueError("IPW covariates are rank deficient (constant column)") + x = x / spread + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", UserWarning) + beta, _ = solve_logit( + x, treated.astype(float), rank_deficient_action="error", epv_threshold=0 + ) + if caught: + raise ValueError(f"IPW propensity fit failed: {caught[0].message}") + z = np.column_stack([np.ones(len(x)), x]) + p = expit(z @ beta) + if not np.isfinite(beta).all() or ((p <= 0) | (p >= 1)).any(): + raise ValueError("IPW propensity fit has non-finite or boundary probabilities") + score_beta = z * (treated - p)[:, None] + if np.max(np.abs(score_beta.sum(axis=0))) > 1e-8 * len(x): + raise ValueError("IPW propensity fit did not solve its score equations") + odds = p / (1 - p) + weights = (~treated) * odds + mass = weights.sum() + mc = weights @ changes / mass + scores = np.column_stack( + [score_beta, treated * (changes - mt), weights * (changes - mc)] + ) + k = z.shape[1] + jacobian = np.zeros((k + 2, k + 2)) + jacobian[:k, :k] = -(z.T * (p * (1 - p))) @ z + jacobian[k, k] = -nt + jacobian[k + 1, :k] = (weights * (changes - mc)) @ z + jacobian[k + 1, k + 1] = -mass + try: + parameter_if = -np.linalg.solve(jacobian, scores.T).T + except np.linalg.LinAlgError as exc: + raise ValueError("IPW score Jacobian is singular") from exc + contributions = parameter_if[:, k] - parameter_if[:, k + 1] + info.update( + propensity_min=float(p.min()), + propensity_max=float(p.max()), + control_ess=float(mass**2 / (weights @ weights)), + ) + effect = mt - mc + if not np.isfinite(effect) or not np.isfinite(contributions).all(): + raise ValueError("Non-finite IPW moments or influence contributions") + return float(effect), contributions, info + + +class ChainedDiD(BaseEstimator): + """Estimate chained ATT(g,t) from overlapping outcome observations. + + Uses normalised IPW comparisons, never-treated controls, PT-post and + fixed identity-weighted GMM. Earlier trend differences remain free. + Each used observation pair must preserve the joint distribution of + covariates and actual outcome changes within its cohort. There is no + correction for covariate-dependent attrition. + + Parameters + ---------- + alpha : float, default=0.05 + Significance level for pointwise intervals and tests. + cluster : str, False or None, default=None + Unit-constant cluster column for one-way CR1 covariance. Without + clustering (None or False), inference uses unit-level HC0 covariance. + Notes + ----- + Treatment is binary and absorbing with zero anticipation. Treatment + dates must be positive stored numeric dates; 0 denotes never treated. + No optimal GMM weighting, survey weights or simultaneous bands are offered. + Pairs need at least two observed units per arm. Exclusions are recorded; + singleton exclusions and retained pairs below 20 units per arm or control + effective sample size 20 trigger a warning. These safeguards do not establish + interval calibration. Identity GMM ignores pair precision, so sparse pairs + can reduce precision and yield anti-conservative intervals. + See Bellégo, Benatia and Dortet-Bernadet, doi:10.1016/j.jeconom.2024.105783. + """ + + def __init__( + self, + alpha: float = 0.05, + cluster: ClusterArg = None, + ) -> None: + self.alpha: float = alpha + self.cluster: ClusterArg = cluster + self.is_fitted_: bool = False + self.results_: Optional[ChainedDiDResults] = None + self._validate_parameters() + + def _validate_parameters(self) -> None: + if isinstance(self.alpha, (bool, np.bool_)) or not isinstance( + self.alpha, (int, float, np.integer, np.floating) + ): + raise ValueError("alpha must be a finite number between 0 and 1") + if not (np.isfinite(float(self.alpha)) and 0 < float(self.alpha) < 1): + raise ValueError("alpha must be a finite number between 0 and 1") + validate_cluster(self.cluster) + + def fit( + self, + data: pd.DataFrame, + outcome: str, + unit: str, + time: str, + first_treat: str, + covariates: Optional[List[str]] = None, + ) -> ChainedDiDResults: + """Fit all supported pairs without balancing or imputing the panel. + + Parameters + ---------- + data : pandas.DataFrame + Long panel. NaN outcomes and absent unit-date rows are unobserved. + Every supplied unit remains in population accounting, including + units with zero or one finite outcome. Infinity is invalid. + outcome, unit, time, first_treat : str + Column names. Cohort membership is supplied, never inferred from + missing outcomes. Every treated date requires a preceding date. + covariates : list of str or None + Complete static numeric covariates, with an automatically added + intercept. Omit for unconditional differences in mean changes. + + Returns + ------- + ChainedDiDResults + Canonical results with per-cell support and full joint covariance. + + Raises + ------ + ValueError + Invalid data, unsupported timing or a failed required nuisance fit. + """ + self.is_fitted_, self.results_ = False, None + self._validate_parameters() + covariates = [] if covariates is None else covariates + if not isinstance(covariates, list) or any(not isinstance(c, str) for c in covariates): + raise ValueError("covariates must be a list of column names") + roles = [outcome, unit, time, first_treat] + covariates + if any(not isinstance(c, str) for c in roles) or len(set(roles)) != len(roles): + raise ValueError("Column roles and covariates must have distinct column names") + if not isinstance(data, pd.DataFrame) or data.empty or not data.columns.is_unique: + raise ValueError("data must be a non-empty DataFrame with unique columns") + cluster = self.cluster if isinstance(self.cluster, str) else None + required = roles + ([cluster] if cluster is not None else []) + missing = [c for c in required if c not in data.columns] + if missing: + raise ValueError(f"Missing columns: {missing}") + static = list( + dict.fromkeys([first_treat] + covariates + ([cluster] if cluster is not None else [])) + ) + if data[[unit, time] + static].isna().any().any(): + raise ValueError("Unit, time, cohort, covariates and cluster metadata must be complete") + if data.duplicated([unit, time]).any(): + raise ValueError("Duplicate unit-date rows are not allowed") + for col in [time, first_treat] + covariates: + if ( + not is_numeric_dtype(data[col]) + or is_complex_dtype(data[col]) + or is_bool_dtype(data[col]) + or not np.isfinite(data[col].to_numpy(dtype=float)).all() + ): + raise ValueError(f"Column {col!r} must contain finite numeric values") + if not is_numeric_dtype(data[outcome]) or is_complex_dtype(data[outcome]): + raise ValueError("Outcomes must be numeric, with NaN allowed for missing values") + outcomes = data[outcome].to_numpy(dtype=float, na_value=np.nan) + if np.isinf(outcomes).any(): + raise ValueError("Infinite outcomes are not allowed") + grouped = data.groupby(unit, sort=False, observed=True) + if (grouped[static].nunique(dropna=False) > 1).any().any(): + raise ValueError("Cohort, covariates and cluster must be constant within each unit") + registry = grouped[static].first() + times = sorted(data[time].unique().tolist()) + cohorts = sorted(registry[first_treat].unique().tolist()) + if 0 not in cohorts or len(cohorts) < 2: + raise ValueError("At least one treated cohort and never-treated controls are required") + cohorts.remove(0) + if any(g <= 0 or g not in times or g == times[0] for g in cohorts): + raise ValueError( + "Each positive treatment date must be stored and have a preceding date" + ) + y = ( + data.pivot(index=unit, columns=time, values=outcome) + .reindex(index=registry.index, columns=times) + .to_numpy(dtype=float, na_value=np.nan) + ) + memberships = registry[first_treat].to_numpy() + x = registry[covariates].to_numpy(dtype=float) if covariates else None + n = len(registry) + cells, influence, catalogue, link_info, ranks = {}, {}, [], [], {} + excluded_links = [] + controls = memberships == 0 + for g in cohorts: + base = times.index(g) - 1 + columns = [j for j in range(len(times)) if j != base] + rows, differences, pair_if, pair_lookup = [], [], [], {} + for s, t in combinations(range(len(times)), 2): + observed = np.isfinite(y[:, s]) & np.isfinite(y[:, t]) + target = memberships == g + nt = int((observed & target).sum()) + nc = int((observed & controls).sum()) + if min(nt, nc) < _MIN_PAIR_COUNT: + reason = ( + "no_treated_observations" + if nt == 0 + else ( + "no_control_observations" + if nc == 0 + else "insufficient_pair_replication" + ) + ) + excluded_links.append( + { + "group": g, + "s": times[s], + "t": times[t], + "n_treated": nt, + "n_control": nc, + "reason": reason, + } + ) + continue + mask = observed & (target | controls) + with np.errstate(over="ignore", invalid="ignore"): + delta = y[mask, t] - y[mask, s] + if not np.isfinite(delta).all(): + raise ValueError( + f"Non-finite outcome changes for cohort {g}, pair {times[s], times[t]}" + ) + estimate, q, info = _pair_ipw( + delta, target[mask], x[mask] if x is not None else None + ) + info["low_support"] = bool(min(nt, nc, info["control_ess"]) < _LOW_PAIR_COUNT) + row = np.zeros(len(columns)) + if s != base: + row[columns.index(s)] = -1 + if t != base: + row[columns.index(t)] = 1 + full_q = np.zeros(n) + full_q[mask] = q + pair_lookup[s, t] = len(rows) + rows.append(row) + differences.append(estimate) + pair_if.append(full_q) + catalogue.append((g, times[s], times[t])) + link_info.append( + {"group": g, "s": times[s], "t": times[t], "att": estimate, **info} + ) + a = np.asarray(rows).reshape(-1, len(columns)) + d = np.asarray(differences) + q = np.column_stack(pair_if) if pair_if else np.empty((n, 0)) + u, singular, vt = np.linalg.svd(a, full_matrices=False) + cutoff = max(a.shape) * np.finfo(float).eps * (singular[0] if len(singular) else 0) + keep = singular > cutoff + basis = vt[keep] + inverse = (basis.T / singular[keep]) @ u[:, keep].T + ranks[g] = int(keep.sum()) + for t, label in enumerate(times): + estimate, contribution, reason = np.nan, np.full(n, np.nan), "unsupported_target" + if t == base: + estimate, contribution, reason = 0.0, np.zeros(n), None + elif t < base: + k = pair_lookup.get((t, base)) + if k is not None: + estimate, contribution, reason = -d[k], -q[:, k], None + else: + reason = "unsupported_direct_pre_comparison" + else: + contrast = np.zeros(len(columns)) + contrast[columns.index(t)] = 1 + residual = contrast - (contrast @ basis.T) @ basis + if np.linalg.norm(residual) <= 4 * max(a.shape) * np.finfo(float).eps: + loading = contrast @ inverse + estimate, contribution, reason = float(loading @ d), q @ loading, None + contribution = contribution - contribution.mean() + cells[g, label] = GroupTimeEffect( + g, + label, + estimate, + np.nan, + np.nan, + np.nan, + (np.nan, np.nan), + int((memberships == g).sum()), + int(controls.sum()), + reason, + ) + influence[g, label] = contribution + singletons = sum(row["reason"] == "insufficient_pair_replication" for row in excluded_links) + low_support = sum(row["low_support"] for row in link_info) + if singletons or low_support: + warnings.warn( + f"ChainedDiD pair support: {singletons} singleton pairs excluded; " + f"{low_support} retained pairs have low support (fewer than 20 units " + "in either arm or control effective sample size below 20). " + "Inspect diagnostics['excluded_links'] and diagnostics['links']. " + "Identity GMM ignores pair precision; pointwise intervals may be " + "anti-conservative. These thresholds do not establish calibration.", + UserWarning, + stacklevel=2, + ) + kit = AggregationKit( + bookkeeping={ + "unit_cohorts": memberships.copy(), + "all_units": registry.index.to_numpy(copy=True), + "clusters": registry[cluster].to_numpy(copy=True) if cluster is not None else None, + "times": times, + "cohorts": cohorts, + "references": {g: times[times.index(g) - 1] for g in cohorts}, + }, + influence=influence, + alpha=self.alpha, + anticipation=0, + cband=False, + ) + result = ChainedDiDResults( + cells, + kit, + { + "catalogue": catalogue, + "links": link_info, + "excluded_links": excluded_links, + "pair_support": { + "minimum_per_arm": _MIN_PAIR_COUNT, + "warning_threshold": _LOW_PAIR_COUNT, + }, + "rank": ranks, + "gmm_weighting": "identity", + "selection": "none", + }, + ) + covariance = result._covariance(result.influence_functions) + result.vcov = covariance + for k, cell in enumerate(cells.values()): + cell.se = float(np.sqrt(covariance[k, k])) + cell.t_stat, cell.p_value, cell.conf_int = safe_inference( + cell.effect, cell.se, self.alpha + ) + result.att, result.se, result.t_stat, result.p_value, result.conf_int = result._overall() + self.results_, self.is_fitted_ = result, True + return result diff --git a/diff_diff/chained_did_results.py b/diff_diff/chained_did_results.py new file mode 100644 index 000000000..ad18392b5 --- /dev/null +++ b/diff_diff/chained_did_results.py @@ -0,0 +1,262 @@ +"""Canonical results and retained-IF aggregation for chained DiD.""" + +from dataclasses import dataclass, field +from typing import Any, ClassVar, Dict, Optional, Tuple + +import numpy as np +import pandas as pd + +from diff_diff.aggregation import AggregationKit, AggregationMixin, AggregationResult +from diff_diff.results_base import ( + _SUMMARY_ALPHA_MESSAGE, + BaseResults, + EventStudyResults, + _require_fit_alpha, +) +from diff_diff.staggered_results import GroupTimeEffect +from diff_diff.utils import safe_inference + + +@dataclass +class ChainedDiDResults(BaseResults, AggregationMixin): + """Chained ATT(g,t), direct earlier diagnostics and analytical inference. + + ``att`` is the cohort-mass-weighted average of all scheduled post-treatment + cells. An unsupported required cell makes its summary unavailable, without + changing its population. ``vcov`` is on the estimator scale, indexed by + ``vcov_index``. ``influence_functions`` contains O(1/n) unit contributions, + including the zero rows of units without an observed change. A non-zero + pre-reference diagnostic is not a violation of maintained PT-post. + The retained aggregation kit contains unit metadata and contributions, + not the outcome panel or an estimator reference. + """ + + group_time_effects: Dict[Tuple[Any, Any], GroupTimeEffect] + _aggregation_kit: AggregationKit = field(repr=False) + diagnostics: Dict[str, Any] + att: float = np.nan + se: float = np.nan + t_stat: float = np.nan + p_value: float = np.nan + conf_int: Tuple[float, float] = (np.nan, np.nan) + vcov: Optional[np.ndarray] = field(default=None, repr=False) + + _AGGREGATE_SUPPORTED: ClassVar[Tuple[str, ...]] = ("simple", "event_study") + _AGGREGATE_BALANCE_E_TYPES: ClassVar[Tuple[str, ...]] = () + + @property + def alpha(self) -> float: + """Fit-time significance level.""" + return self._aggregation_kit.alpha + + @property + def n_units(self) -> int: + """Number of original supplied units, including non-repeat observers.""" + return len(self._aggregation_kit.bookkeeping["unit_cohorts"]) + + @property + def vcov_index(self) -> list: + """Ordered (cohort, date) keys indexing both covariance axes.""" + return list(self.group_time_effects) + + @property + def influence_functions(self) -> np.ndarray: + """O(1/n) unit contributions in vcov_index order.""" + return np.column_stack([self._aggregation_kit.influence[k] for k in self.vcov_index]) + + def _covariance(self, contributions): + q = np.asarray(contributions) + if q.ndim == 1: + q = q[:, None] + clusters = self._aggregation_kit.bookkeeping["clusters"] + if clusters is None: + return q.T @ q + codes, labels = pd.factorize(clusters, sort=False) + if len(labels) < 2: + return np.full((q.shape[1], q.shape[1]), np.nan) + sums = np.zeros((len(labels), q.shape[1])) + np.add.at(sums, codes, q) + return len(labels) / (len(labels) - 1) * (sums.T @ sums) + + def _combine(self, keys): + n = self.n_units + membership = self._aggregation_kit.bookkeeping["unit_cohorts"] + if not keys or any(self.group_time_effects[k].skip_reason is not None for k in keys): + return np.nan, np.full(n, np.nan), np.nan + indicators = np.column_stack([membership == g for g, _ in keys]).astype(float) + masses = indicators.sum(axis=0) + denominator = masses.sum() + weights = self._cell_weights(keys) + effects = np.array([self.group_time_effects[k].effect for k in keys]) + q = np.column_stack([self._aggregation_kit.influence[k] for k in keys]) + centred = indicators - masses / n + weight_if = (centred - centred.sum(axis=1)[:, None] * weights) / denominator + contribution = q @ weights + weight_if @ effects + return float(weights @ effects), contribution, float(denominator) + + def _post_keys(self): + return [(g, t) for g, t in self.group_time_effects if t >= g] + + def _event_keys(self, horizon): + keys = [(g, t) for g, t in self.group_time_effects if t - g == horizon] + references = self._aggregation_kit.bookkeeping["references"] + estimated = [(g, t) for g, t in keys if t != references[g]] + return estimated or keys + + def _cell_weights(self, keys): + membership = self._aggregation_kit.bookkeeping["unit_cohorts"] + masses = np.array([np.count_nonzero(membership == g) for g, _ in keys], dtype=float) + return masses / masses.sum() + + def aggregation_weights(self, level: str = "simple") -> pd.DataFrame: + """Return constituent cell weights, including intended unavailable cells. + + ``level`` is ``'simple'`` or ``'event_study'``. Columns are ``label``, + ``group``, ``time``, ``weight`` and cell ``available``. Unsupported + cells keep their intended weights; the corresponding summary remains + unavailable. Mixed event rows exclude structural reference identities. + The returned table is detached from the fitted results. + """ + if level == "simple": + summaries = {"overall": self._post_keys()} + elif level == "event_study": + summaries = { + e: self._event_keys(e) for e in sorted({t - g for g, t in self.group_time_effects}) + } + else: + raise ValueError("level must be 'simple' or 'event_study'") + return pd.DataFrame( + [ + { + "label": label, + "group": g, + "time": t, + "weight": weight, + "available": self.group_time_effects[g, t].skip_reason is None, + } + for label, keys in summaries.items() + for (g, t), weight in zip(keys, self._cell_weights(keys)) + ], + columns=["label", "group", "time", "weight", "available"], + ) + + def _overall(self): + effect, contribution, _ = self._combine(self._post_keys()) + se = float(np.sqrt(self._covariance(contribution)[0, 0])) + return (effect, se, *safe_inference(effect, se, self.alpha)) + + def _aggregate_compute(self, level, *, weights, balance_e): + if level == "simple": + return AggregationResult( + level="simple", + label=np.array(["overall"]), + target=np.array(["att"]), + att=np.array([self.att]), + se=np.array([self.se]), + t_stat=np.array([self.t_stat]), + p_value=np.array([self.p_value]), + conf_int_lower=np.array([self.conf_int[0]]), + conf_int_upper=np.array([self.conf_int[1]]), + n=np.array([self.n_units]), + weight=np.array([1.0]), + df=np.array([np.nan]), + alpha=self.alpha, + n_kind="units", + estimator="ChainedDiD", + ) + horizons = sorted({t - g for g, t in self.group_time_effects}) + effects, ses, statistics, pvalues, lower, upper, refs, counts, contributions = ( + [] for _ in range(9) + ) + references = self._aggregation_kit.bookkeeping["references"] + for e in horizons: + keys = self._event_keys(e) + estimate, q, _ = self._combine(keys) + reference = all(t == references[g] for g, t in keys) + se = float(np.sqrt(self._covariance(q)[0, 0])) + statistic, pvalue, interval = safe_inference(estimate, se, self.alpha) + effects.append(estimate) + ses.append(se) + statistics.append(statistic) + pvalues.append(pvalue) + lower.append(interval[0]) + upper.append(interval[1]) + refs.append(reference) + counts.append(sum(self.group_time_effects[k].n_treated for k in keys)) + contributions.append(q) + return EventStudyResults( + event_time=np.array(horizons), + att=np.array(effects), + se=np.array(ses), + t_stat=np.array(statistics), + p_value=np.array(pvalues), + conf_int_lower=np.array(lower), + conf_int_upper=np.array(upper), + is_reference=np.array(refs), + n=np.array(counts), + n_kind="units", + alpha=self.alpha, + vcov=self._covariance(np.column_stack(contributions)), + vcov_index=np.array(horizons), + source="ChainedDiD", + base_period="universal", + anticipation=0, + reference_event_times=tuple(sorted({references[g] - g for g in references})), + estimand="ATT(g,t) after treatment; direct unrestricted differences before reference", + post_periods=tuple(e for e in horizons if e >= 0), + ) + + def summary(self, alpha: Optional[float] = None) -> str: + """Describe the fixed fit-time target and pointwise inference.""" + _require_fit_alpha(alpha, self.alpha, message=_SUMMARY_ALPHA_MESSAGE) + return ( + f"Chained DiD (identity GMM, PT-post, never-treated controls)\n" + f"ATT: {self.att:.6g}, SE: {self.se:.6g}, units: {self.n_units}\n" + f"Pointwise normal intervals, alpha={self.alpha}; no attrition correction." + ) + + def print_summary(self) -> None: + """Print the fit-time summary.""" + print(self.summary()) + + def to_dataframe(self, level: str = "group_time") -> pd.DataFrame: + """Return per-cell records or the canonical aggregate table.""" + if level in self._AGGREGATE_SUPPORTED: + return self.aggregate(level).to_dataframe() + if level != "group_time": + raise ValueError("level must be 'group_time', 'simple' or 'event_study'") + return pd.DataFrame( + [ + { + "group": g, + "time": t, + "att": cell.effect, + "se": cell.se, + "t_stat": cell.t_stat, + "p_value": cell.p_value, + "conf_int_lower": cell.conf_int[0], + "conf_int_upper": cell.conf_int[1], + "n_treated": cell.n_treated, + "n_control": cell.n_control, + "available": cell.skip_reason is None, + "status": cell.skip_reason or "ok", + } + for (g, t), cell in self.group_time_effects.items() + ] + ) + + def to_dict(self) -> Dict[str, Any]: + """Serialise canonical inference and explicit cell support.""" + return { + "att": self.att, + "se": self.se, + "t_stat": self.t_stat, + "p_value": self.p_value, + "conf_int": self.conf_int, + "alpha": self.alpha, + "n_units": self.n_units, + "group_time_effects": self.to_dataframe().to_dict(orient="records"), + "vcov_index": self.vcov_index, + "vcov": self.vcov.tolist() if self.vcov is not None else None, + "diagnostics": self.diagnostics, + } diff --git a/diff_diff/guides/llms.txt b/diff_diff/guides/llms.txt index dc0487606..dece0b9de 100644 --- a/diff_diff/guides/llms.txt +++ b/diff_diff/guides/llms.txt @@ -4,6 +4,7 @@ diff-diff offers a catalog of estimators (the `## Estimators` list below is the authoritative roster) covering basic 2x2 DiD, causal duration analysis for absorbing outcomes, modern staggered adoption methods, reversible (non-absorbing) treatments, advanced panel estimators, nonlinear models, sharp regression discontinuity designs, and diagnostic tools. It supports robust and cluster-robust standard errors, wild cluster bootstrap, formula and column-name interfaces, fixed effects (dummy and absorbed), complex survey designs (strata/PSU/FPC, replicate weights, design-based variance), and publication-ready output. The optional Rust backend accelerates compute-intensive estimators like Synthetic DiD and TROP. + - Install: `pip install diff-diff` - License: MIT - Dependencies: numpy, pandas, scipy (no statsmodels dependency) @@ -56,6 +57,7 @@ The site is organized into 5 sections, each with a landing page: - [API Reference](https://diff-diff.readthedocs.io/en/stable/api/index.html): Full API documentation for all estimators, results classes, diagnostics, and utilities ## Estimators +- [ChainedDiD](https://diff-diff.readthedocs.io/en/stable/api/chained_did.html): Experimental native chained DiD for incomplete panels. `ChainedDiD(alpha=0.05, cluster=None).fit(data, outcome, unit, time, first_treat, covariates=None)`. Binary absorbing treatment, never-treated controls, zero anticipation, static covariates, IPW, PT-post and full-catalogue identity GMM. No observation correction: each used pair must preserve covariates and actual changes within cohort. Retains zero-/one-outcome units in population weights; unsupported required cells invalidate their summary. Unit HC0 or cluster CR1 analytical covariance and normal pointwise intervals. Canonical att/se and `aggregate("simple"|"event_study")`. No simultaneous bands, survey, PT-all, OR/DR or arbitrary-attrition support. Direct earlier diagnostics are not PT-post restrictions. Pinned R agreement is bounded: the incomplete continuous-covariate case uses a different propensity-fitting criterion. Small-sample qualification remains open. Pairs require two observed units per arm; exclusions are recorded. Singleton exclusions and retained pairs below 20 units per arm or control ESS 20 warn. These are safeguards, not calibration guarantees: identity GMM ignores pair precision and sparse-pair intervals can be anti-conservative. Mixed diagnostic event rows exclude structural reference identities; aggregation_weights(level) reports constituent weights and availability. - [DifferenceInDifferences](https://diff-diff.readthedocs.io/en/stable/api/estimators.html): Basic 2x2 DiD with robust/cluster-robust SEs, wild bootstrap, formula interface, and fixed effects - [TwoWayFixedEffects](https://diff-diff.readthedocs.io/en/stable/api/estimators.html): Panel data DiD with unit and time fixed effects via within-transformation or dummies; `event_study=True` estimates per-period effects (spec='within'|'pooled') returning the unified EventStudyResults surface diff --git a/diff_diff/visualization/_event_study.py b/diff_diff/visualization/_event_study.py index 7bec7f93b..da5d52c06 100644 --- a/diff_diff/visualization/_event_study.py +++ b/diff_diff/visualization/_event_study.py @@ -86,6 +86,9 @@ def plot_event_study( are normalized (ref effect subtracted) and ref SE is set to NaN. When None and auto-inferred from results, only hollow marker styling is applied (no normalization). If None, tries to infer from results. + ChainedDiD containers preserve their fitted structural references: + selecting an existing reference retains stored intervals; selecting + another period raises rather than reconstructing inference. pre_periods : list, optional List of pre-treatment periods. Used for shading. post_periods : list, optional @@ -188,6 +191,8 @@ def plot_event_study( """ from scipy import stats as scipy_stats + from diff_diff.results_base import EventStudyResults + # Track if reference_period was explicitly provided by user reference_period_explicit = reference_period is not None @@ -213,6 +218,18 @@ def plot_event_study( # If reference was inferred from results, it was NOT explicitly provided if reference_inferred: reference_period_explicit = False + if ( + isinstance(results, EventStudyResults) + and results.source == "ChainedDiD" + and reference_period_explicit + ): + # Chained targets have fitted cohort references; plotting cannot refit their inference. + if reference_period not in results.reference_periods: + raise ValueError( + "ChainedDiD event studies retain their fitted reference periods; " + "plotting cannot renormalise their inference." + ) + reference_period_explicit = False # Channel selection: the band channel when the user wants # simultaneous bands; otherwise the POINTWISE channel - the # producer's stored intervals (NaN preserved), for input routes diff --git a/docs/api/_autosummary/diff_diff.ChainedDiD.rst b/docs/api/_autosummary/diff_diff.ChainedDiD.rst new file mode 100644 index 000000000..b2ba6dbf6 --- /dev/null +++ b/docs/api/_autosummary/diff_diff.ChainedDiD.rst @@ -0,0 +1,17 @@ +diff\_diff.ChainedDiD +===================== + +.. currentmodule:: diff_diff + +.. autoclass:: ChainedDiD + :no-members: + + + .. rubric:: Methods + + .. autosummary:: + + ~ChainedDiD.__init__ + ~ChainedDiD.fit + ~ChainedDiD.get_params + ~ChainedDiD.set_params diff --git a/docs/api/_autosummary/diff_diff.ChainedDiDResults.rst b/docs/api/_autosummary/diff_diff.ChainedDiDResults.rst new file mode 100644 index 000000000..8da4f11d9 --- /dev/null +++ b/docs/api/_autosummary/diff_diff.ChainedDiDResults.rst @@ -0,0 +1,40 @@ +diff\_diff.ChainedDiDResults +============================ + +.. currentmodule:: diff_diff + +.. autoclass:: ChainedDiDResults + :no-members: + + + .. rubric:: Methods + + .. autosummary:: + + ~ChainedDiDResults.__init__ + ~ChainedDiDResults.aggregate + ~ChainedDiDResults.aggregation_weights + ~ChainedDiDResults.print_summary + ~ChainedDiDResults.summary + ~ChainedDiDResults.to_dataframe + ~ChainedDiDResults.to_dict + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~ChainedDiDResults.alpha + ~ChainedDiDResults.att + ~ChainedDiDResults.conf_int + ~ChainedDiDResults.influence_functions + ~ChainedDiDResults.n_units + ~ChainedDiDResults.p_value + ~ChainedDiDResults.se + ~ChainedDiDResults.t_stat + ~ChainedDiDResults.vcov + ~ChainedDiDResults.vcov_index + ~ChainedDiDResults.group_time_effects + ~ChainedDiDResults.diagnostics diff --git a/docs/api/chained_did.rst b/docs/api/chained_did.rst new file mode 100644 index 000000000..f1e4c7e71 --- /dev/null +++ b/docs/api/chained_did.rst @@ -0,0 +1,157 @@ +Chained Difference-in-Differences +================================= + +``ChainedDiD`` reconstructs group-time treatment effects from overlapping +within-unit outcome changes. It implements a restricted native Python version +of Bellégo, Benatia and Dortet-Bernadet's chained difference-in-differences. + +.. warning:: + + This is an experimental implementation. The external pinned ``cdid`` + comparison agrees on eight fixtures; an incomplete continuous-covariate + fixture uses a different propensity-fitting criterion. Default normal + inference still misses four small-sample coverage limits, and sparse-pair + designs can have substantially anti-conservative intervals. See the records + below. Algebraic + tests and the recorded simulation scenarios are distinct evidence; they + do not establish validity for an arbitrary incomplete panel. + +Statistical contract +-------------------- + +Treatment is binary, absorbing and staggered, with zero anticipation and +never-treated controls. ``first_treat`` contains the positive stored treatment +date, or 0 for never-treated units. Every treatment date needs a preceding +stored date. Covariates, when supplied, must be complete and constant within +units. Treatment dates and covariates are not inferred from missing outcomes. + +Every supported ordered pair of stored dates contributes a normalised IPW +comparison. The fixed identity-GMM criterion retains the full comparison +catalogue of replicated pairs, including non-adjacent dates. Identity weighting +ignores pair precision, so adding a sparse comparison can reduce precision. +It does not invert or regularise the +moment covariance and makes no optimal-efficiency claim. PT-post applies from +each cohort's reference date onwards; earlier trend differences remain free. +Reported earlier diagnostics are direct reference comparisons, with their own +support, rather than fitted GMM nuisance parameters. A non-zero earlier +difference does not refute maintained PT-post. + +A sufficient baseline observation assumption is that observing both endpoints +is independent of the joint distribution of covariates and actual outcome +changes within each cohort. This includes treated changes. Without covariates, +the relevant mean restriction suffices. The estimator fits no observation model, +so treatment IPW does not correct arbitrary selective attrition. Conditional +parallel trends, no anticipation, treatment overlap and correct pair-specific +propensity models still need scientific justification. + +Population and missingness +-------------------------- + +NaN outcomes and absent rows are unobserved endpoints; infinite outcomes and +duplicate unit-date rows are errors. The stored calendar includes dates with +only missing outcomes. Every distinct supplied unit remains in the registry, +including units with zero or one finite outcome. A unit absent from the input +is outside the declared study population. A dated row with a missing outcome +and complete static metadata can retain an otherwise unobserved unit. + +``group_time_effects`` and ``to_dataframe()`` expose unavailable targets and +their support reasons. A required unsupported cell makes its overall/event-time +summary unavailable; the estimator does not silently change its population by +dropping that cell. Per-cell treated/control counts describe registry cohort +sizes; observed-pair counts and propensity diagnostics are in ``diagnostics``. + +Pair eligibility and sparse support +----------------------------------- + +Each pair needs at least two observed units in each arm. A singleton has no +within-arm empirical residual variance, so singleton pairs are excluded before +GMM. ``diagnostics['excluded_links']`` records every excluded candidate's counts +and reason; ``diagnostics['catalogue']`` and ``diagnostics['links']`` describe +retained pairs. Exclusions can change estimates or disconnect targets, while +all units remain in registry masses and cluster counts. + +A fit warns once when singleton pairs are excluded or retained pairs have fewer +than 20 units in either arm or control effective sample size below 20. Retained +links expose ``low_support``. Two is a replication minimum and 20 is an +operational warning threshold; neither establishes calibrated inference. Two- +and five-unit arms remain particularly vulnerable. The estimator remains +experimental and statistically unqualified. + +Results and inference +--------------------- + +``att`` averages every scheduled post-treatment cell with weight proportional +to its cohort registry size. ``aggregate('simple')`` relays that headline; +``aggregate('event_study')`` weights eligible cohorts by registry cohort size +at each actual numeric event time. Neither accepts a weighting selector or +``balance_e``. The returned :class:`~diff_diff.EventStudyResults` works with +:func:`~diff_diff.plot_event_study`. + +``aggregation_weights('simple')`` and ``aggregation_weights('event_study')`` +return constituent cohort/date weights and cell availability, including intended +weights for unavailable summaries. The simple container's ``weight=[1.0]`` +describes its one summary row. On irregular calendars, mixed diagnostic rows +exclude structural reference identities before computing weights, counts and +covariance. Pure reference rows remain zero; ``reference_event_times`` retains +all cohort bases, whereas ``reference_periods`` identifies reference-only rows. + +``vcov`` is the full estimator-scale covariance, with both axes indexed by +``vcov_index``. ``influence_functions`` stores O(1/n) original-unit +contributions. Joint uncertainty includes fitted propensity scores, estimated +cohort-share weights, shared units and shared controls. The retained +``AggregationKit`` needs no outcome panel or estimator reference. + +Without ``cluster``, inference uses unit HC0 covariance. With a unit-constant +cluster column, it sums contributions by cluster and applies K/(K-1), using +all study-registry clusters. Intervals are normal, pointwise and computed at +the fit's ``alpha``. Fewer than two clusters or invalid variance yields +unavailable inference while preserving identified estimates. CR1 and normal +intervals do not establish few-cluster validity. Simultaneous bands, +prediction, survey designs, sampling weights, PT-all, +not-yet-treated controls, non-zero anticipation, outcome regression, doubly +robust estimation and conditional missingness corrections are unsupported. + +Plotting preserves stored intervals when selecting an existing structural +reference explicitly. Selecting a different reference is unsupported because +chained event studies retain their fitted cohort reference dates. + +Implementation and validation +----------------------------- + +The methodology is specified in ``docs/methodology/chained-did-pseudocode.md``. +The V01-V16 matrix and its scope are in ``chained-did-proposal.md``. The native +qualification runner is ``benchmarks/python/validate_chained_did.py``; its +manifest records source hashes, settings, analytic targets, failures and Monte +Carlo uncertainty. ``benchmarks/R/benchmark_chained_did.R`` refuses an unmatched +R/package environment and performs no installation. Reference agreement is +not inferred from a missing or refused comparison. + +.. autoclass:: diff_diff.ChainedDiD + :no-index: + :members: + :inherited-members: + +.. autoclass:: diff_diff.ChainedDiDResults + :no-index: + :members: + :inherited-members: + +Reference +--------- + +Bellégo, C., Benatia, D., and Dortet-Bernadet, V. (2025). +"The chained difference-in-differences." *Journal of Econometrics*. +https://doi.org/10.1016/j.jeconom.2024.105783 + +Methodology and validation records +---------------------------------- + +.. toctree:: + :maxdepth: 1 + + ../methodology/chained-did-proposal + ../methodology/chained-did-pseudocode + ../methodology/chained-did-validation + ../methodology/chained-did-validation-followup + ../methodology/chained-did-review-remediation + ../methodology/chained-did-oracle-diagnostic diff --git a/docs/api/index.rst b/docs/api/index.rst index deed96831..704120d13 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -20,6 +20,7 @@ DiD and the power tools keep their own sections: diff_diff.TwoWayFixedEffects diff_diff.SyntheticDiD diff_diff.CallawaySantAnna + diff_diff.ChainedDiD diff_diff.ChaisemartinDHaultfoeuille diff_diff.SunAbraham diff_diff.ImputationDiD @@ -52,6 +53,7 @@ Result containers returned by estimators: diff_diff.DiDResults diff_diff.SyntheticDiDResults diff_diff.CallawaySantAnnaResults + diff_diff.ChainedDiDResults diff_diff.CSBootstrapResults diff_diff.GroupTimeEffect diff_diff.ChaisemartinDHaultfoeuilleResults @@ -367,6 +369,7 @@ Estimators estimators staggered + chained_did chaisemartin_dhaultfoeuille imputation stacked_did diff --git a/docs/doc-deps.yaml b/docs/doc-deps.yaml index 05b506a94..488f6a3ab 100644 --- a/docs/doc-deps.yaml +++ b/docs/doc-deps.yaml @@ -21,6 +21,9 @@ # Members resolve to the first entry (the primary module) for doc lookup. # ────────────���───────────────────────────────────────────────────────── groups: + chained_did: + - diff_diff/chained_did.py + - diff_diff/chained_did_results.py twfe_weights: - diff_diff/twfe_weights.py - diff_diff/twfe_weights_results.py @@ -110,9 +113,37 @@ groups: # Source-to-docs mappings # ────────���─────────────────────────��─────────────────────────────────── sources: + diff_diff/chained_did.py: + drift_risk: high + docs: + - path: docs/api/chained_did.rst + type: api_reference + - path: docs/methodology/REGISTRY.md + section: "ChainedDiD" + type: methodology + - path: docs/methodology/chained-did-pseudocode.md + type: methodology + - path: docs/methodology/chained-did-validation.md + type: methodology + - path: docs/methodology/chained-did-validation-followup.md + type: methodology + - path: docs/methodology/chained-did-review-remediation.md + type: methodology + - path: docs/methodology/chained-did-oracle-diagnostic.md + type: methodology + - path: docs/tutorials/34_chained_did.ipynb + type: tutorial + - path: diff_diff/guides/llms.txt + type: user_guide # ── Base estimators ──────��─────────────────────────────────────────── + diff_diff/chained_did_results.py: + drift_risk: high + docs: + - path: docs/api/chained_did.rst + type: api_reference + diff_diff/estimators.py: drift_risk: medium docs: diff --git a/docs/index.rst b/docs/index.rst index 499771d7b..f421e785b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -185,6 +185,8 @@ Supported Estimators - Chang (2020) double/debiased ML DiD; staggered ATT(g,t) with cross-fitted nuisance learners (panel or declared repeated cross sections; survey/cluster support); Caetano et al. (2026) bad-control score via ``fit(bad_control=)`` (bad-control lane: panel only, ``cluster=`` only) * - :class:`~diff_diff.DurationDiD` - Deaner & Ku (2026) causal duration DiD for binary absorbing outcomes; untreated-hazard restriction (common dynamics or proportional hazards), whole-individual bootstrap bands + * - :class:`~diff_diff.ChainedDiD` + - Bellégo, Benatia & Dortet-Bernadet (2025) chained IPW for overlapping incomplete panels; experimental, identity GMM and PT-post * - :class:`~diff_diff.RegressionDiscontinuity` - Calonico, Cattaneo & Titiunik (2014) sharp/fuzzy RD with robust bias-corrected inference * - :class:`~diff_diff.TROP` diff --git a/docs/methodology/REGISTRY.md b/docs/methodology/REGISTRY.md index 7e4e10c51..8b7769f84 100644 --- a/docs/methodology/REGISTRY.md +++ b/docs/methodology/REGISTRY.md @@ -6939,6 +6939,12 @@ where δ is the target effect. **Reference Period Normalization** +ChainedDiD event-study containers retain their fitted cohort references. +Explicitly selecting an existing structural reference preserves stored intervals, +including unavailable intervals. Selecting a different reference raises +an error because the plotter cannot refit that inference. The general explicit +normalisation behaviour below continues to apply to other producers. + Normalization only occurs when `reference_period` is **explicitly specified** by the user: - **Explicit `reference_period=X`**: Normalizes effects (subtracts ref effect), sets ref SE to NaN @@ -8229,3 +8235,18 @@ estimator-focused: - **v1.2** (2026-03-24): Added Survey-Aware Bootstrap section (Phase 6) - **v1.1** (2026-03-20): Added Survey Data Support section - **v1.0** (2025-01-19): Initial registry with 12 estimators + + +## ChainedDiD + +- **Reference:** Bellégo, Benatia and Dortet-Bernadet, doi:10.1016/j.jeconom.2024.105783. +- **Implementation:** `chained_did.py` constructs all supported pairwise IPW moments and recovers supported post-treatment contrasts with identity GMM. `chained_did_results.py` retains original-unit contributions in the shared AggregationKit. +- **Note:** Experimental restricted implementation: binary absorbing treatment, never-treated controls, zero anticipation, static complete covariates and PT-post. Earlier profile parameters remain unrestricted; reported pre-period cells are separate direct comparisons. +- **Note:** Baseline sampling preserves the joint covariate/actual-change law within each cohort for every used pair; no conditional observation correction is fitted. Treatment IPW does not repair arbitrary attrition. +- **Note:** Pairs require at least two observed units per arm; excluded candidates retain counts and reasons. Singleton exclusions and retained pairs below 20 units per arm or control ESS 20 trigger a warning. These thresholds do not qualify inference. Identity GMM ignores pair precision; sparse pairs can worsen precision and produce anti-conservative intervals. +- **Note:** Mixed event-study diagnostic rows exclude structural reference identities; `aggregation_weights(level)` exposes constituent cohort/date weights and availability. +- **Note:** The full catalogue identity metric is retained. Support is a contrast row-space condition; minimum-norm coordinates are never reported as supported effects. No covariance inverse, regularisation or optimal-efficiency claim. +- **Note:** Link contributions include the pair propensity-score Jacobian and both mean normalisations. Stored contributions are O(1/n); covariance is estimator-scale. Post-fit cohort-share uncertainty and shared controls remain in joint inference. +- **Note:** Every supplied unit remains in registry masses and cluster counts. Missing-outcome rows retain dates/units. Summaries fail closed when a scheduled cell is unsupported, rather than renormalising to a different target. +- **Deviation from R:** Identity GMM is fixed rather than cdid's default optimal weighting. Cluster covariance applies the declared K/(K-1) CR1 factor, which the reference harness reconciles explicitly. Simultaneous bands, PT-all, NYT, OR/DR, observation corrections and survey designs are not implemented. +- **Validation:** Independent hand/stratum and multiplier-covariance tests; planned V01-V16 gates in `chained-did-proposal.md`. The Python qualification manifest records executed scenarios separately. The pinned R environment is now executed: eight fixtures agree, while incomplete continuous covariates use a different propensity-score criterion. Fresh qualification still misses four n=1,000 coverage limits; see `chained-did-validation-followup.md`. The same-panel Gaussian oracle diagnostic in `chained-did-oracle-diagnostic.md` separates covariance-estimation failure from precision loss; its successful design-weight prototype remains benchmark-only and does not qualify production inference. diff --git a/docs/methodology/chained-did-oracle-diagnostic.md b/docs/methodology/chained-did-oracle-diagnostic.md new file mode 100644 index 000000000..c83615b1e --- /dev/null +++ b/docs/methodology/chained-did-oracle-diagnostic.md @@ -0,0 +1,160 @@ +# ChainedDiD sparse-pair covariance diagnostic + +This benchmark separates covariance estimation from the precision of the point +estimator. It does not change the production estimator, its identity GMM metric, +its pair eligibility or its propensity-score criterion. Statistical qualification +is an independent gate. The implementation decision below follows the prespecified experiment. +This is a documented implementation choice within the restricted contract. + +## Model and exact covariance + +Use the existing sparse Gaussian panel: Y_it = a_i + t + D_it + epsilon_it, +where a_i and epsilon_it are independent standard normal, and date innovations +are independent across units and dates. Treatment begins at date 3 for half the +units and ATT(3,3)=1. Observation patterns are fixed independently of outcomes. +For each retained pair p=(s,t), define c_ip as 1/n_Tp for observed treated units, +-1/n_Cp for observed controls, and zero otherwise. Define the error loading +B_(i,u),p = c_ip * (1[u=t] - 1[u=s]). Then + +```text +d_p = sum_i c_ip (Y_it - Y_is) +Omega = B' B +``` + +The unit intercept cancels because each pair's date coefficients sum to zero. +Omega is the exact conditional covariance under this DGP, on the estimator +scale; it includes shared units, shared controls and shared endpoints. It is +not the empirical residual covariance and is not assumed diagonal. + +Anchor the cohort profile at date 2 and form the incidence matrix A. For a fixed +positive diagonal metric H, use the row-space/SVD recovery map + +```text +R_H = pinv(sqrt(H) A) sqrt(H) +l_H = contrast(date 3)' R_H +estimate_H = l_H d +variance_oracle_H = l_H Omega l_H' +variance_HC0_H = (Q l_H')' (Q l_H') +``` + +Q is the original-unit matrix of pair mean residual contributions, with each +arm's own mean and denominator. The native-identity estimate and HC0 variance +must agree with a real ChainedDiD fit, checked during every replication. A +missing earlier coordinate must not alter the supported date 3 contrast. + +Compare H=I with H_pp=n_Tp*n_Cp/(n_Tp+n_Cp). The latter uses the fixed design +counts, not observed outcomes or a fitted covariance. In this homoskedastic +Gaussian design it is proportional to inverse diagonal pair variance. Counts +and observation patterns are deterministic within a case, so no random-weight +term is omitted in this benchmark. This argument does not cover empirical +random missingness, treatment IPW, clusters or heterogeneous outcome variances. +The prototype is not a new public estimator or an optimal-GMM implementation. + +## Prespecified experiment + +Use seed 20261106, SeedSequence([seed,case_index,replication]) and 2,000 replications +per case. Each generated panel is shared by all four variants: identity/HC0, +identity/oracle covariance, design precision/HC0 and design precision/oracle +covariance. Do not compare independently generated samples across variants. + +The ten cases are n=400 with k=0,1,2,5,20,50 extra-date treated units and 19 +extra-date controls; n=4,000 with fixed k=2,5 and 19 controls; and n=4,000 with +proportionally growing k=20,50 and 190 controls. All units have dates 2 and 3; +extra-date units have date 1 and, when k>0, omit date 4. All six existing evidence +JSON files remain unchanged. The new report has a distinct path. + +Use the existing summarize function: count failed replications in unconditional +coverage; report bias/MCSE, RMSE, empirical SD, mean variance, variance ratio, +coverage/MCSE, interval width and zero-null rejection. The declared acceptance +rule remains zero failures, absolute bias below .1 empirical SD, variance ratio +in [.9,1.1], and 95% coverage within three nominal binomial Monte Carlo standard +errors. The oracle covariance is analytically correct here; empirical thresholds +still report finite Monte Carlo fluctuations and do not qualify an empirical +covariance estimator. Record native replay discrepancies and all source hashes. + +## Decision rule + +If known covariance repairs coverage while HC0 fails on the same estimates, +the experiment supports a covariance-estimation failure in this design. If +design weighting reduces theoretical and empirical variance, that establishes +bounded precision evidence, not calibrated feasible inference. Neither result +alone permits a production change across the covariate/cluster contract or a +claim of general statistical qualification. Prefer the agreed, transparent +restricted estimator over an unqualified correction or an oracle-only solution. + +## Executed results + +The report `benchmarks/data/chained_did/sparse-covariance-diagnostic.json` records +20,000 native fits and 80,000 paired variant evaluations, with no fit or target +failures. Every native replay agrees with the independent identity calculation: +maximum point discrepancy is 4.22e-15. The report records variance discrepancies, +all comparison metrics, seeds, source hashes and environment versions. + +| n | Extra-date treated / controls | Identity HC0 coverage | Identity known-covariance coverage | Design precision HC0 coverage | Design / identity exact variance | +| --- | --- | ---: | ---: | ---: | ---: | +| 400 | 0 / 19 | .9495 | .9495 | .9495 | 1.0000 | +| 400 | 1 / 19 | .9470 | .9470 | .9465 | .9996 | +| 400 | 2 / 19 | .8005 | .9560 | .9520 | .2351 | +| 400 | 5 / 19 | .9080 | .9490 | .9395 | .4184 | +| 400 | 20 / 19 | .9530 | .9560 | .9525 | .6846 | +| 400 | 50 / 19 | .9400 | .9450 | .9395 | .7848 | +| 4000 | 2 / 19, fixed | .7260 | .9535 | .9585 | .0283 | +| 4000 | 5 / 19, fixed | .8880 | .9500 | .9570 | .0602 | +| 4000 | 20 / 190, growing | .9345 | .9460 | .9500 | .2351 | +| 4000 | 50 / 190, growing | .9560 | .9620 | .9605 | .4184 | + +At 2,000 replications, the unchanged coverage acceptance interval is +[.93538,.96462]. All oracle and design-precision variants pass the declared +bias, variance and coverage criteria on this ten-case grid. Native identity HC0 +fails five cases. Do not widen the threshold to accept the growing k=20 case. +Observed coverage Monte Carlo standard errors and other metrics remain in the +JSON; the nominal 95% binomial standard error is .00487. Since ATT=1, the +reported rejection of a zero-effect null measures power, not test size. The paired variants share +each panel; cases use separate seeds and are not paired across sample sizes. + +Known covariance repairs coverage on the identical native estimates, which +isolates empirical covariance estimation in this conditional Gaussian model. +Identity weighting also loses precision: with n=4,000 and 2 sparse treated units, +the theoretical design-weighted variance is 2.83% of identity variance. Increasing +the main panel alone does not fix a rare comparison given substantial weight. +Growing support behaves differently and must not be conflated with fixed counts. + +## Implementation decision + +Retain the agreed production identity GMM, full propensity-score/mean Jacobian, +HC0/CR1 covariance and documented sparse-pair safeguards. Keep the design-weighted +prototype in the benchmark only. It improves this specific homoskedastic, +no-covariate, independently sampled design; the experiment does not qualify it +for conditional IPW, heterogeneous variances, random observation counts or +clustered inference. Oracle covariance is not available in applications. + +There is also an algebraic limit to the proposed weighting remedy. In a balanced +panel each cohort has the same arm counts at every pair, so its count-based +metric is a scalar multiple of identity. That scalar cancels from the recovery +map, leaving estimates and analytical covariance unchanged. Therefore this +weighting change cannot repair the historical balanced-panel coverage failure. +A deterministic regression verifies this invariance. Passing the sparse grid +is insufficient grounds to replace the production estimator or declare a +universal finite-sample correction. + +No production source changed and no failed historical evidence was replaced. +A fresh large qualification rerun of the identical production policy would not +supersede its existing high-precision failures. The upstream contribution is +prepared as an experimental restricted estimator with reproducible limits; +statistical qualification remains false. + +## Reproduction + +```bash +PYTHONPATH=. DIFF_DIFF_BACKEND=python OPENBLAS_NUM_THREADS=1 \ + python -m benchmarks.python.diagnose_chained_did_sparse \ + --replications 2000 --workers 4 --seed 20261106 \ + --output /tmp/chained-did-oracle-diagnostic.json +``` + +Choose a new output path; the runner refuses existing and frozen evidence files. +Exit zero indicates completion, not statistical qualification. The derivation +uses the linear GMM structure in the paper's +[appendix A.2.6](https://arxiv.org/html/2301.01085v4); known innovation +variance and the fixed-count prototype are benchmark assumptions, not additional +claims about that paper's empirical inference procedure. diff --git a/docs/methodology/chained-did-proposal.md b/docs/methodology/chained-did-proposal.md new file mode 100644 index 000000000..f0014f6f1 --- /dev/null +++ b/docs/methodology/chained-did-proposal.md @@ -0,0 +1,129 @@ +# Proposal: native Python chained difference-in-differences + +Date: 2 November 2026. Status: experimental implementation; qualification incomplete. +See the [validation record](chained-did-validation.md) and +[oracle diagnostic decision](chained-did-oracle-diagnostic.md) for executed +evidence and limits. The restricted contract below defines this contribution. + +## Problem and contribution + +Add `ChainedDiD` to estimate group-time average treatment effects, ATT(g,t), +from overlapping incomplete panels. Units observed at different pairs of dates +can jointly identify an effect even when no unit spans its reference and target +dates. The contribution is a paper-derived Python implementation, rather than +an R wrapper or a translation of `cdid` source. + +Current upstream `CallawaySantAnna` uses direct within-cell differences or an +optional repeated-cross-section route for unbalanced panels. Neither constructs +the chained comparison system. This proposal targets upstream commit +`c5280ae578f1a60fd4f2a38c0ff8f4f7861bac9a` (version 4.0.0). The historical +validation records retain their original source fingerprints; they do not +qualify the revised catalogue policy below. + +## First-release contract + +| Component | Included | +| --- | --- | +| Design | Binary absorbing treatment, staggered adoption, never-treated controls, zero anticipation | +| Target population | Units represented in the supplied data, including units with fewer than two finite outcomes; explicit missing-outcome rows can retain otherwise unobserved units | +| Covariates | Complete, time-invariant unit attributes; no covariates is the unconditional special case | +| Identification | Conditional PT-post, treatment overlap and the pair-sampling restriction specified in the appendix | +| Treatment adjustment | Normalised inverse probability weighting (IPW), with pair-specific cohort-versus-control logits | +| Comparison system | Every ordered date pair with at least two observed units per arm, including non-adjacent pairs and pre-reference comparisons | +| Combination | Fixed identity-weighted generalised method of moments (GMM); no estimated optimal weights or covariance regularisation | +| Results | Group-time effects, direct universal-reference pre-treatment differentials, overall and event-study summaries, indexed joint covariance, constituent aggregation weights and support diagnostics | +| Inference | Analytical unit-level or one-way clustered covariance; normal pointwise intervals; joint multiplier replay as a validation check | +| Interface | Standalone estimator and result type following the package's fit, parameter, summary and aggregation conventions | + +For the covariate-adjusted version, a sufficient sampling restriction is that +observing both endpoints is independent of the joint distribution of covariates +and actual outcome changes, within each cohort, for every used pair. This is +stronger than merely requiring representative unconditional mean changes. It +allows some selection on outcome levels, but does not correct arbitrary attrition. +These assumptions require scientific justification; diagnostics cannot certify them. + +PT-post leaves earlier cohort-versus-control trend differences unrestricted. +They must cancel when earlier observations help recover a post-treatment target; +the estimator must not set them to zero. The +[methodological pseudocode](chained-did-pseudocode.md) defines that construction, +the population accounting, influence-function scale and aggregation weights. + +Defer PT-all, not-yet-treated controls, non-zero anticipation, outcome regression, +doubly robust methods, conditional observation models, imputation, sampling +weights, complex survey designs, simultaneous bands, +Rust acceleration and custom nuisance learners. These are distinct extensions; +none is required to deliver this first statistical contract. + +Pairs below two observed units in either arm are excluded with counts and a +reason. This replication minimum prevents zero singleton residual variance from +entering GMM. Retained pairs below 20 raw units in either arm or control effective +sample size 20 trigger a warning; 20 is an operational threshold, not evidence of +calibration. Identity GMM ignores pair precision. Sparse-pair intervals can +remain materially anti-conservative after singleton exclusion, so statistical +qualification remains open. Changing the GMM metric is outside this revision. + +## Validation matrix + +The entries below define the acceptance requirements; their executed status is +in the [validation record](chained-did-validation.md). Numerical agreement and +statistical qualification are separate release gates. + +| ID | Design or failure mode | Required evidence | +| --- | --- | --- | +| V01 | Two dates, no covariates | Hand-calculated DiD and covariance; agreement with the existing estimator where contracts coincide | +| V02 | Overlapping two-date windows, no long-panel units | Recover the known long effect by chaining; preserve each link's original unit identifiers | +| V03 | Non-adjacent observed dates | Use the long comparison; do not discard it because an intermediate outcome is absent | +| V04 | Earlier bridge under PT-post | Recover a post-treatment effect by cancelling a non-zero unrestricted pre-reference differential; keep direct pre-period reporting separate from GMM nuisance parameters | +| V05 | Unsupported target | Removing a required bridge makes that target unavailable, while independently supported targets remain available | +| V06 | Overidentified and balanced systems | Compare with independent matrix calculations; preserve all comparison rows even when their covariance is singular | +| V07 | Covariates and shared controls | Check the propensity-score Jacobian, normalisation derivatives and cross-link/cross-cohort covariance | +| V08 | Population accounting | Retain zero-/one-outcome units in cohort masses, aggregation influence functions and cluster counts; reject conflicting metadata | +| V09 | Invalid inputs or failed nuisance fits | Reject duplicate unit-date rows, inconsistent supplied cohort dates, missing covariates, rank failure, separation and non-convergence | +| V10 | Unit and cluster inference | Reproduce the declared covariance scaling; match joint multiplier covariance; mark inference unavailable with fewer than two clusters | +| V11 | R reference | Compare supported cells, estimates, full covariance and aggregates with pinned `cdid`, using IPW, PT-post, never-treated controls, no observation correction and identity GMM | +| V12 | Valid simulation designs | Measure bias, empirical standard deviation, standard-error calibration, pointwise coverage and null rejection rates, with Monte Carlo uncertainty | +| V13 | Violated assumptions | Demonstrate the limits under selective trends/effects, misspecified logits and false PT; do not require diagnostics to detect untestable violations | +| V15 | Sparse comparison pairs | Record exclusions below two units per arm; warn below 20 raw units or control ESS 20; evaluate ATT(3,3)=1 at n=400 with 0/1/2/5/20/50 treated units and 19 controls at an extra date, including bias, variance and coverage with Monte Carlo uncertainty | +| V16 | Irregular multi-cohort calendar | Exclude structural reference zeros from mixed diagnostic rows; verify contributing cohorts, weights, counts and covariance while retaining cohort-base provenance | +| V14 | Package integration | Parameter round trips, stable result schemas, plotting/aggregation compatibility, source-to-doc mappings, tutorial and installed-consumer checks | + +The appendix specifies reference normalisation, tolerances and simulation gates. +No accuracy, efficiency, interval-coverage or runtime claim follows from this +proposal or from passing a reference fixture alone. + +## Delivery and integration + +1. Review this methodological contract and its pseudocode before product code. +2. Implement the comparison catalogue, unweighted and IPW link estimates, complete + unit influence functions and identity-GMM recovery, with V01-V09 fixtures. +3. Complete analytical inference, aggregates, multiplier checks and the external + reference harness; run V10-V13 before claiming statistical qualification. +4. Complete V14 and submit a bounded upstream contribution. + +Candidate new modules are `diff_diff/chained_did.py` and +`diff_diff/chained_did_results.py`. Reuse `linalg.py`, `safe_inference`, the current +aggregation kit and multiplier stream where their contracts fit; do not subclass +`CallawaySantAnna.fit()` or change its estimator behaviour. Production delivery +also reaches exports, tests, R-reference fixtures, API docs, a tutorial, +`docs/references.rst`, methodology registry, bundled guides, estimator catalogues, +`docs/doc-deps.yaml` and a changelog fragment, following upstream contribution +requirements. The experimental implementation includes those production surfaces. + +`cdid` is an external validation reference at commit +`1039120668cceabd8111cad82d32ab3927c2b94f`, version 1.1.0.0, with its documented +`did` 2.5.1 / `DRDID` 1.3.0 environment pinned by the external-reference harness. Its planned +Python implementation is not a dependency or a scheduling constraint. The Python +derivation and implementation remain independent of its GPL-3 source. + +## Sources + +- Bellégo, Benatia and Dortet-Bernadet, *The chained difference-in-differences*, + [published paper](https://doi.org/10.1016/j.jeconom.2024.105783); + [accessible mathematical reference, section 2.2 and appendix A.2](https://arxiv.org/html/2301.01085v4). +- [Upstream unbalanced-panel contract](https://github.com/igerber/diff-diff/blob/ba746948b7fc7d06a5e8cdbf9601e21c3eaac6f0/diff_diff/staggered.py#L449) + and [contribution requirements](https://github.com/igerber/diff-diff/blob/ba746948b7fc7d06a5e8cdbf9601e21c3eaac6f0/CONTRIBUTING.md). +- Pinned `cdid` documentation: + [GMM](https://github.com/dvbn/cdid/blob/1039120668cceabd8111cad82d32ab3927c2b94f/docs/methods/gmm.md), + [parallel trends](https://github.com/dvbn/cdid/blob/1039120668cceabd8111cad82d32ab3927c2b94f/docs/methods/parallel-trends.md), + [treatment adjustment](https://github.com/dvbn/cdid/blob/1039120668cceabd8111cad82d32ab3927c2b94f/docs/methods/treatment.md), + [missing outcomes](https://github.com/dvbn/cdid/blob/1039120668cceabd8111cad82d32ab3927c2b94f/docs/methods/missing-outcomes.md). diff --git a/docs/methodology/chained-did-pseudocode.md b/docs/methodology/chained-did-pseudocode.md new file mode 100644 index 000000000..8cb1843f9 --- /dev/null +++ b/docs/methodology/chained-did-pseudocode.md @@ -0,0 +1,383 @@ +# Chained DiD: methodological pseudocode + +Date: 2 November 2026. Status: proposed derivation for the +[upstream proposal](chained-did-proposal.md); not executable code or validation +evidence. Notation and equations below are an independent specification based on +the paper, not a transcription of the R implementation. Algorithmic choices such +as identity weighting are proposed implementation decisions. + +## 1. Existing Python methodology and the new seam + +The inspected upstream commit is +`ba746948b7fc7d06a5e8cdbf9601e21c3eaac6f0`. Its staggered-estimator flow is: + +```text +validate input and select panel or repeated-cross-section route +construct the cohort, calendar, outcome and covariate structures +for each requested cohort-time cell: + select reference and target dates, and eligible controls + panel route: retain units observed at both selected dates; difference outcomes + repeated-cross-section route: compare the selected outcome-level samples + estimate that cell with its selected treatment adjustment + retain the estimate and its original-unit influence contributions +aggregate cells, including uncertainty in estimated aggregation weights +compute analytical inference and validate joint covariance using multipliers +return results with the retained aggregation inputs +``` + +The chained estimator replaces reference-to-target cell construction with a +catalogue of date-pair comparisons and a linear recovery step. Unit alignment, +aggregation and uncertainty propagation remain explicit. Reuse numerical +primitives rather than treating the existing private cell estimators as a new +public API. In particular, their fallback, clipping and influence-function +conventions must not enter the new estimator without being reconciled below. + +## 2. Inputs, population and assumptions + +Let i index n distinct supplied study units; t indexes the strictly ordered stored +calendar. Inputs follow existing column-name conventions: outcome, unit, time, +first treatment date, optional static covariates and an optional cluster column. +For this first version, calendar labels are finite numeric values, treatment dates +must occur in that calendar and 0 denotes never-treated units. A treated cohort +must have a preceding stored date; first-period treatment is unsupported. + +Build the calendar from all dated rows, including missing outcomes. A missing +row or a NaN outcome supplies no observed endpoint; infinite outcomes are input +errors. Reject duplicate unit-date rows. Unit, date, cohort and active static +metadata must be complete and consistent. Covariates and cluster assignments must +be constant within units; treatment timing is supplied, never inferred from gaps. +This interface accepts cohort dates, not a separate realised treatment-status +column. It rejects conflicting dates within a unit, but callers must verify +that the realised treatment follows the supplied absorbing schedule. Extra +columns are not interpreted as treatment indicators. Preserve caller-owned data. + +The study-unit registry contains every distinct supplied unit, including units +with one or no observed outcomes. A unit absent from the data is outside this +declared population; retaining one dated row with a missing outcome and complete +metadata includes it. Counts and cohort shares use units, not observed rows or +repeat observers. These units represent the cohort population law; the method +does not provide finite-population or survey-design inference. + +For cohort g define D_ig = 1{G_i=g}, C_i = 1{G_i=0}, and b_g as the stored date +immediately preceding g. Let R_i,st indicate finite outcomes at both s and t. +Let X_i include a constant and the declared static covariates. The target is: + +```text +ATT(g,t) = E[Y_t(1) - Y_t(0) | G=g], t >= g. +``` + +Maintain no anticipation, absorbing treatment and conditional parallel untreated +trends from b_g onwards. For every catalogue pair and contributing cohort k, +use the paper's sufficient baseline sampling restriction: + +```text +R_st is independent of (X, Y_t - Y_s), conditional on G=k. +``` + +Here changes are actual outcomes, including treatment effects. Cohort-specific +pair response rates may differ. For no covariates, equality of the relevant mean +changes suffices. With covariates, mean representativeness alone does not preserve +the covariate law needed by IPW. Conditional-on-X missingness without preservation +of that law requires an observation correction, which is outside this release. +Selection on a time-constant outcome-level component is admissible only when it +preserves the required joint restriction. Positive pair response rates, adequate +treatment overlap and correct pair-specific logistic propensity models are also +required. Inference needs independent units or independent clusters, suitable +moments and an increasing number of clusters; few-cluster validity is not claimed. + +## 3. Catalogue and the PT-post parameterisation + +For each g consider all s=g, the maintained PT-post and no-anticipation assumptions make this profile +ATT(g,t). Before b_g it is an unrestricted trend differential, not a treatment +effect or an imposed zero. The baseline sampling restriction makes every pair +comparison target the same covariate and cohort laws, so: + +```text +d_g,st = theta_g(t) - theta_g(s). +``` + +Build A_g with one row per retained pair and a column for every stored date except +b_g: +1 at t, -1 at s, omitting the anchored column. Stack cohorts block-diagonally +into A. Joint covariance is not block-diagonal because cohorts share controls. + +Example: b_g=2, with comparisons 1->2 and 1->4 only. Then +ATT(g,4)=d_g,14-d_g,12. The earlier differential cancels even if it is non-zero. +If 1->2 is absent, 1->4 alone cannot separate the effect from that differential. +This is why earlier parameters remain free under PT-post. + +## 4. Normalised IPW link and its complete influence function + +For each link e=(g,s,t), restrict the nuisance-fit risk set to R_i,st=1 and +D_ig+C_i=1. Fit the unpenalised logistic model: + +```text +p_e(X) = Pr(D_g=1 | X, D_g+C=1, R_st=1) = expit(X' beta_e) +o_e(X) = p_e(X) / (1-p_e(X)) +mu_T,e = sum(R_st D_g deltaY_st) / sum(R_st D_g) +mu_C,e = sum(R_st C o_e deltaY_st) / sum(R_st C o_e) +d_hat_e = mu_T,e - mu_C,e +``` + +This pair-specific propensity accommodates different cohort response rates. An +intercept-only fit gives the ordinary difference in mean observed changes. Reuse +`solve_logit` only with explicit rank/error handling and a checked score residual; +a warning about non-convergence or separation is insufficient for accepting a fit. +Do not fall back to an intercept-only model, clip odds, drop units or trim scores. +Non-finite odds, numerical boundary probabilities, separation, non-convergence +or unresolved singularity must stop the fit. Record propensity range, group counts and control effective +sample size `(sum weights)^2 / sum(weights^2)`; poor overlap remains a limitation. + +Derive uncertainty from the joint estimating equations, not fixed fitted weights. +Let r_i=R_i,st (D_ig+C_i), a_e=(beta_e,mu_T,e,mu_C,e), and define: + +```text +h_beta,i = r_i X_i (D_ig - p_e(X_i)) +h_T,i = R_i,st D_ig (deltaY_i,st - mu_T,e) +h_C,i = R_i,st C_i o_e(X_i) (deltaY_i,st - mu_C,e) +H_e = (1/n) sum_i derivative(h_i, a_e) +IF_a,i,e = -solve(H_e, h_i) +phi_i,e = IF_mu_T,i,e - IF_mu_C,i,e +``` + +All sums use the original n-unit registry, with zero score contributions outside +the risk set; compute changes only inside the observation mask to avoid 0*NaN. +The Jacobian includes the logistic-score and odds derivatives and both mean +normalisations. It must reproduce numerical derivatives on well-conditioned +fixtures. No-outcome units have zero link scores but remain in the registry. +Across links, align phi by original unit identifier, never by local row position. +The convention is `d_hat-d = (1/n) sum_i phi_i + o_p(n^-1/2)`; phi is O(1). + +## 5. Identity-GMM recovery and target support + +Stack d_hat and the n-by-M matrix Phi of link influence functions. Fix the +criterion to the full-catalogue identity metric: + +```text +minimise_theta ||d_hat - A theta||^2. +``` + +Use a rank-revealing SVD, with cutoff `max(A.shape)*machine_epsilon*sigma_max`, +and form A_plus from retained singular directions. For a target contrast c: + +```text +if c is not in rowspace(A): + return unavailable estimate and inference, with a support reason +else: + ell = c' A_plus + tau_hat = ell d_hat + psi_i = ell phi_i +``` + +Test row-space membership by the projection residual using the same retained +SVD basis and a dimension-scaled roundoff tolerance. An empty catalogue has no +supported non-zero contrast. The minimum-norm parameter solution does not make +unsupported parameters estimable; never publish those arbitrary coordinates. +Reference zeros are exact identities and do not enter hypothesis tests. + +Reported pre-treatment cells have a separate, direct construction. For t=2 triggered the retained-support diagnostic, including k=20 and k=50 +because only 19 controls were observed at the extra date. The two-unit case +still has severe undercoverage, so the safeguard does not resolve the wider +finite-sample inference problem. This pilot's `core_thresholds_pass` is false; +it neither qualifies the revised estimator nor replaces the historical study. +Cases use independent SeedSequence-derived seeds, so differences between rows +are not paired comparisons. The JSON also reports bias, its Monte Carlo standard +error, RMSE, variance ratios, interval width and rejection rates. + +To reproduce into a new file from the repository root: + +```bash +PYTHONPATH=. DIFF_DIFF_BACKEND=python OPENBLAS_NUM_THREADS=1 \ + python -m benchmarks.python.validate_chained_did \ + --design sparse --replications 300 --workers 3 --seed 20261105 \ + --output /tmp/chained-did-sparse-pilot.json +``` + +The runner refuses existing output files and the five frozen historical evidence +paths. Exit zero means completion, not statistical acceptance. A larger +prespecified qualification study, hosted CI and a maintainer decision on the +inference policy remain open before merge. + +The [oracle covariance diagnostic](chained-did-oracle-diagnostic.md) subsequently +compares known covariance and fixed design precision weighting on 20,000 fresh +native fits. It isolates two failure mechanisms without changing the agreed +production contract or replacing these historical pilot results. + +## Prepared-branch review (2 November 2026) + +A subsequent user-supplied read-only review examined the prepared contribution +on upstream base `c5280ae5`. It reported 111 passing chained tests with the +pure-Python backend, reproduced the mixed-reference correction, checked all nine +frozen R outputs, and confirmed exposed summary weights and removal of local +process files. It also reported approximately 93% coverage after singleton +exclusion, 78% with two sparse treated units and 91% with five. These are the +reviewer's results, consistent with the recorded pilot, not new runs by the +contribution preparer. The review did not repeat the full suite, wheel, +documentation builds, hosted CI or live R validation. + +This review covers the post-initial-review changes to pair eligibility, +support warnings, mixed-reference aggregation and exposed weights. The separate +submission packaging review did not assess numerical methodology. The review +found the contribution suitable for upstream review after correcting the PR +text, but not ready to merge or statistically qualified. + +Before merge, maintainers must decide whether to accept experimental identity +GMM with the documented undercoverage, or require precision-aware weighting or +another inference policy with further validation. In the larger recorded +diagnostic, coverage with two fixed sparse treated units is 0.8005 at n=400 and +0.7260 at n=4,000; increasing the main panel does not increase rare-pair support. +Maintainers must also decide whether to accept the native observed-pair logit +criterion despite the known continuous-covariate R difference, or require the +cohort-response-calibrated criterion and its corresponding derivation and +validation. The [validation follow-up](chained-did-validation-followup.md) +describes that criterion difference. Hosted CI remains outstanding, and neither +decision is settled by the packaging checks or experimental label. diff --git a/docs/methodology/chained-did-validation-followup.md b/docs/methodology/chained-did-validation-followup.md new file mode 100644 index 000000000..b980122b8 --- /dev/null +++ b/docs/methodology/chained-did-validation-followup.md @@ -0,0 +1,209 @@ +# ChainedDiD validation follow-up + +Date: 2 November 2026. This follows the original qualification record without +replacing it. This document describes historical qualification snapshots, whose +source fingerprints remain unchanged. The current pair eligibility and mixed +reference-row policies are documented in the +[review remediation](chained-did-review-remediation.md); historical experiments +do not establish calibration for that revision. + +## Prespecified experiments + +The original 13 scenarios are rerun with 10,000 replications and fresh seed +20261103. A separate paired diagnostic study uses 2,000 replications, seed +20261104, rotating panels at n=1,000/4,000, and intermittent panels crossing +those sample sizes with independent outcomes or K=50/100 clusters. Both use a +64-bit SeedSequence-derived replication seed, rather than truncating to the +original runner's 32-bit seed. Cases, targets, baseline hashes, environment, +source hashes, failures and observed-rate Monte Carlo uncertainty are recorded. + +Oracle variants know treatment odds and/or population cohort masses. Known +propensity contributions exclude beta estimation uncertainty. Known masses +remove share uncertainty and also change the finite-sample point aggregate. +Each variant retains the same superpopulation target and is assessed against +its own empirical variance. These are benchmark-only diagnostics, not public +estimator options. The original files and their acceptance outcomes are frozen. + +The inherited DGP adds a cluster shock to the unit AR errors when K is positive. +Consequently K=50 versus K=100 holds marginal error variance fixed, whereas K=0 +versus positive K changes both dependence and variance. The latter comparison +does not isolate dependence alone; within-dataset oracle comparisons remain paired. + +## External reference: primitive-score alignment + +The exact reference environment is R 4.5.2, cdid 1.1.0.0 at the pinned commit, +did 2.5.1 and DRDID 1.3.0. The image base is digest-pinned; installed package +versions and image identity are recorded with the executed reference evidence. +The R implementation is used only through public inputs/outputs and documentation. + +The native contract fits an unweighted logit in each observed pair risk set: + +```text +sum_observed Z_i * (D_i - expit(Z_i' beta)) = 0. +``` + +The reference documentation describes observation-route weights, including +cohort response rates. The independently tested alignment criterion is a +cohort-calibrated score with r_k = observed-pair units / eligible cohort units (at least two observed outcomes): + +```text +a_i = D_i/r_g + (1-D_i)/r_0 +sum_observed a_i * Z_i * (D_i - expit(Z_i' beta)) = 0. +``` + +Both use the observed treated mean minus the separately normalised control +odds-weighted mean. Multiplying all control odds by a constant cancels from that +mean. Nevertheless, response-rate calibration can alter fitted logit slopes +with continuous covariates. Under the maintained within-cohort pair sampling +restriction, both logit criteria are correctly specified with different +intercepts; they need not give identical finite-sample points or influence +functions. A saturated binary covariate and equal cohort response rates provide +coinciding cases. The validation-only calibrated-score calculation tests this +hypothesis independently; it supplies point comparisons only, not a new +covariance estimator or a port of R source. + +Structural reference rows are also reported differently: the canonical Python +event-study container sets inference to NaN at reference rows; R records zero +SE. Comparison therefore verifies reference roles and exact zero R reference +SE separately, then compares all estimated-row inference and the complete +aggregate covariance. The predeclared CR1 correction is applied once to the R +raw cluster covariance, with its square root applied once to standard errors. +No tolerance or covariance factor is inferred from agreement. + +## Executed results and disposition + +The fresh qualification record is +`benchmarks/data/chained_did/qualification-followup.json`: 130,000 native fits, +zero core fit failures, all core bias limits satisfied, and all core variance +ratios in [0.95439, 1.03600]. At 10,000 replications the prespecified coverage +acceptance interval is [0.94346, 0.95654]. Four n=1,000 targets miss its lower bound: + +| Observation | Target | Coverage | Variance ratio | +| --- | --- | ---: | ---: | +| Balanced | Overall | 0.9429 | 0.95977 | +| Intermittent, 50 clusters | Overall | 0.9425 | 0.98222 | +| Intermittent, 50 clusters | ATT(2,3) | 0.9390 | 0.97469 | +| Cohort-dependent | ATT(2,3) | 0.9410 | 0.96016 | + +All larger-sample and null scenarios pass. The original rotating ATT(3,4) +coverage failure and the 0.896 variance ratio are not reproduced in this fresh +study; the original record remains unchanged. The follow-up still has +`core_thresholds_pass = false`, so it does not supersede the failed gate with +an acceptance claim. Independent seeds and greater precision refine the +failure pattern rather than erasing it. + +The paired study in `calibration-diagnostics.json` evaluates 16,000 datasets and +four variants per dataset with no fit failures. Knowing cohort masses changes +little in these designs; knowing propensities does not consistently remove +coverage loss. For example, at n=4,000 and K=50, overall absolute standardised +error 95th quantiles are 2.0625 with fitted propensities and 1.9869 with known +propensities, compared with the normal cutoff 1.9600. Fixed population masses +produce 2.0597 and 1.9841 respectively. Other targets still have heavy tails +with oracle propensities. The evidence does not establish an omitted Jacobian +or share term, or justify one universal variance multiplier. Normal/CR1 +finite-sample calibration remains a limitation requiring a separate inference +policy decision; production inference is unchanged. + +### External agreement + +`reference-validation.json` records all nine fixtures, their input hashes, +full public reference outputs, source/environment fingerprints and every +comparison result. Eight fixtures match declared tolerances on registry, +support, all cell estimates, full covariance and simple/event-study aggregates: +two dates, rotating windows, earlier bridges, non-adjacent observations, +balanced binary and continuous covariates, intermittent shared clusters and +zero-/one-outcome registry units. + +The incomplete continuous-covariate fixture remains a failing comparison. +Its native ATT(2,2) differs from R by about -0.001018 and ATT(2,3) by -0.001553; +the maximum covariance difference is 0.00087152. The independent calibrated-score +calculation reproduces every reported R point in this fixture within the +original tolerance. The response-rate denominator is the eligible cohort +(at least two observed outcomes), not the entire registry; inspecting the +public custom-callback inputs confirms the supplied weights. The calibrated +point diagnostic does not reproduce or qualify its covariance. + +The executable `benchmarks/R/inspect_chained_did_weights.R` probe verifies the +first pair's public callback inputs and stops before estimation. Its captured +output is stored under `public_input_probe` in the reference report, bound to +the input and probe source hashes: eligible counts [365, 418], observed pair +counts [301, 257], and weights [365/301, 418/257]. Maximum weight discrepancy +is 2.22e-16. These are public function inputs, not inspected R estimator source. + +Thus V11 has bounded agreement on eight coinciding fixtures and a documented +primitive-score difference on a ninth. It is not blanket parity with `cdid`. +The agreed native risk-set MLE is retained; changing it to the calibrated +criterion would require an explicit methodological revision and its complete +influence-function derivation. No failed comparison was excluded or weakened. + +### Reproduction + +From the repository root, with single-threaded BLAS: + +```bash +DOCKER_BUILDKIT=0 docker build -t diff-diff-cdid-reference:1.1.0.0 \ + benchmarks/environments/chained_did +PYTHONPATH=. DIFF_DIFF_BACKEND=python OPENBLAS_NUM_THREADS=1 \ + python benchmarks/python/compare_chained_did.py \ + --workspace /tmp/chained-did-reference +PYTHONPATH=. DIFF_DIFF_BACKEND=python OPENBLAS_NUM_THREADS=1 \ + python benchmarks/python/diagnose_chained_did.py --mode diagnostic \ + --replications 2000 --seed 20261104 --workers 4 \ + --output /tmp/chained-did-diagnostics.json +PYTHONPATH=. DIFF_DIFF_BACKEND=python OPENBLAS_NUM_THREADS=1 \ + python benchmarks/python/diagnose_chained_did.py --mode qualification \ + --replications 10000 --seed 20261103 --workers 4 \ + --output /tmp/chained-did-followup.json +``` + +The reference command exits 1 because the ninth comparison fails; this is an +identified criterion difference, not an environment installation failure. +Simulation exit 0 means completion, not acceptance: inspect the manifest flag. +The original record paths are protected against overwrite by the follow-up CLI. +The base image and core R versions are pinned. `packages.json` records the full +used environment; the Dockerfile fetches transitive CRAN dependencies at build +time, so rebuilding later is not a hermetic transitive-package restore. +The comparator checks the selected image's complete package/version/build +inventory and R/platform against that manifest before fitting and records the +verified inventory. A changed rebuild fails before generating reference evidence. + +## Engineering verification + +The final affected suite passes 1,302 tests with 14 skips, including estimator, +methodology, validation, base/aggregation, catalogue/naming, documentation and +the changed documentation-consumer regression. Scoped naming-guard entries +identify canonical calendar/aggregation uses and a quoted historical failure; +they do not revive removed API surfaces. + +The final chained estimator, methodology and validation tests pass on isolated +Python 3.9, 3.14 and stable 3.15 environments: 60 tests per environment. Python +3.12 also supplies the simulation and installed-wheel evidence. The wheel +rebuild and isolated installed-consumer checks pass under Python and Rust +backend settings; the chained estimator itself remains native Python. + +Pinned Black 26.10.1 and Ruff 0.17.1 pass across production/tests and the new +validation scripts. Sphinx passes with warnings treated as errors and notebook +execution disabled. Pinned mypy 2.4.0 in an isolated Python 3.14 environment +reports three errors in two unchanged upstream modules; the exact error lines +also occur on the clean target base. The host-inheriting Python 3.12 tool +environment reports 234 errors, likewise identical on that base. Neither +configuration adds errors for the new production modules. + +An independent review found no Critical or Important validation defects. Its +minor package-provenance finding was closed with a failing regression followed +by inventory verification and a full reference rerun. The public callback +evidence is now executable and retained. The slow full repository suite was +not rerun in this sprint; the original validation record distinguishes its +previously reproduced baseline failures. Hosted and non-Linux checks remain unrun. +The attempted broader RST-example replay was stopped at an unrelated expensive +estimator example; the affected documentation consumer passes separately. + +## Upstream disposition + +The native estimator remains experimental. Engineering evidence, matched +reference evidence and statistical qualification are separate gates. The next +methodological decision is whether maintainers want the declared observed-risk +propensity criterion or alignment with the calibrated R criterion, and which +finite-sample inference policy merits a separately prespecified study. +Public OR/DR, attrition corrections and optimal GMM remain outside this work. +The accompanying upstream review draft is prepared locally and has not been posted. diff --git a/docs/methodology/chained-did-validation.md b/docs/methodology/chained-did-validation.md new file mode 100644 index 000000000..8a0fa095b --- /dev/null +++ b/docs/methodology/chained-did-validation.md @@ -0,0 +1,200 @@ +# ChainedDiD validation record + +Date: 2 November 2026. Initial milestone: experimental; **not qualified for upstream merge**. + +This is the historical first-checkpoint record. The [follow-up](chained-did-validation-followup.md) +contains the executed pinned R comparisons, fresh simulations and current +qualification status; the environment-availability statements below describe +this initial checkpoint. Its original failed outcomes are retained. +The implementation has independent algebraic and integration evidence, but the +pinned R comparison is unrun and three small-sample simulation targets miss the +prespecified limits. Passing engineering tests does not close those gates. + +## Scope and reproducibility + +The controlling contract is the [proposal](chained-did-proposal.md) and +[methodological pseudocode](chained-did-pseudocode.md). This implementation uses +PT-post, never-treated controls, pair-specific normalised IPW, all supported date +pairs, identity GMM, unit HC0 or one-way cluster CR1 covariance, and normal +pointwise intervals. It neither fits an observation model nor supplies optimal +GMM weights. All changes are uncommitted against upstream base +`ba746948b7fc7d06a5e8cdbf9601e21c3eaac6f0`. + +The Monte Carlo runner is +`benchmarks/python/validate_chained_did.py`; the complete machine-readable record +is `benchmarks/data/chained_did/qualification.json`. Run from the repository root: + +```bash +PYTHONPATH=. DIFF_DIFF_BACKEND=python OPENBLAS_NUM_THREADS=1 \ + python benchmarks/python/validate_chained_did.py \ + --replications 2000 --workers 4 --seed 20261102 \ + --output /tmp/chained-did-qualification.json +``` + +The JSON includes source fingerprints, versions, seed derivation, analytic +population targets, individual scenario summaries and failures. The command's +successful exit means the study completed; acceptance is the separate +`core_thresholds_pass` field, which is **false** in this record. Do not treat the +exit status alone as statistical acceptance. + +The preliminary study is preserved as `qualification-pre-review.json`. The +final study reruns the same seeds and scenarios after clustering/parameter +boundary fixes and summary-contract fixes. Its fingerprints bind the final +estimator, result module and runner. The final runner separately records +empirical Monte Carlo standard errors for observed coverage/rejection rates and +the nominal binomial standard error used by the unchanged acceptance rule. + +## Acceptance matrix + +| Gate | Evidence and disposition | +| --- | --- | +| V01 | Two-date unconditional estimate and HC0 variance match hand calculations and `CallawaySantAnna` where the contracts coincide. | +| V02 | Rotating two-date windows recover the known longer effect without long-panel units. | +| V03 | Non-adjacent comparisons retain support when intermediate outcomes are absent. | +| V04 | An earlier bridge cancels a non-zero pre-reference differential; a separate test refuses to publish an indirectly supported nuisance coordinate as a direct pre-period diagnostic. | +| V05 | Disconnected cells are unavailable and invalidate summaries that require them; supported cells remain available. | +| V06 | An overidentified six-link graph matches an independent reference-anchored Laplacian calculation and covariance. Balanced systems also enter the simulation study. | +| V07 | Binary-stratum algebra and an independent continuous-covariate score fit with a finite-difference Jacobian match complete contributions. Shared-control cross-cohort covariance is retained. | +| V08 | Zero-/one-outcome units remain in registry masses; summary contributions include estimated-share uncertainty. Conflicting metadata are rejected. | +| V09 | Duplicate rows, missing/non-finite metadata, conflicting cohort dates, separation and rank failure are rejected. A cohort-date interface cannot observe realised treatment reversals: the caller must verify the supplied absorbing schedule. | +| V10 | Hand calculations and joint cluster multiplier replay reproduce HC0/CR1 scaling. A single cluster preserves points but withholds inference. | +| V11 | **Open.** The pinned R environment is unavailable; no `cdid` numerical agreement is claimed. | +| V12 | **Not passed.** Study complete with zero core fit failures; three targets miss the prespecified calibration limits below. | +| V13 | Negative designs demonstrate bias and coverage loss under selective observation, false parallel trends and propensity misspecification. This is sensitivity evidence, not a diagnostic guarantee. | +| V14 | Public exports, parameter/result/aggregation/plotting contracts, source-to-doc routing and affected tutorial have executed evidence. See engineering checks below for remaining environment limits. | + +These are bounded checks, not proofs over every input. In particular, algebraic +agreement does not establish coverage, and a reference implementation would not +certify the observation or causal assumptions. + +## Simulation results + +There are 13 scenarios with 2,000 replications each: four valid observation +patterns (balanced, rotating, intermittent and cohort-dependent) at n=1,000 and +4,000; rotating null scenarios at both sizes; and three negative designs at +n=1,000. Intermittent core scenarios use 50 and 100 independent clusters. The +DGP includes static covariates affecting treatment and trends, serially +correlated outcomes, heterogeneous effects and unrestricted earlier trend +jumps. Targets are the population overall ATT, ATT(2,3) and ATT(3,4). + +Acceptance requires zero failed required fits, absolute bias below 0.1 empirical +standard deviations, estimated-to-empirical variance ratios in [0.9,1.1], and +95% coverage / null rejection within three nominal binomial Monte Carlo +standard errors. For 2,000 replications the coverage acceptance interval is +approximately [0.93538,0.96462]. Thresholds remain as prespecified. + +All n=4,000 core and null targets pass these limits. Core fit failures are zero +at both sizes, but the following n=1,000 targets fail: + +| Observation / clusters | Target | Bias | Empirical SD | Variance ratio | Coverage | Missed limit | +| --- | --- | ---: | ---: | ---: | ---: | --- | +| Rotating / independent units | ATT(3,4) | -0.00201 | 0.26032 | 0.94758 | 0.9340 | Coverage | +| Intermittent / 50 clusters | Overall | -0.00187 | 0.12533 | 0.93308 | 0.9315 | Coverage | +| Intermittent / 50 clusters | ATT(2,3) | -0.00157 | 0.18631 | 0.89639 | 0.9300 | Variance ratio and coverage | + +The failure is concentrated in variance/interval calibration rather than mean +recovery in these designs. The larger-sample results support the tested +asymptotic behaviour, but do not erase smaller-sample failures. The contribution +must remain experimental while this qualification gate is unresolved. Changing +finite-sample corrections or interval rules would be a methodological decision +requiring a revised contract and fresh evidence, not a tolerance adjustment. + +Under the negative selective-observation design, 63 fits fail and unconditional +overall coverage is 0.263, with bias about 0.311. False parallel trends produce +overall bias about 0.752 and zero coverage. Misspecified logits produce overall +bias about 0.102 and coverage 0.862, although one cell still meets its numerical +limits. These results show why treatment IPW cannot justify arbitrary attrition +or substitute for parallel trends. Failure counts remain in coverage denominators. + +## External reference gate + +`benchmarks/R/benchmark_chained_did.R` refuses an unmatched environment and never +installs packages. It requires R 4.5.2, `cdid` 1.1.0.0 at commit +`1039120668cceabd8111cad82d32ab3927c2b94f`, `did` 2.5.1 and `DRDID` 1.3.0. +The available environment has R 4.6.1 and lacks those three packages; invoking +the harness correctly refuses that R version. This tests the guard, not the +reference estimator or output schema. + +A future matched run must compare cell labels/support, estimates, full joint +covariance, direct earlier diagnostics and aggregates at the declared tolerances. +It must reconcile the reference's sqrt(n) matrix with estimator-scale covariance +and separately reconcile Python's declared K/(K-1) cluster correction. Registry +or catalogue differences must be resolved rather than normalised away. + +## Engineering checks + +The final focused run passes 1,112 tests, with 14 skips and 65 warnings, across +the new estimator/methodology/harness tests and existing parameter, aggregation, +documentation and changelog contracts. The three new test modules alone pass +48 tests. The configured non-slow suite is recorded separately below. + +A fresh whole-change review found no critical issues and three input-boundary +defects: duplicate static/cluster metadata aliases, silent disabling of an +empty-named cluster column, and missing fit-time validation after direct +parameter mutation. Each was reproduced with failing regression tests and fixed. +The simple summary also now reports its canonical unit weight; changing summary +alpha uses the shared analytical refusal message. No review findings are deferred. + +The affected notebook executes top to bottom with `nbclient`; +`nbmake` is absent. Sphinx builds with warnings treated as errors and notebook +execution disabled because the affected notebook is executed separately. + +A scoped type check on the two new modules passes with installed mypy 1.18.2 +and `--follow-imports=silent`. That tool differs from upstream's pinned 2.4.0; +a full imported check reports errors in imported modules, so this is not a +claim that the full upstream type gate passes. Available Black/Ruff also differ +from the pinned development versions. No Rust estimator kernel is added. + +The wheel builds and installs into an isolated environment. A consumer outside +the checkout imports both public classes, fits covariate IPW with a shared +cluster/covariate column, aggregates into canonical results, round-trips pickle +and reads the bundled guide. It passes with both Python and Rust backend +settings. This verifies installation and existing backend interoperability; +it does not add or qualify a chained-specific native kernel. + + +### Broad-suite outcome and baseline comparison + +The configured non-slow selection was exercised in two phases. The serial prefix +was interrupted inside `test_methodology_trop.py` after 8,436 passes, 221 skips +and one failure. Its unfinished module and every subsequent collected test file +were rerun using four pytest-xdist workers with work stealing: 6,963 passed, +212 skipped and ten failed. These phase counts overlap and are not a unique +combined test count. The isolated plugin installation changes no project dependency. + +Eight contribution-related failing cases across the two phases were fixed and +retested: the snippet harness mistook an exported result class for a fitted +instance; the naming guard needed the new legal calendar-column surface; the +cluster-policy roster/fit recipe needed the new estimator; and four catalogue +guards still expected 23 estimators. The catalogue now explicitly includes the +23 cut estimators plus experimental `ChainedDiD`. These changes preserve the +strict guards instead of bypassing them. + +Three remaining cases reproduce identically on a separate untouched checkout of +`ba746948b7fc7d06a5e8cdbf9601e21c3eaac6f0`: + +| Case | Observed baseline failure | +| --- | --- | +| `test_naming_guard.py::test_allowlists_are_reachable` | Stale `DMLDiDResults.groups` allowlist entry. | +| `test_rdplot.py::TestRDPlotGoldenParity::test_all_configs` | One senate/esmvpr confidence bound differs by 4.16e-10, exceeding its 1e-9 relative tolerance. | +| `test_rust_backend.py::TestClusterVcovDeterminism::test_raw_kernel_noncontiguous_ids_bit_identical` | The source checkout has no compiled `_rust_backend` module; the raw-kernel test imports it directly even with the Python backend selected. | + +Those cases remain unchanged. Therefore the broad run is not a green upstream +suite. The wheel's isolated Rust-enabled smoke check is separate evidence from +the source checkout's missing extension. Final affected-check results are +recorded below; the reproduced stale-allowlist case is explicitly deselected +there rather than reported as a pass. + + +Final affected run: **1,490 passed, 21 skipped**, 192 warnings, using the Python +backend and four workers. It covers the three new test modules, existing base +and aggregation contracts, source-to-doc routing, changelog, full snippet +harness, naming guard (excluding the separately reproduced stale allowlist +case), cluster policy, diagnostic/estimator catalogues and result serialisation. +All contribution-related failures found by the broad run are resolved. + +Final checks also confirm that the installed wheel contains exactly the final +two implementation modules, that every study source fingerprint matches, and +that `git diff --check` is clean. The graphify AST graph was refreshed locally. +No commits, pushes, PRs or standards changes were made. The implementation lives +in the isolated `feature/chained-did` worktree; original checkout edits are preserved. diff --git a/docs/references.rst b/docs/references.rst index d2918f9fb..820d49a9b 100644 --- a/docs/references.rst +++ b/docs/references.rst @@ -12,9 +12,15 @@ Difference-in-Differences - **Angrist, J. D., & Pischke, J.-S. (2009).** *Mostly Harmless Econometrics: An Empiricist's Companion*. Princeton University Press. Chapter 5: Differences-in-Differences. +Chained Difference-in-Differences +--------------------------------- + +- **Bellégo, C., Benatia, D., & Dortet-Bernadet, V. (2025).** "The chained difference-in-differences." *Journal of Econometrics*. https://doi.org/10.1016/j.jeconom.2024.105783 + Two-Way Fixed Effects --------------------- + - **Wooldridge, J. M. (2010).** *Econometric Analysis of Cross Section and Panel Data* (2nd ed.). MIT Press. - **Imai, K., & Kim, I. S. (2021).** "On the Use of Two-Way Fixed Effects Regression Models for Causal Inference with Panel Data." *Political Analysis*, 29(3), 405-415. https://doi.org/10.1017/pan.2020.33 diff --git a/docs/tutorials/34_chained_did.ipynb b/docs/tutorials/34_chained_did.ipynb new file mode 100644 index 000000000..fccae68e4 --- /dev/null +++ b/docs/tutorials/34_chained_did.ipynb @@ -0,0 +1,158 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "48313625", + "metadata": {}, + "source": [ + "# Chained DiD for incomplete panels\n", + "\n", + "This tutorial constructs an overlapping panel with known treatment effects of 3 and 7. Each unit is observed twice, and no unit spans the reference date and final date. The experimental native estimator combines the observed changes under PT-post and the stated observation assumptions.\n", + "\n", + "The example is a numerical illustration. It does not establish coverage, validate a real missingness process or reproduce the full R `cdid` feature set.", + "\n", + "Pairs need two observed units in each arm; excluded candidates and low-support warnings are recorded in `result.diagnostics`. Identity GMM ignores pair precision, and sparse-pair intervals can remain anti-conservative. The warning threshold of 20 is not a qualification rule. Mixed diagnostic horizons exclude structural reference identities. Use `result.aggregation_weights(\"simple\")` to inspect constituent weights.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "788b551c", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "from diff_diff import ChainedDiD, plot_event_study" + ] + }, + { + "cell_type": "markdown", + "id": "7e982c79", + "metadata": {}, + "source": [ + "## Construct the overlapping windows\n", + "\n", + "Cohort 2 starts treatment at date 2; date 1 is its reference. The untreated trend has a fixed earlier cohort difference, so PT-post does not require a zero earlier diagnostic. Within each observation window the constructed slope deviations average to zero." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e747387a", + "metadata": {}, + "outputs": [], + "source": [ + "rows = []\n", + "for cohort in (0, 2):\n", + " for s, t in ((0, 1), (1, 2), (2, 3)):\n", + " for j, slope in enumerate((-1.5, -0.5, 0.5, 1.5)):\n", + " for date in (s, t):\n", + " effect = {2: 3, 3: 7}.get(date, 0) if cohort else 0\n", + " y = j + (3 + slope) * date\n", + " if cohort:\n", + " y += 10 + 2 * (date >= 1) + effect\n", + " rows.append((f\"{cohort}-{s}-{j}\", date, cohort, y))\n", + "panel = pd.DataFrame(rows, columns=[\"id\", \"date\", \"cohort\", \"outcome\"])\n", + "assert panel.groupby(\"id\").size().eq(2).all()\n", + "panel.head()" + ] + }, + { + "cell_type": "markdown", + "id": "bf29ee40", + "metadata": {}, + "source": [ + "## Fit and inspect the original targets\n", + "\n", + "Without covariates the pairwise IPW comparison reduces to a difference in mean changes. All supported pairs enter the identity-GMM system. The headline weights scheduled post-treatment cells by cohort size; inference uses unit HC0 covariance and normal pointwise intervals." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8aac7b44", + "metadata": {}, + "outputs": [], + "source": [ + "result = ChainedDiD().fit(\n", + " panel, outcome=\"outcome\", unit=\"id\", time=\"date\", first_treat=\"cohort\"\n", + ")\n", + "assert np.isclose(result.group_time_effects[2, 2].effect, 3)\n", + "assert np.isclose(result.group_time_effects[2, 3].effect, 7)\n", + "assert np.isclose(result.group_time_effects[2, 0].effect, -2)\n", + "print(result.summary())\n", + "result.to_dataframe()", + "\n", + "weights = result.aggregation_weights(\"simple\")\n", + "assert np.allclose(weights.weight, [0.5, 0.5])\n", + "weights\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0be693cc", + "metadata": {}, + "outputs": [], + "source": [ + "event = result.aggregate(\"event_study\")\n", + "assert np.allclose(event.att[event.event_time >= 0], [3, 7])\n", + "plot_event_study(event, show=False)\n", + "event.to_dataframe()" + ] + }, + { + "cell_type": "markdown", + "id": "d52786d6", + "metadata": {}, + "source": [ + "## Remove a required bridge\n", + "\n", + "Keep the calendar rows but make date-2 outcomes unobserved. There is then no supported path from the reference date to either post-treatment target. The original summary becomes unavailable; the estimator does not replace it with an average over a different set of effects." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e392dd9a", + "metadata": {}, + "outputs": [], + "source": [ + "broken = panel.copy()\n", + "broken.loc[broken.date == 2, \"outcome\"] = np.nan\n", + "unsupported = ChainedDiD().fit(broken, \"outcome\", \"id\", \"date\", \"cohort\")\n", + "assert np.isnan(unsupported.att)\n", + "assert unsupported.group_time_effects[2, 3].skip_reason == \"unsupported_target\"\n", + "unsupported.to_dataframe()[[\"group\", \"time\", \"att\", \"available\", \"status\"]]" + ] + }, + { + "cell_type": "markdown", + "id": "6230c5d7", + "metadata": {}, + "source": [ + "## Conditions and limits\n", + "\n", + "For covariate adjustment, supply complete static numeric columns through `covariates=[...]`. A sufficient baseline observation condition preserves the joint covariate and actual-change distribution within each cohort for every used pair. Treatment IPW alone does not correct conditional attrition.\n", + "\n", + "A unit-constant `cluster` column selects one-way CR1 covariance. Normal intervals do not guarantee few-cluster validity. Units with fewer than two observed outcomes remain in population accounting when represented in the input. Public bootstrap intervals, PT-all, not-yet-treated controls, OR/DR and survey designs are outside this implementation.\n", + "\n", + "Reference: Bellégo, Benatia and Dortet-Bernadet, [The chained difference-in-differences](https://doi.org/10.1016/j.jeconom.2024.105783). The development validation matrix and source-hashed simulation manifest distinguish executed evidence from unrun reference gates." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/tutorials/index.rst b/docs/tutorials/index.rst index 5e6596ee3..580340170 100644 --- a/docs/tutorials/index.rst +++ b/docs/tutorials/index.rst @@ -261,6 +261,13 @@ Modern estimators for designs the basic toolkit cannot handle. lane, and reading the ATT_X pre-test. + .. grid-item-card:: Chained DiD: Incomplete Panels + :link: 34_chained_did + :link-type: doc + + Recover effects from overlapping two-date observations, inspect + unsupported targets and distinguish treatment IPW from attrition correction. + .. toctree:: :maxdepth: 1 :caption: Advanced Methods @@ -279,6 +286,7 @@ Modern estimators for designs the basic toolkit cannot handle. LWDiD Rolling Transformations <31_lwdid> Double ML DiD (Chang 2020) <32_dml_did> Bad Controls (Caetano et al.) <33_bad_controls> + Chained DiD: Incomplete Panels <34_chained_did> Study Design ------------ diff --git a/tests/test_chained_did.py b/tests/test_chained_did.py new file mode 100644 index 000000000..7e2caa861 --- /dev/null +++ b/tests/test_chained_did.py @@ -0,0 +1,336 @@ +"""Behavioural tests for the native incomplete-panel estimator.""" + +import pickle + +import numpy as np +import pandas as pd +import pytest + +import diff_diff + + +def rotating_panel(windows=((0, 1), (1, 2), (2, 3))): + """Known effects 3 and 7, with a non-zero earlier trend differential.""" + rows = [] + for cohort in (0, 2): + for window in windows: + for j, slope in enumerate((-1.5, -0.5, 0.5, 1.5)): + uid = f"{cohort}-{window}-{j}" + for t in window: + effect = {2: 3, 3: 7}.get(t, 0) if cohort else 0 + y = 10 * bool(cohort) + j + 3 * t + slope * t + y += (2 * (t >= 1) + effect) if cohort else 0 + rows.append((uid, t, cohort, y)) + data = pd.DataFrame(rows, columns=["id", "t", "g", "y"]) + for t in sorted(set(range(4)) - set(data.t)): + data = pd.concat([data, data.iloc[[0]].assign(t=t, y=np.nan)], ignore_index=True) + return data + + +def fit(data, **kwargs): + assert hasattr(diff_diff, "ChainedDiD"), "ChainedDiD must be publicly exported" + return diff_diff.ChainedDiD(**kwargs).fit( + data, outcome="y", unit="id", time="t", first_treat="g" + ) + + +def test_rotating_windows_recover_long_effect_without_long_panel(): + data = rotating_panel() + assert data.groupby("id").size().eq(2).all() + result = fit(data) + assert result.group_time_effects[2, 2].effect == pytest.approx(3) + assert result.group_time_effects[2, 3].effect == pytest.approx(7) + assert result.att == pytest.approx(5) + assert result.group_time_effects[2, 2].se ** 2 == pytest.approx(0.625) + assert result.group_time_effects[2, 3].se ** 2 == pytest.approx(1.25) + assert result.n_units == data.id.nunique() + + +def test_earlier_differential_is_reported_directly_and_not_restricted(): + result = fit(rotating_panel(windows=((0, 1), (0, 3)))) + assert result.group_time_effects[2, 0].effect == pytest.approx(-2) + assert result.group_time_effects[2, 3].effect == pytest.approx(7) + assert result.group_time_effects[2, 2].skip_reason == "unsupported_target" + assert np.isnan(result.att), "overall must not silently drop the missing horizon" + + +def test_nonadjacent_pair_retained_and_missing_calendar_endpoint_preserved(): + data = rotating_panel(windows=((1, 3),)) + result = fit(data) + assert result.group_time_effects[2, 3].effect == pytest.approx(7) + assert np.isnan(result.group_time_effects[2, 2].effect) + assert result.diagnostics["catalogue"] == [(2, 1, 3)] + + +def test_disconnected_effects_are_unavailable_without_fabricated_coordinates(): + result = fit(rotating_panel(windows=((0, 1), (2, 3)))) + assert np.isnan(result.group_time_effects[2, 2].effect) + assert np.isnan(result.group_time_effects[2, 3].effect) + assert np.isnan(result.att) + assert result.group_time_effects[2, 0].effect == pytest.approx(-2) + + +def test_row_order_and_arbitrary_unit_labels_do_not_change_estimates_or_covariance(): + data = rotating_panel() + before = data.copy(deep=True) + a = fit(data) + b = fit(data.sample(frac=1, random_state=4)) + pd.testing.assert_frame_equal(data, before) + np.testing.assert_allclose(a.vcov, b.vcov, atol=1e-12, equal_nan=True) + assert a.att == pytest.approx(b.att) + + +@pytest.mark.parametrize("fault", ["duplicate", "infinite", "missing_id", "changing_cohort"]) +def test_invalid_panel_rejected(fault): + data = rotating_panel() + if fault == "duplicate": + data = pd.concat([data, data.iloc[[0]]], ignore_index=True) + elif fault == "infinite": + data.loc[0, "y"] = np.inf + elif fault == "missing_id": + data.loc[0, "id"] = None + else: + data.loc[0, "g"] = 2 + with pytest.raises(ValueError): + fit(data) + + +def test_parameter_contract_and_canonical_results(): + assert hasattr(diff_diff, "ChainedDiD"), "ChainedDiD must be publicly exported" + estimator = diff_diff.ChainedDiD(alpha=0.1) + assert estimator.get_params() == {"alpha": 0.1, "cluster": None} + with pytest.raises(ValueError): + estimator.set_params(alpha=2) + assert estimator.alpha == 0.1 + result = estimator.fit(rotating_panel(), "y", "id", "t", "g") + assert result.se > 0 + assert 0 <= result.p_value <= 1 + assert result.conf_int[0] < result.att < result.conf_int[1] + assert "Chained" in result.summary() + assert result.to_dict()["att"] == result.att + restored = pickle.loads(pickle.dumps(result)) + assert restored.aggregate("simple").att[0] == pytest.approx(result.att) + + +def test_event_study_uses_canonical_container_with_joint_covariance(): + result = fit(rotating_panel()) + event = result.aggregate("event_study") + assert isinstance(event, diff_diff.EventStudyResults) + assert event.att[event.event_time == 0][0] == pytest.approx(3) + assert event.att[event.event_time == 1][0] == pytest.approx(7) + assert event.is_reference[event.event_time == -1][0] + assert event.vcov is not None + with pytest.raises(ValueError): + result.aggregate("event_study", balance_e=1) + with pytest.raises(ValueError): + result.aggregate("simple", weights="equal") + + +def test_single_cluster_withholds_all_inference_but_preserves_estimate(): + data = rotating_panel().assign(cluster="only") + result = fit(data, cluster="cluster") + assert result.att == pytest.approx(5) + for value in (result.se, result.t_stat, result.p_value, *result.conf_int): + assert np.isnan(value) + + +@pytest.mark.parametrize("alpha", [True, None, "0.05", np.nan, np.inf, 0, 1, -0.1]) +def test_invalid_alpha_rejected(alpha): + with pytest.raises(ValueError, match="alpha"): + diff_diff.ChainedDiD(alpha=alpha) + + +def test_failed_refit_clears_estimator_state_and_existing_results_are_detached(): + estimator = diff_diff.ChainedDiD() + result = estimator.fit(rotating_panel(), "y", "id", "t", "g") + estimator.set_params(alpha=0.1) + assert result.alpha == 0.05 + assert result.aggregate("simple").att[0] == pytest.approx(5) + with pytest.raises(ValueError): + estimator.fit(rotating_panel(), "absent", "id", "t", "g") + assert not estimator.is_fitted_ + assert estimator.results_ is None + + +def test_plotting_accepts_retained_canonical_event_study(): + import matplotlib.pyplot as plt + + result = fit(rotating_panel()) + ax = diff_diff.plot_event_study(result.aggregate("event_study"), show=False) + assert ax is not None + plt.close("all") + + +def test_irregular_numeric_calendar_preserves_actual_horizons(): + data = rotating_panel() + labels = {0: 2000, 1: 2002, 2: 2005, 3: 2009} + data["t"] = data.t.map(labels) + data["g"] = data.g.map({0: 0, 2: 2005}) + result = fit(data) + event = result.aggregate("event_study") + assert event.event_time.tolist() == [-5, -3, 0, 4] + assert event.reference_period == -3 + assert result.group_time_effects[2005, 2009].effect == pytest.approx(7) + + +def test_inconsistent_or_missing_cluster_metadata_rejected(): + data = rotating_panel().assign(cluster="a") + data.loc[0, "cluster"] = "b" + with pytest.raises(ValueError, match="constant"): + fit(data, cluster="cluster") + data.loc[0, "cluster"] = None + with pytest.raises(ValueError, match="complete"): + fit(data, cluster="cluster") + + +@pytest.mark.parametrize("column", ["g", "id", "x"]) +def test_cluster_can_share_a_static_metadata_column(column): + from tests.test_methodology_chained_did import binary_covariate_panel + + data, *_ = binary_covariate_panel() + data["independent_cluster_name"] = data[column] + a = diff_diff.ChainedDiD(cluster=column).fit(data, "y", "id", "t", "g", covariates=["x"]) + b = diff_diff.ChainedDiD(cluster="independent_cluster_name").fit( + data, "y", "id", "t", "g", covariates=["x"] + ) + np.testing.assert_allclose(a.vcov, b.vcov) + + +def test_empty_string_cluster_name_is_used_and_false_remains_explicitly_unclustered(): + data = rotating_panel() + data[""] = np.arange(len(data)) // 8 + data["named"] = data[""] + a, b = fit(data, cluster=""), fit(data, cluster="named") + np.testing.assert_allclose(a.vcov, b.vcov) + off = diff_diff.ChainedDiD(cluster=False) + assert off.get_params()["cluster"] is False + np.testing.assert_allclose(off.fit(data, "y", "id", "t", "g").vcov, fit(data).vcov) + + +@pytest.mark.parametrize("parameter,value", [("alpha", 2), ("alpha", True), ("cluster", True)]) +def test_fit_revalidates_directly_mutated_parameters_and_clears_state(parameter, value): + estimator = diff_diff.ChainedDiD() + estimator.fit(rotating_panel(), "y", "id", "t", "g") + setattr(estimator, parameter, value) + with pytest.raises(ValueError, match=parameter): + estimator.fit(rotating_panel(), "y", "id", "t", "g") + assert not estimator.is_fitted_ + assert estimator.results_ is None + + +def test_simple_aggregate_reports_its_defined_weight_and_summary_refuses_relabelling(): + result = fit(rotating_panel()) + simple = result.aggregate("simple") + np.testing.assert_array_equal(simple.weight, [1.0]) + with pytest.raises(ValueError, match="refit|re-fit") as exc: + result.summary(alpha=0.1) + assert "bootstrap" not in str(exc.value) + + +@pytest.mark.parametrize("arm", [0, 2]) +def test_singleton_pair_is_recorded_and_cannot_distort_connected_post_effects(arm): + data = rotating_panel() + ids = data.loc[(data.g == arm) & (data.t == 0), "id"].unique() + data = data.loc[~data.id.isin(ids[1:])].copy() + with pytest.warns(UserWarning, match="singleton pairs excluded"): + result = fit(data) + assert (2, 0, 1) not in result.diagnostics["catalogue"] + excluded = next( + row for row in result.diagnostics["excluded_links"] if row["s"] == 0 and row["t"] == 1 + ) + assert excluded["reason"] == "insufficient_pair_replication" + assert excluded["n_treated" if arm else "n_control"] == 1 + assert result.diagnostics["pair_support"]["minimum_per_arm"] == 2 + assert result.group_time_effects[2, 0].skip_reason == "unsupported_direct_pre_comparison" + assert result.att == pytest.approx(5) + changed = data.copy() + changed.loc[changed.t == 0, "y"] += 1000 + with pytest.warns(UserWarning): + other = fit(changed) + assert other.att == pytest.approx(result.att) + np.testing.assert_allclose(other.vcov, result.vcov, equal_nan=True) + + +def test_excluding_only_singleton_bridge_withholds_target_and_aggregate(): + data = rotating_panel(windows=((1, 3),)) + ids = data.loc[data.g == 2, "id"].unique() + data = data.loc[~data.id.isin(ids[1:])] + with pytest.warns(UserWarning, match="singleton pairs excluded"): + result = fit(data) + assert result.diagnostics["catalogue"] == [] + cell = result.group_time_effects[2, 3] + assert cell.skip_reason == "unsupported_target" + assert np.isnan(cell.effect) and np.isnan(cell.se) + assert np.isnan(result.att) and np.isnan(result.se) + assert result.n_units == 5 + + +def test_low_count_warning_retains_replicated_pairs_with_diagnostics(): + with pytest.warns(UserWarning, match="retained pairs have low support"): + result = fit(rotating_panel()) + assert len(result.diagnostics["catalogue"]) == 3 + assert all(row["low_support"] for row in result.diagnostics["links"]) + assert result.diagnostics["pair_support"]["warning_threshold"] == 20 + assert result.att == pytest.approx(5) + + +def test_control_effective_sample_size_warns_despite_large_raw_arms(): + x = np.r_[np.ones(95), np.zeros(5), np.ones(2), np.zeros(98)] + g = np.r_[np.full(100, 2), np.zeros(100)] + data = pd.DataFrame( + { + "id": np.repeat(np.arange(200), 2), + "t": np.tile([1, 2], 200), + "g": np.repeat(g, 2), + "x": np.repeat(x, 2), + } + ) + data["y"] = data.t * (data.x + (data.g == 2)) + with pytest.warns(UserWarning, match="retained pairs have low support"): + result = diff_diff.ChainedDiD().fit(data, "y", "id", "t", "g", covariates=["x"]) + link = result.diagnostics["links"][0] + assert link["n_treated"] == link["n_control"] == 100 + assert link["control_ess"] < 20 + assert link["low_support"] + + +def test_mixed_reference_event_row_uses_only_estimated_diagnostics_and_their_weights(): + rows = [] + for g, n in [(0, 30), (3, 20), (5, 10)]: + for i in range(n): + for t in (1, 3, 4, 5): + y = t + i * t / n + (4 * (t >= 3) if g == 5 else 0) + rows.append((f"{g}-{i}", t, g, y)) + result = fit(pd.DataFrame(rows, columns=["id", "t", "g", "y"])) + event = result.aggregate("event_study") + k = np.flatnonzero(event.event_time == -2)[0] + diagnostic = result.group_time_effects[5, 3] + assert event.att[k] == pytest.approx(diagnostic.effect) + assert event.se[k] == pytest.approx(diagnostic.se) + assert event.n[k] == 10 + assert not event.is_reference[k] + assert event.reference_event_times == (-2, -1) + assert event.reference_periods == [-1] + weights = result.aggregation_weights("event_study").query("label == -2") + assert weights[["group", "time"]].to_numpy().tolist() == [[5, 3]] + np.testing.assert_allclose(weights.weight, [1]) + assert result.aggregate("event_study").att[k] == event.att[k] + + +def test_constituent_weights_explain_simple_and_event_aggregates_without_renormalising(): + result = fit(rotating_panel()) + weights = result.aggregation_weights() + assert weights.label.tolist() == ["overall", "overall"] + np.testing.assert_allclose(weights.weight, [0.5, 0.5]) + assert weights.available.all() + assert sum( + row.weight * result.group_time_effects[row.group, row.time].effect + for row in weights.itertuples() + ) == pytest.approx(result.att) + weights.loc[:, "weight"] = 0 + np.testing.assert_allclose(result.aggregation_weights().weight, [0.5, 0.5]) + unavailable = fit(rotating_panel(windows=((1, 3),))).aggregation_weights() + np.testing.assert_allclose(unavailable.weight, [0.5, 0.5]) + assert unavailable.available.tolist() == [False, True] + with pytest.raises(ValueError, match="simple.*event_study"): + result.aggregation_weights("group") diff --git a/tests/test_chained_did_calibration.py b/tests/test_chained_did_calibration.py new file mode 100644 index 000000000..c7a15ce8b --- /dev/null +++ b/tests/test_chained_did_calibration.py @@ -0,0 +1,72 @@ +"""Independent checks for validation-only oracle and follow-up experiments.""" + +import importlib +import importlib.util + +import numpy as np +import pytest + +from diff_diff import ChainedDiD +from tests.test_methodology_chained_did import binary_covariate_panel + + +def study(): + name = "benchmarks.python.diagnose_chained_did" + assert importlib.util.find_spec(name) is not None, "Calibration study must exist" + return importlib.import_module(name) + + +def test_known_propensity_link_omits_estimation_uncertainty_and_matches_ratio_influence(): + data, delta, treated, x = binary_covariate_panel() + odds = np.where(x == 0, 1.5, 2.5) + effects, scores = study().oracle_cells(data, {1: odds}) + wt = (~treated) * odds + mt, mc = delta[treated].mean(), wt @ delta / wt.sum() + expected = treated * (delta - mt) / treated.sum() - wt * (delta - mc) / wt.sum() + assert effects[1, 1] == pytest.approx(mt - mc) + np.testing.assert_allclose(scores[1, 1], expected, atol=1e-12) + + +def test_known_population_weights_change_point_and_remove_share_uncertainty_only(): + memberships = np.array([1, 1, 1, 2, 0, 0]) + cells = {(1, 1): 2.0, (2, 2): 10.0} + q = {k: np.zeros(6) for k in cells} + empirical, empirical_q = study().combine(cells, q, memberships) + known, known_q = study().combine(cells, q, memberships, masses={1: 0.5, 2: 0.5}) + assert empirical == 4 + assert known == 6 + np.testing.assert_allclose(empirical_q, [-0.5, -0.5, -0.5, 1.5, 0, 0]) + np.testing.assert_array_equal(known_q, np.zeros(6)) + + +def test_fitted_contribution_aggregation_matches_production_including_cluster_scaling(): + base = importlib.import_module("benchmarks.python.validate_chained_did") + data, _ = base.generate_data(300, "intermittent", 42, clusters=30) + native = ChainedDiD(cluster="cluster").fit(data, "y", "id", "t", "g", covariates=["x"]) + cells = {k: v.effect for k, v in native.group_time_effects.items() if k[1] >= k[0]} + q = dict(zip(native.vcov_index, native.influence_functions.T)) + registry = data.drop_duplicates("id") + estimate, contribution = study().combine(cells, q, registry.g.to_numpy()) + assert estimate == pytest.approx(native.att) + assert study().standard_error(contribution, registry.cluster.to_numpy()) == pytest.approx( + native.se + ) + + +def test_standardised_error_diagnostics_keep_failed_replications_and_use_observed_rates(): + report = study().summarize(np.array([-2.0, 0.0, 2.0, np.nan]), np.ones(4), 0) + assert report["failures"] == 1 + assert report["coverage"] == 0.25 + assert report["z_sd"] == 2 + assert report["abs_z_95"] > 1.96 + assert report["z_quantiles"][1] == 0 + + +def test_scenario_grid_crosses_sample_and_cluster_counts_and_uses_fresh_seeds(): + cases = study().diagnostic_cases() + clustered = {(c["n"], c["clusters"]) for c in cases if c["clusters"]} + assert clustered == {(1000, 50), (1000, 100), (4000, 50), (4000, 100)} + assert study().replication_seed(20261104, 0, 0) == study().replication_seed(20261104, 0, 0) + seeds = [study().replication_seed(20261104, i, j) for i in range(8) for j in range(50)] + assert len(set(seeds)) == len(seeds) + assert max(seeds) > 2**32 diff --git a/tests/test_chained_did_core.py b/tests/test_chained_did_core.py new file mode 100644 index 000000000..761cd571a --- /dev/null +++ b/tests/test_chained_did_core.py @@ -0,0 +1,53 @@ +"""Restricted contribution boundaries and analytical plotting semantics.""" + +import importlib + +import numpy as np +import pytest + +from diff_diff import ChainedDiD +from diff_diff.chained_did import _pair_ipw +from tests.test_chained_did import rotating_panel + + +@pytest.mark.parametrize( + "kwargs", [{"n_bootstrap": 0}, {"n_bootstrap": 19}, {"seed": None}, {"seed": 1}] +) +def test_core_constructor_rejects_optional_extension_parameters(kwargs): + with pytest.raises(TypeError, match="unexpected keyword argument"): + ChainedDiD(**kwargs) + + +@pytest.mark.parametrize("unavailable", [False, True]) +def test_structural_plot_reference_preserves_analytical_intervals(monkeypatch, unavailable): + data = rotating_panel() + result = ChainedDiD(cluster="id" if unavailable else None).fit(data, "y", "id", "t", "g") + event = result.aggregate("event_study") + if unavailable: + event.conf_int_lower[:] = np.nan + event.conf_int_upper[:] = np.nan + module = importlib.import_module("diff_diff.visualization._event_study") + monkeypatch.setattr(module, "_render_event_study_mpl", lambda df, **kw: df) + inferred = module.plot_event_study(event, show=False) + explicit = module.plot_event_study(event, show=False, reference_period=-1) + np.testing.assert_array_equal(inferred.ci_lower, explicit.ci_lower) + np.testing.assert_array_equal(inferred.ci_upper, explicit.ci_upper) + if unavailable: + assert explicit.ci_lower.isna().all() and explicit.ci_upper.isna().all() + with pytest.raises(ValueError, match="fitted reference"): + module.plot_event_study(event, show=False, reference_period=0) + + +@pytest.mark.parametrize("treated_x1", [0, 1]) +def test_native_pair_distinguishes_separation_from_an_overlapping_counterpart(treated_x1): + x = np.r_[np.zeros(83), np.ones(41 + treated_x1)][:, None] + treated = np.r_[np.zeros(45), np.ones(38), np.zeros(41), np.ones(treated_x1)].astype(bool) + changes = np.linspace(-1, 2, len(treated)) + if not treated_x1: + with pytest.raises(ValueError, match="IPW propensity fit failed:.*did not converge"): + _pair_ipw(changes, treated, x) + else: + effect, contributions, info = _pair_ipw(changes, treated, x) + assert np.isfinite(effect) and np.isfinite(contributions).all() + assert contributions.sum() == pytest.approx(0, abs=1e-10) + assert 0 < info["propensity_min"] <= info["propensity_max"] < 1 diff --git a/tests/test_chained_did_reference.py b/tests/test_chained_did_reference.py new file mode 100644 index 000000000..3bc166cc9 --- /dev/null +++ b/tests/test_chained_did_reference.py @@ -0,0 +1,168 @@ +"""Schema and mismatch checks; these synthetic records do not prove R parity.""" + +import copy +import importlib +import importlib.util + +import numpy as np +import pytest + +from diff_diff import ChainedDiD +from tests.test_chained_did import rotating_panel + + +def comparator(): + name = "benchmarks.python.compare_chained_did" + assert importlib.util.find_spec(name) is not None, "Reference comparator must exist" + return importlib.import_module(name) + + +def reference(result): + event = result.aggregate("event_study") + return { + "metadata": { + "n": result.n_units, + "covariance_scale": "estimator", + "cluster_correction": "none", + }, + "cells": [ + {"group": g, "time": t, "att": cell.effect, "available": cell.skip_reason is None} + for (g, t), cell in result.group_time_effects.items() + ], + "vcov": result.vcov.tolist(), + "overall": {"att": result.att, "se": result.se}, + "events": { + "event_time": event.event_time.tolist(), + "att": event.att.tolist(), + "se": np.where(event.is_reference, 0, event.se).tolist(), + "is_reference": event.is_reference.tolist(), + "vcov": event.vcov.tolist(), + }, + } + + +def test_reference_comparison_matches_by_keys_and_rejects_estimate_and_scale_mismatches(): + fit = ChainedDiD().fit(rotating_panel(), "y", "id", "t", "g") + record = reference(fit) + order = np.arange(len(record["cells"]))[::-1] + record["cells"] = [record["cells"][j] for j in order] + record["vcov"] = np.asarray(record["vcov"])[np.ix_(order, order)].tolist() + assert comparator().compare(fit, record, outcome_sd=1)["passed"] + bad = copy.deepcopy(record) + bad["cells"][0]["att"] += 0.1 + assert not comparator().compare(fit, bad, outcome_sd=1)["passed"] + bad = copy.deepcopy(record) + bad["metadata"]["covariance_scale"] = "sqrt_n" + with pytest.raises(ValueError, match="scale"): + comparator().compare(fit, bad, outcome_sd=1) + + +def test_declared_cluster_cr1_reconciliation_is_applied_once_to_all_inference(): + data = rotating_panel() + data["cluster"] = np.arange(len(data)) // 4 + fit = ChainedDiD(cluster="cluster").fit(data, "y", "id", "t", "g") + record = reference(fit) + k = data.cluster.nunique() + correction = k / (k - 1) + record["vcov"] = (fit.vcov / correction).tolist() + record["overall"]["se"] /= np.sqrt(correction) + record["events"]["vcov"] = (np.asarray(record["events"]["vcov"]) / correction).tolist() + record["events"]["se"] = (np.asarray(record["events"]["se"]) / np.sqrt(correction)).tolist() + assert comparator().compare(fit, record, outcome_sd=1, clusters=k)["passed"] + assert not comparator().compare(fit, record, outcome_sd=1)["passed"] + + +def test_unavailable_cells_and_registry_masses_cannot_be_silently_discarded(): + fit = ChainedDiD().fit(rotating_panel(windows=((0, 1), (0, 3))), "y", "id", "t", "g") + record = reference(fit) + record["cells"][2]["available"] = True + assert not comparator().compare(fit, record, outcome_sd=1)["passed"] + record = reference(fit) + record["metadata"]["n"] += 1 + assert not comparator().compare(fit, record, outcome_sd=1)["passed"] + + +def test_reference_row_reporting_is_reconciled_but_estimated_event_inference_is_checked(): + native = ChainedDiD().fit(rotating_panel(), "y", "id", "t", "g") + record = reference(native) + event = native.aggregate("event_study") + record["events"] = { + "event_time": event.event_time.tolist(), + "att": event.att.tolist(), + "se": np.where(event.is_reference, 0, event.se).tolist(), + "is_reference": event.is_reference.tolist(), + "vcov": event.vcov.tolist(), + } + assert comparator().compare(native, record, outcome_sd=1)["passed"] + record["events"]["se"][-1] *= 1.1 + assert not comparator().compare(native, record, outcome_sd=1)["passed"] + + +def test_calibrated_score_diagnostic_reproduces_external_continuous_pair_points(): + data = comparator().fixtures()["incomplete_continuous_x"] + points = comparator().calibrated_points(data) + # Pinned public R output for this seeded fixture; no R implementation code. + assert points[2, 2] == pytest.approx(3.0406236096388795, abs=1e-8) + assert points[2, 3] == pytest.approx(2.8900418764849487, abs=1e-8) + native = ChainedDiD().fit(data, "y", "id", "t", "g", covariates=["x"]) + assert abs(native.group_time_effects[2, 2].effect - points[2, 2]) > 1e-4 + + +def test_missing_aggregate_evidence_cannot_pass_the_reference_gate(): + native = ChainedDiD().fit(rotating_panel(), "y", "id", "t", "g") + record = reference(native) + del record["events"] + assert not comparator().compare(native, record, outcome_sd=1)["passed"] + + +def test_reference_environment_rejects_changed_missing_and_extra_packages(): + expected = { + "R": "R 4.5.2", + "platform": "linux", + "packages": [ + {"Package": "cdid", "Version": "1.1.0.0", "Built": "4.5.2", "_row": "cdid"}, + {"Package": "dependency", "Version": "1", "Built": "4.5.2"}, + ], + } + actual = copy.deepcopy(expected) + actual["packages"].reverse() + comparator().validate_environment(expected, actual) + for field in ("Version", "Built"): + changed = copy.deepcopy(actual) + changed["packages"][0][field] = "changed" + with pytest.raises(ValueError, match="inventory"): + comparator().validate_environment(expected, changed) + for packages in (actual["packages"][:1], actual["packages"] + [actual["packages"][0]]): + with pytest.raises(ValueError, match="inventory"): + comparator().validate_environment(expected, dict(actual, packages=packages)) + with pytest.raises(ValueError, match="inventory"): + comparator().validate_environment(expected, dict(actual, R="changed")) + + +@pytest.mark.parametrize("name", sorted(comparator().fixtures())) +def test_frozen_public_r_outputs_detect_native_numerical_drift(name): + import hashlib + import json + from pathlib import Path + + tool = comparator() + root = Path(__file__).resolve().parents[1] + frozen = json.loads( + (root / "benchmarks/data/chained_did/reference-validation.json").read_text() + ) + case = frozen["cases"][name] + data = tool.fixtures()[name] + assert ( + hashlib.sha256(data.to_csv(index=False, lineterminator="\n").encode()).hexdigest() + == case["input_sha256"] + ) + native = ChainedDiD(cluster="cluster" if "cluster" in data else None).fit( + data, "y", "id", "t", "g", covariates=["x"] if "x" in data else None + ) + clusters = data.drop_duplicates("id").cluster.nunique() if "cluster" in data else None + outcome = tool.compare( + native, case["reference"], outcome_sd=float(data.y.std()), clusters=clusters + ) + # Pin every check, including the known criterion mismatch, without relaxing tolerances. + assert outcome["checks"] == case["checks"] + assert outcome["passed"] == (name != "incomplete_continuous_x") diff --git a/tests/test_chained_did_sparse_diagnostics.py b/tests/test_chained_did_sparse_diagnostics.py new file mode 100644 index 000000000..ab5b68109 --- /dev/null +++ b/tests/test_chained_did_sparse_diagnostics.py @@ -0,0 +1,144 @@ +"""Exact Gaussian benchmark geometry; these checks do not qualify inference.""" + +import importlib +import importlib.util +import json +import sys + +import numpy as np +import pandas as pd +import pytest + +from benchmarks.python.validate_chained_did import generate_sparse_data + + +def study(): + name = "benchmarks.python.diagnose_chained_did_sparse" + assert importlib.util.find_spec(name) is not None, "Sparse oracle diagnostic must exist" + return importlib.import_module(name) + + +def balanced_dates(dates): + return pd.DataFrame( + [ + (i, t, 3 if i < 2 else 0, (i + 1) * t + (i < 2 and t >= 3)) + for i in range(4) + for t in dates + ], + columns=["id", "t", "g", "y"], + ) + + +def test_two_date_oracle_covariance_is_known_difference_of_means_variance(): + system = study().sparse_system(balanced_dates((2, 3))) + assert system["pairs"] == [(2, 3)] + np.testing.assert_allclose(system["omega"], [[2]]) + np.testing.assert_allclose(system["loadings"], [[1], [1]]) + assert np.allclose(system["error_loadings"].reshape(4, 2, 1).sum(axis=1), 0) + + +def test_oracle_covariance_keeps_shared_endpoint_signs_and_redundant_links(): + system = study().sparse_system(balanced_dates((1, 2, 3))) + np.testing.assert_allclose(system["omega"], [[2, 1, -1], [1, 2, 1], [-1, 1, 2]]) + assert np.linalg.matrix_rank(system["omega"]) == 2 + np.testing.assert_allclose(system["oracle_variances"], [2, 2]) + + +@pytest.mark.parametrize("n,k,c", [(400, 0, 19), (400, 1, 19), (400, 2, 19), (4000, 50, 190)]) +def test_independent_linear_maps_match_native_points_and_hc0_with_sparse_support(n, k, c): + data, _ = generate_sparse_data(n, k, 123, c) + system = study().sparse_system(data) + out = study().evaluate(data, system) + assert out["native_discrepancy"] < 1e-11 + assert out["native_variance_discrepancy"] < 1e-11 + assert out["estimates"][0] == out["estimates"][1] + assert out["estimates"][2] == out["estimates"][3] + assert np.isfinite(out["ses"]).all() + np.testing.assert_allclose(system["metrics"][1], 2 / np.diag(system["omega"])) + other = data.assign(y=data.y * 3 + 100) + np.testing.assert_allclose(study().sparse_system(other)["loadings"], system["loadings"]) + + +def test_unsupported_target_is_not_repaired_by_pseudoinverse(): + data = balanced_dates((1, 2, 3)) + data.loc[data.t == 3, "y"] = np.nan + with pytest.raises(ValueError, match="unsupported"): + study().sparse_system(data) + + +def test_grid_separates_fixed_counts_from_proportional_support(): + grid = study().cases() + assert len(grid) == 10 + assert { + (c["n"], c["sparse_treated"], c["sparse_controls"]) + for c in grid + if c["regime"] == "growing" + } == {(4000, 20, 190), (4000, 50, 190)} + + +def test_small_run_preserves_pairing_and_reports_native_replay_and_failures(): + report = study().run_case(study().cases()[2], 3, 123, 2) + assert report["native_max_abs_error"] < 1e-11 + assert report["native_variance_max_abs_error"] < 1e-11 + assert report["failures"] == {} + assert len(report["variants"]) == 4 + assert all(v["replications"] == 3 and v["failures"] == 0 for v in report["variants"].values()) + assert report["oracle_variance_ratio_design_to_identity"] < 1 + + +def test_cli_protects_existing_and_historical_evidence(tmp_path, monkeypatch): + output = tmp_path / "evidence.json" + output.write_text('{"keep": true}') + monkeypatch.setattr(sys, "argv", ["diagnostic", "--output", str(output)]) + with pytest.raises(SystemExit) as exc: + study().main() + assert exc.value.code == 2 + assert json.loads(output.read_text()) == {"keep": True} + + +def test_unsupported_replications_remain_in_unconditional_coverage(): + report = study().run_case(dict(n=2, sparse_treated=0, sparse_controls=0), 3, 123, 0) + assert sum(report["failures"].values()) == 3 + assert report["oracle_variance_ratio_design_to_identity"] is None + assert all(v["failures"] == 3 and v["coverage"] == 0 for v in report["variants"].values()) + + +def test_cli_completes_small_study_and_binds_exact_source_hashes(tmp_path, monkeypatch): + import hashlib + from pathlib import Path + + output = tmp_path / "diagnostic.json" + monkeypatch.setattr( + sys, + "argv", + ["diagnostic", "--replications", "2", "--workers", "1", "--output", str(output)], + ) + assert study().main() == 0 + report = json.loads(output.read_text()) + assert report["status"] == "complete" + assert not report["statistically_qualified"] + assert len(report["results"]) == 10 + root = Path(study().__file__).resolve().parents[2] + assert all( + hashlib.sha256((root / name).read_bytes()).hexdigest() == digest + for name, digest in report["source_hashes"].items() + ) + + +def test_design_weighting_cannot_change_balanced_equal_count_recovery(): + data = balanced_dates((1, 2, 3, 4)) + system = study().sparse_system(data) + assert np.ptp(system["metrics"][1]) == 0 + np.testing.assert_allclose(system["loadings"][0], system["loadings"][1], atol=1e-14) + result = study().evaluate(data, system) + np.testing.assert_allclose(result["estimates"][:2], result["estimates"][2:], atol=1e-14) + np.testing.assert_allclose(result["ses"][:2], result["ses"][2:], atol=1e-14) + + +def test_native_replay_cannot_accept_unavailable_target_as_a_nan_comparison(): + data = balanced_dates((1, 2, 3, 4)) + system = study().sparse_system(data) + changed = data.copy() + changed.loc[changed.t == 3, "y"] = np.nan + with pytest.raises(ValueError, match="native estimate or covariance"): + study().evaluate(changed, system) diff --git a/tests/test_chained_did_validation.py b/tests/test_chained_did_validation.py new file mode 100644 index 000000000..b100054f9 --- /dev/null +++ b/tests/test_chained_did_validation.py @@ -0,0 +1,134 @@ +"""Small checks for the independent qualification harness, not coverage evidence.""" + +import importlib +import importlib.util + +import numpy as np +import pandas as pd +import pytest + + +def harness(): + name = "benchmarks.python.validate_chained_did" + assert importlib.util.find_spec(name) is not None, "Qualification harness must exist" + return importlib.import_module(name) + + +def test_dgp_keeps_original_registry_and_has_reproducible_analytic_targets(): + data, truth = harness().generate_data(200, "rotating", 42) + again, second_truth = harness().generate_data(200, "rotating", 42) + pd.testing.assert_frame_equal(data, again) + assert truth == second_truth + assert data.id.nunique() == 200 + assert data.groupby("id").y.count().eq(2).all() + assert 1 < truth["overall"] < 4 + assert 2 < truth["g2_t3"] < 3 + assert 3 < truth["g3_t4"] < 4 + _, null_truth = harness().generate_data(200, "rotating", 42, null=True) + assert set(null_truth.values()) == {0} + + +def test_summary_counts_failures_in_coverage_and_reports_monte_carlo_uncertainty(): + result = harness().summarize(np.array([0.0, 0.0, np.nan]), np.array([1.0, 1.0, np.nan]), 0) + assert result["replications"] == 3 + assert result["failures"] == 1 + assert result["coverage"] == pytest.approx(2 / 3) + assert result["coverage_given_valid"] == 1 + assert result["coverage_mcse"] == pytest.approx(np.sqrt((2 / 3) * (1 / 3) / 3)) + assert result["rejection_mcse"] == 0 + assert result["nominal_binomial_mcse"] == pytest.approx(np.sqrt(0.95 * 0.05 / 3)) + + +def test_unknown_observation_design_is_rejected(): + with pytest.raises(ValueError): + harness().generate_data(200, "unknown", 42) + + +@pytest.mark.parametrize("sparse_treated", [0, 1, 2, 5, 20, 50]) +def test_sparse_design_preserves_counts_and_analytic_target(sparse_treated): + data, truth = harness().generate_sparse_data(400, sparse_treated, 123) + other, _ = harness().generate_sparse_data(400, sparse_treated, 123) + pd.testing.assert_frame_equal(data, other) + assert truth == {"g3_t3": 1.0} + assert data.id.nunique() == 400 + at_one = data.loc[data.t == 1].groupby("g").id.nunique() + assert at_one.get(3, 0) == sparse_treated + assert at_one[0] == 19 + assert data.loc[data.t == 2].id.nunique() == 400 + assert data.loc[data.t == 3].id.nunique() == 400 + + +def test_sparse_runner_records_pair_exclusions_and_unconditional_inference_metrics(): + case = dict(n=100, sparse_treated=1, sparse_controls=19, violation="none") + record = harness()._run_case(case, 3, 123, 0) + assert record["failures"] == {} + assert record["targets"]["g3_t3"]["replications"] == 3 + assert record["targets"]["g3_t3"]["truth"] == 1 + assert record["pair_diagnostics"]["replications_with_singleton_exclusions"] == 3 + assert record["pair_diagnostics"]["replications_with_low_support"] == 0 + + +@pytest.mark.parametrize( + "kwargs", [{"n": 3}, {"sparse_treated": -1}, {"sparse_treated": 51}, {"sparse_controls": 51}] +) +def test_sparse_design_rejects_invalid_arm_counts(kwargs): + params = dict(n=100, sparse_treated=1, sparse_controls=19, seed=123) + params.update(kwargs) + with pytest.raises(ValueError, match="counts"): + harness().generate_sparse_data(**params) + + +def test_sparse_cli_records_completed_manifest_and_refuses_overwrite(tmp_path, monkeypatch): + import json + import shutil + import sys + from pathlib import Path + + repo = Path(harness().__file__).resolve().parents[2] + for name in ( + "diff_diff/chained_did.py", + "diff_diff/chained_did_results.py", + "benchmarks/python/validate_chained_did.py", + ): + target = tmp_path / name + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(repo / name, target) + monkeypatch.chdir(tmp_path) + destination = tmp_path / "sparse.json" + args = [ + "validate_chained_did", + "--design", + "sparse", + "--replications", + "2", + "--workers", + "1", + "--output", + str(destination), + ] + monkeypatch.setattr(sys, "argv", args) + assert harness().main() == 0 + record = json.loads(destination.read_text()) + assert record["status"] == "complete" and record["scope"] == "pilot" + assert record["design"] == "sparse" + assert record["source_commit"] is None + assert record["source_uncommitted"] is None + assert len(record["results"]) == 6 + before = destination.read_bytes() + with pytest.raises(SystemExit) as error: + harness().main() + assert error.value.code == 2 + assert destination.read_bytes() == before + + +def test_qualification_cli_protects_frozen_evidence(monkeypatch): + import sys + from pathlib import Path + + frozen = Path(harness().__file__).resolve().parents[1] / "data/chained_did/qualification.json" + before = frozen.read_bytes() + monkeypatch.setattr(sys, "argv", ["validate_chained_did", "--output", str(frozen)]) + with pytest.raises(SystemExit) as error: + harness().main() + assert error.value.code == 2 + assert frozen.read_bytes() == before diff --git a/tests/test_doc_snippets.py b/tests/test_doc_snippets.py index 45a32cf4e..5d9d7c165 100644 --- a/tests/test_doc_snippets.py +++ b/tests/test_doc_snippets.py @@ -569,7 +569,10 @@ def test_lwdid_dr_examples_exercise_dr_path(): fitted = [ v for v in ns.values() - if hasattr(v, "att") and hasattr(v, "se") and np.isscalar(getattr(v, "att")) + if not isinstance(v, type) + and hasattr(v, "att") + and hasattr(v, "se") + and np.isscalar(getattr(v, "att")) ] assert fitted, f"api/lwdid.rst dr block{idx} produced no fitted results" for res in fitted: diff --git a/tests/test_methodology_chained_did.py b/tests/test_methodology_chained_did.py new file mode 100644 index 000000000..735aeb878 --- /dev/null +++ b/tests/test_methodology_chained_did.py @@ -0,0 +1,232 @@ +"""Independent algebraic checks of chained DiD estimating equations.""" + +import numpy as np +import pandas as pd +import pytest +from scipy.optimize import root +from scipy.special import expit + +from diff_diff import ChainedDiD +from diff_diff.bootstrap_chunking import iter_weight_blocks + + +def binary_covariate_panel(): + rows, changes, treated, xs = [], [], [], [] + for g, x, count in ((1, 0, 12), (1, 1, 24), (0, 0, 30), (0, 1, 18)): + for noise in np.linspace(-1, 1, count): + uid = len(changes) + change = (5 + 2 * x if g else 1 + 3 * x) + noise + rows.extend([(uid, 0, g, uid / 10, x), (uid, 1, g, uid / 10 + change, x)]) + changes.append(change) + treated.append(bool(g)) + xs.append(x) + return ( + pd.DataFrame(rows, columns=["id", "t", "g", "y", "x"]), + np.array(changes), + np.array(treated), + np.array(xs), + ) + + +def test_binary_saturated_ipw_matches_stratum_formula_and_complete_influence(): + data, delta, treated, x = binary_covariate_panel() + result = ChainedDiD().fit(data, "y", "id", "t", "g", covariates=["x"]) + nt = treated.sum() + control_means = np.array([delta[(~treated) & (x == k)].mean() for k in (0, 1)]) + expected = delta[treated].mean() - control_means[x[treated]].mean() + q = np.empty(len(delta)) + q[treated] = (delta[treated] - control_means[x[treated]] - expected) / nt + for k in (0, 1): + controls = (~treated) & (x == k) + q[controls] = ( + -(treated & (x == k)).sum() + / (nt * controls.sum()) + * (delta[controls] - control_means[k]) + ) + assert result.att == pytest.approx(expected, abs=1e-10) + np.testing.assert_allclose(result.influence_functions[:, 1], q, atol=1e-10) + assert result.se**2 == pytest.approx(q @ q, abs=1e-11) + + +def test_continuous_covariates_match_independent_numerical_score_jacobian(): + rng = np.random.default_rng(86) + x = rng.normal(size=(180, 2)) + z = np.column_stack([np.ones(len(x)), x]) + treated = rng.random(len(x)) < expit(z @ np.array([0.2, 0.4, -0.3])) + delta = 1 + x[:, 0] + x[:, 1] ** 2 + treated + rng.normal(size=len(x)) + data = pd.DataFrame( + { + "id": np.repeat(np.arange(len(x)), 2), + "t": np.tile([0, 1], len(x)), + "g": np.repeat(treated.astype(int), 2), + "y": np.column_stack([np.zeros(len(x)), delta]).ravel(), + "x1": np.repeat(x[:, 0], 2), + "x2": np.repeat(x[:, 1], 2), + } + ) + fitted = root(lambda beta: z.T @ (treated - expit(z @ beta)), np.zeros(3)) + assert fitted.success + p = expit(z @ fitted.x) + w = (~treated) * p / (1 - p) + mt, mc = delta[treated].mean(), w @ delta / w.sum() + parameters = np.r_[fitted.x, mt, mc] + + def scores(a): + ps = expit(z @ a[:3]) + return np.column_stack( + [ + z * (treated - ps)[:, None], + treated * (delta - a[3]), + (~treated) * ps / (1 - ps) * (delta - a[4]), + ] + ) + + step = 1e-5 + perturbations = np.eye(5) * step + jacobian = np.column_stack( + [ + (scores(parameters + d).sum(axis=0) - scores(parameters - d).sum(axis=0)) / (2 * step) + for d in perturbations + ] + ) + independent = -np.linalg.solve(jacobian, scores(parameters).T).T + q = independent[:, 3] - independent[:, 4] + result = ChainedDiD().fit(data, "y", "id", "t", "g", covariates=["x1", "x2"]) + assert result.att == pytest.approx(mt - mc, abs=1e-8) + np.testing.assert_allclose(result.influence_functions[:, 1], q, atol=1e-8) + + +def test_indirectly_supported_earlier_nuisance_is_not_a_direct_pre_diagnostic(): + from tests.test_chained_did import rotating_panel + + data = rotating_panel() + data["g"] = data.g.map({0: 0, 2: 3}) + result = ChainedDiD().fit(data, "y", "id", "t", "g") + assert result.group_time_effects[3, 3].effect == pytest.approx(4) + assert result.group_time_effects[3, 0].skip_reason == "unsupported_direct_pre_comparison" + assert np.isnan(result.group_time_effects[3, 0].effect) + + +def test_covariate_affine_scaling_does_not_change_fitted_comparison_or_inference(): + data, *_ = binary_covariate_panel() + a = ChainedDiD().fit(data, "y", "id", "t", "g", covariates=["x"]) + data["x"] = 1e8 + 1e4 * data.x + b = ChainedDiD().fit(data, "y", "id", "t", "g", covariates=["x"]) + assert a.att == pytest.approx(b.att, abs=1e-9) + np.testing.assert_allclose(a.vcov, b.vcov, atol=1e-10) + + +@pytest.mark.parametrize("fault", ["separation", "constant", "missing", "changing"]) +def test_failed_or_invalid_covariate_models_are_not_replaced_with_unconditional(fault): + data, *_ = binary_covariate_panel() + if fault == "separation": + data["x"] = data.g + elif fault == "constant": + data["x"] = 1 + elif fault == "missing": + data.loc[0, "x"] = np.nan + else: + data.loc[0, "x"] = 4 + with pytest.raises(ValueError): + ChainedDiD().fit(data, "y", "id", "t", "g", covariates=["x"]) + + +def test_covariance_retains_shared_controls_across_cohorts(): + rows = [] + for g in (0, 1, 2): + for j, noise in enumerate((-1.5, -0.5, 0.5, 1.5)): + for t in (0, 1, 2): + rows.append((f"{g}-{j}", t, g, noise * t + (2 * g if g and t >= g else 0))) + result = ChainedDiD().fit( + pd.DataFrame(rows, columns=["id", "t", "g", "y"]), "y", "id", "t", "g" + ) + j, k = result.vcov_index.index((1, 1)), result.vcov_index.index((2, 2)) + assert result.vcov[j, k] == pytest.approx(5 / 16) + + +def test_registry_only_units_change_aggregation_masses_and_share_influence(): + rows = [] + for g in (0, 1, 2): + for j, noise in enumerate((-1.5, -0.5, 0.5, 1.5)): + for t in (0, 1, 2): + effect = (2 * t if g == 1 else 10) if g and t >= g else 0 + rows.append((f"{g}-{j}", t, g, noise * t + effect)) + for j in range(4): + rows.append((f"registry-{j}", 0, 2, np.nan if j < 2 else 0)) + data = pd.DataFrame(rows, columns=["id", "t", "g", "y"]) + result = ChainedDiD().fit(data, "y", "id", "t", "g") + assert result.n_units == 16 + assert result.att == pytest.approx(6.5) + event = result.aggregate("event_study") + assert event.att[event.event_time == 0][0] == pytest.approx(22 / 3) + assert np.all(result.influence_functions[-4:] == 0) + _, contribution, _ = result._combine(result._post_keys()) + assert np.all(np.abs(contribution[-4:]) > 0) + + +def test_joint_cluster_multiplier_reproduces_the_declared_covariance(): + data, *_ = binary_covariate_panel() + data["cluster"] = data.id // 4 + result = ChainedDiD(cluster="cluster").fit(data, "y", "id", "t", "g", covariates=["x"]) + q = result.influence_functions + labels = data.drop_duplicates("id").cluster.to_numpy() + clusters = np.unique(labels) + sums = np.array([q[labels == c].sum(axis=0) for c in clusters]) + expected = len(clusters) / (len(clusters) - 1) * sums.T @ sums + np.testing.assert_allclose(result.vcov, expected) + draws = np.concatenate( + [ + block + for _, block in iter_weight_blocks( + 30000, len(clusters), "rademacher", np.random.default_rng(4) + ) + ] + ) + perturbations = np.sqrt(len(clusters) / (len(clusters) - 1)) * draws @ sums + np.testing.assert_allclose(np.cov(perturbations, rowvar=False), expected, rtol=0.03, atol=1e-12) + + +def test_two_date_unconditional_matches_hand_calculation_and_existing_estimator(): + from diff_diff import CallawaySantAnna + + data, delta, treated, _ = binary_covariate_panel() + result = ChainedDiD().fit(data, "y", "id", "t", "g") + expected = delta[treated].mean() - delta[~treated].mean() + variance = delta[treated].var() / treated.sum() + delta[~treated].var() / (~treated).sum() + existing = CallawaySantAnna(n_bootstrap=0, estimation_method="ipw").fit( + data, "y", "id", "t", "g" + ) + assert result.att == pytest.approx(expected) + assert result.se**2 == pytest.approx(variance) + assert result.att == pytest.approx(existing.att) + assert result.se == pytest.approx(existing.se) + + +def test_overidentified_catalogue_matches_independent_graph_laplacian_solution(): + rows = [] + pair_means = {(0, 1): 2.0, (0, 2): 6.0, (0, 3): 8.0, (1, 2): 3.0, (1, 3): 7.0, (2, 3): 5.0} + for (s, t), mean in pair_means.items(): + for g in (0, 2): + for j, noise in enumerate((-1.0, 0.0, 1.0)): + uid = f"{s}-{t}-{g}-{j}" + rows.extend([(uid, s, g, 0.0), (uid, t, g, noise + (mean if g else 0))]) + data = pd.DataFrame(rows, columns=["id", "t", "g", "y"]) + # Solve the full-rank reference-anchored graph Laplacian independently of SVD. + incidence = np.array([[-1, 0, 0], [-1, 1, 0], [-1, 0, 1], [0, 1, 0], [0, 0, 1], [0, -1, 1]]) + loading = np.linalg.solve(incidence.T @ incidence, incidence.T) + expected = loading @ np.array(list(pair_means.values())) + result = ChainedDiD().fit(data, "y", "id", "t", "g") + assert result.group_time_effects[2, 2].effect == pytest.approx(expected[1]) + assert result.group_time_effects[2, 3].effect == pytest.approx(expected[2]) + assert len(result.diagnostics["catalogue"]) == 6 + q = np.zeros((data.id.nunique(), 6)) + units = data.id.drop_duplicates().tolist() + for k, (s, t) in enumerate(pair_means): + for g in (0, 2): + for j, noise in enumerate((-1.0, 0.0, 1.0)): + q[units.index(f"{s}-{t}-{g}-{j}"), k] = (1 if g else -1) * noise / 3 + expected_q = q @ loading.T + columns = [result.vcov_index.index((2, t)) for t in (2, 3)] + np.testing.assert_allclose( + result.vcov[np.ix_(columns, columns)], (expected_q.T @ expected_q)[1:, 1:], atol=1e-12 + ) diff --git a/tests/test_naming_guard.py b/tests/test_naming_guard.py index 5b2880780..0bc3c83cb 100644 --- a/tests/test_naming_guard.py +++ b/tests/test_naming_guard.py @@ -511,6 +511,7 @@ def _build_rowed_index(): "BusinessReport.__init__[time]", "CallawaySantAnna.diagnose_propensity[time]", "CallawaySantAnna.fit[time]", + "ChainedDiD.fit[time]", "ChaisemartinDHaultfoeuille.fit[time]", "DMLDiD.fit[time]", "DurationDiD.fit[time]", @@ -1004,6 +1005,10 @@ def _token_family_code_refs(tok): ) CONSUMER_ALLOWLIST = { + ("groups", "docs/methodology/chained-did-validation.md"): ( + "historical baseline-failure evidence quoting DMLDiDResults.groups, " + "not a reader of dCDH's removed unit-id field (M-114)" + ), # ---- 4.0 cut, PR 2: readers that survive the 59 shim removals on purpose ---- # (a) KEPT `__setstate__` legacy-key migrations - a quoted old key is how a # 3.x pickle is recognised; spec section 5 keeps them through 4.0. diff --git a/tests/test_v4_cluster_policy.py b/tests/test_v4_cluster_policy.py index b3df0f7b3..f01d85eca 100644 --- a/tests/test_v4_cluster_policy.py +++ b/tests/test_v4_cluster_policy.py @@ -178,9 +178,10 @@ def _cluster_roster(): CTOR_KW = {"SpilloverDiD": {"rings": [0.0, D_BAR_KM]}, "LPDiD": LP_CTOR} -def test_roster_is_the_sixteen_documented_classes(): +def test_roster_is_the_seventeen_documented_classes(): assert {c.__name__ for c in ROSTER} == { "CallawaySantAnna", + "ChainedDiD", "ChaisemartinDHaultfoeuille", "DMLDiD", "DifferenceInDifferences", @@ -281,6 +282,7 @@ def _fit_recipes(panel, spill_panel): "ImputationDiD": ({}, panel, SA_KW), "TwoStageDiD": ({}, panel, SA_KW), "CallawaySantAnna": ({}, panel, SA_KW), + "ChainedDiD": ({}, panel, SA_KW), "EfficientDiD": ({}, panel, SA_KW), "LPDiD": (LP_CTOR, panel, LP_KW), "SpilloverDiD": ( diff --git a/tests/test_v4_diagnostic_family_docs.py b/tests/test_v4_diagnostic_family_docs.py index 60e344c48..20b13c11d 100644 --- a/tests/test_v4_diagnostic_family_docs.py +++ b/tests/test_v4_diagnostic_family_docs.py @@ -5,8 +5,8 @@ claim that the library ships ESTIMATORS plus a DIAGNOSTIC FAMILY, on every catalogue that used to file ``BaconDecomposition`` as an estimator: -1. ``docs/api/index.rst`` - the Estimators autosummary is exactly the 23 - spec-3.1 estimator classes; ``BaconDecomposition``, ``RDPlot`` and +1. ``docs/api/index.rst`` - the Estimators autosummary contains the 23 + spec-3.1 estimator classes plus experimental ChainedDiD; ``BaconDecomposition``, ``RDPlot`` and ``RDDensityTest`` are catalogued under Diagnostics; the ``bacon`` module page sits in the Diagnostics & Inference toctree; no stub is referenced from two ``:toctree:`` autosummary blocks (the ``-W`` build would warn). @@ -56,12 +56,13 @@ THIS_TEST_REF = "tests/test_v4_diagnostic_family_docs.py" -# The 23 estimator classes of spec section 3.1 (27 -> 24 at 4.0, minus Bacon). +# The 23 cut estimators of spec section 3.1, plus experimental ChainedDiD. ESTIMATOR_ROSTER = { "DifferenceInDifferences", "TwoWayFixedEffects", "SyntheticDiD", "CallawaySantAnna", + "ChainedDiD", "ChaisemartinDHaultfoeuille", "SunAbraham", "ImputationDiD", @@ -193,16 +194,16 @@ def _ids(names): # --------------------------------------------------------------------------- -def test_api_estimators_autosummary_is_exactly_the_23_estimators(): +def test_api_estimators_autosummary_matches_the_current_roster(): body = _section_body(API_INDEX.read_text(), "Estimators") entries = _autosummary_entries(body) counts = _identity_counter(entries) assert max(counts.values()) == 1, "duplicate entry in the Estimators autosummary" assert set(counts) == _ids( ESTIMATOR_ROSTER - ), f"Estimators autosummary drift: {sorted(entries)} vs the spec-3.1 roster" + ), f"Estimators autosummary drift: {sorted(entries)} vs the current estimator roster" assert not (set(counts) & _ids(DIAGNOSTIC_ENTRY_POINTS)) - assert len(entries) == 23 + assert len(entries) == len(ESTIMATOR_ROSTER) def test_api_diagnostics_autosummary_catalogues_bacon_rdplot_and_density_test(): @@ -251,7 +252,7 @@ def test_api_estimators_intro_no_longer_names_the_bacon_diagnostic(): def test_homepage_estimator_table_has_no_bacon_row(): counts = _estimator_table_counts(_section_body(HOMEPAGE.read_text(), "Supported Estimators")) assert "BaconDecomposition" not in counts - assert sum(counts.values()) == 23 + assert sum(counts.values()) == len(ESTIMATOR_ROSTER) def test_homepage_diagnostics_table_lists_the_family_entry_points(): @@ -304,7 +305,7 @@ def test_readme_files_bacon_under_diagnostics_not_estimators(): assert "api/bacon.html" in diagnostics assert "BaconDecomposition" in diagnostics api_count = len(_autosummary_entries(_section_body(API_INDEX.read_text(), "Estimators"))) - assert _bullet_count(estimators) == api_count == 23 + assert _bullet_count(estimators) == api_count == len(ESTIMATOR_ROSTER) def test_llms_txt_files_bacon_under_diagnostics_not_estimators(): @@ -315,7 +316,7 @@ def test_llms_txt_files_bacon_under_diagnostics_not_estimators(): assert "api/bacon.html" not in estimators assert "api/bacon.html" in diagnostics assert "BaconDecomposition" in diagnostics - assert _bullet_count(estimators) == 23 + assert _bullet_count(estimators) == len(ESTIMATOR_ROSTER) # ---------------------------------------------------------------------------