diff --git a/.claude/commands/docs-check.md b/.claude/commands/docs-check.md index 2162c1e67..b86a12fb5 100644 --- a/.claude/commands/docs-check.md +++ b/.claude/commands/docs-check.md @@ -52,7 +52,7 @@ The following estimators/features MUST have documentation: | WooldridgeDiD | "WooldridgeDiD" | "WooldridgeDiD" | wooldridge_etwfe.rst | "Multi-Period and Staggered" | | TROP | "TROP" | "TROP" | trop.rst | "Triply Robust Panel" | | HonestDiD | n/a (in `## Diagnostics`) | n/a (in `## Diagnostics`) | honest_did.rst | "Honest DiD" | -| BaconDecomposition | "BaconDecomposition" | "BaconDecomposition" | bacon.rst | "Multi-Period and Staggered" | +| BaconDecomposition | n/a (in `## Diagnostics and Sensitivity Analysis`) | n/a (in `## Diagnostics & Sensitivity`) | bacon.rst | "Multi-Period and Staggered" | ### Supporting Features (require llms.txt mention + API docs; README mention only if landing-page-relevant) @@ -117,11 +117,11 @@ Determine which checks to run based on `$ARGUMENTS`. For each estimator/diagnostic in the table above: 1. Read `diff_diff/guides/llms.txt` and verify the name appears under the right section: - - **Estimators** (e.g. CallawaySantAnna, SunAbraham, TROP, BaconDecomposition): under `## Estimators` - - **Diagnostics-class** (HonestDiD, and any future diagnostic-only entries): under `## Diagnostics and Sensitivity Analysis` + - **Estimators** (e.g. CallawaySantAnna, SunAbraham, TROP): under `## Estimators` + - **Diagnostics-class** (HonestDiD, BaconDecomposition, RDPlot, RDDensityTest, and any future diagnostic-only entries): under `## Diagnostics and Sensitivity Analysis` 2. Read `README.md` and verify the name appears in the matching flat catalog: - **Estimators**: in the `## Estimators` section - - **Diagnostics-class** (HonestDiD): in the `## Diagnostics & Sensitivity` section + - **Diagnostics-class** (HonestDiD, BaconDecomposition, RDPlot, RDDensityTest): in the `## Diagnostics & Sensitivity` section 3. Report missing entries ```bash diff --git a/.github/workflows/docs-tests.yml b/.github/workflows/docs-tests.yml index bcf86feb6..6bbfc6b7e 100644 --- a/.github/workflows/docs-tests.yml +++ b/.github/workflows/docs-tests.yml @@ -11,6 +11,11 @@ on: - 'tests/test_docs_ia.py' - 'tests/test_v4_matrix.py' - 'tests/test_naming_guard.py' + # Diagnostic-family roster guard (M-090): README-only diffs otherwise + # run no lane that executes it - rust-test.yml fires on tests/** and + # diff_diff/** only. + - 'tests/test_v4_diagnostic_family_docs.py' + - 'README.md' # Changelog-fragment guard: the fragment store, the compiler, and # CHANGELOG.md itself trigger here (their ONLY CI lane - rust-test.yml # deliberately does not fire on fragment-only diffs). @@ -48,6 +53,11 @@ on: - 'tests/test_docs_ia.py' - 'tests/test_v4_matrix.py' - 'tests/test_naming_guard.py' + # Diagnostic-family roster guard (M-090): README-only diffs otherwise + # run no lane that executes it - rust-test.yml fires on tests/** and + # diff_diff/** only. + - 'tests/test_v4_diagnostic_family_docs.py' + - 'README.md' - 'tests/test_changelog_fragments.py' - 'changelog.d/**' - 'CHANGELOG.md' @@ -129,6 +139,13 @@ jobs: # the main test matrix does not trigger on docs-only diffs. run: PYTHONPATH=. DIFF_DIFF_BACKEND=python pytest tests/test_v4_matrix.py -v + - name: Run diagnostic-family roster guard + # Pins the M-090 estimators/diagnostics split on README, llms.txt, + # llms-full.txt, docs/index.rst, docs/api/index.rst and the REGISTRY + # archive framing; docs-only and README-only diffs do not trigger the + # main matrix. + run: PYTHONPATH=. DIFF_DIFF_BACKEND=python pytest tests/test_v4_diagnostic_family_docs.py -v + - name: Run naming-completeness guard # The section-8/section-9 naming guard (spec in the module docstring): # public-surface vocabulary sweep, phase-table agreement in both diff --git a/README.md b/README.md index 11b5b5020..2fe9d1258 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,6 @@ Full guide: `diff_diff.get_llm_guide("practitioner")`. - [LWDiD](https://diff-diff.readthedocs.io/en/stable/api/lwdid.html) - Lee & Wooldridge (2025, 2026) rolling-transformation DiD: unit-specific demean/detrend converts panel to cross-section, staggered adoption, `estimation_method` in `reg`/`ipw`/`dr`/`psm` (the papers' RA/IPW/IPWRA plus propensity-score matching), exact small-N inference on the classical collapsed regression - [DMLDiD](https://diff-diff.readthedocs.io/en/stable/api/dml_did.html) - Chang (2020) double/debiased machine learning DiD: staggered ATT(g,t) with cross-fitted ML nuisance learners (DML2) and Neyman-orthogonal scores, for flexible/high-dimensional covariate adjustment under conditional parallel trends; panel or declared repeated cross sections (`panel=False`); survey/cluster support on both lanes (bad-control lane: panel only, `cluster=` only); Caetano, Callaway, Payne & Sant'Anna (2026) bad-control score via `fit(bad_control=, bad_control_covariates=)` - [DurationDiD](https://diff-diff.readthedocs.io/en/stable/api/duration_did.html) - Deaner & Ku (2026) causal duration DiD for a binary absorbing outcome (spell ended) in a two-group common-timing design: restricts the groups' untreated hazards (`method="cd"` additive gap or `method="ph"` ratio) instead of outcome levels, imputes the treated counterfactual survival, reports the per-date absorption ATT with whole-individual bootstrap pointwise and simultaneous bands plus a fixed-anchor pre-treatment specification test -- [BaconDecomposition](https://diff-diff.readthedocs.io/en/stable/api/bacon.html) - Goodman-Bacon (2021) decomposition for diagnosing TWFE bias in staggered settings ## Diagnostics & Sensitivity @@ -131,6 +130,7 @@ Full guide: `diff_diff.get_llm_guide("practitioner")`. - [Parallel Trends Testing](https://diff-diff.readthedocs.io/en/stable/api/diagnostics.html) - simple and Wasserstein-robust parallel trends tests, equivalence testing (TOST) - [Placebo Tests](https://diff-diff.readthedocs.io/en/stable/api/diagnostics.html) - placebo timing, group, permutation, leave-one-out - [TWFE Weight Diagnostics](https://diff-diff.readthedocs.io/en/stable/api/twfe_weights.html) - Baker et al. (2025) implicit weights a TWFE regression places on each ATT(g,t), against the ATT^O / ATT^simple targets, with the pre-period contribution. Ports Callaway's `twfeweights` (MIT) +- [Bacon Decomposition](https://diff-diff.readthedocs.io/en/stable/api/bacon.html) - Goodman-Bacon (2021) decomposition of the TWFE estimate into its 2x2 comparisons and weights, for diagnosing TWFE bias in staggered settings (`BaconDecomposition`, alias `Bacon`) - [Honest DiD](https://diff-diff.readthedocs.io/en/stable/api/honest_did.html) - Rambachan & Roth (2023) sensitivity analysis: robust CI under PT violations, breakdown values - [Pre-Trends Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/pretrends.html) - Roth (2022) minimum detectable violation and power curves - [Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/power.html) - analytical and simulation-based MDE, sample size, power curves for study design diff --git a/changelog.d/20261101-v4-pr9-m090-docs-reorg.md b/changelog.d/20261101-v4-pr9-m090-docs-reorg.md new file mode 100644 index 000000000..5621302a9 --- /dev/null +++ b/changelog.d/20261101-v4-pr9-m090-docs-reorg.md @@ -0,0 +1,9 @@ +### Documentation + +- `BaconDecomposition` is re-homed into the diagnostics family ([M-090], item 9 of the 4.0 cut): it leaves the estimator catalogues - README `## Estimators`, `llms.txt` `## Estimators`, the `llms-full.txt` Estimators section and its alias line, the API-reference Estimators autosummary and module toctree, and the docs homepage Supported Estimators table - for their Diagnostics groupings. `RDPlot` moves from the API-reference Visualization autosummary into Diagnostics; `RDPlot`, `RDDensityTest` and the TWFE weight diagnostics leave the `llms-full.txt` Estimators section. Docs-only: the import path, the `Bacon` alias, the results class and every number are unchanged. +- The docs homepage gains a "Diagnostics & Sensitivity" table (BaconDecomposition, RDPlot, RDDensityTest, HonestDiD, PreTrendsPower, PowerAnalysis, DiagnosticReport, the parallel-trends / placebo / TWFE-weight functions and the HAD pretest workflow) next to the estimator table; the API-reference Estimators and Diagnostics intros describe the split. +- `docs/methodology/REGISTRY.md` archive framing for the classes removed at 4.0: the MultiPeriodDiD and StaggeredTripleDifference sections and the QDiD class-spelling subsection open with an archive Note, the table of contents marks them *(archived)* and states the archived-sections convention, the cross-reference tables name `ChangesInChanges(method="qdid")` instead of a standalone `QDiD` estimator, and the "20-estimator count" wording is gone. The migration guide gains the [M-090] bullet. + +### Internal + +- Ledger row M-090 flips to `done` with `tests/test_v4_diagnostic_family_docs.py` as its `test_ref`; the test pins the estimators/diagnostics split on README, `llms.txt`, `llms-full.txt`, `docs/index.rst`, `docs/api/index.rst` and the REGISTRY archive framing, and runs in the docs-tests workflow (README-only diffs now trigger that lane). `docs/doc-deps.yaml` points the diagnostic modules at the new homepage table; the `/docs-check` classification table files Bacon as a diagnostics-class entry. diff --git a/diff_diff/guides/llms-full.txt b/diff_diff/guides/llms-full.txt index d178369f5..cd284a231 100644 --- a/diff_diff/guides/llms-full.txt +++ b/diff_diff/guides/llms-full.txt @@ -29,7 +29,7 @@ print(f"ATT: {results.att:.3f} (SE: {results.se:.3f})") - **sklearn-like API**: All estimators use `fit()` method, `get_params()`/`set_params()` for configuration. - **Formula interface**: Supports R-style formulas like `"outcome ~ treated * post"`. - **Results objects**: Rich dataclass containers with `summary()`, `to_dict()`, `to_dataframe()`. -- **Estimator aliases**: Short names available (e.g., `DiD`, `CS`, `SA`, `BJS`, `SDiD`, `TWFE`, `DDD`, `EDiD`, `SCM`, `Bacon`). The former `CDiD`/`Gardner`/`Stacked` aliases were removed in 4.0 - use `ContinuousDiD`/`TwoStageDiD`/`StackedDiD`. +- **Estimator aliases**: Short names available (e.g., `DiD`, `CS`, `SA`, `BJS`, `SDiD`, `TWFE`, `DDD`, `EDiD`, `SCM`); the diagnostic `BaconDecomposition` keeps its `Bacon` alias. The former `CDiD`/`Gardner`/`Stacked` aliases were removed in 4.0 - use `ContinuousDiD`/`TwoStageDiD`/`StackedDiD`. ## Practitioner Workflow (based on Baker et al. 2025) @@ -970,125 +970,6 @@ covariate-balance helper are documented follow-ups; missing rows are dropped WITH a warning (R drops silently); N < 20 falls back to full-range bandwidths exactly as rdrobust does (overriding manual h). -### RDPlot - -RD plot diagnostic (Calonico, Cattaneo & Titiunik 2015), parity-targeting R -rdrobust 4.0.0's `rdplot()`. NOT a treatment-effect estimator: builds the -standard exploratory RD plot - per-side global order-p polynomial fits -(default p=4, uniform kernel over each side's full range; deliberately -different defaults from estimation) plus binned local sample means with a -DATA-DRIVEN number of bins per side. All 8 `binselect` selectors = -{evenly (es) / quantile (qs) spaced} x {IMSE-optimal / mimicking variance -(mv)} x {spacings / polynomial-regression (pr) variance estimators}; the -result reports `J`, `J_IMSE`, `J_MV`, the implied scale `rscale = J/J_IMSE`, -and the CCT 2015 Supplement S.1 WIMSE weights (variance `1/(1+rscale^3)`, -bias `rscale^3/(1+rscale^3)`), matching R's `summary.rdplot`. - -```python -RDPlot( - cutoff: float = 0.0, # Known threshold of the running variable - p: int = 4, # Global polynomial order (R's rdplot default) - nbins: int | tuple | None = None, # Manual bins per side (overrides the selector) - binselect: str = "esmv", # es/espr/esmv/esmvpr/qs/qspr/qsmv/qsmvpr - scale: float | tuple | None = None, # Multiplies the selected J (fractional products take CCT 2015 Eq 2's ceiling; R crashes there) - kernel: str = "uniform", # Weighting for the global fit within h - h: float | tuple | None = None, # Global-fit window; default = full per-side range - support: tuple | None = None, # (lower, upper); only WIDENS the observed range - masspoints: str = "adjust", # >= 20% ties: warn + switch spacings binselect to its *pr sibling ("check" warns only; "off" disables) - ci: float | None = None, # Per-bin CI level for plotted error bars (CI columns are ALWAYS computed, at 95 when None) - covs_drop: bool = True, # Drop collinear covariates with a warning; False = strict error -) -``` - -**fit() parameters:** - -```python -rp.fit( - data: pd.DataFrame, - outcome: str, - running: str, - covariates: list[str] | None = None, # Covariate-adjusted plot (R's covs=, covs_eval="mean"); gammas exposed as covariate_coefficients -) -> RDPlotResult -``` - -The 3.8 names `outcome_col=` and `running_col=` were removed in 4.0 - use `outcome=` and `running=`. - -```python -from diff_diff import RDPlot - -rp = RDPlot(cutoff=0.0) -res = rp.fit(df, outcome="y", running="score") -res.J, res.J_IMSE, res.J_MV # selected / IMSE-optimal / mimicking-variance bins per side -res.rscale # implied scale; WIMSE weights via res.wimse_variance_weight / res.wimse_bias_weight -res.to_dataframe() # vars_bins: R-named per-bin columns (rdplot_mean_bin/mean_x/mean_y/min_bin/max_bin/se_y/N/ci_l/ci_r) -res.vars_poly # 500-point global-fit curve per side -print(res.summary()) # R summary.rdplot layout incl. implied scale + WIMSE weights -res.plot() # optional matplotlib rendering (matplotlib NOT a dependency; ImportError if absent) -``` - -Empty bins are dropped from vars_bins (R's tapply) while bin_avg/bin_med -cover all partition cells; single-observation bins get se = 0 (R's NA -> 0 -fixup); n < 20 raises outright (R behavior - no fallback); a side with -zero outcome variance pins its J to 1 with a warning and echoes R's Inf -J_IMSE. With empty LEFT-side bins, R's own rdplot_min_bin/rdplot_max_bin -come from mirrored grid slots and can disagree with rdplot_mean_bin - an -R quirk replicated for parity (rdplot_mean_bin is the reliable bin -identity). Sampling weights and R's subset= are documented seams (filter -the DataFrame instead). - -### RDDensityTest - -Manipulation (density-discontinuity) test diagnostic (Cattaneo, Jansson & -Ma 2020), parity-targeting R rddensity 3.0. NOT a treatment-effect -estimator: tests the null of NO discontinuity in the running variable's -density at the cutoff (McCrary's manipulation question) via boundary- -adaptive local polynomial density estimation - the density is the slope of -a kernel-weighted local polynomial regression of the rank-based EDF - with -ROBUST BIAS-CORRECTED inference (order q = p+1 estimators at order-p -MSE-optimal bandwidths). A significant test is evidence of sorting around -the cutoff; a null supports (never proves) the RD continuity assumption. -Inference assumes an i.i.d. sample with a smooth, positive density near -the cutoff on each side (CJM Assumption 1); clustered, survey-weighted, or -repeated-observation dependence is unsupported - there is no cluster= -surface. - -```python -RDDensityTest( - cutoff=0.0, - p=2, # density estimation order (1..7) - q=None, # bias-correction order; None -> p+1; q==p = conventional test - fitselect="unrestricted", # or "restricted" (shared CDF/higher derivatives; only the density jumps) - kernel="triangular", # / "epanechnikov" / "uniform" (R spellings accepted) - vcov_type="jackknife", # R's default; or "plugin" (asymptotic) - h=None, # manual bandwidth: scalar or (h_left, h_right) - bwselect="comb", # each / diff / sum / comb (median rule; restricted: min(diff, sum)) - masspoints="adjust", # RD-family surface: adjust (R TRUE) / check (warn only) / off - regularize=True, - n_local_min=None, # None -> 20+p+1; 0 disables the gate (R semantics, ceil applied) - n_unique_min=None, - report_all=False, # also report the conventional (order-p) test, R's all=TRUE -) - -test = RDDensityTest() -res = test.fit(df, running="score") -res.t_stat, res.p_value # robust bias-corrected manipulation test -res.f_left, res.f_right # per-side density estimates at the cutoff (whole-sample scale) -res.bandwidths # the rdbwdensity 4-row h-table (None on manual h) -res.to_dataframe() # tidy left/right/diff table (+ conventional rows with report_all) -print(res.summary()) # R-style block (Conventional row iff report_all=True) -res.to_dict() # JSON-serializable scalars (+ nested 'bandwidths' when estimated) -``` - -Repeated running-variable values are mass-point adjusted with a warning -(matching R massPoints=TRUE; "check" warns without adjusting, "off" is -silent). Degenerate designs fail loud: a side of (almost) only repeated -values, singular local fits, and degenerate selected bandwidths raise -ValueError (R silently returns numerically meaningless output there); -negative/zero/non-finite variance estimates degrade SEs and inference to -NaN with a warning, and finite NEGATIVE density estimates warn. No -confidence-interval fields, matching R's output surface (CIs belong to -rdplotdensity, a documented seam alongside the binomial windows test). - ### StackedDiD Stacked DiD estimator (Wing, Freedman & Hollingsworth 2024). Addresses TWFE bias with corrective Q-weights. @@ -1482,105 +1363,6 @@ results = trop.fit(data, outcome='outcome', treatment='treated', results.print_summary() ``` -### BaconDecomposition - -Goodman-Bacon (2021) decomposition of TWFE into 2x2 DiD comparisons. - -```python -BaconDecomposition( - weights: str = "approximate", # "approximate" or "exact" -) -``` - -**Alias:** `Bacon` - -**fit() parameters:** - -```python -bacon.fit( - data: pd.DataFrame, - outcome: str, - unit: str, - time: str, - first_treat: str, -) -> BaconDecompositionResults -``` - -**Usage:** - -```python -from diff_diff import BaconDecomposition, plot_bacon - -bacon = BaconDecomposition(weights="exact") -results = bacon.fit(data, outcome='outcome', unit='unit', - time='period', first_treat='first_treat') -results.print_summary() -plot_bacon(results) -``` - -### TWFE Weight Diagnostics - -What a TWFE regression implicitly weights on staggered data. Distinct from -`twowayfeweights` (dCDH), which weights (unit, time) cells: these weight -ATT(g,t) parameters. Ported from Brantly Callaway's `twfeweights` R package -(MIT); methodology Baker, Callaway, Cunningham, Goodman-Bacon & Sant'Anna -(2025). - -```python -attgt_weights( - results, # CallawaySantAnnaResults, or a (g,t) frame - type="twfe", # "twfe" | "overall" (ATT^O) | "simple" - data=None, unit=None, time=None, first_treat=None, # frame path only - weights=None, # frame path only; a fitted result uses its own survey weights - anticipation=None, # frame path only; the fit's own otherwise -) -> ATTGTWeightsResult - -decompose_twfe_weights( - data, # balanced long panel (it re-estimates) - outcome=, unit=, time=, first_treat=, - method="fwl", - covariates=None, - base_period="first_period", # or "gmin1" - balance_covariates=None, # enables result.covariate_balance() - weights=None, -) -> TWFEDecompositionResult - -plot_twfe_weights(result, kind="auto") # "weights" | "balance" -``` - -`type="twfe"` requires a fit with `base_period="universal"`, -`control_group="never_treated"` AND no covariates (R twfe_weights' three -restrictions); it raises otherwise. ATT^O and ATT^simple weights are -non-negative and sum to one, so comparing `implied_att` across the three -types shows what the TWFE specification costs. The result records its -`type=` as `.level` (`ATTGTWeightsResult.LEVELS` lists the values). - -Both entry points fail closed on input R never faced: NaN / `-inf` cohort -labels (never-treated is exactly `0` or `+inf`), duplicated or non-finite -ATT(g,t) cells, an incomplete group-time grid (`"twfe"` needs every cohort x -period cell, the CS estimands every post cell), and sampling weights that are -not finite, non-negative and positive-mass. A fitted result is rejected -unless every unit-period outcome cell is present and finite and the estimator -dropped no unit by its own complete-case rules (DMLDiD records this; results -pickled by <= 3.12.0 carry no record and are refused). Gaps the estimator -itself could not fill are handled rather than raised, each with a -`UserWarning`, keyed on the cells' `skip_reason` (`missing_period`, -`zero_treated_control`, `zero_weight_mass`): a cohort with no estimable post -cell under the anticipation window is dropped from the table and the cohort -shares for every `type` (`did`'s first-period drop), and a cohort that keeps -one estimable post cell has its absent cells carved out, so `"overall"` / -`"simple"` average over each cohort's AVAILABLE post cells (`aggte`, -`aggregate()`). The window counts raw time units, as CallawaySantAnna does; -`"twfe"`'s own post window stays `t >= g`. `n_negative_post` / -`negative_post_weight_share` report the -pathology (negative weight on POST cells); `n_negative` counts pre cells too, -and is near-half in every staggered design because the TWFE weights sum to -zero over the full grid. - -`decompose_twfe_weights` takes the raw panel rather than a fitted result -because it re-estimates. It is tied to `attgt_weights` by an identity: -`attgt_weights(cs, type="twfe").implied_att == decompose_twfe_weights(panel, ...).estimate`. - ### WooldridgeDiD Wooldridge (2023, 2025) Extended Two-Way Fixed Effects (ETWFE) estimator. OLS path uses direct saturated-regression coefficients for ATT(g,t). Logit and Poisson QMLE paths use Average Structural Function (ASF) based ATT with delta-method standard errors. @@ -2183,6 +1965,224 @@ Results from `LWDiD.fit()` (both timing modes). Flat-native headline fields `att ## Diagnostics +### BaconDecomposition + +Goodman-Bacon (2021) decomposition of TWFE into 2x2 DiD comparisons. + +```python +BaconDecomposition( + weights: str = "approximate", # "approximate" or "exact" +) +``` + +**Alias:** `Bacon` + +**fit() parameters:** + +```python +bacon.fit( + data: pd.DataFrame, + outcome: str, + unit: str, + time: str, + first_treat: str, +) -> BaconDecompositionResults +``` + +**Usage:** + +```python +from diff_diff import BaconDecomposition, plot_bacon + +bacon = BaconDecomposition(weights="exact") +results = bacon.fit(data, outcome='outcome', unit='unit', + time='period', first_treat='first_treat') +results.print_summary() +plot_bacon(results) +``` + +### TWFE Weight Diagnostics + +What a TWFE regression implicitly weights on staggered data. Distinct from +`twowayfeweights` (dCDH), which weights (unit, time) cells: these weight +ATT(g,t) parameters. Ported from Brantly Callaway's `twfeweights` R package +(MIT); methodology Baker, Callaway, Cunningham, Goodman-Bacon & Sant'Anna +(2025). + +```python +attgt_weights( + results, # CallawaySantAnnaResults, or a (g,t) frame + type="twfe", # "twfe" | "overall" (ATT^O) | "simple" + data=None, unit=None, time=None, first_treat=None, # frame path only + weights=None, # frame path only; a fitted result uses its own survey weights + anticipation=None, # frame path only; the fit's own otherwise +) -> ATTGTWeightsResult + +decompose_twfe_weights( + data, # balanced long panel (it re-estimates) + outcome=, unit=, time=, first_treat=, + method="fwl", + covariates=None, + base_period="first_period", # or "gmin1" + balance_covariates=None, # enables result.covariate_balance() + weights=None, +) -> TWFEDecompositionResult + +plot_twfe_weights(result, kind="auto") # "weights" | "balance" +``` + +`type="twfe"` requires a fit with `base_period="universal"`, +`control_group="never_treated"` AND no covariates (R twfe_weights' three +restrictions); it raises otherwise. ATT^O and ATT^simple weights are +non-negative and sum to one, so comparing `implied_att` across the three +types shows what the TWFE specification costs. The result records its +`type=` as `.level` (`ATTGTWeightsResult.LEVELS` lists the values). + +Both entry points fail closed on input R never faced: NaN / `-inf` cohort +labels (never-treated is exactly `0` or `+inf`), duplicated or non-finite +ATT(g,t) cells, an incomplete group-time grid (`"twfe"` needs every cohort x +period cell, the CS estimands every post cell), and sampling weights that are +not finite, non-negative and positive-mass. A fitted result is rejected +unless every unit-period outcome cell is present and finite and the estimator +dropped no unit by its own complete-case rules (DMLDiD records this; results +pickled by <= 3.12.0 carry no record and are refused). Gaps the estimator +itself could not fill are handled rather than raised, each with a +`UserWarning`, keyed on the cells' `skip_reason` (`missing_period`, +`zero_treated_control`, `zero_weight_mass`): a cohort with no estimable post +cell under the anticipation window is dropped from the table and the cohort +shares for every `type` (`did`'s first-period drop), and a cohort that keeps +one estimable post cell has its absent cells carved out, so `"overall"` / +`"simple"` average over each cohort's AVAILABLE post cells (`aggte`, +`aggregate()`). The window counts raw time units, as CallawaySantAnna does; +`"twfe"`'s own post window stays `t >= g`. `n_negative_post` / +`negative_post_weight_share` report the +pathology (negative weight on POST cells); `n_negative` counts pre cells too, +and is near-half in every staggered design because the TWFE weights sum to +zero over the full grid. + +`decompose_twfe_weights` takes the raw panel rather than a fitted result +because it re-estimates. It is tied to `attgt_weights` by an identity: +`attgt_weights(cs, type="twfe").implied_att == decompose_twfe_weights(panel, ...).estimate`. + +### RDPlot + +RD plot diagnostic (Calonico, Cattaneo & Titiunik 2015), parity-targeting R +rdrobust 4.0.0's `rdplot()`. NOT a treatment-effect estimator: builds the +standard exploratory RD plot - per-side global order-p polynomial fits +(default p=4, uniform kernel over each side's full range; deliberately +different defaults from estimation) plus binned local sample means with a +DATA-DRIVEN number of bins per side. All 8 `binselect` selectors = +{evenly (es) / quantile (qs) spaced} x {IMSE-optimal / mimicking variance +(mv)} x {spacings / polynomial-regression (pr) variance estimators}; the +result reports `J`, `J_IMSE`, `J_MV`, the implied scale `rscale = J/J_IMSE`, +and the CCT 2015 Supplement S.1 WIMSE weights (variance `1/(1+rscale^3)`, +bias `rscale^3/(1+rscale^3)`), matching R's `summary.rdplot`. + +```python +RDPlot( + cutoff: float = 0.0, # Known threshold of the running variable + p: int = 4, # Global polynomial order (R's rdplot default) + nbins: int | tuple | None = None, # Manual bins per side (overrides the selector) + binselect: str = "esmv", # es/espr/esmv/esmvpr/qs/qspr/qsmv/qsmvpr + scale: float | tuple | None = None, # Multiplies the selected J (fractional products take CCT 2015 Eq 2's ceiling; R crashes there) + kernel: str = "uniform", # Weighting for the global fit within h + h: float | tuple | None = None, # Global-fit window; default = full per-side range + support: tuple | None = None, # (lower, upper); only WIDENS the observed range + masspoints: str = "adjust", # >= 20% ties: warn + switch spacings binselect to its *pr sibling ("check" warns only; "off" disables) + ci: float | None = None, # Per-bin CI level for plotted error bars (CI columns are ALWAYS computed, at 95 when None) + covs_drop: bool = True, # Drop collinear covariates with a warning; False = strict error +) +``` + +**fit() parameters:** + +```python +rp.fit( + data: pd.DataFrame, + outcome: str, + running: str, + covariates: list[str] | None = None, # Covariate-adjusted plot (R's covs=, covs_eval="mean"); gammas exposed as covariate_coefficients +) -> RDPlotResult +``` + +The 3.8 names `outcome_col=` and `running_col=` were removed in 4.0 - use `outcome=` and `running=`. + +```python +from diff_diff import RDPlot + +rp = RDPlot(cutoff=0.0) +res = rp.fit(df, outcome="y", running="score") +res.J, res.J_IMSE, res.J_MV # selected / IMSE-optimal / mimicking-variance bins per side +res.rscale # implied scale; WIMSE weights via res.wimse_variance_weight / res.wimse_bias_weight +res.to_dataframe() # vars_bins: R-named per-bin columns (rdplot_mean_bin/mean_x/mean_y/min_bin/max_bin/se_y/N/ci_l/ci_r) +res.vars_poly # 500-point global-fit curve per side +print(res.summary()) # R summary.rdplot layout incl. implied scale + WIMSE weights +res.plot() # optional matplotlib rendering (matplotlib NOT a dependency; ImportError if absent) +``` + +Empty bins are dropped from vars_bins (R's tapply) while bin_avg/bin_med +cover all partition cells; single-observation bins get se = 0 (R's NA -> 0 +fixup); n < 20 raises outright (R behavior - no fallback); a side with +zero outcome variance pins its J to 1 with a warning and echoes R's Inf +J_IMSE. With empty LEFT-side bins, R's own rdplot_min_bin/rdplot_max_bin +come from mirrored grid slots and can disagree with rdplot_mean_bin - an +R quirk replicated for parity (rdplot_mean_bin is the reliable bin +identity). Sampling weights and R's subset= are documented seams (filter +the DataFrame instead). + +### RDDensityTest + +Manipulation (density-discontinuity) test diagnostic (Cattaneo, Jansson & +Ma 2020), parity-targeting R rddensity 3.0. NOT a treatment-effect +estimator: tests the null of NO discontinuity in the running variable's +density at the cutoff (McCrary's manipulation question) via boundary- +adaptive local polynomial density estimation - the density is the slope of +a kernel-weighted local polynomial regression of the rank-based EDF - with +ROBUST BIAS-CORRECTED inference (order q = p+1 estimators at order-p +MSE-optimal bandwidths). A significant test is evidence of sorting around +the cutoff; a null supports (never proves) the RD continuity assumption. +Inference assumes an i.i.d. sample with a smooth, positive density near +the cutoff on each side (CJM Assumption 1); clustered, survey-weighted, or +repeated-observation dependence is unsupported - there is no cluster= +surface. + +```python +RDDensityTest( + cutoff=0.0, + p=2, # density estimation order (1..7) + q=None, # bias-correction order; None -> p+1; q==p = conventional test + fitselect="unrestricted", # or "restricted" (shared CDF/higher derivatives; only the density jumps) + kernel="triangular", # / "epanechnikov" / "uniform" (R spellings accepted) + vcov_type="jackknife", # R's default; or "plugin" (asymptotic) + h=None, # manual bandwidth: scalar or (h_left, h_right) + bwselect="comb", # each / diff / sum / comb (median rule; restricted: min(diff, sum)) + masspoints="adjust", # RD-family surface: adjust (R TRUE) / check (warn only) / off + regularize=True, + n_local_min=None, # None -> 20+p+1; 0 disables the gate (R semantics, ceil applied) + n_unique_min=None, + report_all=False, # also report the conventional (order-p) test, R's all=TRUE +) + +test = RDDensityTest() +res = test.fit(df, running="score") +res.t_stat, res.p_value # robust bias-corrected manipulation test +res.f_left, res.f_right # per-side density estimates at the cutoff (whole-sample scale) +res.bandwidths # the rdbwdensity 4-row h-table (None on manual h) +res.to_dataframe() # tidy left/right/diff table (+ conventional rows with report_all) +print(res.summary()) # R-style block (Conventional row iff report_all=True) +res.to_dict() # JSON-serializable scalars (+ nested 'bandwidths' when estimated) +``` + +Repeated running-variable values are mass-point adjusted with a warning +(matching R massPoints=TRUE; "check" warns without adjusting, "off" is +silent). Degenerate designs fail loud: a side of (almost) only repeated +values, singular local fits, and degenerate selected bandwidths raise +ValueError (R silently returns numerically meaningless output there); +negative/zero/non-finite variance estimates degrade SEs and inference to +NaN with a warning, and finite NEGATIVE density estimates warn. No +confidence-interval fields, matching R's output surface (CIs belong to +rdplotdensity, a documented seam alongside the binomial windows test). + ### Placebo Tests ```python diff --git a/diff_diff/guides/llms.txt b/diff_diff/guides/llms.txt index 3f435fd6a..dc0487606 100644 --- a/diff_diff/guides/llms.txt +++ b/diff_diff/guides/llms.txt @@ -80,7 +80,6 @@ The site is organized into 5 sections, each with a landing page: - [LWDiD](https://diff-diff.readthedocs.io/en/stable/api/lwdid.html): Lee & Wooldridge (2025, 2026) rolling-transformation DiD — unit-specific demean/detrend converts panel to cross-section, supports staggered adoption with flexible control groups. Signature: `LWDiD(rolling='demean', estimation_method='reg', vcov_type='hc1', cluster=None, control_group='not_yet_treated', alpha=0.05, n_bootstrap=0, seed=None, pscore_trim=0.01, n_neighbors=1, caliper=None, with_replacement=True, n_jobs=1).fit(data, outcome, unit, time, treatment, first_treat=None, covariates=None)`. `estimation_method` values: `reg` (papers' RA), `ipw`, `dr` (papers' IPWRA, doubly robust), `psm`; `vcov_type` values: `classical`/`hc1`/`hc2`/`hc3` for `reg`; `ipw`/`dr` accept `hc1` only (influence-function variance); `psm` accepts `hc1` as configuration only - PSM inference is unavailable (NaN) pending an Abadie-Imbens matching variance; cluster-robust inference via the constructor's `cluster=` column (hc1/CR1 only, not a `vcov_type` value; rejected for `psm`). Per-period effects: post-fit `results.aggregate('event_study')`. - [DMLDiD](https://diff-diff.readthedocs.io/en/stable/api/dml_did.html): Chang (2020) double/debiased machine learning DiD — staggered ATT(g,t) with cross-fitted ML nuisances (DML2) and Neyman-orthogonal scores; covariates REQUIRED (conditional parallel trends). Signature: `DMLDiD(propensity_learner='logit', outcome_learner='linear', n_folds=5, control_group='never_treated', anticipation=0, alpha=0.05, n_bootstrap=0, bootstrap_weights=None, seed=None, base_period='varying', cband=True, pscore_trim=0.01, panel=True, cluster=None).fit(data, outcome, unit, time, first_treat, covariates, survey_design=None, bad_control=None, bad_control_covariates=None)`. `panel=False` = declared repeated cross sections (Chang Case 2: level outcomes, row-unique unit IDs, lambda-corrected variance). Survey/cluster support on BOTH designs (bad-control lane: panel only, `cluster=` only): `survey_design=` (pweight full-design TSL — weighted moments, PSU-cohesive folds, design-based variance with t-inference; a library extension of Chang's i.i.d. theory), replicate-weight designs (BRR/Fay/JK1/JKn/SDR — per-cell AND aggregate IF-reweighting variance, df = rank-1; cluster= and bootstrap combinations rejected), and coarser-than-unit `cluster=` (variance/folds only, kernels stay unweighted). Bad controls (Caetano, Callaway, Payne & Sant'Anna 2026): `fit(..., bad_control='x', bad_control_covariates=[outcome])` runs the paper's Neyman-orthogonal doubly-robust score (Eq. 10) - parallel trends conditional on the bad control's untreated path plus covariate unconfoundedness given its base-period value, W and Z - and stores the per-cell `ATT_X(g,t)` pre-test (`results.bad_control_summary()`); a bad control must NOT be in `covariates`. Learners: string names (`linear`/`ridge`/`sieve` regressors, `logit` classifier) or any object with fit/predict(_proba) (sklearn-compatible); `SieveLearner(k_max, criterion)` is exported for adaptive polynomial nuisances. Aggregation is POST-FIT: `results.aggregate('event_study'/'group'/'simple')`, plus `'total'` on panel non-survey fits (RCS and declared-survey fits fail 'total' closed); sup-t bands via bootstrap replay. With seed=None point estimates vary across fits (random folds); set seed for reproducibility. - [DurationDiD](https://diff-diff.readthedocs.io/en/stable/api/duration_did.html): Deaner & Ku (2026) causal duration DiD — two-group, common-timing design for a binary ABSORBING outcome (spell ended; once 1, always 1). Identifies off a restriction on the groups' UNTREATED HAZARDS, never outcome levels: `method='cd'` (constant additive hazard gap) or `method='ph'` (constant hazard ratio, mean-of-ratios estimator), fitted on the pre-treatment cumulative hazards (default equal weights over every eligible pre-treatment date; a window via `pre_periods=`/`pre_period_weights=`). Reports the absorption ATT at every post-treatment date and its uniform average as `att`, with the paper's whole-individual pooled bootstrap (centered pointwise intervals + simultaneous max-|t| band) and a fixed-anchor pre-treatment specification test in `results.pretest`. Signature: `DurationDiD(method='cd', n_bootstrap=1000, alpha=0.05, seed=None).fit(data, outcome, unit, time, treatment, last_pre_period=..., pre_periods=None, pre_period_weights=None)`; `last_pre_period` (the last untreated date) is REQUIRED and never inferred. Balanced equally spaced numeric grid, fixed 0/1 group indicator; no covariates, staggering, censoring, survey or cluster support. Every inference family is fully available or fully withheld (`inference_status`). Not admitted by DiagnosticReport/BusinessReport; `results.aggregate('event_study')` returns the unified container. -- [BaconDecomposition](https://diff-diff.readthedocs.io/en/stable/api/bacon.html): Goodman-Bacon (2021) decomposition for diagnosing TWFE bias in staggered settings ## Diagnostics and Sensitivity Analysis @@ -90,6 +89,7 @@ The site is organized into 5 sections, each with a landing page: - [Parallel Trends Testing](https://diff-diff.readthedocs.io/en/stable/api/diagnostics.html): Simple and Wasserstein-robust parallel trends tests, equivalence testing (TOST) - [Placebo Tests](https://diff-diff.readthedocs.io/en/stable/api/diagnostics.html): Placebo timing, group, permutation, and leave-one-out diagnostics - [TWFE Weight Diagnostics](https://diff-diff.readthedocs.io/en/stable/api/twfe_weights.html): Baker et al. (2025) implicit weights on ATT(g,t) - `attgt_weights(results, type='twfe'|'overall'|'simple')` takes a fitted `CallawaySantAnnaResults` (raw ATT(g,t) frame + panel as fallback) and returns the weight each estimand places on each group-time effect, with the negative-weight share; `decompose_twfe_weights(data, outcome=, unit=, time=, first_treat=, method='fwl', covariates=)` re-derives the TWFE estimate from its ATT(g,t) building blocks with `pre_period_contribution` (the pre-treatment cells' sample contribution, which can be pre-trends or sampling noise), and `result.covariate_balance()` reports implicit-weight covariate balance. Plot with `plot_twfe_weights`. R `twfeweights` 0.9.0 output parity +- [Bacon Decomposition](https://diff-diff.readthedocs.io/en/stable/api/bacon.html): Goodman-Bacon (2021) decomposition of the TWFE estimate into its 2x2 comparisons and weights, for diagnosing TWFE bias in staggered settings (`BaconDecomposition`, alias `Bacon`) - [Honest DiD](https://diff-diff.readthedocs.io/en/stable/api/honest_did.html): Rambachan & Roth (2023) sensitivity analysis — robust CI under parallel trends violations, breakdown values - [Pre-Trends Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/pretrends.html): Roth (2022) Section II.A-B no-individually-significant (NIS) box-probability pretest power + minimum detectable violation; `pretest_form='nis'` (default) implements the paper's primary form, `pretest_form='wald'` retained as paper-supported alternative (Propositions 1+3+4 all apply); linear-violation MDV in Roth's γ units when relative-time labels are threaded through `fit()`; full Σ_22 routing on non-bootstrap CallawaySantAnna and SunAbraham adapters and on admitted CS-/StackedDiD-sourced `aggregate('event_study')` containers (StackedDiD persists its ES VCV in every inference mode) - [Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/power.html): Analytical and simulation-based power analysis — MDE, sample size, power curves for study design diff --git a/docs/api/_autosummary/diff_diff.SunAbrahamResults.rst b/docs/api/_autosummary/diff_diff.SunAbrahamResults.rst index dde7c1698..3e6385ac4 100644 --- a/docs/api/_autosummary/diff_diff.SunAbrahamResults.rst +++ b/docs/api/_autosummary/diff_diff.SunAbrahamResults.rst @@ -38,6 +38,7 @@ ~SunAbrahamResults.event_study_vcov_index ~SunAbrahamResults.inference_df ~SunAbrahamResults.is_significant + ~SunAbrahamResults.n_clusters ~SunAbrahamResults.overall_att ~SunAbrahamResults.overall_conf_int ~SunAbrahamResults.overall_p_value diff --git a/docs/api/bacon.rst b/docs/api/bacon.rst index d80df9c12..12e81aac8 100644 --- a/docs/api/bacon.rst +++ b/docs/api/bacon.rst @@ -30,7 +30,9 @@ variation in treatment timing. *Journal of Econometrics*, 225(2), 254-277. BaconDecomposition ------------------ -Main estimator class for the Goodman-Bacon decomposition. +Entry point of the Goodman-Bacon decomposition diagnostic (a member of the 4.0 +diagnostic family, not an estimator; its results carry the +:class:`~diff_diff.Diagnostic` marker). .. autoclass:: diff_diff.BaconDecomposition :members: diff --git a/docs/api/estimators.rst b/docs/api/estimators.rst index 04ed16d53..a2e16e934 100644 --- a/docs/api/estimators.rst +++ b/docs/api/estimators.rst @@ -17,7 +17,9 @@ for backward compatibility, so you can import any of them using: from diff_diff import DifferenceInDifferences, TwoWayFixedEffects, SyntheticDiD -Most estimators have short aliases (``TROP`` already uses its short canonical name): +Most estimators have short aliases (``TROP`` already uses its short canonical name; +``Bacon`` is the alias of the :class:`~diff_diff.BaconDecomposition` diagnostic, see +:doc:`bacon`): .. code-block:: python diff --git a/docs/api/index.rst b/docs/api/index.rst index 6fed84022..deed96831 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -6,8 +6,11 @@ This section provides complete API documentation for all diff-diff modules. Estimators ---------- -Core causal-inference estimator classes - the DiD family plus synthetic control, -regression discontinuity, and the Goodman-Bacon decomposition diagnostic: +Core causal-inference estimator classes - the DiD family plus synthetic control +and regression discontinuity. The diagnostic family (the Goodman-Bacon +decomposition, RD plots and manipulation tests, placebo tests, TWFE weight +audits) is catalogued under Diagnostics below; parallel-trends testing, Honest +DiD and the power tools keep their own sections: .. autosummary:: :toctree: _autosummary @@ -35,7 +38,6 @@ regression discontinuity, and the Goodman-Bacon decomposition diagnostic: diff_diff.LWDiD diff_diff.DMLDiD diff_diff.DurationDiD - diff_diff.BaconDecomposition diff_diff.RegressionDiscontinuity Results Classes @@ -116,7 +118,6 @@ Plotting functions and plot builders: diff_diff.plot_group_effects diff_diff.plot_sensitivity diff_diff.plot_honest_event_study - diff_diff.RDPlot diff_diff.plot_bacon diff_diff.plot_twfe_weights diff_diff.plot_power_curve @@ -125,12 +126,20 @@ Plotting functions and plot builders: Diagnostics ----------- -Placebo tests and model diagnostics: +The diagnostic family (4.0, spec section 3.5): tools that assess a design or +an identifying assumption rather than estimate a treatment effect - design +decompositions and weight audits, placebo tests, RD plots and manipulation +tests. Class-backed members return :class:`~diff_diff.Diagnostic`-marked +results (the placebo helpers return :class:`~diff_diff.PlaceboTestResults`; +``run_all_placebo_tests`` bundles them in a dict); the raw-dict +parallel-trends helpers and :class:`~diff_diff.TWFEWeightsResult` belong to +the family in the docs only: .. autosummary:: :toctree: _autosummary :nosignatures: + diff_diff.BaconDecomposition diff_diff.run_placebo_test diff_diff.placebo_timing_test diff_diff.placebo_group_test @@ -140,6 +149,7 @@ Placebo tests and model diagnostics: diff_diff.PlaceboTestResults diff_diff.attgt_weights diff_diff.decompose_twfe_weights + diff_diff.RDPlot diff_diff.RDDensityTest Panel Profiling @@ -375,7 +385,6 @@ Estimators lwdid dml_did duration_did - bacon Infrastructure ~~~~~~~~~~~~~~ @@ -400,6 +409,7 @@ Diagnostics & Inference :maxdepth: 2 diagnostics + bacon honest_did power pretrends diff --git a/docs/doc-deps.yaml b/docs/doc-deps.yaml index 62f8b745d..60d3f13dd 100644 --- a/docs/doc-deps.yaml +++ b/docs/doc-deps.yaml @@ -577,6 +577,12 @@ sources: type: user_guide - path: docs/tutorials/28_rdd_scholarship_illusion.ipynb type: tutorial + - path: docs/api/index.rst + type: api_reference + note: "Diagnostics autosummary entry (moved from Visualization)" + - path: docs/index.rst + type: user_guide + note: "Diagnostics & Sensitivity table row" diff_diff/rddensity.py: drift_risk: medium docs: @@ -602,6 +608,9 @@ sources: - path: diff_diff/guides/llms-autonomous.txt section: "RD in-scope routing paragraph" type: user_guide + - path: docs/index.rst + type: user_guide + note: "Diagnostics & Sensitivity table row" diff_diff/_rdrobust_port.py: drift_risk: high docs: @@ -666,6 +675,9 @@ sources: note: "Survey-aware pretest workflow walkthrough (overall + event-study under SurveyDesign(strata=...)); drift-locks `_QUG_DEFERRED_SUFFIX`, the event-study summary QUG-skip note, and joint pretrends/homogeneity horizon labels under stratified-clustered Stute bootstrap (PR #432). Drift-locked at tests/test_t22_had_survey_design_drift.py" - path: docs/migration-4.0.md type: user_guide + - path: docs/index.rst + type: user_guide + note: "Diagnostics & Sensitivity table row" diff_diff/local_linear.py: drift_risk: low @@ -1228,6 +1240,9 @@ sources: type: user_guide - path: docs/r_comparison.rst type: user_guide + - path: docs/index.rst + type: user_guide + note: "Diagnostics & Sensitivity table row" # ── TWFE Weight Diagnostics ───���───────────────────────────────────────── @@ -1250,6 +1265,9 @@ sources: - path: diff_diff/guides/llms-full.txt section: "TWFE Weight Diagnostics" type: user_guide + - path: docs/index.rst + type: user_guide + note: "Diagnostics & Sensitivity table row" diff_diff/twfe_weights_results.py: drift_risk: low docs: @@ -1276,7 +1294,7 @@ sources: type: tutorial note: "TWFE failure-mode demonstration" - path: README.md - section: "Estimators (one-line catalog entry)" + section: "Diagnostics & Sensitivity (one-line catalog entry)" type: user_guide - path: docs/references.rst type: user_guide @@ -1284,10 +1302,16 @@ sources: section: "BaconDecomposition" type: user_guide - path: diff_diff/guides/llms.txt - section: "Estimators" + section: "Diagnostics and Sensitivity Analysis" type: user_guide - path: docs/migration-4.0.md type: user_guide + - path: docs/api/index.rst + type: api_reference + note: "Diagnostics autosummary entry" + - path: docs/index.rst + type: user_guide + note: "Diagnostics & Sensitivity table row" # ── Diagnostics & analysis ───────���───────────────────────────────── @@ -1306,6 +1330,9 @@ sources: type: tutorial - path: docs/migration-4.0.md type: user_guide + - path: docs/index.rst + type: user_guide + note: "Diagnostics & Sensitivity table row" diff_diff/pretrends.py: drift_risk: low @@ -1321,6 +1348,9 @@ sources: type: tutorial - path: docs/migration-4.0.md type: user_guide + - path: docs/index.rst + type: user_guide + note: "Diagnostics & Sensitivity table row" diff_diff/business_report.py: drift_risk: medium @@ -1359,6 +1389,9 @@ sources: type: user_guide - path: docs/migration-4.0.md type: user_guide + - path: docs/index.rst + type: user_guide + note: "Diagnostics & Sensitivity table row" diff_diff/mmm.py: drift_risk: medium @@ -1404,6 +1437,9 @@ sources: type: tutorial - path: docs/migration-4.0.md type: user_guide + - path: docs/index.rst + type: user_guide + note: "Diagnostics & Sensitivity table row" # ── Survey support ───────────────────────────────────────────────── @@ -1509,6 +1545,9 @@ sources: type: internal - path: docs/api/utils.rst type: api_reference + - path: docs/index.rst + type: user_guide + note: "Diagnostics & Sensitivity table row" diff_diff/results.py: drift_risk: medium diff --git a/docs/index.rst b/docs/index.rst index 898d9ee04..499771d7b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -30,7 +30,7 @@ It provides sklearn-like estimators with statsmodels-style output for econometri Key Features ------------ -- **20+ Estimators**: Basic DiD, TWFE, Event Study, Synthetic DiD/Control, modern staggered estimators (Callaway-Sant'Anna, Sun-Abraham, Imputation, Two-Stage, Stacked, LP-DiD), reversible and heterogeneous-adoption designs (dCDH, HAD), distributional methods (Changes-in-Changes), Regression Discontinuity, and Bacon Decomposition diagnostics +- **20+ Estimators**: Basic DiD, TWFE, Event Study, Synthetic DiD/Control, modern staggered estimators (Callaway-Sant'Anna, Sun-Abraham, Imputation, Two-Stage, Stacked, LP-DiD), reversible and heterogeneous-adoption designs (dCDH, HAD), distributional methods (Changes-in-Changes), and Regression Discontinuity - **Modern Inference**: Robust standard errors, cluster-robust SEs, wild cluster bootstrap, and multiplier bootstrap - **Assumption Testing**: Parallel trends tests, placebo tests, Bacon decomposition, and comprehensive diagnostics - **Sensitivity Analysis**: Honest DiD (Rambachan & Roth 2023) for robust inference under parallel trends violations @@ -189,8 +189,43 @@ Supported Estimators - Calonico, Cattaneo & Titiunik (2014) sharp/fuzzy RD with robust bias-corrected inference * - :class:`~diff_diff.TROP` - Triply Robust Panel with factor model adjustment (Athey et al. 2025) + +Diagnostics & Sensitivity +------------------------- + +The diagnostic family assesses a design, an identifying assumption, or the +robustness of an estimate; its members report no treatment-effect inference +row of their own (see the :doc:`API reference ` Diagnostics +section): + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Diagnostic + - Description * - :class:`~diff_diff.BaconDecomposition` - - Goodman-Bacon decomposition diagnostics + - Goodman-Bacon (2021) decomposition of the TWFE estimate into its 2x2 comparisons and weights, for diagnosing TWFE bias in staggered settings + * - :class:`~diff_diff.RDPlot` + - Calonico, Cattaneo & Titiunik (2015) optimal data-driven RD plots: all 8 rdrobust ``binselect`` bin selectors, implied-scale/WIMSE-weight reporting, optional matplotlib rendering + * - :class:`~diff_diff.RDDensityTest` + - Cattaneo, Jansson & Ma (2020) density-discontinuity (manipulation) test: rddensity 3.0 parity, robust bias-corrected inference, unrestricted/restricted models, mass-point adjustment + * - :class:`~diff_diff.HonestDiD` + - Rambachan & Roth (2023) sensitivity analysis: robust CI under parallel-trends violations, breakdown values + * - :class:`~diff_diff.PreTrendsPower` + - Roth (2022) minimum detectable violation and power curves for pre-trends tests + * - :class:`~diff_diff.PowerAnalysis` + - Analytical and simulation-based MDE, sample size, and power curves for study design + * - :class:`~diff_diff.DiagnosticReport` + - Experimental preview: one-call battery of the applicable checks (parallel trends, Bacon read-out, pre-trends power, Honest DiD) for a fitted result, one status row per check in ``summary()``; no pass/fail verdict (see ``docs/methodology/REPORTING.md``) + * - :func:`~diff_diff.check_parallel_trends` + - Simple and Wasserstein-robust parallel trends tests, equivalence testing (TOST) + * - :func:`~diff_diff.run_placebo_test` + - Placebo timing, group, permutation, and leave-one-out tests + * - :func:`~diff_diff.attgt_weights` / :func:`~diff_diff.decompose_twfe_weights` + - Baker et al. (2025) implicit weights a TWFE regression places on each ATT(g,t), against the ATT^O / ATT^simple targets, with the pre-period contribution (ports Callaway's ``twfeweights``) + * - :func:`~diff_diff.did_had_pretest_workflow` + - Bundled QUG / Stute / Yatchew pre-test battery for :class:`~diff_diff.HeterogeneousAdoptionDiD`, dispatched from the panel shape Indices and tables ------------------ diff --git a/docs/methodology/REGISTRY.md b/docs/methodology/REGISTRY.md index 7e0883853..8494b8f67 100644 --- a/docs/methodology/REGISTRY.md +++ b/docs/methodology/REGISTRY.md @@ -10,7 +10,7 @@ This document provides the academic foundations and key implementation requireme 1. [Core DiD Estimators](#core-did-estimators) - [DifferenceInDifferences](#differenceindifferences) - - [MultiPeriodDiD](#multiperioddid) + - [MultiPeriodDiD](#multiperioddid) *(archived - class removed at 4.0)* - [TwoWayFixedEffects](#twowayfixedeffects) 2. [Modern Staggered Estimators](#modern-staggered-estimators) - [CallawaySantAnna](#callawaysantanna) @@ -28,10 +28,10 @@ This document provides the academic foundations and key implementation requireme - [SyntheticDiD](#syntheticdid) - [SyntheticControl](#syntheticcontrol) - [TripleDifference](#tripledifference) - - [StaggeredTripleDifference](#staggeredtripledifference) + - [StaggeredTripleDifference](#staggeredtripledifference) *(archived - class removed at 4.0)* - [TROP](#trop) - [HeterogeneousAdoptionDiD](#heterogeneousadoptiondid) - - [ChangesInChanges (CiC) + QDiD](#changesinchanges-cic) + - [ChangesInChanges (CiC) + QDiD](#changesinchanges-cic) *(QDiD class spelling archived - removed at 4.0)* - [DurationDiD](#durationdid) - [SpilloverDiD](#spilloverdid) 4. [Regression Discontinuity](#regression-discontinuity) @@ -46,6 +46,8 @@ This document provides the academic foundations and key implementation requireme - [PreTrendsPower](#pretrendspower) - [PowerAnalysis](#poweranalysis) +**Archived sections.** Sections marked *archived* are the methodology record of classes removed at the 4.0 cut (ledger rows M-010, M-013, M-015 - [M-090], cut item 9): each opens with an archive Note naming the surviving surface that runs the same estimation core, and the body is kept verbatim in the removed class's vocabulary. They are not live entry points. + --- # Core DiD Estimators @@ -199,7 +201,7 @@ independent full-refit enumeration in `tests/test_wild_bootstrap.py::test_wcr_ma ## MultiPeriodDiD -- **Note:** REMOVED at the 4.0 cut (PR 4c-ii, ledger row M-010; deprecated in 3.9). +- **Note (archived section - class removed at 4.0):** REMOVED at the 4.0 cut (PR 4c-ii, ledger row M-010; deprecated in 3.9). This section is retained as the methodology record of the pooled event-study design, which `TwoWayFixedEffects().fit(..., event_study=True, spec="pooled", post_periods=, reference_period=)` runs unchanged - the @@ -215,8 +217,8 @@ independent full-refit enumeration in `tests/test_wild_bootstrap.py::test_wcr_ma `tests/helpers/event_study_average.py` arithmetic), `period_effects` survives as a FutureWarning view on `EventStudyResults` (row M-016, removed 5.0), and `unit=` auto-clusters the pooled spec at unit. The - methodology below is stated in the removed class's vocabulary; archive - framing lands with [M-090]. + methodology below is stated in the removed class's vocabulary and is kept + verbatim as the archive record ([M-090], 4.0 cut item 9). **Primary source:** Event study methodology - Freyaldenhoven, S., Hansen, C., Pérez, J.P., & Shapiro, J.M. (2021). Visualization, @@ -4406,7 +4408,7 @@ shared verbatim. ## StaggeredTripleDifference -- **Note:** REMOVED at the 4.0 cut (PR 4a, ledger row M-013; deprecated in 3.9). +- **Note (archived section - class removed at 4.0):** REMOVED at the 4.0 cut (PR 4a, ledger row M-013; deprecated in 3.9). This section is retained as the methodology record of the staggered DDD engine, which `TripleDifference().fit(..., unit=, time=, first_treat=, partition=)` runs unchanged - the estimation core moved verbatim to @@ -4415,8 +4417,8 @@ shared verbatim. surface: `eligibility=` is `partition=`, `control_group` takes `"not_yet_treated"` / `"never_treated"` (R's compact spellings died with the class), and `cluster=` raises in staggered mode (see the TripleDifference section). The methodology - below is stated in the engine's own vocabulary; archive framing lands with - [M-090]. + below is stated in the engine's own vocabulary and is kept verbatim as the + archive record ([M-090], 4.0 cut item 9). **Primary source:** [Ortiz-Villavicencio, M., & Sant'Anna, P.H.C. (2025). Better Understanding Triple Differences Estimators. arXiv:2505.09942v3.](https://arxiv.org/abs/2505.09942v3). Paper review on file: `docs/methodology/papers/ortiz-villavicencio-santanna-2025-review.md`. @@ -6048,7 +6050,7 @@ where `Q1` is the R type-1 quantile - the paper's eq. (35)/(A.1) inf-based ceili ### QDiD (quantile DiD comparison estimator) -- **Note:** As of 3.9 this estimator is selected with `ChangesInChanges(method="qdid")`; the standalone `QDiD` class spelling (and the `QDiDResults` alias) was removed at the 4.0 cut (PR 4a, ledger row M-015). Only the class spelling went - the estimator is fully supported and selecting it emits no advisory warning. The existing footnote-21 non-monotonicity `UserWarning` and `practitioner.py`'s "Prefer ChangesInChanges over QDiD" step already carry Athey-Imbens' p. 447 recommendation at the moments a user can act on it; a third warning on every `method="qdid"` fit would blur which of the class/method is dying. Both engines and every number are unchanged by the merge. +- **Note (archived class spelling - removed at 4.0):** As of 3.9 this estimator is selected with `ChangesInChanges(method="qdid")`; the standalone `QDiD` class spelling (and the `QDiDResults` alias) was removed at the 4.0 cut (PR 4a, ledger row M-015). Only the class spelling went - the estimator is fully supported and selecting it emits no advisory warning. The existing footnote-21 non-monotonicity `UserWarning` and `practitioner.py`'s "Prefer ChangesInChanges over QDiD" step already carry Athey-Imbens' p. 447 recommendation at the moments a user can act on it; a third warning on every `method="qdid"` fit would blur which of the class/method is dying. Both engines and every number are unchanged by the merge. **Primary source:** Athey & Imbens (2006), Section 3.3 (pp. 446-447) - the paper formalizes QDiD as the natural comparison estimator and recommends CiC over it: QDiD's justifying model (eq. 22) is not invariant to monotone rescaling of the outcome, forces `U ⊥ (G, T)` (identical unobservable distributions in all four cells), and places testable restrictions on the data (footnote 21 - the implementation warns when the implied counterfactual quantile function is non-monotone on the requested grid; unconditional fits only, see the covariates block above). @@ -7031,7 +7033,7 @@ should be a deliberate user choice. | StackedDiD | Cluster-robust (unit) | Cluster at unit × sub-experiment | | TROP | Block bootstrap | — | | ChangesInChanges | Bootstrap (replicate-SD SEs, normal-approx CIs) | Sup-t uniform bands (fixed 95%, qte parity) | -| QDiD | Bootstrap (replicate-SD SEs, normal-approx CIs) | Sup-t uniform bands (fixed 95%, qte parity) | +| ChangesInChanges(method="qdid") | Bootstrap (replicate-SD SEs, normal-approx CIs) | Sup-t uniform bands (fixed 95%, qte parity) | | BaconDecomposition | N/A (exact decomposition) | Individual 2×2 SEs | | RDDensityTest | Jackknife (leave-one-out, window-restricted; rddensity default) | Plug-in asymptotic (`vcov_type="plugin"`) | | HonestDiD | Inherited from event study | FLCI, C-LF | @@ -7057,7 +7059,7 @@ should be a deliberate user choice. | StackedDiD | stacked-did-weights | `create_sub_exp()` + `compute_weights()` | | TROP | - | (forthcoming) | | ChangesInChanges | qte | `CiC()` | -| QDiD | qte | `QDiD()` | +| ChangesInChanges(method="qdid") | qte | `QDiD()` | | BaconDecomposition | bacondecomp | `bacon()` | | HonestDiD | HonestDiD | `createSensitivityResults()` | | RegressionDiscontinuity | rdrobust | `rdrobust()` + `rdbwselect()` (4.0.0; sharp + fuzzy + covariate-adjusted, nn path; `takeup` = R's `fuzzy=`, `covariates` = R's `covs=`) | @@ -8011,8 +8013,8 @@ variance from the distribution of replicate estimates. HeterogeneousAdoptionDiD (TSL-only survey paths; replicate designs rejected at `fit()`), SyntheticControl (rejects `survey_design` entirely) - - **BaconDecomposition** is diagnostic-only — outside the 20-estimator - count — and likewise rejects replicate designs + - **BaconDecomposition** is diagnostic-only — outside the estimator + roster — and likewise rejects replicate designs - Estimators with replicate support reject replicate + bootstrap (replicate weights provide analytical variance) - **Note:** When invalid replicates are dropped in `compute_replicate_vcov` diff --git a/docs/migration-4.0.md b/docs/migration-4.0.md index 2012db323..0188cd6d0 100644 --- a/docs/migration-4.0.md +++ b/docs/migration-4.0.md @@ -503,6 +503,12 @@ argument contract) and [M-154] (the `SpilloverDiD` `df_convention` knob), both i `n_obs` and `n_control` are no longer in the frame (read `to_dict()`); `ci_lower`/`ci_upper` are `conf_int_lower`/`conf_int_upper`. +- `BaconDecomposition` is re-homed into the diagnostics family ([M-090], landing at 4.0): it + leaves the estimator catalogues (README `## Estimators`, `llms.txt`, the API-reference + Estimators list, the docs homepage table) for their Diagnostics groupings, together with + `RDPlot` in the API reference. Docs-only - the import path, the `Bacon` alias, the results + class and every number are unchanged. + - `BusinessReport.to_dict()` schema `2.0` -> `3.0` (landing at 4.0 with the [M-095] removal): the `assumption` block no longer carries the legacy `clean_control` key; read `control_group`. `DiagnosticReport`'s schema stays `2.0`. diff --git a/docs/v4-deprecations.yaml b/docs/v4-deprecations.yaml index 5ae98c0af..974f8b958 100644 --- a/docs/v4-deprecations.yaml +++ b/docs/v4-deprecations.yaml @@ -1004,10 +1004,11 @@ rows: new: null deprecated_in: "4.0" removed_in: null - status: planned + status: done phase: 5 - code_refs: [README.md, diff_diff/guides/llms.txt, docs/api/index.rst, docs/index.rst, diff_diff/bacon.py, tests/test_docs_ia.py] - notes: "4.0 diagnostic-family docs/roster reorganization (spec section 3.5): Bacon re-homed OUT of the API estimator roster (misfiled today); RDPlot (already documented as a diagnostic) consolidated under the unified family grouping; README/llms.txt/API/alias groupings (incl. docs/api/index.rst AND the docs homepage docs/index.rst, both listing Bacon as an estimator today) and estimator-count claims split into estimators + diagnostics; the flip's terminal test asserts the estimator/diagnostic grouping on the named surfaces. The Phase 2 marker introduction is gated separately by [M-091]." + test_ref: tests/test_v4_diagnostic_family_docs.py + code_refs: [README.md, diff_diff/guides/llms.txt, docs/api/index.rst, docs/index.rst, diff_diff/bacon.py, tests/test_docs_ia.py, docs/methodology/REGISTRY.md, diff_diff/guides/llms-full.txt, docs/api/estimators.rst, docs/api/bacon.rst, docs/migration-4.0.md, docs/doc-deps.yaml, .github/workflows/docs-tests.yml] + notes: "DONE at the 4.0 cut (item 9, 2026-11-01): Bacon moved from the Estimators catalogues (README, llms.txt, llms-full.txt section + alias line, docs/api/index.rst autosummary + module toctree, docs/index.rst table) into the Diagnostics groupings; RDPlot consolidated from the API-index Visualization autosummary into Diagnostics and, with RDDensityTest and the TWFE weight diagnostics, out of the llms-full.txt Estimators section; docs/index.rst gained a Diagnostics & Sensitivity table; /docs-check classification updated; the test_ref runs in the docs-tests workflow (README-only diffs included); REGISTRY archive framing (in-place archive Notes + TOC markers + archived-sections convention paragraph) for MultiPeriodDiD, StaggeredTripleDifference and the QDiD class spelling, cross-reference QDiD rows reworded to ChangesInChanges(method='qdid'); homepage counts stay '20+' (user decision); paper.md untouched (submitted JOSS text already states 23 estimators with Bacon outside the count). Import paths and the Bacon alias unchanged. Pinned by the test_ref. ORIGINAL NOTE: 4.0 diagnostic-family docs/roster reorganization (spec section 3.5): Bacon re-homed OUT of the API estimator roster (misfiled today); RDPlot (already documented as a diagnostic) consolidated under the unified family grouping; README/llms.txt/API/alias groupings (incl. docs/api/index.rst AND the docs homepage docs/index.rst, both listing Bacon as an estimator today) and estimator-count claims split into estimators + diagnostics; the flip's terminal test asserts the estimator/diagnostic grouping on the named surfaces. The Phase 2 marker introduction is gated separately by [M-091]." - id: M-091 kind: behavior group: diagnostic-family diff --git a/docs/v4-design.md b/docs/v4-design.md index 239fed8f8..bf90dc30b 100644 --- a/docs/v4-design.md +++ b/docs/v4-design.md @@ -273,7 +273,12 @@ documented under Diagnostics & Sensitivity - is consolidated under the unified family grouping; README catalog, llms.txt, API reference and alias-table groupings, and estimator-count claims on JOSS-adjacent surfaces all split accordingly. Tracked as a behavior row so the cut -sweep cannot miss it. +sweep cannot miss it. SHIPPED 2026-11-01 (cut item 9; pinned by +`tests/test_v4_diagnostic_family_docs.py`; homepage counts kept at "20+" by +user decision; `paper.md` untouched - it already states 23 estimators with +Bacon outside the count; REGISTRY archive framing is in-place - archive Notes, +TOC markers and an archived-sections convention paragraph - so anchors and +doc-deps section names survive). **RDPlot contract compliance [M-088] [M-089].** `RDPlot.fit()` shipped with `outcome_col` / `running_col` after the ledger froze - the section 8 @@ -1256,6 +1261,11 @@ tests-first PR that flips no rows). complete.) 9. [M-090] docs reorganization: Bacon to the diagnostics family, catalogue counts, JOSS prose, REGISTRY archive framing for the removed classes. + (SHIPPED 2026-11-01: Bacon and RDPlot re-homed on every catalogue surface, + homepage Diagnostics & Sensitivity table, in-place REGISTRY archive framing, + `/docs-check` classification; pinned by + `tests/test_v4_diagnostic_family_docs.py`, which runs in the docs-tests + workflow; row flipped to `done`.) 10. Release: bump to 4.0.0 (closes the cut window), changelog compile, migration-guide retitle, due gate green; re-execute (Rust backend) the tutorials whose clustered-default outputs moved at item 7a and were diff --git a/tests/test_v4_diagnostic_family_docs.py b/tests/test_v4_diagnostic_family_docs.py new file mode 100644 index 000000000..60e344c48 --- /dev/null +++ b/tests/test_v4_diagnostic_family_docs.py @@ -0,0 +1,412 @@ +"""Item 9 of the 4.0 cut - the estimators / diagnostics roster split (ledger row M-090). + +This file is the ``test_ref`` for ledger row M-090 (spec ``docs/v4-design.md`` +section 3.5 and the item-9 line of section 9). It pins the 4.0 public-surface +claim that the library ships ESTIMATORS plus a DIAGNOSTIC FAMILY, on every +catalogue that used to file ``BaconDecomposition`` as an estimator: + +1. ``docs/api/index.rst`` - the Estimators autosummary is exactly the 23 + spec-3.1 estimator classes; ``BaconDecomposition``, ``RDPlot`` and + ``RDDensityTest`` are catalogued under Diagnostics; the ``bacon`` module + page sits in the Diagnostics & Inference toctree; no stub is referenced + from two ``:toctree:`` autosummary blocks (the ``-W`` build would warn). +2. ``docs/index.rst`` - the Supported Estimators table carries no Bacon row, + and the Diagnostics & Sensitivity table lists exactly the class-backed + entry points of the family. +3. ``README.md`` and ``llms.txt`` - Bacon lives in the Diagnostics section, + and the ``## Estimators`` catalogue has one bullet per API estimator. +4. ``llms-full.txt`` - the four diagnostic ``###`` blocks sit under + ``## Diagnostics``, and the alias line no longer files ``Bacon`` as an + estimator alias. +5. ``docs/methodology/REGISTRY.md`` - the archive framing for the classes + removed at 4.0 (MultiPeriodDiD, StaggeredTripleDifference, the QDiD class + spelling) is in place and the "lands with [M-090]" placeholders are gone. +6. The ledger row itself. + +Import discipline: numpy/pandas/scipy/pytest/diff_diff only - the CI test legs +do not install pyyaml, so the ledger is read through ``tests.test_v4_matrix`` +(imported INSIDE the ledger test), and the RST/Markdown surfaces are parsed with +``re``. ``docs/doc-deps.yaml`` is deliberately not parsed here; its integrity is +``tests/test_doc_deps_integrity.py``'s job. +""" + +import re +from collections import Counter + +import pytest + +from diff_diff import get_llm_guide +from tests.test_docs_ia import ( + DOCS, + REPO_ROOT, + _autosummary_entries, + _docname, + _estimator_table_counts, + _resolve, + _section_body, + _toctree_blocks, +) + +# Isolated-install CI (Python Tests workflow) runs from a wheel install with +# no docs/ tree; the repo-surface pins only make sense against a checkout. +pytestmark = pytest.mark.skipif( + not (DOCS / "index.rst").exists(), + reason="docs tree not present (isolated-install CI)", +) + +THIS_TEST_REF = "tests/test_v4_diagnostic_family_docs.py" + +# The 23 estimator classes of spec section 3.1 (27 -> 24 at 4.0, minus Bacon). +ESTIMATOR_ROSTER = { + "DifferenceInDifferences", + "TwoWayFixedEffects", + "SyntheticDiD", + "CallawaySantAnna", + "ChaisemartinDHaultfoeuille", + "SunAbraham", + "ImputationDiD", + "StackedDiD", + "TripleDifference", + "TROP", + "SyntheticControl", + "ContinuousDiD", + "HeterogeneousAdoptionDiD", + "EfficientDiD", + "TwoStageDiD", + "SpilloverDiD", + "WooldridgeDiD", + "LPDiD", + "ChangesInChanges", + "LWDiD", + "DMLDiD", + "DurationDiD", + "RegressionDiscontinuity", +} + +# Class-backed entry points of the diagnostic family (spec section 3.5) that +# the docs homepage catalogues. Function-shaped members (parallel-trends, +# placebo, TWFE weights, the HAD pretest workflow) are docs-family rows too but +# are not classes, so they are not part of this identity pin. +DIAGNOSTIC_ENTRY_POINTS = { + "BaconDecomposition", + "RDPlot", + "RDDensityTest", + "HonestDiD", + "PreTrendsPower", + "PowerAnalysis", + "DiagnosticReport", +} + +# The llms-full.txt ``###`` blocks that moved out of ``## Estimators``. +FULL_GUIDE_DIAGNOSTIC_BLOCKS = ( + "### BaconDecomposition", + "### TWFE Weight Diagnostics", + "### RDPlot", + "### RDDensityTest", +) + +API_INDEX = DOCS / "api" / "index.rst" +HOMEPAGE = DOCS / "index.rst" +README = REPO_ROOT / "README.md" +REGISTRY = DOCS / "methodology" / "REGISTRY.md" + + +# --------------------------------------------------------------------------- +# Helpers (the test_docs_ia ones do not fit these shapes) +# --------------------------------------------------------------------------- + + +def _h3_body(text, title): + """Body of a TILDE-underlined RST H3, from its header to the next header. + + ``_section_body`` matches the FIRST header of that title at any level, so + for ``docs/api/index.rst`` it returns the H2 "Estimators" autosummary + section, never the ``~~~`` "Estimators" heading under Module Documentation. + """ + pattern = rf"^{re.escape(title)}\n~+\n(.*?)(?=^\S[^\n]*\n[-=~]+\n|\Z)" + match = re.search(pattern, text, re.M | re.S) + assert match is not None, f"H3 section {title!r} not found" + return match.group(1) + + +def _first_cells(table_body): + """First-column cells of a list-table body (the ``* - `` lines).""" + return [m.group(1).strip() for m in re.finditer(r"^[ \t]+\* -[ \t]*(.*)$", table_body, re.M)] + + +def _class_targets(table_body): + """``:class:`` targets of a list-table whose rows may also be ``:func:`` rows. + + ``_estimator_table_counts`` demands exactly one ``:class:`` per cell and + skips only a literal ``Estimator`` header, so it rejects the homepage + Diagnostics table (``Diagnostic`` header, ``:func:`` rows). Every first + cell here must be the header, a ``:class:`` cell or a ``:func:`` cell. + """ + targets = Counter() + for cell in _first_cells(table_body): + if cell == "Diagnostic": + continue + classes = re.findall(r":class:`~diff_diff\.(\w+)`", cell) + funcs = re.findall(r":func:`~diff_diff\.(\w+)`", cell) + assert classes or funcs, f"diagnostics table row {cell!r} carries no :class:/:func: target" + if classes: + assert len(classes) == 1, f"diagnostics table row {cell!r} must name one class" + targets[classes[0]] += 1 + return targets + + +def _md_section(text, heading): + """Body of a Markdown section up to the next heading of the same or a higher level.""" + level = len(heading) - len(heading.lstrip("#")) + lines = text.split("\n") + starts = [i for i, line in enumerate(lines) if line.rstrip() == heading] + assert len(starts) == 1, f"heading {heading!r} found {len(starts)} times" + start = starts[0] + end = start + 1 + in_fence = False + while end < len(lines): + line = lines[end] + if line.startswith("```"): + # A ``# comment`` inside a fenced code block is not a heading. + in_fence = not in_fence + elif not in_fence and line.startswith("#"): + hashes = len(line) - len(line.lstrip("#")) + if 0 < hashes <= level and line[hashes : hashes + 1] == " ": + break + end += 1 + return "\n".join(lines[start + 1 : end]) + + +def _identity_counter(qualname_tails): + counter = Counter() + for tail in qualname_tails: + counter[id(_resolve(tail))] += 1 + return counter + + +def _ids(names): + return {id(_resolve(name)) for name in names} + + +# --------------------------------------------------------------------------- +# 1. API reference +# --------------------------------------------------------------------------- + + +def test_api_estimators_autosummary_is_exactly_the_23_estimators(): + body = _section_body(API_INDEX.read_text(), "Estimators") + entries = _autosummary_entries(body) + counts = _identity_counter(entries) + assert max(counts.values()) == 1, "duplicate entry in the Estimators autosummary" + assert set(counts) == _ids( + ESTIMATOR_ROSTER + ), f"Estimators autosummary drift: {sorted(entries)} vs the spec-3.1 roster" + assert not (set(counts) & _ids(DIAGNOSTIC_ENTRY_POINTS)) + assert len(entries) == 23 + + +def test_api_diagnostics_autosummary_catalogues_bacon_rdplot_and_density_test(): + text = API_INDEX.read_text() + diagnostics = set(_autosummary_entries(_section_body(text, "Diagnostics"))) + assert {"BaconDecomposition", "RDPlot", "RDDensityTest"} <= diagnostics + visualization = set(_autosummary_entries(_section_body(text, "Visualization"))) + assert "RDPlot" not in visualization + assert "plot_bacon" in visualization + + +def test_api_index_references_no_stub_from_two_toctree_autosummaries(): + """A stub listed in two ``:toctree:`` blocks is a duplicate toctree reference.""" + all_entries = _autosummary_entries(API_INDEX.read_text()) + dupes = sorted(name for name, n in Counter(all_entries).items() if n > 1) + assert dupes == [], f"autosummary entries listed twice: {dupes}" + + +def test_api_module_toctree_files_bacon_under_diagnostics_and_inference(): + text = API_INDEX.read_text() + estimators_pages = [ + _docname(e) for block in _toctree_blocks(_h3_body(text, "Estimators")) for e in block + ] + diagnostics_pages = [ + _docname(e) + for block in _toctree_blocks(_h3_body(text, "Diagnostics & Inference")) + for e in block + ] + assert "bacon" not in estimators_pages + assert "bacon" in diagnostics_pages + assert "regression_discontinuity" in estimators_pages + + +def test_api_estimators_intro_no_longer_names_the_bacon_diagnostic(): + body = _section_body(API_INDEX.read_text(), "Estimators") + intro = body.split(".. autosummary::")[0] + assert "Goodman-Bacon decomposition diagnostic:" not in intro + assert "Diagnostics" in intro + + +# --------------------------------------------------------------------------- +# 2. Docs homepage +# --------------------------------------------------------------------------- + + +def test_homepage_estimator_table_has_no_bacon_row(): + counts = _estimator_table_counts(_section_body(HOMEPAGE.read_text(), "Supported Estimators")) + assert "BaconDecomposition" not in counts + assert sum(counts.values()) == 23 + + +def test_homepage_diagnostics_table_lists_the_family_entry_points(): + text = HOMEPAGE.read_text() + body = _section_body(text, "Diagnostics & Sensitivity") + targets = _class_targets(body) + assert max(targets.values()) == 1, "duplicate class row in the Diagnostics & Sensitivity table" + assert set(targets) == DIAGNOSTIC_ENTRY_POINTS, sorted(targets) + api_text = API_INDEX.read_text() + all_api = _ids(_autosummary_entries(api_text)) + estimators = _ids(_autosummary_entries(_section_body(api_text, "Estimators"))) + for name in DIAGNOSTIC_ENTRY_POINTS: + obj_id = id(_resolve(name)) + assert obj_id in all_api, f"{name} has no canonical autosummary entry" + assert obj_id not in estimators, f"{name} is still in the Estimators autosummary" + # Function-shaped family members are listed as :func: rows. + for func in ( + "check_parallel_trends", + "run_placebo_test", + "attgt_weights", + "did_had_pretest_workflow", + ): + assert f":func:`~diff_diff.{func}`" in body, f"missing :func: row for {func}" + + +def test_homepage_key_features_estimator_bullet_no_longer_mentions_bacon(): + lines = [ + line for line in HOMEPAGE.read_text().split("\n") if line.startswith("- **20+ Estimators**") + ] + assert len(lines) == 1 + assert "Bacon" not in lines[0] + assert lines[0].rstrip().endswith("Regression Discontinuity") + + +# --------------------------------------------------------------------------- +# 3. README + llms.txt +# --------------------------------------------------------------------------- + + +def _bullet_count(section_body): + return len([line for line in section_body.split("\n") if line.startswith("- [")]) + + +def test_readme_files_bacon_under_diagnostics_not_estimators(): + text = README.read_text() + estimators = _md_section(text, "## Estimators") + diagnostics = _md_section(text, "## Diagnostics & Sensitivity") + assert "BaconDecomposition" not in estimators + assert "api/bacon.html" not in estimators + assert "api/bacon.html" in diagnostics + assert "BaconDecomposition" in diagnostics + api_count = len(_autosummary_entries(_section_body(API_INDEX.read_text(), "Estimators"))) + assert _bullet_count(estimators) == api_count == 23 + + +def test_llms_txt_files_bacon_under_diagnostics_not_estimators(): + text = get_llm_guide("concise") + estimators = _md_section(text, "## Estimators") + diagnostics = _md_section(text, "## Diagnostics and Sensitivity Analysis") + assert "BaconDecomposition" not in estimators + assert "api/bacon.html" not in estimators + assert "api/bacon.html" in diagnostics + assert "BaconDecomposition" in diagnostics + assert _bullet_count(estimators) == 23 + + +# --------------------------------------------------------------------------- +# 4. llms-full.txt +# --------------------------------------------------------------------------- + + +def test_llms_full_diagnostic_blocks_live_under_diagnostics(): + text = get_llm_guide("full") + estimators = _md_section(text, "## Estimators") + diagnostics = _md_section(text, "## Diagnostics") + for heading in FULL_GUIDE_DIAGNOSTIC_BLOCKS: + assert heading not in estimators, f"{heading!r} is still filed under ## Estimators" + assert heading in diagnostics, f"{heading!r} is missing from ## Diagnostics" + # The estimator blocks that neighboured the moved ones stayed put. + assert "### WooldridgeDiD" in estimators + assert "### TROP" in estimators + assert "### StackedDiD" in estimators + + +def test_llms_full_alias_line_does_not_file_bacon_as_an_estimator_alias(): + lines = [ + line + for line in get_llm_guide("full").split("\n") + if line.startswith("- **Estimator aliases**") + ] + assert len(lines) == 1 + match = re.search(r"\((e\.g\., [^)]*)\)", lines[0]) + assert match is not None, lines[0] + alias_list = match.group(1) + assert "`Bacon`" not in alias_list + assert "`BaconDecomposition` keeps its `Bacon` alias" in lines[0] + + +# --------------------------------------------------------------------------- +# 5. REGISTRY archive framing +# --------------------------------------------------------------------------- + +ARCHIVED_SECTIONS = ( + "## MultiPeriodDiD", + "## StaggeredTripleDifference", + "### QDiD (quantile DiD comparison estimator)", +) + + +def test_registry_archive_placeholders_are_gone(): + text = REGISTRY.read_text() + # The old sentence is line-wrapped in both sections, so the match must be + # whitespace-tolerant (a literal substring check passed before any edit). + assert re.search(r"lands\s+with\s+\[M-090\]", text) is None + assert "20-estimator" not in text + + +@pytest.mark.parametrize("heading", ARCHIVED_SECTIONS) +def test_registry_archived_section_opens_with_an_archive_note(heading): + body = _md_section(REGISTRY.read_text(), heading) + first = next(line for line in body.split("\n") if line.strip()) + assert first.startswith("- **Note (archived"), first + if heading.startswith("## "): + # The two removed-class sections cite the archive record; the QDiD + # class-spelling Note keeps its 3.9 text apart from the relabel. + assert "([M-090], 4.0 cut item 9)" in body + + +def test_registry_toc_marks_the_archived_sections_and_states_the_convention(): + text = REGISTRY.read_text() + toc = _md_section(text, "## Table of Contents") + for anchor in ("(#multiperioddid)", "(#staggeredtripledifference)"): + line = next(line for line in toc.split("\n") if anchor in line) + assert "(archived" in line, line + cic = next(line for line in toc.split("\n") if "(#changesinchanges-cic)" in line) + assert "QDiD class spelling archived" in cic + assert "**Archived sections.**" in toc + + +def test_registry_cross_reference_tables_have_no_standalone_qdid_row(): + text = REGISTRY.read_text() + assert re.search(r"^\| QDiD \|", text, re.M) is None + assert text.count('| ChangesInChanges(method="qdid") |') == 2 + + +# --------------------------------------------------------------------------- +# 6. Ledger +# --------------------------------------------------------------------------- + + +def test_ledger_row_m090_is_done_and_points_here(): + from tests.test_v4_matrix import ROWS as LEDGER_ROWS + + row = {r["id"]: r for r in LEDGER_ROWS}["M-090"] + assert row["status"] == "done" + assert row["kind"] == "behavior" + assert row["test_ref"] == THIS_TEST_REF + assert "item 9" in row["notes"]