From 3bdc22116a8d3fd1076582ae93007ddbe9bfc69d Mon Sep 17 00:00:00 2001 From: ChrisW09 <50968720+ChrisW09@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:24:05 +0200 Subject: [PATCH] fix(preprocessor): keep the missing indicator out of the representation With imputation enabled, add_missing_indicator=True and missing_policy='impute_with_indicator' used SimpleImputer(add_indicator=True) as the first pipeline step, so every later step saw the indicator as a second input feature: it was scaled, expanded into basis columns or crossed into polynomial terms, categorical integer codes turned it into 1/2, and get_feature_lineage() crashed while get_feature_info() reported no dimension. The imputer now runs without its indicator, and scikit-learn's MissingIndicator (the one SimpleImputer used internally) runs on the raw input in a parallel FeatureUnion branch, mirroring the separate_state construction. The indicator stays a raw 0/1 column with its previous name, added only for features with missing values at fit; the representation columns are the same as with plain imputation, and lineage reads the indicator width from the fitted branch. Such blocks are now a FeatureUnion, so the representation summary (to_spec() and reproducibility_report()) and the verbose=3 log, which read each block's last step, would have skipped them. A representation_leaf() helper returns the last step of the union's representation branch and both use it; this also lists separate_state blocks, which the summary never did. The fingerprint does not hash the summary. Specs and pickles written by 1.0.0 with add_missing_indicator or impute_with_indicator keep their old block layout and still load, transform, and give the same to_spec(), fingerprint_ and reproducibility_report(). Newly fitted indicator blocks have a different fitted state, so their fingerprint differs from 1.0.0. Fixes #62 --- docs/core_concepts/missing_values.md | 5 ++ pretab/compose/config.py | 5 +- pretab/compose/factory.py | 14 ++- pretab/compose/inspection.py | 26 ++++-- pretab/compose/serialize.py | 11 ++- pretab/preprocessor.py | 16 ++-- tests/integration/test_missing_policy.py | 107 +++++++++++++++++++++++ tests/integration/test_verbosity.py | 11 +++ 8 files changed, 176 insertions(+), 19 deletions(-) diff --git a/docs/core_concepts/missing_values.md b/docs/core_concepts/missing_values.md index 4729edb..393bd04 100644 --- a/docs/core_concepts/missing_values.md +++ b/docs/core_concepts/missing_values.md @@ -21,6 +21,11 @@ pre = Preprocessor( ) ``` +With imputation enabled, the indicator (also used by `missing_policy="impute_with_indicator"`) +is a raw `0`/`1` column computed on the input before imputation and placed next to the +feature's representation. It is never scaled or expanded by the numerical or categorical +method, and it is added only for features that had missing values at `fit`. + ```{note} Disabling imputation lets `NaN` reach the transformer directly. Scalers, splines, and the feature maps (`rbf`, `relu`, `sigmoid`, `tanh`) tolerate it, so an affected row's output is diff --git a/pretab/compose/config.py b/pretab/compose/config.py index b5c7dc7..a9c5ff2 100644 --- a/pretab/compose/config.py +++ b/pretab/compose/config.py @@ -175,10 +175,11 @@ def imputation_plan(self, *, is_numerical: bool) -> dict: booleans and the imputer ``strategy``. When ``missing_policy`` is ``None`` the explicit ``*_imputation`` / ``add_missing_indicator`` parameters stay authoritative (historical behaviour); otherwise ``missing_policy`` decides. + ``add_indicator`` appends a missing indicator built on the raw input next + to the representation (see :func:`~pretab.compose.factory.create_transformer`). ``add_missing_indicator=True`` with imputation disabled for this column kind routes through the standalone ``MissingStateIndicator`` (via - ``separate_state``) instead of the imputer's own indicator, since - ``SimpleImputer.add_indicator`` only takes effect when the imputer runs. + ``separate_state``) instead, preserving its one-column-per-feature output. """ strategy = ( (self.numerical_imputation or "median") diff --git a/pretab/compose/factory.py b/pretab/compose/factory.py index 8ced4d6..5dab8a6 100644 --- a/pretab/compose/factory.py +++ b/pretab/compose/factory.py @@ -11,7 +11,7 @@ import warnings from sklearn.compose import ColumnTransformer -from sklearn.impute import SimpleImputer +from sklearn.impute import MissingIndicator, SimpleImputer from sklearn.pipeline import FeatureUnion, Pipeline from sklearn.preprocessing import MinMaxScaler, StandardScaler @@ -263,7 +263,6 @@ def create_transformer(method: str, *, is_numerical: bool, config: PreprocessorC target_aware=config.target_aware, add_imputer=plan["add_imputer"], imputer_strategy=plan["strategy"], - add_missing_indicator=plan["add_indicator"], output_dim=config.output_dim, adaptive=config.adaptive, min_output_dim=config.min_output_dim if config.adaptive else None, @@ -293,7 +292,6 @@ def create_transformer(method: str, *, is_numerical: bool, config: PreprocessorC method, add_imputer=plan["add_imputer"], imputer_strategy=plan["strategy"], - add_missing_indicator=plan["add_indicator"], **constructor_kwargs, ) @@ -302,6 +300,16 @@ def create_transformer(method: str, *, is_numerical: bool, config: PreprocessorC # Emit a dedicated ``__missing`` column (built on the raw input) alongside # the imputed representation, so the indicator never enters the basis. return FeatureUnion([("representation", pipeline), ("missing", MissingStateIndicator())]) + if plan["add_indicator"]: + # The imputer's missing indicator, built on the raw input next to the + # representation instead of being fed through the scaler and basis as a + # second feature. It keeps SimpleImputer(add_indicator=True)'s semantics + # (a column only for features with missing values at fit) and names. + indicator = MissingIndicator(error_on_new=False) + return FeatureUnion( + [("representation", pipeline), ("missing", indicator)], + verbose_feature_names_out=False, + ) return pipeline diff --git a/pretab/compose/inspection.py b/pretab/compose/inspection.py index 7f3b9b3..77fb9ec 100644 --- a/pretab/compose/inspection.py +++ b/pretab/compose/inspection.py @@ -21,6 +21,7 @@ "build_transformer_summary", "clean_feature_names", "get_output_slices", + "representation_leaf", ] @@ -86,6 +87,19 @@ def _separate_state_branches(transformer): return branches["representation"], branches["missing"] +def representation_leaf(transformer): + """Return the last step of a fitted per-column block's representation. + + For a separate-state / missing-indicator union this is the last step of its + ``"representation"`` pipeline (the raw missing indicator beside it is not + part of the representation); for a pipeline it is the last step, and any + other block is returned unchanged. + """ + separate_state = _separate_state_branches(transformer) + representation = transformer if separate_state is None else separate_state[0] + return representation.steps[-1][1] if hasattr(representation, "steps") else representation + + def build_feature_info(column_transformer, *, embeddings, embedding_dimensions): """Collect per-feature metadata (preprocessing, dimension, categories). @@ -318,12 +332,12 @@ def build_feature_lineage(column_transformer): separate_state = _separate_state_branches(transformer) if separate_state is not None: - representation_pipeline, _missing_indicator = separate_state - union_names = [str(value) for value in transformer.get_feature_names_out(list(columns))] - representation_width = sum(value.startswith("representation__") for value in union_names) - missing_width = sum(value.startswith("missing__") for value in union_names) - if representation_width + missing_width != width: - representation_width = width - missing_width + representation_pipeline, missing_indicator = separate_state + # The missing branch comes last in the union; read its width from the + # fitted indicator, which may emit no column (MissingIndicator only + # marks features that had missing values at fit). + missing_width = len(missing_indicator.get_feature_names_out([str(column) for column in columns])) + representation_width = width - missing_width family, component, uses_target, is_interaction = _resolve_block_representation( representation_pipeline, columns diff --git a/pretab/compose/serialize.py b/pretab/compose/serialize.py index c4b213f..7418224 100644 --- a/pretab/compose/serialize.py +++ b/pretab/compose/serialize.py @@ -29,6 +29,7 @@ from ..core.representation import FeatureLineage, RepresentationSpec from ..exceptions import PretabError, PretabSerializationError from ..placement.base import PlacementResult +from .inspection import representation_leaf from .registry import TransformerSpec SCHEMA_VERSION = 1 @@ -227,7 +228,13 @@ def _library_versions() -> dict: def _representation_summary(preprocessor) -> list: - """Best-effort declarative per-representation summary (family/columns/locations).""" + """Best-effort declarative per-representation summary (family/columns/locations). + + One entry per block whose representation ends in a PreTab transformer. For a + missing-indicator / separate-state block that is the last step of its + representation branch (see :func:`~pretab.compose.inspection.representation_leaf`), + not the union itself. + """ summary: list = [] column_transformer = getattr(preprocessor, "column_transformer_", None) if column_transformer is None: @@ -235,7 +242,7 @@ def _representation_summary(preprocessor) -> list: for name, transformer, columns in column_transformer.transformers_: if name == "remainder": continue - leaf = transformer.steps[-1][1] if hasattr(transformer, "steps") else transformer + leaf = representation_leaf(transformer) spec_fn = getattr(leaf, "get_representation_spec", None) if spec_fn is None: continue diff --git a/pretab/preprocessor.py b/pretab/preprocessor.py index a8193b8..fb2a0da 100644 --- a/pretab/preprocessor.py +++ b/pretab/preprocessor.py @@ -20,6 +20,7 @@ build_transformer_summary, clean_feature_names, get_output_slices, + representation_leaf, ) from .compose.output import ( compute_output_report, @@ -215,10 +216,12 @@ class Preprocessor(TransformerMixin, BaseEstimator): Strategy for the ``SimpleImputer`` that runs *before* every categorical method. ``None`` disables imputation for categorical columns. add_missing_indicator : bool, default=False - If True, append a binary missing-value indicator column for each imputed feature (via the - imputer's ``add_indicator``; a standalone :class:`~pretab.transformers.MissingStateIndicator` - is used instead when imputation is disabled for that column kind). Applies to both - numerical and categorical pipelines. + If True, append a binary (0/1) missing-value indicator column for each imputed feature + that has missing values at ``fit`` (scikit-learn's ``MissingIndicator``, computed on the raw + input next to the representation, so it never passes through the scaler or basis; a + standalone :class:`~pretab.transformers.MissingStateIndicator` is used instead when + imputation is disabled for that column kind). Applies to both numerical and categorical + pipelines. missing_policy : {"error", "propagate", "impute", "impute_with_indicator", "separate_state"} or None, default=None High-level missing-value strategy. ``None`` (default) keeps the explicit ``numerical_imputation`` / ``categorical_imputation`` / ``add_missing_indicator`` @@ -229,7 +232,8 @@ class Preprocessor(TransformerMixin, BaseEstimator): - ``"propagate"`` -- disable imputation so NaNs reach the transformers unchanged (each family applies its own missing-value contract). - ``"impute"`` -- impute with the configured strategy (no indicator). - - ``"impute_with_indicator"`` -- impute and append a missing indicator column. + - ``"impute_with_indicator"`` -- impute and append a raw 0/1 missing indicator column, + as ``add_missing_indicator=True`` does. - ``"separate_state"`` -- impute for the representation *and* emit a dedicated ``__missing`` column per feature that stays outside the ordinary basis, so a downstream model can learn a separate response to missingness. @@ -982,7 +986,7 @@ def get_feature_info(self, verbose=True): def _log_internal_decisions(self): """Log fitted internal decisions (bins / knots / centers) at DEBUG.""" for name, transformer, _columns in self.column_transformer_.transformers_: - last_step = transformer.steps[-1][1] if hasattr(transformer, "steps") else transformer + last_step = representation_leaf(transformer) for attr in ( "thresholds_", "knots_", diff --git a/tests/integration/test_missing_policy.py b/tests/integration/test_missing_policy.py index c7ec70e..a85e1ef 100644 --- a/tests/integration/test_missing_policy.py +++ b/tests/integration/test_missing_policy.py @@ -1,5 +1,7 @@ """Tests for the high-level ``missing_policy`` orchestration knob (P8.5).""" +import json + import numpy as np import pandas as pd import pytest @@ -262,3 +264,108 @@ def test_add_missing_indicator_no_longer_requires_imputation_enabled(frame_with_ assert out.size > 0 assert np.isnan(out).any() assert p.total_output_dim_ > 0 + + +# --- imputer indicator stays outside the representation (issue #62) ----------- + + +def _indicator_values(pre, frame, name): + names = list(pre.get_feature_names_out()) + return np.asarray(pre.transform(frame, return_array=True), dtype=float)[:, names.index(name)] + + +@pytest.mark.parametrize("options", [{"add_missing_indicator": True}, {"missing_policy": "impute_with_indicator"}]) +def test_imputer_indicator_is_a_raw_binary_column(frame_with_nan, y, options): + """Regression guard for issue #62: the indicator was standardized / expanded + by the representation like a second input feature.""" + pre = Preprocessor(numerical_method="standardization", **options).fit(frame_with_nan, y) + values = _indicator_values(pre, frame_with_nan, "num_a__missingindicator_a") + np.testing.assert_array_equal(values, frame_with_nan["a"].isna().astype(float)) + + +@pytest.mark.parametrize("method", ["bspline", "rbf", "pspline", "fourier", "cubicspline", "polynomial"]) +def test_imputer_indicator_adds_exactly_one_column_per_feature_with_missing(frame_with_nan, y, method): + plain = Preprocessor(numerical_method=method, output_dim=5, missing_policy="impute").fit(frame_with_nan, y) + pre = Preprocessor(numerical_method=method, output_dim=5, missing_policy="impute_with_indicator").fit( + frame_with_nan, y + ) + names = list(pre.get_feature_names_out()) + indicators = [name for name in names if "missingindicator" in name] + assert indicators == ["num_a__missingindicator_a", "num_b__missingindicator_b"] + assert [name for name in names if name not in indicators] == list(plain.get_feature_names_out()) + assert pre.output_dims_ == {key: width + 1 for key, width in plain.output_dims_.items()} + + +@pytest.mark.parametrize("method", ["bspline", "rbf", "pspline", "fourier", "cubicspline"]) +def test_imputer_indicator_lineage_and_feature_info(frame_with_nan, y, method): + pre = Preprocessor(numerical_method=method, output_dim=5, add_missing_indicator=True).fit(frame_with_nan, y) + lineage = pre.get_feature_lineage() + assert [record.output_feature for record in lineage] == list(pre.get_feature_names_out()) + missing = [record for record in lineage if record.family == "missing_state"] + assert [record.output_feature for record in missing] == ["num_a__missingindicator_a", "num_b__missingindicator_b"] + numerical, _, _ = pre.get_feature_info(verbose=False) + assert {feature: info["dimension"] for feature, info in numerical.items()} == pre.output_dims_ + + +def test_imputer_indicator_on_categorical_integer_codes(y): + frame = pd.DataFrame({"c": ["x", "y", np.nan, "x", "y", "x"]}) + pre = Preprocessor(categorical_method="int", add_missing_indicator=True).fit(frame, y) + assert list(pre.get_feature_names_out()) == ["cat_c", "cat_c__missingindicator_c"] + np.testing.assert_array_equal(_indicator_values(pre, frame, "cat_c__missingindicator_c"), frame["c"].isna()) + + +def test_imputer_indicator_only_marks_features_with_missing_values_at_fit(clean_frame, frame_with_nan, y): + pre = Preprocessor(numerical_method="minmax", add_missing_indicator=True).fit(clean_frame, y) + assert not any("missing" in name for name in pre.get_feature_names_out()) + assert np.asarray(pre.transform(frame_with_nan, return_array=True)).shape == (6, 2) + + +# --- representation summary of missing-indicator / separate-state blocks -------- + + +@pytest.mark.parametrize( + "options", + [ + {"add_missing_indicator": True}, + {"missing_policy": "impute_with_indicator"}, + {"missing_policy": "separate_state"}, + ], +) +def test_missing_state_blocks_report_their_representation(frame_with_nan, y, options): + """The spec summary read each block's last step, which for these blocks is the + FeatureUnion of the representation and the missing indicator, so to_spec() and + reproducibility_report() listed no representation for them.""" + plain = _bspline(missing_policy="impute").fit(frame_with_nan, y) + pre = _bspline(**options).fit(frame_with_nan, y) + + assert pre.reproducibility_report()["representations"] == {"a": "bspline", "b": "bspline"} + assert pre.to_spec()["representations"] == plain.to_spec()["representations"] + + +def test_block_with_the_imputer_built_in_indicator_still_loads_and_reports(frame_with_nan, y): + """A preprocessor fitted by PreTab 1.0.0 with add_missing_indicator has the + imputer's built-in indicator in front of the representation (a pipeline, not + a union). Such an object still transforms, round-trips through its spec with + the same fingerprint, and lists its representations.""" + from sklearn.pipeline import Pipeline + + pre = _bspline(add_missing_indicator=True).fit(frame_with_nan, y) + + def legacy_block(transformer, columns): + if not isinstance(transformer, Pipeline): + transformer = dict(transformer.transformer_list)["representation"] + block = clone(transformer) + assert isinstance(block, Pipeline) + return block.set_params(imputer__add_indicator=True).fit(frame_with_nan[columns], y) + + pre.column_transformer_.transformers_ = [ + (name, legacy_block(transformer, columns), columns) + for name, transformer, columns in pre.column_transformer_.transformers_ + ] + out = np.asarray(pre.transform(frame_with_nan, return_array=True)) + restored = Preprocessor.from_spec(json.loads(json.dumps(pre.to_spec()))) + + assert out.shape == (6, 2 * 2 * 8) # the old layout expands the indicator too + np.testing.assert_array_equal(np.asarray(restored.transform(frame_with_nan, return_array=True)), out) + assert restored.fingerprint_ == pre.fingerprint_ + assert restored.reproducibility_report()["representations"] == {"a": "bspline", "b": "bspline"} diff --git a/tests/integration/test_verbosity.py b/tests/integration/test_verbosity.py index 3c44e1e..0ddb6f8 100644 --- a/tests/integration/test_verbosity.py +++ b/tests/integration/test_verbosity.py @@ -119,6 +119,17 @@ def test_verbose_3_logs_internal_decisions(sample_data, caplog): assert "thresholds_" in debug_text or "total_output_dim_" in debug_text +@pytest.mark.parametrize("options", [{"add_missing_indicator": True}, {"missing_policy": "separate_state"}]) +def test_verbose_3_logs_internal_decisions_of_missing_state_blocks(sample_data, caplog, options): + """These blocks are a FeatureUnion of the representation and the missing + indicator; the union itself has no fitted thresholds, so none were logged.""" + X, y = sample_data + caplog.set_level(logging.DEBUG, logger="pretab") + Preprocessor(numerical_method="ple", verbose=3, **options).fit(X, y) + debug_text = "\n".join(r.getMessage() for r in caplog.records if r.levelno == logging.DEBUG) + assert "num_num1.thresholds_" in debug_text + + def test_verbose_true_behaves_like_level_1(sample_data, caplog): X, y = sample_data caplog.set_level(logging.DEBUG, logger="pretab")