Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion docs/core_concepts/target_awareness.md
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,16 @@ this for the training features specifically. It splits the training data into fo
fresh copy of the transformer on all folds _except_ one, and uses that copy to transform the
held-out fold, so every training row is transformed by a model that never saw its own target.
`transform` on genuinely new data (a validation or test set) instead uses one model fit on all
the training data, since there is no leakage risk there.
the training data, since there is no leakage risk there. Both return the same kind of output,
the one the wrapped transformer produces: a dense array of the same dtype, a sparse matrix in
the same format, or a pandas/Polars DataFrame. Fold outputs are restored to the input row
order. A pandas output keeps the input index when the wrapped transformer preserves it;
otherwise it uses a fresh row index.

Cross-fitting currently requires matrix output. A wrapped `Preprocessor` configured with
`output_structure="blocks"` raises an explicit error; use `output_structure="matrix"` instead.
Each fold must return a 2D output with one row per held-out sample, a fixed width, and, for
DataFrames, the same column labels in the same order as the all-data fit.

A wrapped `Preprocessor` is not refit from scratch on each fold: every fold reuses its all-data
fit and refits only the target-aware representations and subsequent pipeline steps. Column
Expand Down
99 changes: 92 additions & 7 deletions pretab/core/supervised.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,11 @@
import sys
import warnings
from dataclasses import replace
from typing import cast
from typing import Any, cast

import numpy as np
import pandas as pd
from scipy import sparse as sp
from sklearn.base import BaseEstimator, TransformerMixin, clone
from sklearn.model_selection import KFold, StratifiedKFold
from sklearn.utils.validation import check_is_fitted
Expand Down Expand Up @@ -89,6 +91,41 @@ def warn_target_leakage(estimator, y) -> None:
)


def _stack_folds(blocks, test_indices, X) -> Any:
"""Stack the out-of-fold ``blocks`` and put their rows back in the order of ``X``.

``blocks[i]`` holds the transformed rows ``test_indices[i]``. The result keeps
the kind of output the wrapped transformer returns: sparse blocks stay sparse
(in the format of the first block), pandas / polars DataFrames keep their
columns and dtypes, and dense blocks take their common dtype (``object`` for
strings left unchanged). A pandas DataFrame keeps the index of ``X`` when the
blocks carry it (scikit-learn's ``set_output`` copies the input index);
otherwise it is renumbered from 0, as ``transform`` numbers the rows of a
transformer that builds its own index (such as a Preprocessor).
"""
rows = np.concatenate(test_indices)
# order[i] is the position of input row i among the stacked fold rows.
order = np.empty_like(rows)
order[rows] = np.arange(len(rows))
first = blocks[0]
if all(sp.issparse(block) for block in blocks):
# Older SciPy vstack returns a matrix even when its inputs are sparse arrays.
return type(first)(sp.vstack(blocks).tocsr()[order])
if all(isinstance(block, pd.DataFrame) for block in blocks):
stacked = pd.concat(blocks).iloc[order]
index = getattr(X, "index", None)
return stacked if index is not None and stacked.index.equals(index) else stacked.reset_index(drop=True)
if is_polars_frame(first):
import polars as pl # type: ignore

return pl.concat(blocks)[order]
dense = [block.toarray() if sp.issparse(block) else np.asarray(block) for block in blocks]
out = np.empty((len(rows), dense[0].shape[1]), dtype=np.result_type(*dense))
for block, test_idx in zip(dense, test_indices, strict=True):
out[test_idx] = block
return out


def _as_2d(X):
"""Return ``X`` as 2D input for the wrapped transformer.

Expand Down Expand Up @@ -119,6 +156,8 @@ class CrossFittedTransformer(RepresentationSpecMixin, TransformerMixin, BaseEsti
training fold and used to transform the held-out fold, so every training row
is encoded by a model that never saw its own target. :meth:`transform`
(for unseen data) uses ``estimator_``, a single transformer fit on all data.
Both return the wrapped transformer's kind of output: a dense array of the
same dtype, a sparse matrix in the same format, or a DataFrame.

Parameters
----------
Expand Down Expand Up @@ -215,16 +254,46 @@ def _fit_fold(self, X, y):
return fold

def fit_transform(self, X, y=None):
"""Fit and return leakage-free out-of-fold features for the training data."""
"""Fit and return leakage-free out-of-fold features for the training data.

Returns
-------
array-like of shape (n_samples, n_features_out)
The out-of-fold features, in the row order of ``X`` and in the kind of
output :meth:`transform` returns: a sparse matrix in the same format, a
DataFrame, or a dense array with the folds' dtype (``object`` when the
wrapped transformer leaves strings unchanged).

Raises
------
IncompatibleParamsError
If the wrapped transformer returns an unsupported dict of blocks,
or if a fold's output has an invalid shape, a different width, or
DataFrame columns other than the all-data fit's.
"""
X_arr, y_arr = self._fit_full(X, y)
width = len(self.estimator_.get_feature_names_out())
out = np.empty((X_arr.shape[0], width), dtype=float)
names = list(self.estimator_.get_feature_names_out())
width = len(names)
blocks, test_indices = [], []
splitter = self._make_splitter()
token = _cross_fit_active.set(True)
try:
for train_idx, test_idx in splitter.split(X_arr, y_arr):
fold = self._fit_fold(_take_rows(X_arr, train_idx), y_arr[train_idx])
fold_out = np.asarray(fold.transform(_take_rows(X_arr, test_idx)))
fold_out = fold.transform(_take_rows(X_arr, test_idx))
if isinstance(fold_out, dict):
raise IncompatibleParamsError(
"Cross-fitting stacks the out-of-fold features into one array; the "
"wrapped transformer returned a dict of blocks. Use output_structure="
"'matrix' on a wrapped Preprocessor."
)
if not hasattr(fold_out, "shape"):
fold_out = np.asarray(fold_out)
if len(fold_out.shape) != 2 or fold_out.shape[0] != len(test_idx):
raise IncompatibleParamsError(
f"Cross-fitting received invalid output shape {fold_out.shape}; "
f"expected a 2D output with {len(test_idx)} rows, one per held-out sample."
)
if fold_out.shape[1] != width:
raise IncompatibleParamsError(
"Cross-fitting requires a fixed output width across folds; expected "
Expand All @@ -233,10 +302,26 @@ def fit_transform(self, X, y=None):
"indicator columns learned from the data); configure it to produce the "
"same columns on every fold."
)
out[test_idx] = fold_out
# DataFrames are stacked by column label, so a fold that kept other
# columns at the same width (e.g. other top categories) cannot be aligned.
columns = getattr(fold_out, "columns", None)
if columns is not None and list(columns) != names:
differing = sorted(set(columns).symmetric_difference(names), key=str)
detail = (
f"columns differ from the all-data fit in {differing}"
if differing
else f"column order differs from the all-data fit: expected {names}, got {list(columns)}"
)
raise IncompatibleParamsError(
f"Cross-fitting requires the same output columns across folds; a fold's {detail}. The wrapped "
"transformer learns its columns from the rows it is fit on; configure it "
"to produce the same columns on every fold."
)
blocks.append(fold_out)
test_indices.append(test_idx)
finally:
_cross_fit_active.reset(token)
return out
return _stack_folds(blocks, test_indices, X_arr)

def get_feature_names_out(self, input_features=None):
"""Delegate output feature names to the all-data ``estimator_``."""
Expand Down
Loading
Loading