Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions docs/core_concepts/target_awareness.md
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,25 @@ search.best_method_
best_preprocessor_` and `search.best_estimator_` are refit on all the data with `bspline` and
ready to call `.predict(X_new)`.

When several rows belong to the same unit, such as repeated measurements of one patient, a
row-level split validates each candidate on units it has already seen in training. Pass a
group splitter as `cv` and the group labels to `fit`, so no group is on both sides of a
train/validation split:

```python
from sklearn.model_selection import GroupKFold

patient_id = np.repeat(np.arange(30), 10) # 30 patients with 10 rows each

search = RepresentationSearchCV(Ridge(), methods=["minmax", "bspline"], cv=GroupKFold(5))
search.fit(X, y, groups=patient_id)
```

Any other keyword argument to `fit`, such as `sample_weight`, is passed to the estimator's
`fit` on every fold (restricted to that fold's training rows) and on the final refit. The
`Preprocessor` and the scorer do not see it, so placement and validation scores are
unweighted.

```{note}
This is deliberately narrow: it only searches the single `numerical_method` axis with one
global method for every numerical column, not a per-column `feature_preprocessing` search.
Expand Down
42 changes: 36 additions & 6 deletions pretab/compose/search.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from sklearn.base import BaseEstimator, clone, is_classifier
from sklearn.metrics import check_scoring
from sklearn.model_selection import BaseCrossValidator, check_cv
from sklearn.utils.validation import check_is_fitted
from sklearn.utils.validation import _check_method_params, check_is_fitted

from ..core._typing import PredictorLike
from ..core.validation import is_polars_frame, polars_to_pandas
Expand Down Expand Up @@ -43,7 +43,9 @@ class RepresentationSearchCV(BaseEstimator):
Candidate ``numerical_method`` values to search over.
cv : int or cross-validation generator, default=5
Cross-validation splitting strategy passed to
:func:`sklearn.model_selection.check_cv`.
:func:`sklearn.model_selection.check_cv`. A group splitter such as
:class:`~sklearn.model_selection.GroupKFold` needs ``groups`` at
:meth:`fit`.
scoring : str or callable or None, default=None
Scoring passed to :func:`sklearn.metrics.check_scoring`; ``None`` uses the
estimator's ``score`` method.
Expand Down Expand Up @@ -88,12 +90,36 @@ def _make_preprocessor(self, method):
params.setdefault("task", "classification")
return Preprocessor(numerical_method=method, **params)

def fit(self, X, y=None):
def fit(self, X, y=None, *, groups=None, **fit_params):
"""Search over ``methods`` and refit the best representation on all data.

``X`` may be a pandas or polars DataFrame or an array. A polars frame is
converted to pandas once, before the fold rows are taken, so every fold's
Preprocessor detects the same column types as the final refit.

Parameters
----------
X : pandas.DataFrame, polars.DataFrame or array-like of shape (n_samples, n_features)
Input features, transformed by each candidate :class:`Preprocessor`.
y : array-like of shape (n_samples,)
Target values. Required.
groups : array-like of shape (n_samples,), default=None
Group labels passed to ``cv.split``, so a group splitter (for example
:class:`~sklearn.model_selection.GroupKFold`) keeps every group on one
side of each train/validation split. Ignored by splitters that do not
use groups, and not passed to the estimator.
**fit_params : dict
Parameters passed to the ``fit`` method of ``estimator`` on every fold
and on the final refit, for example ``sample_weight``. Array-like values
of length ``n_samples`` are restricted to the training rows of each fold.
They are not passed to the :class:`Preprocessor` (its target-aware
placement is unweighted) or to the scorer (validation scores are
unweighted).

Returns
-------
self : RepresentationSearchCV
Fitted search.
"""
methods = list(self.methods)
if not methods:
Expand All @@ -106,10 +132,13 @@ def fit(self, X, y=None):
# as a pandas frame's do (see polars_to_pandas).
X = polars_to_pandas(X)
n_samples = X.shape[0] if hasattr(X, "shape") else len(X)
# As in scikit-learn's searches: per-sample fit params are made indexable
# so each fold can take its training rows.
fit_params = _check_method_params(X, params=fit_params)
cv = cast(BaseCrossValidator, check_cv(self.cv, y_arr, classifier=is_classifier(self.estimator)))
# Reuse the same held-out rows for every candidate, including splitters
# whose random state advances on each call to split().
splits = list(cv.split(np.zeros(n_samples), y_arr))
splits = list(cv.split(np.zeros(n_samples), y_arr, groups))

cv_results: dict[str, float] = {}
best_score = -np.inf
Expand All @@ -120,7 +149,8 @@ def fit(self, X, y=None):
pre = self._make_preprocessor(method)
est = cast(PredictorLike, clone(self.estimator))
x_train = pre.fit_transform(_row_subset(X, train_idx), y_arr[train_idx], return_array=True)
est.fit(x_train, y_arr[train_idx])
fold_params = _check_method_params(X, params=fit_params, indices=train_idx)
est.fit(x_train, y_arr[train_idx], **fold_params)
scorer = cast("Callable[..., float]", check_scoring(est, scoring=self.scoring))
x_test = pre.transform(_row_subset(X, test_idx), return_array=True)
fold_scores.append(scorer(est, x_test, y_arr[test_idx]))
Expand All @@ -135,7 +165,7 @@ def fit(self, X, y=None):
self.best_score_ = best_score
self.best_preprocessor_ = self._make_preprocessor(best_method)
x_all = self.best_preprocessor_.fit_transform(X, y_arr, return_array=True)
self.best_estimator_ = cast(PredictorLike, clone(self.estimator)).fit(x_all, y_arr)
self.best_estimator_ = cast(PredictorLike, clone(self.estimator)).fit(x_all, y_arr, **fit_params)
return self

def predict(self, X):
Expand Down
86 changes: 84 additions & 2 deletions tests/compose/test_search.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,10 @@
import numpy as np
import pandas as pd
import pytest
from sklearn.base import BaseEstimator, RegressorMixin
from sklearn.exceptions import NotFittedError
from sklearn.linear_model import LinearRegression, LogisticRegression
from sklearn.model_selection import KFold
from sklearn.linear_model import LinearRegression, LogisticRegression, Ridge
from sklearn.model_selection import GroupKFold, GroupShuffleSplit, KFold, LeaveOneGroupOut

from pretab import RepresentationSearchCV
from pretab.exceptions import InvalidParamError
Expand Down Expand Up @@ -126,6 +127,87 @@ def test_get_params_and_clone_preserve_config():
assert cloned.get_params()["cv"] == 3


class _RowRecorder(RegressorMixin, BaseEstimator):
"""Mean regressor whose ``fit`` records the ``row_id`` fit parameter it gets."""

def fit(self, X, y, row_id=None):
self.row_id_ = row_id
self.mean_ = float(np.mean(y))
return self

def predict(self, X):
return np.full(len(X), self.mean_)


def _recording_scorer(fold_rows):
"""Scorer that stores the ``row_id`` of each fold's fitted estimator."""

def score(est, x_test, y_test):
fold_rows.append(np.asarray(est.row_id_))
return 0.0

return score


@pytest.fixture
def grouped_data(nonlinear_data):
"""``nonlinear_data`` with 20 groups of 10 rows (e.g. 20 patients)."""
X, y = nonlinear_data
return X, y, np.repeat(np.arange(20), 10)


@pytest.mark.parametrize(
"cv",
[GroupKFold(n_splits=4), LeaveOneGroupOut(), GroupShuffleSplit(n_splits=3, test_size=0.25, random_state=0)],
ids=["group_kfold", "leave_one_group_out", "group_shuffle_split"],
)
def test_group_splitter_receives_groups(grouped_data, cv):
"""Group splitters always failed: fit() took no groups and split() got none."""
X, y, groups = grouped_data
fold_rows = []
search = _search(_RowRecorder(), ["minmax"], cv=cv, scoring=_recording_scorer(fold_rows))
search.fit(X, y, groups=groups, row_id=np.arange(len(X)))

for rows, (train, test) in zip(fold_rows, cv.split(X, y, groups), strict=True):
np.testing.assert_array_equal(rows, train)
# No group is on both sides of a train/validation split.
assert set(groups[rows]).isdisjoint(groups[test])


@pytest.mark.filterwarnings("ignore:The groups parameter is ignored:UserWarning")
def test_groups_leave_non_group_splits_unchanged(grouped_data):
X, y, groups = grouped_data
plain = _search(LinearRegression(), ["standardization", "bspline"]).fit(X, y)
grouped = _search(LinearRegression(), ["standardization", "bspline"]).fit(X, y, groups=groups)
assert grouped.cv_results_ == plain.cv_results_


def test_fit_params_are_split_per_fold_and_passed_to_the_refit(nonlinear_data):
X, y = nonlinear_data
row_id = np.arange(len(X))
cv = KFold(n_splits=4, shuffle=True, random_state=0)
fold_rows = []
search = _search(_RowRecorder(), ["minmax"], cv=cv, scoring=_recording_scorer(fold_rows))
search.fit(X, y, row_id=row_id)

# Each fold's estimator sees only its training rows; the refit sees all rows.
for rows, (train, _) in zip(fold_rows, cv.split(X), strict=True):
np.testing.assert_array_equal(rows, train)
np.testing.assert_array_equal(search.best_estimator_.row_id_, row_id)


def test_sample_weight_reaches_the_estimator(nonlinear_data):
X, y = nonlinear_data
weights = np.linspace(0.1, 2.0, len(X))
search = _search(Ridge(), ["bspline"]).fit(X, y, sample_weight=weights)

x_all = search.best_preprocessor_.transform(X, return_array=True)
expected = Ridge().fit(x_all, y, sample_weight=weights)
np.testing.assert_allclose(search.best_estimator_.coef_, expected.coef_)
# The fold fits are weighted as well, so the validation scores change.
assert search.cv_results_ != _search(Ridge(), ["bspline"]).fit(X, y).cv_results_


@pytest.fixture
def class_data():
rng = np.random.default_rng(0)
Expand Down
Loading