diff --git a/docs/core_concepts/target_awareness.md b/docs/core_concepts/target_awareness.md index df17127..3b74e94 100644 --- a/docs/core_concepts/target_awareness.md +++ b/docs/core_concepts/target_awareness.md @@ -195,6 +195,25 @@ search.best_method_ best_preprocessor_` and `search.best_estimator_` are refit on all the data with `bspline` and ready to call `.predict(X_new)`. +When several rows belong to the same unit, such as repeated measurements of one patient, a +row-level split validates each candidate on units it has already seen in training. Pass a +group splitter as `cv` and the group labels to `fit`, so no group is on both sides of a +train/validation split: + +```python +from sklearn.model_selection import GroupKFold + +patient_id = np.repeat(np.arange(30), 10) # 30 patients with 10 rows each + +search = RepresentationSearchCV(Ridge(), methods=["minmax", "bspline"], cv=GroupKFold(5)) +search.fit(X, y, groups=patient_id) +``` + +Any other keyword argument to `fit`, such as `sample_weight`, is passed to the estimator's +`fit` on every fold (restricted to that fold's training rows) and on the final refit. The +`Preprocessor` and the scorer do not see it, so placement and validation scores are +unweighted. + ```{note} This is deliberately narrow: it only searches the single `numerical_method` axis with one global method for every numerical column, not a per-column `feature_preprocessing` search. diff --git a/pretab/compose/search.py b/pretab/compose/search.py index df03818..16680fa 100644 --- a/pretab/compose/search.py +++ b/pretab/compose/search.py @@ -15,7 +15,7 @@ from sklearn.base import BaseEstimator, clone, is_classifier from sklearn.metrics import check_scoring from sklearn.model_selection import BaseCrossValidator, check_cv -from sklearn.utils.validation import check_is_fitted +from sklearn.utils.validation import _check_method_params, check_is_fitted from ..core._typing import PredictorLike from ..core.validation import is_polars_frame, polars_to_pandas @@ -43,7 +43,9 @@ class RepresentationSearchCV(BaseEstimator): Candidate ``numerical_method`` values to search over. cv : int or cross-validation generator, default=5 Cross-validation splitting strategy passed to - :func:`sklearn.model_selection.check_cv`. + :func:`sklearn.model_selection.check_cv`. A group splitter such as + :class:`~sklearn.model_selection.GroupKFold` needs ``groups`` at + :meth:`fit`. scoring : str or callable or None, default=None Scoring passed to :func:`sklearn.metrics.check_scoring`; ``None`` uses the estimator's ``score`` method. @@ -88,12 +90,36 @@ def _make_preprocessor(self, method): params.setdefault("task", "classification") return Preprocessor(numerical_method=method, **params) - def fit(self, X, y=None): + def fit(self, X, y=None, *, groups=None, **fit_params): """Search over ``methods`` and refit the best representation on all data. ``X`` may be a pandas or polars DataFrame or an array. A polars frame is converted to pandas once, before the fold rows are taken, so every fold's Preprocessor detects the same column types as the final refit. + + Parameters + ---------- + X : pandas.DataFrame, polars.DataFrame or array-like of shape (n_samples, n_features) + Input features, transformed by each candidate :class:`Preprocessor`. + y : array-like of shape (n_samples,) + Target values. Required. + groups : array-like of shape (n_samples,), default=None + Group labels passed to ``cv.split``, so a group splitter (for example + :class:`~sklearn.model_selection.GroupKFold`) keeps every group on one + side of each train/validation split. Ignored by splitters that do not + use groups, and not passed to the estimator. + **fit_params : dict + Parameters passed to the ``fit`` method of ``estimator`` on every fold + and on the final refit, for example ``sample_weight``. Array-like values + of length ``n_samples`` are restricted to the training rows of each fold. + They are not passed to the :class:`Preprocessor` (its target-aware + placement is unweighted) or to the scorer (validation scores are + unweighted). + + Returns + ------- + self : RepresentationSearchCV + Fitted search. """ methods = list(self.methods) if not methods: @@ -106,10 +132,13 @@ def fit(self, X, y=None): # as a pandas frame's do (see polars_to_pandas). X = polars_to_pandas(X) n_samples = X.shape[0] if hasattr(X, "shape") else len(X) + # As in scikit-learn's searches: per-sample fit params are made indexable + # so each fold can take its training rows. + fit_params = _check_method_params(X, params=fit_params) cv = cast(BaseCrossValidator, check_cv(self.cv, y_arr, classifier=is_classifier(self.estimator))) # Reuse the same held-out rows for every candidate, including splitters # whose random state advances on each call to split(). - splits = list(cv.split(np.zeros(n_samples), y_arr)) + splits = list(cv.split(np.zeros(n_samples), y_arr, groups)) cv_results: dict[str, float] = {} best_score = -np.inf @@ -120,7 +149,8 @@ def fit(self, X, y=None): pre = self._make_preprocessor(method) est = cast(PredictorLike, clone(self.estimator)) x_train = pre.fit_transform(_row_subset(X, train_idx), y_arr[train_idx], return_array=True) - est.fit(x_train, y_arr[train_idx]) + fold_params = _check_method_params(X, params=fit_params, indices=train_idx) + est.fit(x_train, y_arr[train_idx], **fold_params) scorer = cast("Callable[..., float]", check_scoring(est, scoring=self.scoring)) x_test = pre.transform(_row_subset(X, test_idx), return_array=True) fold_scores.append(scorer(est, x_test, y_arr[test_idx])) @@ -135,7 +165,7 @@ def fit(self, X, y=None): self.best_score_ = best_score self.best_preprocessor_ = self._make_preprocessor(best_method) x_all = self.best_preprocessor_.fit_transform(X, y_arr, return_array=True) - self.best_estimator_ = cast(PredictorLike, clone(self.estimator)).fit(x_all, y_arr) + self.best_estimator_ = cast(PredictorLike, clone(self.estimator)).fit(x_all, y_arr, **fit_params) return self def predict(self, X): diff --git a/tests/compose/test_search.py b/tests/compose/test_search.py index c3e065f..b64181a 100644 --- a/tests/compose/test_search.py +++ b/tests/compose/test_search.py @@ -9,9 +9,10 @@ import numpy as np import pandas as pd import pytest +from sklearn.base import BaseEstimator, RegressorMixin from sklearn.exceptions import NotFittedError -from sklearn.linear_model import LinearRegression, LogisticRegression -from sklearn.model_selection import KFold +from sklearn.linear_model import LinearRegression, LogisticRegression, Ridge +from sklearn.model_selection import GroupKFold, GroupShuffleSplit, KFold, LeaveOneGroupOut from pretab import RepresentationSearchCV from pretab.exceptions import InvalidParamError @@ -126,6 +127,87 @@ def test_get_params_and_clone_preserve_config(): assert cloned.get_params()["cv"] == 3 +class _RowRecorder(RegressorMixin, BaseEstimator): + """Mean regressor whose ``fit`` records the ``row_id`` fit parameter it gets.""" + + def fit(self, X, y, row_id=None): + self.row_id_ = row_id + self.mean_ = float(np.mean(y)) + return self + + def predict(self, X): + return np.full(len(X), self.mean_) + + +def _recording_scorer(fold_rows): + """Scorer that stores the ``row_id`` of each fold's fitted estimator.""" + + def score(est, x_test, y_test): + fold_rows.append(np.asarray(est.row_id_)) + return 0.0 + + return score + + +@pytest.fixture +def grouped_data(nonlinear_data): + """``nonlinear_data`` with 20 groups of 10 rows (e.g. 20 patients).""" + X, y = nonlinear_data + return X, y, np.repeat(np.arange(20), 10) + + +@pytest.mark.parametrize( + "cv", + [GroupKFold(n_splits=4), LeaveOneGroupOut(), GroupShuffleSplit(n_splits=3, test_size=0.25, random_state=0)], + ids=["group_kfold", "leave_one_group_out", "group_shuffle_split"], +) +def test_group_splitter_receives_groups(grouped_data, cv): + """Group splitters always failed: fit() took no groups and split() got none.""" + X, y, groups = grouped_data + fold_rows = [] + search = _search(_RowRecorder(), ["minmax"], cv=cv, scoring=_recording_scorer(fold_rows)) + search.fit(X, y, groups=groups, row_id=np.arange(len(X))) + + for rows, (train, test) in zip(fold_rows, cv.split(X, y, groups), strict=True): + np.testing.assert_array_equal(rows, train) + # No group is on both sides of a train/validation split. + assert set(groups[rows]).isdisjoint(groups[test]) + + +@pytest.mark.filterwarnings("ignore:The groups parameter is ignored:UserWarning") +def test_groups_leave_non_group_splits_unchanged(grouped_data): + X, y, groups = grouped_data + plain = _search(LinearRegression(), ["standardization", "bspline"]).fit(X, y) + grouped = _search(LinearRegression(), ["standardization", "bspline"]).fit(X, y, groups=groups) + assert grouped.cv_results_ == plain.cv_results_ + + +def test_fit_params_are_split_per_fold_and_passed_to_the_refit(nonlinear_data): + X, y = nonlinear_data + row_id = np.arange(len(X)) + cv = KFold(n_splits=4, shuffle=True, random_state=0) + fold_rows = [] + search = _search(_RowRecorder(), ["minmax"], cv=cv, scoring=_recording_scorer(fold_rows)) + search.fit(X, y, row_id=row_id) + + # Each fold's estimator sees only its training rows; the refit sees all rows. + for rows, (train, _) in zip(fold_rows, cv.split(X), strict=True): + np.testing.assert_array_equal(rows, train) + np.testing.assert_array_equal(search.best_estimator_.row_id_, row_id) + + +def test_sample_weight_reaches_the_estimator(nonlinear_data): + X, y = nonlinear_data + weights = np.linspace(0.1, 2.0, len(X)) + search = _search(Ridge(), ["bspline"]).fit(X, y, sample_weight=weights) + + x_all = search.best_preprocessor_.transform(X, return_array=True) + expected = Ridge().fit(x_all, y, sample_weight=weights) + np.testing.assert_allclose(search.best_estimator_.coef_, expected.coef_) + # The fold fits are weighted as well, so the validation scores change. + assert search.cv_results_ != _search(Ridge(), ["bspline"]).fit(X, y).cv_results_ + + @pytest.fixture def class_data(): rng = np.random.default_rng(0)