Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion docs/core_concepts/target_awareness.md
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,9 @@ to fit it) leaks information the same way an unguarded supervised transformer do
`numerical_method` values, refits the best one on all the data, and keeps every candidate's
scoring honest by fitting a fresh `Preprocessor` per fold. With a classifier as the estimator,
every `Preprocessor` uses `task="classification"`, so target-aware methods place against the
class labels; a `task` set in `preprocessor_params` takes precedence.
class labels; a `task` set in `preprocessor_params` takes precedence. The candidate names are
checked when `fit` starts, so a misspelled method raises before any candidate is
cross-validated.

```python
import numpy as np
Expand Down
67 changes: 61 additions & 6 deletions pretab/compose/search.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,9 @@

from ..core._typing import PredictorLike
from ..core.validation import is_polars_frame, polars_to_pandas
from ..exceptions import InvalidParamError
from ..exceptions import InvalidParamError, invalid_param_error
from ..preprocessor import Preprocessor
from .registry import NUMERICAL_ALIASES, NUMERICAL_METHODS, resolve_method

__all__ = ["RepresentationSearchCV"]

Expand Down Expand Up @@ -64,7 +65,12 @@ class RepresentationSearchCV(BaseEstimator):
estimator : estimator
Downstream supervised estimator fit on the transformed features.
methods : sequence of str
Candidate ``numerical_method`` values to search over.
Candidate ``numerical_method`` values to search over, e.g.
``["minmax", "bspline"]``. Names are resolved like
``Preprocessor(numerical_method=...)`` (aliases and ``None`` included) and
checked when :meth:`fit` starts, before any candidate is cross-validated.
A single string is rejected instead of being read as a sequence of
one-letter names.
cv : int or cross-validation generator, default=5
Cross-validation splitting strategy passed to
:func:`sklearn.model_selection.check_cv`. A group splitter such as
Expand All @@ -76,7 +82,8 @@ class RepresentationSearchCV(BaseEstimator):
preprocessor_params : dict or None, default=None
Extra keyword arguments forwarded to every :class:`Preprocessor`. When
``estimator`` is a classifier, ``task`` defaults to ``"classification"`` so
target-aware placement treats ``y`` as class labels.
target-aware placement treats ``y`` as class labels. ``numerical_method``
cannot be set here: the search sets it to each entry of ``methods``.
random_state : int or None, default=None
Seed forwarded to each :class:`Preprocessor`.

Expand Down Expand Up @@ -121,6 +128,48 @@ def _make_preprocessor(self, method):
params.setdefault("task", "classification")
return Preprocessor(numerical_method=method, **params)

def _validated_methods(self) -> list:
"""Return ``methods`` as a list, checked before any candidate is fitted.

A typo would otherwise only surface inside the first fold of that
candidate, after every candidate listed before it has been
cross-validated, and a single string would be searched letter by letter.
"""
name = type(self).__name__
if isinstance(self.methods, str):
raise invalid_param_error(
name,
"methods",
self.methods,
f"must be a sequence of numerical method names, not a single string; pass [{self.methods!r}] "
"to search one method",
)
methods = list(self.methods)
if not methods:
raise InvalidParamError("methods must be a non-empty sequence of numerical methods.")
for method in methods:
# None is Preprocessor's spelling of "none" (leave numerical columns unchanged).
known = method is None or (
isinstance(method, str)
and resolve_method(method, NUMERICAL_METHODS, NUMERICAL_ALIASES) in NUMERICAL_METHODS
)
if not known:
raise invalid_param_error(
name,
"methods",
method,
"every entry must name a numerical preprocessing method",
valid=set(NUMERICAL_METHODS),
)
if "numerical_method" in (self.preprocessor_params or {}):
raise invalid_param_error(
name,
"preprocessor_params",
self.preprocessor_params,
"must not set 'numerical_method': the search sets it to each entry of methods",
)
return methods

def fit(self, X, y=None, *, groups=None, **fit_params):
"""Search over ``methods`` and refit the best representation on all data.

Expand Down Expand Up @@ -151,10 +200,16 @@ def fit(self, X, y=None, *, groups=None, **fit_params):
-------
self : RepresentationSearchCV
Fitted search.

Raises
------
InvalidParamError
If ``methods`` is empty, a single string, or has an entry that does not
name a numerical method; if ``preprocessor_params`` sets
``numerical_method``; or if ``y`` is None. Raised before any candidate
is fitted.
"""
methods = list(self.methods)
if not methods:
raise InvalidParamError("methods must be a non-empty sequence of numerical methods.")
methods = self._validated_methods()
if y is None:
raise InvalidParamError("RepresentationSearchCV requires y at fit time; got y=None.")
y_arr = np.asarray(y).ravel()
Expand Down
48 changes: 48 additions & 0 deletions tests/compose/test_search.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,54 @@ def test_empty_methods_raises(nonlinear_data):
RepresentationSearchCV(LinearRegression(), methods=[]).fit(X, y)


def test_methods_as_a_single_string_is_rejected(nonlinear_data):
"""A string was iterated letter by letter: "bspline" failed as the unknown method 'b'."""
X, y = nonlinear_data
with pytest.raises(InvalidParamError, match=r"not a single string; pass \['bspline'\]"):
RepresentationSearchCV(LinearRegression(), methods="bspline").fit(X, y)


class _CountingRegressor(RegressorMixin, BaseEstimator):
"""Counts fits on the class, so clones made by the search are counted too."""

n_fits = 0

def fit(self, X, y):
type(self).n_fits += 1
return self

def predict(self, X):
return np.zeros(len(X))


@pytest.mark.parametrize("unknown", ["no-such-method", "one-hot", "tensorspline", 3])
def test_unknown_method_is_rejected_before_any_candidate_is_fitted(nonlinear_data, unknown):
"""An unknown name used to fail only after the candidates before it were cross-validated."""
X, y = nonlinear_data
_CountingRegressor.n_fits = 0
search = _search(_CountingRegressor(), ["minmax", unknown])
with pytest.raises(InvalidParamError, match=r"RepresentationSearchCV\.methods = .* is invalid"):
search.fit(X, y)
assert _CountingRegressor.n_fits == 0
assert not hasattr(search, "cv_results_")


def test_method_aliases_and_none_stay_valid(nonlinear_data):
X, y = nonlinear_data
search = _search(LinearRegression(), ["std", "B-Spline", None]).fit(X, y)
assert list(search.cv_results_) == ["std", "B-Spline", None]


def test_numerical_method_in_preprocessor_params_is_rejected(nonlinear_data):
"""It collided with the searched method in a bare TypeError from Preprocessor()."""
X, y = nonlinear_data
search = RepresentationSearchCV(
LinearRegression(), methods=["minmax"], preprocessor_params={"numerical_method": "ple"}
)
with pytest.raises(InvalidParamError, match="must not set 'numerical_method'"):
search.fit(X, y)


def test_requires_y_at_fit(nonlinear_data):
X, _ = nonlinear_data
with pytest.raises(InvalidParamError):
Expand Down