From e9d7c7e5d81205f0e27a1e634cd57496f1b3b089 Mon Sep 17 00:00:00 2001 From: ChrisW09 <50968720+ChrisW09@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:24:08 +0200 Subject: [PATCH 1/2] fix(preprocessor): route columns to the ColumnTransformer by label Preprocessor passed each column label to scikit-learn's ColumnTransformer as the column selector. ColumnTransformer reads an integer selector as a position, so integer labels that differ from their positions (after dropping, reordering or selecting columns of pd.DataFrame(array) or read_csv(header=None)) silently swapped columns: data, per-feature methods from feature_preprocessing, names and lineage referred to different columns. Non-contiguous labels or swapped columns of different types crashed. The ColumnTransformer is now fitted and applied on a frame whose labels are strings (a shallow relabelled copy only when needed, so the caller's frame is unchanged), and every step selects its column by that string label. Labels that collide as strings, such as 1 and "1", raise PretabDataError. output_dims_ and get_feature_info stay keyed by the original labels. A NumPy array passed to transform after a fit on a frame with non-string labels is still matched by position, as before. Fixes #55 --- pretab/compose/factory.py | 10 ++- pretab/compose/feature_detection.py | 35 +++++++- pretab/preprocessor.py | 73 ++++++++++++++-- tests/compose/test_feature_detection.py | 11 ++- tests/integration/test_column_labels.py | 106 ++++++++++++++++++++++++ 5 files changed, 223 insertions(+), 12 deletions(-) create mode 100644 tests/integration/test_column_labels.py diff --git a/pretab/compose/factory.py b/pretab/compose/factory.py index 8ced4d6..e394ceb 100644 --- a/pretab/compose/factory.py +++ b/pretab/compose/factory.py @@ -316,17 +316,21 @@ def build_column_transformer( Numerical features are prefixed ``num_`` and categorical features ``cat_`` to match the transformer names the Preprocessor exposes; untransformed columns - pass through via ``remainder="passthrough"``. + pass through via ``remainder="passthrough"``. Each step selects its column by + the label's string form: scikit-learn reads an integer selector as a + *position*, so the Preprocessor fits and transforms the ColumnTransformer on a + frame whose labels are strings (see + :func:`~pretab.compose.feature_detection.with_string_labels`). """ transformers = [] for feature in numerical_features: method = config.method_for(feature, is_numerical=True) pipeline = create_transformer(method, is_numerical=True, config=config) - transformers.append((f"num_{feature}", pipeline, [feature])) + transformers.append((f"num_{feature}", pipeline, [str(feature)])) for feature in categorical_features: method = config.method_for(feature, is_numerical=False) pipeline = create_transformer(method, is_numerical=False, config=config) - transformers.append((f"cat_{feature}", pipeline, [feature])) + transformers.append((f"cat_{feature}", pipeline, [str(feature)])) return ColumnTransformer( transformers=transformers, remainder="passthrough", diff --git a/pretab/compose/feature_detection.py b/pretab/compose/feature_detection.py index ebdb1cc..21709d1 100644 --- a/pretab/compose/feature_detection.py +++ b/pretab/compose/feature_detection.py @@ -5,12 +5,14 @@ sequence of delegations rather than inlining the classification heuristic. """ +from collections import Counter + import numpy as np import pandas as pd from ..exceptions import PretabDataError, invalid_param_error -__all__ = ["detect_column_types", "to_dataframe"] +__all__ = ["detect_column_types", "to_dataframe", "with_string_labels"] def to_dataframe(X, *, copy: bool = False) -> pd.DataFrame: @@ -41,6 +43,37 @@ def to_dataframe(X, *, copy: bool = False) -> pd.DataFrame: return X +def with_string_labels(X: pd.DataFrame) -> pd.DataFrame: + """Return ``X`` with every column label converted to its ``str`` form. + + scikit-learn's :class:`~sklearn.compose.ColumnTransformer` treats an integer + column selector as a *position*, so routing a column by an integer label (as + in ``pd.DataFrame(array)`` or ``read_csv(header=None)`` after reordering or + dropping a column) would select the wrong column. The Preprocessor therefore + fits and transforms its ColumnTransformer on string labels. ``X`` itself is + returned when every label is already a string; otherwise a shallow copy with + relabelled columns, so the caller's frame is never modified. + + Raises + ------ + PretabDataError + If two labels share a string form (e.g. ``1`` and ``"1"``), which would + make the columns indistinguishable. + """ + if all(isinstance(label, str) for label in X.columns): + return X + labels = [str(label) for label in X.columns] + collisions = sorted(label for label, count in Counter(labels).items() if count > 1) + if collisions: + raise PretabDataError( + f"Column labels must stay unique when converted to strings; {collisions} would collide.\n" + "Fix: rename the columns so their string forms are unique." + ) + relabelled = X.copy(deep=False) + relabelled.columns = pd.Index(labels) + return relabelled + + def detect_column_types(X, *, cat_cutoff, treat_all_integers_as_numerical, estimator_name="Preprocessor"): """Classify each column of ``X`` as numerical or categorical. diff --git a/pretab/preprocessor.py b/pretab/preprocessor.py index a8193b8..066012c 100644 --- a/pretab/preprocessor.py +++ b/pretab/preprocessor.py @@ -13,7 +13,7 @@ from .compose.config import PreprocessorConfig from .compose.factory import build_column_transformer -from .compose.feature_detection import detect_column_types, to_dataframe +from .compose.feature_detection import detect_column_types, to_dataframe, with_string_labels from .compose.inspection import ( build_feature_info, build_feature_lineage, @@ -305,7 +305,9 @@ class Preprocessor(TransformerMixin, BaseEstimator): ---------- column_transformer\_ : ColumnTransformer The internal scikit-learn column transformer that handles feature-wise preprocessing. - Set when the preprocessor is fitted. + Set when the preprocessor is fitted. It is fitted on the string form of every column + label (scikit-learn reads an integer column selector as a position), so a column + labelled ``1`` appears there as ``"1"``. n_features_in\_ : int Number of input features seen during ``fit``. total_output_dim\_ : int @@ -478,6 +480,13 @@ def fit(self, X, y=None, embeddings=None): ------- self : Preprocessor Fitted instance of the preprocessor. + + Raises + ------ + PretabDataError + If ``X`` has a duplicate column label, or two column labels share a string + form (e.g. ``1`` and ``"1"``): every column is routed by the string form of + its label, so such columns could not be told apart. """ if self.is_frozen(): @@ -580,7 +589,7 @@ def fit(self, X, y=None, embeddings=None): categorical_features, sparse_threshold=sparse_threshold, ) - self.column_transformer_.fit(X, y) + self.column_transformer_.fit(self._column_transformer_input(X), y) self.n_features_in_ = X.shape[1] self._enforce_output_budget(X.shape[0]) @@ -613,7 +622,10 @@ def transform(self, X, embeddings=None, return_array=None): Parameters ---------- X : pandas.DataFrame, numpy.ndarray, or dict - Input features to transform. + Input features to transform. DataFrame and dict columns are matched by + label, integer labels included. A NumPy array is matched by position + after a fit on a frame with non-string labels (such as + ``pd.DataFrame(array)``). embeddings : np.ndarray or list of np.ndarray, optional External embeddings to attach to dictionary output. Required when embeddings were supplied during ``fit`` and unsupported when the @@ -636,7 +648,10 @@ def transform(self, X, embeddings=None, return_array=None): check_is_fitted(self) + is_array = isinstance(X, np.ndarray) X = to_dataframe(X, copy=True) + if is_array: + X = self._label_array_by_position(X) if self.missing_policy == "error": self._reject_missing(X) @@ -647,7 +662,7 @@ def transform(self, X, embeddings=None, return_array=None): output_kind = container if container in ("pandas", "polars") else ("array" if resolved_return_array else "dict") validate_embedding_request(embeddings, expected=self.embeddings_, output_kind=output_kind) - transformed_X = self.column_transformer_.transform(X) + transformed_X = self.column_transformer_.transform(self._column_transformer_input(X)) if not sp.issparse(transformed_X): transformed_X = np.asarray(transformed_X) if self.dtype is not None: @@ -753,8 +768,11 @@ def get_feature_names_out(self, input_features=None): Parameters ---------- - input_features : array-like of str or None, default=None + input_features : array-like or None, default=None Input feature names. Passed through to the underlying column transformer. + Each name is compared by its string form, the form the column transformer is + fitted on, so the labels seen at fit are accepted as given (e.g. ``[0, 1]`` + after fitting on ``pd.DataFrame(array)``). Returns ------- @@ -763,6 +781,8 @@ def get_feature_names_out(self, input_features=None): """ check_is_fitted(self) + if input_features is not None: + input_features = [str(feature) for feature in input_features] raw_names = self.column_transformer_.get_feature_names_out(input_features) return np.array(clean_feature_names(self.column_transformer_, raw_names)) @@ -807,6 +827,7 @@ def output_dims_(self) -> dict: check_is_fitted(self) column_transformer = self.column_transformer_ output_indices = column_transformer.output_indices_ + input_labels = self._input_labels() dims: dict = {} for name, _transformer, columns in column_transformer.transformers_: span = output_indices[name] @@ -819,9 +840,44 @@ def output_dims_(self) -> dict: feature = full.split("__", 1)[-1] if "__" in full else full dims[feature] = dims.get(feature, 0) + 1 else: - dims[columns[0]] = width + dims[input_labels.get(columns[0], columns[0])] = width return dims + @staticmethod + def _column_transformer_input(X): + """Adapt a frame to what the internal ColumnTransformer expects. + + Column labels become strings, because scikit-learn reads an integer + column selector as a position (see + :func:`~pretab.compose.feature_detection.with_string_labels`). The + caller's frame is never modified. + """ + return with_string_labels(X) + + def _input_labels(self) -> dict: + """Map each ColumnTransformer column back to the label it has in the input. + + The ColumnTransformer is fitted on the string form of every label (see + :func:`~pretab.compose.feature_detection.with_string_labels`), so a + non-string label such as ``1`` appears there as ``"1"``. + """ + return {str(label): label for label in (*self.numerical_features_, *self.categorical_features_)} + + def _label_array_by_position(self, X): + """Give the columns of array input the labels seen at fit, by position. + + :func:`~pretab.compose.feature_detection.to_dataframe` names array columns + ``feature_0``, ``feature_1``, .... After a fit on a frame with a non-string + label, such as ``pd.DataFrame(array)``, an array carries no label to route + by, so its columns are matched by position, as scikit-learn does for a + frame without string feature names. Any other input is returned unchanged. + """ + fitted = [str(label) for label in getattr(self.column_transformer_, "feature_names_in_", [])] + input_labels = self._input_labels() + if len(fitted) != X.shape[1] or all(isinstance(input_labels.get(label), str) for label in fitted): + return X + return X.set_axis(fitted, axis=1) + def _output_itemsize(self) -> int: """Bytes per element of the dense transformed array. @@ -967,6 +1023,9 @@ def get_feature_info(self, verbose=True): embeddings=self.embeddings_, embedding_dimensions=self.embedding_dimensions_, ) + input_labels = self._input_labels() + numerical_feature_info = {input_labels.get(key, key): info for key, info in numerical_feature_info.items()} + categorical_feature_info = {input_labels.get(key, key): info for key, info in categorical_feature_info.items()} if verbose: configure_logging(1) diff --git a/tests/compose/test_feature_detection.py b/tests/compose/test_feature_detection.py index 670e8c4..ea10974 100644 --- a/tests/compose/test_feature_detection.py +++ b/tests/compose/test_feature_detection.py @@ -4,7 +4,7 @@ import pandas as pd import pytest -from pretab.compose.feature_detection import detect_column_types, to_dataframe +from pretab.compose.feature_detection import detect_column_types, to_dataframe, with_string_labels from pretab.exceptions import InvalidParamError, PretabDataError @@ -105,3 +105,12 @@ def test_invalid_cat_cutoff_type_raises(): df = pd.DataFrame({"x": [1, 2, 3]}) with pytest.raises(InvalidParamError): detect_column_types(df, cat_cutoff="bad", treat_all_integers_as_numerical=False) + + +def test_with_string_labels_relabels_without_touching_the_input(): + frame = pd.DataFrame({1: [1.0], "b": [2.0]}) + relabelled = with_string_labels(frame) + assert list(relabelled.columns) == ["1", "b"] + assert list(frame.columns) == [1, "b"] + string_frame = pd.DataFrame({"a": [1.0]}) + assert with_string_labels(string_frame) is string_frame diff --git a/tests/integration/test_column_labels.py b/tests/integration/test_column_labels.py new file mode 100644 index 0000000..9a139c5 --- /dev/null +++ b/tests/integration/test_column_labels.py @@ -0,0 +1,106 @@ +"""Column labels the ColumnTransformer cannot take verbatim. + +Regression guards for issue #55: integer labels were read as *positions* by +scikit-learn's ColumnTransformer, silently swapping columns. +""" + +from typing import cast + +import numpy as np +import pandas as pd +import pytest + +from pretab import Preprocessor +from pretab.exceptions import PretabDataError + + +@pytest.fixture +def int_labelled(): + # Labels are a permutation of the positions: label 1 sits at position 0. + return pd.DataFrame({1: [10.0, 20, 30, 40, 50, 60], 0: [1.0, 2, 3, 4, 5, 6]}) + + +def _passthrough(**kwargs): + return Preprocessor(numerical_method="none", scaling="none", output_structure="blocks", **kwargs) + + +def _blocks(out): + return cast("dict[str, np.ndarray]", out) + + +def test_integer_labels_route_each_column_to_its_own_block(int_labelled): + out = _blocks(_passthrough().fit_transform(int_labelled)) + np.testing.assert_array_equal(out["num_1"].ravel(), int_labelled[1].to_numpy()) + np.testing.assert_array_equal(out["num_0"].ravel(), int_labelled[0].to_numpy()) + # The ColumnTransformer works on a relabelled copy; the caller's frame keeps its labels. + assert list(int_labelled.columns) == [1, 0] + + +def test_feature_preprocessing_with_integer_keys_targets_the_labelled_column(int_labelled): + pre = Preprocessor(feature_preprocessing={1: "minmax", 0: "none"}, scaling="none", output_structure="blocks") + out = _blocks(pre.fit_transform(int_labelled)) + np.testing.assert_allclose(out["num_1"].ravel(), np.linspace(0.0, 1.0, 6)) + np.testing.assert_array_equal(out["num_0"].ravel(), int_labelled[0].to_numpy()) + + +def test_non_contiguous_integer_labels_fit_and_transform(): + frame = pd.DataFrame(np.random.default_rng(0).normal(size=(20, 3))).drop(columns=[0]) + pre = Preprocessor(numerical_method="minmax").fit(frame) + assert pre.get_feature_names_out().tolist() == ["num_1", "num_2"] + out = np.asarray(pre.transform(frame)) + assert out.shape == (20, 2) + np.testing.assert_allclose(out[:, 0], (frame[1] - frame[1].min()) / np.ptp(frame[1])) + + +def test_integer_labels_of_different_column_types_reach_their_own_pipeline(): + # Routed by position, the string column landed in the numerical pipeline and fit + # failed with "Cannot use median strategy with non-numeric data". + frame = pd.DataFrame({1: list("abcabc"), 0: [1.0, 2, 3, 4, 5, 6]}) + pre = Preprocessor(numerical_method="minmax", scaling="none", categorical_method="int").fit(frame) + assert pre.get_feature_names_out().tolist() == ["num_0", "cat_1"] + out = np.asarray(pre.transform(frame), dtype=float) + np.testing.assert_allclose(out[:, 0], np.linspace(0.0, 1.0, 6)) + np.testing.assert_array_equal(out[:, 1], [1.0, 2.0, 3.0, 1.0, 2.0, 3.0]) + default = Preprocessor().fit(frame, np.arange(6.0)) + assert (default.numerical_features_, default.categorical_features_) == ([0], [1]) + np.testing.assert_array_equal(np.asarray(default.transform(frame))[:, -1], [1.0, 2.0, 3.0, 1.0, 2.0, 3.0]) + + +def test_integer_labels_are_matched_by_label_at_transform(int_labelled): + pre = _passthrough(output_format="dense").fit(int_labelled) + reordered = int_labelled[[0, 1]] + np.testing.assert_array_equal( + _blocks(pre.transform(reordered))["num_1"], _blocks(pre.transform(int_labelled))["num_1"] + ) + + +def test_array_input_after_an_integer_labelled_fit_is_matched_by_position(int_labelled): + pre = _passthrough().fit(int_labelled) + out = _blocks(pre.transform(int_labelled.to_numpy())) + np.testing.assert_array_equal(out["num_1"].ravel(), int_labelled[1].to_numpy()) + np.testing.assert_array_equal(out["num_0"].ravel(), int_labelled[0].to_numpy()) + + +def test_integer_label_metadata_uses_the_input_labels(int_labelled): + pre = Preprocessor(numerical_method="minmax").fit(int_labelled) + assert pre.get_feature_names_out().tolist() == ["num_1", "num_0"] + assert pre.output_dims_ == {1: 1, 0: 1} + numerical, _, _ = pre.get_feature_info(verbose=False) + assert list(numerical) == [1, 0] + lineage = [(record.output_feature, record.source_features) for record in pre.get_feature_lineage()] + assert lineage == [("num_1", ("1",)), ("num_0", ("0",))] + + +def test_labels_colliding_as_strings_are_rejected(): + frame = pd.DataFrame({1: [1.0, 2.0, 3.0], "1": [4.0, 5.0, 6.0]}) + with pytest.raises(PretabDataError, match="unique when converted to strings"): + Preprocessor(numerical_method="minmax").fit(frame) + + +def test_get_feature_names_out_accepts_the_integer_labels_seen_at_fit(): + """The ColumnTransformer is fitted on string labels, so ``input_features`` given as + the original integer labels must still be accepted (it already was before).""" + X = pd.DataFrame(np.random.default_rng(0).normal(size=(40, 2))) + pre = Preprocessor(numerical_method="minmax").fit(X) + np.testing.assert_array_equal(pre.get_feature_names_out([0, 1]), pre.get_feature_names_out()) + From 69254e793442cf975867605f3a80e181f12eff74 Mon Sep 17 00:00:00 2001 From: ChrisW09 <50968720+ChrisW09@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:12:58 +0200 Subject: [PATCH 2/2] style: apply ruff formatting Remove a trailing blank line at the end of the column-label tests. --- tests/integration/test_column_labels.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_column_labels.py b/tests/integration/test_column_labels.py index 9a139c5..9a5de88 100644 --- a/tests/integration/test_column_labels.py +++ b/tests/integration/test_column_labels.py @@ -103,4 +103,3 @@ def test_get_feature_names_out_accepts_the_integer_labels_seen_at_fit(): X = pd.DataFrame(np.random.default_rng(0).normal(size=(40, 2))) pre = Preprocessor(numerical_method="minmax").fit(X) np.testing.assert_array_equal(pre.get_feature_names_out([0, 1]), pre.get_feature_names_out()) -