Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 16 additions & 2 deletions pretab/compose/inspection.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,16 @@ def _probe_input(step, value):
return pd.DataFrame(np.full((1, len(names)), value), columns=names)


def _block_width(column_transformer, step_name):
"""Return the fitted output width of a ColumnTransformer step, or None when unknown.

Every per-column block holds one feature, so its width in ``output_indices_``
is that feature's output dimension.
"""
span = column_transformer.output_indices_.get(step_name)
return None if span is None else span.stop - span.start


def representation_leaf(transformer):
"""Return the last step of a fitted per-column block's representation.

Expand Down Expand Up @@ -274,7 +284,9 @@ def build_feature_info(column_transformer, *, embeddings, embedding_dimensions):
feature_name,
exc,
)
dimension = None
# The probe value is outside the step's domain (e.g. Box-Cox
# needs strictly positive input): use the fitted block width.
dimension = _block_width(column_transformer, name)
if separate_state_dimension is not None:
dimension = separate_state_dimension
numerical_feature_info[feature_name] = {
Expand Down Expand Up @@ -322,7 +334,9 @@ def build_feature_info(column_transformer, *, embeddings, embedding_dimensions):
feature_name,
exc,
)
dimension = None
# The probe value is outside the step's domain (e.g. Box-Cox
# needs strictly positive input): use the fitted block width.
dimension = _block_width(column_transformer, name)
if separate_state_dimension is not None:
dimension = separate_state_dimension
if name.startswith("cat_"):
Expand Down
12 changes: 12 additions & 0 deletions tests/compose/test_inspection.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,18 @@ def test_build_feature_info_probes_a_step_fitted_on_the_frame_without_warning(
assert numerical["age"]["dimension"] == dimension


@pytest.mark.parametrize("scaling", [None, "standardization", "minmax"])
def test_build_feature_info_reports_the_width_of_a_step_the_probe_cannot_transform(make_config, sample_frame, scaling):
"""Box-Cox rejects the non-positive probe value used without min-max scaling, so
its dimension was reported as None; the fitted block width is used instead."""
config = make_config(numerical_method="box-cox", scaling=scaling)
ct = build_column_transformer(config, ["age"], ["city"]).fit(sample_frame)

numerical, _, _ = build_feature_info(ct, embeddings=False, embedding_dimensions={})

assert numerical["age"]["dimension"] == 1


def test_build_feature_info_reports_a_zero_width_block():
# scikit-learn's imputer default drops a column without observed values, so
# the encoder after it receives no column at all.
Expand Down
19 changes: 19 additions & 0 deletions tests/integration/test_preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -768,3 +768,22 @@ def test_boxcox_survives_cross_validation():
X, y = data[0], data[1]
scores = cross_val_score(make_pipeline(Preprocessor(numerical_method="box-cox"), Ridge()), X, y, cv=5)
assert np.isfinite(scores).all()


@pytest.mark.parametrize("scaling", [None, "standardization", "minmax"])
@pytest.mark.parametrize(
"numerical_method",
["box-cox", "yeo-johnson", "polynomial", "robust", "custombin", "fourier", "bspline", "rbf", "none"],
)
def test_feature_info_dimension_matches_output_dims(numerical_method, scaling):
"""get_feature_info reported dimension=None for Box-Cox unless scaling was "minmax"."""
rng = np.random.default_rng(0)
df = pd.DataFrame({"a": rng.normal(size=60), "b": rng.exponential(size=60), "c": rng.choice(["x", "y"], size=60)})
pre = Preprocessor(
numerical_method=numerical_method, scaling=scaling, target_aware=False, placement_strategy="uniform"
).fit(df)

numerical_info, categorical_info, _ = pre.get_feature_info(verbose=False)

reported = {feature: info["dimension"] for feature, info in {**numerical_info, **categorical_info}.items()}
assert reported == pre.output_dims_