Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/core_concepts/resolution_and_placement.md
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,11 @@ placement subsystem so no transformer re-implements it, and it is driven by two
| `True` | `"cart"` | Split points from a per-feature decision tree fit against `y`. |
| `True` | `"lightgbm"` | Split points aggregated from gradient-boosted trees (needs the `lightgbm` extra). |

On tied data, such as a zero-inflated, top-coded or low-cardinality feature, several quantiles
coincide or fall on the range boundary. The spline families keep their interior knots unique
and strictly inside the observed range, topping a short set up with evenly spaced interior
points, so no basis function collapses to a point and leaves a dead column.

```{warning}
The unsupervised and target-aware rows are mutually exclusive. Combining them, for example
`target_aware=True` with `placement_strategy="quantile"`, raises an error.
Expand Down
41 changes: 41 additions & 0 deletions pretab/core/knots.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
"quantile_knots",
"select_knots",
"spanning_knots",
"supplement_interior_knots",
"uniform_knots",
]

Expand Down Expand Up @@ -142,3 +143,43 @@ def select_knots(knots: np.ndarray, count: int) -> np.ndarray:
return knots
idx = np.linspace(0, len(knots) - 1, count).round().astype(int)
return knots[idx]


def supplement_interior_knots(x: np.ndarray, knots: np.ndarray, count: int) -> np.ndarray:
"""Return unique knots strictly inside the range of ``x``, topped up to ``count``.

Knots equal to (or outside) ``min(x)`` / ``max(x)`` and repeated knots give a
spline basis function with zero-width support -- a dead column -- so they are
dropped first. Every remaining knot is kept; when fewer than ``count`` remain,
the shortfall is filled with evenly spread quantile and uniform candidates
that are themselves strictly interior and not already present. On heavily tied
data the quantile candidates coincide with the tied values (often the range
boundary), so the uniform candidates guarantee the top-up for any feature with
a positive range. Never down-samples: callers trim an overfull set themselves.

Parameters
----------
x : ndarray
Values of a single feature (finite).
knots : ndarray
Candidate interior knots, e.g. quantile knots or selector locations.
count : int
Minimum number of interior knots to return.

Returns
-------
ndarray
Sorted, unique knots strictly inside ``(min(x), max(x))``.
"""
x = np.asarray(x, dtype=float)
x_min, x_max = x.min(), x.max()
knots = np.asarray(knots, dtype=float)
knots = np.unique(knots[(knots > x_min) & (knots < x_max)])
missing = count - len(knots)
if missing <= 0:
return knots

candidates = np.unique(np.concatenate([quantile_knots(x, count), uniform_knots(x, count)]))
candidates = candidates[(candidates > x_min) & (candidates < x_max)]
candidates = np.setdiff1d(candidates, knots)
return np.unique(np.concatenate([knots, select_knots(candidates, missing)]))
46 changes: 21 additions & 25 deletions pretab/expansion/spline/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,8 @@
from ...core.knots import (
basis_to_knots,
generate_internal_knots,
quantile_knots,
select_knots,
uniform_knots,
supplement_interior_knots,
)
from ...core.parameters import UNSET, validate_placement, validate_task
from ...core.policy import RepresentationPolicy, resolve_out_of_range
Expand Down Expand Up @@ -65,7 +64,10 @@ class BaseSplineTransformer(BasePreTabTransformer):
knot_locations : ndarray or None, default=None
Explicit internal knot locations applied to every feature. Takes priority
over ``target_aware`` placement and the automatic strategy, and overrides
``output_dim``.
``output_dim``. Like every interior knot, they must be unique and lie
strictly inside a feature's range: repeated knots and knots on or outside
the range are dropped and replaced by strictly interior quantile / uniform
knots.

target_aware : bool, default=False
If True, knots are placed by a target-aware selector built from
Expand All @@ -76,6 +78,11 @@ class BaseSplineTransformer(BasePreTabTransformer):
When ``target_aware=True``, the selector: ``"cart"`` or ``"lightgbm"``.
When ``target_aware=False``, the spacing rule: ``"uniform"`` spaces knots
evenly across the range, ``"quantile"`` places them at data quantiles.
Interior knots are always unique and strictly inside the feature's range.
On tied data (top-coded, zero-inflated or low-cardinality features)
quantiles repeat or fall on the boundary; such knots are replaced by
strictly interior quantile / uniform knots, so no basis function
collapses to a point.

task : {"regression", "classification"} or None, default=None
Task passed to the target-aware selector when ``target_aware=True``. ``None`` is
Expand Down Expand Up @@ -197,30 +204,22 @@ def _generate_knots(self, x: np.ndarray, n_knots: int, strategy: str) -> np.ndar
def _adjust_internal_knots(
self, x: np.ndarray, internal_knots: np.ndarray, min_knots: int, max_knots: int
) -> np.ndarray:
"""Clip, deduplicate and rebalance internal knots to the allowed count."""
internal_knots = np.clip(internal_knots, x.min(), x.max())
internal_knots = np.unique(np.sort(internal_knots))
if len(internal_knots) < min_knots:
internal_knots = self._supplement_knots(x, internal_knots, min_knots)
"""Restrict, deduplicate and rebalance internal knots to the allowed count.

Interior knots must lie strictly inside the data range and be unique: a
knot on the boundary or a repeated knot collapses a basis function's
support to a point, giving a dead column (and, at the boundary, all-zero
B/M-spline rows at ``x_max``). Such knots are dropped and the set is
topped up to ``min_knots`` before an overfull set is trimmed.
"""
internal_knots = self._supplement_knots(x, internal_knots, min_knots)
if len(internal_knots) > max_knots:
internal_knots = select_knots(internal_knots, max_knots)
return internal_knots

def _supplement_knots(self, x: np.ndarray, internal_knots: np.ndarray, target_count: int) -> np.ndarray:
"""Add quantile and uniform candidates until ``target_count`` knots exist."""
if target_count <= len(internal_knots):
return internal_knots

candidates = [internal_knots]
if target_count > 0:
candidates.append(quantile_knots(x, target_count))
candidates.append(uniform_knots(x, target_count))

combined = np.unique(np.concatenate(candidates))
combined = np.sort(combined)
if len(combined) < target_count:
combined = uniform_knots(x, target_count)
return select_knots(np.asarray(combined), target_count)
"""Keep the strictly interior, unique knots and top them up to ``target_count``."""
return supplement_interior_knots(x, internal_knots, target_count)

def _column_knots(
self,
Expand Down Expand Up @@ -257,9 +256,6 @@ def _column_knots(
internal_knots = self._generate_knots(x_valid, n_internal, strategy)
internal_knots = self._adjust_internal_knots(x_valid, internal_knots, min_knots, max_knots)

internal_knots = np.clip(internal_knots, x_min, x_max)
internal_knots = np.unique(np.sort(internal_knots))

boundary_left = np.repeat(x_min, self.degree + 1)
boundary_right = np.repeat(x_max, self.degree + 1)
return np.concatenate([boundary_left, internal_knots, boundary_right])
Expand Down
2 changes: 2 additions & 0 deletions pretab/expansion/spline/cubic_regression.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ class CubicRegressionSplineTransformer(SplineBasisMixin, TransformerMixin, BaseE
When ``target_aware=True``, the selector: ``"cart"`` or ``"lightgbm"``.
When ``target_aware=False``, the spacing: ``"uniform"`` spaces knots evenly
across the range, ``"quantile"`` places them at evenly spaced data quantiles.
On tied data, repeated quantiles and quantiles on the range boundary are
replaced by unique, strictly interior quantile / uniform knots.

task : {"regression", "classification"} or None, default=None
Task forwarded to the target-aware selector when ``target_aware=True``. ``None`` is
Expand Down
17 changes: 12 additions & 5 deletions pretab/expansion/spline/mixins.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
import numpy as np

from ...core.base import BasePreTabTransformer
from ...core.knots import generate_internal_knots, select_knots, spanning_knots
from ...core.knots import generate_internal_knots, select_knots, spanning_knots, supplement_interior_knots
from ...exceptions import IncompatibleParamsError, PretabDataError


Expand Down Expand Up @@ -84,13 +84,19 @@ def _place_spanning_knots(self, x, y, n_basis, strategy, selector, task, min_int
window before bracketing.
"""
x, y = self._finite_column(x, y)
x_min, x_max = x.min(), x.max()
if selector is not None:
interior = self._place_interior_knots(
x, y, n_basis - 2, strategy, selector, task, min_interior, max_interior
)
x_min, x_max = x.min(), x.max()
return np.concatenate([[x_min], interior, [x_max]])
return spanning_knots(x, n_basis, strategy)
knots = spanning_knots(x, n_basis, strategy)
if len(knots) <= 2:
return knots
# On tied data quantile knots repeat or land on the endpoints; keep the
# endpoints once and make the interior unique and strictly inside.
interior = supplement_interior_knots(x, knots[1:-1], n_basis - 2)
return np.concatenate([[x_min], interior, [x_max]])

def _place_interior_knots(self, x, y, n_interior, strategy, selector, task, min_interior=None, max_interior=None):
"""Return the interior knots (endpoints excluded) for one feature.
Expand All @@ -106,7 +112,8 @@ def _place_interior_knots(self, x, y, n_interior, strategy, selector, task, min_
On the adaptive selector path ``min_interior`` / ``max_interior`` clamp the
data-driven count into that window instead. Without a ``selector``,
``n_interior`` knots are placed with
:func:`pretab.core.knots.generate_internal_knots`.
:func:`pretab.core.knots.generate_internal_knots` and made unique and
strictly interior (see :func:`pretab.core.knots.supplement_interior_knots`).
"""
x, y = self._finite_column(x, y)
if selector is not None:
Expand All @@ -121,7 +128,7 @@ def _place_interior_knots(self, x, y, n_interior, strategy, selector, task, min_
min_interior = max_interior = n_interior
selected = self._clamp_interior_knots(x, selected, min_interior, max_interior, strategy)
return selected
return generate_internal_knots(x, n_interior, strategy)
return supplement_interior_knots(x, generate_internal_knots(x, n_interior, strategy), n_interior)

def _clamp_interior_knots(self, x, knots, min_count, max_count, strategy):
"""Clamp a data-driven set of interior knots into ``[min_count, max_count]``.
Expand Down
3 changes: 2 additions & 1 deletion pretab/expansion/spline/multivariate/tensor_product.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ class TensorProductSplineTransformer(SplineBasisMixin, TransformerMixin, BaseEst

placement_strategy : {"uniform", "quantile"}, default="uniform"
Interior-knot placement rule for each marginal dimension. ``"uniform"``
spaces knots evenly; ``"quantile"`` uses data quantiles.
spaces knots evenly; ``"quantile"`` uses data quantiles, made unique and
strictly interior on tied data.

.. note::
The tensor-product spline is a penalized (difference-penalty) spline
Expand Down
4 changes: 3 additions & 1 deletion pretab/expansion/spline/natural_cubic.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,9 @@ class NaturalCubicSplineTransformer(SplineBasisMixin, TransformerMixin, BaseEsti
When ``target_aware=True``, the selector: ``"cart"`` or ``"lightgbm"``.
When ``target_aware=False``, the spacing: ``"uniform"`` reproduces the
historical evenly spaced knots, ``"quantile"`` places them at evenly spaced
data quantiles.
data quantiles. The range endpoints are kept once; on tied data, repeated
interior quantiles and quantiles on the boundary are replaced by unique,
strictly interior quantile / uniform knots.

task : {"regression", "classification"} or None, default=None
Task forwarded to the target-aware selector when ``target_aware=True``. ``None`` is
Expand Down
157 changes: 157 additions & 0 deletions tests/expansion/spline/test_tied_knot_placement.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
"""Knot placement on tied data (top-coded, zero-inflated, low-cardinality features).

Regression guards for issue #56: quantile knots on tied data coincide with each
other and with the range boundary, which collapsed basis functions to a point
(dead columns) and, for B/M-splines, left every row at ``x_max`` all zero.
"""

import numpy as np
import pandas as pd
import pytest

from pretab import Preprocessor
from pretab.core.knots import supplement_interior_knots
from pretab.transformers import (
BSplineTransformer,
CubicRegressionSplineTransformer,
ISplineTransformer,
MSplineTransformer,
NaturalCubicSplineTransformer,
TensorProductSplineTransformer,
)


@pytest.fixture
def top_coded():
# ~40% of the rows sit exactly on the cap of 100.
return np.minimum(np.random.default_rng(0).uniform(0, 160, (1000, 1)), 100.0)


@pytest.fixture
def zero_inflated():
rng = np.random.default_rng(1)
return np.where(rng.random((1000, 1)) < 0.6, 0.0, rng.exponential(5.0, (1000, 1)))


def _interior(knots, degree):
return knots[degree + 1 : len(knots) - degree - 1]


def _dense_grid(X, n=2001):
return np.linspace(X.min(), X.max(), n).reshape(-1, 1)


@pytest.mark.parametrize("cls", [BSplineTransformer, MSplineTransformer, ISplineTransformer])
@pytest.mark.parametrize("data", ["top_coded", "zero_inflated"])
def test_bmi_interior_knots_are_unique_and_strictly_inside(cls, data, request):
X = request.getfixturevalue(data)
transformer = cls(output_dim=6).fit(X)
knots = transformer.knots_[0]
interior = _interior(knots, transformer.degree)

assert len(interior) == 6 - transformer.degree - 1
assert len(np.unique(interior)) == len(interior)
assert (interior > X.min()).all() and (interior < X.max()).all()
# Boundary multiplicity stays exactly degree + 1.
assert (knots == X.min()).sum() == transformer.degree + 1
assert (knots == X.max()).sum() == transformer.degree + 1
assert (np.abs(transformer.transform(_dense_grid(X))).max(axis=0) > 0).all()


def test_bspline_is_a_partition_of_unity_at_the_cap(top_coded):
transformer = BSplineTransformer(output_dim=6).fit(top_coded)
basis = transformer.transform(top_coded)
np.testing.assert_allclose(basis.sum(axis=1), 1.0, atol=1e-9)
# Values on and beyond the cap (clipped onto x_max) keep a full basis row.
np.testing.assert_allclose(transformer.transform(np.array([[100.0], [150.0]])).sum(axis=1), 1.0, atol=1e-9)


def test_mspline_keeps_every_row_at_the_cap(top_coded):
transformer = MSplineTransformer(output_dim=6).fit(top_coded)
basis = transformer.transform(top_coded)
assert (basis.sum(axis=1) > 0).all()
assert (basis.max(axis=0) > 0).all()
assert (transformer.transform(np.array([[100.0], [150.0]])).sum(axis=1) > 0).all()


def test_preprocessor_bspline_has_no_dead_column_on_a_top_coded_feature(top_coded):
df = pd.DataFrame({"x": top_coded[:, 0]})
pre = Preprocessor(numerical_method="bspline", target_aware=False, placement_strategy="quantile").fit(df)
out = np.asarray(pre.transform(df), dtype=float)
assert (np.abs(out).max(axis=0) > 0).all()
assert (np.abs(out).sum(axis=1) > 0).all()


def test_ispline_equals_one_at_the_cap(top_coded):
transformer = ISplineTransformer(output_dim=6).fit(top_coded)
np.testing.assert_allclose(transformer.transform(np.array([[100.0]])), 1.0, atol=1e-12)


@pytest.mark.parametrize("cls", [NaturalCubicSplineTransformer, CubicRegressionSplineTransformer])
def test_cubic_families_place_unique_knots_on_zero_inflated_data(cls, zero_inflated):
transformer = cls(output_dim=6, placement_strategy="quantile").fit(zero_inflated)
knots = transformer.knots_[0]
assert len(np.unique(knots)) == len(knots)
basis = transformer.transform(_dense_grid(zero_inflated))
assert np.linalg.matrix_rank(basis) == basis.shape[1]
assert (np.abs(basis).max(axis=0) > 0).all()


def test_natural_spline_keeps_both_endpoints_once(zero_inflated):
knots = NaturalCubicSplineTransformer(output_dim=6, placement_strategy="quantile").fit(zero_inflated).knots_[0]
assert len(knots) == 7
assert knots[0] == zero_inflated.min() and knots[-1] == zero_inflated.max()
assert (knots[1:-1] > knots[0]).all() and (knots[1:-1] < knots[-1]).all()


@pytest.mark.parametrize("output_dim", [6, 8])
def test_tensor_product_has_no_degenerate_columns_on_zero_inflated_marginal(zero_inflated, output_dim):
X = np.hstack([zero_inflated, np.random.default_rng(2).normal(size=(len(zero_inflated), 1))])
transformer = TensorProductSplineTransformer(output_dim=output_dim, placement_strategy="quantile").fit(X)
g0 = np.linspace(X[:, 0].min(), X[:, 0].max(), 200)
g1 = np.linspace(X[:, 1].min(), X[:, 1].max(), 200)
grid = np.array(np.meshgrid(g0, g1)).reshape(2, -1).T
assert (np.abs(transformer.transform(grid)).max(axis=0) > 0).all()


@pytest.mark.parametrize(
("levels", "p"),
[
([0.0, 1.0], None),
([0.0, 1.0, 2.0], None),
# Most of the mass on the two end levels: both quantile knots hit the boundary.
([1.0, 2.0, 3.0, 4.0], [0.4, 0.1, 0.1, 0.4]),
],
)
def test_low_cardinality_feature_keeps_a_full_rank_bspline_basis(levels, p):
X = np.random.default_rng(3).choice(levels, size=(300, 1), p=p)
transformer = BSplineTransformer(output_dim=6).fit(X)
basis = transformer.transform(_dense_grid(X))
assert np.linalg.matrix_rank(basis) == 6


def test_four_level_feature_keeps_every_row_at_a_wide_bspline_basis():
# Integers 0-3 at output_dim=8: the quantile knots were 0, 1, 2, 3, so the
# first and last basis functions collapsed and every row at x = 3 was zero.
X = np.random.default_rng(0).integers(0, 4, (1000, 1)).astype(float)
transformer = BSplineTransformer(output_dim=8).fit(X)
basis = transformer.transform(X)
np.testing.assert_allclose(basis.sum(axis=1), 1.0, atol=1e-9)
# Four distinct values support at most rank 4 on the training rows.
assert np.linalg.matrix_rank(basis) == 4
assert np.linalg.matrix_rank(transformer.transform(_dense_grid(X))) == 8


def test_supplement_interior_knots_drops_boundary_and_duplicate_knots():
x = np.array([0.0, 0.0, 0.0, 1.0, 2.0, 10.0, 10.0])
knots = supplement_interior_knots(x, np.array([0.0, 2.0, 2.0, 10.0, 12.0]), 3)
assert len(knots) == 3
assert 2.0 in knots # an existing interior knot is always kept
assert (knots > 0.0).all() and (knots < 10.0).all()
assert len(np.unique(knots)) == 3


def test_supplement_interior_knots_never_down_samples():
x = np.linspace(0, 1, 50)
knots = np.array([0.1, 0.2, 0.3, 0.4])
np.testing.assert_array_equal(supplement_interior_knots(x, knots, 2), knots)
Loading