diff --git a/docs/core_concepts/configuration.md b/docs/core_concepts/configuration.md index c5fa21d..8fbb584 100644 --- a/docs/core_concepts/configuration.md +++ b/docs/core_concepts/configuration.md @@ -51,6 +51,19 @@ example `{"feature_0": "rbf"}`. Check `numerical_features_` / `categorical_featu overrides, rather than guessing the column order. ``` +At `transform`, array columns are matched by position, so the training and serving +containers may differ, as in scikit-learn. A `Preprocessor` fitted on a string-labelled +`DataFrame` accepts the same data as an array of the same width (for example `df.to_numpy()` +in NumPy-based serving or an explainer such as `shap.KernelExplainer`). It warns that `X` +does not have valid feature names and re-infers the numeric columns of an object array. +A fit on non-string column labels accepts arrays by position without that warning. +One fitted on an array accepts a `DataFrame` or dict of the same width the same way, +with a warning that `X` has feature names when its labels differ from the fitted synthetic +labels. After a `DataFrame` fit, subsequent frames are still matched by label, even when +the fitted labels are literally `feature_0`, `feature_1`, .... Input matched by position +with a different number of columns raises the usual +`X has 4 features, but Preprocessor is expecting 3 features as input.` error. + ## Per-feature overrides Columns rarely want identical treatment. The `feature_preprocessing` dict assigns a strategy diff --git a/pretab/preprocessor.py b/pretab/preprocessor.py index 497ba77..00c00b9 100644 --- a/pretab/preprocessor.py +++ b/pretab/preprocessor.py @@ -650,10 +650,16 @@ def transform(self, X, embeddings=None, return_array=None): Parameters ---------- X : pandas.DataFrame, numpy.ndarray, or dict - Input features to transform. DataFrame and dict columns are matched by - label, integer labels included. A NumPy array is matched by position - after a fit on a frame with non-string labels (such as - ``pd.DataFrame(array)``). + Input features to transform. A DataFrame or dict is matched to the fitted + columns by label, integer labels included. A NumPy array is matched by + position: after a fit on string-labelled columns it takes those names + (numeric columns of an object array are + re-inferred), with scikit-learn's usual warning that ``X`` does not have + valid feature names. Likewise, a DataFrame passed after a fit on an array + is matched by position, with a warning that ``X`` has feature names when + its labels differ from the fitted synthetic labels. After a fit on + non-string labels, an array is matched by position without a feature-name + warning. embeddings : np.ndarray or list of np.ndarray, optional External embeddings to attach to dictionary output. Required when embeddings were supplied during ``fit`` and unsupported when the @@ -673,14 +679,17 @@ def transform(self, X, embeddings=None, return_array=None): ``output_format`` resolves to ``"sparse"``; or a pandas / polars DataFrame when configured via :meth:`set_output` (a pandas DataFrame keeps the index of ``X``). + + Raises + ------ + PretabDataError + If input matched by position has a different number of columns than + ``n_features_in_``: it could never be routed correctly. """ check_is_fitted(self) - is_array = isinstance(X, np.ndarray) - X = to_dataframe(X, copy=True) - if is_array: - X = self._label_array_by_position(X) + X = self._align_input(X) if self.missing_policy == "error": self._reject_missing(X) @@ -893,6 +902,57 @@ def output_dims_(self) -> dict: dims[input_labels.get(columns[0], columns[0])] = width return dims + def _align_input(self, X): + """Return ``X`` as a DataFrame whose columns line up with the fitted columns. + + A NumPy array is matched by position. Once the preprocessor was fitted on + named columns, an array of the fitted width takes those column labels + (numeric columns of an object array are re-inferred), and a DataFrame + passed to a preprocessor fitted on an array is matched by position too -- + with scikit-learn's usual warning about missing or unexpected feature + names. A fit on non-string labels accepts arrays without a feature-name + warning. Any other DataFrame is matched by column label, even when its + fitted labels are literally ``feature_0``, ``feature_1``, .... + + Raises + ------ + PretabDataError + If array-like input matched by position has a different number of + columns than seen during ``fit``: it could never be routed correctly. + """ + # Synthetic labels mark an array fit only when no input feature names were recorded. + fitted_labels = getattr(self.column_transformer_, "feature_names_in_", None) + has_feature_names = hasattr(self, "feature_names_in_") + positional_labels = [f"feature_{i}" for i in range(self.n_features_in_)] + fitted_on_array = ( + not has_feature_names and fitted_labels is not None and list(fitted_labels) == positional_labels + ) + + is_array = isinstance(X, np.ndarray) + X = to_dataframe(X, copy=True) + if not (is_array or (fitted_on_array and list(X.columns) != positional_labels)): + return X + if X.shape[1] != self.n_features_in_: + raise PretabDataError( + f"X has {X.shape[1]} features, but {type(self).__name__} is expecting " + f"{self.n_features_in_} features as input." + ) + if is_array and has_feature_names: + warnings.warn( + f"X does not have valid feature names, but {type(self).__name__} was fitted with feature names", + UserWarning, + stacklevel=3, + ) + elif not is_array and fitted_on_array: + warnings.warn( + f"X has feature names, but {type(self).__name__} was fitted without feature names", + UserWarning, + stacklevel=3, + ) + if fitted_labels is not None and list(X.columns) != list(fitted_labels): + X = X.set_axis(list(fitted_labels), axis=1) + return X.infer_objects() if is_array else X + @staticmethod def _column_transformer_input(X): """Adapt a frame to what the internal ColumnTransformer expects. @@ -915,21 +975,6 @@ def _input_labels(self) -> dict: """ return {str(label): label for label in (*self.numerical_features_, *self.categorical_features_)} - def _label_array_by_position(self, X): - """Give the columns of array input the labels seen at fit, by position. - - :func:`~pretab.compose.feature_detection.to_dataframe` names array columns - ``feature_0``, ``feature_1``, .... After a fit on a frame with a non-string - label, such as ``pd.DataFrame(array)``, an array carries no label to route - by, so its columns are matched by position, as scikit-learn does for a - frame without string feature names. Any other input is returned unchanged. - """ - fitted = [str(label) for label in getattr(self.column_transformer_, "feature_names_in_", [])] - input_labels = self._input_labels() - if len(fitted) != X.shape[1] or all(isinstance(input_labels.get(label), str) for label in fitted): - return X - return X.set_axis(fitted, axis=1) - def _output_itemsize(self) -> int: """Bytes per element of the dense transformed array. diff --git a/tests/integration/test_column_labels.py b/tests/integration/test_column_labels.py index 9a7db59..6e0e88d 100644 --- a/tests/integration/test_column_labels.py +++ b/tests/integration/test_column_labels.py @@ -162,6 +162,7 @@ def test_integer_labels_are_matched_by_label_at_transform(int_labelled): ) +@pytest.mark.filterwarnings("error:.*feature names.*:UserWarning") def test_array_input_after_an_integer_labelled_fit_is_matched_by_position(int_labelled): pre = _passthrough().fit(int_labelled) out = _blocks(pre.transform(int_labelled.to_numpy())) diff --git a/tests/integration/test_preprocessor.py b/tests/integration/test_preprocessor.py index 2fdad6b..284678d 100644 --- a/tests/integration/test_preprocessor.py +++ b/tests/integration/test_preprocessor.py @@ -379,6 +379,88 @@ def test_unknown_feature_preprocessing_key_raises(sample_data): Preprocessor(feature_preprocessing={"nnum1": "minmax"}).fit(X, y) +# --- positional input after a named fit (and vice versa) (issue #78) --------------- + + +@pytest.fixture +def mixed_named_frame(): + rng = np.random.default_rng(0) + frame = pd.DataFrame( + { + "age": rng.normal(40, 10, 100), + "income": rng.normal(50, 10, 100), + "city": rng.choice(["a", "b", "c"], 100), + } + ) + return frame, rng.normal(size=100) + + +def test_frame_fit_accepts_an_array_of_the_fitted_width(mixed_named_frame): + """Regression guard for issue #78: a DataFrame-fitted preprocessor rejected the same + data as an array ('columns are missing'), breaking NumPy-based serving and explainers.""" + from sklearn.linear_model import Ridge + from sklearn.pipeline import make_pipeline + + X, y = mixed_named_frame + pipe = make_pipeline(Preprocessor(random_state=0), Ridge()).fit(X, y) + with pytest.warns(UserWarning, match="does not have valid feature names"): + from_array = pipe.predict(X.to_numpy()) + np.testing.assert_allclose(from_array, pipe.predict(X)) + + +@pytest.mark.parametrize("as_dict", [False, True]) +def test_array_fit_accepts_a_frame_of_the_fitted_width(mixed_named_frame, as_dict): + X, y = mixed_named_frame + numeric = X[["age", "income"]] + pre = Preprocessor(numerical_method="minmax").fit(numeric.to_numpy(), y) + incoming = numeric.rename(columns={"age": "u", "income": "v"}) + if as_dict: + incoming = incoming.to_dict(orient="list") + with pytest.warns(UserWarning, match="fitted without feature names"): + from_frame = pre.transform(incoming) + np.testing.assert_array_equal(from_frame, pre.transform(numeric.to_numpy())) + + +@pytest.mark.parametrize("n_features", [1, 4]) +def test_positional_input_of_the_wrong_width_is_rejected(mixed_named_frame, n_features): + X, y = mixed_named_frame + pre = Preprocessor(random_state=0).fit(X, y) + incoming = pd.DataFrame(np.zeros((len(X), n_features))) + with pytest.raises(PretabDataError, match=f"X has {n_features} features, but Preprocessor is expecting 3"): + pre.transform(incoming.to_numpy()) + numeric = X[["age", "income"]] + array_fit = Preprocessor(numerical_method="minmax").fit(numeric.to_numpy(), y) + with pytest.raises(PretabDataError, match=f"X has {n_features} features, but Preprocessor is expecting 2"): + array_fit.transform(incoming) + + +@pytest.mark.parametrize("as_dict", [False, True]) +def test_synthetic_named_frame_is_not_treated_as_an_array_fit(as_dict): + frame = pd.DataFrame({"feature_0": [1.0, 2, 3, 4], "feature_1": [10.0, 20, 30, 40]}) + pre = Preprocessor(numerical_method="minmax").fit(frame.to_dict(orient="list") if as_dict else frame) + assert pre.feature_names_in_.tolist() == ["feature_0", "feature_1"] + expected = pre.transform(frame) + np.testing.assert_array_equal(pre.transform(frame[["feature_1", "feature_0"]]), expected) + with pytest.raises(ValueError, match="columns are missing"): + pre.transform(frame.rename(columns={"feature_0": "age", "feature_1": "income"})) + with pytest.warns(UserWarning, match="does not have valid feature names"): + from_array = pre.transform(frame.to_numpy(dtype=object)) + np.testing.assert_array_equal(from_array, expected) + + +@pytest.mark.parametrize("dtype", ["bool", "boolean"]) +def test_boolean_frame_fit_accepts_an_array_without_mutating_input(bool_frame, dtype): + frame = bool_frame.astype({"flag": dtype}) + if dtype == "boolean": + frame.loc[2, "flag"] = pd.NA + original = frame.copy(deep=True) + pre = Preprocessor(numerical_method="minmax", categorical_method="one-hot").fit(frame) + with pytest.warns(UserWarning, match="does not have valid feature names"): + from_array = pre.transform(frame.to_numpy(na_value=np.nan)) + np.testing.assert_array_equal(from_array, pre.transform(frame)) + pd.testing.assert_frame_equal(frame, original) + + # --- get_feature_names_out(input_features) (issue #65) ----------------------------