Repository: dirty-cat/dirty_cat

Issue:
One hot encoding of Supervectorizer for columns with 2 values
Just something I observed using SuperVectorizer, on a dataset with columns having mostly 2 categorical values. As documented, it runs one hot encoding on them, and this blows up the feature space. This is probably an edge case, and the example I have here is maybe an extreme one. For just two values it seemed excessive to do one-hot encoding.

```
from dirty_cat import SuperVectorizer
from sklearn.datasets import fetch_openml
X, y = fetch_openml("kr-vs-kp", return_X_y=True)
X.shape
(3196, 36)
X.describe()
       bkblk bknwy bkon8 bkona bkspr bkxbq bkxcr bkxwp blxwp bxqsq cntxt dsopp dwipd  ... simpl skach skewr skrxp spcop stlmt thrsk wkcti wkna8 wknck wkovl wkpos wtoeg
count   3196  3196  3196  3196  3196  3196  3196  3196  3196  3196  3196  3196  3196  ...  3196  3196  3196  3196  3196  3196  3196  3196  3196  3196  3196  3196  3196
unique     2     2     2     2     2     2     2     2     2     2     2     2     2  ...     2     2     2     2     2     2     2     2     2     2     2     2     2
top        f     f     f     f     f     f     f     f     f     f     f     f     l  ...     f     f     t     f     f     f     f     f     f     f     t     t     n
freq    2839  2971  3076  2874  2129  1722  2026  2500  1980  2225  1817  2860  2205  ...  1975  3185  2216  3021  3195  3149  3060  2631  3021  1984  2007  2345  2407

[4 rows x 36 columns]

sv = SuperVectorizer()
X_tx = sv.fit_transform(X)
X_tx.shape
(3196, 73)

```

Discussion:

Patch:
diff --git a/CHANGES.rst b/CHANGES.rst
index a1f7651..da2fc09 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -7,14 +7,21 @@ Major changes
 * New encoder: :class:`DatetimeEncoder` can transform a datetime column into several numerical
     columns (year, month, day, hour, minute, second, ...). It is now the default transformer used
     in the :class:`SuperVectorizer` for datetime columns.
-* The :class:`SuperVectorizer` has seen some major improvements and bug fixes.
+* The :class:`SuperVectorizer` has seen some major improvements and bug fixes
+  - Fixes the automatic casting logic in ``transform``.
+  - **Behavior change** To avoid dimensionality explosion when a feature has two unique values,
+    the default encoder (:class:`OneHotEncoder`) now drops one of the two
+    vectors (see parameter `drop="if_binary"`).
+  - ``fit_transform`` and ``transform`` can now return unencoded features,
+    like the :class:`ColumnTransformer`'s behavior.
+    Previously, a ``RuntimeError`` was raised.
 * **Backward-incompatible change in the SuperVectorizer**: to apply ``remainder``
   to features (with the ``*_transformer`` parameters), the value ``'remainder'``
   must be passed, instead of ``None`` in previous versions.
   ``None`` now indicates that we want to use the default transformer.
 * Support for Python 3.6 and 3.7 has been dropped. Python >= 3.8 is now required.
 * Bumped minimum dependencies:
-  - sklearn>=0.22
+  - sklearn>=0.23
   - scipy>=1.4.0
   - numpy>=1.17.3
   - pandas>=1.2.0
diff --git a/README.rst b/README.rst
index e0e232c..26a7a14 100644
--- a/README.rst
+++ b/README.rst
@@ -38,7 +38,7 @@ dirty_cat requires:
 - Python (>= 3.8)
 - NumPy (>= 1.17.3)
 - SciPy (>= 1.4.0)
-- scikit-learn (>= 0.22.0)
+- scikit-learn (>= 0.23.0)
 - pandas (>= 1.2.0)
 
 User installation
diff --git a/dirty_cat/gap_encoder.py b/dirty_cat/gap_encoder.py
index a0471af..b06d622 100644
--- a/dirty_cat/gap_encoder.py
+++ b/dirty_cat/gap_encoder.py
@@ -35,21 +35,12 @@ from dirty_cat.utils import Version
 
 from .utils import check_input
 
-if Version(sklearn_version) == Version("0.22"):
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore")
-        from sklearn.cluster.k_means_ import _k_init
-elif Version(sklearn_version) < Version("0.24"):
+if Version(sklearn_version) < Version("0.24"):
     from sklearn.cluster._kmeans import _k_init
 else:
     from sklearn.cluster import kmeans_plusplus
 
-if Version(sklearn_version) == Version("0.22"):
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore")
-        from sklearn.decomposition.nmf import _beta_divergence
-else:
-    from sklearn.decomposition._nmf import _beta_divergence
+from sklearn.decomposition._nmf import _beta_divergence
 
 
 class GapEncoderColumn(BaseEstimator, TransformerMixin):
diff --git a/dirty_cat/super_vectorizer.py b/dirty_cat/super_vectorizer.py
index 6643dc7..6c4e8b4 100644
--- a/dirty_cat/super_vectorizer.py
+++ b/dirty_cat/super_vectorizer.py
@@ -96,16 +96,18 @@ class SuperVectorizer(ColumnTransformer):
 
     cardinality_threshold : int, default=40
         Two lists of features will be created depending on this value: strictly
-        under this value, the low cardinality categorical values, and above or
-        equal, the high cardinality categorical values.
+        under this value, the low cardinality categorical features, and above or
+        equal, the high cardinality categorical features.
         Different transformers will be applied to these two groups,
         defined by the parameters `low_card_cat_transformer` and
         `high_card_cat_transformer` respectively.
+        Note: currently, missing values are counted as a single unique value
+        (so they count in the cardinality).
 
     low_card_cat_transformer : typing.Optional[typing.Union[sklearn.base.TransformerMixin, typing.Literal["drop", "remainder", "passthrough"]]], default=None  # noqa
         Transformer used on categorical/string features with low cardinality
         (threshold is defined by `cardinality_threshold`).
-        Can either be a transformer object instance (e.g. `OneHotEncoder()`),
+        Can either be a transformer object instance (e.g. `OneHotEncoder(drop="if_binary")`),
         a `Pipeline` containing the preprocessing steps,
         'drop' for dropping the columns,
         'remainder' for applying `remainder`,
@@ -226,6 +228,16 @@ class SuperVectorizer(ColumnTransformer):
     imputed_columns_: typing.List[str]
         The list of columns in which we imputed the missing values.
 
+    Notes
+    -----
+    The column order of the input data is not guaranteed to be the same
+    as the output data (returned by `transform`).
+    This is a due to the way the ColumnTransformer works.
+    However, the output column order will always be the same for different
+    calls to `transform` on a same fitted SuperVectorizer instance.
+    For example, if input data has columns ['name', 'job', 'year], then output
+    columns might be shuffled, e.g., ['job', 'year', 'name'], but every call
+    to `transform` will return this order.
     """
 
     transformers_: List[Tuple[str, Union[str, TransformerMixin], List[str]]]
@@ -290,7 +302,7 @@ class SuperVectorizer(ColumnTransformer):
         if isinstance(self.low_card_cat_transformer, sklearn.base.TransformerMixin):
             self.low_card_cat_transformer_ = clone(self.low_card_cat_transformer)
         elif self.low_card_cat_transformer is None:
-            self.low_card_cat_transformer_ = OneHotEncoder()
+            self.low_card_cat_transformer_ = OneHotEncoder(drop="if_binary")
         elif self.low_card_cat_transformer == "remainder":
             self.low_card_cat_transformer_ = self.remainder
         else:
@@ -406,6 +418,11 @@ class SuperVectorizer(ColumnTransformer):
     def fit_transform(self, X, y=None):
         """
         Fit all transformers, transform the data, and concatenate the results.
+        In practice, it (1) converts features to their best possible types
+        if `auto_cast=True`, (2) classify columns based on their data type,
+        (3) replaces "false missing" (see function `_replace_false_missing`),
+        and imputes categorical columns depending on `impute_missing`, and
+        finally, transforms X.
 
         Parameters
         ----------
@@ -470,16 +487,21 @@ class SuperVectorizer(ColumnTransformer):
         ).columns.to_list()
 
         # Classify categorical columns by cardinality
+        _nunique_values = {  # Cache results
+            col: X[col].nunique() for col in categorical_columns
+        }
         low_card_cat_columns = [
             col
             for col in categorical_columns
-            if X[col].nunique() < self.cardinality_threshold
+            if _nunique_values[col] < self.cardinality_threshold
         ]
         high_card_cat_columns = [
             col
             for col in categorical_columns
-            if X[col].nunique() >= self.cardinality_threshold
+            if _nunique_values[col] >= self.cardinality_threshold
         ]
+        # Clear cache
+        del _nunique_values
 
         # Next part: construct the transformers
         # Create the list of all the transformers.
@@ -603,24 +625,10 @@ class SuperVectorizer(ColumnTransformer):
         typing.List[str]
             Feature names.
         """
-        if Version(sklearn_version) < Version("0.23"):
-            try:
-                if Version(sklearn_version) < Version("1.0"):
-                    ct_feature_names = super().get_feature_names()
-                else:
-                    ct_feature_names = super().get_feature_names_out()
-            except NotImplementedError:
-                raise NotImplementedError(
-                    "Prior to sklearn 0.23, get_feature_names with "
-                    '"passthrough" is unsupported. To use the method, '
-                    'either make sure there is no "passthrough" in the '
-                    "transformers, or update your copy of scikit-learn. "
-                )
+        if Version(sklearn_version) < Version("1.0"):
+            ct_feature_names = super().get_feature_names()
         else:
-            if Version(sklearn_version) < Version("1.0"):
-                ct_feature_names = super().get_feature_names()
-            else:
-                ct_feature_names = super().get_feature_names_out()
+            ct_feature_names = super().get_feature_names_out()
         all_trans_feature_names = []
 
         for name, trans, cols, _ in self._iter(fitted=True):
diff --git a/dirty_cat/utils.py b/dirty_cat/utils.py
index c5b8442..d7f780f 100644
--- a/dirty_cat/utils.py
+++ b/dirty_cat/utils.py
@@ -77,8 +77,8 @@ class Version:
 
     Examples:
     >>> # Standard usage
-    >>> Version(sklearn.__version__) > Version('0.22')
-    >>> Version(sklearn.__version__) > '0.22'
+    >>> Version(sklearn.__version__) > Version('0.23')
+    >>> Version(sklearn.__version__) > '0.23'
     >>> # In general, pass the version as numbers separated by dots.
     >>> Version('1.5') <= Version('1.6.5')
     >>> Version('1.5') <= '1.6.5'
diff --git a/requirements-min.txt b/requirements-min.txt
index eff1e8c..75702d6 100644
--- a/requirements-min.txt
+++ b/requirements-min.txt
@@ -1,6 +1,6 @@
 numpy==1.17.3
 scipy==1.4.0
-scikit-learn==0.22.0
+scikit-learn==0.23.0
 pytest
 pytest-cov
 coverage
diff --git a/setup.py b/setup.py
index 73bf659..a015caa 100755
--- a/setup.py
+++ b/setup.py
@@ -39,7 +39,7 @@ if __name__ == "__main__":
         packages=find_packages(),
         package_data={"dirty_cat": ["VERSION.txt"]},
         install_requires=[
-            "scikit-learn>=0.21",
+            "scikit-learn>=0.23",
             "numpy>=1.16",
             "scipy>=1.2",
             "pandas>=1.2.0",


Test patch:
diff --git a/dirty_cat/test/test_super_vectorizer.py b/dirty_cat/test/test_super_vectorizer.py
index 4b04c77..c97e826 100644
--- a/dirty_cat/test/test_super_vectorizer.py
+++ b/dirty_cat/test/test_super_vectorizer.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Tuple
+from typing import Any, Tuple
 
 import numpy as np
 import pandas as pd
@@ -145,15 +145,7 @@ def _get_datetimes_dataframe() -> pd.DataFrame:
     )
 
 
-def _test_possibilities(
-    X,
-    expected_transformers_df: Dict[str, List[str]],
-    expected_transformers_2: Dict[str, List[str]],
-    expected_transformers_np_no_cast: Dict[str, List[int]],
-    expected_transformers_series: Dict[str, List[str]],
-    expected_transformers_plain: Dict[str, List[str]],
-    expected_transformers_np_cast: Dict[str, List[int]],
-):
+def _test_possibilities(X):
     """
     Do a bunch of tests with the SuperVectorizer.
     We take some expected transformers results as argument. They're usually
@@ -167,10 +159,18 @@ def _test_possibilities(
         numerical_transformer=StandardScaler(),
     )
     # Warning: order-dependant
+    expected_transformers_df = {
+        "numeric": ["int", "float"],
+        "low_card_cat": ["str1", "cat1"],
+        "high_card_cat": ["str2", "cat2"],
+    }
     vectorizer_base.fit_transform(X)
     check_same_transformers(expected_transformers_df, vectorizer_base.transformers)
 
     # Test with higher cardinality threshold and no numeric transformer
+    expected_transformers_2 = {
+        "low_card_cat": ["str1", "str2", "cat1", "cat2"],
+    }
     vectorizer_default = SuperVectorizer()  # Using default values
     vectorizer_default.fit_transform(X)
     check_same_transformers(expected_transformers_2, vectorizer_default.transformers)
@@ -178,12 +178,20 @@ def _test_possibilities(
     # Test with a numpy array
     arr = X.to_numpy()
     # Instead of the columns names, we'll have the column indices.
+    expected_transformers_np_no_cast = {
+        "low_card_cat": [2, 4],
+        "high_card_cat": [3, 5],
+        "numeric": [0, 1],
+    }
     vectorizer_base.fit_transform(arr)
     check_same_transformers(
         expected_transformers_np_no_cast, vectorizer_base.transformers
     )
 
     # Test with pandas series
+    expected_transformers_series = {
+        "low_card_cat": ["cat1"],
+    }
     vectorizer_base.fit_transform(X["cat1"])
     check_same_transformers(expected_transformers_series, vectorizer_base.transformers)
 
@@ -196,9 +204,19 @@ def _test_possibilities(
     )
     X_str = X.astype("object")
     # With pandas
+    expected_transformers_plain = {
+        "high_card_cat": ["str2", "cat2"],
+        "low_card_cat": ["str1", "cat1"],
+        "numeric": ["int", "float"],
+    }
     vectorizer_cast.fit_transform(X_str)
     check_same_transformers(expected_transformers_plain, vectorizer_cast.transformers)
     # With numpy
+    expected_transformers_np_cast = {
+        "numeric": [0, 1],
+        "low_card_cat": [2, 4],
+        "high_card_cat": [3, 5],
+    }
     vectorizer_cast.fit_transform(X_str.to_numpy())
     check_same_transformers(expected_transformers_np_cast, vectorizer_cast.transformers)
 
@@ -208,43 +226,7 @@ def test_with_clean_data():
     Defines the expected returns of the vectorizer in different settings,
     and runs the tests with a clean dataset.
     """
-    X = _get_clean_dataframe()
-    # Define the transformers we'll use throughout the test.
-    expected_transformers_df = {
-        "numeric": ["int", "float"],
-        "low_card_cat": ["str1", "cat1"],
-        "high_card_cat": ["str2", "cat2"],
-    }
-    expected_transformers_2 = {
-        "low_card_cat": ["str1", "str2", "cat1", "cat2"],
-    }
-    expected_transformers_np_no_cast = {
-        "low_card_cat": [2, 4],
-        "high_card_cat": [3, 5],
-        "numeric": [0, 1],
-    }
-    expected_transformers_series = {
-        "low_card_cat": ["cat1"],
-    }
-    expected_transformers_plain = {
-        "high_card_cat": ["str2", "cat2"],
-        "low_card_cat": ["str1", "cat1"],
-        "numeric": ["int", "float"],
-    }
-    expected_transformers_np_cast = {
-        "numeric": [0, 1],
-        "low_card_cat": [2, 4],
-        "high_card_cat": [3, 5],
-    }
-    _test_possibilities(
-        X,
-        expected_transformers_df,
-        expected_transformers_2,
-        expected_transformers_np_no_cast,
-        expected_transformers_series,
-        expected_transformers_plain,
-        expected_transformers_np_cast,
-    )
+    _test_possibilities(_get_clean_dataframe())
 
 
 def test_with_dirty_data() -> None:
@@ -252,44 +234,7 @@ def test_with_dirty_data() -> None:
     Defines the expected returns of the vectorizer in different settings,
     and runs the tests with a dataset containing missing values.
     """
-    X = _get_dirty_dataframe()
-    # Define the transformers we'll use throughout the test.
-    expected_transformers_df = {
-        "numeric": ["int", "float"],
-        "low_card_cat": ["str1", "cat1"],
-        "high_card_cat": ["str2", "cat2"],
-    }
-    expected_transformers_2 = {
-        "low_card_cat": ["str1", "str2", "cat1", "cat2"],
-    }
-    expected_transformers_np_no_cast = {
-        "low_card_cat": [2, 4],
-        "high_card_cat": [3, 5],
-        "numeric": [0, 1],
-    }
-    expected_transformers_series = {
-        "low_card_cat": ["cat1"],
-    }
-    expected_transformers_plain = {
-        "high_card_cat": ["str2", "cat2"],
-        "low_card_cat": ["str1", "cat1"],
-        "numeric": ["int", "float"],
-    }
-    expected_transformers_np_cast = {
-        "numeric": [0, 1],
-        "low_card_cat": [2, 4],
-        "high_card_cat": [3, 5],
-    }
-
-    _test_possibilities(
-        X,
-        expected_transformers_df,
-        expected_transformers_2,
-        expected_transformers_np_no_cast,
-        expected_transformers_series,
-        expected_transformers_plain,
-        expected_transformers_np_cast,
-    )
+    _test_possibilities(_get_dirty_dataframe())
 
 
 def test_auto_cast() -> None:
@@ -376,49 +321,39 @@ def test_get_feature_names_out() -> None:
     vec_w_pass = SuperVectorizer(remainder="passthrough")
     vec_w_pass.fit(X)
 
-    if Version(sklearn.__version__) < Version("0.23"):
-        with pytest.raises(NotImplementedError):
-            # Prior to sklearn 0.23, ColumnTransformer.get_feature_names
-            # with "passthrough" transformer(s) raises a NotImplementedError
-            assert vec_w_pass.get_feature_names_out()
+    # In this test, order matters. If it doesn't, convert to set.
+    expected_feature_names_pass = [
+        "str1_public",
+        "str2_chef",
+        "str2_lawyer",
+        "str2_manager",
+        "str2_officer",
+        "str2_teacher",
+        "cat1_yes",
+        "cat2_20K+",
+        "cat2_30K+",
+        "cat2_40K+",
+        "cat2_50K+",
+        "cat2_60K+",
+        "int",
+        "float",
+    ]
+    if Version(sklearn.__version__) < Version("1.0"):
+        assert vec_w_pass.get_feature_names() == expected_feature_names_pass
     else:
-        # In this test, order matters. If it doesn't, convert to set.
-        expected_feature_names_pass = [
-            "str1_private",
-            "str1_public",
-            "str2_chef",
-            "str2_lawyer",
-            "str2_manager",
-            "str2_officer",
-            "str2_teacher",
-            "cat1_no",
-            "cat1_yes",
-            "cat2_20K+",
-            "cat2_30K+",
-            "cat2_40K+",
-            "cat2_50K+",
-            "cat2_60K+",
-            "int",
-            "float",
-        ]
-        if Version(sklearn.__version__) < Version("1.0"):
-            assert vec_w_pass.get_feature_names() == expected_feature_names_pass
-        else:
-            assert vec_w_pass.get_feature_names_out() == expected_feature_names_pass
+        assert vec_w_pass.get_feature_names_out() == expected_feature_names_pass
 
     vec_w_drop = SuperVectorizer(remainder="drop")
     vec_w_drop.fit(X)
 
     # In this test, order matters. If it doesn't, convert to set.
     expected_feature_names_drop = [
-        "str1_private",
         "str1_public",
         "str2_chef",
         "str2_lawyer",
         "str2_manager",
         "str2_officer",
         "str2_teacher",
-        "cat1_no",
         "cat1_yes",
         "cat2_20K+",
         "cat2_30K+",
@@ -448,7 +383,11 @@ def test_transform() -> None:
     s = [34, 5.5, "private", "manager", "yes", "60K+"]
     x = np.array(s).reshape(1, -1)
     x_trans = sup_vec.transform(x)
-    assert (x_trans == [[1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 34, 5.5]]).all()
+    assert x_trans.tolist() == [
+        [0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 34.0, 5.5]
+    ]
+    # To understand the list above:
+    # print(dict(zip(sup_vec.get_feature_names_out(), x_trans.tolist()[0])))
 
 
 def test_fit_transform_equiv() -> None:
@@ -492,13 +431,20 @@ def test_passthrough():
         auto_cast=False,
     )
 
-    X_enc_dirty = sv.fit_transform(X_dirty)
-    X_enc_clean = sv.fit_transform(X_clean)
+    X_enc_dirty = pd.DataFrame(
+        sv.fit_transform(X_dirty), columns=sv.get_feature_names_out()
+    )
+    X_enc_clean = pd.DataFrame(
+        sv.fit_transform(X_clean), columns=sv.get_feature_names_out()
+    )
+    # Reorder encoded arrays' columns (see SV's doc "Notes" section as to why)
+    X_enc_dirty = X_enc_dirty[X_dirty.columns]
+    X_enc_clean = X_enc_clean[X_clean.columns]
 
     dirty_flat_df = X_dirty.to_numpy().ravel().tolist()
-    dirty_flat_trans_df = X_enc_dirty.ravel().tolist()
+    dirty_flat_trans_df = X_enc_dirty.to_numpy().ravel().tolist()
     assert all(map(_is_equal, zip(dirty_flat_df, dirty_flat_trans_df)))
-    assert (X_clean.to_numpy() == X_enc_clean).all()
+    assert (X_clean.to_numpy() == X_enc_clean.to_numpy()).all()
 
 
 if __name__ == "__main__":