Repository: dirty-cat/dirty_cat
Pull request #238 (2022-02-14 11:38:52)

Change auto_cast in SuperVectorizer
Uses pandas [`TextParser`](https://github.com/pandas-dev/pandas/blob/06d230151e6f18fdb8139d09abf539867a8cd481/pandas/io/parsers/readers.py#L1289) (used in `read_csv`, as per @GaelVaroquaux suggestion in #234) instead of `convert_dtypes` in the SuperVectorizer `_auto_cast` method. This has several advantages:

- Type detection works on string columns, e.g `["3", "2", "1"]` would get detected as integer. Solves #234 
-  Missing values encoded as strings get treated as missing values, and don't prevent type detection. For instance,` [1, 2, "", 4]` is detected as integers. (solves  #231). Which strings get detected as Nans is easy to change (see below).
- Datetime columns are converted to datetimes, and the format is inferred by Pandas. Solve the first part of #233.

Some comments:
- TODO: I'd like to enable the user to specify a specific date format if he wants to.
- Strings detected as NaNs are the default [`STR_NA_VALUES`](https://github.com/pandas-dev/pandas/blob/main/pandas/_libs/parsers.pyx#L1350), and I've added `[None, " ", "?", "..."]`. I think we could find a more principled way to choose which strings to include.
- I haven't properly measured the speed of the conversion.
- To use TextParser, I need to convert the input data into a list of list. I don't think this is a problem, but tell me if you think otherwise.
- All variables which are not numerics or datetimes are converted to object (and thus handled as categorical in the SuperVectorizer).
- Right now, if a datetime column contains mixed timezones, it is encoded as an object.


Patch:
diff --git a/CHANGES.rst b/CHANGES.rst
index 63dcc0e..378e453 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,17 @@
+Release 0.2.1
+=============
+
+Major changes
+-------------
+
+Notes
+-----
+
+* Improvements to the :class:`SuperVectorizer`
+
+    - Type detection works better: handles dates, numerics columns encoded as strings, or numeric columns containing strings for missing values
+
+
 Release 0.2.0
 =============
 
diff --git a/dirty_cat/super_vectorizer.py b/dirty_cat/super_vectorizer.py
index e2b2dd6..81610d6 100644
--- a/dirty_cat/super_vectorizer.py
+++ b/dirty_cat/super_vectorizer.py
@@ -23,7 +23,6 @@ from sklearn.preprocessing import OneHotEncoder
 
 from dirty_cat import GapEncoder
 
-
 _sklearn_loose_version = LooseVersion(sklearn.__version__)
 
 
@@ -171,34 +170,25 @@ class SuperVectorizer(ColumnTransformer):
     @staticmethod
     def _auto_cast(X: pd.DataFrame) -> pd.DataFrame:
         """
-        Takes a pandas DataFrame and tries to convert its columns to the best
+        Takes a dataframe and tries to convert its columns to the best
         possible data type.
 
         Parameters
         ----------
-        X: pd.DataFrame
-            Input data as a pandas DataFrame.
+        X : {dataframe} of shape (n_samples, n_features)
+            The data to be transformed.
 
         Returns
         -------
-        array
-            The same array, with its columns casted to the best possible
+        pd.DataFrame
+            The same pandas DataFrame, with its columns casted to the best possible
             data type.
-            Columns with missing values won't be modified.
         """
         from pandas.core.dtypes.base import ExtensionDtype
+
+        # Handle missing values
         for col in X.columns:
-            dtype = X[col].dtype
             contains_missing: bool = _has_missing_values(X[col])
-            if not contains_missing:
-                X[col] = X[col].convert_dtypes()
-            # Cast pandas dtypes to numpy dtypes
-            # for earlier versions of sklearn
-            if issubclass(dtype.__class__, ExtensionDtype):
-                try:
-                    X[col] = X[col].astype(dtype.type, errors='ignore')
-                except (TypeError, ValueError):
-                    pass
             # Convert pandas' NaN value (pd.NA) to numpy NaN value (np.nan)
             # because the former tends to raise all kind of issues when dealing
             # with scikit-learn (as of version 0.24).
@@ -208,7 +198,30 @@ class SuperVectorizer(ColumnTransformer):
                 if pd.api.types.is_numeric_dtype(X[col]):
                     X[col] = X[col].astype(np.float64)
                 X[col].fillna(value=np.nan, inplace=True)
+        STR_NA_VALUES = ['null', '', '1.#QNAN', '#NA', 'nan', '#N/A N/A', '-1.#QNAN', '<NA>', '-1.#IND', '-nan', 'n/a',
+                         '-NaN', '1.#IND', 'NULL', 'NA', 'N/A', '#N/A', 'NaN']  # taken from pandas.io.parsers (version 1.1.4)
+        X = X.replace(STR_NA_VALUES + [None, "?", "..."],
+                      np.nan)
+        X = X.replace(r'^\s+$', np.nan, regex=True) # replace whitespace only
 
+        # Convert to best possible data type
+        for col in X.columns:
+            if not pd.api.types.is_datetime64_any_dtype(X[col]): # we don't want to cast datetime64
+                try:
+                    X[col] = pd.to_numeric(X[col], errors='raise')
+                except:
+                    # Only try to convert to datetime if the variable isn't numeric.
+                    try:
+                        X[col] = pd.to_datetime(X[col], errors='raise')
+                    except:
+                        pass
+            # Cast pandas dtypes to numpy dtypes
+            # for earlier versions of sklearn
+            if issubclass(X[col].dtype.__class__, ExtensionDtype):
+                try:
+                    X[col] = X[col].astype(X[col].dtype.type, errors='ignore')
+                except (TypeError, ValueError):
+                    pass
         return X
 
     def transform(self, X) -> np.ndarray:
@@ -229,30 +242,26 @@ class SuperVectorizer(ColumnTransformer):
             sparse matrices.
 
         """
-        # Convert to pandas DataFrame if not already.
         if not isinstance(X, pd.DataFrame):
             X = pd.DataFrame(X)
-            # Auto cast the imported DataFrame to avoid having issues with
-            # `object` dtype when we will cast columns to the fitted types.
-            X = self._auto_cast(X)
-            # Check the number of columns matches the fitted array's.
-            if X.shape[1] != len(self.columns_):
-                raise ValueError("Passed array does not match column count of "
-                                 f"array seen at fit time. Got {X.shape[0]} "
-                                 f"columns, expected {len(self.columns_)}")
         else:
             # Create a copy to avoid altering the original data.
             X = X.copy()
+        # Auto cast the imported data to avoid having issues with
+        # `object` dtype when we will cast columns to the fitted types.
+        if self.auto_cast:
+            X = self._auto_cast(X)
+            self.types_ = {c: t for c, t in zip(X.columns, X.dtypes)}
+        if X.shape[1] != len(self.columns_):
+            raise ValueError("Passed array does not match column count of "
+                             f"array seen at fit time. Got {X.shape[1]} "
+                             f"columns, expected {len(self.columns_)}")
+
         # If the DataFrame does not have named columns already,
         # apply the learnt columns
-        if isinstance(X.columns, pd.RangeIndex):
+        if pd.api.types.is_numeric_dtype(X.columns):
             X.columns = self.columns_
 
-        if self.auto_cast:
-            # Enforce types of the fitted input array.
-            for col, to_type in self.types_.items():
-                X[col] = X[col].astype(to_type)
-
         for col in self.imputed_columns_:
             X[col] = _replace_missing_in_col(X[col])
 
@@ -432,7 +441,7 @@ class SuperVectorizer(ColumnTransformer):
             return ct_feature_names
 
         return all_trans_feature_names
-    
+
     def get_feature_names_out(self, input_features=None) -> List[str]:
         """
         Ensures compatibility with sklearn >= 1.0, and returns the output of


Test patch:
diff --git a/dirty_cat/test/test_super_vectorizer.py b/dirty_cat/test/test_super_vectorizer.py
index 13e0db9..066c23d 100644
--- a/dirty_cat/test/test_super_vectorizer.py
+++ b/dirty_cat/test/test_super_vectorizer.py
@@ -18,6 +18,17 @@ def check_same_transformers(expected_transformers: dict, actual_transformers: li
     actual_transformers_dict = dict([(name, cols) for name, trans, cols in actual_transformers])
     assert actual_transformers_dict == expected_transformers
 
+def type_equality(expected_type, actual_type):
+    """
+    Checks that the expected type is equal to the actual type, assuming object and str types
+    are equivalent (considered as categorical by the supervectorizer).
+    """
+    if (isinstance(expected_type , object) or  isinstance(expected_type, str))\
+            and (isinstance(actual_type, object) or isinstance(actual_type, str)):
+        return True
+    else:
+        return expected_type == actual_type
+
 
 def _get_clean_dataframe():
     """
@@ -49,6 +60,55 @@ def _get_dirty_dataframe():
         'cat2': pd.Series(['20K+', '40K+', '60K+', '30K+', np.nan], dtype='object'),
     })
 
+def _get_numpy_array():
+    return np.array([["15", "56", pd.NA, "12", ""],
+            ["?", "2.4", "6.2", "10.45", np.nan],
+            ['public', np.nan, 'private', 'private', pd.NA],
+            ['officer', 'manager', None, 'chef', 'teacher'],
+            [np.nan, 'yes', 'no', 'yes', 'no'],
+            ['20K+', '40K+', '60K+', '30K+', np.nan]]).T
+
+def _get_list_of_lists():
+    return _get_numpy_array().tolist()
+
+def _get_datetimes_dataframe():
+    """
+    Creates a DataFrame with various date format, already converted or to be converted.
+    """
+    return pd.DataFrame({
+        "pd_datetime": [
+            pd.Timestamp("2019-01-01"),
+            pd.Timestamp("2019-01-02"),
+            pd.Timestamp("2019-01-03"),
+            pd.Timestamp("2019-01-04"),
+            pd.Timestamp("2019-01-05")],
+        "np_datetime": [np.datetime64('2018-01-01'),
+                        np.datetime64('2018-01-02'),
+                        np.datetime64('2018-01-03'),
+                        np.datetime64('2018-01-04'),
+                        np.datetime64('2018-01-05')],
+        "dmy-": ['11-12-2029',
+                 '02-12-2012',
+                 '11-09-2012',
+                 '13-02-2000',
+                 '10-11-2001'],
+        # "mdy-": ['11-13-2013',
+        #          '02-12-2012',
+        #          '11-31-2012',
+        #          '05-02-2000',
+        #          '10-11-2001'],
+        "ymd/": ['2014/12/31',
+                 '2001/11/23',
+                 '2005/02/12',
+                 '1997/11/01',
+                 '2011/05/05'],
+        "ymd/_hms:": ["2014/12/31 00:31:01",
+                      "2014/12/30 00:31:12",
+                      "2014/12/31 23:31:23",
+                      "2015/12/31 01:31:34",
+                      "2014/01/31 00:32:45"],
+    })
+
 
 def _test_possibilities(
         X,
@@ -183,6 +243,7 @@ def test_with_dirty_data():
         'low_card_cat': [2, 4],
         'high_card_cat': [3, 5],
     }
+
     _test_possibilities(
         X,
         expected_transformers_df,
@@ -194,6 +255,81 @@ def test_with_dirty_data():
     )
 
 
+def test_auto_cast():
+    """
+    Tests that the SuperVectorizer automatic type detection works as expected.
+    """
+    vectorizer = SuperVectorizer()
+
+    # Test datetime detection
+    X = _get_datetimes_dataframe()
+
+    expected_types_datetimes = {
+        "pd_datetime": "datetime64[ns]",
+        "np_datetime": "datetime64[ns]",
+        "dmy-": "datetime64[ns]",
+        "ymd/": "datetime64[ns]",
+        "ymd/_hms:": "datetime64[ns]",
+    }
+    X_trans = vectorizer._auto_cast(X)
+    for col in X_trans.columns:
+        assert expected_types_datetimes[col] == X_trans[col].dtype
+
+    # Test other types detection
+
+    expected_types_clean_dataframe = {
+        "int": "int64",
+        "float": "float64",
+        "str1": "object",
+        "str2": "object",
+        "cat1": "object",
+        "cat2": "object"
+    }
+
+    X = _get_clean_dataframe()
+    X_trans = vectorizer._auto_cast(X)
+    for col in X_trans.columns:
+        assert type_equality(expected_types_clean_dataframe[col], X_trans[col].dtype)
+
+    # Test that missing values don't prevent type detection
+    expected_types_dirty_dataframe = {
+        "int": "float64",  # int type doesn't support nans
+        "float": "float64",
+        "str1": "object",
+        "str2": "object",
+        "cat1": "object",
+        "cat2": "object"
+    }
+
+    X = _get_dirty_dataframe()
+    X_trans = vectorizer._auto_cast(X)
+    for col in X_trans.columns:
+        assert type_equality(expected_types_dirty_dataframe[col], X_trans[col].dtype)
+
+def test_with_arrays():
+    """
+    Check that the SuperVectorizer works if we input a list of lists or a numpy array.
+    """
+    expected_transformers = {
+        'numeric': [0, 1],
+        'low_card_cat': [2, 4],
+        'high_card_cat': [3, 5],
+    }
+    vectorizer =  SuperVectorizer(
+        cardinality_threshold=4,
+        # we must have n_samples = 5 >= n_components
+        high_card_cat_transformer=GapEncoder(n_components=2),
+        numerical_transformer=StandardScaler(),
+    )
+
+    X = _get_numpy_array()
+    vectorizer.fit_transform(X)
+    check_same_transformers(expected_transformers, vectorizer.transformers)
+
+    X = _get_list_of_lists()
+    vectorizer.fit_transform(X)
+    check_same_transformers(expected_transformers, vectorizer.transformers)
+
 def test_get_feature_names():
     X = _get_clean_dataframe()
 
@@ -250,7 +386,7 @@ def test_transform():
     assert (x_trans == [[1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 34, 5.5]]).all()
 
 
-def fit_transform_equiv():
+def test_fit_transform_equiv():
     """
     We will test the equivalence between using `.fit_transform(X)`
     and `.fit(X).transform(X).`
@@ -269,11 +405,9 @@ def fit_transform_equiv():
     enc1_x2 = sup_vec3.fit_transform(X2)
     enc2_x2 = sup_vec4.fit(X2).transform(X2)
 
-    assert enc1_x1 == enc2_x1
-    assert sup_vec1 == sup_vec2
+    assert np.allclose(enc1_x1, enc2_x1, rtol=0, atol=0, equal_nan=True)
 
-    assert enc1_x2 == enc2_x2
-    assert sup_vec3 == sup_vec4
+    assert np.allclose(enc1_x2, enc2_x2, rtol=0, atol=0, equal_nan=True)
 
 
 if __name__ == '__main__':
@@ -283,6 +417,12 @@ if __name__ == '__main__':
     print('start test_super_vectorizer with dirty df')
     test_with_dirty_data()
     print('test_super_vectorizer with dirty df passed')
+    print('start test_auto_cast')
+    test_auto_cast()
+    print('test_auto_cast passed')
+    print("start test_with_arrays")
+    test_with_arrays()
+    print("test_with_arrays passed")
     print('start test_get_feature_names')
     test_get_feature_names()
     print('test_get_feature_names passed')
@@ -290,7 +430,7 @@ if __name__ == '__main__':
     test_fit()
     print('test_fit passed')
     print('start fit_transform_equiv')
-    fit_transform_equiv()
+    test_fit_transform_equiv()
     print('fit_transform_equiv passed')
 
     print('Done')