Repository: dirty-cat/dirty_cat
Pull request #132 (2020-07-31 10:41:54)

[MRG] minhash handle missing values
References Issue #124 Add support for missing values in the encoders

Changes: Added missing values support for minhash encoder


Patch:
diff --git a/dirty_cat/minhash_encoder.py b/dirty_cat/minhash_encoder.py
index 2fbee5c..ca597ae 100644
--- a/dirty_cat/minhash_encoder.py
+++ b/dirty_cat/minhash_encoder.py
@@ -24,11 +24,12 @@ from sklearn.cluster import KMeans
 from sklearn.feature_extraction.text import CountVectorizer, HashingVectorizer
 from sklearn.neighbors import NearestNeighbors
 from sklearn.base import BaseEstimator, TransformerMixin
-from sklearn.utils import check_random_state, murmurhash3_32 
+from sklearn.utils import check_random_state, murmurhash3_32
 
 from .fast_hash import ngram_min_hash
 from .utils import LRUDict
 
+
 class MinHashEncoder(BaseEstimator, TransformerMixin):
     """
     Encode string categorical features as a numeric array, minhash method
@@ -49,21 +50,29 @@ class MinHashEncoder(BaseEstimator, TransformerMixin):
         might have some concern with its entropy.
     minmax_hash : bool, default=False
         if True, return min hash and max hash concatenated.
+    handle_missing : 'error' or '' (default)
+        Whether to raise an error or impute with blank string '' if missing
+        values (NaN) are present during fit (default is to impute).
+        When this parameter is set to '', and a missing value is encountered
+        during fit_transform, the resulting encoded columns for this feature
+        will be all zeros.
     References
     ----------
     For a detailed description of the method, see
     `Encoding high-cardinality string categorical variables
     <https://hal.inria.fr/hal-02171256v4>`_ by Cerda, Varoquaux (2019).
-    
+
     """
 
     def __init__(self, n_components=30, ngram_range=(2, 4),
-                 hashing='fast', minmax_hash=False):
+                 hashing='fast', minmax_hash=False,
+                 handle_missing=''):
         self.ngram_range = ngram_range
         self.n_components = n_components
         self.hashing = hashing
         self.minmax_hash = minmax_hash
         self.count = 0
+        self.handle_missing = handle_missing
 
     def get_unique_ngrams(self, string, ngram_range):
         """ Return the set of unique n-grams of a string.
@@ -114,11 +123,10 @@ class MinHashEncoder(BaseEstimator, TransformerMixin):
             min_hashes = np.minimum(min_hashes, hash_array)
         return min_hashes/(2**32-1)
 
-    def get_hash(self, string):
+    def get_fast_hash(self, string):
         """
-        Encode a string with murmur or fast hashing function.
-        fast hashing supports both min_hash and minmax_hash encoding,
-        whereas murmur only supports min_hash encoding.
+        Encode a string with fast hashing function.
+        fast hashing supports both min_hash and minmax_hash encoding.
         Parameters
         ----------
         string : str
@@ -128,27 +136,13 @@ class MinHashEncoder(BaseEstimator, TransformerMixin):
         array, shape (n_components, )
             The encoded string, using specified encoding scheme.
         """
-        if self.hashing == 'fast':
-            if self.minmax_hash:
-                assert self.n_components % 2 == 0,\
-                       "n_components should be even when minmax_hash=True"
-                return np.concatenate([ngram_min_hash(string, self.ngram_range,
-                                                      seed, return_minmax=True)
-                                      for seed in range(self.n_components//2)])
-            else:
-                return np.array([ngram_min_hash(string, self.ngram_range, seed)
-                                for seed in range(self.n_components)])
-
-        elif self.hashing == 'murmur':
-            assert not(self.minmax_hash),\
-                   "minmax_hash not implemented with murmur"
-            return self.minhash(
-                    string, n_components=self.n_components,
-                    ngram_range=self.ngram_range)
+        if self.minmax_hash:
+            return np.concatenate([ngram_min_hash(string, self.ngram_range,
+                                   seed, return_minmax=True)
+                                   for seed in range(self.n_components//2)])
         else:
-            raise ValueError("hashing function must be 'fast' or"
-                             "'murmur', got '{}'"
-                             "".format(self.hashing))
+            return np.array([ngram_min_hash(string, self.ngram_range, seed)
+                            for seed in range(self.n_components)])
 
     def fit(self, X, y=None):
         """
@@ -158,11 +152,11 @@ class MinHashEncoder(BaseEstimator, TransformerMixin):
         ----------
         X : array-like, shape (n_samples, ) or (n_samples, 1)
             The string data to encode.
-        
+
         Returns
         -------
         self
-            The fitted MinHashEncoder instance. 
+            The fitted MinHashEncoder instance.
         """
         self.hash_dict = LRUDict(capacity=2**10)
         return self
@@ -185,14 +179,51 @@ class MinHashEncoder(BaseEstimator, TransformerMixin):
             X = X[:, 0]
         # Check if first item has str or np.str_ type
         assert isinstance(X[0], str), "ERROR: Input data is not string."
+        if self.minmax_hash:
+            assert self.n_components % 2 == 0,\
+                    "n_components should be even when minmax_hash=True"
+        if self.hashing == 'murmur':
+            assert not(self.minmax_hash),\
+                   "minmax_hash not implemented with murmur"
+        if self.handle_missing not in ['error', '']:
+            template = ("handle_missing should be either 'error' or "
+                        "'', got %s")
+            raise ValueError(template % self.handle_missing)
         X_out = np.zeros((len(X), self.n_components))
+        X = X.reshape(-1)
 
         # TODO Parallel run here
-        for i, x in enumerate(X):
-            if x not in self.hash_dict:
-                self.hash_dict[x] = self.get_hash(x)
+        nan_idx = []
+
+        if self.hashing == 'fast':
+            for i, x in enumerate(X):
+                if isinstance(x, float):
+                    nan_idx.append(i)
+                elif x not in self.hash_dict:
+                    X_out[i, :] = self.hash_dict[x] = self.get_fast_hash(x)
+                else:
+                    X_out[i, :] = self.hash_dict[x]
+
+        elif self.hashing == 'murmur':
+            for i, x in enumerate(X):
+                if isinstance(x, float):
+                    nan_idx.append(i)
+                elif x not in self.hash_dict:
+                    X_out[i, :] = self.hash_dict[x] = self.minhash(
+                        x,
+                        n_components=self.n_components,
+                        ngram_range=self.ngram_range)
+                else:
+                    X_out[i, :] = self.hash_dict[x]
+
+        else:
+            raise ValueError("hashing function must be 'fast' or"
+                             "'murmur', got '{}'"
+                             "".format(self.hashing))
+
+        if self.handle_missing == 'error' and nan_idx:
+            msg = ("Found missing values in input data; set "
+                   "handle_missing='' to encode with missing values")
+            raise ValueError(msg)
 
-        for i, x in enumerate(X):
-            X_out[i, :] = self.hash_dict[x]
-            
         return X_out


Test patch:
diff --git a/dirty_cat/test/test_minhash_encoder.py b/dirty_cat/test/test_minhash_encoder.py
index 4ae6f9a..b780e65 100644
--- a/dirty_cat/test/test_minhash_encoder.py
+++ b/dirty_cat/test/test_minhash_encoder.py
@@ -1,5 +1,6 @@
-import numpy as np
 import time
+import numpy as np
+import pytest
 
 from sklearn.datasets import fetch_20newsgroups
 from dirty_cat import MinHashEncoder
@@ -56,18 +57,20 @@ def test_get_unique_ngrams():
     enc = MinHashEncoder(n_components=2)
     ngrams = enc.get_unique_ngrams(string, ngram_range)
     assert ngrams == true_ngrams
-        
+
+
 def profile_encoder(Encoder, hashing='fast', minmax_hash=False):
     # not an unit test
-    
+
     from dirty_cat import datasets
     import pandas as pd
     employee_salaries = datasets.fetch_employee_salaries()
-    data = pd.read_csv(employee_salaries['path'])
-    X = data['Employee Position Title'].tolist()
+    data = employee_salaries['data']
+    X = data['employee_position_title'].tolist()
     X = X * 10
     t0 = time.time()
-    encoder = Encoder(n_components=50, hashing=hashing, minmax_hash=minmax_hash)
+    encoder = Encoder(n_components=50, hashing=hashing,
+                      minmax_hash=minmax_hash)
     encoder.fit(X)
     y = encoder.transform(X)
     assert y.shape == (len(X), 50)
@@ -75,6 +78,52 @@ def profile_encoder(Encoder, hashing='fast', minmax_hash=False):
     return eta
 
 
+@pytest.mark.parametrize("input_type, missing, hashing", [
+    ['list', '', 'fast'],
+    ['numpy', 'error', 'fast'],
+    ['pandas', '', 'murmur'],
+    ['numpy', '', 'fast']])
+def test_missing_values(input_type, missing, hashing):
+    X = ['Red',
+         np.nan,
+         'green',
+         'blue',
+         'green',
+         'green',
+         'blue',
+         float('nan')]
+    n = 3
+    z = np.zeros(n)
+
+    if input_type == 'numpy':
+        X = np.array(X, dtype=object)
+    elif input_type == 'pandas':
+        pd = pytest.importorskip("pandas")
+        X = pd.DataFrame(X)
+
+    encoder = MinHashEncoder(n_components=n, hashing=hashing,
+                             minmax_hash=False, handle_missing=missing)
+    if missing == 'error':
+        encoder.fit(X)
+        if input_type in ['numpy', 'pandas']:
+            with pytest.raises(ValueError, match=r"missing"
+                               " values in input"):
+                encoder.transform(X)
+    elif missing == '':
+        encoder.fit(X)
+        y = encoder.transform(X)
+        if input_type == 'list':
+            assert np.allclose(y[1], y[-1])
+        else:
+            assert np.array_equal(y[1], z)
+            assert np.array_equal(y[-1], z)
+    else:
+        with pytest.raises(ValueError, match=r"handle_missing"
+                           " should be either 'error' or ''"):
+            encoder.fit_transform(X)
+    return
+
+
 if __name__ == '__main__':
     print('start test')
     test_MinHashEncoder()
@@ -86,7 +135,7 @@ if __name__ == '__main__':
     for _ in range(3):
         print('time profile_encoder(MinHashEncoder, hashing=fast) with minmax')
         print("{:.4} seconds".format(profile_encoder(MinHashEncoder,
-                         hashing='fast', minmax_hash=True)))
+                                     hashing='fast', minmax_hash=True)))
     print('time profile_encoder(MinHashEncoder, hashing=murmur)')
     print("{:.4} seconds".format(profile_encoder(MinHashEncoder, hashing='murmur')))