Repository: dirty-cat/dirty_cat
Pull request #242 (2022-03-18 15:20:08)

Added test of analyzer and corrected the gap encoder options
Thanks to @GaelVaroquaux, we noticed that the output of the gap encoder was identical when the parameter analyzer = "word" or  analyzer = "char". This is not possible as the encoding method should be different.

A test was added to test_gap_encoder.py to ensure that this is no longer possible.

Line 580 was missing the analyzer parameter which created the problem of identical outputs.

Finally, the Hashing Vectorizor on line 864 didn't take into account the change of the analyzer parameter, but always stayed "char", is this normal?

Thanks!


Patch:
diff --git a/CHANGES.rst b/CHANGES.rst
index 378e453..28ef8a9 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -11,6 +11,7 @@ Notes
 
     - Type detection works better: handles dates, numerics columns encoded as strings, or numeric columns containing strings for missing values
 
+* Fixed a bug that resulted in the **GapEncoder** ignoring the analyzer argument.
 
 Release 0.2.0
 =============
diff --git a/dirty_cat/gap_encoder.py b/dirty_cat/gap_encoder.py
index 762c0d5..1747b54 100644
--- a/dirty_cat/gap_encoder.py
+++ b/dirty_cat/gap_encoder.py
@@ -171,7 +171,7 @@ class GapEncoderColumn(BaseEstimator, TransformerMixin):
                 size=(self.n_components, self.n_vocab))
         elif self.init == 'k-means':
             prototypes = get_kmeans_prototypes(
-                X, self.n_components, random_state=self.random_state)
+                X, self.n_components, analyzer=self.analyzer, random_state=self.random_state)
             W = self.ngrams_count_.transform(prototypes).A + .1
             if self.add_words:
                 W2 = self.word_count_.transform(prototypes).A + .1
@@ -577,6 +577,7 @@ class GapEncoder(BaseEstimator, TransformerMixin):
         return GapEncoderColumn(
             ngram_range=self.ngram_range,
             n_components=self.n_components,
+            analyzer = self.analyzer,
             gamma_shape_prior=self.gamma_shape_prior,
             gamma_scale_prior=self.gamma_scale_prior,
             rho=self.rho,
@@ -852,7 +853,7 @@ def batch_lookup(lookup, n=1):
         yield (unq_indices, indices)
 
 
-def get_kmeans_prototypes(X, n_prototypes, hashing_dim=128,
+def get_kmeans_prototypes(X, n_prototypes, analyzer='char', hashing_dim=128,
                           ngram_range=(2, 4), sparse=False,
                           sample_weight=None, random_state=None):
     """
@@ -861,7 +862,7 @@ def get_kmeans_prototypes(X, n_prototypes, hashing_dim=128,
       - k-means clustering
       - nearest neighbor
     """
-    vectorizer = HashingVectorizer(analyzer='char', norm=None,
+    vectorizer = HashingVectorizer(analyzer=analyzer, norm=None,
                                    alternate_sign=False,
                                    ngram_range=ngram_range,
                                    n_features=hashing_dim)


Test patch:
diff --git a/dirty_cat/test/test_gap_encoder.py b/dirty_cat/test/test_gap_encoder.py
index 4390cd9..23144a2 100644
--- a/dirty_cat/test/test_gap_encoder.py
+++ b/dirty_cat/test/test_gap_encoder.py
@@ -1,12 +1,43 @@
 import time
 import numpy as np
-from numpy.random import random
 import pytest
 import pandas as pd
 
 from sklearn.datasets import fetch_20newsgroups
 from dirty_cat import GapEncoder
 
+@pytest.mark.parametrize("init1, analyzer1, analyzer2",[
+    ('k-means++', 'char', 'word'),
+    ('random', 'char', 'word'),
+    ('k-means', 'char', 'word')
+])
+def test_analyzer(init1, analyzer1, analyzer2):
+    """" Test if the output is different when the analyzer is 'word' or 'char'.
+        If it is, no error ir raised. 
+    """
+    add_words = False
+    n_samples = 70
+    X_txt = fetch_20newsgroups(subset='train')['data'][:n_samples]
+    X = np.array([X_txt, X_txt]).T
+    n_components = 10
+    # Test first analyzer output:
+    encoder = GapEncoder(
+        n_components=n_components, init='k-means++',
+        analyzer=analyzer1, add_words=add_words,
+        random_state=42, rescale_W=True)
+    encoder.fit(X)
+    y = encoder.transform(X)
+    
+    # Test the other analyzer output:
+    encoder = GapEncoder(
+        n_components=n_components, init='k-means++',
+        analyzer=analyzer2, add_words=add_words,
+        random_state=42)
+    encoder.fit(X)
+    y2 = encoder.transform(X)
+    
+    # Test inequality btw analyzer word and char ouput:
+    np.testing.assert_raises(AssertionError, np.testing.assert_array_equal, y, y2)
 
 @pytest.mark.parametrize("hashing, init, analyzer, add_words", [
     (False, 'k-means++', 'word', True),
@@ -157,6 +188,10 @@ def profile_encoder(Encoder, init):
 
 
 if __name__ == '__main__':
+    
+    print('test_analyzer')
+    test_analyzer('k-means++', 'char_wb', 'word')
+    print('test_analyzer passed')
     print('start test_gap_encoder')
     test_gap_encoder(True, 'k-means++', 'char', False)
     print('test_gap_encoder passed')