Repository: dirty-cat/dirty_cat
Pull request #444 (2022-12-20 10:47:43)

ENH Improve coverage of SimilarityEncoder
Fix part of #371 

Coverage is up from 90 to 99%


Patch:
diff --git a/CHANGES.rst b/CHANGES.rst
index 3c1af98..3a817e6 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -10,7 +10,7 @@ Major changes
   on string similarities and the nearest neighbors matches are found for each category. :pr:`291` by :user:`Jovan Stojanovic <jovan-stojanovic>` and :user:`Leo Grinsztajn <LeoGrin>`
 * **datasets.fetching**: contains a new function :func:`fetch_world_bank_indicator` that can be used to download any indicator from the World Bank Open Data platform. It only needs the indicator ID that can be found on the website. :pr:`291` by :user:`Jovan Stojanovic <jovan-stojanovic>`
 * Unnecessary API has been made private: everything (files, functions, classes) starting with an underscore shouldn't be imported in your code. :pr:`331` by :user:`Lilian Boulard <LilianBoulard>`
-* The MinHashEncoder now supports a `n_jobs` parameter to parallelize the hashes computation. :pr:`267` by `Leo Grinsztajn <LeoGrin>` and `Lilian Boulard <LilianBoulard>`.
+* The MinHashEncoder now supports a `n_jobs` parameter to parallelize the hashes computation. :pr:`267` by :user:`Leo Grinsztajn <LeoGrin>` and :user:`Lilian Boulard <LilianBoulard>`.
 
 Minor changes
 -------------
diff --git a/build_tools/circle/build_doc.sh b/build_tools/circle/build_doc.sh
index 0da832c..b66372d 100755
--- a/build_tools/circle/build_doc.sh
+++ b/build_tools/circle/build_doc.sh
@@ -108,7 +108,7 @@ fi
 # Install dependencies with miniconda
 wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \
    -O miniconda.sh
-chmod +x miniconda.sh && ./miniconda.sh -b -p $MINICONDA_PATH
+chmod +x miniconda.sh && bash ./miniconda.sh -b -p $MINICONDA_PATH
 export PATH="$MINICONDA_PATH/bin:$PATH"
 conda update --yes --quiet conda
 
diff --git a/dirty_cat/_gap_encoder.py b/dirty_cat/_gap_encoder.py
index 61302f2..0b595b7 100644
--- a/dirty_cat/_gap_encoder.py
+++ b/dirty_cat/_gap_encoder.py
@@ -40,6 +40,9 @@ else:
 
 from sklearn.decomposition._nmf import _beta_divergence
 
+# Ignore lines too long, as some things in the docstring cannot be cut.
+# flake8: noqa: E501
+
 
 class GapEncoderColumn(BaseEstimator, TransformerMixin):
 
@@ -539,74 +542,92 @@ class GapEncoderColumn(BaseEstimator, TransformerMixin):
 
 
 class GapEncoder(BaseEstimator, TransformerMixin):
-    """
+    """Constructs latent topics with continuous encoding.
+
     This encoder can be understood as a continuous encoding on a set of latent
     categories estimated from the data. The latent categories are built by
     capturing combinations of substrings that frequently co-occur.
 
-    The GapEncoder supports online learning on batches of data for
-    scalability through the partial_fit method.
+    The :class:`~dirty_cat.GapEncoder` supports online learning on batches of
+    data for scalability through the :func:`~dirty_cat.GapEncoder.partial_fit`
+    method.
 
     Parameters
     ----------
-    n_components : int, default=10
+    n_components : int, optional, default=10
         Number of latent categories used to model string data.
-    batch_size : int, default=128
+    batch_size : int, optional, default=128
         Number of samples per batch.
-    gamma_shape_prior : float, default=1.1
+    gamma_shape_prior : float, optional, default=1.1
         Shape parameter for the Gamma prior distribution.
-    gamma_scale_prior : float, default=1.0
+    gamma_scale_prior : float, optional, default=1.0
         Scale parameter for the Gamma prior distribution.
-    rho : float, default=0.95
-        Weight parameter for the update of the W matrix.
-    rescale_rho : bool, default=False
-        If true, use rho ** (batch_size / len(X)) instead of rho to obtain an
+    rho : float, optional, default=0.95
+        Weight parameter for the update of the *W* matrix.
+    rescale_rho : bool, optional, default=False
+        If true, use ``rho ** (batch_size / len(X))`` instead of rho to obtain an
         update rate per iteration that is independent of the batch size.
-    hashing : bool, default=False
-        If true, HashingVectorizer is used instead of CountVectorizer.
-        It has the advantage of being very low memory scalable to large
+    hashing : bool, optional, default=False
+        If true, :class:`~sklearn.feature_extraction.text.HashingVectorizer`
+        is used instead of :class:`~sklearn.feature_extraction.text.CountVectorizer`.
+        It has the advantage of being very low memory, scalable to large
         datasets as there is no need to store a vocabulary dictionary in
         memory.
-    hashing_n_features : int, default=2**12
-        Number of features for the HashingVectorizer. Only relevant if
-        hashing=True.
-    init : typing.Literal["k-means++", "random", "k-means"], default='k-means++'
+    hashing_n_features : int, optional, default=2**12
+        Number of features for the :class:`~sklearn.feature_extraction.text.HashingVectorizer`.
+        Only relevant if `hashing=True`.
+    init : {"k-means++", "random", "k-means"}, optional, default='k-means++'
         Initialization method of the W matrix.
-        Options: {'k-means++', 'random', 'k-means'}.
-        If init='k-means++', we use the init method of sklearn.cluster.KMeans.
-        If init='random', topics are initialized with a Gamma distribution.
-        If init='k-means', topics are initialized with a KMeans on the n-grams
+        If `init='k-means++'`, we use the init method of :class:`~sklearn.cluster.KMeans`.
+        If `init='random'`, topics are initialized with a Gamma distribution.
+        If `init='k-means'`, topics are initialized with a KMeans on the n-grams
         counts. This usually makes convergence faster but is a bit slower.
     tol : float, default=1e-4
-        Tolerance for the convergence of the matrix W.
-    min_iter : int, default=2
+        Tolerance for the convergence of the matrix *W*.
+    min_iter : int, optional, default=2
         Minimum number of iterations on the input data.
-    max_iter : int, default=5
+    max_iter : int, optional, default=5
         Maximum number of iterations on the input data.
-    ngram_range : typing.Tuple[int, int], default=(2, 4)
+    ngram_range : int 2-tuple, optional, default=(2, 4)
         The range of ngram length that will be used to build the
         bag-of-n-grams representation of the input data.
-    analyzer : typing.Literal["word", "char", "char_wb"], default='char'.
-        Analyzer parameter for the CountVectorizer/HashingVectorizer.
-        Options: {‘word’, ‘char’, ‘char_wb’}, describing whether the matrix V
-        to factorize should be made of word counts or character n-gram counts.
+    analyzer : {"word", "char", "char_wb"}, optional, default='char'
+        Analyzer parameter for the :class:`~sklearn.feature_extraction.text.HashingVectorizer`
+        / :class:`~sklearn.feature_extraction.text.CountVectorizer`.
+        Describes whether the matrix *V* to factorize should be made of word counts
+        or character n-gram counts.
         Option ‘char_wb’ creates character n-grams only from text inside word
         boundaries; n-grams at the edges of words are padded with space.
-    add_words : bool, default=False
+    add_words : bool, optional, default=False
         If true, add the words counts to the bag-of-n-grams representation
         of the input data.
-    random_state : typing.Optional[Union[int, RandomState]], default=None
-        Pass an int for reproducible output across multiple function calls.
-    rescale_W : bool, default=True
-        If true, the weight matrix W is rescaled at each iteration
+    random_state : int, :class:`numpy.random.RandomState` or None, optional, default=None
+        RNG seed for reproducible output across multiple function calls.
+    rescale_W : bool, optional, default=True
+        If true, the weight matrix *W* is rescaled at each iteration
         to have a l1 norm equal to 1 for each row.
     max_iter_e_step : int, default=20
         Maximum number of iterations to adjust the activations h at each step.
-    handle_missing : typing.Literal["error", "empty_impute"], default=empty_impute
-        Whether to raise an error or impute with empty string '' if missing
+    handle_missing : {"error", "empty_impute"}, optional, default='empty_impute'
+        Whether to raise an error or impute with empty string ``''`` if missing
         values (NaN) are present during fit (default is to impute).
         In the inverse transform, the missing category will be denoted as None.
 
+    Attributes
+    ----------
+    rho_: float
+        Effective update rate for the W matrix
+    fitted_models_: list of GapEncoderColumn
+        Column-wise fitted GapEncoders
+    column_names_: list of str
+        Column names of the data the Gap was fitted on
+
+    References
+    ----------
+    For a detailed description of the method, see
+    `Encoding high-cardinality string categorical variables
+    <https://hal.inria.fr/hal-02171256v4>`_ by Cerda, Varoquaux (2019).
+
     Examples
     --------
     >>> enc = GapEncoder(n_components=2)
@@ -624,8 +645,8 @@ class GapEncoder(BaseEstimator, TransformerMixin):
     >>> enc.get_feature_names()
     ['england, london, uk', 'france, paris, pqris']
 
-    He got it right, reccuring topics are "London" and "England" on the
-    one side and and "Paris" and "France" on the other.
+    It got it right, reccuring topics are "London" and "England" on the
+    one side and "Paris" and "France" on the other.
 
     As this is a continuous encoding, we can look at the level of
     activation of each topic for each category:
@@ -641,19 +662,6 @@ class GapEncoder(BaseEstimator, TransformerMixin):
           [ 0.05002016,  4.54997983]])
 
     The higher the value, the bigger the correspondance with the topic.
-
-    Attributes
-    ----------
-    rho_: float
-    fitted_models_: typing.List[GapEncoderColumn]
-    column_names_: typing.List[str]
-
-    References
-    ----------
-    For a detailed description of the method, see
-    `Encoding high-cardinality string categorical variables
-    <https://hal.inria.fr/hal-02171256v4>`_ by Cerda, Varoquaux (2019).
-
     """
 
     rho_: float
diff --git a/dirty_cat/_similarity_encoder.py b/dirty_cat/_similarity_encoder.py
index 93e91cc..f622acf 100644
--- a/dirty_cat/_similarity_encoder.py
+++ b/dirty_cat/_similarity_encoder.py
@@ -337,7 +337,11 @@ class SimilarityEncoder(OneHotEncoder):
         self.similarity = None
 
         if not isinstance(categories, list):
-            assert categories in [None, "auto", "k-means", "most_frequent"]
+            if categories not in ["auto", "k-means", "most_frequent"]:
+                raise ValueError(
+                    f"Got categories={self.categories}, but expected "
+                    "any of {'auto', 'k-means', 'most_frequent'}. "
+                )
         if categories in ["k-means", "most_frequent"] and (
             n_prototypes is None or n_prototypes == 0
         ):
@@ -522,7 +526,7 @@ class SimilarityEncoder(OneHotEncoder):
                     "Found missing values in input data; set "
                     "handle_missing='' to encode with missing values. "
                 )
-            if self.handle_missing != "error":
+            else:
                 X = X.fillna(self.handle_missing)
         elif not hasattr(X, "dtype") and isinstance(X, list):
             X = np.asarray(X, dtype=object)


Test patch:
diff --git a/dirty_cat/tests/test_docstrings.py b/dirty_cat/tests/test_docstrings.py
index 5c4b2a6..0a683aa 100644
--- a/dirty_cat/tests/test_docstrings.py
+++ b/dirty_cat/tests/test_docstrings.py
@@ -18,7 +18,6 @@ from numpydoc.validate import validate
 DOCSTRING_TEMP_IGNORE_SET = {
     "dirty_cat._datetime_encoder.DatetimeEncoder.get_feature_names",
     "dirty_cat._datetime_encoder.DatetimeEncoder.get_feature_names_out",
-    "dirty_cat._gap_encoder.GapEncoder",
     "dirty_cat._gap_encoder.GapEncoder.fit",
     "dirty_cat._gap_encoder.GapEncoder.get_feature_names",
     "dirty_cat._gap_encoder.GapEncoder.get_feature_names_out",
diff --git a/dirty_cat/tests/test_similarity_encoder.py b/dirty_cat/tests/test_similarity_encoder.py
index e3a0927..292c7c2 100644
--- a/dirty_cat/tests/test_similarity_encoder.py
+++ b/dirty_cat/tests/test_similarity_encoder.py
@@ -6,7 +6,7 @@ import pytest
 from sklearn import __version__ as sklearn_version
 
 from dirty_cat import SimilarityEncoder
-from dirty_cat._similarity_encoder import get_kmeans_prototypes
+from dirty_cat._similarity_encoder import get_kmeans_prototypes, ngram_similarity_matrix
 from dirty_cat._string_distances import ngram_similarity
 from dirty_cat._utils import parse_version
 
@@ -49,6 +49,25 @@ def test_fast_ngram_similarity() -> None:
     assert np.allclose(feature_matrix, feature_matrix_fast)
 
 
+def test_parameters():
+    X = [["foo"], ["baz"]]
+    X2 = [["foo"], ["bar"]]
+    with pytest.raises(ValueError, match=r"Got handle_unknown="):
+        SimilarityEncoder(handle_unknown="bb").fit(X)
+    with pytest.raises(ValueError, match=r"Got hashing_dim="):
+        SimilarityEncoder(hashing_dim="bb").fit(X)
+    with pytest.raises(ValueError, match=r"Got categories="):
+        SimilarityEncoder(categories="bb")
+    with pytest.raises(ValueError, match=r"Unsorted categories "):
+        SimilarityEncoder(categories=[["cat2", "cat1"], ["cat3", "cat4"]]).fit(X)
+    with pytest.raises(ValueError, match=r"Found unknown categories "):
+        SimilarityEncoder(categories=[["fooo", "loo"]], handle_unknown="error").fit(X)
+    with pytest.raises(ValueError, match=r"Found unknown categories "):
+        sim = SimilarityEncoder(categories=[["baz", "foo"]], handle_unknown="error")
+        sim.fit(X)
+        sim.transform(X2)
+
+
 def _test_missing_values(input_type, missing):
     observations = [["a", "b"], ["b", "a"], ["b", np.nan], ["a", "c"], [np.nan, "a"]]
     encoded = np.array(
@@ -146,6 +165,12 @@ def _test_similarity(
         ).reshape(-1, 1)
         X_test = np.array([["Aa", "aAa", "aaa", "aaab", " aaa  c"]]).reshape(-1, 1)
 
+        if categories == "auto":
+            with pytest.warns(UserWarning, match=r"n_prototypes parameter ignored"):
+                SimilarityEncoder(
+                    categories=categories,
+                    n_prototypes=n_prototypes,
+                )
         try:
             model = SimilarityEncoder(
                 hashing_dim=hashing_dim,
@@ -157,7 +182,7 @@ def _test_similarity(
         except ValueError as e:
             assert (
                 e.__str__()
-                == "n_prototypes expected None or a positive non null integer"
+                == "n_prototypes expected None or a positive non null integer. "
             )
             return
 
@@ -193,8 +218,9 @@ def test_similarity_encoder() -> None:
                 hashing_dim=2**16,
                 categories=category,
             )
+            _test_similarity(ngram_similarity, categories=category, n_prototypes=4)
         else:
-            for i in range(1, 4):
+            for i in range(0, 4):
                 _test_similarity(
                     ngram_similarity,
                     categories=category,
@@ -217,8 +243,18 @@ def test_similarity_encoder() -> None:
 
 def test_kmeans_protoypes() -> None:
     X_test = np.array(["cbbba", "baaac", "accc"])
-    proto = get_kmeans_prototypes(X_test, 3)
+    proto = get_kmeans_prototypes(X_test, 3, sparse=True)
     assert np.array_equal(np.sort(proto), np.sort(X_test))
+    X_test_2 = np.array(["aa", "bb", "cc", "bbb"])
+    with pytest.warns(UserWarning, match=r"number of unique prototypes is lower "):
+        get_kmeans_prototypes(X_test_2, 4)
+
+
+def test_ngram_similarity_matrix() -> None:
+    X1 = np.array(["cat1", "cat2", "cat3"])
+    X2 = np.array(["cata1", "caat2", "ccat3"])
+    sim = ngram_similarity_matrix(X1, X2, ngram_range=(2, 2), hashing_dim=5)
+    assert sim.shape == (len(X1), len(X2))
 
 
 def test_reproducibility() -> None:
@@ -233,6 +269,16 @@ def test_reproducibility() -> None:
         assert np.array_equal(prototypes, sim_enc.fit(X).categories_[0])
 
 
+def test_fit_transform():
+    X = [["foo"], ["baz"]]
+    y = ["foo", "bar"]
+    tr1 = SimilarityEncoder().fit_transform(X, y)
+    sim = SimilarityEncoder()
+    sim.fit(X, y)
+    tr2 = sim.transform(X, y)
+    assert np.array_equal(tr1, tr2)
+
+
 def test_get_features() -> None:
     # See https://github.com/dirty-cat/dirty_cat/issues/168
     sim_enc = SimilarityEncoder(random_state=435)