Repository: dirty-cat/dirty_cat
Pull request #435 (2022-12-16 15:32:29)

Fix em dash in website titles
Currently, page titles are rendered with the code ``&mdash`` in the title, which is very ugly.
The dash is already rendered by sphinx or whatnot, so we should remove it.


Patch:
diff --git a/CHANGES.rst b/CHANGES.rst
index 3c1af98..3a817e6 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -10,7 +10,7 @@ Major changes
   on string similarities and the nearest neighbors matches are found for each category. :pr:`291` by :user:`Jovan Stojanovic <jovan-stojanovic>` and :user:`Leo Grinsztajn <LeoGrin>`
 * **datasets.fetching**: contains a new function :func:`fetch_world_bank_indicator` that can be used to download any indicator from the World Bank Open Data platform. It only needs the indicator ID that can be found on the website. :pr:`291` by :user:`Jovan Stojanovic <jovan-stojanovic>`
 * Unnecessary API has been made private: everything (files, functions, classes) starting with an underscore shouldn't be imported in your code. :pr:`331` by :user:`Lilian Boulard <LilianBoulard>`
-* The MinHashEncoder now supports a `n_jobs` parameter to parallelize the hashes computation. :pr:`267` by `Leo Grinsztajn <LeoGrin>` and `Lilian Boulard <LilianBoulard>`.
+* The MinHashEncoder now supports a `n_jobs` parameter to parallelize the hashes computation. :pr:`267` by :user:`Leo Grinsztajn <LeoGrin>` and :user:`Lilian Boulard <LilianBoulard>`.
 
 Minor changes
 -------------
diff --git a/build_tools/circle/build_doc.sh b/build_tools/circle/build_doc.sh
index 0da832c..b66372d 100755
--- a/build_tools/circle/build_doc.sh
+++ b/build_tools/circle/build_doc.sh
@@ -108,7 +108,7 @@ fi
 # Install dependencies with miniconda
 wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \
    -O miniconda.sh
-chmod +x miniconda.sh && ./miniconda.sh -b -p $MINICONDA_PATH
+chmod +x miniconda.sh && bash ./miniconda.sh -b -p $MINICONDA_PATH
 export PATH="$MINICONDA_PATH/bin:$PATH"
 conda update --yes --quiet conda
 
diff --git a/dirty_cat/_datetime_encoder.py b/dirty_cat/_datetime_encoder.py
index 29e2ff5..afa6186 100644
--- a/dirty_cat/_datetime_encoder.py
+++ b/dirty_cat/_datetime_encoder.py
@@ -70,6 +70,26 @@ class DatetimeEncoder(BaseEstimator, TransformerMixin):
     col_names_: typing.Optional[typing.List[str]]
         List of the names of the features of the input data,
         if input data was a pandas DataFrame, otherwise None.
+
+    Examples
+    --------
+    >>> enc = DatetimeEncoder()
+
+    Let's encode the following dates:
+
+    >>> X = [['2022-10-15'], ['2021-12-25'], ['2020-05-18'], ['2019-10-15 12:00:00']]
+
+    >>> enc.fit(X)
+    DatetimeEncoder()
+
+    The encoder will output a transformed array
+    with four columns (year, month, day and hour):
+
+    >>> enc.transform(X)
+    array([[2022.,   10.,   15.,    0.],
+           [2021.,   12.,   25.,    0.],
+           [2020.,    5.,   18.,    0.],
+           [2019.,   10.,   15.,   12.]])
     """
 
     n_features_in_: int
diff --git a/dirty_cat/_gap_encoder.py b/dirty_cat/_gap_encoder.py
index 61302f2..0b595b7 100644
--- a/dirty_cat/_gap_encoder.py
+++ b/dirty_cat/_gap_encoder.py
@@ -40,6 +40,9 @@ else:
 
 from sklearn.decomposition._nmf import _beta_divergence
 
+# Ignore lines too long, as some things in the docstring cannot be cut.
+# flake8: noqa: E501
+
 
 class GapEncoderColumn(BaseEstimator, TransformerMixin):
 
@@ -539,74 +542,92 @@ class GapEncoderColumn(BaseEstimator, TransformerMixin):
 
 
 class GapEncoder(BaseEstimator, TransformerMixin):
-    """
+    """Constructs latent topics with continuous encoding.
+
     This encoder can be understood as a continuous encoding on a set of latent
     categories estimated from the data. The latent categories are built by
     capturing combinations of substrings that frequently co-occur.
 
-    The GapEncoder supports online learning on batches of data for
-    scalability through the partial_fit method.
+    The :class:`~dirty_cat.GapEncoder` supports online learning on batches of
+    data for scalability through the :func:`~dirty_cat.GapEncoder.partial_fit`
+    method.
 
     Parameters
     ----------
-    n_components : int, default=10
+    n_components : int, optional, default=10
         Number of latent categories used to model string data.
-    batch_size : int, default=128
+    batch_size : int, optional, default=128
         Number of samples per batch.
-    gamma_shape_prior : float, default=1.1
+    gamma_shape_prior : float, optional, default=1.1
         Shape parameter for the Gamma prior distribution.
-    gamma_scale_prior : float, default=1.0
+    gamma_scale_prior : float, optional, default=1.0
         Scale parameter for the Gamma prior distribution.
-    rho : float, default=0.95
-        Weight parameter for the update of the W matrix.
-    rescale_rho : bool, default=False
-        If true, use rho ** (batch_size / len(X)) instead of rho to obtain an
+    rho : float, optional, default=0.95
+        Weight parameter for the update of the *W* matrix.
+    rescale_rho : bool, optional, default=False
+        If true, use ``rho ** (batch_size / len(X))`` instead of rho to obtain an
         update rate per iteration that is independent of the batch size.
-    hashing : bool, default=False
-        If true, HashingVectorizer is used instead of CountVectorizer.
-        It has the advantage of being very low memory scalable to large
+    hashing : bool, optional, default=False
+        If true, :class:`~sklearn.feature_extraction.text.HashingVectorizer`
+        is used instead of :class:`~sklearn.feature_extraction.text.CountVectorizer`.
+        It has the advantage of being very low memory, scalable to large
         datasets as there is no need to store a vocabulary dictionary in
         memory.
-    hashing_n_features : int, default=2**12
-        Number of features for the HashingVectorizer. Only relevant if
-        hashing=True.
-    init : typing.Literal["k-means++", "random", "k-means"], default='k-means++'
+    hashing_n_features : int, optional, default=2**12
+        Number of features for the :class:`~sklearn.feature_extraction.text.HashingVectorizer`.
+        Only relevant if `hashing=True`.
+    init : {"k-means++", "random", "k-means"}, optional, default='k-means++'
         Initialization method of the W matrix.
-        Options: {'k-means++', 'random', 'k-means'}.
-        If init='k-means++', we use the init method of sklearn.cluster.KMeans.
-        If init='random', topics are initialized with a Gamma distribution.
-        If init='k-means', topics are initialized with a KMeans on the n-grams
+        If `init='k-means++'`, we use the init method of :class:`~sklearn.cluster.KMeans`.
+        If `init='random'`, topics are initialized with a Gamma distribution.
+        If `init='k-means'`, topics are initialized with a KMeans on the n-grams
         counts. This usually makes convergence faster but is a bit slower.
     tol : float, default=1e-4
-        Tolerance for the convergence of the matrix W.
-    min_iter : int, default=2
+        Tolerance for the convergence of the matrix *W*.
+    min_iter : int, optional, default=2
         Minimum number of iterations on the input data.
-    max_iter : int, default=5
+    max_iter : int, optional, default=5
         Maximum number of iterations on the input data.
-    ngram_range : typing.Tuple[int, int], default=(2, 4)
+    ngram_range : int 2-tuple, optional, default=(2, 4)
         The range of ngram length that will be used to build the
         bag-of-n-grams representation of the input data.
-    analyzer : typing.Literal["word", "char", "char_wb"], default='char'.
-        Analyzer parameter for the CountVectorizer/HashingVectorizer.
-        Options: {‘word’, ‘char’, ‘char_wb’}, describing whether the matrix V
-        to factorize should be made of word counts or character n-gram counts.
+    analyzer : {"word", "char", "char_wb"}, optional, default='char'
+        Analyzer parameter for the :class:`~sklearn.feature_extraction.text.HashingVectorizer`
+        / :class:`~sklearn.feature_extraction.text.CountVectorizer`.
+        Describes whether the matrix *V* to factorize should be made of word counts
+        or character n-gram counts.
         Option ‘char_wb’ creates character n-grams only from text inside word
         boundaries; n-grams at the edges of words are padded with space.
-    add_words : bool, default=False
+    add_words : bool, optional, default=False
         If true, add the words counts to the bag-of-n-grams representation
         of the input data.
-    random_state : typing.Optional[Union[int, RandomState]], default=None
-        Pass an int for reproducible output across multiple function calls.
-    rescale_W : bool, default=True
-        If true, the weight matrix W is rescaled at each iteration
+    random_state : int, :class:`numpy.random.RandomState` or None, optional, default=None
+        RNG seed for reproducible output across multiple function calls.
+    rescale_W : bool, optional, default=True
+        If true, the weight matrix *W* is rescaled at each iteration
         to have a l1 norm equal to 1 for each row.
     max_iter_e_step : int, default=20
         Maximum number of iterations to adjust the activations h at each step.
-    handle_missing : typing.Literal["error", "empty_impute"], default=empty_impute
-        Whether to raise an error or impute with empty string '' if missing
+    handle_missing : {"error", "empty_impute"}, optional, default='empty_impute'
+        Whether to raise an error or impute with empty string ``''`` if missing
         values (NaN) are present during fit (default is to impute).
         In the inverse transform, the missing category will be denoted as None.
 
+    Attributes
+    ----------
+    rho_: float
+        Effective update rate for the W matrix
+    fitted_models_: list of GapEncoderColumn
+        Column-wise fitted GapEncoders
+    column_names_: list of str
+        Column names of the data the Gap was fitted on
+
+    References
+    ----------
+    For a detailed description of the method, see
+    `Encoding high-cardinality string categorical variables
+    <https://hal.inria.fr/hal-02171256v4>`_ by Cerda, Varoquaux (2019).
+
     Examples
     --------
     >>> enc = GapEncoder(n_components=2)
@@ -624,8 +645,8 @@ class GapEncoder(BaseEstimator, TransformerMixin):
     >>> enc.get_feature_names()
     ['england, london, uk', 'france, paris, pqris']
 
-    He got it right, reccuring topics are "London" and "England" on the
-    one side and and "Paris" and "France" on the other.
+    It got it right, reccuring topics are "London" and "England" on the
+    one side and "Paris" and "France" on the other.
 
     As this is a continuous encoding, we can look at the level of
     activation of each topic for each category:
@@ -641,19 +662,6 @@ class GapEncoder(BaseEstimator, TransformerMixin):
           [ 0.05002016,  4.54997983]])
 
     The higher the value, the bigger the correspondance with the topic.
-
-    Attributes
-    ----------
-    rho_: float
-    fitted_models_: typing.List[GapEncoderColumn]
-    column_names_: typing.List[str]
-
-    References
-    ----------
-    For a detailed description of the method, see
-    `Encoding high-cardinality string categorical variables
-    <https://hal.inria.fr/hal-02171256v4>`_ by Cerda, Varoquaux (2019).
-
     """
 
     rho_: float
diff --git a/dirty_cat/_similarity_encoder.py b/dirty_cat/_similarity_encoder.py
index bbc3927..f622acf 100644
--- a/dirty_cat/_similarity_encoder.py
+++ b/dirty_cat/_similarity_encoder.py
@@ -80,7 +80,7 @@ def _ngram_similarity_one_sample_inplace(
     se_dict[unq_X[i]] = similarity.reshape(-1)
 
 
-def ngram_similarity(
+def ngram_similarity_matrix(
     X,
     cats: List[str],
     ngram_range: Tuple[int, int],
@@ -337,7 +337,11 @@ class SimilarityEncoder(OneHotEncoder):
         self.similarity = None
 
         if not isinstance(categories, list):
-            assert categories in [None, "auto", "k-means", "most_frequent"]
+            if categories not in ["auto", "k-means", "most_frequent"]:
+                raise ValueError(
+                    f"Got categories={self.categories}, but expected "
+                    "any of {'auto', 'k-means', 'most_frequent'}. "
+                )
         if categories in ["k-means", "most_frequent"] and (
             n_prototypes is None or n_prototypes == 0
         ):
@@ -522,7 +526,7 @@ class SimilarityEncoder(OneHotEncoder):
                     "Found missing values in input data; set "
                     "handle_missing='' to encode with missing values. "
                 )
-            if self.handle_missing != "error":
+            else:
                 X = X.fillna(self.handle_missing)
         elif not hasattr(X, "dtype") and isinstance(X, list):
             X = np.asarray(X, dtype=object)
@@ -560,7 +564,7 @@ class SimilarityEncoder(OneHotEncoder):
             if fast:
                 encoded_Xj = self._ngram_similarity_fast(Xlist[j], j)
             else:
-                encoded_Xj = ngram_similarity(
+                encoded_Xj = ngram_similarity_matrix(
                     Xlist[j],
                     categories,
                     ngram_range=(min_n, max_n),
diff --git a/dirty_cat/datasets/_fetching.py b/dirty_cat/datasets/_fetching.py
index f373f54..ace1134 100644
--- a/dirty_cat/datasets/_fetching.py
+++ b/dirty_cat/datasets/_fetching.py
@@ -237,9 +237,8 @@ def _fetch_world_bank_data(
               saved as a CSV file.
 
     """
-    path = f"{data_directory}/{indicator_id}.csv"
-    Path(data_directory).mkdir(parents=True, exist_ok=True)
-    csv_path = Path(path).resolve()
+    csv_path = (data_directory / f"{indicator_id}.csv").resolve()
+    data_directory.mkdir(parents=True, exist_ok=True)
     url = f"https://api.worldbank.org/v2/en/indicator/{indicator_id}?downloadformat=csv"  # noqa
     if csv_path.is_file():
         df = pd.read_csv(csv_path, nrows=0)
diff --git a/doc/conf.py b/doc/conf.py
index ab3a59b..334d8c8 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -20,11 +20,13 @@ import os
 import shutil
 from datetime import datetime
 import sys
+
 # If extensions (or modules to document with autodoc) are in another
 # directory, add these directories to sys.path here. If the directory
 # is relative to the documentation root, use os.path.abspath to make it
 # absolute, like shown here.
 sys.path.insert(0, os.path.abspath("sphinxext"))
+from github_link import make_linkcode_resolve
 
 
 # -- Copy files for docs --------------------------------------------------
@@ -50,10 +52,10 @@ extensions = [
     "sphinx.ext.doctest",
     "sphinx.ext.intersphinx",
     "sphinx.ext.mathjax",
-    "sphinx.ext.viewcode",
     "sphinx.ext.githubpages",
     "numpydoc",
     "sphinx_issues",
+    "sphinx.ext.linkcode",
     "sphinx.ext.autodoc.typehints",
     "sphinx_gallery.gen_gallery",
 ]
@@ -174,7 +176,8 @@ html_static_path = ["_static"]
 
 
 # Modify the title to get good social-media links
-html_title = "&mdash; Dirty cat"
+html_title = "— dirty_cat"
+
 
 # -- Options for HTMLHelp output ------------------------------------------
 
@@ -307,3 +310,11 @@ html_js_files = ["scrolltoc.js"]
 
 # we use the issues path for PRs since the issues URL will forward
 issues_github_path = "dirty-cat/dirty_cat"
+
+# The following is used by sphinx.ext.linkcode to provide links to github
+linkcode_resolve = make_linkcode_resolve(
+    "dirty_cat",
+    "https://github.com/dirty-cat/"
+    "dirty-cat/blob/{revision}/"
+    "{package}/{path}#L{lineno}",
+)
diff --git a/doc/sphinxext/github_link.py b/doc/sphinxext/github_link.py
new file mode 100644
index 0000000..3ad3743
--- /dev/null
+++ b/doc/sphinxext/github_link.py
@@ -0,0 +1,83 @@
+"""
+Taken from https://github.com/scikit-learn/scikit-learn/blob/3d4ee9e9dc384ea11724e270b51bd26a1f227f0c/doc/sphinxext/github_link.py
+"""
+
+from operator import attrgetter
+import inspect
+import subprocess
+import os
+import sys
+from functools import partial
+
+REVISION_CMD = "git rev-parse --short HEAD"
+
+
+def _get_git_revision():
+    try:
+        revision = subprocess.check_output(REVISION_CMD.split()).strip()
+    except (subprocess.CalledProcessError, OSError):
+        print("Failed to execute git to get revision")
+        return None
+    return revision.decode("utf-8")
+
+
+def _linkcode_resolve(domain, info, package, url_fmt, revision):
+    """Determine a link to online source for a class/method/function
+    This is called by sphinx.ext.linkcode
+    An example with a long-untouched module that everyone has
+    >>> _linkcode_resolve('py', {'module': 'tty',
+    ...                          'fullname': 'setraw'},
+    ...                   package='tty',
+    ...                   url_fmt='http://hg.python.org/cpython/file/'
+    ...                           '{revision}/Lib/{package}/{path}#L{lineno}',
+    ...                   revision='xxxx')
+    'http://hg.python.org/cpython/file/xxxx/Lib/tty/tty.py#L18'
+    """
+
+    if revision is None:
+        return
+    if domain not in ("py", "pyx"):
+        return
+    if not info.get("module") or not info.get("fullname"):
+        return
+
+    class_name = info["fullname"].split(".")[0]
+    module = __import__(info["module"], fromlist=[class_name])
+    obj = attrgetter(info["fullname"])(module)
+
+    # Unwrap the object to get the correct source
+    # file in case that is wrapped by a decorator
+    obj = inspect.unwrap(obj)
+
+    try:
+        fn = inspect.getsourcefile(obj)
+    except Exception:
+        fn = None
+    if not fn:
+        try:
+            fn = inspect.getsourcefile(sys.modules[obj.__module__])
+        except Exception:
+            fn = None
+    if not fn:
+        return
+
+    fn = os.path.relpath(fn, start=os.path.dirname(__import__(package).__file__))
+    try:
+        lineno = inspect.getsourcelines(obj)[1]
+    except Exception:
+        lineno = ""
+    return url_fmt.format(revision=revision, package=package, path=fn, lineno=lineno)
+
+
+def make_linkcode_resolve(package, url_fmt):
+    """Returns a linkcode_resolve function for the given URL format
+    revision is a git commit reference (hash or name)
+    package is the name of the root module of the package
+    url_fmt is along the lines of ('https://github.com/USER/PROJECT/'
+                                   'blob/{revision}/{package}/'
+                                   '{path}#L{lineno}')
+    """
+    revision = _get_git_revision()
+    return partial(
+        _linkcode_resolve, revision=revision, package=package, url_fmt=url_fmt
+    )


Test patch:
diff --git a/dirty_cat/tests/test_docstrings.py b/dirty_cat/tests/test_docstrings.py
index 5c4b2a6..0a683aa 100644
--- a/dirty_cat/tests/test_docstrings.py
+++ b/dirty_cat/tests/test_docstrings.py
@@ -18,7 +18,6 @@ from numpydoc.validate import validate
 DOCSTRING_TEMP_IGNORE_SET = {
     "dirty_cat._datetime_encoder.DatetimeEncoder.get_feature_names",
     "dirty_cat._datetime_encoder.DatetimeEncoder.get_feature_names_out",
-    "dirty_cat._gap_encoder.GapEncoder",
     "dirty_cat._gap_encoder.GapEncoder.fit",
     "dirty_cat._gap_encoder.GapEncoder.get_feature_names",
     "dirty_cat._gap_encoder.GapEncoder.get_feature_names_out",
diff --git a/dirty_cat/tests/test_similarity_encoder.py b/dirty_cat/tests/test_similarity_encoder.py
index e3a0927..292c7c2 100644
--- a/dirty_cat/tests/test_similarity_encoder.py
+++ b/dirty_cat/tests/test_similarity_encoder.py
@@ -6,7 +6,7 @@ import pytest
 from sklearn import __version__ as sklearn_version
 
 from dirty_cat import SimilarityEncoder
-from dirty_cat._similarity_encoder import get_kmeans_prototypes
+from dirty_cat._similarity_encoder import get_kmeans_prototypes, ngram_similarity_matrix
 from dirty_cat._string_distances import ngram_similarity
 from dirty_cat._utils import parse_version
 
@@ -49,6 +49,25 @@ def test_fast_ngram_similarity() -> None:
     assert np.allclose(feature_matrix, feature_matrix_fast)
 
 
+def test_parameters():
+    X = [["foo"], ["baz"]]
+    X2 = [["foo"], ["bar"]]
+    with pytest.raises(ValueError, match=r"Got handle_unknown="):
+        SimilarityEncoder(handle_unknown="bb").fit(X)
+    with pytest.raises(ValueError, match=r"Got hashing_dim="):
+        SimilarityEncoder(hashing_dim="bb").fit(X)
+    with pytest.raises(ValueError, match=r"Got categories="):
+        SimilarityEncoder(categories="bb")
+    with pytest.raises(ValueError, match=r"Unsorted categories "):
+        SimilarityEncoder(categories=[["cat2", "cat1"], ["cat3", "cat4"]]).fit(X)
+    with pytest.raises(ValueError, match=r"Found unknown categories "):
+        SimilarityEncoder(categories=[["fooo", "loo"]], handle_unknown="error").fit(X)
+    with pytest.raises(ValueError, match=r"Found unknown categories "):
+        sim = SimilarityEncoder(categories=[["baz", "foo"]], handle_unknown="error")
+        sim.fit(X)
+        sim.transform(X2)
+
+
 def _test_missing_values(input_type, missing):
     observations = [["a", "b"], ["b", "a"], ["b", np.nan], ["a", "c"], [np.nan, "a"]]
     encoded = np.array(
@@ -146,6 +165,12 @@ def _test_similarity(
         ).reshape(-1, 1)
         X_test = np.array([["Aa", "aAa", "aaa", "aaab", " aaa  c"]]).reshape(-1, 1)
 
+        if categories == "auto":
+            with pytest.warns(UserWarning, match=r"n_prototypes parameter ignored"):
+                SimilarityEncoder(
+                    categories=categories,
+                    n_prototypes=n_prototypes,
+                )
         try:
             model = SimilarityEncoder(
                 hashing_dim=hashing_dim,
@@ -157,7 +182,7 @@ def _test_similarity(
         except ValueError as e:
             assert (
                 e.__str__()
-                == "n_prototypes expected None or a positive non null integer"
+                == "n_prototypes expected None or a positive non null integer. "
             )
             return
 
@@ -193,8 +218,9 @@ def test_similarity_encoder() -> None:
                 hashing_dim=2**16,
                 categories=category,
             )
+            _test_similarity(ngram_similarity, categories=category, n_prototypes=4)
         else:
-            for i in range(1, 4):
+            for i in range(0, 4):
                 _test_similarity(
                     ngram_similarity,
                     categories=category,
@@ -217,8 +243,18 @@ def test_similarity_encoder() -> None:
 
 def test_kmeans_protoypes() -> None:
     X_test = np.array(["cbbba", "baaac", "accc"])
-    proto = get_kmeans_prototypes(X_test, 3)
+    proto = get_kmeans_prototypes(X_test, 3, sparse=True)
     assert np.array_equal(np.sort(proto), np.sort(X_test))
+    X_test_2 = np.array(["aa", "bb", "cc", "bbb"])
+    with pytest.warns(UserWarning, match=r"number of unique prototypes is lower "):
+        get_kmeans_prototypes(X_test_2, 4)
+
+
+def test_ngram_similarity_matrix() -> None:
+    X1 = np.array(["cat1", "cat2", "cat3"])
+    X2 = np.array(["cata1", "caat2", "ccat3"])
+    sim = ngram_similarity_matrix(X1, X2, ngram_range=(2, 2), hashing_dim=5)
+    assert sim.shape == (len(X1), len(X2))
 
 
 def test_reproducibility() -> None:
@@ -233,6 +269,16 @@ def test_reproducibility() -> None:
         assert np.array_equal(prototypes, sim_enc.fit(X).categories_[0])
 
 
+def test_fit_transform():
+    X = [["foo"], ["baz"]]
+    y = ["foo", "bar"]
+    tr1 = SimilarityEncoder().fit_transform(X, y)
+    sim = SimilarityEncoder()
+    sim.fit(X, y)
+    tr2 = sim.transform(X, y)
+    assert np.array_equal(tr1, tr2)
+
+
 def test_get_features() -> None:
     # See https://github.com/dirty-cat/dirty_cat/issues/168
     sim_enc = SimilarityEncoder(random_state=435)