Repository: dirty-cat/dirty_cat
Pull request #438 (2022-12-16 16:18:13)

DOC Ensures that `GapEncoder` passes numpydoc validation
Part of #345.

LilianBoulard: There's one thing I'm missing on the checklist of the numpydoc guidelines, and that is a one-liner to describe the GapEncoder.

If you have any idea, please let me know :)
jovan-stojanovic: > There's one thing I'm missing on the checklist of the numpydoc guidelines, and that is a one-liner to describe the GapEncoder.
> 
> If you have any idea, please let me know :)

"This encoder use the data to construct latent topics with continuous encoding." Would this fit?

Patch:
diff --git a/CHANGES.rst b/CHANGES.rst
index 3c1af98..3a817e6 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -10,7 +10,7 @@ Major changes
   on string similarities and the nearest neighbors matches are found for each category. :pr:`291` by :user:`Jovan Stojanovic <jovan-stojanovic>` and :user:`Leo Grinsztajn <LeoGrin>`
 * **datasets.fetching**: contains a new function :func:`fetch_world_bank_indicator` that can be used to download any indicator from the World Bank Open Data platform. It only needs the indicator ID that can be found on the website. :pr:`291` by :user:`Jovan Stojanovic <jovan-stojanovic>`
 * Unnecessary API has been made private: everything (files, functions, classes) starting with an underscore shouldn't be imported in your code. :pr:`331` by :user:`Lilian Boulard <LilianBoulard>`
-* The MinHashEncoder now supports a `n_jobs` parameter to parallelize the hashes computation. :pr:`267` by `Leo Grinsztajn <LeoGrin>` and `Lilian Boulard <LilianBoulard>`.
+* The MinHashEncoder now supports a `n_jobs` parameter to parallelize the hashes computation. :pr:`267` by :user:`Leo Grinsztajn <LeoGrin>` and :user:`Lilian Boulard <LilianBoulard>`.
 
 Minor changes
 -------------
diff --git a/dirty_cat/_datetime_encoder.py b/dirty_cat/_datetime_encoder.py
index 29e2ff5..afa6186 100644
--- a/dirty_cat/_datetime_encoder.py
+++ b/dirty_cat/_datetime_encoder.py
@@ -70,6 +70,26 @@ class DatetimeEncoder(BaseEstimator, TransformerMixin):
     col_names_: typing.Optional[typing.List[str]]
         List of the names of the features of the input data,
         if input data was a pandas DataFrame, otherwise None.
+
+    Examples
+    --------
+    >>> enc = DatetimeEncoder()
+
+    Let's encode the following dates:
+
+    >>> X = [['2022-10-15'], ['2021-12-25'], ['2020-05-18'], ['2019-10-15 12:00:00']]
+
+    >>> enc.fit(X)
+    DatetimeEncoder()
+
+    The encoder will output a transformed array
+    with four columns (year, month, day and hour):
+
+    >>> enc.transform(X)
+    array([[2022.,   10.,   15.,    0.],
+           [2021.,   12.,   25.,    0.],
+           [2020.,    5.,   18.,    0.],
+           [2019.,   10.,   15.,   12.]])
     """
 
     n_features_in_: int
diff --git a/dirty_cat/_gap_encoder.py b/dirty_cat/_gap_encoder.py
index 61302f2..0b595b7 100644
--- a/dirty_cat/_gap_encoder.py
+++ b/dirty_cat/_gap_encoder.py
@@ -40,6 +40,9 @@ else:
 
 from sklearn.decomposition._nmf import _beta_divergence
 
+# Ignore lines too long, as some things in the docstring cannot be cut.
+# flake8: noqa: E501
+
 
 class GapEncoderColumn(BaseEstimator, TransformerMixin):
 
@@ -539,74 +542,92 @@ class GapEncoderColumn(BaseEstimator, TransformerMixin):
 
 
 class GapEncoder(BaseEstimator, TransformerMixin):
-    """
+    """Constructs latent topics with continuous encoding.
+
     This encoder can be understood as a continuous encoding on a set of latent
     categories estimated from the data. The latent categories are built by
     capturing combinations of substrings that frequently co-occur.
 
-    The GapEncoder supports online learning on batches of data for
-    scalability through the partial_fit method.
+    The :class:`~dirty_cat.GapEncoder` supports online learning on batches of
+    data for scalability through the :func:`~dirty_cat.GapEncoder.partial_fit`
+    method.
 
     Parameters
     ----------
-    n_components : int, default=10
+    n_components : int, optional, default=10
         Number of latent categories used to model string data.
-    batch_size : int, default=128
+    batch_size : int, optional, default=128
         Number of samples per batch.
-    gamma_shape_prior : float, default=1.1
+    gamma_shape_prior : float, optional, default=1.1
         Shape parameter for the Gamma prior distribution.
-    gamma_scale_prior : float, default=1.0
+    gamma_scale_prior : float, optional, default=1.0
         Scale parameter for the Gamma prior distribution.
-    rho : float, default=0.95
-        Weight parameter for the update of the W matrix.
-    rescale_rho : bool, default=False
-        If true, use rho ** (batch_size / len(X)) instead of rho to obtain an
+    rho : float, optional, default=0.95
+        Weight parameter for the update of the *W* matrix.
+    rescale_rho : bool, optional, default=False
+        If true, use ``rho ** (batch_size / len(X))`` instead of rho to obtain an
         update rate per iteration that is independent of the batch size.
-    hashing : bool, default=False
-        If true, HashingVectorizer is used instead of CountVectorizer.
-        It has the advantage of being very low memory scalable to large
+    hashing : bool, optional, default=False
+        If true, :class:`~sklearn.feature_extraction.text.HashingVectorizer`
+        is used instead of :class:`~sklearn.feature_extraction.text.CountVectorizer`.
+        It has the advantage of being very low memory, scalable to large
         datasets as there is no need to store a vocabulary dictionary in
         memory.
-    hashing_n_features : int, default=2**12
-        Number of features for the HashingVectorizer. Only relevant if
-        hashing=True.
-    init : typing.Literal["k-means++", "random", "k-means"], default='k-means++'
+    hashing_n_features : int, optional, default=2**12
+        Number of features for the :class:`~sklearn.feature_extraction.text.HashingVectorizer`.
+        Only relevant if `hashing=True`.
+    init : {"k-means++", "random", "k-means"}, optional, default='k-means++'
         Initialization method of the W matrix.
-        Options: {'k-means++', 'random', 'k-means'}.
-        If init='k-means++', we use the init method of sklearn.cluster.KMeans.
-        If init='random', topics are initialized with a Gamma distribution.
-        If init='k-means', topics are initialized with a KMeans on the n-grams
+        If `init='k-means++'`, we use the init method of :class:`~sklearn.cluster.KMeans`.
+        If `init='random'`, topics are initialized with a Gamma distribution.
+        If `init='k-means'`, topics are initialized with a KMeans on the n-grams
         counts. This usually makes convergence faster but is a bit slower.
     tol : float, default=1e-4
-        Tolerance for the convergence of the matrix W.
-    min_iter : int, default=2
+        Tolerance for the convergence of the matrix *W*.
+    min_iter : int, optional, default=2
         Minimum number of iterations on the input data.
-    max_iter : int, default=5
+    max_iter : int, optional, default=5
         Maximum number of iterations on the input data.
-    ngram_range : typing.Tuple[int, int], default=(2, 4)
+    ngram_range : int 2-tuple, optional, default=(2, 4)
         The range of ngram length that will be used to build the
         bag-of-n-grams representation of the input data.
-    analyzer : typing.Literal["word", "char", "char_wb"], default='char'.
-        Analyzer parameter for the CountVectorizer/HashingVectorizer.
-        Options: {‘word’, ‘char’, ‘char_wb’}, describing whether the matrix V
-        to factorize should be made of word counts or character n-gram counts.
+    analyzer : {"word", "char", "char_wb"}, optional, default='char'
+        Analyzer parameter for the :class:`~sklearn.feature_extraction.text.HashingVectorizer`
+        / :class:`~sklearn.feature_extraction.text.CountVectorizer`.
+        Describes whether the matrix *V* to factorize should be made of word counts
+        or character n-gram counts.
         Option ‘char_wb’ creates character n-grams only from text inside word
         boundaries; n-grams at the edges of words are padded with space.
-    add_words : bool, default=False
+    add_words : bool, optional, default=False
         If true, add the words counts to the bag-of-n-grams representation
         of the input data.
-    random_state : typing.Optional[Union[int, RandomState]], default=None
-        Pass an int for reproducible output across multiple function calls.
-    rescale_W : bool, default=True
-        If true, the weight matrix W is rescaled at each iteration
+    random_state : int, :class:`numpy.random.RandomState` or None, optional, default=None
+        RNG seed for reproducible output across multiple function calls.
+    rescale_W : bool, optional, default=True
+        If true, the weight matrix *W* is rescaled at each iteration
         to have a l1 norm equal to 1 for each row.
     max_iter_e_step : int, default=20
         Maximum number of iterations to adjust the activations h at each step.
-    handle_missing : typing.Literal["error", "empty_impute"], default=empty_impute
-        Whether to raise an error or impute with empty string '' if missing
+    handle_missing : {"error", "empty_impute"}, optional, default='empty_impute'
+        Whether to raise an error or impute with empty string ``''`` if missing
         values (NaN) are present during fit (default is to impute).
         In the inverse transform, the missing category will be denoted as None.
 
+    Attributes
+    ----------
+    rho_: float
+        Effective update rate for the W matrix
+    fitted_models_: list of GapEncoderColumn
+        Column-wise fitted GapEncoders
+    column_names_: list of str
+        Column names of the data the Gap was fitted on
+
+    References
+    ----------
+    For a detailed description of the method, see
+    `Encoding high-cardinality string categorical variables
+    <https://hal.inria.fr/hal-02171256v4>`_ by Cerda, Varoquaux (2019).
+
     Examples
     --------
     >>> enc = GapEncoder(n_components=2)
@@ -624,8 +645,8 @@ class GapEncoder(BaseEstimator, TransformerMixin):
     >>> enc.get_feature_names()
     ['england, london, uk', 'france, paris, pqris']
 
-    He got it right, reccuring topics are "London" and "England" on the
-    one side and and "Paris" and "France" on the other.
+    It got it right, reccuring topics are "London" and "England" on the
+    one side and "Paris" and "France" on the other.
 
     As this is a continuous encoding, we can look at the level of
     activation of each topic for each category:
@@ -641,19 +662,6 @@ class GapEncoder(BaseEstimator, TransformerMixin):
           [ 0.05002016,  4.54997983]])
 
     The higher the value, the bigger the correspondance with the topic.
-
-    Attributes
-    ----------
-    rho_: float
-    fitted_models_: typing.List[GapEncoderColumn]
-    column_names_: typing.List[str]
-
-    References
-    ----------
-    For a detailed description of the method, see
-    `Encoding high-cardinality string categorical variables
-    <https://hal.inria.fr/hal-02171256v4>`_ by Cerda, Varoquaux (2019).
-
     """
 
     rho_: float
diff --git a/dirty_cat/_similarity_encoder.py b/dirty_cat/_similarity_encoder.py
index bbc3927..93e91cc 100644
--- a/dirty_cat/_similarity_encoder.py
+++ b/dirty_cat/_similarity_encoder.py
@@ -80,7 +80,7 @@ def _ngram_similarity_one_sample_inplace(
     se_dict[unq_X[i]] = similarity.reshape(-1)
 
 
-def ngram_similarity(
+def ngram_similarity_matrix(
     X,
     cats: List[str],
     ngram_range: Tuple[int, int],
@@ -560,7 +560,7 @@ class SimilarityEncoder(OneHotEncoder):
             if fast:
                 encoded_Xj = self._ngram_similarity_fast(Xlist[j], j)
             else:
-                encoded_Xj = ngram_similarity(
+                encoded_Xj = ngram_similarity_matrix(
                     Xlist[j],
                     categories,
                     ngram_range=(min_n, max_n),
diff --git a/dirty_cat/datasets/_fetching.py b/dirty_cat/datasets/_fetching.py
index f373f54..ace1134 100644
--- a/dirty_cat/datasets/_fetching.py
+++ b/dirty_cat/datasets/_fetching.py
@@ -237,9 +237,8 @@ def _fetch_world_bank_data(
               saved as a CSV file.
 
     """
-    path = f"{data_directory}/{indicator_id}.csv"
-    Path(data_directory).mkdir(parents=True, exist_ok=True)
-    csv_path = Path(path).resolve()
+    csv_path = (data_directory / f"{indicator_id}.csv").resolve()
+    data_directory.mkdir(parents=True, exist_ok=True)
     url = f"https://api.worldbank.org/v2/en/indicator/{indicator_id}?downloadformat=csv"  # noqa
     if csv_path.is_file():
         df = pd.read_csv(csv_path, nrows=0)
diff --git a/doc/conf.py b/doc/conf.py
index ab3a59b..b30a284 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -25,6 +25,7 @@ import sys
 # is relative to the documentation root, use os.path.abspath to make it
 # absolute, like shown here.
 sys.path.insert(0, os.path.abspath("sphinxext"))
+from github_link import make_linkcode_resolve
 
 
 # -- Copy files for docs --------------------------------------------------
@@ -50,10 +51,10 @@ extensions = [
     "sphinx.ext.doctest",
     "sphinx.ext.intersphinx",
     "sphinx.ext.mathjax",
-    "sphinx.ext.viewcode",
     "sphinx.ext.githubpages",
     "numpydoc",
     "sphinx_issues",
+    "sphinx.ext.linkcode",
     "sphinx.ext.autodoc.typehints",
     "sphinx_gallery.gen_gallery",
 ]
@@ -307,3 +308,11 @@ html_js_files = ["scrolltoc.js"]
 
 # we use the issues path for PRs since the issues URL will forward
 issues_github_path = "dirty-cat/dirty_cat"
+
+# The following is used by sphinx.ext.linkcode to provide links to github
+linkcode_resolve = make_linkcode_resolve(
+    "dirty_cat",
+    "https://github.com/dirty-cat/"
+    "dirty-cat/blob/{revision}/"
+    "{package}/{path}#L{lineno}",
+)
diff --git a/doc/sphinxext/github_link.py b/doc/sphinxext/github_link.py
new file mode 100644
index 0000000..3ad3743
--- /dev/null
+++ b/doc/sphinxext/github_link.py
@@ -0,0 +1,83 @@
+"""
+Taken from https://github.com/scikit-learn/scikit-learn/blob/3d4ee9e9dc384ea11724e270b51bd26a1f227f0c/doc/sphinxext/github_link.py
+"""
+
+from operator import attrgetter
+import inspect
+import subprocess
+import os
+import sys
+from functools import partial
+
+REVISION_CMD = "git rev-parse --short HEAD"
+
+
+def _get_git_revision():
+    try:
+        revision = subprocess.check_output(REVISION_CMD.split()).strip()
+    except (subprocess.CalledProcessError, OSError):
+        print("Failed to execute git to get revision")
+        return None
+    return revision.decode("utf-8")
+
+
+def _linkcode_resolve(domain, info, package, url_fmt, revision):
+    """Determine a link to online source for a class/method/function
+    This is called by sphinx.ext.linkcode
+    An example with a long-untouched module that everyone has
+    >>> _linkcode_resolve('py', {'module': 'tty',
+    ...                          'fullname': 'setraw'},
+    ...                   package='tty',
+    ...                   url_fmt='http://hg.python.org/cpython/file/'
+    ...                           '{revision}/Lib/{package}/{path}#L{lineno}',
+    ...                   revision='xxxx')
+    'http://hg.python.org/cpython/file/xxxx/Lib/tty/tty.py#L18'
+    """
+
+    if revision is None:
+        return
+    if domain not in ("py", "pyx"):
+        return
+    if not info.get("module") or not info.get("fullname"):
+        return
+
+    class_name = info["fullname"].split(".")[0]
+    module = __import__(info["module"], fromlist=[class_name])
+    obj = attrgetter(info["fullname"])(module)
+
+    # Unwrap the object to get the correct source
+    # file in case that is wrapped by a decorator
+    obj = inspect.unwrap(obj)
+
+    try:
+        fn = inspect.getsourcefile(obj)
+    except Exception:
+        fn = None
+    if not fn:
+        try:
+            fn = inspect.getsourcefile(sys.modules[obj.__module__])
+        except Exception:
+            fn = None
+    if not fn:
+        return
+
+    fn = os.path.relpath(fn, start=os.path.dirname(__import__(package).__file__))
+    try:
+        lineno = inspect.getsourcelines(obj)[1]
+    except Exception:
+        lineno = ""
+    return url_fmt.format(revision=revision, package=package, path=fn, lineno=lineno)
+
+
+def make_linkcode_resolve(package, url_fmt):
+    """Returns a linkcode_resolve function for the given URL format
+    revision is a git commit reference (hash or name)
+    package is the name of the root module of the package
+    url_fmt is along the lines of ('https://github.com/USER/PROJECT/'
+                                   'blob/{revision}/{package}/'
+                                   '{path}#L{lineno}')
+    """
+    revision = _get_git_revision()
+    return partial(
+        _linkcode_resolve, revision=revision, package=package, url_fmt=url_fmt
+    )


Test patch:
diff --git a/dirty_cat/tests/test_docstrings.py b/dirty_cat/tests/test_docstrings.py
index 5c4b2a6..0a683aa 100644
--- a/dirty_cat/tests/test_docstrings.py
+++ b/dirty_cat/tests/test_docstrings.py
@@ -18,7 +18,6 @@ from numpydoc.validate import validate
 DOCSTRING_TEMP_IGNORE_SET = {
     "dirty_cat._datetime_encoder.DatetimeEncoder.get_feature_names",
     "dirty_cat._datetime_encoder.DatetimeEncoder.get_feature_names_out",
-    "dirty_cat._gap_encoder.GapEncoder",
     "dirty_cat._gap_encoder.GapEncoder.fit",
     "dirty_cat._gap_encoder.GapEncoder.get_feature_names",
     "dirty_cat._gap_encoder.GapEncoder.get_feature_names_out",