Repository: dirty-cat/dirty_cat
Pull request #465 (2023-01-17 21:09:31)

ENH: more didactic deduplicate example
- Add the functions to the generated docs
- Incorporate the random number generation seed accordingly to scikit-learn's standard
- A bit of phrasing

GaelVaroquaux: I applied the suggested change and resolved conflicts. Thanks!

Patch:
diff --git a/dirty_cat/datasets/__init__.py b/dirty_cat/datasets/__init__.py
index 92f3ace..0dfc30f 100644
--- a/dirty_cat/datasets/__init__.py
+++ b/dirty_cat/datasets/__init__.py
@@ -10,9 +10,14 @@ from ._fetching import (
     fetch_traffic_violations,
     fetch_world_bank_indicator,
     get_data_dir,
+)
+
+
+from ._generating import (
     make_deduplication_data,
 )
 
+
 __all__ = [
     "DatasetAll",
     "DatasetInfoOnly",
diff --git a/dirty_cat/datasets/_fetching.py b/dirty_cat/datasets/_fetching.py
index 41c61ca..b0eeb77 100644
--- a/dirty_cat/datasets/_fetching.py
+++ b/dirty_cat/datasets/_fetching.py
@@ -820,47 +820,3 @@ def fetch_world_bank_indicator(
     )
 
 
-def make_deduplication_data(
-    examples: List[str], entries_per_example: List[int], prob_mistake_per_letter: float
-) -> List[str]:
-    """Duplicates examples with spelling mistakes.
-    Characters are misspelled with probability `prob_mistake_per_letter`.
-
-    Parameters
-    ----------
-    examples : List[str]
-        examples to duplicate
-    entries_per_example : List[int]
-        number of duplications per example
-    prob_mistake_per_letter : float in [0, 1]
-        probability of misspelling a character in duplications
-
-    Returns
-    -------
-    List[str]
-        list of duplicated examples with spelling mistakes
-    """
-
-    import string
-
-    import numpy as np
-
-    data = []
-    for example, n_ex in zip(examples, entries_per_example):
-        len_ex = len(example)
-        # generate a 2D array of chars of size (n_ex, len_ex)
-        str_as_list = np.array([list(example)] * n_ex)
-        # randomly choose which characters are misspelled
-        idxes = np.where(
-            np.random.random(len(example[0]) * n_ex) < prob_mistake_per_letter
-        )[0]
-        # and randomly pick with which character to replace
-        replacements = [
-            string.ascii_lowercase[i]
-            for i in np.random.choice(np.arange(26), len(idxes)).astype(int)
-        ]
-        # introduce spelling mistakes at right examples and char locations per example
-        str_as_list[idxes // len_ex, idxes % len_ex] = replacements
-        # go back to 1d array of strings
-        data.append(np.ascontiguousarray(str_as_list).view(f"U{len_ex}").ravel())
-    return np.concatenate(data).tolist()
diff --git a/dirty_cat/datasets/_generating.py b/dirty_cat/datasets/_generating.py
new file mode 100644
index 0000000..d379a33
--- /dev/null
+++ b/dirty_cat/datasets/_generating.py
@@ -0,0 +1,60 @@
+"""
+Functions that generate example data.
+
+"""
+from __future__ import annotations
+
+from typing import List, Optional, Union
+import string
+
+import numpy as np
+
+from sklearn.utils import check_random_state
+
+def make_deduplication_data(
+    examples: List[str],
+    entries_per_example: List[int],
+    prob_mistake_per_letter: float,
+    random_state: Optional[Union[int, np.random.RandomState]] = None,
+) -> List[str]:
+    """Duplicates examples with spelling mistakes.
+    Characters are misspelled with probability `prob_mistake_per_letter`.
+
+    Parameters
+    ----------
+    examples : List[str]
+        examples to duplicate
+    entries_per_example : List[int]
+        number of duplications per example
+    prob_mistake_per_letter : float in [0, 1]
+        probability of misspelling a character in duplications
+    random_state : int, RandomState instance or None, default=None
+        Determines random number generation for dataset noise. Pass an int
+        for reproducible output across multiple function calls.
+
+    Returns
+    -------
+    List[str]
+        list of duplicated examples with spelling mistakes
+    """
+    rng = check_random_state(random_state)
+
+    data = []
+    for example, n_ex in zip(examples, entries_per_example):
+        len_ex = len(example)
+        # generate a 2D array of chars of size (n_ex, len_ex)
+        str_as_list = np.array([list(example)] * n_ex)
+        # randomly choose which characters are misspelled
+        idxes = np.where(
+            rng.random(len(example[0]) * n_ex) < prob_mistake_per_letter
+        )[0]
+        # and randomly pick with which character to replace
+        replacements = [
+            string.ascii_lowercase[i]
+            for i in np.random.choice(np.arange(26), len(idxes)).astype(int)
+        ]
+        # introduce spelling mistakes at right examples and char locations per example
+        str_as_list[idxes // len_ex, idxes % len_ex] = replacements
+        # go back to 1d array of strings
+        data.append(np.ascontiguousarray(str_as_list).view(f"U{len_ex}").ravel())
+    return np.concatenate(data).tolist()
diff --git a/doc/index.rst b/doc/index.rst
index 25289f5..de61a71 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -142,10 +142,27 @@ Joining tables
    :nosignatures:
 
    fuzzy_join
+
+.. autosummary::
+   :toctree: generated/
+   :template: class.rst
+   :nosignatures:
+
    FeatureAugmenter
 
-Data download
--------------
+Deduplication: merging variants of the same entry
+---------------------------------------------------
+
+.. autosummary::
+   :toctree: generated/
+   :template: function.rst
+   :nosignatures:
+
+   deduplicate
+
+
+Data download and generation
+----------------------------
 
 .. autosummary::
    :toctree: generated/
@@ -161,6 +178,7 @@ Data download
    datasets.fetch_drug_directory
    datasets.fetch_world_bank_indicator
    datasets.get_data_dir
+   datasets.make_deduplication_data
 
 About
 =====
diff --git a/examples/05_fuzzy_joining_and_FeatureAugmenter.py b/examples/05_fuzzy_joining_and_FeatureAugmenter.py
old mode 100755
new mode 100644
diff --git a/examples/06_deduplication.py b/examples/06_deduplication.py
index 95b8677..0c3fcf8 100644
--- a/examples/06_deduplication.py
+++ b/examples/06_deduplication.py
@@ -2,25 +2,26 @@
 Deduplicating misspelled categories with deduplicate
 ====================================================
 
-A common step in data analyses is grouping or analyzing data conditional on a
-categorical variable. In real world datasets, there often will be slight
-misspellings in the category names: This happens when, for example, data input
-*should* use a drop down menu, but users are forced to input the category name
-by hand. Misspellings happen and analyzing the resulting data using a simple
-`GROUP BY` is not possible anymore.
-
-This problem is however the perfect use case of *unsupervised learning*, a
-category of various statical methods that find structure in data without
-providing explicit labels/categories of the data a-priori. Specifically
-clustering of the distance between strings can be used to find clusters
-of strings that are similar to each other (e.g. differ only by a misspelling)
-and hence gives us an easy tool to flag potentially misspelled category names
-in an unsupervised manner.
+Real world datasets often come with slight misspellings in the category
+names, for instance if the category is manually input. Such misspellings
+break many data-analyses steps that require exact matching, such as a
+'GROUP BY'.
+
+Merging the multiple variants of the same category or entity is known as
+*deduplication*. It is performed by the :func:`~dirty_cat.deduplicate`
+function.
+
+Deduplication relies on *unsupervised learning*, to find structure in
+data without providing explicit labels/categories of the data a-priori.
+Specifically clustering of the distance between strings can be used to
+find clusters of strings that are similar to each other (e.g. differ only
+by a misspelling) and hence gives us an easy tool to flag potentially
+misspelled category names in an unsupervised manner.
 """
 
 ###############################################################################
-# An example
-# ----------
+# An example dataset
+# -------------------
 #
 # Imagine the following example:
 # As a data scientist, our job is to analyze the data from a hospital ward.
@@ -34,13 +35,8 @@ in an unsupervised manner.
 
 
 import numpy as np
-import pandas as pd
-import matplotlib.pyplot as plt
 from dirty_cat.datasets import make_deduplication_data
 
-# set seed for reproducibility
-np.random.seed(123)
-
 # our three medication names
 medications = ["Contrivan", "Genericon", "Zipholan"]
 entries_per_medications = [500, 100, 1500]
@@ -48,18 +44,25 @@ entries_per_medications = [500, 100, 1500]
 # 5% probability of a typo per letter
 prob_mistake_per_letter = 0.05
 
-data = make_deduplication_data(
-    medications, entries_per_medications, prob_mistake_per_letter
+duplicated_names = make_deduplication_data(
+    medications, entries_per_medications, prob_mistake_per_letter,
+    random_state=42, # set seed for reproducibility
 )
 # we extract the unique medication names in the data & how often they appear
-unique_examples, counts = np.unique(data, return_counts=True)
+unique_examples, counts = np.unique(duplicated_names, return_counts=True)
 # and build a series out of them
+import pandas as pd
 ex_series = pd.Series(counts, index=unique_examples)
 
+# This is our data:
+ex_series.head()
+
 ###############################################################################
 # Visualize the data
 # ------------------
 
+import matplotlib.pyplot as plt
+
 ex_series.plot.barh(figsize=(10, 15))
 plt.xlabel("Medication name")
 plt.ylabel("Counts")
@@ -73,14 +76,6 @@ plt.ylabel("Counts")
 # name will be closest to either the correctly or incorrectly spelled orginal
 # medication name - and therefore form clusters.
 
-
-from dirty_cat import deduplicate
-from dirty_cat._deduplicate import compute_ngram_distance
-from scipy.spatial.distance import squareform
-
-ngram_distances = compute_ngram_distance(unique_examples)
-square_distances = squareform(ngram_distances)
-
 ###############################################################################
 # We can visualize the pair-wise distance between all medication names
 # --------------------------------------------------------------------
@@ -90,6 +85,12 @@ square_distances = squareform(ngram_distances)
 # a lighter color means a larger distance. We can see that we are dealing with three
 # clusters - the original medication names and their misspellings that cluster around them.
 
+from dirty_cat import compute_ngram_distance
+from scipy.spatial.distance import squareform
+
+ngram_distances = compute_ngram_distance(unique_examples)
+square_distances = squareform(ngram_distances)
+
 import seaborn as sns
 
 fig, axes = plt.subplots(1, 1, figsize=(12, 12))
@@ -98,15 +99,19 @@ sns.heatmap(
 )
 
 ###############################################################################
-# Clustering to suggest corrections of misspelled names
+# Deduplication: suggest corrections of misspelled names
 # -----------------------------------------------------
 #
+# The :func:`~dirty_cat.deduplicate` function uses clustering based on
+# string similarities to group duplicated names
+#
 # The number of clusters will need some adjustment depending on the data you have.
 # If no fixed number of clusters is given, `deduplicate` tries to set it automatically
 # via the `silhouette score <https://scikit-learn.org/stable/modules/clustering.html#silhouette-coefficient>`_.
 
+from dirty_cat import deduplicate
 
-deduplicated_data = deduplicate(data)
+deduplicated_data = deduplicate(duplicated_names)
 
 ###############################################################################
 # We can visualize the distribution of categories in the deduplicated data:
@@ -131,7 +136,7 @@ plt.ylabel("Counts")
 # (potentially) correct categories as values.
 
 # create a table that maps original -> corrected categories
-translation_table = pd.Series(deduplicated_data, index=data)
+translation_table = pd.Series(deduplicated_data, index=duplicated_names)
 
 # remove duplicates in the original data
 translation_table = translation_table[~translation_table.index.duplicated(keep="first")]
@@ -154,8 +159,10 @@ def print_corrections(spell_correct):
 print_corrections(translation_table)
 
 ###############################################################################
-# In case we want to adapt the translation table post-hoc we can easily do so:
+# In case we want to adapt the translation table post-hoc we can easily
+# modified it manually and apply it, for instance modifying the
+# correspondance for the last entry as such:
 
-translation_table["Gszericon"] = "Completely new category"
-new_deduplicated_data = translation_table[data]
-assert (new_deduplicated_data == "Completely new category").sum() > 0
+translation_table.iloc[-1] = "Completely new category"
+new_deduplicated_names = translation_table[duplicated_names]
+assert (new_deduplicated_names == "Completely new category").sum() > 0


Test patch:
diff --git a/dirty_cat/datasets/tests/test_fetching.py b/dirty_cat/datasets/tests/test_fetching.py
index 721c29c..561dcaa 100644
--- a/dirty_cat/datasets/tests/test_fetching.py
+++ b/dirty_cat/datasets/tests/test_fetching.py
@@ -29,7 +29,6 @@ from dirty_cat.datasets._fetching import (
 from dirty_cat.datasets._fetching import (
     fetch_world_bank_indicator as fetch_world_bank_indicator,
 )
-from dirty_cat.datasets._fetching import make_deduplication_data
 from dirty_cat.datasets._utils import get_data_dir as _get_data_dir
 
 
@@ -456,23 +455,3 @@ def test_fetch_world_bank_indicator():
         shutil.rmtree(path=str(test_data_dir), ignore_errors=True)
 
 
-def test_make_deduplication_data():
-    import numpy as np
-
-    np.random.seed(123)
-    assert make_deduplication_data(["abc", "cba", "test1"], [3, 2, 1], 0.3) == [
-        "agr",
-        "abc",
-        "abc",
-        "cba",
-        "cba",
-        "test1",
-    ]
-    assert make_deduplication_data(["abc", "cba", "test1"], [1, 2, 3], 0.8) == [
-        "pbc",
-        "pza",
-        "cba",
-        "erxt1",
-        "test1",
-        "test1",
-    ]
diff --git a/dirty_cat/datasets/tests/test_generating.py b/dirty_cat/datasets/tests/test_generating.py
new file mode 100644
index 0000000..d1d399f
--- /dev/null
+++ b/dirty_cat/datasets/tests/test_generating.py
@@ -0,0 +1,29 @@
+"""
+Tests generating.py (synthetic dataset generation).
+"""
+
+import numpy as np
+
+from dirty_cat.datasets._generating import make_deduplication_data
+
+
+
+def test_make_deduplication_data():
+
+    np.random.seed(123)
+    assert make_deduplication_data(["abc", "cba", "test1"], [3, 2, 1], 0.3) == [
+        "agr",
+        "abc",
+        "abc",
+        "cba",
+        "cba",
+        "test1",
+    ]
+    assert make_deduplication_data(["abc", "cba", "test1"], [1, 2, 3], 0.8) == [
+        "pbc",
+        "pza",
+        "cba",
+        "erxt1",
+        "test1",
+        "test1",
+    ]