Repository: dirty-cat/dirty_cat
Pull request #456 (2023-01-09 10:58:12)

Remove pandas `FutureWarning` in `SuperVectorizer`
Solves #454 


Patch:
diff --git a/.github/ISSUE_TEMPLATE/documentation.md b/.github/ISSUE_TEMPLATE/documentation.md
index ed69f9c..d24e63e 100644
--- a/.github/ISSUE_TEMPLATE/documentation.md
+++ b/.github/ISSUE_TEMPLATE/documentation.md
@@ -10,4 +10,4 @@ assignees: ''
 <!--List pages that would be impacted by the enhancement-->
 
 Affected pages:
-- 
+-
diff --git a/.gitignore b/.gitignore
index dc2445f..746bffc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -64,4 +64,3 @@ doc/CONTRIBUTING.rst
 
 # Pkl files for benchmarks
 benchmarks/*.pkl
-
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 71fc470..58dc8c3 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -3,12 +3,16 @@ repos:
     rev: v4.3.0
     hooks:
     -   id: check-yaml
+        exclude: doc/
     -   id: end-of-file-fixer
+        exclude: doc/
     -   id: trailing-whitespace
+        exclude: doc/
 -   repo: https://github.com/psf/black
     rev: 22.8.0
     hooks:
     -   id: black
+        exclude: doc/
 -   repo: https://github.com/pycqa/flake8
     rev: 5.0.4
     hooks:
@@ -16,7 +20,7 @@ repos:
         files: dirty_cat/
         types: [file, python]
 -   repo: https://github.com/PyCQA/isort
-    rev: 5.10.1
+    rev: 5.12.0
     hooks:
     -   id: isort
         files: dirty_cat/
diff --git a/CHANGES.rst b/CHANGES.rst
index 8300c48..07665d3 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -2,22 +2,31 @@
 
 .. currentmodule:: dirty_cat
 
-Release 0.4.0 (upcoming)
-=========================
+Release 0.5.0
+==============
+
+Release 0.4.0 (beta)
+====================
 
 Major changes
 -------------
 
 * New experimental feature: joining tables using :func:`fuzzy_join` by approximate key matching. Matches are based
   on string similarities and the nearest neighbors matches are found for each category.
-:pr:`291` by :user:`Jovan Stojanovic <jovan-stojanovic>` and :user:`Leo Grinsztajn <LeoGrin>`
-* **datasets.fetching**: contains a new function :func:`fetch_world_bank_indicator`
-  that can be used to download any indicator from the World Bank Open Data platform.
-  It only needs the indicator ID that can be found on the website. :pr:`291` by :user:`Jovan Stojanovic <jovan-stojanovic>`
+  :pr:`291` by :user:`Jovan Stojanovic <jovan-stojanovic>` and :user:`Leo Grinsztajn <LeoGrin>`
+* New experimental feature: :class:`FeatureAugmenter`, a transformer
+  that augments with :func:`fuzzy_join` the number of features in a main table by using information from auxilliary tables.
+  :pr:`409` by :user:`Jovan Stojanovic <jovan-stojanovic>`
+* **datasets.fetching**: contains a new function :func:`fetch_world_bank_indicator` that can be used to download any indicator
+  from the World Bank Open Data platform, the indicator ID that can be found there.
+  :pr:`291` by :user:`Jovan Stojanovic <jovan-stojanovic>`
 * Unnecessary API has been made private: everything (files, functions, classes)
   starting with an underscore shouldn't be imported in your code. :pr:`331` by :user:`Lilian Boulard <LilianBoulard>`
 * The :class:`MinHashEncoder` now supports a `n_jobs` parameter to parallelize
   the hashes computation. :pr:`267` by :user:`Leo Grinsztajn <LeoGrin>` and :user:`Lilian Boulard <LilianBoulard>`.
+* New experimental feature: deduplicating misspelled categories using :func:`deduplicate` by clustering string distances.
+  This function works best when there are significantly more duplicates than underlying categories.
+  :pr:`339` by :user:`Moritz Boos <mjboos>`.
 
 Minor changes
 -------------
diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
index 2301656..b4e1709 100644
--- a/CODE_OF_CONDUCT.md
+++ b/CODE_OF_CONDUCT.md
@@ -13,4 +13,3 @@ all priceless contributions.
 
 We abide by the principles of openness, respect, and consideration of others of
 the Python Software Foundation: https://www.python.org/psf/codeofconduct/
-
diff --git a/benchmarks/bench_minhash.py b/benchmarks/bench_minhash.py
index bbffa6f..d891c51 100644
--- a/benchmarks/bench_minhash.py
+++ b/benchmarks/bench_minhash.py
@@ -66,7 +66,7 @@ class MinHashEncoder(BaseEstimator, TransformerMixin):
     ----------
     hash_dict_ : LRUDict
         Computed hashes.
-    
+
     Examples
     --------
     >>> enc = MinHashEncoder(n_components=5)
@@ -343,8 +343,9 @@ class MinHashEncoder(BaseEstimator, TransformerMixin):
                     unique_x[idx_slice],
                     hash_func,
                 )
-                for idx_slice in gen_even_slices(len(unique_x), 
-                n_jobs * self.batch_per_job)
+                for idx_slice in gen_even_slices(
+                    len(unique_x), n_jobs * self.batch_per_job
+                )
             )
             # Match the hashes of the unique value to the original values
             X_out = np.concatenate(unique_x_trans)[indices_x].reshape(
@@ -395,8 +396,9 @@ def benchmark(
     batch_per_job: int,
 ) -> None:
     X = data[dataset_size]
-    MinHashEncoder(batch=batched, n_jobs=n_jobs, 
-    batch_per_job=batch_per_job).fit(X).transform(X)
+    MinHashEncoder(batch=batched, n_jobs=n_jobs, batch_per_job=batch_per_job).fit(
+        X
+    ).transform(X)
 
 
 def plot(res: pd.DataFrame):
@@ -408,17 +410,25 @@ def plot(res: pd.DataFrame):
         memories = eval(str(ser["memory"]))
         _, _, kwargs = parse_func_repr(ser["call"])
         for time, memory in zip(times, memories):
-            rows.append((kwargs["batched"], kwargs["batch_per_job"], 
-            kwargs["n_jobs"], time, memory))
+            rows.append(
+                (
+                    kwargs["batched"],
+                    kwargs["batch_per_job"],
+                    kwargs["n_jobs"],
+                    time,
+                    memory,
+                )
+            )
 
-    df = pd.DataFrame(rows, columns=["batched", "batch_per_job", 
-    "n_jobs", "time", "memory"])
+    df = pd.DataFrame(
+        rows, columns=["batched", "batch_per_job", "n_jobs", "time", "memory"]
+    )
 
     # Create a new columns merging batched and batch_per_job
     # If batch is False, ignore batch_per_job
     df["config"] = df.apply(
         lambda row: f"batched={row['batched']}, batch_per_job={row['batch_per_job']}"
-        if row["batched"] == 'True'
+        if row["batched"] == "True"
         else "batched=False",
         axis=1,
     )
diff --git a/benchmarks/get_results.py b/benchmarks/get_results.py
index 5406673..94a6cfe 100644
--- a/benchmarks/get_results.py
+++ b/benchmarks/get_results.py
@@ -12,16 +12,16 @@ from pathlib import Path
 
 here = Path(__file__).parent
 
-expected_end = '_results.csv'
+expected_end = "_results.csv"
 
 param_name = "cardinality_threshold"
 for file in here.iterdir():
     if file.name.endswith(expected_end):
         df: pd.DataFrame = pd.read_csv(file)
-        name = file.name[:-len(expected_end)]
+        name = file.name[: -len(expected_end)]
 
-        card_unq = np.unique(df['param_sv__cardinality_threshold'])
-        comp_unq = np.unique(df['param_sv__high_card_str_transformer__n_components'])
+        card_unq = np.unique(df["param_sv__cardinality_threshold"])
+        comp_unq = np.unique(df["param_sv__high_card_str_transformer__n_components"])
 
         card_amnt = len(card_unq)
         comp_amnt = len(comp_unq)
@@ -35,18 +35,18 @@ for file in here.iterdir():
         ax.set_xticks(np.arange(comp_amnt))
         ax.set_yticks(np.arange(card_amnt))
         # ... and label them with the respective list entries
-        ax.set_xticklabels([f'n_components={v}' for v in comp_unq])
-        ax.set_yticklabels([f'cardinality={v}' for v in card_unq])
+        ax.set_xticklabels([f"n_components={v}" for v in comp_unq])
+        ax.set_yticklabels([f"cardinality={v}" for v in card_unq])
 
         # Rotate the tick labels and set their alignment.
-        plt.setp(ax.get_xticklabels(), rotation=45, ha="right",
-                 rotation_mode="anchor")
+        plt.setp(ax.get_xticklabels(), rotation=45, ha="right", rotation_mode="anchor")
 
         # Loop over data dimensions and create text annotations.
         for i in range(card_amnt):
             for j in range(comp_amnt):
-                text = ax.text(j, i, round(scores[i, j], 3),
-                               ha="center", va="center", color="w")
+                text = ax.text(
+                    j, i, round(scores[i, j], 3), ha="center", va="center", color="w"
+                )
 
         ax.set_title(name)
         fig.tight_layout()
diff --git a/benchmarks/similarity_scores_time_benchmark.py b/benchmarks/similarity_scores_time_benchmark.py
index dc782cc..3010e16 100644
--- a/benchmarks/similarity_scores_time_benchmark.py
+++ b/benchmarks/similarity_scores_time_benchmark.py
@@ -12,7 +12,8 @@ Parameters that are modified:
 
 # We filter out the warning asking us to specify the solver in the logistic_regression
 import warnings
-warnings.simplefilter(action='ignore', category=FutureWarning)
+
+warnings.simplefilter(action="ignore", category=FutureWarning)
 
 from time import time
 
@@ -30,10 +31,17 @@ from dirty_cat.datasets import fetch_traffic_violations
 data = fetch_traffic_violations()
 
 col_to_ohe = [
-    "alcohol", "arrest_type", "belts",
-    "commercial_license", "commercial_vehicle",
-    "fatal", "gender", "hazmat",
-    "property_damage", "race", "work_zone",
+    "alcohol",
+    "arrest_type",
+    "belts",
+    "commercial_license",
+    "commercial_vehicle",
+    "fatal",
+    "gender",
+    "hazmat",
+    "property_damage",
+    "race",
+    "work_zone",
 ]
 
 col_to_pass = ["year"]
@@ -51,25 +59,43 @@ X.reset_index(inplace=True, drop=True)
 y.reset_index(inplace=True, drop=True)
 
 transformers = [
-    ('one_hot', OneHotEncoder(sparse=False, handle_unknown='ignore'), col_to_ohe),
+    ("one_hot", OneHotEncoder(sparse=False, handle_unknown="ignore"), col_to_ohe),
     ("pass", "passthrough", col_to_pass),
 ]
 
 
-def benchmark(X_b, y_b, strat='k-means', limit=50000, n_proto=100, hash_dim=None, ngram_range=(3, 3)):
+def benchmark(
+    X_b,
+    y_b,
+    strat="k-means",
+    limit=50000,
+    n_proto=100,
+    hash_dim=None,
+    ngram_range=(3, 3),
+):
     X_b = X_b[:limit].copy()
     y_b = y_b[:limit].copy()
 
-    if strat == 'k-means':
-        sim_enc = SimilarityEncoder(ngram_range=ngram_range, categories='k-means',
-                                    hashing_dim=hash_dim, n_prototypes=n_proto, random_state=3498)
+    if strat == "k-means":
+        sim_enc = SimilarityEncoder(
+            ngram_range=ngram_range,
+            categories="k-means",
+            hashing_dim=hash_dim,
+            n_prototypes=n_proto,
+            random_state=3498,
+        )
     else:
-        sim_enc = SimilarityEncoder(ngram_range=ngram_range, categories='most_frequent',
-                                    hashing_dim=hash_dim, n_prototypes=n_proto, random_state=3498)
+        sim_enc = SimilarityEncoder(
+            ngram_range=ngram_range,
+            categories="most_frequent",
+            hashing_dim=hash_dim,
+            n_prototypes=n_proto,
+            random_state=3498,
+        )
 
     column_trans = ColumnTransformer(
-        transformers=transformers + [('sim_enc', sim_enc, ['description'])],
-        remainder='drop'
+        transformers=transformers + [("sim_enc", sim_enc, ["description"])],
+        remainder="drop",
     )
 
     t0 = time()
@@ -86,29 +112,33 @@ def benchmark(X_b, y_b, strat='k-means', limit=50000, n_proto=100, hash_dim=None
     return t_score_1, m_score, t_score_2
 
 
-def plot(bench, title=''):
-    sns.set(style='ticks', palette='muted')
-    hash_dims = ['Count', '2 ** 14', '2 ** 16', '2 ** 18', '2 ** 20']
+def plot(bench, title=""):
+    sns.set(style="ticks", palette="muted")
+    hash_dims = ["Count", "2 ** 14", "2 ** 16", "2 ** 18", "2 ** 20"]
     scores = []
     vectorizer = []
     strategy = []
 
     for i, e in enumerate(bench):
         vectorizer.extend([hash_dims[i % 5]] * (2 * len(e[0][1])))
-        strategy.extend(['k-means'] * len(e[0][1]))
-        strategy.extend(['most-frequent'] * len(e[1][1]))
+        strategy.extend(["k-means"] * len(e[0][1]))
+        strategy.extend(["most-frequent"] * len(e[1][1]))
         scores.extend(e[0][1])
         scores.extend(e[1][1])
 
-    df = pd.DataFrame(columns=['vectorizer', 'strategy', 'score'])
-    df['vectorizer'] = vectorizer
-    df['strategy'] = strategy
-    df['score'] = scores
+    df = pd.DataFrame(columns=["vectorizer", "strategy", "score"])
+    df["vectorizer"] = vectorizer
+    df["strategy"] = strategy
+    df["score"] = scores
 
     first = plt.figure()
-    ax = sns.boxplot(x='vectorizer', y='score', hue='strategy', data=df)
-    ax.set(title=title, xlabel='Vectorizer used', ylabel='Mean score on 10 cross validations')
-    ax.legend(loc='center left', bbox_to_anchor=(1, 0.5))
+    ax = sns.boxplot(x="vectorizer", y="score", hue="strategy", data=df)
+    ax.set(
+        title=title,
+        xlabel="Vectorizer used",
+        ylabel="Mean score on 10 cross validations",
+    )
+    ax.legend(loc="center left", bbox_to_anchor=(1, 0.5))
     first.tight_layout()
 
     vectorizer.clear()
@@ -118,41 +148,65 @@ def plot(bench, title=''):
 
     for i, e in enumerate(bench):
         vectorizer.extend([hash_dims[i % 5]] * 4)
-        strategy.extend(['K-means vect', 'K-means X-val', 'MF vect', 'MF X-val'])
+        strategy.extend(["K-means vect", "K-means X-val", "MF vect", "MF X-val"])
         times.extend([e[0][0], e[0][2] / 20, e[1][0], e[1][2] / 20])
 
-    df = pd.DataFrame(columns=['vectorizer', 'strategy/operation', 'time'])
-    df['vectorizer'] = vectorizer
-    df['strategy/operation'] = strategy
-    df['time'] = times
+    df = pd.DataFrame(columns=["vectorizer", "strategy/operation", "time"])
+    df["vectorizer"] = vectorizer
+    df["strategy/operation"] = strategy
+    df["time"] = times
 
     second = plt.figure()
-    ax1 = sns.barplot(x='vectorizer', y='time', hue='strategy/operation', data=df)
-    ax1.set(title=title, xlabel='Vectorizer used', ylabel='Time in seconds')
-    ax1.legend(loc='center left', bbox_to_anchor=(1, 0.5))
+    ax1 = sns.barplot(x="vectorizer", y="time", hue="strategy/operation", data=df)
+    ax1.set(title=title, xlabel="Vectorizer used", ylabel="Time in seconds")
+    ax1.legend(loc="center left", bbox_to_anchor=(1, 0.5))
     second.tight_layout()
 
-    title = title.replace(' ', '_').replace(':', '-').replace(',', '_').lower()
-    first.savefig(title + '_score.png')
-    second.savefig(title + '_time.png')
+    title = title.replace(" ", "_").replace(":", "-").replace(",", "_").lower()
+    first.savefig(title + "_score.png")
+    second.savefig(title + "_time.png")
     # first.show()
     # second.show(t)
 
 
 def loop(proto):
-    limits = sorted([X['description'].nunique(), 10000, 20000, 50000, 100000])
-    hash_dims = [None, 2 ** 14, 2 ** 16, 2 ** 18, 2 ** 20]
+    limits = sorted([X["description"].nunique(), 10000, 20000, 50000, 100000])
+    hash_dims = [None, 2**14, 2**16, 2**18, 2**20]
     bench = list()
     ngram = [(3, 3), (2, 4)]
     for limit in limits:
         for r in ngram:
             for h in hash_dims:
-                bench.append((benchmark(X, y, strat='k-means', limit=limit, n_proto=proto, hash_dim=h, ngram_range=r),
-                              benchmark(X, y, strat='most-frequent', limit=limit, n_proto=proto, hash_dim=h, ngram_range=r)))
-            title = 'N-gram range: %s, Rows: %d, Prototypes: %d, 20 CV' % (r.__str__(), limit, proto)
+                bench.append(
+                    (
+                        benchmark(
+                            X,
+                            y,
+                            strat="k-means",
+                            limit=limit,
+                            n_proto=proto,
+                            hash_dim=h,
+                            ngram_range=r,
+                        ),
+                        benchmark(
+                            X,
+                            y,
+                            strat="most-frequent",
+                            limit=limit,
+                            n_proto=proto,
+                            hash_dim=h,
+                            ngram_range=r,
+                        ),
+                    )
+                )
+            title = "N-gram range: %s, Rows: %d, Prototypes: %d, 20 CV" % (
+                r.__str__(),
+                limit,
+                proto,
+            )
             plot(bench, title)
             bench.clear()
 
 
-if __name__ == '__main__':
-    loop(100)
\ No newline at end of file
+if __name__ == "__main__":
+    loop(100)
diff --git a/benchmarks/supervectorizer_tuning.py b/benchmarks/supervectorizer_tuning.py
index f726c0f..cd1031f 100644
--- a/benchmarks/supervectorizer_tuning.py
+++ b/benchmarks/supervectorizer_tuning.py
@@ -11,9 +11,15 @@ from sklearn.model_selection import GridSearchCV
 from sklearn.pipeline import Pipeline
 
 from dirty_cat import SuperVectorizer
-from dirty_cat.datasets import fetch_open_payments, fetch_drug_directory, \
-    fetch_road_safety, fetch_midwest_survey, fetch_medical_charge, \
-    fetch_employee_salaries, fetch_traffic_violations
+from dirty_cat.datasets import (
+    fetch_open_payments,
+    fetch_drug_directory,
+    fetch_road_safety,
+    fetch_midwest_survey,
+    fetch_medical_charge,
+    fetch_employee_salaries,
+    fetch_traffic_violations,
+)
 
 from pathlib import Path
 from functools import wraps
@@ -23,25 +29,25 @@ from typing import List, Tuple
 
 def get_classification_datasets() -> List[Tuple[dict, str]]:
     return [
-        (fetch_open_payments(), 'open_payments'),
+        (fetch_open_payments(), "open_payments"),
         # (fetch_drug_directory(), 'drug_directory),
-        (fetch_road_safety(), 'road_safety'),
-        (fetch_midwest_survey(), 'midwest_survey'),
-        (fetch_traffic_violations(), 'traffic_violations'),
+        (fetch_road_safety(), "road_safety"),
+        (fetch_midwest_survey(), "midwest_survey"),
+        (fetch_traffic_violations(), "traffic_violations"),
     ]
 
 
 def get_regression_datasets() -> List[Tuple[dict, str]]:
     return [
-        (fetch_medical_charge(), 'medical_charge'),
-        (fetch_employee_salaries(), 'employee_salaries'),
+        (fetch_medical_charge(), "medical_charge"),
+        (fetch_employee_salaries(), "employee_salaries"),
     ]
 
 
 def get_dataset(info) -> Tuple[pd.DataFrame, pd.Series]:
-    df = pd.read_csv(info['path'], **info['read_csv_kwargs'])
-    y = df[info['y']]
-    X = df.drop(info['y'], axis=1).astype(str)
+    df = pd.read_csv(info["path"], **info["read_csv_kwargs"])
+    y = df[info["y"]]
+    X = df.drop(info["y"], axis=1).astype(str)
     return X, y
 
 
@@ -53,12 +59,12 @@ def set_logging(func):
         logger = logging.getLogger()
         logger.setLevel(logging_level)
 
-        formatter = logging.Formatter('%(asctime)s - [%(levelname)s] %(message)s')
-        formatter.datefmt = '%m/%d/%Y %H:%M:%S'
+        formatter = logging.Formatter("%(asctime)s - [%(levelname)s] %(message)s")
+        formatter.datefmt = "%m/%d/%Y %H:%M:%S"
 
-        path = Path(__file__).parent / f'tuning_{str(datetime.now())[:10]}.log'
+        path = Path(__file__).parent / f"tuning_{str(datetime.now())[:10]}.log"
 
-        fh = logging.FileHandler(filename=path, mode='w')
+        fh = logging.FileHandler(filename=path, mode="w")
         fh.setLevel(logging_level)
         fh.setFormatter(formatter)
 
@@ -70,28 +76,35 @@ def set_logging(func):
         # logger.addHandler(sh)
 
         return func(*args, **kwargs)
+
     return wrapper
 
 
 @set_logging
 def main():
-    logging.info('Launching !')
+    logging.info("Launching !")
 
     card_possibilities = [20, 30, 40, 50]
     n_comp_possibilities = [10, 30, 50]
 
-    logging.debug('Creating pipelines')
-    regression_pipeline = Pipeline([
-        ('sv', SuperVectorizer()),
-        ('estimator', RandomForestRegressor()),
-    ])
-    classification_pipeline = Pipeline([
-        ('sv', SuperVectorizer()),
-        ('estimator', RandomForestClassifier()),
-    ])
-
-    logging.debug(f'With cardinality possibilities: {card_possibilities} '
-                  f'and n_components possibilities: {n_comp_possibilities}')
+    logging.debug("Creating pipelines")
+    regression_pipeline = Pipeline(
+        [
+            ("sv", SuperVectorizer()),
+            ("estimator", RandomForestRegressor()),
+        ]
+    )
+    classification_pipeline = Pipeline(
+        [
+            ("sv", SuperVectorizer()),
+            ("estimator", RandomForestClassifier()),
+        ]
+    )
+
+    logging.debug(
+        f"With cardinality possibilities: {card_possibilities} "
+        f"and n_components possibilities: {n_comp_possibilities}"
+    )
     for pipeline, datasets in zip(
         [
             regression_pipeline,
@@ -100,25 +113,25 @@ def main():
         [
             get_regression_datasets(),
             get_classification_datasets(),
-        ]
+        ],
     ):
         for info, name in datasets:
             X, y = get_dataset(info)
-            if name != 'traffic_violations':
+            if name != "traffic_violations":
                 continue
 
-            csv_path = Path('.').resolve() / f'{name}_results.csv'
+            csv_path = Path(".").resolve() / f"{name}_results.csv"
             if csv_path.exists():
                 # If the results already exist, we'll skip to the next
-                logging.debug(f'Skipping {name} as {csv_path!s} was found')
+                logging.debug(f"Skipping {name} as {csv_path!s} was found")
                 continue
 
-            logging.debug(f'Running search on {name}')
+            logging.debug(f"Running search on {name}")
             grid = GridSearchCV(
                 estimator=pipeline,
                 param_grid={
-                    'sv__cardinality_threshold': card_possibilities,
-                    'sv__high_card_str_transformer__n_components': n_comp_possibilities,
+                    "sv__cardinality_threshold": card_possibilities,
+                    "sv__high_card_str_transformer__n_components": n_comp_possibilities,
                 },
                 n_jobs=30,
             )
@@ -126,8 +139,8 @@ def main():
 
             df = pd.DataFrame(grid.cv_results_)
             df.to_csv(csv_path)
-            logging.info(f'Saved search results in {csv_path!s}')
+            logging.info(f"Saved search results in {csv_path!s}")
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()
diff --git a/dirty_cat/VERSION.txt b/dirty_cat/VERSION.txt
index ca91a08..88b0e14 100644
--- a/dirty_cat/VERSION.txt
+++ b/dirty_cat/VERSION.txt
@@ -1,1 +1,1 @@
-0.4.dev0
+0.4.0b1
diff --git a/dirty_cat/__init__.py b/dirty_cat/__init__.py
index dc13251..40f30b8 100644
--- a/dirty_cat/__init__.py
+++ b/dirty_cat/__init__.py
@@ -15,6 +15,8 @@ except ModuleNotFoundError:
     )
 
 from ._datetime_encoder import DatetimeEncoder
+from ._deduplicate import compute_ngram_distance, deduplicate
+from ._feature_augmenter import FeatureAugmenter
 from ._fuzzy_join import fuzzy_join
 from ._gap_encoder import GapEncoder
 from ._minhash_encoder import MinHashEncoder
@@ -27,11 +29,14 @@ with open(_Path(__file__).parent / "VERSION.txt") as _fh:
 
 
 __all__ = [
-    "SimilarityEncoder",
-    "TargetEncoder",
-    "MinHashEncoder",
-    "GapEncoder",
     "DatetimeEncoder",
-    "SuperVectorizer",
+    "FeatureAugmenter",
     "fuzzy_join",
+    "GapEncoder",
+    "MinHashEncoder",
+    "SimilarityEncoder",
+    "SuperVectorizer",
+    "TargetEncoder",
+    "deduplicate",
+    "compute_ngram_distance",
 ]
diff --git a/dirty_cat/_datetime_encoder.py b/dirty_cat/_datetime_encoder.py
index 247ec8f..c117276 100644
--- a/dirty_cat/_datetime_encoder.py
+++ b/dirty_cat/_datetime_encoder.py
@@ -62,12 +62,12 @@ class DatetimeEncoder(BaseEstimator, TransformerMixin):
     ----------
     n_features_in_: int
         Number of features in the data seen during fit.
-    n_features_out_: int
+    n_features_out_ : int
         Number of features of the transformed data.
-    features_per_column_: typing.Dict[int, typing.List[str]]
+    features_per_column_ : mapping of int to list of str
         Dictionary mapping the index of the original columns
         to the list of features extracted for each column.
-    col_names_: typing.Optional[typing.List[str]]
+    col_names_ : None or list of str
         List of the names of the features of the input data,
         if input data was a pandas DataFrame, otherwise None.
 
@@ -154,7 +154,9 @@ class DatetimeEncoder(BaseEstimator, TransformerMixin):
                 ) // pd.Timedelta("1s")
 
     def fit(self, X, y=None) -> "DatetimeEncoder":
-        """Fit the DatetimeEncoder to X. In practice, just stores which extracted features are not constant.
+        """Fit the instance to X.
+
+        In practice, just stores which extracted features are not constant.
 
         Parameters
         ----------
@@ -165,8 +167,8 @@ class DatetimeEncoder(BaseEstimator, TransformerMixin):
 
         Returns
         -------
-        DatetimeEncoder
-            Fitted DatetimeEncoder instance.
+        :class:`~dirty_cat.DatetimeEncoder`
+            Fitted :class:`~dirty_cat.DatetimeEncoder` instance (self).
         """
         self._validate_keywords()
         # Columns to extract for each column,
@@ -218,7 +220,7 @@ class DatetimeEncoder(BaseEstimator, TransformerMixin):
 
         Returns
         -------
-        np.ndarray, shape `(n_samples, n_features_out_)`
+        :obj:`~numpy.ndarray`, shape (n_samples, n_features_out_)
             Transformed input.
         """
         check_is_fitted(
diff --git a/dirty_cat/_deduplicate.py b/dirty_cat/_deduplicate.py
new file mode 100644
index 0000000..fda228f
--- /dev/null
+++ b/dirty_cat/_deduplicate.py
@@ -0,0 +1,182 @@
+"""
+Implements deduplication based on clustering string distance matrices.
+This works best if there is a number of underlying categories that
+sometimes appear in the data with small variations and/or misspellings.
+"""
+from typing import List, Literal, Optional, Sequence, Tuple, Union
+
+import numpy as np
+import pandas as pd
+from scipy.cluster.hierarchy import fcluster, linkage
+from scipy.spatial.distance import pdist, squareform
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics import silhouette_score
+
+
+def compute_ngram_distance(
+    unique_words: Union[Sequence[str], np.ndarray],
+    ngram_range: Tuple[int, int] = (2, 4),
+    analyzer: str = "char_wb",
+) -> np.ndarray:
+    """Compute the condensed pair-wise n-gram distance between `unique_words`.
+
+    Parameters
+    ----------
+    unique_words : Union[Sequence[str], np.ndarray]
+        Sequence or array of unique words from the original data.
+    ngram_range : Tuple[int, int], optional, default=(2,4)
+        The n-gram range to compute the distance in.
+    analyzer : str, optional, default=`char_wb`
+        Analyzer to extract n-grams.
+
+    Returns
+    -------
+    np.ndarray
+        An n-times-(n-1)/2 array of n-gram tf-idf distances between `unique_words`.
+
+    Notes
+    -----
+    Extracts n-grams of all elements in `unique_words`, calculates the
+    term frequency-inverse document frequency (tf-idf) for each n-gram, then
+    computes the pair-wise euclidean distance between elements based on their n-gram
+    tf-idf representation.
+    """
+    encoded = TfidfVectorizer(ngram_range=ngram_range, analyzer=analyzer).fit_transform(
+        unique_words
+    )
+
+    distance_mat = pdist(encoded.todense(), metric="euclidean")
+    return distance_mat
+
+
+def _guess_clusters(Z: np.ndarray, distance_mat: np.ndarray) -> int:
+    """Finds the number of clusters that maximize the silhouette score
+    when clustering `distance_mat`.
+
+    Parameters
+    ----------
+    Z : np.ndarray
+        hierarchical linkage matrix, specifies which clusters to merge.
+    distance_mat : np.ndarray
+        distance matrix either in square or condensed form.
+
+    Returns
+    -------
+    int
+        number of clusters that maximize the silhouette score.
+    """
+    max_clusters = distance_mat.shape[0]
+    n_clusters = np.arange(2, max_clusters)
+    # silhouette score needs a redundant distance matrix
+    redundant_dist = squareform(distance_mat)
+    silhouette_scores = []
+    for n_clust in n_clusters:
+        labels = fcluster(Z, n_clust, criterion="maxclust")
+        silhouette_avg = silhouette_score(redundant_dist, labels, metric="precomputed")
+        silhouette_scores.append(silhouette_avg)
+    return n_clusters[np.argmax(silhouette_scores)]
+
+
+def _create_spelling_correction(
+    unique_words: Union[Sequence[str], np.ndarray],
+    counts: Union[Sequence[int], np.ndarray],
+    clusters: Sequence[int],
+) -> pd.Series:
+    """Creates a pandas Series that map each cluster member to the most
+    frequent cluster member. The assumption is that the most common spelling
+    is the correct one.
+
+    Parameters
+    ----------
+    unique_words : Union[Sequence[str], np.ndarray]
+        A sequence or array of unique words in the original data.
+    counts : Union[Sequence[int], np.ndarray]
+        A sequence or array of counts of how often each unique word appears in
+        the original data.
+    clusters : Sequence[int]
+        A sequence of ints, indicating cluster membership of each unique word
+        in `count_series`.
+
+    Returns
+    -------
+    pd.Series
+        Series with unique (original) words as indices and (estimated)
+        corrected spelling of each word as values.
+    """
+    count_series = pd.Series(counts, index=unique_words)
+    original_spelling: List[str] = []
+    corrected_spelling: List[str] = []
+    for cluster in np.unique(clusters):
+        sorted_spellings = (
+            count_series.loc[clusters == cluster]
+            .sort_values(ascending=False)
+            .index.tolist()
+        )
+        original_spelling.extend(sorted_spellings)
+        # assumes spelling that occurs most frequently in cluster is correct
+        corrected_spelling.extend(sorted_spellings[:1] * len(sorted_spellings))
+    pd_spell_correct = pd.Series(corrected_spelling, index=original_spelling)
+    return pd_spell_correct
+
+
+def deduplicate(
+    data: Sequence[str],
+    n_clusters: Optional[int] = None,
+    ngram_range: Tuple[int, int] = (2, 4),
+    analyzer: Literal["word", "char", "char_wb"] = "char_wb",
+    method: Literal[
+        "single", "complete", "average", "centroid", "median", "ward"
+    ] = "average",
+) -> List[str]:
+    """Deduplicate data by hierarchically clustering similar strings.
+
+    Parameters
+    ----------
+    data : Sequence[str]
+        The data to be deduplicated.
+    n_clusters : Optional[int], optional, default=None
+        Number of clusters to use for hierarchical clustering, if None use the
+        number of clusters that lead to the lowest silhouette score.
+    ngram_range : Tuple[int, int], optional, default=(2, 4)
+        Range to use for computing n-gram distance.
+    analyzer : typing.Literal["word", "char", "char_wb"], optional, default=`char_wb`
+        Analyzer parameter for the CountVectorizer used for the string
+        similarities.
+        Options: {`word`, `char`, `char_wb`}, describing whether the matrix V
+        to factorize should be made of word counts or character n-gram counts.
+        Option `char_wb` creates character n-grams only from text inside word
+        boundaries; n-grams at the edges of words are padded with space.
+    method : str, optional, default=`average`
+        Linkage method parameter to use for merging clusters via scipy's
+        `linkage` method.
+        Options: {`single`, `complete`, `average`, `centroid`, `median`, `ward`},
+        describing different methods to calculate the distance between two clusters.
+        Option `average` calculates the distance between two clusters as the average
+        distance between data points in the first and second cluster.
+
+    Returns
+    -------
+    List[str]
+       The deduplicated data.
+
+    Notes
+    -----
+    Deduplication is done by first computing the n-gram distance between unique
+    categories in data, then performing hierarchical clustering on this distance
+    matrix, and choosing the most frequent element in each cluster as the
+    'correct' spelling. This method works best if the true number of
+    categories is significantly smaller than the number of observed spellings.
+    """
+    unique_words, counts = np.unique(data, return_counts=True)
+    distance_mat = compute_ngram_distance(
+        unique_words, ngram_range=ngram_range, analyzer=analyzer
+    )
+
+    Z = linkage(distance_mat, method=method, optimal_ordering=True)
+    if n_clusters is None:
+        n_clusters = _guess_clusters(Z, distance_mat)
+    clusters = fcluster(Z, n_clusters, criterion="maxclust")
+
+    translation_table = _create_spelling_correction(unique_words, counts, clusters)
+    unrolled_corrections = translation_table[data]
+    return unrolled_corrections
diff --git a/dirty_cat/_feature_augmenter.py b/dirty_cat/_feature_augmenter.py
new file mode 100644
index 0000000..ec5afef
--- /dev/null
+++ b/dirty_cat/_feature_augmenter.py
@@ -0,0 +1,197 @@
+"""
+Transformer that allows multiple fuzzy joins to be performed on a table.
+The principle is as follows:
+  1. The main table and the key column name are provided at initialisation.
+  2. The auxilliary tables are provided for fitting, and will be joined sequentially
+  when the transform is called.
+It is advised to use hyper-parameter tuning tools such as scikit-learn's
+GridSearchCV to determine the best `match_score` parameter, as this can
+significantly improve your results.
+(see example 'Fuzzy joining dirty tables with the FeatureAugmenter' for an illustration)
+For more information on how the join is performed, see fuzzy_join's documentation.
+"""
+
+from typing import List, Literal, Tuple
+
+import pandas as pd
+from sklearn.base import BaseEstimator, TransformerMixin
+
+from dirty_cat._fuzzy_join import fuzzy_join
+
+
+class FeatureAugmenter(BaseEstimator, TransformerMixin):
+    """Transformer augmenting the number of features in a table by joining multiple tables.
+
+    Given a list of tables and key column names,
+    fuzzy join them to the main table.
+
+    Parameters
+    ----------
+    tables : list of 2-tuples of (:class:`~pandas.DataFrame`, str)
+        List of (table, column name) tuples
+        specyfying the transformer objects to be applied.
+        table: str
+            Name of the table to be joined.
+        column: str
+            Name of table column to join on.
+    main_key : str
+        The key column name in the main table on which
+        the join will be performed.
+    match_score : float, default=0
+        Distance score between the closest matches that will be accepted.
+        In a [0, 1] interval. 1 means that only a perfect match will be
+        accepted, and zero means that the closest match will be accepted,
+        no matter how distant.
+    analyzer : typing.Literal["word", "char", "char_wb"], default=`char_wb`
+        Analyzer parameter for the CountVectorizer used for the string
+        similarities.
+        Options: {`word`, `char`, `char_wb`}, describing whether the matrix V
+        to factorize should be made of word counts or character n-gram counts.
+        Option `char_wb` creates character n-grams only from text inside word
+        boundaries; n-grams at the edges of words are padded with space.
+    ngram_range : tuple (min_n, max_n), default=(2, 4)
+        The lower and upper boundary of the range of n-values for different
+        n-grams used in the string similarity. All values of n such
+        that min_n <= n <= max_n will be used.
+
+    Examples
+    --------
+    >>> X = pd.DataFrame(['France', 'Germany', 'Italy'], columns=['Country'])
+    >>> X
+    Country
+    0   France
+    1  Germany
+    2    Italy
+
+    >>> aux_table_1 = pd.DataFrame([['Germany', 84_000_000], ['France', 68_000_000], ['Italy', 59_000_000]], columns=['Country', 'Population']) # noqa
+    >>> aux_table_1
+       Country  Population
+    0  Germany    84000000
+    1   France    68000000
+    2    Italy    59000000
+
+    >>> aux_table_2 = pd.DataFrame([['French Republic', 2937], ['Italy', 2099], ['Germany', 4223], ['UK', 3186]], columns=['Country name', 'GDP (billion)']) # noqa
+    >>> aux_table_2
+        Country name  GDP (billion)
+    0   French Republic      2937
+    1        Italy           2099
+    2      Germany           4223
+    3           UK           3186
+
+    >>> aux_table_3 = pd.DataFrame([['France', 'Paris'], ['Italia', 'Rome'], ['Germany', 'Berlin']], columns=['Countries', 'Capital']) # noqa
+    >>> aux_table_3
+      Countries Capital
+    0    France   Paris
+    1     Italia   Rome
+    2   Germany  Berlin
+
+    >>> aux_tables = [(aux_table_1, "Country"), (aux_table_2, "Country name"), (aux_table_3, "Countries")] # noqa
+
+    >>> fa = FeatureAugmenter(tables=aux_tables, main_key='Country')
+
+    >>> augmented_table = fa.fit_transform(X)
+    >>> augmented_table
+        Country Country_aux  Population Country name  GDP (billion) Countries Capital
+    0   France      France    68000000  French Republic       2937    France   Paris
+    1  Germany     Germany    84000000      Germany           4223   Germany  Berlin
+    2    Italy       Italy    59000000        Italy           2099    Italia    Rome
+    """
+
+    def __init__(
+        self,
+        tables: List[Tuple[pd.DataFrame, str]],
+        main_key: str,
+        match_score: float = 0.0,
+        analyzer: Literal["word", "char", "char_wb"] = "char_wb",
+        ngram_range: Tuple[int, int] = (2, 4),
+    ):
+        self.tables = tables
+        self.main_key = main_key
+        self.match_score = match_score
+        self.analyzer = analyzer
+        self.ngram_range = ngram_range
+
+    def fit(self, X, y=None) -> "FeatureAugmenter":
+        """Fit the Feature Augmenter to the main table.
+
+        In practice, just checks if the key columns in X,
+        the main table, and in the auxilliary tables exist.
+
+        Parameters
+        ----------
+        X : DataFrame, shape [n_samples, n_features]
+            The main table, to be joined to the
+            auxilliary ones.
+        y : array-like of shape (n_samples,...), default=None
+            Targets for supervised learning.
+
+        Returns
+        -------
+        FeatureAugmenter
+            Fitted FeatureAugmenter instance.
+        """
+
+        if self.main_key not in X.columns:
+            raise ValueError(
+                f"Got main_key={self.main_key!r}, but column not in {list(X.columns)}."
+            )
+
+        for pairs in self.tables:
+            if pairs[1] not in pairs[0].columns:
+                raise ValueError(
+                    f"Got column key {pairs[1]!r}, "
+                    f"but column not in {pairs[0].columns}."
+                )
+        return self
+
+    def transform(self, X, y=None) -> pd.DataFrame:
+        """Transform X using the specified encoding scheme.
+
+        Parameters
+        ----------
+        X : DataFrame, shape [n_samples, n_features]
+            The main table, to be joined to the
+            auxilliary ones.
+        y : array-like of shape (n_samples,...), default=None
+            Targets for supervised learning.
+
+        Returns
+        -------
+        DataFrame
+            The final joined table.
+        """
+
+        for pairs in self.tables:
+            # TODO: Add an option to fuzzy_join on multiple columns at once
+            # (will be if len(inter_col)!=0)
+            aux_table = pairs[0]
+            X = fuzzy_join(
+                X,
+                aux_table,
+                left_on=self.main_key,
+                right_on=pairs[1],
+                match_score=self.match_score,
+                analyzer=self.analyzer,
+                ngram_range=self.ngram_range,
+                suffixes=("", "_aux"),
+            )
+        return X
+
+    def fit_transform(self, X, y=None) -> pd.DataFrame:
+        """
+        Fit the FeatureAugmenter to X, then transforms it.
+
+        Parameters
+        ----------
+        X : DataFrame, shape [n_samples, n_features]
+            The main table, to be joined to the
+            auxilliary ones.
+        y : array-like of shape (n_samples,...), default=None
+            Targets for supervised learning.
+
+        Returns
+        -------
+        DataFrame
+            The final joined table.
+        """
+        return self.fit(X).transform(X)
diff --git a/dirty_cat/_fuzzy_join.py b/dirty_cat/_fuzzy_join.py
index 0054b36..00b442d 100644
--- a/dirty_cat/_fuzzy_join.py
+++ b/dirty_cat/_fuzzy_join.py
@@ -1,8 +1,8 @@
 """
 Fuzzy joining tables using string columns.
 The principle is as follows:
-  1. We embed and transform the key string columns using CountVectorizer
-  and TfifdTransformer.
+  1. We embed and transform the key string columns using
+  HashingVectorizer and TfifdTransformer.
   2. For each category, we use the nearest neighbor method to find its closest
   neighbor and establish a match.
   3. We match the tables using the previous information.
@@ -17,7 +17,11 @@ from typing import Literal, Tuple, Union
 import numpy as np
 import pandas as pd
 from scipy.sparse import vstack
-from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
+from sklearn.feature_extraction.text import (
+    HashingVectorizer,
+    TfidfTransformer,
+    _VectorizerMixin,
+)
 from sklearn.neighbors import NearestNeighbors
 
 
@@ -28,6 +32,7 @@ def fuzzy_join(
     left_on: Union[str, None] = None,
     right_on: Union[str, None] = None,
     on: Union[str, None] = None,
+    encoder: Union[Literal["hashing"], _VectorizerMixin] = None,
     analyzer: Literal["word", "char", "char_wb"] = "char_wb",
     ngram_range: Tuple[int, int] = (2, 4),
     return_score: bool = False,
@@ -42,56 +47,61 @@ def fuzzy_join(
 
     Parameters
     ----------
-    left : pandas.DataFrame
+    left : :class:`~pandas.DataFrame`
         A table to merge.
-    right : pandas.DataFrame
+    right : :class:`~pandas.DataFrame`
         A table used to merge with.
     how: typing.Literal["left", "right"], default=`left`
-        Type of merge to be performed. Note that unlike pandas' merge,
+        Type of merge to be performed. Note that unlike :func:`~pandas.merge`,
         only "left" and "right" are supported so far, as the fuzzy-join comes
         with its own mechanism to resolve lack of correspondence between
         left and right tables.
-    left_on : typing.Union[str, None]
+    left_on : str, optional, default=None
         Name of left table column to join.
-    right_on : typing.Union[str, None]
+    right_on : str, optional, default=None
         Name of right table key column to join
         with left table key column.
-    on : typing.Union[str, None]
+    on : str, optional, default=None
         Name of common left and right table join key columns.
         Must be found in both DataFrames. Use only if `left_on`
         and `right_on` parameters are not specified.
-    analyzer : typing.Literal["word", "char", "char_wb"], default=`char_wb`
-        Analyzer parameter for the CountVectorizer used for the string
-        similarities.
+    encoder: Union[Literal["hashing"], _VectorizerMixin], default=None,
+        Encoder parameter for the Vectorizer.
+        Options: {None, `_VectorizerMixin`}. If None, the
+        encoder will use the `HashingVectorizer`. It is possible to pass a
+        `_VectorizerMixin` custom object to tweak the parameters of the encoder.
+    analyzer : {"word", "char", "char_wb"}, optional, default=`char_wb`
+        Analyzer parameter for the HashingVectorizer passed to
+        the encoder and used for the string similarities.
         Options: {`word`, `char`, `char_wb`}, describing whether the matrix V
         to factorize should be made of word counts or character n-gram counts.
         Option `char_wb` creates character n-grams only from text inside word
         boundaries; n-grams at the edges of words are padded with space.
-    ngram_range : tuple (min_n, max_n), default=(2, 4)
+    ngram_range : int 2-tuple, optional, default=(2, 4)
         The lower and upper boundary of the range of n-values for different
         n-grams used in the string similarity. All values of n such
         that min_n <= n <= max_n will be used.
     return_score : boolean, default=True
         Whether to return matching score based on the distance between
-        nearest matched categories.
-    match_score : float, default=0
+        the nearest matched categories.
+    match_score : float, default=0.0
         Distance score between the closest matches that will be accepted.
-        In a [0, 1] interval. Closer to 1 means the matches need to be very
-        close to be accepted, and closer to 0 that a bigger matching distance
-        is tolerated.
+        In a [0, 1] interval. 1 means that only a perfect match will be
+        accepted, and zero means that the closest match will be accepted,
+        no matter how distant.
     drop_unmatched : boolean, default=False
         Remove categories for which a match was not found in the two tables.
     sort : boolean, default=False
         Sort the join keys lexicographically in the result DataFrame.
         If False, the order of the join keys depends on the join type
         (`how` keyword).
-    suffixes : typing.Tuple[str, str], default=('_x', '_y')
+    suffixes : str 2-tuple, default=('_x', '_y')
         A list of strings indicating the suffix to add when overlaping
         column names.
 
     Returns
     -------
-    df_joined: pandas.DataFrame
+    df_joined : :class:`~pandas.DataFrame`
         The joined table returned as a DataFrame. If `return_score` is True,
         another column will be added to the DataFrame containing the
         matching scores.
@@ -99,13 +109,12 @@ def fuzzy_join(
     Notes
     -----
     For regular joins, the output of fuzzy_join is identical
-    to pandas.merge, except that both key columns are returned.
+    to :func:`~pandas.merge`, except that both key columns are returned.
 
-    Joining on indexes and multiple columns is not
-    supported.
+    Joining on indexes and multiple columns is not supported.
 
-    When return_score=True, the returned DataFrame gives
-    the distances between closest matches in a [0, 1] interval.
+    When `return_score=True`, the returned :class:`~pandas.DataFrame` gives
+    the distances between the closest matches in a [0, 1] interval.
     0 corresponds to no matching n-grams, while 1 is a
     perfect match.
 
@@ -154,7 +163,6 @@ def fuzzy_join(
     2  nana  3   NaN  NaN  0.532717
 
     As expected, the category "nana" has no exact match (`match_score=1`).
-
     """
 
     warnings.warn("This feature is still experimental.")
@@ -164,6 +172,10 @@ def fuzzy_join(
             f"analyzer should be either 'char', 'word' or 'char_wb', got {analyzer!r}",
         )
 
+    if encoder is not None:
+        if not issubclass(encoder.__class__, _VectorizerMixin):
+            raise ValueError(f"encoder should be a vectorizer object, got {encoder!r}")
+
     if how not in ["left", "right"]:
         raise ValueError(
             f"how should be either 'left' or 'right', got {how!r}",
@@ -208,10 +220,13 @@ def fuzzy_join(
     main_col_clean = main_table[main_col].astype(str)
     aux_col_clean = aux_table[aux_col].astype(str)
 
-    enc = CountVectorizer(analyzer=analyzer, ngram_range=ngram_range)
-
     all_cats = pd.concat([main_col_clean, aux_col_clean], axis=0).unique()
 
+    if encoder is None:
+        enc = HashingVectorizer(analyzer=analyzer, ngram_range=ngram_range)
+    else:
+        enc = encoder
+
     enc_cv = enc.fit(all_cats)
     main_enc = enc_cv.transform(main_col_clean)
     aux_enc = enc_cv.transform(aux_col_clean)
diff --git a/dirty_cat/_gap_encoder.py b/dirty_cat/_gap_encoder.py
index 0cea641..014d6bd 100644
--- a/dirty_cat/_gap_encoder.py
+++ b/dirty_cat/_gap_encoder.py
@@ -30,6 +30,7 @@ from sklearn.neighbors import NearestNeighbors
 from sklearn.utils import check_random_state, gen_batches
 from sklearn.utils.extmath import row_norms, safe_sparse_dot
 from sklearn.utils.fixes import _object_dtype_isnan
+from sklearn.utils.validation import check_is_fitted
 
 from ._utils import check_input, parse_version
 
@@ -512,6 +513,7 @@ class GapEncoderColumn(BaseEstimator, TransformerMixin):
         H : 2-d array, shape (n_samples, n_topics)
             Transformed input.
         """
+        check_is_fitted(self, "H_dict_")
         # Check if first item has str or np.str_ type
         assert isinstance(X[0], str), "Input data is not string. "
         unq_X = np.unique(X)
@@ -760,7 +762,7 @@ class GapEncoder(BaseEstimator, TransformerMixin):
 
     def fit(self, X, y=None) -> "GapEncoder":
         """
-        Fit the GapEncoder on batches of X.
+        Fit the instance on batches of X.
 
         Parameters
         ----------
@@ -771,10 +773,15 @@ class GapEncoder(BaseEstimator, TransformerMixin):
 
         Returns
         -------
-        GapEncoder
-            Fitted GapEncoder instance.
+        :class:`~dirty_cat.GapEncoder`
+            Fitted :class:`~dirty_cat.GapEncoder` instance (self).
         """
 
+        # Check that n_samples >= n_components
+        if len(X) < self.n_components:
+            raise ValueError(
+                f"n_samples={len(X)} should be >= n_components={self.n_components}. "
+            )
         # Copy parameter rho
         self.rho_ = self.rho
         # If X is a dataframe, store its column names
@@ -810,7 +817,7 @@ class GapEncoder(BaseEstimator, TransformerMixin):
         H : 2-d array, shape (n_samples, n_topics * n_features)
             Transformed input.
         """
-
+        check_is_fitted(self, "fitted_models_")
         # Check input data shape
         X = check_input(X)
         X = self._handle_missing(X)
@@ -873,7 +880,7 @@ class GapEncoder(BaseEstimator, TransformerMixin):
 
                 - if the input data was a dataframe, its column names are used,
                 - otherwise, 'col1', ..., 'colN' are used as prefixes.
-                
+
             Prefixes can be manually set by passing a list for col_names.
         n_labels : int, default=3
             The number of labels used to describe each topic.
diff --git a/dirty_cat/_minhash_encoder.py b/dirty_cat/_minhash_encoder.py
index 9518c02..4a56562 100644
--- a/dirty_cat/_minhash_encoder.py
+++ b/dirty_cat/_minhash_encoder.py
@@ -22,6 +22,7 @@ import numpy as np
 from joblib import Parallel, delayed, effective_n_jobs
 from sklearn.base import BaseEstimator, TransformerMixin
 from sklearn.utils import gen_even_slices, murmurhash3_32
+from sklearn.utils.validation import check_is_fitted
 
 from ._fast_hash import ngram_min_hash
 from ._string_distances import get_unique_ngrams
@@ -70,7 +71,7 @@ class MinHashEncoder(BaseEstimator, TransformerMixin):
     ----------
     hash_dict_ : LRUDict
         Computed hashes.
-    
+
     Examples
     --------
     >>> enc = MinHashEncoder(n_components=5)
@@ -139,7 +140,7 @@ class MinHashEncoder(BaseEstimator, TransformerMixin):
 
         Returns
         -------
-        ndarray of shape (n_components, )
+        :obj:`~numpy.ndarray` of shape (n_components, )
             The encoded string.
         """
         min_hashes = np.ones(self.n_components) * np.infty
@@ -168,7 +169,7 @@ class MinHashEncoder(BaseEstimator, TransformerMixin):
 
         Returns
         -------
-        ndarray of shape (n_components, )
+        :obj:`~numpy.ndarray` of shape (n_components, )
             The encoded string, using specified encoding scheme.
         """
         if self.minmax_hash:
@@ -189,21 +190,22 @@ class MinHashEncoder(BaseEstimator, TransformerMixin):
     def _compute_hash_batched(
         self, batch: Collection[str], hash_func: Callable[[str], np.ndarray]
     ):
-        """Function called to compute the hashes of a batch of strings.
+        """
+        Function called to compute the hashes of a batch of strings.
 
-        Check if the string is in the hash dictionary, if not, compute the hash using
-        the specified hashing function and add it to the dictionary.
+        Check if the string is in the hash dictionary, if not, compute the hash
+        using the specified hashing function and add it to the dictionary.
 
         Parameters
         ----------
-        batch : iterable of str
+        batch : collection of str
             The batch of strings to encode.
         hash_func : callable
             Hashing function to use on the string.
 
         Returns
         -------
-        np.array of shape (n_samples, n_components)
+        :obj:`~numpy.ndarray` of shape (n_samples, n_components)
             The encoded strings, using specified encoding scheme.
         """
         res = np.zeros((len(batch), self.n_components))
@@ -257,9 +259,10 @@ class MinHashEncoder(BaseEstimator, TransformerMixin):
 
         Returns
         -------
-        ndarray of shape (n_samples, n_components)
+        :obj:`~numpy.ndarray` of shape (n_samples, n_components)
             Transformed input.
         """
+        check_is_fitted(self, "hash_dict_")
         X = check_input(X)
         if self.minmax_hash:
             if self.n_components % 2 != 0:
@@ -316,8 +319,7 @@ class MinHashEncoder(BaseEstimator, TransformerMixin):
                 unique_x[idx_slice],
                 hash_func,
             )
-            for idx_slice in gen_even_slices(len(unique_x), 
-            n_jobs)
+            for idx_slice in gen_even_slices(len(unique_x), n_jobs)
         )
 
         # Match the hashes of the unique value to the original values
diff --git a/dirty_cat/_similarity_encoder.py b/dirty_cat/_similarity_encoder.py
index 6fc8ce4..83c6511 100644
--- a/dirty_cat/_similarity_encoder.py
+++ b/dirty_cat/_similarity_encoder.py
@@ -27,6 +27,7 @@ from sklearn.neighbors import NearestNeighbors
 from sklearn.preprocessing import OneHotEncoder
 from sklearn.utils import check_random_state
 from sklearn.utils.fixes import _object_dtype_isnan
+from sklearn.utils.validation import check_is_fitted
 
 from ._string_distances import get_ngram_count, preprocess
 from ._utils import parse_version
@@ -62,7 +63,7 @@ def _ngram_similarity_one_sample_inplace(
     i : str
         The index of x_count_vector in the csr count matrix
     ngram_range : tuple
-        n-grams to use for the decomposition, where `n_min <= n <= n_max`.
+        n-grams to use for the decomposition, where ``n_min <= n <= n_max``.
     """
     nonzero_idx = x_count_vector.indices
     nonzero_vals = x_count_vector.data
@@ -219,7 +220,7 @@ class SimilarityEncoder(OneHotEncoder):
            categorical variable
         - 'k-means' : Computes the K nearest neighbors of K-mean centroids
            in order to choose the prototype categories
-           
+
         The categories used can be found in the ``categories_`` attribute.
     dtype : number type, default np.float64
         Desired dtype of output.
@@ -359,7 +360,7 @@ class SimilarityEncoder(OneHotEncoder):
 
         Parameters
         ----------
-        prototypes : typing.List[str]
+        prototypes : list of str
             The list of values for a category variable.
 
         Returns
@@ -372,7 +373,7 @@ class SimilarityEncoder(OneHotEncoder):
 
     def fit(self, X, y=None) -> "SimilarityEncoder":
         """
-        Fit the SimilarityEncoder to X.
+        Fit the instance to X.
 
         Parameters
         ----------
@@ -383,8 +384,8 @@ class SimilarityEncoder(OneHotEncoder):
 
         Returns
         -------
-        SimilarityEncoder
-            The fitted SimilarityEncoder instance.
+        :class:`~dirty_cat.SimilarityEncoder`
+            The fitted :class:`~dirty_cat.SimilarityEncoder` instance (self).
         """
 
         if self.handle_missing not in ["error", ""]:
@@ -521,7 +522,7 @@ class SimilarityEncoder(OneHotEncoder):
         X_new : 2-d array, shape [n_samples, n_features_new]
             Transformed input.
         """
-
+        check_is_fitted(self, "categories_")
         if hasattr(X, "iloc") and X.isna().values.any():
             if self.handle_missing == "error":
                 raise ValueError(
@@ -586,18 +587,19 @@ class SimilarityEncoder(OneHotEncoder):
         """
         Fast computation of ngram similarity.
 
-        SimilarityEncoder.transform uses the count vectors of the vocabulary in
-        its computations. In ngram_similarity, these count vectors have to be
+        :func:`~dirty_cat.SimilarityEncoder.transform` uses the count vectors
+        of the vocabulary in its computations.
+        In `ngram_similarity`, these count vectors have to be
         re-computed each time, which can slow down the execution. In this
-        method, the count vectors are recovered from the
-        ``vocabulary_count_matrices`` attribute of the SimilarityEncoder,
+        method, the count vectors are recovered from
+        :attr:`~dirty_cat.SimilarityEncoder.vocabulary_count_matrices`,
         speeding up the execution.
 
         Parameters
         ----------
-        X: typing.Union[list, np.array]
+        X : list or :obj:`numpy.ndarray`
             Observations being transformed.
-        col_idx: int
+        col_idx : int
             The column index of X in the original feature matrix.
         """
         vectorizer = self.vectorizers_[col_idx]
diff --git a/dirty_cat/_super_vectorizer.py b/dirty_cat/_super_vectorizer.py
index f012c9a..3733916 100644
--- a/dirty_cat/_super_vectorizer.py
+++ b/dirty_cat/_super_vectorizer.py
@@ -15,6 +15,7 @@ from sklearn import __version__ as sklearn_version
 from sklearn.base import TransformerMixin, clone
 from sklearn.compose import ColumnTransformer
 from sklearn.preprocessing import OneHotEncoder
+from sklearn.utils.validation import check_is_fitted
 
 from dirty_cat import DatetimeEncoder, GapEncoder
 from dirty_cat._utils import parse_version
@@ -87,10 +88,10 @@ class SuperVectorizer(ColumnTransformer):
     """Easily transform a heterogeneous array to a numerical one.
 
     Easily transforms a heterogeneous data table
-    (such as a :class:`pandas.DataFrame`) to a numerical array for machine
+    (such as a :class:`~pandas.DataFrame`) to a numerical array for machine
     learning. For this it transforms each column depending on its data type.
     It provides a simplified interface for the
-    :class:`sklearn.compose.ColumnTransformer`; more documentation of
+    :class:`~sklearn.compose.ColumnTransformer`; more documentation of
     attributes and functions are available in its doc.
 
     .. versionadded:: 0.2.0
@@ -111,46 +112,52 @@ class SuperVectorizer(ColumnTransformer):
     low_card_cat_transformer : {"drop", "remainder", "passthrough"} or Transformer, optional
         Transformer used on categorical/string features with low cardinality
         (threshold is defined by `cardinality_threshold`).
-        Can either be a transformer object instance (e.g. `OneHotEncoder()`),
-        a `Pipeline` containing the preprocessing steps,
+        Can either be a transformer object instance
+        (e.g. :class:`~sklearn.preprocessing.OneHotEncoder`),
+        a :class:`~sklearn.pipeline.Pipeline` containing the preprocessing steps,
         'drop' for dropping the columns,
         'remainder' for applying `remainder`,
         'passthrough' to return the unencoded columns,
-        or None to use the default transformer (`OneHotEncoder(drop="if_binary")`).
+        or `None` to use the default transformer
+        (:class:`~sklearn.preprocessing.OneHotEncoder(drop="if_binary")`).
         Features classified under this category are imputed based on the
         strategy defined with `impute_missing`.
 
     high_card_cat_transformer : {"drop", "remainder", "passthrough"} or Transformer, optional
         Transformer used on categorical/string features with high cardinality
         (threshold is defined by `cardinality_threshold`).
-        Can either be a transformer object instance (e.g. `GapEncoder()`),
-        a `Pipeline` containing the preprocessing steps,
+        Can either be a transformer object instance
+        (e.g. :class:`~dirty_cat.GapEncoder`),
+        a :class:`~sklearn.pipeline.Pipeline` containing the preprocessing steps,
         'drop' for dropping the columns,
         'remainder' for applying `remainder`,
         'passthrough' to return the unencoded columns,
-        or None to use the default transformer (`GapEncoder(n_components=30)`).
+        or `None` to use the default transformer
+        (:class:`~dirty_cat.GapEncoder(n_components=30)`).
         Features classified under this category are imputed based on the
         strategy defined with `impute_missing`.
 
     numerical_transformer : {"drop", "remainder", "passthrough"} or Transformer, optional
         Transformer used on numerical features.
-        Can either be a transformer object instance (e.g. `StandardScaler()`),
-        a `Pipeline` containing the preprocessing steps,
+        Can either be a transformer object instance
+        (e.g. :class:`~sklearn.preprocessing.StandardScaler`),
+        a :class:`~sklearn.pipeline.Pipeline` containing the preprocessing steps,
         'drop' for dropping the columns,
         'remainder' for applying `remainder`,
         'passthrough' to return the unencoded columns,
-        or None to use the default transformer (here nothing, so 'passthrough').
+        or `None` to use the default transformer (here nothing, so 'passthrough').
         Features classified under this category are not imputed at all
         (regardless of `impute_missing`).
 
     datetime_transformer : {"drop", "remainder", "passthrough"} or Transformer, optional
         Transformer used on datetime features.
-        Can either be a transformer object instance (e.g. `DatetimeEncoder()`),
-        a `Pipeline` containing the preprocessing steps,
+        Can either be a transformer object instance
+        (e.g. :class:`~dirty_cat.DatetimeEncoder`),
+        a :class:`~sklearn.pipeline.Pipeline` containing the preprocessing steps,
         'drop' for dropping the columns,
         'remainder' for applying `remainder`,
         'passthrough' to return the unencoded columns,
-        or None to use the default transformer (`DatetimeEncoder()`).
+        or `None` to use the default transformer (:class:`~dirty_cat.DatetimeEncoder()`).
         Features classified under this category are not imputed at all
         (regardless of `impute_missing`).
 
@@ -168,18 +175,18 @@ class SuperVectorizer(ColumnTransformer):
         When imputed, missing values are replaced by the string 'missing'.
         As imputation logic for numerical features can be quite intricate,
         it is left to the user to manage.
-        See also attribute `imputed_columns_`.
+        See also attribute :attr:`~dirty_cat.SuperVectorizer.imputed_columns_`.
 
     remainder : {"drop", "passthrough"} or Transformer, optional, default='drop'
         By default, only the specified columns in `transformers` are
         transformed and combined in the output, and the non-specified
-        columns are dropped. (default ``'drop'``).
-        By specifying ``remainder='passthrough'``, all remaining columns that
+        columns are dropped. (default 'drop').
+        By specifying `remainder='passthrough'`, all remaining columns that
         were not specified in `transformers` will be automatically passed
         through. This subset of columns is concatenated with the output of
         the transformers.
-        By setting ``remainder`` to be an estimator, the remaining
-        non-specified columns will use the ``remainder`` estimator. The
+        By setting `remainder` to be an estimator, the remaining
+        non-specified columns will use the `remainder` estimator. The
         estimator must support :term:`fit` and :term:`transform`.
         Note that using this feature requires that the DataFrame columns
         input at :term:`fit` and :term:`transform` have identical order.
@@ -187,7 +194,7 @@ class SuperVectorizer(ColumnTransformer):
     sparse_threshold : float, optional, default=0.3
         If the output of the different transformers contains sparse matrices,
         these will be stacked as a sparse matrix if the overall density is
-        lower than this value. Use sparse_threshold=0 to always return dense.
+        lower than this value. Use `sparse_threshold=0` to always return dense.
         When the transformed output consists of all dense data, the stacked
         result will be dense, and this keyword will be ignored.
 
@@ -217,7 +224,7 @@ class SuperVectorizer(ColumnTransformer):
         If there are remaining columns, the final element is a tuple of the
         form:
         ('remainder', transformer, remaining_columns) corresponding to the
-        ``remainder`` parameter. If there are remaining columns, then
+        `remainder` parameter. If there are remaining columns, then
         ``len(transformers_)==len(transformers)+1``, otherwise
         ``len(transformers_)==len(transformers)``.
 
@@ -375,7 +382,7 @@ class SuperVectorizer(ColumnTransformer):
                 # Some numerical dtypes like Int64 or Float64 only support
                 # pd.NA, so they must be converted to np.float64 before.
                 if pd.api.types.is_numeric_dtype(X[col]):
-                    X.loc[:, col] = X[col].astype(np.float64)
+                    X[col] = X[col].astype(np.float64)
                 X[col].fillna(value=np.nan, inplace=True)
 
         # Convert to the best possible data type
@@ -384,12 +391,12 @@ class SuperVectorizer(ColumnTransformer):
             if not pd.api.types.is_datetime64_any_dtype(X[col]):
                 # we don't want to cast datetime64
                 try:
-                    X.loc[:, col] = pd.to_numeric(X[col], errors="raise")
+                    X[col] = pd.to_numeric(X[col], errors="raise")
                 except (ValueError, TypeError):
                     # Only try to convert to datetime
                     # if the variable isn't numeric.
                     try:
-                        X.loc[:, col] = pd.to_datetime(
+                        X[col] = pd.to_datetime(
                             X[col], errors="raise", infer_datetime_format=True
                         )
                     except (ValueError, TypeError):
@@ -398,7 +405,7 @@ class SuperVectorizer(ColumnTransformer):
             # for earlier versions of sklearn. FIXME: which ?
             if issubclass(X[col].dtype.__class__, ExtensionDtype):
                 try:
-                    X.loc[:, col] = X[col].astype(X[col].dtype.type, errors="ignore")
+                    X[col] = X[col].astype(X[col].dtype.type, errors="ignore")
                 except (TypeError, ValueError):
                     pass
             self.types_.update({col: X[col].dtype})
@@ -412,15 +419,15 @@ class SuperVectorizer(ColumnTransformer):
         Does the same thing as `_auto_cast`, but applies learnt info.
         """
         for col in X.columns:
-            X.loc[:, col] = _replace_false_missing(X[col])
+            X[col] = _replace_false_missing(X[col])
             if _has_missing_values(X[col]):
                 if pd.api.types.is_numeric_dtype(X[col]):
-                    X.loc[:, col] = X[col].astype(np.float64)
+                    X[col] = X[col].astype(np.float64)
                 X[col].fillna(value=np.nan, inplace=True)
         for col in self.imputed_columns_:
-            X.loc[:, col] = _replace_missing_in_cat_col(X[col])
+            X[col] = _replace_missing_in_cat_col(X[col])
         for col, dtype in self.types_.items():
-            X.loc[:, col] = X[col].astype(dtype)
+            X[col] = X[col].astype(dtype)
         return X
 
     def fit_transform(self, X, y=None):
@@ -541,7 +548,7 @@ class SuperVectorizer(ColumnTransformer):
                     for col in X.columns:
                         # Only impute categorical columns
                         if col in categorical_columns:
-                            X.loc[:, col] = _replace_missing_in_cat_col(X[col])
+                            X[col] = _replace_missing_in_cat_col(X[col])
                             self.imputed_columns_.append(col)
 
                 elif self.impute_missing == "auto":
@@ -557,7 +564,7 @@ class SuperVectorizer(ColumnTransformer):
                             for col in cols:
                                 # Only impute categorical columns
                                 if col in categorical_columns:
-                                    X.loc[:, col] = _replace_missing_in_cat_col(X[col])
+                                    X[col] = _replace_missing_in_cat_col(X[col])
                                     self.imputed_columns_.append(col)
 
         # If there was missing values imputation, we cast the DataFrame again,
@@ -601,6 +608,7 @@ class SuperVectorizer(ColumnTransformer):
             any result is a sparse matrix, everything will be converted to
             sparse matrices.
         """
+        check_is_fitted(self, attributes=["columns_"])
         if X.shape[1] != len(self.columns_):
             raise ValueError(
                 "Passed array does not match column count of "
diff --git a/dirty_cat/_target_encoder.py b/dirty_cat/_target_encoder.py
index 73fdeb1..e87e411 100644
--- a/dirty_cat/_target_encoder.py
+++ b/dirty_cat/_target_encoder.py
@@ -31,8 +31,8 @@ class TargetEncoder(BaseEstimator, TransformerMixin):
         Categories (unique values) per feature:
 
         - 'auto' : Determine categories automatically from the training data.
-        - list : ``categories[i]`` holds the categories expected in the i-th 
-          column. The passed categories must be sorted and should not mix 
+        - list : ``categories[i]`` holds the categories expected in the i-th
+          column. The passed categories must be sorted and should not mix
           strings and numeric values.
 
         The categories used can be found in the ``categories_`` attribute.
@@ -44,8 +44,8 @@ class TargetEncoder(BaseEstimator, TransformerMixin):
         Whether to raise an error or ignore if a unknown categorical feature is
         present during transform (default is to raise). When this parameter
         is set to 'ignore' and an unknown category is encountered during
-        transform, the resulting one-hot encoded columns for this feature
-        will be all zeros.
+        transform, the encoded columns for this feature
+        will be assigned the prior mean of the target variable.
     handle_missing : typing.Literal["error", ""], default=""
         Whether to raise an error or impute with blank string '' if missing
         values (NaN) are present during fit (default is to impute).
@@ -71,7 +71,7 @@ class TargetEncoder(BaseEstimator, TransformerMixin):
     --------
     >>> enc = TargetEncoder(handle_unknown='ignore')
     >>> X = [['male'], ['Male'], ['Female'], ['male'], ['Female']]
-    >>> y = np.ndarray([1, 2, 3, 4, 5])
+    >>> y = np.array([1, 2, 3, 4, 5])
 
     >>> enc.fit(X, y)
     TargetEncoder(handle_unknown='ignore')
@@ -81,16 +81,19 @@ class TargetEncoder(BaseEstimator, TransformerMixin):
     >>> enc.categories_
     [array(['Female', 'Male', 'male'], dtype='<U6')]
 
-    We look at the encoded categories :
+    We will encode the following categories, of which the first two are unknown :
+
+    >>> X2 = [['MALE'], ['FEMALE'], ['Female'], ['male'], ['Female']]
 
-    >>> enc.transform(X)
-    array([[3.44444444],
-        [2.11111111],
-        [3.61538462],
-        [3.44444444],
-        [3.61538462]])
+    >>> enc.transform(X2)
+    array([[3.        ],
+        [3.        ],
+        [3.54545455],
+        [2.72727273],
+        [3.54545455]])
 
     As expected, they were encoded according to their influence on y.
+    The unknown categories were assigned the mean of the target variable.
 
     """
 
@@ -121,7 +124,7 @@ class TargetEncoder(BaseEstimator, TransformerMixin):
 
     def fit(self, X, y) -> "TargetEncoder":
         """
-        Fit the TargetEncoder to X.
+        Fit the instance to X.
 
         Parameters
         ----------
@@ -132,8 +135,8 @@ class TargetEncoder(BaseEstimator, TransformerMixin):
 
         Returns
         -------
-        TargetEncoder
-            Fitted TargetEncoder instance.
+        :class:`~dirty_cat.TargetEncoder`
+            Fitted :class:`~dirty_cat.TargetEncoder` instance (self).
         """
         X = check_input(X)
         self.n_features_in_ = X.shape[1]
@@ -224,7 +227,7 @@ class TargetEncoder(BaseEstimator, TransformerMixin):
 
         Returns
         -------
-        2-d np.ndarray
+        2-d :class:`~numpy.ndarray`
             Transformed input.
         """
         check_is_fitted(self, attributes=["n_features_in_"])
diff --git a/dirty_cat/datasets/__init__.py b/dirty_cat/datasets/__init__.py
index ff8ccc4..0dfc30f 100644
--- a/dirty_cat/datasets/__init__.py
+++ b/dirty_cat/datasets/__init__.py
@@ -12,6 +12,12 @@ from ._fetching import (
     get_data_dir,
 )
 
+
+from ._generating import (
+    make_deduplication_data,
+)
+
+
 __all__ = [
     "DatasetAll",
     "DatasetInfoOnly",
@@ -24,4 +30,5 @@ __all__ = [
     "fetch_traffic_violations",
     "fetch_world_bank_indicator",
     "get_data_dir",
+    "make_deduplication_data",
 ]
diff --git a/dirty_cat/datasets/_generating.py b/dirty_cat/datasets/_generating.py
new file mode 100644
index 0000000..d379a33
--- /dev/null
+++ b/dirty_cat/datasets/_generating.py
@@ -0,0 +1,60 @@
+"""
+Functions that generate example data.
+
+"""
+from __future__ import annotations
+
+from typing import List, Optional, Union
+import string
+
+import numpy as np
+
+from sklearn.utils import check_random_state
+
+def make_deduplication_data(
+    examples: List[str],
+    entries_per_example: List[int],
+    prob_mistake_per_letter: float,
+    random_state: Optional[Union[int, np.random.RandomState]] = None,
+) -> List[str]:
+    """Duplicates examples with spelling mistakes.
+    Characters are misspelled with probability `prob_mistake_per_letter`.
+
+    Parameters
+    ----------
+    examples : List[str]
+        examples to duplicate
+    entries_per_example : List[int]
+        number of duplications per example
+    prob_mistake_per_letter : float in [0, 1]
+        probability of misspelling a character in duplications
+    random_state : int, RandomState instance or None, default=None
+        Determines random number generation for dataset noise. Pass an int
+        for reproducible output across multiple function calls.
+
+    Returns
+    -------
+    List[str]
+        list of duplicated examples with spelling mistakes
+    """
+    rng = check_random_state(random_state)
+
+    data = []
+    for example, n_ex in zip(examples, entries_per_example):
+        len_ex = len(example)
+        # generate a 2D array of chars of size (n_ex, len_ex)
+        str_as_list = np.array([list(example)] * n_ex)
+        # randomly choose which characters are misspelled
+        idxes = np.where(
+            rng.random(len(example[0]) * n_ex) < prob_mistake_per_letter
+        )[0]
+        # and randomly pick with which character to replace
+        replacements = [
+            string.ascii_lowercase[i]
+            for i in np.random.choice(np.arange(26), len(idxes)).astype(int)
+        ]
+        # introduce spelling mistakes at right examples and char locations per example
+        str_as_list[idxes // len_ex, idxes % len_ex] = replacements
+        # go back to 1d array of strings
+        data.append(np.ascontiguousarray(str_as_list).view(f"U{len_ex}").ravel())
+    return np.concatenate(data).tolist()
diff --git a/doc/_static/custom.css b/doc/_static/custom.css
index b161abf..63a95cb 100644
--- a/doc/_static/custom.css
+++ b/doc/_static/custom.css
@@ -72,6 +72,15 @@ div.flex-content p {
     margin-right: 1ex;
 }
 
+div.flex-content ul a span.std {
+    float: right;
+   }
+
+/* Avoid an overlap of the float with the line below, eg on narrow screens */
+div.flex-content ul li {
+    clear: both;
+}
+
 span.container-title {
     font-weight: bold;
     color: black;
diff --git a/doc/index.rst b/doc/index.rst
index 282059b..dd6db8f 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -1,7 +1,7 @@
 
-===============================================
-dirty_cat: machine learning on dirty categories
-===============================================
+=================================================
+dirty_cat: machine learning with dirty categories
+=================================================
 
 .. toctree::
    :maxdepth: 2
@@ -10,10 +10,12 @@ dirty_cat: machine learning on dirty categories
 
 .. container:: larger-container
 
-    `dirty_cat` facilitates machine-learning on non-curated categories:
-    **robust to morphological variants**, such as typos.
-    See `the first example notebook <https://dirty-cat.github.io/stable/auto_examples/01_dirty_categories.html>`_
-    for a more in-depth look into the problem and the library's capabilities.
+    `dirty_cat` facilitates machine-learning with non-curated categories:
+    **robust to morphological variants**, such as typos. See
+    :ref:`examples <usage_examples>`, such as `the first one
+    <https://dirty-cat.github.io/stable/auto_examples/01_dirty_categories.html>`_,
+    for an introduction to problems of dirty categories or misspelled
+    entities.
 
 |
 
@@ -40,11 +42,38 @@ pipeline you can tweak.
 
 * :class:`GapEncoder`, scalable and interpretable, where each encoding
   dimension corresponds to a topic that summarizes substrings captured.
+  :ref:`Example <example_gap_encoder>`
 
 * :class:`SimilarityEncoder`, a simple modification of one-hot encoding
   to capture the strings.
+  :ref:`Example <example_similarity_encoder>`
 
-* :class:`MinHashEncoder`, very scalable
+* :class:`MinHashEncoder`, very scalable.
+  :ref:`Example <example_minhash_encoder>`
+
+.. raw:: html
+
+    </div>
+    <div class="flex-content">
+    <span class="container-title">Joining tables on non-normalized categories</span>
+
+* :func:`fuzzy_join`, approximate matching using morphological similarity.
+  :ref:`Example <example_fuzzy_join>`
+
+* :class:`FeatureAugmenter`, a scikit-learn transformer for joining multiple tables.
+  :ref:`Example <example_feature_augmenter>`
+
+.. raw:: html
+
+    </div>
+    <div class="flex-content">
+    <span class="container-title">Deduplicating dirty categories</span>
+
+:func:`deduplicate`, merging categories of similar morphology (spelling).
+
+.. rst-class:: centered
+
+     :ref:`An example <example_deduplication>`
 
 .. raw:: html
 
@@ -61,7 +90,18 @@ pipeline you can tweak.
 
     :Installing: ``$ pip install --user --upgrade dirty_cat``
 
+.. _usage_examples:
 
+Usage examples
+==============
+
+.. container:: larger-container
+
+  .. include:: auto_examples/index.rst
+    :start-line: 5
+    :end-before: .. rst-class:: sphx-glr-signature
+
+|
 
 .. raw:: html
 
@@ -78,17 +118,6 @@ pipeline you can tweak.
     </div>
 
 
-Usage examples
-==============
-
-.. container:: larger-container
-
-  .. include:: auto_examples/index.rst
-    :start-line: 5
-    :end-before: .. rst-class:: sphx-glr-signature
-
-|
-
 For a detailed description of the problem of encoding dirty categorical data,
 see `Similarity encoding for learning with dirty categorical variables
 <https://hal.inria.fr/hal-01806175>`_ [1]_ and `Encoding high-cardinality
@@ -141,8 +170,26 @@ Joining tables
 
    fuzzy_join
 
-Data download
--------------
+.. autosummary::
+   :toctree: generated/
+   :template: class.rst
+   :nosignatures:
+
+   FeatureAugmenter
+
+Deduplication: merging variants of the same entry
+---------------------------------------------------
+
+.. autosummary::
+   :toctree: generated/
+   :template: function.rst
+   :nosignatures:
+
+   deduplicate
+
+
+Data download and generation
+----------------------------
 
 .. autosummary::
    :toctree: generated/
@@ -158,6 +205,7 @@ Data download
    datasets.fetch_drug_directory
    datasets.fetch_world_bank_indicator
    datasets.get_data_dir
+   datasets.make_deduplication_data
 
 About
 =====
diff --git a/examples/01_dirty_categories.py b/examples/01_dirty_categories.py
index d3bcd69..e7ef704 100644
--- a/examples/01_dirty_categories.py
+++ b/examples/01_dirty_categories.py
@@ -153,6 +153,8 @@ import numpy as np
 np.unique(y)
 
 ###############################################################################
+# .. _example_minhash_encoder:
+#
 # We will now experiment with encoders specially made for handling
 # dirty columns:
 
diff --git a/examples/02_investigating_dirty_categories.py b/examples/02_investigating_dirty_categories.py
index f22f64a..96bf957 100644
--- a/examples/02_investigating_dirty_categories.py
+++ b/examples/02_investigating_dirty_categories.py
@@ -67,6 +67,8 @@ values = data[["employee_position_title", "gender"]]
 values.insert(0, "current_annual_salary", employee_salaries.y)
 
 ###############################################################################
+# .. _example_similarity_encoder:
+#
 # String similarity between entries
 # ---------------------------------
 #
@@ -174,6 +176,8 @@ print(X_dirty.head(), end="\n\n")
 print(f"Number of dirty entries = {len(X_dirty)}")
 
 ###############################################################################
+# .. _example_gap_encoder:
+#
 # Encoding dirty job titles
 # .........................
 #
diff --git a/examples/05_joining_tables_with_FuzzyJoin.py b/examples/05_fuzzy_joining_and_FeatureAugmenter.py
old mode 100755
new mode 100644
similarity index 74%
rename from examples/05_joining_tables_with_FuzzyJoin.py
rename to examples/05_fuzzy_joining_and_FeatureAugmenter.py
index 6f1d561..d7eefad
--- a/examples/05_joining_tables_with_FuzzyJoin.py
+++ b/examples/05_fuzzy_joining_and_FeatureAugmenter.py
@@ -1,6 +1,6 @@
 """
-Merging dirty tables: fuzzy join
-================================
+Fuzzy joining dirty tables and the FeatureAugmenter
+=====================================================
 
 Here we show how to combine data from different sources,
 with a vocabulary not well normalized.
@@ -8,16 +8,22 @@ with a vocabulary not well normalized.
 Joining is difficult: one entry on one side does not have
 an exact match on the other side.
 
-In this example, the |fj| function allows us to join
-tables without cleaning the data by taking into account the
-label variations.
+The |fj| function enables to join tables without cleaning the data by
+accounting for the label variations.
 
 To illustrate, we will join data from the `2022 World Happiness Report <https://worldhappiness.report/>`_.
 with tables provided in `the World Bank open data platform <https://data.worldbank.org/>`_
-in order to create a satisfying first prediction model.
+in order to create a first prediction model.
+
+Moreover, the |fa| is a scikit-learn Transformer that makes it easy to
+use such fuzzy joining multiple tables to bring in information in a
+machine-learning pipeline. In particular, it enables tuning parameters of
+|fj| to find the matches that maximize prediction accuracy.
 
 
 .. |fj| replace:: :func:`~dirty_cat.fuzzy_join`
+
+.. |fa| replace:: :func:`~dirty_cat.FeatureAugmenter`
 """
 
 ###############################################################################
@@ -104,12 +110,19 @@ gdppc.sort_values(by="Country Name").tail(7)
 #
 
 ###############################################################################
+# .. _example_fuzzy_join:
+#
 # 1. Joining GDP per capita table
 # ...............................
 #
 # To join them with dirty_cat, we only need to do the following:
 from dirty_cat import fuzzy_join
 
+# We will ignore the warnings:
+import warnings
+
+warnings.filterwarnings("ignore")
+
 df1 = fuzzy_join(
     df,  # our table to join
     gdppc,  # the table to join with
@@ -269,10 +282,6 @@ plt.show()
 ###############################################################################
 # 3. Joining legal rights strength table
 # ......................................
-# .. topic:: Note:
-#
-#    Here, we use the `keep='left'` option to keep only the left key matching
-#    column, so as not to have unnecessary overlaping column with country names.
 #
 # And the table with a measure of legal rights strength in the country:
 df3 = fuzzy_join(
@@ -341,10 +350,7 @@ cv_results_t = cross_validate(hgdb, X, y, cv=cv, scoring="r2")
 
 cv_r2_t = cv_results_t["test_score"]
 
-print(
-    f"Mean R2 score with {len(X.columns) - 2} feature columns is"
-    f" {cv_r2_t.mean():.2f} +- {cv_r2_t.std():.2f}"
-)
+print(f"Mean R2 score is {cv_r2_t.mean():.2f} +- {cv_r2_t.std():.2f}")
 
 #################################################################
 # We have a satisfying first result: an R2 of 0.66!
@@ -358,3 +364,100 @@ print(
 #
 # Now up to you, try improving our model by adding information into it and
 # beating our result!
+
+#######################################################################
+# Using the |fa| to fuzzy join multiple tables
+# --------------------------------------------
+# A faster way to merge different tables from the World Bank
+# to `X` is to use the |fa|.
+#
+# The |fa| is a transformer that can easily chain joins of tables on
+# a main table.
+
+#######################################################################
+# .. _example_feature_augmenter:
+#
+# Instantiating the transformer
+# .............................
+
+y = df["Happiness score"]
+#######################################################################
+# We gather the auxilliary tables into a
+# list of (tables, keys) for the `tables` parameter.
+# An instance of the transformer with the necessary information is:
+from dirty_cat import FeatureAugmenter
+
+fa = FeatureAugmenter(
+    tables=[
+        (gdppc, "Country Name"),
+        (life_exp, "Country Name"),
+        (legal_rights, "Country Name"),
+    ],
+    main_key="Country",
+)
+
+#################################################################
+# Fitting and transforming into the final table
+# .............................................
+# To get our final joined table we will fit and transform the main table (df)
+# with our create instance of the |fa|:
+df_final = fa.fit_transform(df)
+
+df_final.head(10)
+
+##########################################################################
+# And that's it! As previously, we now have a big table
+# ready for machine learning.
+# Let's create our machine learning pipeline:
+from sklearn.pipeline import make_pipeline
+from sklearn.compose import make_column_transformer
+
+# We include only the columns that will be pertinent for our regression:
+encoder = make_column_transformer(
+    (
+        "passthrough",
+        [
+            "GDP per capita (current US$)",
+            "Life expectancy at birth, total (years)",
+            "Strength of legal rights index (0=weak to 12=strong)",
+        ],
+    ),
+    remainder="drop",
+)
+
+pipeline = make_pipeline(fa, encoder, HistGradientBoostingRegressor())
+
+##########################################################################
+# And the best part is that we are now able to evaluate the paramaters of the |fj|.
+# For instance, the ``match_score`` was manually picked and can now be
+# introduced into a grid search:
+
+from sklearn.model_selection import GridSearchCV
+
+# We will test four possible values of match_score:
+params = {"featureaugmenter__match_score": [0.2, 0.3, 0.4, 0.5]}
+
+grid = GridSearchCV(pipeline, param_grid=params)
+grid.fit(df, y)
+
+print(grid.best_params_)
+##########################################################################
+# The grid searching gave us the best value of 0.5 for the parameter
+# ``match_score``. Let's use this value in our regression:
+#
+
+print(f"Mean R2 score with pipeline is {grid.score(df, y):.2f}")
+
+##########################################################################
+#
+# .. topic:: Note:
+#
+#    Here, ``grid.score()`` takes directly the best model
+#    (with ``match_score=0.5``) that was found during the grid search.
+#    Thus, it is equivalent to fixing the ``match_score`` to 0.5 and
+#    refitting the pipeline on the data.
+#
+#
+# Great, by evaluating the correct ``match_score`` we improved our
+# results significantly!
+#
diff --git a/examples/06_deduplication.py b/examples/06_deduplication.py
new file mode 100644
index 0000000..02aa10a
--- /dev/null
+++ b/examples/06_deduplication.py
@@ -0,0 +1,175 @@
+"""
+Deduplicating misspelled categories with deduplicate
+====================================================
+
+Real world datasets often come with slight misspellings in the category
+names, for instance if the category is manually input. Such misspellings
+break many data-analyses steps that require exact matching, such as a
+'GROUP BY'.
+
+Merging the multiple variants of the same category or entity is known as
+*deduplication*. It is performed by the |dd| function.
+
+Deduplication relies on *unsupervised learning*, to find structure in
+data without providing explicit labels/categories of the data a-priori.
+Specifically clustering of the distance between strings can be used to
+find clusters of strings that are similar to each other (e.g. differ only
+by a misspelling) and hence gives us an easy tool to flag potentially
+misspelled category names in an unsupervised manner.
+
+
+.. |dd| replace:: :func:`~dirty_cat.deduplicate`
+"""
+
+###############################################################################
+# An example dataset
+# -------------------
+#
+# Imagine the following example:
+# As a data scientist, our job is to analyze the data from a hospital ward.
+# We notice that most of the cases involve the prescription of one of three different medications:
+#  "Contrivan", "Genericon", or "Zipholan".
+# However, data entry is manual and - either because the prescribing doctor's handwriting
+# was hard to decipher, or due to mistakes during data input - there are multiple
+# spelling mistakes for these three medications.
+#
+# Let's generate some example data that demonstrate this.
+
+
+import numpy as np
+from dirty_cat.datasets import make_deduplication_data
+
+# our three medication names
+medications = ["Contrivan", "Genericon", "Zipholan"]
+entries_per_medications = [500, 100, 1500]
+
+# 5% probability of a typo per letter
+prob_mistake_per_letter = 0.05
+
+duplicated_names = make_deduplication_data(
+    medications,
+    entries_per_medications,
+    prob_mistake_per_letter,
+    random_state=42,  # set seed for reproducibility
+)
+# we extract the unique medication names in the data & how often they appear
+unique_examples, counts = np.unique(duplicated_names, return_counts=True)
+# and build a series out of them
+import pandas as pd
+
+ex_series = pd.Series(counts, index=unique_examples)
+
+# This is our data:
+ex_series.head()
+
+###############################################################################
+# Visualize the data
+# ------------------
+
+import matplotlib.pyplot as plt
+
+ex_series.plot.barh(figsize=(10, 15))
+plt.xlabel("Medication name")
+plt.ylabel("Counts")
+
+###############################################################################
+# We can now see clearly the structure of the data: The three original medications
+# are the most common ones, however there are many spelling mistakes and hence
+# many slight variations of the names of the original medications.
+#
+# The idea is to use the fact that the string-distance of each misspelled medication
+# name will be closest to either the correctly or incorrectly spelled orginal
+# medication name - and therefore form clusters.
+
+###############################################################################
+# We can visualize the pair-wise distance between all medication names
+# --------------------------------------------------------------------
+#
+# Below we use a heatmap to visualize the pairwise-distance between medication names.
+# A darker color means that two medication names are closer together (i.e. more similar),
+# a lighter color means a larger distance. We can see that we are dealing with three
+# clusters - the original medication names and their misspellings that cluster around them.
+
+from dirty_cat import compute_ngram_distance
+from scipy.spatial.distance import squareform
+
+ngram_distances = compute_ngram_distance(unique_examples)
+square_distances = squareform(ngram_distances)
+
+import seaborn as sns
+
+fig, axes = plt.subplots(1, 1, figsize=(12, 12))
+sns.heatmap(
+    square_distances, yticklabels=ex_series.index, xticklabels=ex_series.index, ax=axes
+)
+
+###############################################################################
+# .. _example_deduplication:
+#
+# Deduplication: suggest corrections of misspelled names
+# ------------------------------------------------------
+#
+# The |dd| function uses clustering based on
+# string similarities to group duplicated names
+#
+# The number of clusters will need some adjustment depending on the data you have.
+# If no fixed number of clusters is given, |dd| tries to set it automatically
+# via the `silhouette score <https://scikit-learn.org/stable/modules/clustering.html#silhouette-coefficient>`_.
+
+from dirty_cat import deduplicate
+
+deduplicated_data = deduplicate(duplicated_names)
+
+###############################################################################
+# We can visualize the distribution of categories in the deduplicated data:
+
+deduplicated_unique_examples, deduplicated_counts = np.unique(
+    deduplicated_data, return_counts=True
+)
+deduplicated_series = pd.Series(deduplicated_counts, index=deduplicated_unique_examples)
+
+deduplicated_series.plot.barh(figsize=(10, 15))
+plt.xlabel("Medication name")
+plt.ylabel("Counts")
+
+###############################################################################
+# In this example we can correct all spelling mistakes by using the ideal number
+# of clusters as determined by the silhouette score.
+#
+# However, often the translation/deduplication won't be perfect and will require some tweaks.
+# In this case, we can construct and update a translation table based on the data
+# returned by |dd|.
+# It consists of the (potentially) misspelled category names as indices and the
+# (potentially) correct categories as values.
+
+# create a table that maps original -> corrected categories
+translation_table = pd.Series(deduplicated_data, index=duplicated_names)
+
+# remove duplicates in the original data
+translation_table = translation_table[~translation_table.index.duplicated(keep="first")]
+
+###############################################################################
+# Since the number of correct spellings will likely be much smaller than the
+# number of original categories, we can print the estimated cluster and their
+# most common exemplars (the guessed correct spelling):
+
+
+def print_corrections(spell_correct):
+    correct = np.unique(spell_correct.values)
+    for c in correct:
+        print(
+            f"Guessed correct spelling: {c!r} for "
+            f"{spell_correct[spell_correct==c].index.values}"
+        )
+
+
+print_corrections(translation_table)
+
+###############################################################################
+# In case we want to adapt the translation table post-hoc we can easily
+# modified it manually and apply it, for instance modifying the
+# correspondance for the last entry as such:
+
+translation_table.iloc[-1] = "Completely new category"
+new_deduplicated_names = translation_table[duplicated_names]
+assert (new_deduplicated_names == "Completely new category").sum() > 0
diff --git a/examples/README.txt b/examples/README.txt
index 8d0935a..822da6f 100644
--- a/examples/README.txt
+++ b/examples/README.txt
@@ -1,2 +1,2 @@
 Using dirty_cat
-===============
\ No newline at end of file
+===============
diff --git a/setup.cfg b/setup.cfg
index 88b6413..2ec8020 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -23,6 +23,7 @@ project_urls =
     Source = https://github.com/dirty-cat/dirty_cat
 
 [options]
+include_package_data = True
 packages = find:
 install_requires =
     scikit-learn>=0.23.0


Test patch:
diff --git a/dirty_cat/datasets/tests/test_fetching.py b/dirty_cat/datasets/tests/test_fetching.py
index 2c8605d..561dcaa 100644
--- a/dirty_cat/datasets/tests/test_fetching.py
+++ b/dirty_cat/datasets/tests/test_fetching.py
@@ -453,3 +453,5 @@ def test_fetch_world_bank_indicator():
 
     finally:
         shutil.rmtree(path=str(test_data_dir), ignore_errors=True)
+
+
diff --git a/dirty_cat/datasets/tests/test_generating.py b/dirty_cat/datasets/tests/test_generating.py
new file mode 100644
index 0000000..d1d399f
--- /dev/null
+++ b/dirty_cat/datasets/tests/test_generating.py
@@ -0,0 +1,29 @@
+"""
+Tests generating.py (synthetic dataset generation).
+"""
+
+import numpy as np
+
+from dirty_cat.datasets._generating import make_deduplication_data
+
+
+
+def test_make_deduplication_data():
+
+    np.random.seed(123)
+    assert make_deduplication_data(["abc", "cba", "test1"], [3, 2, 1], 0.3) == [
+        "agr",
+        "abc",
+        "abc",
+        "cba",
+        "cba",
+        "test1",
+    ]
+    assert make_deduplication_data(["abc", "cba", "test1"], [1, 2, 3], 0.8) == [
+        "pbc",
+        "pza",
+        "cba",
+        "erxt1",
+        "test1",
+        "test1",
+    ]
diff --git a/dirty_cat/tests/test_datetime_encoder.py b/dirty_cat/tests/test_datetime_encoder.py
index 22705ea..124ae9d 100644
--- a/dirty_cat/tests/test_datetime_encoder.py
+++ b/dirty_cat/tests/test_datetime_encoder.py
@@ -1,5 +1,7 @@
+import pytest
 import numpy as np
 import pandas as pd
+from sklearn.exceptions import NotFittedError
 
 from dirty_cat._datetime_encoder import DatetimeEncoder
 
@@ -340,3 +342,14 @@ def test_transform():
     X = get_constant_date_array()
     enc = DatetimeEncoder(add_day_of_the_week=True)
     assert enc.fit_transform(X).shape[1] == 0
+
+def test_check_fitted_datetime_encoder():
+    """Test that calling transform before fit raises an error"""
+    X = get_datetime_array()[:, 0].reshape(-1, 1)
+    enc = DatetimeEncoder(add_day_of_the_week=True)
+    with pytest.raises(NotFittedError):
+        enc.transform(X)
+    
+    # Check that it works after fit
+    enc.fit(X)
+    enc.transform(X)
diff --git a/dirty_cat/tests/test_deduplicate.py b/dirty_cat/tests/test_deduplicate.py
new file mode 100644
index 0000000..572996a
--- /dev/null
+++ b/dirty_cat/tests/test_deduplicate.py
@@ -0,0 +1,124 @@
+import string
+from typing import List
+
+import numpy as np
+import pandas as pd
+import pytest
+from scipy.cluster.hierarchy import linkage
+from scipy.spatial.distance import squareform
+
+from dirty_cat._deduplicate import (
+    _create_spelling_correction,
+    _guess_clusters,
+    compute_ngram_distance,
+    deduplicate,
+)
+
+
+def generate_example_data(examples, entries_per_example, prob_mistake_per_letter, rng):
+    """Helper function to generate data consisting of multiple entries per example.
+    Characters are misspelled with probability `prob_mistake_per_letter`"""
+
+    data = []
+    for example, n_ex in zip(examples, entries_per_example):
+        len_ex = len(example)
+        # generate a 2D array of chars of size (n_ex, len_ex)
+        str_as_list = np.array([list(example)] * n_ex)
+        # randomly choose which characters are misspelled
+        idxes = np.where(
+            rng.random_sample(len(example[0]) * n_ex) < prob_mistake_per_letter
+        )[0]
+        # and randomly pick with which character to replace
+        replacements = [
+            string.ascii_lowercase[i]
+            for i in rng.choice(np.arange(26), len(idxes)).astype(int)
+        ]
+        # introduce spelling mistakes at right examples and char locations per example
+        str_as_list[idxes // len_ex, idxes % len_ex] = replacements
+        # go back to 1d array of strings
+        data.append(np.ascontiguousarray(str_as_list).view(f"U{len_ex}").ravel())
+    return np.concatenate(data)
+
+
+@pytest.mark.parametrize(
+    "entries_per_category, prob_mistake_per_letter",
+    [[[500, 100, 1500], 0.05], [[100, 100], 0.02], [[200, 50, 30, 200, 800], 0.01]],
+)
+def test_deduplicate(
+    entries_per_category: List[int],
+    prob_mistake_per_letter: float,
+    seed: int = 123,
+) -> None:
+    rng = np.random.RandomState(seed)
+
+    # hard coded to fix ground truth string similarities
+    clean_categories = [
+        "Example Category",
+        "Generic",
+        "Random Word",
+        "Pretty similar category",
+        "Final cluster",
+    ]
+    n_clusters = len(entries_per_category)
+    clean_categories = clean_categories[:n_clusters]
+    data = generate_example_data(
+        clean_categories, entries_per_category, prob_mistake_per_letter, rng
+    )
+    deduplicated_data = np.array(deduplicate(data, n_clusters=None))
+    assert deduplicated_data.shape[0] == data.shape[0]
+    recovered_categories = np.unique(deduplicated_data)
+    assert recovered_categories.shape[0] == n_clusters
+    assert np.isin(clean_categories, recovered_categories).all()
+    deduplicated_data = deduplicate(data, n_clusters=n_clusters)
+    translation_table = pd.Series(deduplicated_data, index=data)
+    translation_table = translation_table[
+        ~translation_table.index.duplicated(keep="first")
+    ]
+    assert np.isin(np.unique(deduplicated_data), recovered_categories).all()
+    assert np.alltrue(translation_table[data] == np.array(deduplicated_data))
+    deduplicated_other_analyzer = np.array(
+        deduplicate(data, n_clusters=n_clusters, analyzer="char")
+    )
+    unique_other_analyzer = np.unique(deduplicated_other_analyzer)
+    assert np.isin(unique_other_analyzer, recovered_categories).all()
+
+
+def test_compute_ngram_distance() -> None:
+    words = np.array(["aac", "aaa", "aaab", "aaa", "aaab", "aaa", "aaab", "aaa"])
+    distance = compute_ngram_distance(words)
+    distance = squareform(distance)
+    assert distance.shape[0] == words.shape[0]
+    assert np.allclose(np.diag(distance), 0)
+    for un_word in np.unique(words):
+        assert np.allclose(distance[words == un_word][:, words == un_word], 0)
+
+
+def test__guess_clusters() -> None:
+    words = np.array(["aac", "aaa", "aaab", "aaa", "aaab", "aaa", "aaab", "aaa"])
+    distance = compute_ngram_distance(words)
+    Z = linkage(distance, method="average")
+    n_clusters = _guess_clusters(Z, distance)
+    assert n_clusters == len(np.unique(words))
+
+
+def test__create_spelling_correction(seed: int = 123) -> None:
+    rng = np.random.RandomState(seed)
+    n_clusters = 3
+    samples_per_cluster = 10
+    counts = np.concatenate(
+        [rng.randint(0, 100, samples_per_cluster) for _ in range(n_clusters)]
+    )
+    clusters = (
+        np.repeat(np.arange(n_clusters), samples_per_cluster).astype("int").tolist()
+    )
+    spelling_correction = _create_spelling_correction(
+        counts.astype("str"),
+        counts,
+        clusters,
+    )
+    # check that the most common sample per cluster is chosen as the 'correct' spelling
+    for n in np.arange(n_clusters):
+        assert (
+            spelling_correction.values[clusters == n].astype("int")
+            == counts[clusters == n].max()
+        ).all()
diff --git a/dirty_cat/tests/test_docstrings.py b/dirty_cat/tests/test_docstrings.py
index 0a683aa..9afaebe 100644
--- a/dirty_cat/tests/test_docstrings.py
+++ b/dirty_cat/tests/test_docstrings.py
@@ -10,7 +10,7 @@ should also remove their corresponding references from the list.
 import inspect
 import re
 from importlib import import_module
-from typing import Optional
+from typing import Callable, Optional
 
 import pytest
 from numpydoc.validate import validate
@@ -18,7 +18,6 @@ from numpydoc.validate import validate
 DOCSTRING_TEMP_IGNORE_SET = {
     "dirty_cat._datetime_encoder.DatetimeEncoder.get_feature_names",
     "dirty_cat._datetime_encoder.DatetimeEncoder.get_feature_names_out",
-    "dirty_cat._gap_encoder.GapEncoder.fit",
     "dirty_cat._gap_encoder.GapEncoder.get_feature_names",
     "dirty_cat._gap_encoder.GapEncoder.get_feature_names_out",
     "dirty_cat._gap_encoder.GapEncoder.partial_fit",
@@ -26,43 +25,41 @@ DOCSTRING_TEMP_IGNORE_SET = {
     "dirty_cat._gap_encoder.GapEncoder.transform",
     "dirty_cat._minhash_encoder.MinHashEncoder",
     "dirty_cat._minhash_encoder.MinHashEncoder.fit",
-    "dirty_cat._minhash_encoder.MinHashEncoder._get_fast_hash",
-    "dirty_cat._minhash_encoder.MinHashEncoder._get_murmur_hash",
-    "dirty_cat._minhash_encoder.MinHashEncoder.transform",
     "dirty_cat._similarity_encoder.SimilarityEncoder",
-    "dirty_cat._similarity_encoder.SimilarityEncoder.fit",
-    "dirty_cat._similarity_encoder.SimilarityEncoder.transform",
     "dirty_cat._similarity_encoder.SimilarityEncoder.fit_transform",
     "dirty_cat._super_vectorizer.SuperVectorizer.fit_transform",
-    "dirty_cat._super_vectorizer.SuperVectorizer.transform",
-    "dirty_cat._super_vectorizer.SuperVectorizer._auto_cast",
-    "dirty_cat._super_vectorizer.SuperVectorizer._apply_cast",
     "dirty_cat._super_vectorizer.SuperVectorizer.get_feature_names",
     "dirty_cat._super_vectorizer.SuperVectorizer.get_feature_names_out",
+    "dirty_cat._super_vectorizer.SuperVectorizer.transform",
     "dirty_cat._target_encoder.TargetEncoder",
-    "dirty_cat._target_encoder.TargetEncoder.fit",
-    "dirty_cat._target_encoder.TargetEncoder.transform",
+    "dirty_cat._fuzzy_join.fuzzy_join",
     # The following are not documented in dirty_cat (and thus are out of scope)
     # They are usually inherited from other libraries.
     "dirty_cat._super_vectorizer.SuperVectorizer.fit",
     "dirty_cat._super_vectorizer.SuperVectorizer.set_params",
     "dirty_cat._super_vectorizer.SuperVectorizer.named_transformers_",
+    # The following are internal functions
+    "dirty_cat._check_dependencies.check_dependencies",
 }
 
 
-def all_estimators():
+def get_public_classes():
     module = import_module("dirty_cat")
     classes = inspect.getmembers(module, inspect.isclass)
-    classes = [(name, est_cls) for name, est_cls in classes if not name.startswith("_")]
+    classes = [(name, cls) for name, cls in classes if not name.startswith("_")]
     return sorted(classes, key=lambda x: x[0])
 
 
-def get_all_methods():
-    estimators = all_estimators()
+def get_public_functions():
+    module = import_module("dirty_cat")
+    funcs = inspect.getmembers(module, inspect.isfunction)
+    funcs = [(name, func) for name, func in funcs if not name.startswith("_")]
+    return sorted(funcs, key=lambda x: x[0])
+
+
+def get_methods_to_validate():
+    estimators = get_public_classes()
     for name, Estimator in estimators:
-        if name.startswith("_"):
-            # skip private classes
-            continue
         methods = []
         for name in dir(Estimator):
             if name.startswith("_"):
@@ -76,8 +73,16 @@ def get_all_methods():
             yield Estimator, method
 
 
+def get_functions_to_validate():
+    functions = get_public_functions()
+    for name, func in functions:
+        yield func, name
+
+
 def repr_errors(res, estimator=None, method: Optional[str] = None) -> str:
-    """Pretty print original docstring and the obtained errors
+    """
+    Pretty print original docstring and the obtained errors
+
     Parameters
     ----------
     res : dict
@@ -86,6 +91,7 @@ def repr_errors(res, estimator=None, method: Optional[str] = None) -> str:
         estimator object or None
     method : str
         if estimator is not None, either the method name or None.
+
     Returns
     -------
     str
@@ -104,7 +110,7 @@ def repr_errors(res, estimator=None, method: Optional[str] = None) -> str:
         try:
             obj_signature = str(inspect.signature(obj))
         except TypeError:
-            # In particular we can't parse the signature of properties
+            # In particular, we can't parse the signature of properties
             obj_signature = (
                 "\nParsing of the method signature failed, "
                 "possibly because this is a property."
@@ -162,8 +168,8 @@ def filter_errors(errors, method, Estimator=None):
         yield code, message
 
 
-@pytest.mark.parametrize("Estimator, method", get_all_methods())
-def test_docstring(Estimator, method, request):
+@pytest.mark.parametrize("Estimator, method", get_methods_to_validate())
+def test_estimator_docstrings(Estimator: object, method: str, request):
     base_import_path = Estimator.__module__
     import_path = [base_import_path, Estimator.__name__]
     if method is not None:
@@ -181,9 +187,25 @@ def test_docstring(Estimator, method, request):
     res["errors"] = list(filter_errors(res["errors"], method, Estimator=Estimator))
 
     if res["errors"]:
-        msg = repr_errors(res, Estimator, method)
+        raise ValueError(repr_errors(res, Estimator, method))
 
-        raise ValueError(msg)
+
+@pytest.mark.parametrize("func, name", get_functions_to_validate())
+def test_function_docstrings(func: Callable, name: str, request):
+    import_path = ".".join([func.__module__, name])
+    print(import_path)
+
+    if import_path in DOCSTRING_TEMP_IGNORE_SET:
+        request.applymarker(
+            pytest.mark.xfail(run=False, reason="TODO pass numpydoc validation")
+        )
+
+    res = validate(import_path)
+
+    res["errors"] = list(filter_errors(res["errors"], name))
+
+    if res["errors"]:
+        raise ValueError(repr_errors(res, method=name))
 
 
 if __name__ == "__main__":
diff --git a/dirty_cat/tests/test_feature_augmenter.py b/dirty_cat/tests/test_feature_augmenter.py
new file mode 100644
index 0000000..4840fa1
--- /dev/null
+++ b/dirty_cat/tests/test_feature_augmenter.py
@@ -0,0 +1,92 @@
+import pandas as pd
+import pytest
+
+from dirty_cat import FeatureAugmenter
+
+
+def test_feature_augmenter():
+    main_table = pd.DataFrame(
+        [
+            "France",
+            "Germany",
+            "Italy",
+        ],
+        columns=["Country"],
+    )
+
+    aux_table_1 = pd.DataFrame(
+        [
+            ["Germany", 84_000_000],
+            ["French Republic", 68_000_000],
+            ["Italia", 59_000_000],
+        ],
+        columns=["Country", "Population"],
+    )
+
+    aux_table_2 = pd.DataFrame(
+        [
+            ["France", 2937],
+            ["Italy", 2099],
+            ["Germany", 4223],
+            ["UK", 3186],
+        ],
+        columns=["Country name", "GDP (billion)"],
+    )
+
+    aux_table_3 = pd.DataFrame(
+        [
+            ["La France", "Paris"],
+            ["Italy", "Rome"],
+            ["Germany", "Berlin"],
+        ],
+        columns=["Countries", "Capital"],
+    )
+
+    aux_tables = [
+        (aux_table_1, "Country"),
+        (aux_table_2, "Country name"),
+        (aux_table_3, "Countries"),
+    ]
+
+    fa = FeatureAugmenter(tables=aux_tables, main_key="Country")
+
+    fa.fit(main_table)
+
+    number_of_cols = tuple(
+        map(
+            sum,
+            zip(
+                main_table.shape,
+                aux_table_1.shape,
+                aux_table_2.shape,
+                aux_table_3.shape,
+            ),
+        )
+    )[1]
+
+    big_table = fa.transform(main_table)
+    assert big_table.shape == (main_table.shape[0], number_of_cols)
+
+    big_table = fa.fit_transform(main_table)
+    assert big_table.shape == (main_table.shape[0], number_of_cols)
+
+    false_fa = FeatureAugmenter(tables=aux_tables, main_key="Countryy")
+
+    with pytest.raises(
+        ValueError,
+        match=r"Got main_key",
+    ):
+        false_fa.fit(main_table)
+
+    false_aux_tables = [
+        (aux_table_1, "Countrys"),
+        (aux_table_2, "Country name"),
+        (aux_table_3, "Countries"),
+    ]
+
+    false_fa2 = FeatureAugmenter(tables=false_aux_tables, main_key="Country")
+    with pytest.raises(
+        ValueError,
+        match=r"Got column key",
+    ):
+        false_fa2.fit(main_table)
diff --git a/dirty_cat/tests/test_fuzzy_join.py b/dirty_cat/tests/test_fuzzy_join.py
index d90a132..11ea7cc 100644
--- a/dirty_cat/tests/test_fuzzy_join.py
+++ b/dirty_cat/tests/test_fuzzy_join.py
@@ -1,6 +1,7 @@
 import numpy as np
 import pandas as pd
 import pytest
+from sklearn.feature_extraction.text import HashingVectorizer
 
 from dirty_cat import fuzzy_join
 
@@ -201,3 +202,42 @@ def test_fuzzy_join_pandas_comparison():
     result_s_fj.rename(columns={"key_x": "key"}, inplace=True)
 
     pd.testing.assert_frame_equal(result_s, result_s_fj)
+
+
+def test_correct_encoder():
+    """Test that the encoder error checking is working as intended."""
+
+    class TestVectorizer(HashingVectorizer):
+        """Implements a custom vectorizer to check if the `encoder`
+        parameter uses the passed instance as expected.
+        Raises an error when `fit` is called.
+        """
+
+        def fit(self, X):
+            raise AssertionError("Custom vectorizer was called as intended.")
+
+    left = pd.DataFrame(
+        {
+            "key": ["K0", "K1", "K2", "K3"],
+            "A": ["A0", "A1", "A2", "A3"],
+            "B": ["B0", "B1", "B2", "B3"],
+        }
+    )
+
+    right = pd.DataFrame(
+        {
+            "key": ["K0", "K1", "K2", "K3"],
+            "C": ["C0", "C1", "C2", "C3"],
+            "D": ["D0", "D1", "D2", "D3"],
+        }
+    )
+
+    enc = TestVectorizer()
+
+    with pytest.raises(
+        AssertionError, match=r"Custom vectorizer was called as intended."
+    ):
+        fuzzy_join(left, right, on="key", how="left", encoder=enc)
+
+    with pytest.raises(ValueError, match=r"encoder should be a vectorizer object"):
+        fuzzy_join(left, right, on="key", how="left", encoder="awrongencoder")
diff --git a/dirty_cat/tests/test_gap_encoder.py b/dirty_cat/tests/test_gap_encoder.py
index e9be30f..77c2c1f 100644
--- a/dirty_cat/tests/test_gap_encoder.py
+++ b/dirty_cat/tests/test_gap_encoder.py
@@ -2,6 +2,7 @@ import numpy as np
 import pandas as pd
 import pytest
 from sklearn import __version__ as sklearn_version
+from sklearn.exceptions import NotFittedError
 
 from dirty_cat import GapEncoder
 from dirty_cat._utils import parse_version
@@ -195,3 +196,23 @@ def test_missing_values(missing: str) -> None:
             r"'error' or 'zero_impute', got 'aaa'",
         ):
             enc.fit_transform(observations)
+
+
+def test_check_fitted_gap_encoder():
+    """Test that calling transform before fit raises an error"""
+    X = np.array([["alice"], ["bob"]])
+    enc = GapEncoder(n_components=2, random_state=42)
+    with pytest.raises(NotFittedError):
+        enc.transform(X)
+
+    # Check that it works after fit
+    enc.fit(X)
+    enc.transform(X)
+
+
+def test_small_sample():
+    """Test that having n_samples < n_components raises an error"""
+    X = np.array([["alice"], ["bob"]])
+    enc = GapEncoder(n_components=3, random_state=42)
+    with pytest.raises(ValueError, match="should be >= n_components"):
+        enc.fit_transform(X)
diff --git a/dirty_cat/tests/test_minhash_encoder.py b/dirty_cat/tests/test_minhash_encoder.py
index e88d380..0aa504f 100644
--- a/dirty_cat/tests/test_minhash_encoder.py
+++ b/dirty_cat/tests/test_minhash_encoder.py
@@ -6,6 +6,7 @@ import numpy as np
 import pandas as pd
 import pytest
 from sklearn.utils._testing import assert_array_equal, skip_if_no_parallel
+from sklearn.exceptions import NotFittedError
 
 from dirty_cat import MinHashEncoder
 
@@ -192,6 +193,7 @@ class DummyBackend(DEFAULT_JOBLIB_BACKEND):  # type: ignore
     The `count` attribute is used to check that the backend is used.
     Copied from https://github.com/scikit-learn/scikit-learn/blob/36958fb240fbe435673a9e3c52e769f01f36bec0/sklearn/ensemble/tests/test_forest.py  # noqa
     """
+
     def __init__(self, *args, **kwargs):
         self.count = 0
         super().__init__(*args, **kwargs)
@@ -216,7 +218,7 @@ def test_backend_respected():
 
     with joblib.parallel_backend("testing") as (ba, n_jobs):
         encoder.fit_transform(X)
-        
+
     assert ba.count > 0
 
 
@@ -242,3 +244,15 @@ def test_correct_arguments():
     with pytest.raises(ValueError, match=r"n_components should be even"):
         encoder = MinHashEncoder(n_components=3, minmax_hash=True)
         encoder.fit_transform(X)
+
+def test_check_fitted_minhash_encoder():
+    """Test that calling transform before fit raises an error"""
+    encoder = MinHashEncoder(n_components=3)
+    X = np.array(["a", "b", "c", "d", "e", "f", "g", "h"])[:, None]
+    with pytest.raises(NotFittedError):
+        encoder.transform(X)
+    
+    # Check that it works after fitting
+    encoder.fit(X)
+    encoder.transform(X)
+
diff --git a/dirty_cat/tests/test_similarity_encoder.py b/dirty_cat/tests/test_similarity_encoder.py
index 292c7c2..51bc929 100644
--- a/dirty_cat/tests/test_similarity_encoder.py
+++ b/dirty_cat/tests/test_similarity_encoder.py
@@ -4,6 +4,7 @@ import numpy as np
 import numpy.testing
 import pytest
 from sklearn import __version__ as sklearn_version
+from sklearn.exceptions import NotFittedError
 
 from dirty_cat import SimilarityEncoder
 from dirty_cat._similarity_encoder import get_kmeans_prototypes, ngram_similarity_matrix
@@ -385,3 +386,12 @@ def test_get_features() -> None:
         "x0_}",
         "x0_~",
     ]
+
+def test_check_fitted_super_vectorizer():
+    """Test that calling transform before fit raises an error"""
+    sim_enc = SimilarityEncoder()
+    X = np.array(["%s" % chr(i) for i in range(32, 127)]).reshape((-1, 1))
+    with pytest.raises(NotFittedError):
+        sim_enc.transform(X)
+    sim_enc.fit(X)
+    sim_enc.transform(X)
diff --git a/dirty_cat/tests/test_super_vectorizer.py b/dirty_cat/tests/test_super_vectorizer.py
index ce600c5..0806c0d 100644
--- a/dirty_cat/tests/test_super_vectorizer.py
+++ b/dirty_cat/tests/test_super_vectorizer.py
@@ -7,6 +7,7 @@ import sklearn
 from sklearn.exceptions import NotFittedError
 from sklearn.preprocessing import StandardScaler
 from sklearn.utils.validation import check_is_fitted
+from sklearn.exceptions import NotFittedError
 
 from dirty_cat import GapEncoder, SuperVectorizer
 from dirty_cat._utils import parse_version
@@ -445,3 +446,15 @@ def test_passthrough():
     dirty_flat_trans_df = X_enc_dirty.to_numpy().ravel().tolist()
     assert all(map(_is_equal, zip(dirty_flat_df, dirty_flat_trans_df)))
     assert (X_clean.to_numpy() == X_enc_clean.to_numpy()).all()
+
+def test_check_fitted_super_vectorizer():
+    """Test that calling transform before fit raises an error"""
+    X = _get_clean_dataframe()
+    sv = SuperVectorizer()
+    with pytest.raises(NotFittedError):
+        sv.transform(X)
+    
+    # Test that calling transform after fit works
+    sv.fit(X)
+    sv.transform(X)
+
diff --git a/dirty_cat/tests/test_target_encoder.py b/dirty_cat/tests/test_target_encoder.py
index e45b396..cf64192 100644
--- a/dirty_cat/tests/test_target_encoder.py
+++ b/dirty_cat/tests/test_target_encoder.py
@@ -1,5 +1,6 @@
 import numpy as np
 import pytest
+from sklearn.exceptions import NotFittedError
 
 from dirty_cat import _target_encoder
 
@@ -275,3 +276,15 @@ def test_errors():
         enc4.transform(X2)
     with pytest.raises(ValueError, match="Found unknown categories "):
         enc4.transform(X3)
+
+def test_check_fitted_target_encoder():
+    """Test that calling transform before fit raises an error"""
+    enc = _target_encoder.TargetEncoder()
+    X = [[1], [2], [2], [3], [4]]
+    y = np.array([1, 2, 3, 4, 5])
+
+    with pytest.raises(NotFittedError, match="This TargetEncoder instance is not"):
+        enc.transform(X)
+    
+    enc.fit(X, y)
+    enc.transform(X)
diff --git a/dirty_cat/tests/utils.py b/dirty_cat/tests/utils.py
index baf55bd..c6c3756 100644
--- a/dirty_cat/tests/utils.py
+++ b/dirty_cat/tests/utils.py
@@ -8,6 +8,7 @@ def generate_data(
     n_samples,
     as_list=False,
     random_state: Optional[Union[int, float, str, bytes, bytearray]] = None,
+    sample_length: int = 100,
 ) -> np.ndarray:
     if random_state is not None:
         random.seed(random_state)
@@ -15,7 +16,7 @@ def generate_data(
     str_list = []
     for i in range(n_samples):
         random_string = "category "
-        for _ in range(100):
+        for _ in range(sample_length):
             random_integer = random.randint(1, MAX_LIMIT)
             random_string += chr(random_integer)
             if random_integer < 50: