Repository: feature-engine/feature_engine

Issue:
`CountFrequencyEncoder` could output zeros for new categories
The `CountFrequencyEncoder` has an `errors` argument which can either raise an error or output NaNs when encountering new categories. For this particular class, it'd make sense (perhaps even as a default behavior) to output zeros when a new category is encoded instead of generating NaNs.

Comments:

Patch:
diff --git a/feature_engine/encoding/_docstrings.py b/feature_engine/encoding/_docstrings.py
index 20e312e..d2d9617 100644
--- a/feature_engine/encoding/_docstrings.py
+++ b/feature_engine/encoding/_docstrings.py
@@ -15,8 +15,8 @@ _ignore_format_docstring = """ignore_format: bool, default=False
 
 _errors_docstring = """errors: string, default='ignore'
         Indicates what to do, when categories not present in the train set are
-        encountered during transform. If 'raise', then rare categories will raise an
-        error. If 'ignore', then rare categories will be set as NaN and a warning will
+        encountered during transform. If 'raise', then unseen categories will raise an
+        error. If 'ignore', then unseen categories will be set as NaN and a warning will
         be raised instead.
     """.rstrip()
 
diff --git a/feature_engine/encoding/_helper_functions.py b/feature_engine/encoding/_helper_functions.py
new file mode 100644
index 0000000..6d4b415
--- /dev/null
+++ b/feature_engine/encoding/_helper_functions.py
@@ -0,0 +1,6 @@
+def check_parameter_errors(errors, accepted_values):
+    if errors not in accepted_values:
+        raise ValueError(
+            f"errors takes only values {', '.join(accepted_values)}."
+            f"Got {errors} instead."
+        )
diff --git a/feature_engine/encoding/base_encoder.py b/feature_engine/encoding/base_encoder.py
index 973e6aa..5290d84 100644
--- a/feature_engine/encoding/base_encoder.py
+++ b/feature_engine/encoding/base_encoder.py
@@ -17,6 +17,7 @@ from feature_engine.encoding._docstrings import (
     _ignore_format_docstring,
     _variables_docstring,
 )
+from feature_engine.encoding._helper_functions import check_parameter_errors
 from feature_engine.get_feature_names_out import _get_feature_names_out
 from feature_engine.tags import _return_tags
 from feature_engine.variable_manipulation import (
@@ -80,12 +81,7 @@ class CategoricalInitExpandedMixin(CategoricalInitMixin):
         ignore_format: bool = False,
         errors: str = "ignore",
     ) -> None:
-        if errors not in ["raise", "ignore"]:
-            raise ValueError(
-                "errors takes only values 'raise' and 'ignore ."
-                f"Got {errors} instead."
-            )
-
+        check_parameter_errors(errors, ["raise", "ignore"])
         super().__init__(variables, ignore_format)
         self.errors = errors
 
diff --git a/feature_engine/encoding/count_frequency.py b/feature_engine/encoding/count_frequency.py
index d73bfff..6243a58 100644
--- a/feature_engine/encoding/count_frequency.py
+++ b/feature_engine/encoding/count_frequency.py
@@ -1,6 +1,7 @@
 # Authors: Soledad Galli <solegalli@protonmail.com>
 # License: BSD 3 clause
 
+from collections import defaultdict
 from typing import List, Optional, Union
 
 import pandas as pd
@@ -22,11 +23,17 @@ from feature_engine.encoding._docstrings import (
     _transform_docstring,
     _variables_docstring,
 )
+from feature_engine.encoding._helper_functions import check_parameter_errors
 from feature_engine.encoding.base_encoder import (
-    CategoricalInitExpandedMixin,
+    CategoricalInitMixin,
     CategoricalMethodsMixin,
 )
 
+_errors_docstring = (
+    _errors_docstring
+    + """ If `'encode'`, unseen categories will be encoded as 0 (zero)."""
+)
+
 
 @Substitution(
     ignore_format=_ignore_format_docstring,
@@ -39,7 +46,7 @@ from feature_engine.encoding.base_encoder import (
     transform=_transform_docstring,
     inverse_transform=_inverse_transform_docstring,
 )
-class CountFrequencyEncoder(CategoricalInitExpandedMixin, CategoricalMethodsMixin):
+class CountFrequencyEncoder(CategoricalInitMixin, CategoricalMethodsMixin):
     """
     The CountFrequencyEncoder() replaces categories by either the count or the
     percentage of observations per category.
@@ -102,11 +109,11 @@ class CountFrequencyEncoder(CategoricalInitExpandedMixin, CategoricalMethodsMixi
 
     Notes
     -----
-    NAN are introduced when encoding categories that were not present in the training
-    dataset. If this happens, try grouping infrequent categories using the
-    RareLabelEncoder().
+    NAN will be introduced when encoding categories that were not present in the
+    training set. If this happens, try grouping infrequent categories using the
+    RareLabelEncoder(), or set `errors='encode'`.
 
-    There is a similar implementation in the the open-source package
+    There is a similar implementation in the open-source package
     `Category encoders <https://contrib.scikit-learn.org/category_encoders/>`_
 
     See Also
@@ -125,11 +132,14 @@ class CountFrequencyEncoder(CategoricalInitExpandedMixin, CategoricalMethodsMixi
 
         if encoding_method not in ["count", "frequency"]:
             raise ValueError(
-                "encoding_method takes only values 'count' and 'frequency'"
+                "encoding_method takes only values 'count' and 'frequency'. "
+                f"Got {encoding_method} instead."
             )
-        super().__init__(variables, ignore_format, errors)
 
+        check_parameter_errors(errors, ["ignore", "raise", "encode"])
+        super().__init__(variables, ignore_format)
         self.encoding_method = encoding_method
+        self.errors = errors
 
     def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None):
         """
@@ -149,15 +159,17 @@ class CountFrequencyEncoder(CategoricalInitExpandedMixin, CategoricalMethodsMixi
         self._get_feature_names_in(X)
 
         self.encoder_dict_ = {}
+        dct_init = defaultdict(lambda: 0) if self.errors == "encode" else {}
 
         # learn encoding maps
         for var in self.variables_:
             if self.encoding_method == "count":
-                self.encoder_dict_[var] = X[var].value_counts().to_dict()
+                self.encoder_dict_[var] = X[var].value_counts().to_dict(dct_init)
 
             elif self.encoding_method == "frequency":
-                n_obs = float(len(X))
-                self.encoder_dict_[var] = (X[var].value_counts() / n_obs).to_dict()
+                self.encoder_dict_[var] = (
+                    X[var].value_counts(normalize=True).to_dict(dct_init)
+                )
 
         self._check_encoding_dictionary()
 


Test patch:
diff --git a/tests/test_encoding/test_count_frequency_encoder.py b/tests/test_encoding/test_count_frequency_encoder.py
index b47fd97..a4b88f8 100644
--- a/tests/test_encoding/test_count_frequency_encoder.py
+++ b/tests/test_encoding/test_count_frequency_encoder.py
@@ -1,3 +1,6 @@
+import warnings
+
+from numpy import nan
 import pandas as pd
 import pytest
 
@@ -111,23 +114,22 @@ def test_automatically_select_variables_encode_with_frequency(df_enc):
     pd.testing.assert_frame_equal(X, transf_df)
 
 
-def test_error_if_encoding_method_not_permitted_value():
+@pytest.mark.parametrize("enc_method", ["arbitrary", False, 1])
+def test_error_if_encoding_method_not_permitted_value(enc_method):
     with pytest.raises(ValueError):
-        CountFrequencyEncoder(encoding_method="arbitrary")
+        CountFrequencyEncoder(encoding_method=enc_method)
 
 
-def test_error_if_input_df_contains_categories_not_present_in_fit_df(
-    df_enc, df_enc_rare
-):
-    # test case 3: when dataset to be transformed contains categories not present in
-    # training dataset
+def test_warning_when_df_contains_unseen_categories(df_enc, df_enc_rare):
+    # dataset to be transformed contains categories not present in
+    # training dataset (unseen categories), errors set to ignore.
 
     msg = "During the encoding, NaN values were introduced in the feature(s) var_A."
 
-    # check for warning when rare_labels equals 'ignore'
+    # check for warning when errors equals 'ignore'
+    encoder = CountFrequencyEncoder(errors="ignore")
+    encoder.fit(df_enc)
     with pytest.warns(UserWarning) as record:
-        encoder = CountFrequencyEncoder(errors="ignore")
-        encoder.fit(df_enc)
         encoder.transform(df_enc_rare)
 
     # check that only one warning was raised
@@ -135,32 +137,140 @@ def test_error_if_input_df_contains_categories_not_present_in_fit_df(
     # check that the message matches
     assert record[0].message.args[0] == msg
 
-    # check for error when rare_labels equals 'raise'
-    with pytest.raises(ValueError) as record:
-        encoder = CountFrequencyEncoder(errors="raise")
 
-        encoder.fit(df_enc)
+def test_error_when_df_contains_unseen_categories(df_enc, df_enc_rare):
+    # dataset to be transformed contains categories not present in
+    # training dataset (unseen categories), errors set to raise.
+
+    msg = "During the encoding, NaN values were introduced in the feature(s) var_A."
+
+    encoder = CountFrequencyEncoder(errors="raise")
+    encoder.fit(df_enc)
+
+    # check for exception when errors equals 'raise'
+    with pytest.raises(ValueError) as record:
         encoder.transform(df_enc_rare)
 
     # check that the error message matches
     assert str(record.value) == msg
 
+    # check for no error and no warning when errors equals 'encode'
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        encoder = CountFrequencyEncoder(errors="encode")
+        encoder.fit(df_enc)
+        encoder.transform(df_enc_rare)
+
+
+def test_no_error_triggered_when_df_contains_unseen_categories_and_errors_is_encode(
+    df_enc, df_enc_rare
+):
+    # dataset to be transformed contains categories not present in
+    # training dataset (unseen categories).
+
+    # check for no error and no warning when errors equals 'encode'
+    warnings.simplefilter("error")
+    encoder = CountFrequencyEncoder(errors="encode")
+    encoder.fit(df_enc)
+    with warnings.catch_warnings():
+        encoder.transform(df_enc_rare)
 
-def test_fit_raises_error_if_df_contains_na(df_enc_na):
+
+@pytest.mark.parametrize("errors", ["raise", "ignore", "encode"])
+def test_fit_raises_error_if_df_contains_na(errors, df_enc_na):
     # test case 4: when dataset contains na, fit method
+    encoder = CountFrequencyEncoder(errors=errors)
     with pytest.raises(ValueError):
-        encoder = CountFrequencyEncoder()
         encoder.fit(df_enc_na)
 
 
-def test_transform_raises_error_if_df_contains_na(df_enc, df_enc_na):
+@pytest.mark.parametrize("errors", ["raise", "ignore", "encode"])
+def test_transform_raises_error_if_df_contains_na(errors, df_enc, df_enc_na):
     # test case 4: when dataset contains na, transform method
+    encoder = CountFrequencyEncoder(errors=errors)
+    encoder.fit(df_enc)
     with pytest.raises(ValueError):
-        encoder = CountFrequencyEncoder()
-        encoder.fit(df_enc)
         encoder.transform(df_enc_na)
 
 
+def test_zero_encoding_for_new_categories():
+    df_fit = pd.DataFrame({
+        "col1": ["a", "a", "b", "a", "c"],
+        "col2": ["1", "2", "3", "1", "2"]
+    })
+    df_transf = pd.DataFrame({
+        "col1": ["a", "d", "b", "a", "c"],
+        "col2": ["1", "2", "3", "1", "4"]
+    })
+    encoder = CountFrequencyEncoder(errors="encode").fit(df_fit)
+    result = encoder.transform(df_transf)
+
+    # check that no NaNs are added
+    assert pd.isnull(result).sum().sum() == 0
+
+    # check that the counts are correct for both new and old
+    expected_result = pd.DataFrame({
+        "col1": [3, 0, 1, 3, 1],
+        "col2": [2, 2, 1, 2, 0]
+    })
+    pd.testing.assert_frame_equal(result, expected_result)
+
+
+def test_zero_encoding_for_unseen_categories_if_errors_is_encode():
+    df_fit = pd.DataFrame(
+        {"col1": ["a", "a", "b", "a", "c"], "col2": ["1", "2", "3", "1", "2"]}
+    )
+    df_transform = pd.DataFrame(
+        {"col1": ["a", "d", "b", "a", "c"], "col2": ["1", "2", "3", "1", "4"]}
+    )
+
+    # count encoding
+    encoder = CountFrequencyEncoder(errors="encode").fit(df_fit)
+    result = encoder.transform(df_transform)
+
+    # check that no NaNs are added
+    assert pd.isnull(result).sum().sum() == 0
+
+    # check that the counts are correct
+    expected_result = pd.DataFrame({"col1": [3, 0, 1, 3, 1], "col2": [2, 2, 1, 2, 0]})
+    pd.testing.assert_frame_equal(result, expected_result)
+
+    # with frequency
+    encoder = CountFrequencyEncoder(encoding_method="frequency", errors="encode").fit(
+        df_fit
+    )
+    result = encoder.transform(df_transform)
+
+    # check that no NaNs are added
+    assert pd.isnull(result).sum().sum() == 0
+
+    # check that the frequencies are correct
+    expected_result = pd.DataFrame(
+        {"col1": [0.6, 0, 0.2, 0.6, 0.2], "col2": [0.4, 0.4, 0.2, 0.4, 0]}
+    )
+    pd.testing.assert_frame_equal(result, expected_result)
+
+
+def test_nan_encoding_for_new_categories_if_errors_is_ignore():
+    df_fit = pd.DataFrame(
+        {"col1": ["a", "a", "b", "a", "c"], "col2": ["1", "2", "3", "1", "2"]}
+    )
+    df_transf = pd.DataFrame(
+        {"col1": ["a", "d", "b", "a", "c"], "col2": ["1", "2", "3", "1", "4"]}
+    )
+    encoder = CountFrequencyEncoder(errors="ignore").fit(df_fit)
+    result = encoder.transform(df_transf)
+
+    # check that no NaNs are added
+    assert pd.isnull(result).sum().sum() == 2
+
+    # check that the counts are correct for both new and old
+    expected_result = pd.DataFrame(
+        {"col1": [3, nan, 1, 3, 1], "col2": [2, 2, 1, 2, nan]}
+    )
+    pd.testing.assert_frame_equal(result, expected_result)
+
+
 def test_ignore_variable_format_with_frequency(df_vartypes):
     encoder = CountFrequencyEncoder(
         encoding_method="frequency", variables=None, ignore_format=True
@@ -252,6 +362,7 @@ def test_variables_cast_as_category(df_enc_category_dtypes):
     assert X["var_A"].dtypes == float
 
 
-def test_error_if_rare_labels_not_permitted_value():
+@pytest.mark.parametrize("errors", ["empanada", False, 1])
+def test_exception_if_errors_gets_not_permitted_value(errors):
     with pytest.raises(ValueError):
-        CountFrequencyEncoder(errors="empanada")
+        CountFrequencyEncoder(errors=errors)