Repository: feature-engine/feature_engine

Issue:
`CountFrequencyEncoder` could output zeros for new categories
The `CountFrequencyEncoder` has an `errors` argument which can either raise an error or output NaNs when encountering new categories. For this particular class, it'd make sense (perhaps even as a default behavior) to output zeros when a new category is encoded instead of generating NaNs.

Discussion:
solegalli: Will have a look when I am back from hols, apologies for the delay
david-cortes: Redone with the new base classes for categoricals.
david-cortes: I am not sure what would be the right way to handle explicit missing values that are not from unseen categories.

I think ideally whether to handle them should be configured through a different option than this one, and ideally one of the ways of handling them would be to encode them into a separate category "missing" or similar. However pandas functions and methods don't lend themselves well towards handling missing values and that would require a lot more modifications.
solegalli: Hi @david-cortes 

Sorry for the delay. Please see this PR: https://github.com/david-cortes/feature_engine/pull/1

I mostly reorganized the code you wrote.

The conflict is probably through the main branches not being in sync. If you can't resolve it let me know.

Thank you so much for the great contribution!
david-cortes: Updated, but I'm not sure if the merge conflict solving I made here preserved the changes that you wanted.
david-cortes: @solegalli What linter are you using for indentation?
solegalli: black
david-cortes: Reformatted with black and isort.
solegalli: Hi @david-cortes 

Not sure all the files were reformatted. Also, can we remove the base transformer from this PR?

Thank you!
david-cortes: If I run either `black` or `isort` on this package, it will reformat pretty much all the files, not just the ones connected with this PR.
david-cortes: The base class `CategoricalInitExpandedMixin` is used by other transformers too.
david-cortes: Updated.
solegalli: Thank you! merging as we speak.

Patch:
diff --git a/feature_engine/encoding/_docstrings.py b/feature_engine/encoding/_docstrings.py
index 20e312e..d2d9617 100644
--- a/feature_engine/encoding/_docstrings.py
+++ b/feature_engine/encoding/_docstrings.py
@@ -15,8 +15,8 @@ _ignore_format_docstring = """ignore_format: bool, default=False
 
 _errors_docstring = """errors: string, default='ignore'
         Indicates what to do, when categories not present in the train set are
-        encountered during transform. If 'raise', then rare categories will raise an
-        error. If 'ignore', then rare categories will be set as NaN and a warning will
+        encountered during transform. If 'raise', then unseen categories will raise an
+        error. If 'ignore', then unseen categories will be set as NaN and a warning will
         be raised instead.
     """.rstrip()
 
diff --git a/feature_engine/encoding/_helper_functions.py b/feature_engine/encoding/_helper_functions.py
new file mode 100644
index 0000000..6d4b415
--- /dev/null
+++ b/feature_engine/encoding/_helper_functions.py
@@ -0,0 +1,6 @@
+def check_parameter_errors(errors, accepted_values):
+    if errors not in accepted_values:
+        raise ValueError(
+            f"errors takes only values {', '.join(accepted_values)}."
+            f"Got {errors} instead."
+        )
diff --git a/feature_engine/encoding/base_encoder.py b/feature_engine/encoding/base_encoder.py
index 973e6aa..5290d84 100644
--- a/feature_engine/encoding/base_encoder.py
+++ b/feature_engine/encoding/base_encoder.py
@@ -17,6 +17,7 @@ from feature_engine.encoding._docstrings import (
     _ignore_format_docstring,
     _variables_docstring,
 )
+from feature_engine.encoding._helper_functions import check_parameter_errors
 from feature_engine.get_feature_names_out import _get_feature_names_out
 from feature_engine.tags import _return_tags
 from feature_engine.variable_manipulation import (
@@ -80,12 +81,7 @@ class CategoricalInitExpandedMixin(CategoricalInitMixin):
         ignore_format: bool = False,
         errors: str = "ignore",
     ) -> None:
-        if errors not in ["raise", "ignore"]:
-            raise ValueError(
-                "errors takes only values 'raise' and 'ignore ."
-                f"Got {errors} instead."
-            )
-
+        check_parameter_errors(errors, ["raise", "ignore"])
         super().__init__(variables, ignore_format)
         self.errors = errors
 
diff --git a/feature_engine/encoding/count_frequency.py b/feature_engine/encoding/count_frequency.py
index d73bfff..6243a58 100644
--- a/feature_engine/encoding/count_frequency.py
+++ b/feature_engine/encoding/count_frequency.py
@@ -1,6 +1,7 @@
 # Authors: Soledad Galli <solegalli@protonmail.com>
 # License: BSD 3 clause
 
+from collections import defaultdict
 from typing import List, Optional, Union
 
 import pandas as pd
@@ -22,11 +23,17 @@ from feature_engine.encoding._docstrings import (
     _transform_docstring,
     _variables_docstring,
 )
+from feature_engine.encoding._helper_functions import check_parameter_errors
 from feature_engine.encoding.base_encoder import (
-    CategoricalInitExpandedMixin,
+    CategoricalInitMixin,
     CategoricalMethodsMixin,
 )
 
+_errors_docstring = (
+    _errors_docstring
+    + """ If `'encode'`, unseen categories will be encoded as 0 (zero)."""
+)
+
 
 @Substitution(
     ignore_format=_ignore_format_docstring,
@@ -39,7 +46,7 @@ from feature_engine.encoding.base_encoder import (
     transform=_transform_docstring,
     inverse_transform=_inverse_transform_docstring,
 )
-class CountFrequencyEncoder(CategoricalInitExpandedMixin, CategoricalMethodsMixin):
+class CountFrequencyEncoder(CategoricalInitMixin, CategoricalMethodsMixin):
     """
     The CountFrequencyEncoder() replaces categories by either the count or the
     percentage of observations per category.
@@ -102,11 +109,11 @@ class CountFrequencyEncoder(CategoricalInitExpandedMixin, CategoricalMethodsMixi
 
     Notes
     -----
-    NAN are introduced when encoding categories that were not present in the training
-    dataset. If this happens, try grouping infrequent categories using the
-    RareLabelEncoder().
+    NAN will be introduced when encoding categories that were not present in the
+    training set. If this happens, try grouping infrequent categories using the
+    RareLabelEncoder(), or set `errors='encode'`.
 
-    There is a similar implementation in the the open-source package
+    There is a similar implementation in the open-source package
     `Category encoders <https://contrib.scikit-learn.org/category_encoders/>`_
 
     See Also
@@ -125,11 +132,14 @@ class CountFrequencyEncoder(CategoricalInitExpandedMixin, CategoricalMethodsMixi
 
         if encoding_method not in ["count", "frequency"]:
             raise ValueError(
-                "encoding_method takes only values 'count' and 'frequency'"
+                "encoding_method takes only values 'count' and 'frequency'. "
+                f"Got {encoding_method} instead."
             )
-        super().__init__(variables, ignore_format, errors)
 
+        check_parameter_errors(errors, ["ignore", "raise", "encode"])
+        super().__init__(variables, ignore_format)
         self.encoding_method = encoding_method
+        self.errors = errors
 
     def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None):
         """
@@ -149,15 +159,17 @@ class CountFrequencyEncoder(CategoricalInitExpandedMixin, CategoricalMethodsMixi
         self._get_feature_names_in(X)
 
         self.encoder_dict_ = {}
+        dct_init = defaultdict(lambda: 0) if self.errors == "encode" else {}
 
         # learn encoding maps
         for var in self.variables_:
             if self.encoding_method == "count":
-                self.encoder_dict_[var] = X[var].value_counts().to_dict()
+                self.encoder_dict_[var] = X[var].value_counts().to_dict(dct_init)
 
             elif self.encoding_method == "frequency":
-                n_obs = float(len(X))
-                self.encoder_dict_[var] = (X[var].value_counts() / n_obs).to_dict()
+                self.encoder_dict_[var] = (
+                    X[var].value_counts(normalize=True).to_dict(dct_init)
+                )
 
         self._check_encoding_dictionary()
 


Test patch:
diff --git a/tests/test_encoding/test_count_frequency_encoder.py b/tests/test_encoding/test_count_frequency_encoder.py
index b47fd97..a4b88f8 100644
--- a/tests/test_encoding/test_count_frequency_encoder.py
+++ b/tests/test_encoding/test_count_frequency_encoder.py
@@ -1,3 +1,6 @@
+import warnings
+
+from numpy import nan
 import pandas as pd
 import pytest
 
@@ -111,23 +114,22 @@ def test_automatically_select_variables_encode_with_frequency(df_enc):
     pd.testing.assert_frame_equal(X, transf_df)
 
 
-def test_error_if_encoding_method_not_permitted_value():
+@pytest.mark.parametrize("enc_method", ["arbitrary", False, 1])
+def test_error_if_encoding_method_not_permitted_value(enc_method):
     with pytest.raises(ValueError):
-        CountFrequencyEncoder(encoding_method="arbitrary")
+        CountFrequencyEncoder(encoding_method=enc_method)
 
 
-def test_error_if_input_df_contains_categories_not_present_in_fit_df(
-    df_enc, df_enc_rare
-):
-    # test case 3: when dataset to be transformed contains categories not present in
-    # training dataset
+def test_warning_when_df_contains_unseen_categories(df_enc, df_enc_rare):
+    # dataset to be transformed contains categories not present in
+    # training dataset (unseen categories), errors set to ignore.
 
     msg = "During the encoding, NaN values were introduced in the feature(s) var_A."
 
-    # check for warning when rare_labels equals 'ignore'
+    # check for warning when errors equals 'ignore'
+    encoder = CountFrequencyEncoder(errors="ignore")
+    encoder.fit(df_enc)
     with pytest.warns(UserWarning) as record:
-        encoder = CountFrequencyEncoder(errors="ignore")
-        encoder.fit(df_enc)
         encoder.transform(df_enc_rare)
 
     # check that only one warning was raised
@@ -135,32 +137,140 @@ def test_error_if_input_df_contains_categories_not_present_in_fit_df(
     # check that the message matches
     assert record[0].message.args[0] == msg
 
-    # check for error when rare_labels equals 'raise'
-    with pytest.raises(ValueError) as record:
-        encoder = CountFrequencyEncoder(errors="raise")
 
-        encoder.fit(df_enc)
+def test_error_when_df_contains_unseen_categories(df_enc, df_enc_rare):
+    # dataset to be transformed contains categories not present in
+    # training dataset (unseen categories), errors set to raise.
+
+    msg = "During the encoding, NaN values were introduced in the feature(s) var_A."
+
+    encoder = CountFrequencyEncoder(errors="raise")
+    encoder.fit(df_enc)
+
+    # check for exception when errors equals 'raise'
+    with pytest.raises(ValueError) as record:
         encoder.transform(df_enc_rare)
 
     # check that the error message matches
     assert str(record.value) == msg
 
+    # check for no error and no warning when errors equals 'encode'
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        encoder = CountFrequencyEncoder(errors="encode")
+        encoder.fit(df_enc)
+        encoder.transform(df_enc_rare)
+
+
+def test_no_error_triggered_when_df_contains_unseen_categories_and_errors_is_encode(
+    df_enc, df_enc_rare
+):
+    # dataset to be transformed contains categories not present in
+    # training dataset (unseen categories).
+
+    # check for no error and no warning when errors equals 'encode'
+    warnings.simplefilter("error")
+    encoder = CountFrequencyEncoder(errors="encode")
+    encoder.fit(df_enc)
+    with warnings.catch_warnings():
+        encoder.transform(df_enc_rare)
 
-def test_fit_raises_error_if_df_contains_na(df_enc_na):
+
+@pytest.mark.parametrize("errors", ["raise", "ignore", "encode"])
+def test_fit_raises_error_if_df_contains_na(errors, df_enc_na):
     # test case 4: when dataset contains na, fit method
+    encoder = CountFrequencyEncoder(errors=errors)
     with pytest.raises(ValueError):
-        encoder = CountFrequencyEncoder()
         encoder.fit(df_enc_na)
 
 
-def test_transform_raises_error_if_df_contains_na(df_enc, df_enc_na):
+@pytest.mark.parametrize("errors", ["raise", "ignore", "encode"])
+def test_transform_raises_error_if_df_contains_na(errors, df_enc, df_enc_na):
     # test case 4: when dataset contains na, transform method
+    encoder = CountFrequencyEncoder(errors=errors)
+    encoder.fit(df_enc)
     with pytest.raises(ValueError):
-        encoder = CountFrequencyEncoder()
-        encoder.fit(df_enc)
         encoder.transform(df_enc_na)
 
 
+def test_zero_encoding_for_new_categories():
+    df_fit = pd.DataFrame({
+        "col1": ["a", "a", "b", "a", "c"],
+        "col2": ["1", "2", "3", "1", "2"]
+    })
+    df_transf = pd.DataFrame({
+        "col1": ["a", "d", "b", "a", "c"],
+        "col2": ["1", "2", "3", "1", "4"]
+    })
+    encoder = CountFrequencyEncoder(errors="encode").fit(df_fit)
+    result = encoder.transform(df_transf)
+
+    # check that no NaNs are added
+    assert pd.isnull(result).sum().sum() == 0
+
+    # check that the counts are correct for both new and old
+    expected_result = pd.DataFrame({
+        "col1": [3, 0, 1, 3, 1],
+        "col2": [2, 2, 1, 2, 0]
+    })
+    pd.testing.assert_frame_equal(result, expected_result)
+
+
+def test_zero_encoding_for_unseen_categories_if_errors_is_encode():
+    df_fit = pd.DataFrame(
+        {"col1": ["a", "a", "b", "a", "c"], "col2": ["1", "2", "3", "1", "2"]}
+    )
+    df_transform = pd.DataFrame(
+        {"col1": ["a", "d", "b", "a", "c"], "col2": ["1", "2", "3", "1", "4"]}
+    )
+
+    # count encoding
+    encoder = CountFrequencyEncoder(errors="encode").fit(df_fit)
+    result = encoder.transform(df_transform)
+
+    # check that no NaNs are added
+    assert pd.isnull(result).sum().sum() == 0
+
+    # check that the counts are correct
+    expected_result = pd.DataFrame({"col1": [3, 0, 1, 3, 1], "col2": [2, 2, 1, 2, 0]})
+    pd.testing.assert_frame_equal(result, expected_result)
+
+    # with frequency
+    encoder = CountFrequencyEncoder(encoding_method="frequency", errors="encode").fit(
+        df_fit
+    )
+    result = encoder.transform(df_transform)
+
+    # check that no NaNs are added
+    assert pd.isnull(result).sum().sum() == 0
+
+    # check that the frequencies are correct
+    expected_result = pd.DataFrame(
+        {"col1": [0.6, 0, 0.2, 0.6, 0.2], "col2": [0.4, 0.4, 0.2, 0.4, 0]}
+    )
+    pd.testing.assert_frame_equal(result, expected_result)
+
+
+def test_nan_encoding_for_new_categories_if_errors_is_ignore():
+    df_fit = pd.DataFrame(
+        {"col1": ["a", "a", "b", "a", "c"], "col2": ["1", "2", "3", "1", "2"]}
+    )
+    df_transf = pd.DataFrame(
+        {"col1": ["a", "d", "b", "a", "c"], "col2": ["1", "2", "3", "1", "4"]}
+    )
+    encoder = CountFrequencyEncoder(errors="ignore").fit(df_fit)
+    result = encoder.transform(df_transf)
+
+    # check that no NaNs are added
+    assert pd.isnull(result).sum().sum() == 2
+
+    # check that the counts are correct for both new and old
+    expected_result = pd.DataFrame(
+        {"col1": [3, nan, 1, 3, 1], "col2": [2, 2, 1, 2, nan]}
+    )
+    pd.testing.assert_frame_equal(result, expected_result)
+
+
 def test_ignore_variable_format_with_frequency(df_vartypes):
     encoder = CountFrequencyEncoder(
         encoding_method="frequency", variables=None, ignore_format=True
@@ -252,6 +362,7 @@ def test_variables_cast_as_category(df_enc_category_dtypes):
     assert X["var_A"].dtypes == float
 
 
-def test_error_if_rare_labels_not_permitted_value():
+@pytest.mark.parametrize("errors", ["empanada", False, 1])
+def test_exception_if_errors_gets_not_permitted_value(errors):
     with pytest.raises(ValueError):
-        CountFrequencyEncoder(errors="empanada")
+        CountFrequencyEncoder(errors=errors)