Repository: feature-engine/feature_engine
Pull request #308 (2021-09-13 17:05:21)

Add match variables closes #132
Add transformer to ensure that columns in test dataset are similar to train dataset.

solegalli: closes #132 

Check that PR for discussions, planning and code revisions

Patch:
diff --git a/README.md b/README.md
index 8c520f1..c911439 100644
--- a/README.md
+++ b/README.md
@@ -56,6 +56,7 @@ More resources will be added as they appear online!
 * Variable Creation
 * Variable Selection
 * Scikit-learn Wrappers
+* Preprocessing
 
 ### Imputing Methods
 * MeanMedianImputer
@@ -115,6 +116,8 @@ More resources will be added as they appear online!
  * RecursiveFeatureElimination
  * RecursiveFeatureAddition
 
+### Preprocessing
+ * MatchVariables
 
 ## Installing
 
diff --git a/docs/index.rst b/docs/index.rst
index 3f07c9e..5572b90 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -23,6 +23,7 @@ Feature-engine includes transformers for:
 - Outlier capping or removal
 - Variable combination
 - Variable selection
+- Preprocessing
 
 Feature-engine allows you to select the variables you want to transform within each
 transformer. This way, different engineering procedures can be easily applied to
@@ -166,6 +167,10 @@ Feature Selection:
 - :doc:`selection/RecursiveFeatureElimination`: selects features recursively, by evaluating model performance
 - :doc:`selection/RecursiveFeatureAddition`: selects features recursively, by evaluating model performance
 
+Preprocessing:
+~~~~~~~~~~~~~~
+
+- :doc:`preprocessing/MatchVariables`: ensures that columns in test set match those in train set
 
 Getting Help
 ------------
@@ -237,6 +242,7 @@ The `issues <https://github.com/feature-engine/feature_engine/issues/>`_ and
    outliers/index
    creation/index
    selection/index
+   preprocessing/index
    wrappers/index
 
 .. toctree::
diff --git a/docs/preprocessing/MatchVariables.rst b/docs/preprocessing/MatchVariables.rst
new file mode 100644
index 0000000..3999480
--- /dev/null
+++ b/docs/preprocessing/MatchVariables.rst
@@ -0,0 +1,168 @@
+MatchVariables
+==============
+
+API Reference
+-------------
+
+.. autoclass:: feature_engine.preprocessing.MatchVariables
+    :members:
+
+
+Example
+-------
+
+MatchVariables() ensures that the columns in the test set are identical to those
+in the train set.
+
+If the test set contains additional columns, they are dropped. Alternatively, if the
+test set lacks columns that were present in the train set, they will be added with a
+value determined by the user, for example np.nan.
+
+
+.. code:: python
+
+    import numpy as np
+    import pandas as pd
+
+    from feature_engine.preprocessing import MatchVariables
+
+
+    # Load dataset
+    def load_titanic():
+        data = pd.read_csv('https://www.openml.org/data/get_csv/16826755/phpMYEkMl')
+        data = data.replace('?', np.nan)
+        data['cabin'] = data['cabin'].astype(str).str[0]
+        data['pclass'] = data['pclass'].astype('O')
+        data['age'] = data['age'].astype('float')
+        data['fare'] = data['fare'].astype('float')
+        data['embarked'].fillna('C', inplace=True)
+        data.drop(
+            labels=['name', 'ticket', 'boat', 'body', 'home.dest'],
+            axis=1, inplace=True,
+        )
+        return data
+
+    # load data as pandas dataframe
+    data = load_titanic()
+
+    # Split test and train
+    train = data.iloc[0:1000, :]
+    test = data.iloc[1000:, :]
+
+    # set up the transformer
+    match_cols = MatchVariables(missing_values="ignore")
+
+    # learn the variables in the train set
+    match_cols.fit(train)
+
+    # the transformer stores the input variables
+    match_cols.input_features_
+
+
+.. code:: python
+
+    ['pclass',
+     'survived',
+     'sex',
+     'age',
+     'sibsp',
+     'parch',
+     'fare',
+     'cabin',
+     'embarked']
+
+
+.. code:: python
+
+    # Let's drop some columns in the test set for the demo
+    test_t = test.drop(["sex", "age"], axis=1)
+
+    test_t.head()
+
+.. code:: python
+
+         pclass  survived  sibsp  parch     fare cabin embarked
+    1000      3         1      0      0   7.7500     n        Q
+    1001      3         1      2      0  23.2500     n        Q
+    1002      3         1      2      0  23.2500     n        Q
+    1003      3         1      2      0  23.2500     n        Q
+    1004      3         1      0      0   7.7875     n        Q
+
+
+.. code:: python
+
+    # the transformer adds the columns back
+    test_tt = match_cols.transform(test_t)
+
+    test_tt.head()
+
+.. code:: python
+
+    The following variables are added to the DataFrame: ['sex', 'age']
+
+         pclass  survived  sex  age  sibsp  parch     fare cabin embarked
+    1000      3         1  NaN  NaN      0      0   7.7500     n        Q
+    1001      3         1  NaN  NaN      2      0  23.2500     n        Q
+    1002      3         1  NaN  NaN      2      0  23.2500     n        Q
+    1003      3         1  NaN  NaN      2      0  23.2500     n        Q
+    1004      3         1  NaN  NaN      0      0   7.7875     n        Q
+
+
+
+Note how the missing columns were added back to the transformed test set, with
+missing values, in the position (i.e., order) in which they were in the train set.
+
+Similarly, if the test set contained additional columns, those would be removed:
+
+.. code:: python
+
+    # let's add some columns for the demo
+    test_t[['var_a', 'var_b']] = 0
+
+    test_t.head()
+
+.. code:: python
+
+         pclass  survived  sibsp  parch     fare cabin embarked  var_a  var_b
+    1000      3         1      0      0   7.7500     n        Q      0      0
+    1001      3         1      2      0  23.2500     n        Q      0      0
+    1002      3         1      2      0  23.2500     n        Q      0      0
+    1003      3         1      2      0  23.2500     n        Q      0      0
+    1004      3         1      0      0   7.7875     n        Q      0      0
+
+
+.. code:: python
+
+    test_tt = match_cols.transform(test_t)
+
+    test_tt.head()
+
+.. code:: python
+
+    The following variables are added to the DataFrame: ['age', 'sex']
+    The following variables are dropped from the DataFrame: ['var_a', 'var_b']
+
+         pclass  survived  sex  age  sibsp  parch     fare cabin embarked
+    1000      3         1  NaN  NaN      0      0   7.7500     n        Q
+    1001      3         1  NaN  NaN      2      0  23.2500     n        Q
+    1002      3         1  NaN  NaN      2      0  23.2500     n        Q
+    1003      3         1  NaN  NaN      2      0  23.2500     n        Q
+    1004      3         1  NaN  NaN      0      0   7.7875     n        Q
+
+
+Now, the transformer simultaneously added the missing columns with NA as values and
+removed the additional columns from the resulting dataset.
+
+These transformer is useful in "predict then optimize type of problems". In such cases,
+a machine learning model is trained on a certain dataset, with certain input features.
+Then, test sets are "post-processed" according to scenarios that want to be modelled.
+For example, "what would have happened if the customer received an email campaign"?
+where the variable "receive_campaign" would be turned from 0 -> 1.
+
+While creating these modelling datasets, a lot of meta data e.g., "scenario number",
+"time scenario was generated", etc, could be added to the data. Then we need to pass
+these data over to the model to obtain the modelled prediction.
+
+MatchVariables() provides an easy an elegant way to remove the additional metadeta,
+while returning datasets with the input features in the correct order, allowing the
+different scenarios to be modelled directly inside a machine learning pipeline.
\ No newline at end of file
diff --git a/docs/preprocessing/index.rst b/docs/preprocessing/index.rst
new file mode 100644
index 0000000..b5bd923
--- /dev/null
+++ b/docs/preprocessing/index.rst
@@ -0,0 +1,12 @@
+.. -*- mode: rst -*-
+
+Preprocessing
+=============
+
+Feature-engine's preprocessing transformers apply general data pre-processing
+and transformation procedures.
+
+.. toctree::
+   :maxdepth: 2
+
+   MatchVariables
diff --git a/feature_engine/preprocessing/__init__.py b/feature_engine/preprocessing/__init__.py
new file mode 100644
index 0000000..c477ab2
--- /dev/null
+++ b/feature_engine/preprocessing/__init__.py
@@ -0,0 +1,10 @@
+"""
+The module preprocessing includes classes and functions for general data pre-processing
+and transformation.
+"""
+
+from .match_columns import MatchVariables
+
+__all__ = [
+    "MatchVariables",
+]
diff --git a/feature_engine/preprocessing/match_columns.py b/feature_engine/preprocessing/match_columns.py
new file mode 100644
index 0000000..2f8fe60
--- /dev/null
+++ b/feature_engine/preprocessing/match_columns.py
@@ -0,0 +1,197 @@
+from typing import List, Union
+
+import numpy as np
+import pandas as pd
+from sklearn.base import BaseEstimator, TransformerMixin
+from sklearn.utils.validation import check_is_fitted
+
+from feature_engine.dataframe_checks import _check_contains_na, _is_dataframe
+from feature_engine.validation import _return_tags
+
+
+class MatchVariables(BaseEstimator, TransformerMixin):
+    """
+    MatchVariables() ensures that the same variables observed in the train set
+    are present in the test set. If the dataset to transform contains variables that
+    were not present in the train set, they are dropped. If the dataset to transform
+    lacks variables that were present in the train set, these variables are added to
+    the dataframe with a value determined by the user (np.nan by default).
+
+    .. code-block:: python
+
+        train = pd.DataFrame({
+            "Name": ["tom", "nick", "krish", "jack"],
+            "City": ["London", "Manchester", "Liverpool", "Bristol"],
+            "Age": [20, 21, 19, 18],
+            "Marks": [0.9, 0.8, 0.7, 0.6],
+        })
+
+        test = pd.DataFrame({
+            "Name": ["tom", "sam", "nick"],
+            "Age": [20, 22, 23],
+            "Marks": [0.9, 0.7, 0.6],
+            "Hobbies": ["tennis", "rugby", "football"]
+        })
+
+        match_columns = MatchVariables()
+
+        match_columns.fit(train)
+
+        df_transformed = match_columns.transform(test)
+
+        df_transformed
+
+            Name    City  Age  Marks
+        0    tom  np.nan   20    0.9
+        1    sam  np.nan   22    0.7
+        2   nick  np.nan   23    0.6
+
+
+    The order of the variables in the transformed dataset is also adjusted to match
+    that observed in the train set.
+
+    Parameters
+    ----------
+    fill_value: integer, float or string. Default=np.nan
+        The values for the variables that will be added to the transformed dataset.
+
+    missing_values: string, default='ignore'
+        Indicates if missing values should be ignored or raised. If 'ignore', the
+        transformer will ignore missing data when transforming the data. If 'raise' the
+        transformer will return an error if the training or the datasets to transform
+        contain missing values.
+
+    verbose: bool, default=True
+        If True, the transformer will print out the names of the variables that are
+        added and / or removed from the dataset.
+
+    Attributes
+    ----------
+    input_features_:
+        The variables present in the train set, in the order observed during fit.
+
+    n_features_in_:
+        The number of features in the train set used in fit.
+
+    Methods
+    -------
+    fit:
+        Identify the variable names in the train set.
+    transform:
+        Add or delete variables to match those observed in the train set.
+    fit_transform:
+        Fit to the data. Then transform it.
+    """
+
+    def __init__(
+        self,
+        fill_value: Union[str, int, float] = np.nan,
+        missing_values: str = "raise",
+        verbose: bool = True,
+    ):
+
+        if missing_values not in ["raise", "ignore"]:
+            raise ValueError(
+                "missing_values takes only values 'raise' or 'ignore'."
+                f"Got '{missing_values} instead.")
+
+        if not isinstance(verbose, bool):
+            raise ValueError("verbose takes only booleans True and False."
+                             f"Got '{verbose} instead.")
+
+        # note: np.nan is an instance of float!!!
+        if not isinstance(fill_value, (str, int, float)):
+            raise ValueError("fill_value takes integers, floats or strings."
+                             f"Got '{fill_value} instead.")
+
+        self.fill_value = fill_value
+        self.missing_values = missing_values
+        self.verbose = verbose
+
+    def fit(self, X: pd.DataFrame, y: pd.Series = None):
+        """Learns and stores the names of the variables in the training dataset.
+
+        Parameters
+        ----------
+
+        X: pandas dataframe of shape = [n_samples, n_features]
+            The input dataframe
+
+        y: None
+            y is not needed for this transformer. You can pass y or None.
+
+        Raises
+        ------
+        TypeError
+           - If the input is not a Pandas DataFrame
+
+        Returns
+        -------
+        self
+        """
+        X = _is_dataframe(X)
+
+        if self.missing_values == "raise":
+            # check if dataset contains na
+            _check_contains_na(X, X.columns)
+
+        # save input features
+        self.input_features_: List[Union[str, int]] = X.columns.tolist()
+
+        self.n_features_in_ = X.shape[1]
+
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        """Drops variables that were not seen in the train set and adds variables that
+         were in the train set but not in the data to transform.
+
+        Parameters
+        ----------
+        X: pandas dataframe of shape = [n_samples, n_features]
+            The data to transform.
+
+        Returns
+        -------
+        X_transformed: Pandas dataframe, shape = [n_samples, n_features]
+             The dataframe with variables that match those observed in the train set.
+        """
+        check_is_fitted(self)
+
+        X = _is_dataframe(X)
+
+        if self.missing_values == "raise":
+            # check if dataset contains na
+            _check_contains_na(X, self.input_features_)
+
+        _columns_to_drop = list(set(X.columns) - set(self.input_features_))
+        _columns_to_add = list(set(self.input_features_) - set(X.columns))
+
+        if self.verbose:
+            if len(_columns_to_add) > 0:
+                print("The following variables are added to the DataFrame: "
+                      f"{_columns_to_add}")
+            if len(_columns_to_drop) > 0:
+                print(
+                    "The following variables are dropped from the DataFrame: "
+                    f"{_columns_to_drop}")
+
+        X = X.drop(_columns_to_drop, axis=1)
+
+        X = X.reindex(columns=self.input_features_, fill_value=self.fill_value)
+
+        return X
+
+    # for the check_estimator tests
+    def _more_tags(self):
+        tags_dict = _return_tags()
+
+        msg = "input shape of dataframes in fit and transform can differ"
+        tags_dict["_xfail_checks"]["check_transformer_general"] = msg
+
+        msg = (
+            "transformer takes categorical variables, and inf cannot be determined"
+            "on these variables. Thus, check is not implemented")
+        tags_dict["_xfail_checks"]["check_estimators_nan_inf"] = msg
+
+        return tags_dict


Test patch:
diff --git a/tests/test_preprocessing/__init__.py b/tests/test_preprocessing/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/test_preprocessing/test_check_estimator_preprocessing.py b/tests/test_preprocessing/test_check_estimator_preprocessing.py
new file mode 100644
index 0000000..b082b52
--- /dev/null
+++ b/tests/test_preprocessing/test_check_estimator_preprocessing.py
@@ -0,0 +1,14 @@
+import pytest
+from sklearn.utils.estimator_checks import check_estimator
+
+from feature_engine.preprocessing import MatchVariables
+
+
+@pytest.mark.parametrize(
+    "Estimator",
+    [
+        MatchVariables(),
+    ],
+)
+def test_all_transformers(Estimator):
+    return check_estimator(Estimator)
diff --git a/tests/test_preprocessing/test_match_columns.py b/tests/test_preprocessing/test_match_columns.py
new file mode 100644
index 0000000..ec1be0a
--- /dev/null
+++ b/tests/test_preprocessing/test_match_columns.py
@@ -0,0 +1,183 @@
+import numpy as np
+import pandas as pd
+import pytest
+from sklearn.exceptions import NotFittedError
+
+from feature_engine.preprocessing import MatchVariables
+
+_params_fill_value = [
+    (1, [1, 1, 1, 1], [1, 1, 1, 1]),
+    (0.1, [0.1, 0.1, 0.1, 0.1], [0.1, 0.1, 0.1, 0.1]),
+    ("none", ["none", "none", "none",
+              "none"], ["none", "none", "none", "none"]),
+    (np.nan, [np.nan, np.nan, np.nan,
+              np.nan], [np.nan, np.nan, np.nan, np.nan]),
+]
+
+_params_allowed = [
+    ([0, 1], "ignore", True),
+    ("nan", "hola", True),
+    ("nan", "ignore", "hallo"),
+]
+
+
+@pytest.mark.parametrize("fill_value, expected_studies, expected_age",
+                         _params_fill_value)
+def test_drop_and_add_columns(fill_value, expected_studies, expected_age,
+                              df_vartypes, df_na):
+    train = df_na.copy()
+    test = df_vartypes.copy()
+    test = test.drop("Age", axis=1)  # to add more than one column
+
+    # adding columns to test if they are removed
+    for new_col in ["test1", "test2"]:
+        test.loc[:, new_col] = new_col
+
+    match_columns = MatchVariables(
+        fill_value=fill_value,
+        missing_values="ignore",
+    )
+    match_columns.fit(train)
+
+    transformed_df = match_columns.transform(test)
+
+    expected_result = pd.DataFrame({
+        "Name": ["tom", "nick", "krish", "jack"],
+        "City": ["London", "Manchester", "Liverpool", "Bristol"],
+        "Studies": expected_studies,
+        "Age": expected_age,
+        "Marks": [0.9, 0.8, 0.7, 0.6],
+        "dob": pd.date_range("2020-02-24", periods=4, freq="T"),
+    })
+
+    # test init params
+    if fill_value is np.nan:
+        assert match_columns.fill_value is np.nan
+    else:
+        assert match_columns.fill_value == fill_value
+    assert match_columns.verbose is True
+    assert match_columns.missing_values == "ignore"
+    # test fit attrs
+    assert list(match_columns.input_features_) == list(train.columns)
+    assert match_columns.n_features_in_ == 6
+    # test transform output
+    pd.testing.assert_frame_equal(expected_result, transformed_df)
+
+
+@pytest.mark.parametrize("fill_value, expected_studies, expected_age",
+                         _params_fill_value)
+def test_columns_addition_when_more_columns_in_train_than_test(
+        fill_value, expected_studies, expected_age, df_vartypes, df_na):
+
+    train = df_na.copy()
+    test = df_vartypes.copy()
+    test = test.drop("Age", axis=1)  # to add more than one column
+
+    match_columns = MatchVariables(
+        fill_value=fill_value,
+        missing_values="ignore",
+    )
+    match_columns.fit(train)
+
+    transformed_df = match_columns.transform(test)
+
+    expected_result = pd.DataFrame({
+        "Name": ["tom", "nick", "krish", "jack"],
+        "City": ["London", "Manchester", "Liverpool", "Bristol"],
+        "Studies": expected_studies,
+        "Age": expected_age,
+        "Marks": [0.9, 0.8, 0.7, 0.6],
+        "dob": pd.date_range("2020-02-24", periods=4, freq="T"),
+    })
+
+    # test init params
+    if fill_value is np.nan:
+        assert match_columns.fill_value is np.nan
+    else:
+        assert match_columns.fill_value == fill_value
+    assert match_columns.verbose is True
+    assert match_columns.missing_values == "ignore"
+    # test fit attrs
+    assert list(match_columns.input_features_) == list(train.columns)
+    assert match_columns.n_features_in_ == 6
+    # test transform output
+    pd.testing.assert_frame_equal(expected_result, transformed_df)
+
+
+def test_drop_columns_when_more_columns_in_test_than_train(df_vartypes, df_na):
+    train = df_vartypes.copy()
+    train = train.drop("City", axis=1)  # to remove more than one column
+    test = df_na.copy()
+
+    match_columns = MatchVariables(missing_values="ignore")
+    match_columns.fit(train)
+
+    transformed_df = match_columns.transform(test)
+
+    expected_result = test.drop(columns=["Studies", "City"])
+
+    # test init params
+    assert match_columns.fill_value is np.nan
+    assert match_columns.verbose is True
+    assert match_columns.missing_values == "ignore"
+    # test fit attrs
+    assert list(match_columns.input_features_) == list(train.columns)
+    assert match_columns.n_features_in_ == 4
+    # test transform output
+    pd.testing.assert_frame_equal(expected_result, transformed_df)
+
+
+@pytest.mark.parametrize("fill_value, missing_values, verbose",
+                         _params_allowed)
+def test_error_if_param_values_not_allowed(fill_value, missing_values,
+                                           verbose):
+    with pytest.raises(ValueError):
+        MatchVariables(fill_value=fill_value,
+                       missing_values=missing_values,
+                       verbose=verbose)
+
+
+def test_verbose_print_out(capfd, df_vartypes, df_na):
+
+    match_columns = MatchVariables(missing_values="ignore",
+                                   verbose=True)
+
+    train = df_na.copy()
+    train.loc[:, "new_variable"] = 5
+
+    match_columns.fit(train)
+    match_columns.transform(df_vartypes)
+
+    out, err = capfd.readouterr()
+    assert (out == "The following variables are added to the DataFrame: "
+            "['new_variable', 'Studies']\n"
+            or out == "The following variables are added to the DataFrame: "
+            "['Studies', 'new_variable']\n")
+
+    match_columns.fit(df_vartypes)
+    match_columns.transform(train)
+
+    out, err = capfd.readouterr()
+    assert (out == "The following variables are dropped from the DataFrame: "
+            "['new_variable', 'Studies']\n" or out
+            == "The following variables are dropped from the DataFrame: "
+            "['Studies', 'new_variable']\n")
+
+
+def test_raises_error_if_na_in_df(df_na, df_vartypes):
+    # when dataset contains na, fit method
+    with pytest.raises(ValueError):
+        transformer = MatchVariables()
+        transformer.fit(df_na)
+
+    # when dataset contains na, transform method
+    with pytest.raises(ValueError):
+        transformer = MatchVariables()
+        transformer.fit(df_vartypes)
+        transformer.transform(df_na)
+
+
+def test_non_fitted_error(df_vartypes):
+    with pytest.raises(NotFittedError):
+        transformer = MatchVariables()
+        transformer.transform(df_vartypes)