Repository: solegalli/feature_engine

Issue:
feature_selection: select by individual feature performance with a ML algo
A transformer that implements this:

https://github.com/solegalli/feature-selection-for-machine-learning/blob/master/06-Filter-other-metrics/06.1-Univariate-roc-auc.ipynb

Should be able to take pretty much any ML algorithm

Comments:

Patch:
diff --git a/docs/index.rst b/docs/index.rst
index 7601d5b..0d95b18 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -176,9 +176,9 @@ Feature Selection:
 - :doc:`selection/DropDuplicateFeatures`: drops duplicated variables from a dataframe
 - :doc:`selection/DropCorrelatedFeatures`: drops correlated variables from a dataframe
 - :doc:`selection/ShuffleFeaturesSelector`: selects features by evaluating model performance after feature shuffling
+- :doc:`selection/SelectBySingleFeaturePerformance`: selects features based on their performance on univariate estimators
 - :doc:`selection/RecursiveFeatureElimination`: selects features recursively, by evaluating model performance
 
-
 Getting Help
 ------------
 
diff --git a/docs/selection/SelectBySingleFeaturePerformance.rst b/docs/selection/SelectBySingleFeaturePerformance.rst
new file mode 100644
index 0000000..5b3c632
--- /dev/null
+++ b/docs/selection/SelectBySingleFeaturePerformance.rst
@@ -0,0 +1,56 @@
+SelectBySingleFeaturePerformance
+================================
+
+The SelectBySingleFeaturePerformance()selects features based on the performance of
+machine learning models trained using individual features. In other words, selects
+features based on their individual performance, returned by estimators trained on
+only that particular feature.
+
+.. code:: python
+
+    import pandas as pd
+    from sklearn.datasets import load_diabetes
+    from sklearn.linear_model import LinearRegression
+    from feature_engine.selection import SelectBySingleFeaturePerformance
+
+    # load dataset
+    diabetes_X, diabetes_y = load_diabetes(return_X_y=True)
+    X = pd.DataFrame(diabetes_X)
+    y = pd.DataFrame(diabetes_y)
+
+    # initialize feature selector
+    sel = SelectBySingleFeaturePerformance(
+            estimator=LinearRegression(), scoring="r2", cv=3, threshold=0.01)
+
+    # fit transformer
+    sel.fit(X, y)
+
+    sel.selected_features_
+
+.. code:: python
+
+    [0, 2, 3, 4, 5, 6, 7, 8, 9]
+
+..  code:: python
+
+    sel.feature_importance_
+
+.. code:: python
+
+    {0: 0.029231969375784466,
+    1: -0.003738551760264386,
+    2: 0.336620809987693,
+    3: 0.19219056680145055,
+    4: 0.037115559827549806,
+    5: 0.017854228256932614,
+    6: 0.15153886177526896,
+    7: 0.17721609966501747,
+    8: 0.3149462084418813,
+    9: 0.13876602125792703}
+
+
+API Reference
+-------------
+
+.. autoclass:: feature_engine.selection.SelectBySingleFeaturePerformance
+    :members:
\ No newline at end of file
diff --git a/docs/selection/index.rst b/docs/selection/index.rst
index ec698bc..bbf6b46 100644
--- a/docs/selection/index.rst
+++ b/docs/selection/index.rst
@@ -14,4 +14,5 @@ Or in other words to select subsets of variables.
    DropDuplicateFeatures
    DropCorrelatedFeatures
    ShuffleFeaturesSelector
-   RecursiveFeatureElimination
\ No newline at end of file
+   SelectBySingleFeaturePerformance
+   RecursiveFeatureElimination
diff --git a/feature_engine/selection/__init__.py b/feature_engine/selection/__init__.py
index 4d17261..deaf2e7 100644
--- a/feature_engine/selection/__init__.py
+++ b/feature_engine/selection/__init__.py
@@ -5,8 +5,9 @@ from .drop_features import DropFeatures
 from .drop_constant_features import DropConstantFeatures
 from .drop_duplicate_features import DropDuplicateFeatures
 from .drop_correlated_features import DropCorrelatedFeatures
-from .recursive_feature_elimination import RecursiveFeatureElimination
 from .shuffle_features import ShuffleFeaturesSelector
+from .single_feature_performance_selection import SelectBySingleFeaturePerformance
+from .recursive_feature_elimination import RecursiveFeatureElimination
 
 __all__ = [
     "DropFeatures",
@@ -14,5 +15,6 @@ __all__ = [
     "DropDuplicateFeatures",
     "DropCorrelatedFeatures",
     "ShuffleFeaturesSelector",
-    "RecursiveFeatureElimination"
+    "SelectBySingleFeaturePerformance",
+    "RecursiveFeatureElimination",
 ]
diff --git a/feature_engine/selection/single_feature_performance_selection.py b/feature_engine/selection/single_feature_performance_selection.py
new file mode 100644
index 0000000..953bece
--- /dev/null
+++ b/feature_engine/selection/single_feature_performance_selection.py
@@ -0,0 +1,175 @@
+from sklearn.base import BaseEstimator, TransformerMixin
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.model_selection import cross_validate
+from sklearn.utils.validation import check_is_fitted
+
+from feature_engine.dataframe_checks import (
+    _is_dataframe,
+    _check_input_matches_training_df,
+)
+from feature_engine.variable_manipulation import (
+    _define_variables,
+    _find_numerical_variables,
+)
+
+
+class SelectBySingleFeaturePerformance(BaseEstimator, TransformerMixin):
+    """
+
+    SelectBySingleFeaturePerformance selects features based on the performance obtained
+    from a machine learning model trained utilising a single feature. In other words,
+    it trains a machine learning model for every single feature, utilising that
+    individual feature, then determines each model performance. If the performance of
+    the model based on the single feature is greater than a user specified threshold,
+    then the feature is retained, otherwise removed.
+
+    The models trained on the individual features are trained using cross-validation.
+    The performance metric to evaluate and the machine learning model to train are
+    specified by the user.
+
+    Parameters
+    ----------
+
+    variables : str or list, default=None
+        The list of variable(s) to be evaluated.
+        If None, the transformer will evaluate all numerical variables in the dataset.
+
+    estimator: object, default = RandomForestClassifier()
+        A Scikit-learn estimator for regression or classification.
+
+    scoring: str, default='roc_auc'
+        Desired metric to optimise the performance for the estimator. Comes from
+        sklearn.metrics. See the model evaluation documentation for more options:
+        https://scikit-learn.org/stable/modules/model_evaluation.html
+
+    threshold: float, int, default = 0.5
+        The value that defines if a feature will be kept or removed. Note that for
+        metrics like roc-auc, r2_score and accuracy, the thresholds will be floats
+        between 0 and 1. For metrics like the mean_square_error and the
+        root_mean_square_error the threshold will be a big number.
+        The threshold must be defined by the user.
+
+    cv : int, default=3
+        Desired number of cross-validation fold to be used to fit the estimator.
+
+    Attributes
+    ----------
+
+    selected_features_: list
+        The selected features.
+
+    feature_performance_: dict
+        A dictionary containing the feature name as key and the performance of the
+        model trained on each feature as value.
+
+    Methods
+    -------
+
+    fit: finds important features
+
+    transform: removes non-important / non-selected features
+
+    fit_transform: finds and removes non-important features
+
+    """
+
+    def __init__(
+        self,
+        estimator=RandomForestClassifier(),
+        scoring="roc_auc",
+        cv=3,
+        threshold=0.5,
+        variables=None,
+    ):
+
+        if not isinstance(cv, int) or cv < 1:
+            raise ValueError("cv can only take positive integers bigger than 1")
+
+        if not isinstance(threshold, (int, float)):
+            raise ValueError("threshold can only be integer or float")
+
+        self.variables = _define_variables(variables)
+        self.estimator = estimator
+        self.scoring = scoring
+        self.threshold = threshold
+        self.cv = cv
+
+    def fit(self, X, y):
+        """
+
+        Args
+        ----
+
+        X: pandas dataframe of shape = [n_samples, n_features]
+           The input dataframe
+
+        y: array-like of shape (n_samples)
+           Target variable. Required to train the estimator.
+
+
+        Returns
+        -------
+
+        self
+        """
+
+        # check input dataframe
+        X = _is_dataframe(X)
+
+        # find numerical variables or check variables entered by user
+        self.variables = _find_numerical_variables(X, self.variables)
+
+        # list to collect selected features
+        self.selected_features_ = []
+
+        self.feature_performance_ = {}
+
+        # train a model for every feature
+        for feature in self.variables:
+            model = cross_validate(
+                self.estimator,
+                X[feature].to_frame(),
+                y,
+                cv=self.cv,
+                return_estimator=False,
+                scoring=self.scoring,
+            )
+
+            if model["test_score"].mean() > self.threshold:
+                self.selected_features_.append(feature)
+
+            self.feature_performance_[feature] = model["test_score"].mean()
+
+        self.input_shape_ = X.shape
+
+        return self
+
+    def transform(self, X):
+        """
+        Removes non-selected features.
+
+        Args
+        ----
+
+        X: pandas dataframe of shape = [n_samples, n_features].
+            The input dataframe from which feature values will be train.
+
+
+        Returns
+        -------
+
+        X_transformed: pandas dataframe
+            of shape = [n_samples, selected_features]
+            Pandas dataframe with the selected features.
+        """
+
+        # check if fit is performed prior to transform
+        check_is_fitted(self)
+
+        # check if input is a dataframe
+        X = _is_dataframe(X)
+
+        # check if number of columns in test dataset matches to train dataset
+        _check_input_matches_training_df(X, self.input_shape_[1])
+
+        return X[self.selected_features_]


Test patch:
diff --git a/tests/test_selection/test_single_feature_performance_selection.py b/tests/test_selection/test_single_feature_performance_selection.py
new file mode 100644
index 0000000..731834f
--- /dev/null
+++ b/tests/test_selection/test_single_feature_performance_selection.py
@@ -0,0 +1,168 @@
+import numpy as np
+import pandas as pd
+import pytest
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.exceptions import NotFittedError
+from sklearn.linear_model import LinearRegression
+from sklearn.tree import DecisionTreeRegressor
+
+from feature_engine.selection import SelectBySingleFeaturePerformance
+
+
+def test_default_parameters(df_test):
+    X, y = df_test
+    sel = SelectBySingleFeaturePerformance(RandomForestClassifier(random_state=1))
+    sel.fit(X, y)
+
+    # expected result
+    Xtransformed = X.copy()
+    Xtransformed.drop("var_3", 1, inplace=True)
+    Xtransformed.drop("var_10", 1, inplace=True)
+
+    # test init params
+    assert sel.variables == [
+        "var_0",
+        "var_1",
+        "var_2",
+        "var_3",
+        "var_4",
+        "var_5",
+        "var_6",
+        "var_7",
+        "var_8",
+        "var_9",
+        "var_10",
+        "var_11",
+    ]
+    assert sel.threshold == 0.5
+    assert sel.cv == 3
+    assert sel.scoring == "roc_auc"
+    # test fit attrs
+    assert sel.selected_features_ == [
+        "var_0",
+        "var_1",
+        "var_2",
+        "var_4",
+        "var_5",
+        "var_6",
+        "var_7",
+        "var_8",
+        "var_9",
+        "var_11",
+    ]
+    assert sel.feature_performance_ == {
+        "var_0": 0.5957642619540211,
+        "var_1": 0.5365534287221033,
+        "var_2": 0.5001855546283257,
+        "var_3": 0.4752954458526748,
+        "var_4": 0.9780875304971691,
+        "var_5": 0.5065441419357082,
+        "var_6": 0.9758243290622809,
+        "var_7": 0.994571685008432,
+        "var_8": 0.5164434795458892,
+        "var_9": 0.9543427678969847,
+        "var_10": 0.47404183834906727,
+        "var_11": 0.5227164067525513,
+    }
+    # test transform output
+    pd.testing.assert_frame_equal(sel.transform(X), Xtransformed)
+
+
+def test_regression_cv_3_and_r2(load_diabetes_dataset):
+    #  test for regression using cv=3, and the r2 as metric.
+    X, y = load_diabetes_dataset
+    sel = SelectBySingleFeaturePerformance(
+        estimator=LinearRegression(), scoring="r2", cv=3, threshold=0.01
+    )
+    sel.fit(X, y)
+
+    # expected output
+    Xtransformed = pd.DataFrame(X[[0, 2, 3, 4, 5, 6, 7, 8, 9]].copy())
+
+    performance_dict = {
+        0: 0.029,
+        1: -0.004,
+        2: 0.337,
+        3: 0.192,
+        4: 0.037,
+        5: 0.018,
+        6: 0.152,
+        7: 0.177,
+        8: 0.315,
+        9: 0.139,
+    }
+
+    # test init params
+    assert sel.cv == 3
+    assert sel.variables == list(X.columns)
+    assert sel.scoring == "r2"
+    assert sel.threshold == 0.01
+    # fit params
+    assert sel.selected_features_ == [0, 2, 3, 4, 5, 6, 7, 8, 9]
+    assert all(
+        np.round(sel.feature_performance_[f], 3) == performance_dict[f]
+        for f in sel.feature_performance_.keys()
+    )
+    # test transform output
+    pd.testing.assert_frame_equal(sel.transform(X), Xtransformed)
+
+
+def test_regression_cv_2_and_mse(load_diabetes_dataset):
+    #  test for regression using cv=2, and the neg_mean_squared_error as metric.
+    # add suitable threshold for regression mse
+
+    X, y = load_diabetes_dataset
+    sel = SelectBySingleFeaturePerformance(
+        estimator=DecisionTreeRegressor(random_state=0),
+        scoring="neg_mean_squared_error",
+        cv=2,
+        threshold=10,
+    )
+    # fit transformer
+    sel.fit(X, y)
+
+    # expected output
+    Xtransformed = X.copy()
+    Xtransformed.drop(
+        Xtransformed.columns[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]], axis=1, inplace=True
+    )
+
+    # test init params
+    assert sel.cv == 2
+    assert sel.variables == list(X.columns)
+    assert sel.scoring == "neg_mean_squared_error"
+    assert sel.threshold == 10
+    # fit params
+    assert sel.selected_features_ == []
+    assert sel.feature_performance_ == {
+        0: -7657.154138192973,
+        1: -5966.662211695372,
+        2: -6613.779604700854,
+        3: -6502.621725718592,
+        4: -9415.586278197177,
+        5: -11760.999622926094,
+        6: -6592.584431571728,
+        7: -5270.563893676307,
+        8: -7641.414795123177,
+        9: -6287.557824391035,
+    }
+    # test transform output
+    print(sel.transform(X))
+    pd.testing.assert_frame_equal(sel.transform(X), Xtransformed)
+
+
+def test_non_fitted_error(df_test):
+    # when fit is not called prior to transform
+    with pytest.raises(NotFittedError):
+        sel = SelectBySingleFeaturePerformance()
+        sel.transform(df_test)
+
+
+def test_raises_cv_error():
+    with pytest.raises(ValueError):
+        SelectBySingleFeaturePerformance(cv=0)
+
+
+def test_raises_threshold_error():
+    with pytest.raises(ValueError):
+        SelectBySingleFeaturePerformance(threshold=None)