Repository: DistrictDataLabs/yellowbrick

Issue:
Recursive Feature Elimination
This [recursive feature elimination heatmap](http://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.RFE.html#sklearn.feature_selection.RFE) would be a cool feature selection visualization to implement in yb - possibly we can hoist some of #164 into a base class (see #58) for feature importance visualizers?


Discussion:
I had this idea that we could do a combo plot of [rfecv features scores](http://scikit-learn.org/stable/auto_examples/feature_selection/plot_rfe_with_cross_validation.html#sphx-glr-auto-examples-feature-selection-plot-rfe-with-cross-validation-py) along with the heatmap. I was putting together some code to do this, however when I got to the heatmap I discovered that was specific to the `digits` dataset, the heatmap was just showing the importance of pixels in a 2D array. 

We can/should still use the ranking attribute to show which features are ranked higher relative to the rest, but I'm not sure how to do that. In the meantime, I'll do the RFECV visualizer to get that in our library. 

Resolving patch:
diff --git a/docs/api/features/rfecv.py b/docs/api/features/rfecv.py
new file mode 100644
--- /dev/null
+++ b/docs/api/features/rfecv.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+# Generates RFECV visualizations for the documentation
+
+import os
+import pandas as pd
+import matplotlib.pyplot as plt
+
+from sklearn.svm import SVC
+from yellowbrick.features import RFECV
+from sklearn.datasets import make_classification
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.model_selection import StratifiedKFold
+
+CWD    = os.path.dirname(__file__)
+DATA   = os.path.join(CWD, "..", "..", "..", "examples", "data")
+IMAGES = os.path.join(CWD, "images")
+
+
+def rfecv_sklearn_example(image="rfecv_sklearn_example.png"):
+    X, y = make_classification(
+        n_samples=1000, n_features=25, n_informative=3, n_redundant=2,
+        n_repeated=0, n_classes=8, n_clusters_per_class=1, random_state=0
+    )
+
+    _, ax = plt.subplots()
+
+    oz = RFECV(SVC(kernel='linear', C=1), ax=ax)
+    oz.fit(X, y)
+    oz.poof(outpath=os.path.join(IMAGES, image))
+
+
+def rfecv_credit_example(image="rfecv_credit.png"):
+    data = pd.read_csv(os.path.join(DATA, "credit", "credit.csv"))
+
+    target = "default"
+    features = [col for col in data.columns if col != target]
+
+    X = data[features]
+    y = data[target]
+
+    _, ax = plt.subplots()
+    cv = StratifiedKFold(5)
+    oz = RFECV(RandomForestClassifier(), ax=ax, cv=cv, scoring='f1_weighted')
+    oz.fit(X, y)
+    oz.poof(outpath=os.path.join(IMAGES, image))
+
+
+if __name__ == '__main__':
+    rfecv_sklearn_example()
+    rfecv_credit_example()
diff --git a/yellowbrick/features/__init__.py b/yellowbrick/features/__init__.py
--- a/yellowbrick/features/__init__.py
+++ b/yellowbrick/features/__init__.py
@@ -25,3 +25,4 @@
 from .jointplot import  JointPlotVisualizer
 from .pca import PCADecomposition, pca_decomposition
 from .importances import FeatureImportances, feature_importances
+from .rfecv import RFECV, rfecv
diff --git a/yellowbrick/features/rfecv.py b/yellowbrick/features/rfecv.py
new file mode 100644
--- /dev/null
+++ b/yellowbrick/features/rfecv.py
@@ -0,0 +1,331 @@
+# yellowbrick.features.rfecv
+# Visualize the number of features selected with recursive feature elimination
+#
+# Author:  Benjamin Bengfort <benjamin@bengfort.com>
+# Created: Tue Apr 03 17:31:37 2018 -0400
+#
+# ID: rfecv.py [] benjamin@bengfort.com $
+
+"""
+Visualize the number of features selected using recursive feature elimination
+"""
+
+##########################################################################
+## Imports
+##########################################################################
+
+import numpy as np
+
+from yellowbrick.base import ModelVisualizer
+from yellowbrick.exceptions import YellowbrickValueError
+
+from sklearn.utils import check_X_y
+from sklearn.feature_selection import RFE
+from sklearn.model_selection import cross_val_score
+
+
+##########################################################################
+## Recursive Feature Elimination
+##########################################################################
+
+class RFECV(ModelVisualizer):
+    """
+    Recursive Feature Elimination, Cross-Validated (RFECV) feature selection.
+
+    Selects the best subset of features for the suplied estimator by removing
+    0 to N features (where N is the number of features) using recursive
+    feature elimination, then selecting the best subset based on the
+    cross-validation score of the model. Recursive feature elimination
+    eliminates n features from a model by fitting the model multiple times and
+    at each step, removing the weakest features, determined by either the
+    ``coef_`` or ``feature_importances_`` attribute of the fitted model.
+
+    The visualization plots the score relative to each subset and shows trends
+    in feature elimination. If the feature elimination CV score is flat, then
+    potentially there are not enough features in the model. An ideal curve is
+    when the score jumps from low to high as the number of features removed
+    increases, then slowly decreases again from the optimal number of
+    features.
+
+    Parameters
+    ----------
+    model : a scikit-learn estimator
+        An object that implements ``fit`` and provides information about the
+        relative importance of features with either a ``coef_`` or
+        ``feature_importances_`` attribute.
+
+        Note that the object is cloned for each validation.
+
+    ax : matplotlib.Axes object, optional
+        The axes object to plot the figure on.
+
+    step : int or float, optional (default=1)
+        If greater than or equal to 1, then step corresponds to the (integer)
+        number of features to remove at each iteration. If within (0.0, 1.0),
+        then step corresponds to the percentage (rounded down) of features to
+        remove at each iteration.
+
+    groups : array-like, with shape (n_samples,), optional
+        Group labels for the samples used while splitting the dataset into
+        train/test set.
+
+    cv : int, cross-validation generator or an iterable, optional
+        Determines the cross-validation splitting strategy.
+        Possible inputs for cv are:
+
+          - None, to use the default 3-fold cross-validation,
+          - integer, to specify the number of folds.
+          - An object to be used as a cross-validation generator.
+          - An iterable yielding train/test splits.
+
+        see the scikit-learn
+        `cross-validation guide <http://scikit-learn.org/stable/modules/cross_validation.html>`_
+        for more information on the possible strategies that can be used here.
+
+    scoring : string, callable or None, optional, default: None
+        A string or scorer callable object / function with signature
+        ``scorer(estimator, X, y)``. See scikit-learn model evaluation
+        documentation for names of possible metrics.
+
+    kwargs : dict
+        Keyword arguments that are passed to the base class and may influence
+        the visualization as defined in other Visualizers.
+
+    Attributes
+    ----------
+    n_features_ : int
+        The number of features in the selected subset
+
+    support_ : array of shape [n_features]
+        A mask of the selected features
+
+    ranking_ : array of shape [n_features]
+        The feature ranking, such that ``ranking_[i]`` corresponds to the
+        ranked position of feature i. Selected features are assigned rank 1.
+
+    cv_scores_ : array of shape [n_subsets_of_features, n_splits]
+        The cross-validation scores for each subset of features and splits in
+        the cross-validation strategy.
+
+    rfe_estimator_ : sklearn.feature_selection.RFE
+        A fitted RFE estimator wrapping the original estimator. All estimator
+        functions such as ``predict()`` and ``score()`` are passed through to
+        this estimator (it rewraps the original model).
+
+    Notes
+    -----
+    This model wraps ``sklearn.feature_selection.RFE`` and not
+    ``sklearn.feature_selection.RFECV`` because access to the internals of the
+    CV and RFE estimators is required for the visualization. The visualizer
+    does take similar arguments, however it does not expose the same internal
+    attributes.
+
+    Additionally, the RFE model can be accessed via the ``rfe_estimator_``
+    attribute. Once fitted, the visualizer acts as a wrapper for this
+    estimator and not for the original model passed to the model. This way the
+    visualizer model can be used to make predictions.
+
+    .. caution:: This visualizer requires a model that has either a ``coef_``
+        or ``feature_importances_`` attribute when fitted.
+    """
+
+    def __init__(self, model, ax=None, step=1, groups=None, cv=None,
+                 scoring=None, **kwargs):
+
+        # Initialize the model visualizer
+        super(RFECV, self).__init__(model, ax=ax, **kwargs)
+
+        # Set parameters
+        self.set_params(step=step, groups=groups, cv=cv, scoring=scoring)
+
+    def fit(self, X, y=None):
+        """
+        Fits the RFECV with the wrapped model to the specified data and draws
+        the rfecv curve with the optimal number of features found.
+
+        Parameters
+        ----------
+        X : array-like, shape (n_samples, n_features)
+            Training vector, where n_samples is the number of samples and
+            n_features is the number of features.
+
+        y : array-like, shape (n_samples) or (n_samples, n_features), optional
+            Target relative to X for classification or regression.
+
+        Returns
+        -------
+        self : instance
+            Returns the instance of the RFECV visualizer.
+        """
+        X, y = check_X_y(X, y, "csr")
+        n_features = X.shape[1]
+
+        # This check is kind of unnecessary since RFE will do it, but it's
+        # nice to get it out of the way ASAP and raise a meaningful error.
+        if 0.0 < self.step < 1.0:
+            step = int(max(1, self.step * n_features))
+        else:
+            step = int(self.step)
+
+        if step < 0:
+            raise YellowbrickValueError("step must be >0")
+
+        # Create the RFE model
+        rfe = RFE(self.estimator, step=step)
+        n_feature_subsets = np.arange(1, n_features+1)
+
+        # Create the cross validation params
+        # TODO: handle random state
+        cv_params = {
+            key: self.get_params()[key]
+            for key in ('groups', 'cv', 'scoring')
+        }
+
+        # Perform cross-validation for each feature subset
+        scores = []
+        for n_features_to_select in n_feature_subsets:
+            rfe.set_params(n_features_to_select=n_features_to_select)
+            scores.append(cross_val_score(rfe, X, y, **cv_params))
+
+        # Convert scores to array
+        self.cv_scores_ = np.array(scores)
+
+        # Find the best RFE model
+        bestidx = self.cv_scores_.mean(axis=1).argmax()
+        self.n_features_ = n_feature_subsets[bestidx]
+
+        # Fit the final RFE model for the number of features
+        self.rfe_estimator_ = rfe
+        self.rfe_estimator_.set_params(n_features_to_select=self.n_features_)
+        self.rfe_estimator_.fit(X, y)
+
+        # Rewrap the visualizer to use the rfe estimator
+        self._wrapped = self.rfe_estimator_
+
+        # Hoist the RFE params to the visualizer
+        self.support_ = self.rfe_estimator_.support_
+        self.ranking_ = self.rfe_estimator_.ranking_
+
+        self.draw()
+        return self
+
+    def draw(self, **kwargs):
+        """
+        Renders the rfecv curve.
+        """
+        # Compute the curves
+        x = np.arange(1, len(self.cv_scores_)+1)
+        means = self.cv_scores_.mean(axis=1)
+        sigmas = self.cv_scores_.std(axis=1)
+
+
+        # Plot one standard deviation above and below the mean
+        self.ax.fill_between(x, means - sigmas, means+sigmas, alpha=0.25)
+
+        # Plot the curve
+        self.ax.plot(x, means, 'o-')
+
+        # Plot the maximum number of features
+        self.ax.axvline(
+            self.n_features_, c='k', ls='--',
+            label="n_features = {}\nscore = {:0.3f}".format(
+                self.n_features_, self.cv_scores_.mean(axis=1).max()
+            )
+        )
+
+        return self.ax
+
+    def finalize(self, **kwargs):
+        """
+        Add the title, legend, and other visual final touches to the plot.
+        """
+        # Set the title of the figure
+        self.set_title('RFECV for {}'.format(self.name))
+
+        # Add the legend
+        self.ax.legend(frameon=True, loc='best')
+
+        # Set the axis labels
+        self.ax.set_xlabel('Number of Features Selected')
+        self.ax.set_ylabel('Score')
+
+
+##########################################################################
+## Quick Methods
+##########################################################################
+
+def rfecv(model, X, y, ax=None, step=1, groups=None, cv=None,
+          scoring=None, **kwargs):
+    """
+    Performs recursive feature elimination with cross-validation to determine
+    an optimal number of features for a model. Visualizes the feature subsets
+    with respect to the cross-validation score.
+
+    This helper function is a quick wrapper to utilize the RFECV visualizer
+    for one-off analysis.
+
+    Parameters
+    ----------
+    model : a scikit-learn estimator
+        An object that implements ``fit`` and provides information about the
+        relative importance of features with either a ``coef_`` or
+        ``feature_importances_`` attribute.
+
+        Note that the object is cloned for each validation.
+
+    X : array-like, shape (n_samples, n_features)
+        Training vector, where n_samples is the number of samples and
+        n_features is the number of features.
+
+    y : array-like, shape (n_samples) or (n_samples, n_features), optional
+        Target relative to X for classification or regression.
+
+    ax : matplotlib.Axes object, optional
+        The axes object to plot the figure on.
+
+    step : int or float, optional (default=1)
+        If greater than or equal to 1, then step corresponds to the (integer)
+        number of features to remove at each iteration. If within (0.0, 1.0),
+        then step corresponds to the percentage (rounded down) of features to
+        remove at each iteration.
+
+    groups : array-like, with shape (n_samples,), optional
+        Group labels for the samples used while splitting the dataset into
+        train/test set.
+
+    cv : int, cross-validation generator or an iterable, optional
+        Determines the cross-validation splitting strategy.
+        Possible inputs for cv are:
+
+          - None, to use the default 3-fold cross-validation,
+          - integer, to specify the number of folds.
+          - An object to be used as a cross-validation generator.
+          - An iterable yielding train/test splits.
+
+        see the scikit-learn
+        `cross-validation guide <http://scikit-learn.org/stable/modules/cross_validation.html>`_
+        for more information on the possible strategies that can be used here.
+
+    scoring : string, callable or None, optional, default: None
+        A string or scorer callable object / function with signature
+        ``scorer(estimator, X, y)``. See scikit-learn model evaluation
+        documentation for names of possible metrics.
+
+    kwargs : dict
+        Keyword arguments that are passed to the base class and may influence
+        the visualization as defined in other Visualizers. These arguments are
+        also passed to the `poof()` method, e.g. can pass a path to save the
+        figure to.
+
+    Returns
+    -------
+    ax : matplotlib axes
+        Returns the axes that the rfecv were drawn on.
+    """
+    # Initialize the visualizer
+    oz = RFECV(model, ax=ax, step=step, groups=groups, cv=cv, scoring=scoring)
+
+    # Fit and poof the visualizer
+    oz.fit(X, y)
+    oz.poof(**kwargs)
+    return oz.ax


Test patch:
diff --git a/tests/baseline_images/test_features/test_rfecv/test_pandas_integration.png b/tests/baseline_images/test_features/test_rfecv/test_pandas_integration.png
new file mode 100644
Binary files /dev/null and b/tests/baseline_images/test_features/test_rfecv/test_pandas_integration.png differ
diff --git a/tests/baseline_images/test_features/test_rfecv/test_quick_method.png b/tests/baseline_images/test_features/test_rfecv/test_quick_method.png
new file mode 100644
Binary files /dev/null and b/tests/baseline_images/test_features/test_rfecv/test_quick_method.png differ
diff --git a/tests/baseline_images/test_features/test_rfecv/test_rfecv_classification.png b/tests/baseline_images/test_features/test_rfecv/test_rfecv_classification.png
new file mode 100644
Binary files /dev/null and b/tests/baseline_images/test_features/test_rfecv/test_rfecv_classification.png differ
diff --git a/tests/test_features/test_rfecv.py b/tests/test_features/test_rfecv.py
new file mode 100644
--- /dev/null
+++ b/tests/test_features/test_rfecv.py
@@ -0,0 +1,156 @@
+# tests.test_feautures.test_rfecv
+# Tests for the RFECV visualizer
+#
+# Author:  Benjamin Bengfort <benjamin@bengfort.com>
+# Created: Tue Apr 03 17:35:16 2018 -0400
+#
+# ID: test_rfecv.py [] benjamin@bengfort.com $
+
+"""
+Tests for the RFECV visualizer
+"""
+
+##########################################################################
+## Imports
+##########################################################################
+
+import pytest
+
+from tests.base import VisualTestCase
+from tests.dataset import DatasetMixin, Dataset
+
+from yellowbrick.features.rfecv import *
+from yellowbrick.exceptions import YellowbrickValueError
+
+from sklearn.svm import SVC
+from sklearn.model_selection import ShuffleSplit
+from sklearn.model_selection import StratifiedKFold
+from sklearn.datasets import make_classification
+from sklearn.linear_model import LogisticRegression
+from sklearn.ensemble import RandomForestClassifier
+
+try:
+    import pandas as pd
+except ImportError:
+    pd = None
+
+try:
+    from unittest.mock import patch
+except ImportError:
+    from mock import patch
+
+
+##########################################################################
+## Fixtures
+##########################################################################
+
+@pytest.fixture(scope="class")
+def dataset(request):
+    """
+    Creates a multiclass classification dataset fixture for RFECV
+    """
+    X, y = make_classification(
+        n_samples=600, n_features=15, n_informative=7, n_redundant=4,
+        n_repeated=0, n_classes=8, n_clusters_per_class=1, random_state=0
+    )
+
+    dataset = Dataset(X, y)
+    request.cls.dataset = dataset
+
+
+##########################################################################
+## Test Cases
+##########################################################################
+
+@pytest.mark.usefixtures("dataset")
+class TestRFECV(VisualTestCase, DatasetMixin):
+    """
+    Test the RFECV visualizer
+    """
+
+    @patch.object(RFECV, 'draw')
+    def test_fit(self, mock_draw):
+        """
+        Assert that fit returns self and creates expected properties with NB
+        """
+        X, y = self.dataset
+        params = (
+            "n_features_", "support_", "ranking_",
+            "cv_scores_", "rfe_estimator_",
+        )
+
+        rf = RandomForestClassifier()
+        oz = RFECV(rf)
+        for param in params:
+            assert not hasattr(oz, param)
+
+        # Assert original estimator is wrapped
+        assert oz._wrapped is rf
+
+        assert oz.fit(X, y) is oz
+        mock_draw.assert_called_once()
+
+        for param in params:
+            assert hasattr(oz, param)
+
+        # Assert rfe estimator is now wrapped
+        assert oz._wrapped is not rf
+        assert oz._wrapped is oz.rfe_estimator_
+
+    def test_rfecv_classification(self):
+        """
+        Test image closeness on a classification dataset with an SVM
+        """
+        cv = ShuffleSplit(3, random_state=21)
+        oz = RFECV(SVC(kernel="linear", C=1), cv=cv)
+        oz.fit(self.dataset.X, self.dataset.y)
+        oz.poof()
+
+        self.assert_images_similar(oz)
+
+    @pytest.mark.filterwarnings('ignore:F-score is ill-defined')
+    def test_quick_method(self):
+        """
+        Test the recv quick method works with LogisticRegression
+        """
+        cv = ShuffleSplit(2, random_state=14)
+        model = LogisticRegression()
+        X, y = self.dataset
+
+        ax = rfecv(model, X, y, step=3, cv=cv, scoring='f1_weighted')
+
+        self.assert_images_similar(ax=ax)
+
+    @pytest.mark.skipif(pd is None, reason="test requires pandas")
+    def test_pandas_integration(self):
+        """
+        Test on a real dataset with pandas DataFrame and Series
+        """
+        df = self.load_pandas("occupancy")
+
+        target = "occupancy"
+        features = [
+            'temperature', 'relative humidity', 'light', 'C02', 'humidity'
+        ]
+
+        X = df[features]
+        y = df[target]
+
+        assert isinstance(X, pd.DataFrame)
+        assert isinstance(y, pd.Series)
+
+        cv = StratifiedKFold(n_splits=4, random_state=32)
+        oz = RFECV(RandomForestClassifier(random_state=83), cv=cv)
+        oz.fit(X, y)
+        oz.poof()
+
+        self.assert_images_similar(oz)
+
+    def test_valid_step(self):
+        """
+        Test step hyperparam validation
+        """
+        # TODO: parametrize when unittest is removed
+        with pytest.raises(YellowbrickValueError):
+            oz = RFECV(SVC(kernel="lnear"), step=-1)
+            oz.fit(self.dataset.X, self.dataset.y)
diff --git a/tests/test_utils/test_wrapper.py b/tests/test_utils/test_wrapper.py
--- a/tests/test_utils/test_wrapper.py
+++ b/tests/test_utils/test_wrapper.py
@@ -17,11 +17,10 @@
 ## Imports
 ##########################################################################
 
-import unittest
-
 from yellowbrick.base import Visualizer
 from yellowbrick.utils.wrapper import *
 from sklearn.naive_bayes import MultinomialNB
+from sklearn.naive_bayes import GaussianNB
 
 try:
     from unittest import mock
@@ -70,7 +69,10 @@ def foo(self, a, b):
 ## Wrapper Test Case
 ##########################################################################
 
-class WrapperTests(unittest.TestCase):
+class TestWrapper(object):
+    """
+    Test the object Wrapper mixin utility
+    """
 
     def test_wrapper_methods(self):
         """
@@ -79,9 +81,9 @@ def test_wrapper_methods(self):
         obj = WrappedEstimator()
 
         # Assert that all the wrapper methods are called
-        self.assertTrue(obj.draw())
-        self.assertEqual(obj.foo(2,2), 4)
-        self.assertIsNotNone(obj.estimator)
+        assert obj.draw()
+        assert obj.foo(2,2) == 4
+        assert obj.estimator is not None
 
     def test_super_methods(self):
         """
@@ -96,7 +98,7 @@ def test_super_methods(self):
         obj.poof()
         obj.set_title()
 
-        self.assertIsNone(obj.ax)
+        assert obj.ax is None
         obj.fit.assert_called_once_with()
         obj.finalize.assert_called_once_with()
         obj.poof.assert_called_once_with()
@@ -104,7 +106,7 @@ def test_super_methods(self):
 
     def test_wrapped_methods(self):
         """
-        Assert that wrapped estimator methods are calle d
+        Assert that wrapped estimator methods are called
         """
         obj = WrappedEstimator()
 
@@ -116,3 +118,21 @@ def test_wrapped_methods(self):
         obj._wrapped.predict.assert_called_once_with()
         obj._wrapped.predict_proba.assert_called_once_with()
         obj._wrapped.score.assert_called_once_with()
+
+    def test_rewrap_object(self):
+        """
+        Test the ability to "rewrap" an object on demand
+        """
+        obj = WrappedEstimator()
+        old = obj._wrapped
+        new = mock.MagicMock(spec=GaussianNB())
+
+        obj.predict()
+        old.predict.assert_called_once()
+        new.assert_not_called()
+
+        # rewrap
+        obj._wrapped = new
+        obj.predict()
+        old.predict.assert_called_once()
+        new.predict.assert_called_once()