Repository: scikit-learn/scikit-learn
Pull request: ENH add score_samples method in base search CV and add tests
#### Reference Issues/PRs
Fixes https://github.com/scikit-learn/scikit-learn/issues/12542.
Building on and closes https://github.com/scikit-learn/scikit-learn/pull/16275.


#### What does this implement/fix? Explain your changes.
This adds the ability to call `score_samples` on the estimator with the best found parameters in *SearchCV instances.

Problem statement:
Add a score_samples method to Pipeline and *SearchCV
Hello,

scikit-learn 0.20 provides more consistent outlier detection API.

- covariance.EllipticEnvelope
- svm.OneClassSVM
- ensemble.IsolationForest
- neighbors.LocalOutlierFactor

So I would like to use `Pipeline`, `GridSearchCV`, and `RandomizedSearchCV` that have a `score_samples` method.

Thank you.

Hints from issue discussion:
Does #13806 close this or is there something more to address?
Fixed in #13806
I think the *SearchCV are still missing, right?
I am working on this issue as part of the sklearn Paris Sprint 2020.
it looks like the related PR is stale. I will build on this PR and take this for the sprint

Commit message: address some linting issues

Patch to review:
diff --git a/sklearn/model_selection/_search.py b/sklearn/model_selection/_search.py
index f0ead220fafe6..3aabf809db37d 100644
--- a/sklearn/model_selection/_search.py
+++ b/sklearn/model_selection/_search.py
@@ -457,6 +457,26 @@ def score(self, X, y=None):
         score = self.scorer_[self.refit] if self.multimetric_ else self.scorer_
         return score(self.best_estimator_, X, y)
 
+    @if_delegate_has_method(delegate=('best_estimator_', 'estimator'))
+    def score_samples(self, X):
+        """Call score_samples on the estimator with the best found parameters.
+
+        Only available if ``refit=True`` and the underlying estimator supports
+        ``score_samples``.
+
+        Parameters
+        ----------
+        X : iterable
+            Data to predict on. Must fulfill input requirements
+            of the underlying estimator.
+
+        Returns
+        -------
+        y_score : ndarray, shape (n_samples,)
+        """
+        self._check_is_fitted('score_samples')
+        return self.best_estimator_.score_samples(X)
+
     def _check_is_fitted(self, method_name):
         if not self.refit:
             raise NotFittedError('This %s instance was initialized '
@@ -861,9 +881,9 @@ class GridSearchCV(BaseSearchCV):
     Important members are fit, predict.
 
     GridSearchCV implements a "fit" and a "score" method.
-    It also implements "predict", "predict_proba", "decision_function",
-    "transform" and "inverse_transform" if they are implemented in the
-    estimator used.
+    It also implements "score_samples", "predict", "predict_proba",
+    "decision_function", "transform" and "inverse_transform" if they are
+    implemented in the estimator used.
 
     The parameters of the estimator used to apply these methods are optimized
     by cross-validated grid-search over a parameter grid.
@@ -1174,9 +1194,9 @@ class RandomizedSearchCV(BaseSearchCV):
     """Randomized search on hyper parameters.
 
     RandomizedSearchCV implements a "fit" and a "score" method.
-    It also implements "predict", "predict_proba", "decision_function",
-    "transform" and "inverse_transform" if they are implemented in the
-    estimator used.
+    It also implements "score_samples", "predict", "predict_proba",
+    "decision_function", "transform" and "inverse_transform" if they are
+    implemented in the estimator used.
 
     The parameters of the estimator used to apply these methods are optimized
     by cross-validated search over parameter settings.
diff --git a/sklearn/model_selection/tests/test_search.py b/sklearn/model_selection/tests/test_search.py
index 8a20872ac762a..f7328ca678c91 100644
--- a/sklearn/model_selection/tests/test_search.py
+++ b/sklearn/model_selection/tests/test_search.py
@@ -56,6 +56,7 @@
 from sklearn.tree import DecisionTreeClassifier
 from sklearn.cluster import KMeans
 from sklearn.neighbors import KernelDensity
+from sklearn.neighbors import LocalOutlierFactor
 from sklearn.neighbors import KNeighborsClassifier
 from sklearn.metrics import f1_score
 from sklearn.metrics import recall_score
@@ -76,6 +77,7 @@
 # to test hyperparameter search on user-defined classifiers.
 class MockClassifier:
     """Dummy classifier to test the parameter search algorithms"""
+
     def __init__(self, foo_param=0):
         self.foo_param = foo_param
 
@@ -160,8 +162,8 @@ def test_parameter_grid():
         # tuple + chain transforms {"a": 1, "b": 2} to ("a", 1, "b", 2)
         points = set(tuple(chain(*(sorted(p.items())))) for p in grid2)
         assert (points ==
-                     set(("bar", x, "foo", y)
-                         for x, y in product(params2["bar"], params2["foo"])))
+                set(("bar", x, "foo", y)
+                    for x, y in product(params2["bar"], params2["foo"])))
     assert_grid_iter_equals_getitem(grid2)
 
     # Special case: empty grid (useful to get default estimator settings)
@@ -690,8 +692,8 @@ def test_gridsearch_nd():
     # Pass X as list in GridSearchCV
     X_4d = np.arange(10 * 5 * 3 * 2).reshape(10, 5, 3, 2)
     y_3d = np.arange(10 * 7 * 11).reshape(10, 7, 11)
-    check_X = lambda x: x.shape[1:] == (5, 3, 2)
-    check_y = lambda x: x.shape[1:] == (7, 11)
+    def check_X(x): return x.shape[1:] == (5, 3, 2)
+    def check_y(x): return x.shape[1:] == (7, 11)
     clf = CheckingClassifier(
         check_X=check_X, check_y=check_y, methods_to_check=["fit"],
     )
@@ -1097,6 +1099,64 @@ def compare_refit_methods_when_refit_with_acc(search_multi, search_acc, refit):
         assert getattr(search_multi, key) == getattr(search_acc, key)
 
 
+@pytest.mark.parametrize('search_cv',
+                         [RandomizedSearchCV(estimator=DecisionTreeClassifier(),
+                                             param_distributions={'max_depth': [5, 10]}),
+                          GridSearchCV(estimator=DecisionTreeClassifier(),
+                                       param_grid={'max_depth': [5, 10]})
+                          ])
+def test_search_cv_score_samples_error(search_cv):
+    X, y = make_blobs(n_samples=100, n_features=4, random_state=42)
+    search_cv.fit(X, y)
+
+    # Make sure to error out when underlying estimator does not implement
+    # the method `score_samples`
+    err_msg = ("'DecisionTreeClassifier' object has no attribute "
+               "'score_samples'")
+
+    with pytest.raises(AttributeError, match=err_msg):
+        search_cv.score_samples(X)
+
+
+@pytest.mark.parametrize('search_cv',
+                         [RandomizedSearchCV(estimator=LocalOutlierFactor(novelty=True),
+                                             param_distributions={
+                                                 'n_neighbors': [5, 10]},
+                                             scoring="precision"),
+                          GridSearchCV(estimator=LocalOutlierFactor(novelty=True),
+                                       param_grid={'n_neighbors': [5, 10]},
+                                       scoring="precision")
+                          ])
+def test_search_cv_score_samples_method(search_cv):
+    # Set parameters
+    rng = np.random.RandomState(42)
+    n_samples = 300
+    outliers_fraction = 0.15
+    n_outliers = int(outliers_fraction * n_samples)
+    n_inliers = n_samples - n_outliers
+
+    # Create dataset
+    X = make_blobs(n_samples=n_inliers, n_features=2, centers=[[0, 0], [0, 0]],
+                   cluster_std=0.5, random_state=0)[0]
+    # Add some noisy points
+    X = np.concatenate([X, rng.uniform(low=-6, high=6,
+                                       size=(n_outliers, 2))], axis=0)
+
+    # Define labels to be able to score the estimator with `search_cv`
+    y_true = np.array([1] * n_samples)
+    y_true[-n_outliers:] = [-1] * n_outliers
+
+    # Fit on data
+    search_cv.fit(X, y_true)
+
+    # Fit the estimator
+    lof = search_cv.best_estimator_
+
+    # Verify that the stand alone estimator yields the same results
+    # as the ones obtained with *SearchCV
+    assert_allclose(search_cv.score_samples(X), lof.score_samples(X))
+
+
 def test_search_cv_results_rank_tie_breaking():
     X, y = make_blobs(n_samples=50, random_state=42)
 


Code review comments:
diff_hunk: @@ -1097,6 +1099,64 @@ def compare_refit_methods_when_refit_with_acc(search_multi, search_acc, refit):
         assert getattr(search_multi, key) == getattr(search_acc, key)
 
 
+@pytest.mark.parametrize('search_cv',
+                         [RandomizedSearchCV(estimator=DecisionTreeClassifier(),
+                                             param_distributions={'max_depth': [5, 10]}),
+                          GridSearchCV(estimator=DecisionTreeClassifier(),
+                                       param_grid={'max_depth': [5, 10]})
+                          ])
+def test_search_cv_score_samples_error(search_cv):
+    X, y = make_blobs(n_samples=100, n_features=4, random_state=42)
+    search_cv.fit(X, y)
+
+    # Make sure to error out when underlying estimator does not implement
+    # the method `score_samples`
+    err_msg = ("'DecisionTreeClassifier' object has no attribute "
+               "'score_samples'")
+
+    with pytest.raises(AttributeError, match=err_msg):
+        search_cv.score_samples(X)
+
+
+@pytest.mark.parametrize('search_cv',
+                         [RandomizedSearchCV(estimator=LocalOutlierFactor(novelty=True),
+                                             param_distributions={
+                                                 'n_neighbors': [5, 10]},
+                                             scoring="precision"),
+                          GridSearchCV(estimator=LocalOutlierFactor(novelty=True),
+                                       param_grid={'n_neighbors': [5, 10]},
+                                       scoring="precision")
+                          ])
+def test_search_cv_score_samples_method(search_cv):
+    # Set parameters
+    rng = np.random.RandomState(42)
+    n_samples = 300
+    outliers_fraction = 0.15
+    n_outliers = int(outliers_fraction * n_samples)
+    n_inliers = n_samples - n_outliers
+
+    # Create dataset
+    X = make_blobs(n_samples=n_inliers, n_features=2, centers=[[0, 0], [0, 0]],
+                   cluster_std=0.5, random_state=0)[0]
+    # Add some noisy points
+    X = np.concatenate([X, rng.uniform(low=-6, high=6,
+                                       size=(n_outliers, 2))], axis=0)
+
+    # Define labels to be able to score the estimator with `search_cv`
+    y_true = np.array([1] * n_samples)
+    y_true[-n_outliers:] = [-1] * n_outliers
path: sklearn/model_selection/tests/test_search.py
text: @user1:
Nit

```suggestion
    y_true[-n_outliers:] = -1
```