Repository: mne-tools/mne-python

Issue:
BUG: _SearchLight when 'scoring=roc_auc' & y not in (0, 1)
@kaichogami we missed a necessary feature when adding the `scoring` parameter to the SearchLight in the last PR:  currently, we encounter an error if `scoring='roc_auc'` but `y` is not (0, 1):

``` python
import numpy as np
from mne.decoding.search_light import _SearchLight
from sklearn.linear_model import LogisticRegression

sl = _SearchLight(LogisticRegression(), scoring='roc_auc')
X = np.random.rand(100, 20, 2)
y = np.random.randint(0, 2, 100)
sl.fit(X, y)
sl.score(X, y)  # works

y = np.random.randint(2, 4, 100)
sl.fit(X, y)
sl.score(X, y)  # crashes
```

I suggest to add a sklearn.preprocessing.LabelEncoder() at fitting and use it before scoring. @agramfort do you think it's the right approach?



Discussion:
Ok, I'll work on that.

I think we should use `LabelEncoder` only when number of classes of y is 2, in fit.

PR in #3717

I believe that this has to be solved at sklearn level: https://github.com/scikit-learn/scikit-learn/pull/6874
it's too complicated at sklearn level. We need to raise an error in MNE if
y is not [0, 1] and scoring == 'roc_auc' explaining that make_scorer with
the good pos_label set needs to be used

@agramfort that's bleeding edge IMO. It's not complicated to sort this at sklearn level, we just need to add LabelBinarizer within the roc if and only if len(set(y)) == 2
maybe but really the easy fix is in mne. We can do all this in our code.


Resolving patch:
diff --git a/mne/decoding/search_light.py b/mne/decoding/search_light.py
--- a/mne/decoding/search_light.py
+++ b/mne/decoding/search_light.py
@@ -237,6 +237,7 @@ def score(self, X, y):
                              'X.shape[-1]')
 
         scoring = _make_scorer(self.scoring)
+        y = _fix_auc(scoring, y)
 
         # For predictions/transforms the parallelization is across the data and
         # not across the estimators to avoid memory load.
@@ -534,6 +535,8 @@ def score(self, X, y):
         n_jobs = min(n_jobs, X.shape[-1])
         X_splits = np.array_split(X, n_jobs, axis=-1)
         scoring = _make_scorer(self.scoring)
+        y = _fix_auc(scoring, y)
+
         score = parallel(p_func(self.estimators_, scoring, x, y)
                          for x in X_splits)
 
@@ -638,3 +641,20 @@ def _gl_score(estimators, scoring, X, y):
                 score = np.zeros(shape, dtype)
             score[ii, jj, ...] = _score
     return score
+
+
+def _fix_auc(scoring, y):
+    from sklearn.preprocessing import LabelEncoder
+    # This fixes sklearn's inability to compute roc_auc when y not in [0, 1]
+    # scikit-learn/scikit-learn#6874
+    if scoring is not None:
+        if (
+            hasattr(scoring, '_score_func') and
+            hasattr(scoring._score_func, '__name__') and
+            scoring._score_func.__name__ == 'roc_auc_score'
+        ):
+            if np.ndim(y) != 1 or len(set(y)) != 2:
+                raise ValueError('roc_auc scoring can only be computed for '
+                                 'two-class problems.')
+            y = LabelEncoder().fit_transform(y)
+    return y


Test patch:
diff --git a/mne/decoding/tests/test_search_light.py b/mne/decoding/tests/test_search_light.py
--- a/mne/decoding/tests/test_search_light.py
+++ b/mne/decoding/tests/test_search_light.py
@@ -66,6 +66,20 @@ def test_SearchLight():
         sl.fit(X, y)
         assert_raises(err, sl.score, X, y)
 
+    # Check sklearn's roc_auc fix: scikit-learn/scikit-learn#6874
+    # -- 3 class problem
+    sl = _SearchLight(LogisticRegression(), scoring='roc_auc')
+    y = np.arange(len(X)) % 3
+    sl.fit(X, y)
+    assert_raises(ValueError, sl.score, X, y)
+    # -- 2 class problem not in [0, 1]
+    y = np.arange(len(X)) % 2 + 1
+    sl.fit(X, y)
+    score = sl.score(X, y)
+    assert_array_equal(score, [roc_auc_score(y - 1, _y_pred - 1)
+                               for _y_pred in sl.decision_function(X).T])
+    y = np.arange(len(X)) % 2
+
     for method, scoring in [
             ('predict_proba', 'roc_auc'), ('predict', roc_auc_score)]:
         sl1 = _SearchLight(LogisticRegression(), scoring=scoring)
@@ -165,6 +179,20 @@ def test_GeneralizationLight():
         gl.fit(X, y)
         assert_raises(err, gl.score, X, y)
 
+    # Check sklearn's roc_auc fix: scikit-learn/scikit-learn#6874
+    # -- 3 class problem
+    gl = _GeneralizationLight(LogisticRegression(), scoring='roc_auc')
+    y = np.arange(len(X)) % 3
+    gl.fit(X, y)
+    assert_raises(ValueError, gl.score, X, y)
+    # -- 2 class problem not in [0, 1]
+    y = np.arange(len(X)) % 2 + 1
+    gl.fit(X, y)
+    score = gl.score(X, y)
+    manual_score = [[roc_auc_score(y - 1, _y_pred) for _y_pred in _y_preds]
+                    for _y_preds in gl.decision_function(X).transpose(1, 2, 0)]
+    assert_array_equal(score, manual_score)
+
     # n_jobs
     gl = _GeneralizationLight(LogisticRegression(), n_jobs=2)
     gl.fit(X, y)