Repository: nilearn/nilearn

Issue:
Decoder.score() behaviour in classification mode
Hello!

I'm not sure if this issue is a documentation or feature request issue: the `.score` seems a bit confusing for me as a new-comer. 

I used the Decoder object in a classification tasks. After fitting the model, I used the  `.score` method, expecting it to compute the mean accuracy (as it is done in scikit-learn for classifiers). I got lucky its value came back negative, it made me look at the documentation and the fact that it computes R^2 coefficient.

I think it should be a bit more explicit to the user that `.score` in a classification fashion does not compute the mean accuracy:

- either by changing the behaviour (which seems tricky as it comes from inherited scikit-learn RegressorMixin)
- or by explicitly warn the user when running the method (which should be easier by using the `self.is_classification` flag) and make a visible warning in the documentation



Discussion:
Thanks for pointing this out! IMO it's a bug so I would go for your first
suggestion -- change the behavior. Either by having the _BaseDecoder inherint
from LinearModel rather than LinearRegression if possible, or by overriding the
score method.
WDYT @tbng  and @thomasbazeille ?
I might be wrong but I would expect `.score` to use whatever scoring strategy was provided to the decoder object (`decoder.scoring`). So I'd be in favour of overriding the `score` method to rely on the user-defined strategy.
> I might be wrong but I would expect `.score` to use whatever scoring strategy was provided to the decoder object

Good point! I agree that's the most sensible behaviour

Agreed. Thx for raising the point.

Resolving patch:
diff --git a/nilearn/decoding/decoder.py b/nilearn/decoding/decoder.py
--- a/nilearn/decoding/decoder.py
+++ b/nilearn/decoding/decoder.py
@@ -31,6 +31,7 @@
 from sklearn.utils import check_random_state
 from sklearn.utils.extmath import safe_sparse_dot
 from sklearn.utils.validation import check_is_fitted, check_X_y
+from sklearn.metrics import get_scorer
 
 from nilearn._utils import CacheMixin
 from nilearn._utils.cache_mixin import _check_memory
@@ -387,6 +388,7 @@ def __init__(self, estimator='svc', mask=None, cv=10, param_grid=None,
         self.n_jobs = n_jobs
         self.verbose = verbose
 
+
     def fit(self, X, y, groups=None):
         """Fit the decoder (learner).
 
@@ -465,6 +467,10 @@ def fit(self, X, y, groups=None):
             in the inner cross validation loop. The grid is empty
             when Dummy estimators are provided.
 
+        'scorer_' : function
+            Scorer function used on the held out data to choose the best
+            parameters for the model.
+
         `cv_scores_` : dict, (classes, n_folds)
             Scores (misclassification) for each parameter, and on each fold
 
@@ -490,7 +496,14 @@ def fit(self, X, y, groups=None):
             self.n_outputs_ = y.shape[1]
 
         # Setup scorer
-        scorer = check_scoring(self.estimator, self.scoring)
+        if self.scoring is not None:
+            self.scorer_ = check_scoring(self.estimator,
+                                         self.scoring)
+        else:
+            if self.is_classification:
+                self.scorer_ = get_scorer("accuracy")
+            else:
+                self.scorer_ = get_scorer("r2")
 
         # Setup cross-validation object. Default is StratifiedKFold when groups
         # is None. If groups is specified but self.cv is not set to custom CV
@@ -550,7 +563,7 @@ def fit(self, X, y, groups=None):
                 X=X, y=y[:, c], train=train, test=test,
                 param_grid=self.param_grid,
                 is_classification=self.is_classification, selector=selector,
-                scorer=scorer, mask_img=self.mask_img_, class_index=c,
+                scorer=self.scorer_, mask_img=self.mask_img_, class_index=c,
                 clustering_percentile=self.clustering_percentile)
             for c, (train, test) in itertools.product(
                 range(n_problems), self.cv_))
@@ -583,6 +596,31 @@ def fit(self, X, y, groups=None):
                 if not self.n_outputs_ > 1:
                     self.dummy_output_ = self.dummy_output_[0, :][np.newaxis, :]
 
+    def score(self, X, y, *args):
+        """Compute the prediction score using the scoring
+        metric defined by the scoring attribute.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix}, shape = (n_samples, n_features)
+            Samples.
+
+        y : array-like
+            Target values.
+
+        args : Optional arguments that can be passed to
+            scoring metrics. Example: sample_weight.
+
+        Returns
+        -------
+        score : float
+            Prediction score.
+
+        """
+        check_is_fitted(self, "coef_")
+        check_is_fitted(self, "masker_")
+        return self.scorer_(self, X, y, *args)
+
     def decision_function(self, X):
         """Predict class labels for samples in X.
 


Test patch:
diff --git a/nilearn/decoding/tests/test_decoder.py b/nilearn/decoding/tests/test_decoder.py
--- a/nilearn/decoding/tests/test_decoder.py
+++ b/nilearn/decoding/tests/test_decoder.py
@@ -22,11 +22,12 @@
 from sklearn.datasets import load_iris, make_classification, make_regression
 from sklearn.ensemble import RandomForestClassifier
 from sklearn.linear_model import LogisticRegression, RidgeClassifierCV, RidgeCV
-from sklearn.metrics import accuracy_score, r2_score, roc_auc_score
+from sklearn.metrics import accuracy_score, r2_score, roc_auc_score, get_scorer
 from sklearn.model_selection import KFold, LeaveOneGroupOut
 from sklearn.preprocessing import StandardScaler
 from sklearn.svm import SVR, LinearSVC
 from sklearn.dummy import DummyClassifier, DummyRegressor
+from sklearn.exceptions import NotFittedError
 
 try:
     from sklearn.metrics import check_scoring
@@ -173,6 +174,8 @@ def test_decoder_binary_classification():
     model = Decoder(mask=NiftiMasker())
     model.fit(X, y)
     y_pred = model.predict(X)
+    assert model.scoring == "roc_auc"
+    assert model.score(X, y) == 1.0
     assert accuracy_score(y, y_pred) > 0.95
 
     # decoder object use predict_proba for scoring with logistic model
@@ -185,8 +188,26 @@ def test_decoder_binary_classification():
     model = Decoder(estimator='dummy_classifier', mask=mask)
     model.fit(X, y)
     y_pred = model.predict(X)
+    assert model.scoring == "roc_auc"
+    assert model.score(X, y) == 0.5
     assert accuracy_score(y, y_pred) == 0.5
 
+    # Set scoring of decoder with a callable
+    accuracy_scorer = get_scorer('accuracy')
+    model = Decoder(estimator='dummy_classifier',
+                    mask=mask,
+                    scoring=accuracy_scorer)
+    model.fit(X, y)
+    y_pred = model.predict(X)
+    assert model.scoring == accuracy_scorer
+    assert model.score(X, y) == accuracy_score(y, y_pred)
+
+    # An error should be raise when trying to compute
+    # the score whithout having called fit first.
+    model = Decoder(estimator='dummy_classifier', mask=mask)
+    with pytest.raises(NotFittedError, match="This Decoder instance is not fitted yet."):
+        model.score(X, y)
+
     # decoder object use other strategy for dummy classifier
     param = dict(strategy='stratified')
     dummy_classifier.set_params(**param)
@@ -218,6 +239,22 @@ def test_decoder_binary_classification():
     model = Decoder(estimator=dummy_classifier, mask=mask)
     pytest.raises(NotImplementedError, model.fit, X, y)
 
+    # Raises an error with unknown scoring metrics
+    model = Decoder(estimator=dummy_classifier,
+                    mask=mask,
+                    scoring="foo")
+    with pytest.raises(ValueError,
+                       match="'foo' is not a valid scoring value"):
+        model.fit(X, y)
+
+    # Default scoring
+    model = Decoder(estimator='dummy_classifier',
+                    scoring=None)
+    assert model.scoring is None
+    model.fit(X, y)
+    assert model.scorer_ == get_scorer("accuracy")
+    assert model.score(X, y) == 0.5
+
     # check different screening_percentile value
     for screening_percentile in [100, 20, None]:
         model = Decoder(mask=mask, screening_percentile=screening_percentile)
@@ -258,11 +295,15 @@ def test_decoder_multiclass_classification():
     assert accuracy_score(y, y_pred) > 0.95
 
     # check classification with masker object and dummy classifier
-    model = Decoder(estimator='dummy_classifier', mask=NiftiMasker())
+    model = Decoder(estimator='dummy_classifier',
+                    mask=NiftiMasker(),
+                    scoring="accuracy")
     model.fit(X, y)
     y_pred = model.predict(X)
+    assert model.scoring == "accuracy"
     # 4-class classification
     assert accuracy_score(y, y_pred) > 0.2
+    assert model.score(X, y) == accuracy_score(y, y_pred)
 
     # check different screening_percentile value
     for screening_percentile in [100, 20, None]:
@@ -280,6 +321,7 @@ def test_decoder_multiclass_classification():
                                    cv=5)
             model.fit(X, y)
             y_pred = model.predict(X)
+            assert model.scoring == "roc_auc"
             assert accuracy_score(y, y_pred) > 0.9
 
     # check cross-validation scheme and fit attribute with groups enabled
@@ -327,8 +369,18 @@ def test_decoder_regression():
                              scoring='r2', screening_percentile=1)
     model.fit(X, y)
     y_pred = model.predict(X)
+    assert model.scoring == "r2"
     assert r2_score(y, y_pred) <= 0.
- 
+    assert model.score(X, y) == r2_score(y, y_pred)
+
+    # Check that default scoring metric for regression is r2
+    model = DecoderRegressor(estimator='dummy_regressor',
+                             mask=mask,
+                             scoring=None)
+    model.fit(X, y)
+    y_pred = model.predict(X)
+    assert model.score(X, y) == r2_score(y, y_pred)
+
     # decoder object use other strategy for dummy regressor
     param = dict(strategy='median')
     dummy_regressor.set_params(**param)
@@ -355,7 +407,9 @@ def test_decoder_regression():
                               cv=10)
         model.fit(X, y)
         y_pred = model.predict(X)
+        assert model.scoring == "r2"
         assert r2_score(y, y_pred) > 0.95
+        assert model.score(X, y) == r2_score(y, y_pred)
 
 
 def test_decoder_apply_mask():