Repository: nilearn/nilearn

Issue:
Improve performance of the MetaEstimator Decoder object 
Also in the discussion with @jeromedockes we agree on a future PR to improve performance of the Decoder object when using `LogisticRegression` estimator, in particular change from `LogisticRegression` to `LogisticRegressionCV`.

_Originally posted by @tbng in https://github.com/nilearn/nilearn/pull/2000#issuecomment-549411503_


Discussion:
@kchawla-pi One thing I notice that I have the privilege to work on some beautiful PR numbers.
Hahaha I noticed that as well when i was writing this issue!! :~)
Also there is a discussion about possibility of adding Variance Thresholder to `Decoder` object. See: https://github.com/nilearn/nilearn/pull/2000/files#r275962577
I see that as a low-priority thing; Could be added later.
@tbng We should do it now while you have the momentum (after merging the original), just in a separate PR so it is easy to track and review. So You can open another issue (and PR) for this.
@tbng #2000 Has been merged int master. Congrats!!! That was a lot of work you did there.
As a special prize for your wonderful work, you now get to work on THIS issue!! Yay!
Congrats ! B

Resolving patch:
diff --git a/nilearn/_utils/docs.py b/nilearn/_utils/docs.py
--- a/nilearn/_utils/docs.py
+++ b/nilearn/_utils/docs.py
@@ -118,17 +118,17 @@ def custom_function(vertices):
 
 docdict['classifier_options'] = f"""
 
-        - `svc`: :class:`{svc} <sklearn.svm.SVC>` with L2 penalty.
+        - `svc`: :class:`{svc} <sklearn.svm.LinearSVC>` with L2 penalty.
             .. code-block:: python
 
                 svc = LinearSVC(penalty='l2',
                                 max_iter=1e4)
 
-        - `svc_l2`: :class:`{svc} <sklearn.svm.SVC>` with L2 penalty.
+        - `svc_l2`: :class:`{svc} <sklearn.svm.LinearSVC>` with L2 penalty.
             .. note::
                 Same as option `svc`.
 
-        - `svc_l1`: :class:`{svc} <sklearn.svm.SVC>` with L1 penalty.
+        - `svc_l1`: :class:`{svc} <sklearn.svm.LinearSVC>` with L1 penalty.
             .. code-block:: python
 
                 svc_l1 = LinearSVC(penalty='l1',
@@ -136,23 +136,23 @@ def custom_function(vertices):
                                    max_iter=1e4)
 
         - `logistic`: \
-            :class:`{logistic} <sklearn.linear_model.LogisticRegression>` \
+            :class:`{logistic} <sklearn.linear_model.LogisticRegressionCV>` \
             with L2 penalty.
             .. code-block:: python
 
-                logistic = LogisticRegression(penalty='l2',
+                logistic = LogisticRegressionCV(penalty='l2',
                                               solver='liblinear')
 
         - `logistic_l1`: \
-            :class:`{logistic} <sklearn.linear_model.LogisticRegression>` \
+            :class:`{logistic} <sklearn.linear_model.LogisticRegressionCV>` \
             with L1 penalty.
             .. code-block:: python
 
-                logistic_l1 = LogisticRegression(penalty='l1',
+                logistic_l1 = LogisticRegressionCV(penalty='l1',
                                                  solver='liblinear')
 
         - `logistic_l2`: \
-            :class:`{logistic} <sklearn.linear_model.LogisticRegression>` \
+            :class:`{logistic} <sklearn.linear_model.LogisticRegressionCV>` \
             with L2 penalty
             .. note::
                 Same as option `logistic`.
diff --git a/nilearn/decoding/decoder.py b/nilearn/decoding/decoder.py
--- a/nilearn/decoding/decoder.py
+++ b/nilearn/decoding/decoder.py
@@ -31,7 +31,7 @@
 from sklearn.dummy import DummyClassifier, DummyRegressor
 from sklearn.linear_model import (
     LinearRegression,
-    LogisticRegression,
+    LogisticRegressionCV,
     RidgeClassifierCV,
     RidgeCV,
 )
@@ -53,9 +53,9 @@
     svc_l1=LinearSVC(penalty="l1", dual=False, max_iter=10000),
     svc_l2=LinearSVC(penalty="l2", max_iter=10000),
     svc=LinearSVC(penalty="l2", max_iter=10000),
-    logistic_l1=LogisticRegression(penalty="l1", solver="liblinear"),
-    logistic_l2=LogisticRegression(penalty="l2", solver="liblinear"),
-    logistic=LogisticRegression(penalty="l2", solver="liblinear"),
+    logistic_l1=LogisticRegressionCV(penalty="l1", solver="liblinear"),
+    logistic_l2=LogisticRegressionCV(penalty="l2", solver="liblinear"),
+    logistic=LogisticRegressionCV(penalty="l2", solver="liblinear"),
     ridge_classifier=RidgeClassifierCV(),
     ridge_regressor=RidgeCV(),
     ridge=RidgeCV(),
@@ -111,6 +111,9 @@ def _check_param_grid(estimator, X, y, param_grid=None):
     else:
         if isinstance(estimator, (RidgeCV, RidgeClassifierCV)):
             param_grid = _wrap_param_grid(param_grid, "alphas")
+        elif isinstance(estimator, LogisticRegressionCV):
+            param_grid = _replace_param_grid_key(param_grid, "C", "Cs")
+            param_grid = _wrap_param_grid(param_grid, "Cs")
 
     return param_grid
 
@@ -152,7 +155,7 @@ def _default_param_grid(estimator, X, y):
             raise NotImplementedError(message)
     elif not isinstance(
         estimator,
-        (LogisticRegression, LinearSVC, RidgeCV, RidgeClassifierCV, SVR),
+        (LogisticRegressionCV, LinearSVC, RidgeCV, RidgeClassifierCV, SVR),
     ):
         raise ValueError(
             "Invalid estimator. The supported estimators are:"
@@ -162,7 +165,7 @@ def _default_param_grid(estimator, X, y):
     # use l1_min_c to get lower bound for estimators with L1 penalty
     if hasattr(estimator, "penalty") and (estimator.penalty == "l1"):
         # define loss function
-        if isinstance(estimator, LogisticRegression):
+        if isinstance(estimator, LogisticRegressionCV):
             loss = "log"
         elif isinstance(estimator, LinearSVC):
             loss = "squared_hinge"
@@ -176,7 +179,16 @@ def _default_param_grid(estimator, X, y):
     # define sensible default for different types of estimators
     if isinstance(estimator, (RidgeCV, RidgeClassifierCV)):
         param_grid["alphas"] = [np.geomspace(1e-3, 1e4, 8)]
-    elif isinstance(estimator, (LogisticRegression, LinearSVC, SVR)):
+    elif isinstance(estimator, LogisticRegressionCV):
+        # min_c value is set to 0.5 unless the estimator uses L1 penalty,
+        # in which case min_c is computed with sklearn.svm.l1_min_c(),
+        # so for L2 penalty, param_grid["Cs"] is either 1e-3, ..., 1e4, and
+        # for L1 penalty the values are obtained in a more data-driven way
+        param_grid["Cs"] = [np.geomspace(2e-3, 2e4, 8) * min_c]
+    elif isinstance(estimator, (LinearSVC, SVR)):
+        # similar logic as above:
+        # - for L2 penalty this is [1, 10, 100]
+        # - for L1 penalty the values depend on the data
         param_grid["C"] = np.array([2, 20, 200]) * min_c
     else:
         param_grid = {}
@@ -187,10 +199,10 @@ def _default_param_grid(estimator, X, y):
 def _wrap_param_grid(param_grid, param_name):
     """Wrap a parameter's sequence of values with an outer list.
 
-    This can be
-    desirable for models featuring built-in cross-validation, as it would leave
-    it to the model's internal (optimized) cross-validation to loop over
-    hyperparameter values. Does nothing if the parameter is already wrapped.
+    This can be desirable for models featuring built-in cross-validation,
+    as it would leave it to the model's internal (optimized) cross-validation
+    to loop over hyperparameter values. Does nothing if the parameter is
+    already wrapped.
 
     Parameters
     ----------
@@ -202,7 +214,7 @@ def _wrap_param_grid(param_grid, param_name):
 
     Returns
     -------
-    param_grid_wrapped: dict of str to sequence, or sequence of such
+    dict of str to sequence, or sequence of such
         The updated parameter grid
     """
     if param_grid is None:
@@ -239,6 +251,53 @@ def _wrap_param_grid(param_grid, param_name):
     return new_param_grid
 
 
+def _replace_param_grid_key(param_grid, key_to_replace, new_key):
+    """Replace a parameter name by another one.
+
+    Parameters
+    ----------
+    param_grid : dict of str to sequence, or sequence of such
+        The parameter grid to process, as a dictionary mapping estimator
+        parameters to sequences of allowed values.
+    key_to_replace : str
+        Name of parameter to replace
+    new_key : str
+        New parameter name. If this key already exists in the parameter grid,
+        it is overwritten
+
+    Returns
+    -------
+    dict of str to sequence, or sequence of such
+        The updated parameter grid
+    """
+    # ensure param_grid is a list so that we can loop over it
+    input_is_dict = isinstance(param_grid, dict)
+    if input_is_dict:
+        param_grid = [param_grid]
+
+    # replace old key by new key if needed
+    new_param_grid = []
+    for param_grid_item in param_grid:
+        param_grid_item = dict(param_grid_item)  # make a new dict
+        if key_to_replace in param_grid_item:
+            warnings.warn(
+                f'The "{key_to_replace}" parameter in "param_grid" is'
+                f' being replaced by "{new_key}" due to a change in the'
+                " choice of underlying scikit-learn estimator. In a future"
+                " version, this will result in an error.",
+                DeprecationWarning,
+                stacklevel=13,
+            )
+            param_grid_item[new_key] = param_grid_item.pop(key_to_replace)
+        new_param_grid.append(param_grid_item)
+
+    # return a dict if input was a dict
+    if input_is_dict:
+        new_param_grid = new_param_grid[0]
+
+    return new_param_grid
+
+
 def _check_estimator(estimator):
     if not isinstance(estimator, str):
         warnings.warn(
@@ -341,6 +400,8 @@ def _parallel_fit(
 
             if isinstance(estimator, (RidgeCV, RidgeClassifierCV)):
                 params["best_alpha"] = estimator.alpha_
+            elif isinstance(estimator, LogisticRegressionCV):
+                params["best_C"] = estimator.C_.item()
             best_params = params
 
             # fill in any missing param from param_grid
@@ -596,8 +657,9 @@ def fit(self, X, y, groups=None):
             when Dummy estimators are provided. Note: if the estimator used its
             built-in cross-validation, this will include an additional key for
             the single best value estimated by the built-in cross-validation
-            (e.g., 'best_alpha' for RidgeCV/RidgeClassifierCV), in addition to
-            the input list of values.
+            ('best_C' for LogisticRegressionCV and 'best_alpha' for
+            RidgeCV/RidgeClassifierCV), in addition to the input list of
+            values.
 
         'scorer_' : function
             Scorer function used on the held out data to choose the best


Test patch:
diff --git a/nilearn/decoding/tests/test_decoder.py b/nilearn/decoding/tests/test_decoder.py
--- a/nilearn/decoding/tests/test_decoder.py
+++ b/nilearn/decoding/tests/test_decoder.py
@@ -41,7 +41,11 @@
 from sklearn.dummy import DummyClassifier, DummyRegressor
 from sklearn.ensemble import RandomForestClassifier
 from sklearn.exceptions import NotFittedError
-from sklearn.linear_model import LogisticRegression, RidgeClassifierCV, RidgeCV
+from sklearn.linear_model import (
+    LogisticRegressionCV,
+    RidgeClassifierCV,
+    RidgeCV,
+)
 from sklearn.metrics import (
     accuracy_score,
     check_scoring,
@@ -153,8 +157,8 @@ def test_check_param_grid_regression(regressor, param):
 @pytest.mark.parametrize(
     "classifier, param",
     [
-        (LogisticRegression(penalty="l1"), "C"),
-        (LogisticRegression(penalty="l2"), "C"),
+        (LogisticRegressionCV(penalty="l1"), ["Cs"]),
+        (LogisticRegressionCV(penalty="l2"), ["Cs"]),
         (RidgeClassifierCV(), ["alphas"]),
     ],
 )
@@ -170,6 +174,30 @@ def test_check_param_grid_classification(rand_X_Y, classifier, param):
     assert list(param_grid.keys()) == list(param)
 
 
+@pytest.mark.parametrize(
+    "param_grid_input",
+    [
+        {"C": [1, 10, 100]},
+        {"Cs": [1, 10, 100]},
+        [{"C": [1, 10, 100]}, {"fit_intercept": [False]}],
+    ],
+)
+def test_check_param_grid_replacement(rand_X_Y, param_grid_input):
+    X, Y = rand_X_Y
+    param_to_replace = "C"
+    param_replaced = "Cs"
+    param_grid_output = _check_param_grid(
+        LogisticRegressionCV(),
+        X,
+        Y,
+        param_grid_input,
+    )
+    for params in ParameterGrid(param_grid_output):
+        assert param_to_replace not in params
+        if param_replaced not in params:
+            assert params in ParameterGrid(param_grid_input)
+
+
 @pytest.mark.parametrize("estimator", ["log_l1", RandomForestClassifier()])
 def test_non_supported_estimator_error(rand_X_Y, estimator):
     """Raise the error when using a non supported estimator."""
@@ -383,6 +411,7 @@ def test_parallel_fit(rand_X_Y):
     [
         (RidgeCV(), "alphas", "best_alpha", False),
         (RidgeClassifierCV(), "alphas", "best_alpha", True),
+        (LogisticRegressionCV(), "Cs", "best_C", True),
     ],
 )
 def test_parallel_fit_builtin_cv(
@@ -438,7 +467,7 @@ def test_parallel_fit_builtin_cv(
         clustering_percentile=100,
     )
 
-    assert isinstance(best_param[fitted_param_name], (float, int))
+    assert isinstance(best_param[fitted_param_name], numbers.Number)
 
 
 def test_decoder_param_grid_sequence(binary_classification_data):