Repository: nilearn/nilearn

Issue:
Decoder object failing weirdly when screening percentile too low.
On Nilearn master, running the Decoder object on Haxby example (see code below) encounters a weird bug. With screening_percentile=20, it runs smoothly but with screening_percentile=10 or lower it yields a completely unrelated error. 

I don't get what is the bug here. Any ideas @tbng ?

```
from nilearn.datasets import fetch_haxby
data_files = fetch_haxby()

# Load behavioral data
import pandas as pd
behavioral = pd.read_csv(data_files.session_target[0], sep=" ")

# Restrict to face and house conditions
conditions = behavioral['labels']
condition_mask = conditions.isin(['face', 'house'])

# Split data into train and test samples, using the chunks
condition_mask_train = (condition_mask) & (behavioral['chunks'] <= 6)
condition_mask_test = (condition_mask) & (behavioral['chunks'] > 6)

# Apply this sample mask to X (fMRI data) and y (behavioral labels)
# Because the data is in one single large 4D image, we need to use
# index_img to do the split easily
from nilearn.image import index_img
func_filenames = data_files.func[0]
X_train = index_img(func_filenames, condition_mask_train)
X_test = index_img(func_filenames, condition_mask_test)
y_train = conditions[condition_mask_train].values
y_test = conditions[condition_mask_test].values


# Compute the mean epi to be used for the background of the plotting
from nilearn.image import mean_img
background_img = mean_img(func_filenames)

##############################################################################
# Fit fREM
# --------------------------------------
from nilearn.decoding import fREMClassifier
from nilearn.decoding import Decoder
dec = Decoder(screening_percentile=5)
dec.fit(X_train, y_train)
```


Full traceback :
```
ValueError                                Traceback (most recent call last)
<ipython-input-135-b9478a772005> in <module>()
----> 1 dec.fit(X_train, y_train)

~/Documents/Stage/nilearn/nilearn/decoding/decoder.py in fit(self, X, y, groups)
    496                 c, self.screening_percentile_, self.clustering_percentile)
    497             for c, (train, test) in itertools.product(
--> 498                 range(n_problems), self.cv_))
    499 
    500         coefs, intercepts = self._fetch_parallel_fit_outputs(

~/anaconda2/envs/py36/lib/python3.6/site-packages/joblib/parallel.py in __call__(self, iterable)
    919             # remaining jobs.
    920             self._iterating = False
--> 921             if self.dispatch_one_batch(iterator):
    922                 self._iterating = self._original_iterator is not None
    923 

~/anaconda2/envs/py36/lib/python3.6/site-packages/joblib/parallel.py in dispatch_one_batch(self, iterator)
    757                 return False
    758             else:
--> 759                 self._dispatch(tasks)
    760                 return True
    761 

~/anaconda2/envs/py36/lib/python3.6/site-packages/joblib/parallel.py in _dispatch(self, batch)
    714         with self._lock:
    715             job_idx = len(self._jobs)
--> 716             job = self._backend.apply_async(batch, callback=cb)
    717             # A job can complete so quickly than its callback is
    718             # called before we get here, causing self._jobs to

~/anaconda2/envs/py36/lib/python3.6/site-packages/joblib/_parallel_backends.py in apply_async(self, func, callback)
    180     def apply_async(self, func, callback=None):
    181         """Schedule a func to be run"""
--> 182         result = ImmediateResult(func)
    183         if callback:
    184             callback(result)

~/anaconda2/envs/py36/lib/python3.6/site-packages/joblib/_parallel_backends.py in __init__(self, batch)
    547         # Don't delay the application, to avoid keeping the input
    548         # arguments in memory
--> 549         self.results = batch()
    550 
    551     def get(self):

~/anaconda2/envs/py36/lib/python3.6/site-packages/joblib/parallel.py in __call__(self)
    223         with parallel_backend(self._backend, n_jobs=self._n_jobs):
    224             return [func(*args, **kwargs)
--> 225                     for func, args, kwargs in self.items]
    226 
    227     def __len__(self):

~/anaconda2/envs/py36/lib/python3.6/site-packages/joblib/parallel.py in <listcomp>(.0)
    223         with parallel_backend(self._backend, n_jobs=self._n_jobs):
    224             return [func(*args, **kwargs)
--> 225                     for func, args, kwargs in self.items]
    226 
    227     def __len__(self):

~/anaconda2/envs/py36/lib/python3.6/site-packages/joblib/memory.py in __call__(self, *args, **kwargs)
    353 
    354     def __call__(self, *args, **kwargs):
--> 355         return self.func(*args, **kwargs)
    356 
    357     def call_and_shelve(self, *args, **kwargs):

~/Documents/Stage/nilearn/nilearn/decoding/decoder.py in _parallel_fit(estimator, X, y, train, test, param_grid, is_classification, scorer, mask_img, class_index, screening_percentile, clustering_percentile)
    180     for param in ParameterGrid(param_grid):
    181         estimator = clone(estimator).set_params(**param)
--> 182         estimator.fit(X_train, y_train)
    183 
    184         if is_classification:

~/anaconda2/envs/py36/lib/python3.6/site-packages/sklearn/svm/classes.py in fit(self, X, y, sample_weight)
    227         X, y = check_X_y(X, y, accept_sparse='csr',
    228                          dtype=np.float64, order="C",
--> 229                          accept_large_sparse=False)
    230         check_classification_targets(y)
    231         self.classes_ = np.unique(y)

~/anaconda2/envs/py36/lib/python3.6/site-packages/sklearn/utils/validation.py in check_X_y(X, y, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, multi_output, ensure_min_samples, ensure_min_features, y_numeric, warn_on_dtype, estimator)
    717                     ensure_min_features=ensure_min_features,
    718                     warn_on_dtype=warn_on_dtype,
--> 719                     estimator=estimator)
    720     if multi_output:
    721         y = check_array(y, 'csr', force_all_finite=True, ensure_2d=False,

~/anaconda2/envs/py36/lib/python3.6/site-packages/sklearn/utils/validation.py in check_array(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, warn_on_dtype, estimator)
    556                              " a minimum of %d is required%s."
    557                              % (n_features, array.shape, ensure_min_features,
--> 558                                 context))
    559 
    560     if warn_on_dtype and dtype_orig is not None and array.dtype != dtype_orig:

ValueError: Found array with 0 feature(s) (shape=(112, 0)) while a minimum of 1 is required.
```

Decoder with ridge_regressor fails on oasis vbm example
On Nilearn master

To replicate : in Voxel-Based Morphometry on Oasis dataset example, changing
```
decoder = DecoderRegressor(estimator='svr', mask=mask,
                           scoring='neg_mean_absolute_error',
                           screening_percentile=1,
                           n_jobs=1)
``` 
by 
``` 
decoder = DecoderRegressor(estimator='ridge_regressor', mask=mask,
                           scoring='neg_mean_absolute_error',
                           screening_percentile=1,
                           n_jobs=1)
``` 
yields an error, `best_coef` returned is 1D instead of expected 2D array `

Any thoughts @tbng ? It's not related to my PR #2327 but I can include fix if needed

Full traceback: 
```
----> 1 decoder.fit(gm_imgs_train, age_train)

~/nilearn/nilearn/decoding/decoder.py in fit(self, X, y, groups)
    454                 self.mask_img_, c, self.screening_percentile_)
    455             for c, (train, test) in itertools.product(
--> 456                 range(n_problems), self.cv_))
    457 
    458         coefs, intercepts = self._fetch_parallel_fit_outputs(

~/miniconda3/lib/python3.7/site-packages/joblib/parallel.py in __call__(self, iterable)
   1002             # remaining jobs.
   1003             self._iterating = False
-> 1004             if self.dispatch_one_batch(iterator):
   1005                 self._iterating = self._original_iterator is not None
   1006 

~/miniconda3/lib/python3.7/site-packages/joblib/parallel.py in dispatch_one_batch(self, iterator)
    833                 return False
    834             else:
--> 835                 self._dispatch(tasks)
    836                 return True
    837 

~/miniconda3/lib/python3.7/site-packages/joblib/parallel.py in _dispatch(self, batch)
    752         with self._lock:
    753             job_idx = len(self._jobs)
--> 754             job = self._backend.apply_async(batch, callback=cb)
    755             # A job can complete so quickly than its callback is
    756             # called before we get here, causing self._jobs to

~/miniconda3/lib/python3.7/site-packages/joblib/_parallel_backends.py in apply_async(self, func, callback)
    207     def apply_async(self, func, callback=None):
    208         """Schedule a func to be run"""
--> 209         result = ImmediateResult(func)
    210         if callback:
    211             callback(result)

~/miniconda3/lib/python3.7/site-packages/joblib/_parallel_backends.py in __init__(self, batch)
    588         # Don't delay the application, to avoid keeping the input
    589         # arguments in memory
--> 590         self.results = batch()
    591 
    592     def get(self):

~/miniconda3/lib/python3.7/site-packages/joblib/parallel.py in __call__(self)
    254         with parallel_backend(self._backend, n_jobs=self._n_jobs):
    255             return [func(*args, **kwargs)
--> 256                     for func, args, kwargs in self.items]
    257 
    258     def __len__(self):

~/miniconda3/lib/python3.7/site-packages/joblib/parallel.py in <listcomp>(.0)
    254         with parallel_backend(self._backend, n_jobs=self._n_jobs):
    255             return [func(*args, **kwargs)
--> 256                     for func, args, kwargs in self.items]
    257 
    258     def __len__(self):

~/miniconda3/lib/python3.7/site-packages/joblib/memory.py in __call__(self, *args, **kwargs)
    353 
    354     def __call__(self, *args, **kwargs):
--> 355         return self.func(*args, **kwargs)
    356 
    357     def call_and_shelve(self, *args, **kwargs):

~/nilearn/nilearn/decoding/decoder.py in _parallel_fit(estimator, X, y, train, test, param_grid, is_classification, scorer, mask_img, class_index, screening_percentile)
    177 
    178     if (selector is not None) and do_screening:
--> 179         best_coef = selector.inverse_transform(best_coef)
    180 
    181     return class_index, best_coef, best_intercept, best_param, best_score

~/miniconda3/lib/python3.7/site-packages/sklearn/feature_selection/_base.py in inverse_transform(self, X)
    113 
    114         support = self.get_support()
--> 115         X = check_array(X, dtype=None)
    116         if support.sum() != X.shape[1]:
    117             raise ValueError("X has a different shape than during fitting.")

~/miniconda3/lib/python3.7/site-packages/sklearn/utils/validation.py in check_array(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, warn_on_dtype, estimator)
    554                     "Reshape your data either using array.reshape(-1, 1) if "
    555                     "your data has a single feature or array.reshape(1, -1) "
--> 556                     "if it contains a single sample.".format(array))
    557 
    558         # in the future np.flexible dtypes will be handled like object dtypes

ValueError: Expected 2D array, got 1D array instead:
array=[-0.1109591  -0.11897708 -0.07845441 ... -0.06480181 -0.10047668
 -0.13377442].
Reshape your data either using array.reshape(-1, 1) if your data has a single feature or array.reshape(1, -1) if it contains a single sample.
``` 




Discussion:
there seems to be a bug in the decoder: the `adjust_screening_percentile` is done twice, once in the decoder's `fit` and once in `_parallel_fit`, when `check_feature_screening` is called.
also, can you check if the masking is working well? when I run the script you posted the masking seems to fail for me: almost all the image (not just the brain) is kept inside the mask (as a result the adjustment reduces the screening percentile a lot but that's not the biggest problem)
BTW, I think the documentation for `screening_percentile` could be improved a lot:
```
    screening_percentile: int, float, optional, in the closed interval [0, 100]
        Perform an univariate feature selection based on the Anova F-value for
        the input data. A float according to a percentile of the highest
        scores. Default: 20.
```
it should state that it is the percentage of brain volume that is kept, and that is expressed wrt a full mni template brain, and adjusted if the volume of your brain mask differs (for example what happens when you provide a mask selecting an ROI?)
Maybe related : #854
the bug is described above. do you want to open a PR? or should I do it?
On 28.04.2020 06:24, jeromedockes wrote:
>the bug is described above. do you want to open a PR? or should I do it?

Sorry I am a bit busy these days. I will try to investigate and open a PR tomorrow.

>-- 
>You are receiving this because you were mentioned.
>Reply to this email directly or view it on GitHub:
>https://github.com/nilearn/nilearn/issues/2414#issuecomment-620605798


> Sorry I am a bit busy these days. I will try to investigate and open a PR tomorrow.

no rush!

I can confirm that this bug can be replicated this bug with my machine with the script.

> there seems to be a bug in the decoder: the `adjust_screening_percentile` is done twice, once in the decoder's `fit` and once in `_parallel_fit`, when `check_feature_screening` is called.

Good catch, but it is actually the `_adjust_screening_percentile` function get called twice? Once in `Decoder.fit()` and one more time in `_parallel_fit` -- a nested call for this function. This function seems to check the volume of the mask and adjust the screening percentile on that, so it is possible that the `screening_percentile` if inputed as very small (like 0.05 in the above script) get corrected twice might be even smaller.  

> also, can you check if the masking is working well? when I run the script you posted the masking seems to fail for me: almost all the image (not just the brain) is kept inside the mask (as a result the adjustment reduces the screening percentile a lot but that's not the biggest problem)

Also related that the `_adjust_screening_percentile` takes mask image as one of the input, so the quality of the mask has direct influence on the output.
> BTW, I think the documentation for `screening_percentile` could be improved a lot:
> 
> ```
>     screening_percentile: int, float, optional, in the closed interval [0, 100]
>         Perform an univariate feature selection based on the Anova F-value for
>         the input data. A float according to a percentile of the highest
>         scores. Default: 20.
> ```
> 
> it should state that it is the percentage of brain volume that is kept, and that is expressed wrt a full mni template brain, and adjusted if the volume of your brain mask differs (for example what happens when you provide a mask selecting an ROI?)

Yes as least we can improve this one quickly in the PR.
linked with #2360 
@thomasbazeille if you fix it , do it in a new PR, not the existing one.
This is prbably related to `Decoder` when having `RidgeCV` as an estimator will set the `param_grid` to an empty dict. I can open a PR for a fix.

Resolving patch:
diff --git a/nilearn/decoding/decoder.py b/nilearn/decoding/decoder.py
--- a/nilearn/decoding/decoder.py
+++ b/nilearn/decoding/decoder.py
@@ -22,7 +22,7 @@
 from sklearn.base import RegressorMixin
 from sklearn.linear_model import LogisticRegression
 from sklearn.linear_model.base import LinearModel
-from sklearn.linear_model.ridge import RidgeCV, RidgeClassifierCV, _BaseRidgeCV
+from sklearn.linear_model.ridge import RidgeClassifierCV, RidgeCV, _BaseRidgeCV
 from sklearn.model_selection import (LeaveOneGroupOut, ParameterGrid,
                                      ShuffleSplit, StratifiedShuffleSplit,
                                      check_cv)
@@ -32,12 +32,6 @@
 from sklearn.utils.extmath import safe_sparse_dot
 from sklearn.utils.validation import check_is_fitted, check_X_y
 
-try:
-    from sklearn.metrics import check_scoring
-except ImportError:
-    # for scikit-learn 0.18 and 0.19
-    from sklearn.metrics.scorer import check_scoring
-
 from nilearn._utils import CacheMixin
 from nilearn._utils.cache_mixin import _check_memory
 from nilearn._utils.param_validation import (_adjust_screening_percentile,
@@ -45,6 +39,12 @@
 from nilearn.input_data.masker_validation import check_embedded_nifti_masker
 from nilearn.regions.rena_clustering import ReNA
 
+try:
+    from sklearn.metrics import check_scoring
+except ImportError:
+    # for scikit-learn 0.18 and 0.19
+    from sklearn.metrics.scorer import check_scoring
+
 
 SUPPORTED_ESTIMATORS = dict(
     svc_l1=LinearSVC(penalty='l1', dual=False, max_iter=1e4),
@@ -140,7 +140,7 @@ def _check_estimator(estimator):
 
 
 def _parallel_fit(estimator, X, y, train, test, param_grid, is_classification,
-                  scorer, mask_img, class_index, screening_percentile,
+                  selector, scorer, mask_img, class_index,
                   clustering_percentile):
     """Find the best estimator for a fold within a job.
     This function tries several parameters for the estimator for the train and
@@ -163,11 +163,6 @@ def _parallel_fit(estimator, X, y, train, test, param_grid, is_classification,
         X_train = clustering.fit_transform(X_train)
         X_test = clustering.transform(X_test)
 
-    # Check if the size of the mask image and the number of features allow
-    # to perform feature screening.
-
-    selector = check_feature_screening(screening_percentile, mask_img,
-                                       is_classification)
     do_screening = (X_train.shape[1] > 100) and selector is not None
 
     if do_screening:
@@ -191,7 +186,7 @@ def _parallel_fit(estimator, X, y, train, test, param_grid, is_classification,
         # Store best parameters and estimator coefficients
         if (best_score is None) or (score >= best_score):
             best_score = score
-            best_coef = estimator.coef_
+            best_coef = np.reshape(estimator.coef_, (1, -1))
             best_intercept = estimator.intercept_
             best_param = param
 
@@ -253,8 +248,10 @@ class _BaseDecoder(LinearModel, RegressorMixin, CacheMixin):
         for clustering_percentile equal to 10.
 
     screening_percentile: int, float, optional, in the closed interval [0, 100]
-        Perform a univariate feature selection based on the Anova F-value for
-        the input data. A float according to a percentile of the highest
+        The percentage of brain volume that will be kept with respect to a full
+        MNI template. In particular, if it is lower than 100, a univariate
+        feature selection based on the Anova F-value for the input data will be
+        perfomed. A float according to a percentile of the highest
         scores. Default: 20.
 
     scoring: str, callable or None, optional. Default None
@@ -333,6 +330,7 @@ class _BaseDecoder(LinearModel, RegressorMixin, CacheMixin):
     nilearn.decoding.fREMRegressor: State of the art regression pipeline
         for Neuroimaging
     nilearn.decoding.SpaceNetClassifier: Graph-Net and TV-L1 priors/penalties
+
     """
 
     def __init__(self, estimator='svc', mask=None, cv=10, param_grid=None,
@@ -476,9 +474,18 @@ def fit(self, X, y, groups=None):
         else:
             n_problems = 1
 
+        # Check if the size of the mask image and the number of features allow
+        # to perform feature screening.
+        selector = check_feature_screening(self.screening_percentile,
+                                           self.mask_img_,
+                                           self.is_classification)
         # Return a suitable screening percentile according to the mask image
-        self.screening_percentile_ = _adjust_screening_percentile(
-            self.screening_percentile, self.mask_img_, verbose=self.verbose)
+        if hasattr(selector, 'percentile'):
+            self.screening_percentile_ = selector.percentile
+        elif self.screening_percentile is None:
+            self.screening_percentile_ = 100.0
+        else:
+            self.screening_percentile_ = self.screening_percentile
 
         n_final_features = int(X.shape[1] * self.screening_percentile_
                                * self.clustering_percentile / 10000)
@@ -493,9 +500,12 @@ def fit(self, X, y, groups=None):
 
         parallel_fit_outputs = parallel(
             delayed(self._cache(_parallel_fit))(
-                self.estimator, X, y[:, c], train, test, self.param_grid,
-                self.is_classification, scorer, self.mask_img_,
-                c, self.screening_percentile_, self.clustering_percentile)
+                estimator=self.estimator,
+                X=X, y=y[:, c], train=train, test=test,
+                param_grid=self.param_grid,
+                is_classification=self.is_classification, selector=selector,
+                scorer=scorer, mask_img=self.mask_img_, class_index=c,
+                clustering_percentile=self.clustering_percentile)
             for c, (train, test) in itertools.product(
                 range(n_problems), self.cv_))
 
@@ -699,8 +709,10 @@ class Decoder(_BaseDecoder):
         for example: https://scikit-learn.org/stable/modules/grid_search.html
 
     screening_percentile: int, float, optional, in the closed interval [0, 100]
-        Perform an univariate feature selection based on the Anova F-value for
-        the input data. A float according to a percentile of the highest
+        The percentage of brain volume that will be kept with respect to a full
+        MNI template. In particular, if it is lower than 100, a univariate
+        feature selection based on the Anova F-value for the input data will be
+        performed. A float according to a percentile of the highest
         scores. Default: 20.
 
     scoring: str, callable or None, optional. Default: 'roc_auc'
@@ -831,8 +843,10 @@ class DecoderRegressor(_BaseDecoder):
         for example: https://scikit-learn.org/stable/modules/grid_search.html
 
     screening_percentile: int, float, optional, in the closed interval [0, 100]
-        Perform a univariate feature selection based on the Anova F-value for
-        the input data. A float according to a percentile of the highest
+        The percentage of brain volume that will be kept with respect to a full
+        MNI template. In particular, if it is lower than 100, a univariate
+        feature selection based on the Anova F-value for the input data will be
+        performed. A float according to a percentile of the highest
         scores. Default: 20.
 
     scoring: str, callable or None, optional. Default: 'r2'
@@ -973,8 +987,10 @@ class fREMRegressor(_BaseDecoder):
         equal to 10. Default: 10.
 
     screening_percentile: int, float, optional, in closed interval [0, 100]
-        Perform a univariate feature selection based on the Anova F-value for
-        the input data. A float according to a percentile of the highest
+        The percentage of brain volume that will be kept with respect to a full
+        MNI template. In particular, if it is lower than 100, a univariate
+        feature selection based on the Anova F-value for the input data will be
+        performed. A float according to a percentile of the highest
         scores. Default: 20.
 
     scoring: str, callable or None, optional. Default: 'r2'
@@ -1127,8 +1143,10 @@ class fREMClassifier(_BaseDecoder):
         cluster_percentile equal to 10. Default: 10.
 
     screening_percentile: int, float, optional, in closed interval [0, 100]
-        Perform an univariate feature selection based on the Anova F-value for
-        the input data. A float according to a percentile of the highest
+        The percentage of brain volume that will be kept with respect to a full
+        MNI template. In particular, if it is lower than 100, a univariate
+        feature selection based on the Anova F-value for the input data will be
+        performed. A float according to a percentile of the highest
         scores. Default: 20.
 
     scoring: str, callable or None, optional. Default: 'roc_auc'


Test patch:
diff --git a/nilearn/decoding/tests/test_decoder.py b/nilearn/decoding/tests/test_decoder.py
--- a/nilearn/decoding/tests/test_decoder.py
+++ b/nilearn/decoding/tests/test_decoder.py
@@ -10,23 +10,22 @@
 
 import warnings
 
-import pytest
 import numpy as np
-
+import pytest
+from nilearn._utils.param_validation import check_feature_screening
+from nilearn.decoding.decoder import (Decoder, DecoderRegressor, _BaseDecoder,
+                                      _check_estimator, _check_param_grid,
+                                      _parallel_fit, fREMClassifier,
+                                      fREMRegressor)
+from nilearn.decoding.tests.test_same_api import to_niimgs
+from nilearn.input_data import NiftiMasker
 from sklearn.datasets import load_iris, make_classification, make_regression
 from sklearn.ensemble import RandomForestClassifier
-from sklearn.linear_model import LogisticRegression, RidgeCV, RidgeClassifierCV
+from sklearn.linear_model import LogisticRegression, RidgeClassifierCV, RidgeCV
 from sklearn.metrics import accuracy_score, r2_score
 from sklearn.model_selection import KFold, LeaveOneGroupOut
-from sklearn.svm import SVR, LinearSVC
 from sklearn.preprocessing import StandardScaler
-
-from nilearn.decoding.decoder import (_BaseDecoder, Decoder, DecoderRegressor,
-                                      fREMRegressor, fREMClassifier,
-                                      _check_estimator, _check_param_grid,
-                                      _parallel_fit)
-from nilearn.decoding.tests.test_same_api import to_niimgs
-from nilearn.input_data import NiftiMasker
+from sklearn.svm import SVR, LinearSVC
 
 try:
     from sklearn.metrics import check_scoring
@@ -51,10 +50,10 @@
                'logistic_l2': (logistic_l2, 'C'),
                'ridge_classifier': (ridge_classifier, [])}
 # Create a test dataset
-rand = np.random.RandomState(0)
-X = rand.rand(100, 10)
+rng = np.random.RandomState(0)
+X = rng.rand(100, 10)
 # Create different targets
-y_regression = rand.rand(100)
+y_regression = rng.rand(100)
 y_classification = np.hstack([[-1] * 50, [1] * 50])
 y_classification_str = np.hstack([['face'] * 50, ['house'] * 50])
 y_multiclass = np.hstack([[0] * 35, [1] * 30, [2] * 35])
@@ -134,6 +133,9 @@ def test_parallel_fit():
     estimator = svr
     svr_params = [[1e-1, 1e0, 1e1], [1e-1, 1e0, 5e0, 1e1]]
     scorer = check_scoring(estimator, 'r2')  #  define a scorer
+    # Define a screening selector
+    selector = check_feature_screening(screening_percentile=None,
+                                       mask_img=None, is_classification=False)
     for params in svr_params:
         param_grid = {}
         param_grid['C'] = np.array(params)
@@ -143,7 +145,7 @@ def test_parallel_fit():
                                           is_classification=False,
                                           scorer=scorer, mask_img=None,
                                           class_index=1,
-                                          screening_percentile=None,
+                                          selector=selector,
                                           clustering_percentile=100)))
     # check that every element of the output tuple is the same for both tries
     for a, b in zip(outputs[0], outputs[1]):
@@ -171,17 +173,16 @@ def test_decoder_binary_classification():
     assert accuracy_score(y, y_pred) > 0.95
 
     # check different screening_percentile value
-    for screening_percentile in [100, 20]:
+    for screening_percentile in [100, 20, None]:
         model = Decoder(mask=mask, screening_percentile=screening_percentile)
         model.fit(X, y)
         y_pred = model.predict(X)
         assert accuracy_score(y, y_pred) > 0.95
 
-    screening_percentile = 90
     for clustering_percentile in [100, 99]:
         model = fREMClassifier(estimator='logistic_l2', mask=mask,
                                clustering_percentile=clustering_percentile,
-                               screening_percentile=screening_percentile, cv=5)
+                               screening_percentile=90, cv=5)
         model.fit(X, y)
         y_pred = model.predict(X)
         assert accuracy_score(y, y_pred) > 0.9
@@ -211,20 +212,18 @@ def test_decoder_multiclass_classification():
     assert accuracy_score(y, y_pred) > 0.95
 
     # check different screening_percentile value
-    for screening_percentile in [100, 20]:
+    for screening_percentile in [100, 20, None]:
         model = Decoder(mask=mask, screening_percentile=screening_percentile)
         model.fit(X, y)
         y_pred = model.predict(X)
         assert accuracy_score(y, y_pred) > 0.95
 
     # check fREM with clustering or not
-    screening_percentile = 90
     for clustering_percentile in [100, 99]:
         for estimator in ['svc_l2', 'svc_l1']:
-            screening_percentile = 90
             model = fREMClassifier(estimator=estimator, mask=mask,
                                    clustering_percentile=clustering_percentile,
-                                   screening_percentile=screening_percentile,
+                                   screening_percentile=90,
                                    cv=5)
             model.fit(X, y)
             y_pred = model.predict(X)
@@ -257,33 +256,39 @@ def test_decoder_classification_string_label():
 
 
 def test_decoder_regression():
-    X, y = make_regression(n_samples=200, n_features=125,
-                           n_informative=5, noise=0.2, random_state=42)
+    dim = 30
+    X, y = make_regression(n_samples=100, n_features=dim**3, n_informative=dim,
+                           noise=1.5, bias=1.0, random_state=42)
     X = StandardScaler().fit_transform(X)
-    y = (y - y.mean()) / y.std()
-    X, mask = to_niimgs(X, [5, 5, 5])
-    for regressor_ in regressors:
-        for screening_percentile in [100, 20]:
-            model = DecoderRegressor(estimator=regressor_, mask=mask,
+    X, mask = to_niimgs(X, [dim, dim, dim])
+    for reg in regressors:
+        for screening_percentile in [100, 20, 1, None]:
+            model = DecoderRegressor(estimator=reg, mask=mask,
                                      screening_percentile=screening_percentile)
             model.fit(X, y)
             y_pred = model.predict(X)
             assert r2_score(y, y_pred) > 0.95
-        for clustering_percentile in [100, 99]:
-            screening_percentile = 90
-            model = fREMRegressor(estimator=regressor_, mask=mask,
-                                  clustering_percentile=clustering_percentile,
-                                  screening_percentile=screening_percentile,
-                                  cv=10)
-            model.fit(X, y)
-            y_pred = model.predict(X)
-            assert r2_score(y, y_pred) > 0.95
+
+    dim = 5
+    X, y = make_regression(n_samples=100, n_features=dim**3, n_informative=dim,
+                           noise=1.5, bias=1.0, random_state=42)
+    X = StandardScaler().fit_transform(X)
+    X, mask = to_niimgs(X, [dim, dim, dim])
+
+    for clustering_percentile in [100, 99]:
+        model = fREMRegressor(estimator=reg, mask=mask,
+                              clustering_percentile=clustering_percentile,
+                              screening_percentile=90,
+                              cv=10)
+        model.fit(X, y)
+        y_pred = model.predict(X)
+        assert r2_score(y, y_pred) > 0.95
 
 
 def test_decoder_apply_mask():
-    X_init, y = make_classification(
-        n_samples=200, n_features=125, scale=3.0,
-        n_informative=5, n_classes=4, random_state=42)
+    X_init, y = make_classification(n_samples=200, n_features=125, scale=3.0,
+                                    n_informative=5, n_classes=4,
+                                    random_state=42)
     X, _ = to_niimgs(X_init, [5, 5, 5])
     model = Decoder(mask=NiftiMasker())