Repository: mne-tools/mne-python

Issue:
BUG: joblib error in spatio_temporal_cluster_1samp_test
Running `spatio_temporal_cluster_1samp_test` with `n_jobs=8` using the default stat_fun `ttest_1samp_no_p` from the current master, I'm getting the error "The task could not be sent to the workers as it is too large for `send_bytes`."  

The matrix is not that all that large, 33x545x20484, and the function runs fine if I remove subjects to reduce it to 23x545x20484 or if I set `n_jobs=1`.  I continue to get the error after doing `mne.set_cache_dir('/tmp')`. The error message looks exactly like one reported in an old joblib issue: https://github.com/joblib/joblib/issues/344 

Operator error is always a possibility, but nothing obvious stands out to me (but then again I am the operator).  Complete command line and error message is below:

``T_obs, clusters, cluster_p_values, H0 = clu = \
spatio_temporal_cluster_1samp_test(X1, connectivity=connectivity, n_permutations=256, n_jobs=8, threshold=t_threshold, check_disjoint=True,                                   verbose='DEBUG')`` 

Traceback (most recent call last):
  File "/export/research/analysis/human/jhouck/shared/tools/anaconda/envs/mnedev/lib/python3.6/site-packages/joblib/externals/loky/backend/queues.py", line 156, in _feed
    send_bytes(obj_)
  File "/export/research/analysis/human/jhouck/shared/tools/anaconda/envs/mnedev/lib/python3.6/multiprocessing/connection.py", line 200, in send_bytes
    self._send_bytes(m[offset:offset + size])
  File "/export/research/analysis/human/jhouck/shared/tools/anaconda/envs/mnedev/lib/python3.6/multiprocessing/connection.py", line 393, in _send_bytes
    header = struct.pack("!i", n)
struct.error: 'i' format requires -2147483648 <= number <= 2147483647
"""

The above exception was the direct cause of the following exception:

RuntimeError                              Traceback (most recent call last)
<ipython-input-31-b9583e70ef55> in <module>
      2     spatio_temporal_cluster_1samp_test(X1a, connectivity=connectivity, n_permutations=1, n_jobs=8,
      3                                        threshold=t_threshold, buffer_size=2000, check_disjoint=True,
----> 4                                        verbose='DEBUG') 

</export/research/analysis/human/jhouck/shared/tools/mne-python/mne/externals/decorator.py:decorator-gen-186> in spatio_temporal_cluster_1samp_test(X, threshold, n_permutations, tail, stat_fun, connectivity, n_jobs, seed, max_step, spatial_exclude, step_down_p, t_power, out_type, check_disjoint, buffer_size, verbose)

/export/research/analysis/human/jhouck/shared/tools/mne-python/mne/utils/_logging.py in wrapper(*args, **kwargs)
     86             # set it back if we get an exception
     87             with use_log_level(verbose_level):
---> 88                 return function(*args, **kwargs)
     89         return function(*args, **kwargs)
     90     return FunctionMaker.create(

/export/research/analysis/human/jhouck/shared/tools/mne-python/mne/stats/cluster_level.py in spatio_temporal_cluster_1samp_test(X, threshold, n_permutations, tail, stat_fun, connectivity, n_jobs, seed, max_step, spatial_exclude, step_down_p, t_power, out_type, check_disjoint, buffer_size, verbose)
   1322         n_jobs=n_jobs, seed=seed, max_step=max_step, exclude=exclude,
   1323         step_down_p=step_down_p, t_power=t_power, out_type=out_type,
-> 1324         check_disjoint=check_disjoint, buffer_size=buffer_size)
   1325 
   1326 

</export/research/analysis/human/jhouck/shared/tools/mne-python/mne/externals/decorator.py:decorator-gen-185> in permutation_cluster_1samp_test(X, threshold, n_permutations, tail, stat_fun, connectivity, verbose, n_jobs, seed, max_step, exclude, step_down_p, t_power, out_type, check_disjoint, buffer_size)

/export/research/analysis/human/jhouck/shared/tools/mne-python/mne/utils/_logging.py in wrapper(*args, **kwargs)
     87             with use_log_level(verbose_level):
     88                 return function(*args, **kwargs)
---> 89         return function(*args, **kwargs)
     90     return FunctionMaker.create(
     91         function, 'return decfunc(%(signature)s)',

/export/research/analysis/human/jhouck/shared/tools/mne-python/mne/stats/cluster_level.py in permutation_cluster_1samp_test(X, threshold, n_permutations, tail, stat_fun, connectivity, verbose, n_jobs, seed, max_step, exclude, step_down_p, t_power, out_type, check_disjoint, buffer_size)
   1202         max_step=max_step, exclude=exclude, step_down_p=step_down_p,
   1203         t_power=t_power, out_type=out_type, check_disjoint=check_disjoint,
-> 1204         buffer_size=buffer_size)
   1205 
   1206 

/export/research/analysis/human/jhouck/shared/tools/mne-python/mne/stats/cluster_level.py in _permutation_cluster_test(***failed resolving arguments***)
    883                                 t_power, order, sample_shape, buffer_size,
    884                                 progress_bar.subset(idx))
--> 885                 for idx, order in split_list(orders, n_jobs, idx=True))
    886         # include original (true) ordering
    887         if tail == -1:  # up tail

/export/research/analysis/human/jhouck/shared/tools/anaconda/envs/mnedev/lib/python3.6/site-packages/joblib/parallel.py in __call__(self, iterable)
    928 
    929             with self._backend.retrieval_context():
--> 930                 self.retrieve()
    931             # Make sure that we get a last message telling us we are done
    932             elapsed_time = time.time() - self._start_time

/export/research/analysis/human/jhouck/shared/tools/anaconda/envs/mnedev/lib/python3.6/site-packages/joblib/parallel.py in retrieve(self)
    831             try:
    832                 if getattr(self._backend, 'supports_timeout', False):
--> 833                     self._output.extend(job.get(timeout=self.timeout))
    834                 else:
    835                     self._output.extend(job.get())

/export/research/analysis/human/jhouck/shared/tools/anaconda/envs/mnedev/lib/python3.6/site-packages/joblib/_parallel_backends.py in wrap_future_result(future, timeout)
    519         AsyncResults.get from multiprocessing."""
    520         try:
--> 521             return future.result(timeout=timeout)
    522         except LokyTimeoutError:
    523             raise TimeoutError()

/export/research/analysis/human/jhouck/shared/tools/anaconda/envs/mnedev/lib/python3.6/concurrent/futures/_base.py in result(self, timeout)
    430                 raise CancelledError()
    431             elif self._state == FINISHED:
--> 432                 return self.__get_result()
    433             else:
    434                 raise TimeoutError()

/export/research/analysis/human/jhouck/shared/tools/anaconda/envs/mnedev/lib/python3.6/concurrent/futures/_base.py in __get_result(self)
    382     def __get_result(self):
    383         if self._exception:
--> 384             raise self._exception
    385         else:
    386             return self._result

RuntimeError: The task could not be sent to the workers as it is too large for `send_bytes`.


Discussion:
it's a joblib issue that should be reported on joblib issue tracker.
to help joblib devs you should see if you can make toy script that
does not need mne and that replicates the pb.

FWIW if I borrow @kingjr's test code from the above-referenced issue , it also crashed with ndarrays of this size with `max_nbytes=None` and ran without error with `max_nbytes='1M'`.   

If I do `mne.set_memmap_min_size('1M')` before running `spatio_temporal_cluster_1samp_test` it will now run without error.  Out of curiosity, what is the default for MNE_MEMMAP_MIN_SIZE?  Mine was coming back as None, which might be a problem with my setup. 
*max_nbytes='1M' is the default in joblib *

https://joblib.readthedocs.io/en/latest/generated/joblib.Parallel.html

so we should not be smarter than this in MNE. Maybe it needs to be
investigated....

>

It looks like this has come up before in MNE as well (https://github.com/mne-tools/mne-python/issues/3190) and the outcome was to leave the joblib default unmolested. 

There is a big angry warning in parallel_func that should display if n_jobs>1 and cache_dir is not set (line 91 in parallel.py), but I don't see that message when I run `spatio_temporal_cluster_1samp_test` under those conditions. E.g.:
```
import numpy as np
from mne.stats import spatio_temporal_cluster_1samp_test
import mne
from scipy import stats as stats
mmap_test1 = mne.get_config('MNE_MEMMAP_MIN_SIZE')  
mne.set_cache_dir(None)
testdata = np.random.rand(33, 545, 20484)
n_subjects = len(testdata)
p_threshold = 0.05 
t_threshold = -stats.distributions.t.ppf(p_threshold / 2., n_subjects - 1)
T_obs, clusters, cluster_p_values, H0 = clu = \
    spatio_temporal_cluster_1samp_test(testdata, connectivity=None, n_permutations=1, n_jobs=8,
                                       threshold=t_threshold, check_disjoint=True,
                                       verbose='DEBUG') 
```
will produce the error message from the first post ("The task could not be sent to the workers as it is too large for `send_bytes`." etc.) but nothing about the cache dir.  For me this is consistent across logging levels.


you should have a look at:

https://github.com/mne-tools/mne-python/blob/master/mne/parallel.py#L79

to see if something can be fixed.

@jhouck I can run this code:
```
import time
import numpy as np
import mne
src_fname = (mne.datasets.sample.data_path() +
             '/subjects/fsaverage/bem/fsaverage-ico-5-src.fif')
X = np.random.RandomState(0).randn(33, 545, 20484) * 0.1
src = mne.read_source_spaces(src_fname)
connectivity = mne.spatial_src_connectivity(src)
t0 = time.time()
mne.stats.spatio_temporal_cluster_1samp_test(
    X, connectivity=connectivity, n_jobs=2, buffer_size=None, verbose=True,
    max_step=1)
print(time.time() - t0)
```
This was the size that broke things for you. Are you on a 32-bit system by any chance? If so we can probably write a suitable error message for this.

Resolving patch:
diff --git a/mne/parallel.py b/mne/parallel.py
--- a/mne/parallel.py
+++ b/mne/parallel.py
@@ -109,7 +109,7 @@ def parallel_func(func, n_jobs, max_nbytes='auto', pre_dispatch='n_jobs',
             kwargs['max_nbytes'] = max_nbytes
 
         n_jobs = check_n_jobs(n_jobs)
-        parallel = Parallel(n_jobs, **kwargs)
+        parallel = _check_wrapper(Parallel(n_jobs, **kwargs))
         my_func = delayed(func)
 
     if total is not None:
@@ -122,6 +122,22 @@ def parallel_progress(op_iter):
     return parallel_out, my_func, n_jobs
 
 
+def _check_wrapper(fun):
+    def run(*args, **kwargs):
+        try:
+            return fun(*args, **kwargs)
+        except RuntimeError as err:
+            msg = str(err.args[0]) if err.args else ''
+            if msg.startswith('The task could not be sent to the workers'):
+                raise RuntimeError(
+                    msg + ' Consider using joblib memmap caching to get '
+                    'around this problem. See mne.set_mmap_min_size, '
+                    'mne.set_cache_dir, and buffer_size parallel function '
+                    'arguments (if applicable).')
+            raise
+    return run
+
+
 def check_n_jobs(n_jobs, allow_cuda=False):
     """Check n_jobs in particular for negative values.
 


Test patch: