Repository: nilearn/nilearn

Issue:
"Operation timed out" during fetch_abide_pcp()
It appears tricky to download the full ABIDE datasets as it stands today. There may be 3 different problems:
1) the download freezed several times with different Wifi connections
2) already downloaded parts of previously failed fetch attemps are not resumed
3) Operation timed out error occurs every once in a while

``` python

<ipython-input-4-565d7d34308f> in <module>()
----> 1 ds.fetch_abide_pcp()

/Users/admin/anaconda/lib/python2.7/site-packages/nilearn/datasets/func.pyc in fetch_abide_pcp(data_dir, n_subjects, pipeline, band_pass_filtering, global_signal_regression, derivatives, quality_checked, url, verbose, **kwargs)
   1248                   '/'.join([url, derivative, file_id + '_' + derivative + ext]),
   1249                   {}) for file_id in file_ids]
-> 1250         files = _fetch_files(data_dir, files, verbose=verbose)
   1251         # Load derivatives if needed
   1252         if ext == '.1D':

/Users/admin/anaconda/lib/python2.7/site-packages/nilearn/datasets/utils.pyc in _fetch_files(data_dir, files, resume, mock, verbose)
    730                                   password=opts.get('password', None),
    731                                   handlers=opts.get('handlers', []),
--> 732                                   overwrite=overwrite)
    733             if 'move' in opts:
    734                 # XXX: here, move is supposed to be a dir, it can be a name

/Users/admin/anaconda/lib/python2.7/site-packages/nilearn/datasets/utils.pyc in _fetch_file(url, data_dir, resume, overwrite, md5sum, username, password, handlers, verbose)
    560             local_file = open(temp_full_name, "wb")
    561         _chunk_read_(data, local_file, report_hook=(verbose > 0),
--> 562                      initial_size=initial_size, verbose=verbose)
    563         # temp file must be closed prior to the move
    564         if not local_file.closed:

/Users/admin/anaconda/lib/python2.7/site-packages/nilearn/datasets/utils.pyc in _chunk_read_(response, local_file, chunk_size, report_hook, initial_size, total_size, verbose)
    153     t0 = time_last_display = time.time()
    154     while True:
--> 155         chunk = response.read(chunk_size)
    156         bytes_so_far += len(chunk)
    157         time_last_read = time.time()

/Users/admin/anaconda/lib/python2.7/socket.pyc in read(self, size)
    382                 # fragmentation issues on many platforms.
    383                 try:
--> 384                     data = self._sock.recv(left)
    385                 except error, e:
    386                     if e.args[0] == EINTR:

/Users/admin/anaconda/lib/python2.7/httplib.pyc in read(self, amt)
    610         # connection, and the user is reading more bytes than will be provided
    611         # (for example, reading in 1k chunks)
--> 612         s = self.fp.read(amt)
    613         if not s and amt:
    614             # Ideally, we would raise IncompleteRead if the content-length

/Users/admin/anaconda/lib/python2.7/socket.pyc in read(self, size)
    382                 # fragmentation issues on many platforms.
    383                 try:
--> 384                     data = self._sock.recv(left)
    385                 except error, e:
    386                     if e.args[0] == EINTR:

/Users/admin/anaconda/lib/python2.7/ssl.pyc in recv(self, buflen, flags)
    732                     "non-zero flags not allowed in calls to recv() on %s" %
    733                     self.__class__)
--> 734             return self.read(buflen)
    735         else:
    736             return self._sock.recv(buflen, flags)

/Users/admin/anaconda/lib/python2.7/ssl.pyc in read(self, len, buffer)
    619                 v = self._sslobj.read(len, buffer)
    620             else:
--> 621                 v = self._sslobj.read(len or 1024)
    622             return v
    623         except SSLError as x:

error: [Errno 60] Operation timed out

```



Discussion:
True, I noticed this problem recently. Gimme 5 minutes tu prepare a fix.


Resolving patch:
diff --git a/nilearn/datasets/func.py b/nilearn/datasets/func.py
--- a/nilearn/datasets/func.py
+++ b/nilearn/datasets/func.py
@@ -1247,10 +1247,14 @@ def fetch_abide_pcp(data_dir=None, n_subjects=None, pipeline='cpac',
     results['phenotypic'] = pheno
     for derivative in derivatives:
         ext = '.1D' if derivative.startswith('rois') else '.nii.gz'
-        files = [(file_id + '_' + derivative + ext,
-                  '/'.join([url, derivative, file_id + '_' + derivative + ext]),
-                  {}) for file_id in file_ids]
-        files = _fetch_files(data_dir, files, verbose=verbose)
+        files = []
+        for file_id in file_ids:
+            file_ = [(
+                file_id + '_' + derivative + ext,
+                '/'.join([url, derivative, file_id + '_' + derivative + ext]),
+                {}
+            )]
+            files.append(_fetch_files(data_dir, file_, verbose=verbose))
         # Load derivatives if needed
         if ext == '.1D':
             files = [np.loadtxt(f) for f in files]


Test patch: