Repository: nilearn/nilearn

Issue:
ADHD dataset fetcher is not working
The problem has been reported by several people.

Debug output is:

`````` python

Dataset created in /Users/admin/nilearn_data/adhd

Downloading data from http://connectir.projects.nitrc.org/adhd40_p1.tar.gz ...
Error while fetching file adhd40_p1.tar.gz. Dataset fetching aborted.
---------------------------------------------------------------------------
HTTPError                                 Traceback (most recent call last)
<ipython-input-4-51958e38ffc8> in <module>()
----> 1 ds.fetch_adhd()

/git/nilearn/nilearn/datasets.pyc in fetch_adhd(n_subjects, data_dir, url, resume, verbose)
   1644                                 verbose=verbose)
   1645     subjects_funcs = _fetch_files(data_dir, subjects_funcs, resume=resume,
-> 1646                                   verbose=verbose)
   1647     subjects_confounds = _fetch_files(data_dir, subjects_confounds,
   1648             resume=resume, verbose=verbose)

/git/nilearn/nilearn/datasets.pyc in _fetch_files(data_dir, files, resume, mock, verbose)
    697                                   username=opts.get('username', None),
    698                                   password=opts.get('password', None),
--> 699                                   handlers=opts.get('handlers', []))
    700             if 'move' in opts:
    701                 # XXX: here, move is supposed to be a dir, it can be a name

/git/nilearn/nilearn/datasets.pyc in _fetch_file(url, data_dir, resume, overwrite, md5sum, username, password, handlers, verbose)
    526             initial_size = local_file_size
    527         else:
--> 528             data = url_opener.open(request)
    529             local_file = open(temp_full_name, "wb")
    530         _chunk_read_(data, local_file, report_hook=(verbose > 0),

/Users/admin/anaconda/lib/python2.7/urllib2.pyc in open(self, fullurl, data, timeout)
    435         for processor in self.process_response.get(protocol, []):
    436             meth = getattr(processor, meth_name)
--> 437             response = meth(req, response)
    438 
    439         return response

/Users/admin/anaconda/lib/python2.7/urllib2.pyc in http_response(self, request, response)
    548         if not (200 <= code < 300):
    549             response = self.parent.error(
--> 550                 'http', request, response, code, msg, hdrs)
    551 
    552         return response

/Users/admin/anaconda/lib/python2.7/urllib2.pyc in error(self, proto, *args)
    473         if http_err:
    474             args = (dict, 'default', 'http_error_default') + orig_args
--> 475             return self._call_chain(*args)
    476 
    477 # XXX probably also want an abstract factory that knows when it makes

/Users/admin/anaconda/lib/python2.7/urllib2.pyc in _call_chain(self, chain, kind, meth_name, *args)
    407             func = getattr(handler, meth_name)
    408 
--> 409             result = func(*args)
    410             if result is not None:
    411                 return result

/Users/admin/anaconda/lib/python2.7/urllib2.pyc in http_error_default(self, req, fp, code, msg, hdrs)
    556 class HTTPDefaultErrorHandler(BaseHandler):
    557     def http_error_default(self, req, fp, code, msg, hdrs):
--> 558         raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
    559 
    560 class HTTPRedirectHandler(BaseHandler):

HTTPError: HTTP Error 404: Not Found```
``````



Discussion:
also issues with abide 

Regarding ADHD, the file has not been hosted by us in the first place. I can re-upload it tomorrow.

Regarding Abide, please precise your problem. Some configurations are not available. The problem has been reported but, since we don't host the data, we can't fix the problem.


Resolving patch:
diff --git a/nilearn/datasets.py b/nilearn/datasets.py
--- a/nilearn/datasets.py
+++ b/nilearn/datasets.py
@@ -655,6 +655,7 @@ def _fetch_files(data_dir, files, resume=True, mock=False, verbose=1):
     #   files that must be downloaded will be in this directory. If a corrupted
     #   file is found, or a file is missing, this working directory will be
     #   deleted.
+    files = list(files)
     files_pickle = cPickle.dumps([(file_, url) for file_, url, _ in files])
     files_md5 = hashlib.md5(files_pickle).hexdigest()
     temp_dir = os.path.join(data_dir, files_md5)
@@ -1588,78 +1589,67 @@ def fetch_adhd(n_subjects=None, data_dir=None, url=None, resume=True,
     """
 
     if url is None:
-        url = 'http://connectir.projects.nitrc.org'
-    f1 = url + '/adhd40_p1.tar.gz'
-    f2 = url + '/adhd40_p2.tar.gz'
-    f3 = url + '/adhd40_p3.tar.gz'
-    f4 = url + '/adhd40_p4.tar.gz'
-    f1_opts = {'uncompress': True}
-    f2_opts = {'uncompress': True}
-    f3_opts = {'uncompress': True}
-    f4_opts = {'uncompress': True}
-
-    fname = '%s_rest_tshift_RPI_voreg_mni.nii.gz'
-    rname = '%s_regressors.csv'
-
-    # Subjects ID per file
-    sub1 = ['3902469', '7774305', '3699991']
-    sub2 = ['2014113', '4275075', '1019436', '3154996', '3884955', '0027034',
-            '4134561', '0027018', '6115230', '0027037', '8409791', '0027011']
-    sub3 = ['3007585', '8697774', '9750701', '0010064', '0021019', '0010042',
-            '0010128', '2497695', '4164316', '1552181', '4046678', '0023012']
-    sub4 = ['1679142', '1206380', '0023008', '4016887', '1418396', '2950754',
-            '3994098', '3520880', '1517058', '9744150', '1562298', '3205761',
-            '3624598']
-    subs = sub1 + sub2 + sub3 + sub4
-
-    subjects_funcs = \
-        [(os.path.join('data', i, fname % i), f1, f1_opts) for i in sub1] + \
-        [(os.path.join('data', i, fname % i), f2, f2_opts) for i in sub2] + \
-        [(os.path.join('data', i, fname % i), f3, f3_opts) for i in sub3] + \
-        [(os.path.join('data', i, fname % i), f4, f4_opts) for i in sub4]
-
-    subjects_confounds = \
-        [(os.path.join('data', i, rname % i), f1, f1_opts) for i in sub1] + \
-        [(os.path.join('data', i, rname % i), f2, f2_opts) for i in sub2] + \
-        [(os.path.join('data', i, rname % i), f3, f3_opts) for i in sub3] + \
-        [(os.path.join('data', i, rname % i), f4, f4_opts) for i in sub4]
-
-    phenotypic = [('ADHD200_40subs_motion_parameters_and_phenotypics.csv', f1,
-        f1_opts)]
-
-    max_subjects = len(subjects_funcs)
-    # Check arguments
+        url = 'https://www.nitrc.org/frs/download.php/'
+
+    # Preliminary checks and declarations
+    dataset_name = 'adhd'
+    data_dir = _get_dataset_dir(dataset_name, data_dir=data_dir,
+                                verbose=verbose)
+    ids = ['0010042', '0010064', '0010128', '0021019', '0023008', '0023012',
+           '0027011', '0027018', '0027034', '0027037', '1019436', '1206380',
+           '1418396', '1517058', '1552181', '1562298', '1679142', '2014113',
+           '2497695', '2950754', '3007585', '3154996', '3205761', '3520880',
+           '3624598', '3699991', '3884955', '3902469', '3994098', '4016887',
+           '4046678', '4134561', '4164316', '4275075', '6115230', '7774305',
+           '8409791', '8697774', '9744150', '9750701']
+    nitrc_ids = range(7782, 7822)
+    max_subjects = len(ids)
     if n_subjects is None:
         n_subjects = max_subjects
     if n_subjects > max_subjects:
         warnings.warn('Warning: there are only %d subjects' % max_subjects)
         n_subjects = max_subjects
+    ids = ids[:n_subjects]
+    nitrc_ids = nitrc_ids[:n_subjects]
 
-    subs = subs[:n_subjects]
-    subjects_funcs = subjects_funcs[:n_subjects]
-    subjects_confounds = subjects_confounds[:n_subjects]
-
-    dataset_name = 'adhd'
-    data_dir = _get_dataset_dir(dataset_name, data_dir=data_dir,
-                                verbose=verbose)
-    subjects_funcs = _fetch_files(data_dir, subjects_funcs, resume=resume,
-                                  verbose=verbose)
-    subjects_confounds = _fetch_files(data_dir, subjects_confounds,
-            resume=resume, verbose=verbose)
-    phenotypic = _fetch_files(data_dir, phenotypic, resume=resume,
-                              verbose=verbose)[0]
+    opts = dict(uncompress=True)
 
+    # Dataset description
     fdescr = _get_dataset_descr(dataset_name)
 
-    # Load phenotypic data
+    # First, get the metadata
+    phenotypic = ('ADHD200_40subs_motion_parameters_and_phenotypics.csv',
+        url + '7781/adhd40_metadata.tgz', opts)
+
+    phenotypic = _fetch_files(data_dir, [phenotypic], resume=resume,
+                              verbose=verbose)[0]
+
+    ## Load the csv file
     phenotypic = np.genfromtxt(phenotypic, names=True, delimiter=',',
                                dtype=None)
+
     # Keep phenotypic information for selected subjects
-    isubs = np.asarray(subs, dtype=int)
+    int_ids = np.asarray(ids, dtype=int)
     phenotypic = phenotypic[[np.where(phenotypic['Subject'] == i)[0][0]
-                             for i in isubs]]
+                             for i in int_ids]]
+
+    # Download dataset files
+
+    archives = [url + '%i/adhd40_%s.tgz' % (ni, ii)
+                for ni, ii in zip(nitrc_ids, ids)]
+    functionals = ['data/%s/%s_rest_tshift_RPI_voreg_mni.nii.gz' % (i, i)
+                   for i in ids]
+    confounds = ['data/%s/%s_regressors.csv' % (i, i) for i in ids]
+
+    functionals = _fetch_files(
+        data_dir, zip(functionals, archives, (opts,) * n_subjects),
+        resume=resume, verbose=verbose)
+
+    confounds = _fetch_files(
+        data_dir, zip(confounds, archives, (opts,) * n_subjects),
+        resume=resume, verbose=verbose)
 
-    return Bunch(func=subjects_funcs, confounds=subjects_confounds,
+    return Bunch(func=functionals, confounds=confounds,
                  phenotypic=phenotypic, description=fdescr)
 
 


Test patch:
diff --git a/nilearn/tests/test_datasets.py b/nilearn/tests/test_datasets.py
--- a/nilearn/tests/test_datasets.py
+++ b/nilearn/tests/test_datasets.py
@@ -438,7 +438,7 @@ def test_fetch_adhd():
                                n_subjects=12, verbose=0)
     assert_equal(len(adhd.func), 12)
     assert_equal(len(adhd.confounds), 12)
-    assert_equal(len(url_request.urls), 2)
+    assert_equal(len(url_request.urls), 13)  # Subjects + phenotypic
 
 
 @with_setup(setup_mock)