Repository: nilearn/nilearn

Issue:
Bug in _uncompress_file
Hi!
(...still nilearn-sprinting in the train back to Marseille ;) )

I found a bug when trying to download .tar.gz archive files from osf.

I got an error when using _fetch_files:

```
from nilearn.datasets.utils import _fetch_files
opts = {'uncompress': True}
data_dir = '~/nilearn_data'
url = 'https://osf.io/vym9k/download'
_fetch_files(data_dir, [('sphere_right.gii', url, opts)])
```

After digging, I realized the problem is in _uncompress_file: the temporary downloaded file is actually called 'download' on disk (because of the url that's given by osf), and  _uncompress_file does not extract the files if the archive was a .tar.gz one (it does not return an error, and it erases the 'download' file)

Note that:
- if the archive is a zip file, everything works... (as in the fetch_icbm152_2009 function); the temporary file is also called 'download' on disk, but _uncompress_file handles it properly
- if the file on disk is actually called 'whatever.tar.gz' and you run _uncompress_file on it, it also handles it properly

Soooooooooo, to reproduce the problem, the easiest is that:
1. you type https://osf.io/vym9k/download in your browser to get the file
2. you rename the file manually in your unix shell
```
> mv sphere_mesh.tar.gz download
```
3. you try to use _uncompress_file on it
```
from nilearn.datasets.utils import _uncompress_file
_uncompress_file('download')
```
It will: 1. not extract the files in the archive, 2. erase the 'download' file, 3. not return an error.

I hope this is clear enough...

Sylvain




Discussion:
@SylvainTakerkart  asked me to replace on OSF the two .tar.gz files with the .zip files and send the two new links.
On it.


InterTVA: https://osf.io/aekp7/download
sphere mesh: https://osf.io/b79fy/download
Thanks! I can then move forward with writing my fetchers (just created a PR for the sphere meshes)...

But this bug will still need to be looked at at some point...

Resolving patch:
diff --git a/nilearn/datasets/func.py b/nilearn/datasets/func.py
--- a/nilearn/datasets/func.py
+++ b/nilearn/datasets/func.py
@@ -1910,7 +1910,8 @@ def _fetch_development_fmri_participants(data_dir, url, verbose):
     return participants
 
 
-def _fetch_development_fmri_functional(participants, data_dir, url, verbose):
+def _fetch_development_fmri_functional(participants, data_dir, url, resume,
+                                       verbose):
     """Helper function to fetch_development_fmri.
 
     This function helps in downloading functional MRI data in Nifti
@@ -1932,6 +1933,9 @@ def _fetch_development_fmri_functional(participants, data_dir, url, verbose):
         Override download URL. Used for test only (or if you setup a mirror of
         the data). Default: None
 
+    resume: bool, optional (default True)
+        Whether to resume download of a partly-downloaded file.
+
     verbose: int
         Defines the level of verbosity of the output.
 
@@ -1981,13 +1985,15 @@ def _fetch_development_fmri_functional(participants, data_dir, url, verbose):
         func_url = url.format(this_osf_id['key_b'][0])
         func_file = [(func.format(participant_id, participant_id), func_url,
                       {'move': func.format(participant_id)})]
-        path_to_func = _fetch_files(data_dir, func_file, verbose=verbose)[0]
+        path_to_func = _fetch_files(data_dir, func_file, resume=resume,
+                                    verbose=verbose)[0]
         funcs.append(path_to_func)
     return funcs, regressors
 
 
 def fetch_development_fmri(n_subjects=None, reduce_confounds=True,
-                           data_dir=None, resume=True, verbose=1):
+                           data_dir=None, resume=True, verbose=1,
+                           age_group='both'):
     """Fetch movie watching based brain development dataset (fMRI)
 
     The data is downsampled to 4mm resolution for convenience. The origin of
@@ -2019,6 +2025,12 @@ def fetch_development_fmri(n_subjects=None, reduce_confounds=True,
     verbose: int, optional (default 1)
         Defines the level of verbosity of the output.
 
+    age_group: str, optional (default 'both')
+        Which age group to fetch
+        - 'adults' = fetch adults only (n=33, ages 18-39)
+        - 'child' = fetch children only (n=122, ages 3-12)
+        - 'both' = fetch full sample (n=155)
+
     Returns
     -------
     data: Bunch
@@ -2044,6 +2056,10 @@ def fetch_development_fmri(n_subjects=None, reduce_confounds=True,
 
     Preprocessing details: https://osf.io/wjtyq/
 
+    Note that if n_subjects > 2, and age_group is 'both',
+    fetcher will return a ratio of children and adults representative
+    of the total sample.
+
     References
     ----------
     Please cite this paper if you are using this dataset.
@@ -2070,52 +2086,104 @@ def fetch_development_fmri(n_subjects=None, reduce_confounds=True,
                                                         url=None,
                                                         verbose=verbose)
 
-    max_subjects = len(participants)
-    if n_subjects is None:
-        n_subjects = max_subjects
-
-    if (isinstance(n_subjects, numbers.Number) and
-            ((n_subjects > max_subjects) or (n_subjects < 1))):
-        warnings.warn("Wrong value for n_subjects={0}. The maximum "
-                      "value will be used instead n_subjects={1}"
-                      .format(n_subjects, max_subjects))
-        n_subjects = max_subjects
+    adult_count, child_count = _filter_func_regressors_by_participants(
+            participants, age_group)  # noqa: E126
+    max_subjects = adult_count + child_count
 
-    # Download functional and regressors based on participants
-    child_count = participants['Child_Adult'].tolist().count('child')
-    adult_count = participants['Child_Adult'].tolist().count('adult')
+    n_subjects = _set_invalid_n_subjects_to_max(n_subjects,
+                                                max_subjects,
+                                                age_group)
 
     # To keep the proportion of children versus adults
-    n_child = np.round(float(n_subjects) / max_subjects * child_count).astype(int)
-    n_adult = np.round(float(n_subjects) / max_subjects * adult_count).astype(int)
+    percent_total = float(n_subjects) / max_subjects
+    n_child = np.round(percent_total * child_count).astype(int)
+    n_adult = np.round(percent_total * adult_count).astype(int)
 
-    # First, restrict the csv files to the adequate number of subjects
-    child_ids = participants[participants['Child_Adult'] ==
-                             'child']['participant_id'][:n_child]
-    adult_ids = participants[participants['Child_Adult'] ==
-                             'adult']['participant_id'][:n_adult]
-    ids = np.hstack([child_ids, adult_ids])
-    participants = participants[np.in1d(participants['participant_id'],
-                                        ids)]
+    # We want to return adults by default (i.e., `age_group=both`) or
+    # if explicitly requested.
+    if (age_group != 'child') and (n_subjects == 1):
+        n_adult, n_child = 1, 0
+
+    if (age_group == 'both') and (n_subjects == 2):
+        n_adult, n_child = 1, 1
+
+    participants = _filter_csv_by_n_subjects(participants, n_adult, n_child)
 
     funcs, regressors = _fetch_development_fmri_functional(participants,
                                                            data_dir=data_dir,
                                                            url=None,
+                                                           resume=resume,
                                                            verbose=verbose)
 
     if reduce_confounds:
-        reduced_regressors = []
-        for in_file in regressors:
-            out_file = in_file.replace('desc-confounds',
-                                       'desc-reducedConfounds')
-            if not os.path.isfile(out_file):
-                confounds = np.recfromcsv(in_file, delimiter='\t')
-                selected_confounds = confounds[keep_confounds]
-                header = '\t'.join(selected_confounds.dtype.names)
-                np.savetxt(out_file, np.array(selected_confounds.tolist()),
-                           header=header, delimiter='\t', comments='')
-            reduced_regressors.append(out_file)
-        regressors = reduced_regressors
-
+        regressors = _reduce_confounds(regressors, keep_confounds)
     return Bunch(func=funcs, confounds=regressors, phenotypic=participants,
                  description=fdescr)
+
+
+def _filter_func_regressors_by_participants(participants, age_group):
+    """ Filter functional and regressors based on participants
+    """
+    valid_age_groups = ('both', 'child', 'adult')
+    if age_group not in valid_age_groups:
+        raise ValueError("Wrong value for age_group={0}. "
+                         "Valid arguments are: {1}".format(age_group,
+                                                           valid_age_groups)
+                         )
+
+    child_adult = participants['Child_Adult'].tolist()
+
+    if age_group != 'adult':
+        child_count = child_adult.count('child')
+    else:
+        child_count = 0
+
+    if age_group != 'child':
+        adult_count = child_adult.count('adult')
+    else:
+        adult_count = 0
+    return adult_count, child_count
+
+
+def _filter_csv_by_n_subjects(participants, n_adult, n_child):
+    """Restrict the csv files to the adequate number of subjects
+    """
+    child_ids = participants[participants['Child_Adult'] ==
+                             'child']['participant_id'][:n_child]
+    adult_ids = participants[participants['Child_Adult'] ==
+                             'adult']['participant_id'][:n_adult]
+    ids = np.hstack([adult_ids, child_ids])
+    participants = participants[np.in1d(participants['participant_id'], ids)]
+    participants = participants[np.argsort(participants, order='Child_Adult')]
+    return participants
+
+
+def _set_invalid_n_subjects_to_max(n_subjects, max_subjects, age_group):
+    """ If n_subjects is invalid, sets it to max.
+    """
+    if n_subjects is None:
+        n_subjects = max_subjects
+
+    if (isinstance(n_subjects, numbers.Number) and
+            ((n_subjects > max_subjects) or (n_subjects < 1))):
+        warnings.warn("Wrong value for n_subjects={0}. The maximum "
+                      "value (for age_group={1}) will be used instead: "
+                      "n_subjects={2}"
+                      .format(n_subjects, age_group, max_subjects))
+        n_subjects = max_subjects
+    return n_subjects
+
+
+def _reduce_confounds(regressors, keep_confounds):
+    reduced_regressors = []
+    for in_file in regressors:
+        out_file = in_file.replace('desc-confounds',
+                                   'desc-reducedConfounds')
+        if not os.path.isfile(out_file):
+            confounds = np.recfromcsv(in_file, delimiter='\t')
+            selected_confounds = confounds[keep_confounds]
+            header = '\t'.join(selected_confounds.dtype.names)
+            np.savetxt(out_file, np.array(selected_confounds.tolist()),
+                       header=header, delimiter='\t', comments='')
+        reduced_regressors.append(out_file)
+    return reduced_regressors


Test patch:
diff --git a/nilearn/datasets/tests/test_func.py b/nilearn/datasets/tests/test_func.py
--- a/nilearn/datasets/tests/test_func.py
+++ b/nilearn/datasets/tests/test_func.py
@@ -6,10 +6,13 @@
 
 import os
 import uuid
+
 import numpy as np
 import json
 import nibabel
 import gzip
+
+import pytest
 from sklearn.utils import check_random_state
 
 from nose import with_setup
@@ -609,6 +612,7 @@ def test_fetch_development_fmri_functional():
     funcs, confounds = func._fetch_development_fmri_functional(csv,
                                                                data_dir=tst.tmpdir,
                                                                url=local_url,
+                                                               resume=True,
                                                                verbose=1)
     assert_equal(len(funcs), 8)
     assert_equal(len(confounds), 8)
@@ -633,3 +637,47 @@ def test_fetch_development_fmri():
                                        verbose=1)
     confounds = np.recfromcsv(data.confounds[0], delimiter='\t')
     assert_equal(len(confounds[0]), 28)
+
+    # check first subject is an adult
+    data = func.fetch_development_fmri(n_subjects=1, reduce_confounds=False,
+                                       verbose=1)
+    age_group = data.phenotypic['Child_Adult'][0]
+    assert_equal(age_group, 'adult')
+
+    # check first subject is an child if requested with age_group
+    data = func.fetch_development_fmri(n_subjects=1, reduce_confounds=False,
+                                       verbose=1, age_group='child')
+    age_group = data.phenotypic['Child_Adult'][0]
+    assert_equal(age_group, 'child')
+
+    # check one of each age group returned if n_subject == 2
+    # and age_group == 'both
+    data = func.fetch_development_fmri(n_subjects=2, reduce_confounds=False,
+                                       verbose=1, age_group='both')
+    age_group = data.phenotypic['Child_Adult']
+    assert(all(age_group == ['adult', 'child']))
+
+    # check age_group
+    data = func.fetch_development_fmri(n_subjects=2, reduce_confounds=False,
+                                       verbose=1, age_group='child')
+    assert(all([x == 'child' for x in data.phenotypic['Child_Adult']]))
+
+
+@with_setup(tst.setup_tmpdata, tst.teardown_tmpdata)
+def test_fetch_development_fmri_invalid_n_subjects():
+    max_subjects = 155
+    n_subjects = func._set_invalid_n_subjects_to_max(n_subjects=-1,
+                                                     max_subjects=max_subjects,
+                                                     age_group='adult')
+    assert n_subjects == max_subjects
+    with pytest.warns(UserWarning, match='Wrong value for n_subjects='):
+        func._set_invalid_n_subjects_to_max(n_subjects=-1,
+                                            max_subjects=max_subjects,
+                                            age_group='adult')
+
+
+@with_setup(tst.setup_tmpdata, tst.teardown_tmpdata)
+def test_fetch_development_fmri_exception():
+    with pytest.raises(ValueError, match='Wrong value for age_group'):
+        func._filter_func_regressors_by_participants(participants='junk',
+                                                     age_group='junk for test')