Repository: nipy/nipype

Issue:
pybids 0.6.4 causes test failure in BIDSDataGrabber
### Summary

The error in https://travis-ci.org/nipy/nipype/builds/408021599:

```
______________________________ test_bids_grabber _______________________________
[gw0] linux -- Python 3.5.5 /home/travis/virtualenv/python3.5.5/bin/python
tmpdir = local('/tmp/pytest-of-travis/pytest-0/popen-gw0/test_bids_grabber0')
    @pytest.mark.skipif(not have_pybids,
                        reason="Pybids is not installed")
    @pytest.mark.skipif(sys.version_info < (3, 0),
                        reason="Pybids no longer supports Python 2")
    @pytest.mark.skipif(not dist_is_editable('pybids'),
                        reason="Pybids is not installed in editable mode")
    def test_bids_grabber(tmpdir):
        tmpdir.chdir()
        bg = nio.BIDSDataGrabber()
        bg.inputs.base_dir = os.path.join(datadir, 'ds005')
        bg.inputs.subject = '01'
        results = bg.run()
        assert os.path.basename(results.outputs.anat[0]) == 'sub-01_T1w.nii.gz'
>       assert os.path.basename(results.outputs.func[0]) == (
            'sub-01_task-mixedgamblestask_run-01_bold.nii.gz')
E       AssertionError: assert 'sub-01_task-...01_events.tsv' == 'sub-01_task-m...1_bold.nii.gz'
E         - sub-01_task-mixedgamblestask_run-01_events.tsv
E         ?                                     ^^^ ^^ ^^^
E         + sub-01_task-mixedgamblestask_run-01_bold.nii.gz
E         ?                                     ^^^^^ ^^ ^^
/home/travis/build/nipy/nipype/nipype/interfaces/tests/test_io.py:596: AssertionError
```

@adelavega @tyarkoni Flagging you in this in case the cause is obvious.

It's not immediately clear whether the correct fix will be in pybids or nipype.

### Execution environment

Travis CI



Discussion:
Okay, it's indexing the derivatives first, so it's the new test data that's causing the issue.

```Python
In [1]: import os

In [2]: from nipype.interfaces import io as nio

In [3]: from nipype.interfaces.tests.test_io import datadir

In [4]: bg = nio.BIDSDataGrabber()

In [5]: bg.inputs.base_dir = os.path.join(datadir, 'ds005')

In [6]: bg.inputs.subject = '01'

In [7]: results = bg.run()

In [8]: results.outputs.func
Out[8]: 
['.../bids/tests/data/ds005/derivatives/events/func/sub-01_task-mixedgamblestask_run-01_events.tsv',
 '.../bids/tests/data/ds005/sub-01/func/sub-01_task-mixedgamblestask_run-01_bold.nii.gz',
 '.../bids/tests/data/ds005/sub-01/func/sub-01_task-mixedgamblestask_run-01_events.tsv',
 '.../bids/tests/data/ds005/sub-01/func/sub-01_task-mixedgamblestask_run-02_bold.nii.gz',
 '.../bids/tests/data/ds005/sub-01/func/sub-01_task-mixedgamblestask_run-02_events.tsv',
 '.../bids/tests/data/ds005/sub-01/func/sub-01_task-mixedgamblestask_run-03_bold.nii.gz',
 '.../bids/tests/data/ds005/sub-01/func/sub-01_task-mixedgamblestask_run-03_events.tsv']
```

Now to see whether it's pybids incorrectly indexing derivatives or nipype incorrectly calling `BIDSLayout`...
@tyarkoni I think this is a pybids problem. Opening an issue over there.

```Python
In [9]: import bids

In [10]: layout = bids.BIDSLayout(os.path.join(datadir, 'ds005'))

In [11]: layout.get(subject='01', task='mixedgamblestask', modality='func')
Out[11]: 
[File(filename='.../bids/tests/data/ds005/derivatives/events/func/sub-01_task-mixedgamblestask_run-01_events.tsv', subject='01', task='mixedgamblestask', run='1', type='events', modality='func'),
 File(filename='.../bids/tests/data/ds005/sub-01/func/sub-01_task-mixedgamblestask_run-01_bold.nii.gz', subject='01', task='mixedgamblestask', run='1', type='bold', modality='func'),
 ...]
```

Resolving patch:
diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py
--- a/nipype/interfaces/io.py
+++ b/nipype/interfaces/io.py
@@ -2755,6 +2755,8 @@ class BIDSDataGrabberInputSpec(DynamicTraitedSpec):
                                  desc='Generate exception if list is empty '
                                  'for a given field')
     return_type = traits.Enum('file', 'namedtuple', usedefault=True)
+    strict = traits.Bool(desc='Return only BIDS "proper" files (e.g., '
+                              'ignore derivatives/, sourcedata/, etc.)')
 
 
 class BIDSDataGrabber(IOBase):
@@ -2801,8 +2803,10 @@ def __init__(self, infields=None, **kwargs):
         super(BIDSDataGrabber, self).__init__(**kwargs)
 
         if not isdefined(self.inputs.output_query):
-            self.inputs.output_query = {"func": {"modality": "func"},
-                                        "anat": {"modality": "anat"}}
+            self.inputs.output_query = {
+                "func": {"modality": "func", 'extensions': ['nii', '.nii.gz']},
+                "anat": {"modality": "anat", 'extensions': ['nii', '.nii.gz']},
+                }
 
         # If infields is empty, use all BIDS entities
         if infields is None and have_pybids:
@@ -2828,7 +2832,10 @@ def _run_interface(self, runtime):
         return runtime
 
     def _list_outputs(self):
-        layout = gb.BIDSLayout(self.inputs.base_dir)
+        exclude = None
+        if self.inputs.strict:
+            exclude = ['derivatives/', 'code/', 'sourcedata/']
+        layout = gb.BIDSLayout(self.inputs.base_dir, exclude=exclude)
 
         # If infield is not given nm input value, silently ignore
         filters = {}


Test patch:
diff --git a/nipype/interfaces/tests/test_io.py b/nipype/interfaces/tests/test_io.py
--- a/nipype/interfaces/tests/test_io.py
+++ b/nipype/interfaces/tests/test_io.py
@@ -592,9 +592,9 @@ def test_bids_grabber(tmpdir):
     bg.inputs.base_dir = os.path.join(datadir, 'ds005')
     bg.inputs.subject = '01'
     results = bg.run()
-    assert os.path.basename(results.outputs.anat[0]) == 'sub-01_T1w.nii.gz'
-    assert os.path.basename(results.outputs.func[0]) == (
-        'sub-01_task-mixedgamblestask_run-01_bold.nii.gz')
+    assert 'sub-01_T1w.nii.gz' in map(os.path.basename, results.outputs.anat)
+    assert 'sub-01_task-mixedgamblestask_run-01_bold.nii.gz' in \
+        map(os.path.basename, results.outputs.func)
 
 
 @pytest.mark.skipif(not have_pybids,
@@ -610,7 +610,7 @@ def test_bids_fields(tmpdir):
     bg.inputs.subject = '01'
     bg.inputs.output_query['dwi'] = dict(modality='dwi')
     results = bg.run()
-    assert os.path.basename(results.outputs.dwi[0]) == 'sub-01_dwi.nii.gz'
+    assert 'sub-01_dwi.nii.gz' in map(os.path.basename, results.outputs.dwi)
 
 
 @pytest.mark.skipif(not have_pybids,