Repository: nipy/nipype

Issue:
BIDSDataGrabber doesn't populate outputspec when outfields is defined
### Summary
When defining custom fields BIDSDataGrabber doesn't add the corresponding traits to output_spec, so it's not possible to link nodes in a workflow.

### Actual behavior
```python
In [1]: from nipype.interfaces.bids_utils import BIDSDataGrabber

In [2]: bg = BIDSDataGrabber(infields=['infield1', 'infield2'], outfields=['outfield1', 'outfield2'])

In [3]: bg._outputs()
Out[3]: 
```
### Expected behavior

```python
In [1]: from nipype.interfaces.bids_utils import BIDSDataGrabber

In [2]: bg = BIDSDataGrabber(infields=['infield1', 'infield2'], outfields=['outfield1', 'outfield2'])

In [3]: bg._outputs()
Out[3]: 

outfield1 = <undefined>
outfield2 = <undefined>

```

I started working on a fix for this, and it seems to me that perhaps subclassing IOBase instead of BaseInterface might be a solution. But I was wondering what you think it would be better.


Discussion:

Resolving patch:
diff --git a/nipype/interfaces/__init__.py b/nipype/interfaces/__init__.py
--- a/nipype/interfaces/__init__.py
+++ b/nipype/interfaces/__init__.py
@@ -10,5 +10,5 @@
                         absolute_import)
 __docformat__ = 'restructuredtext'
 
-from .io import DataGrabber, DataSink, SelectFiles
+from .io import DataGrabber, DataSink, SelectFiles, BIDSDataGrabber
 from .utility import IdentityInterface, Rename, Function, Select, Merge
diff --git a/nipype/interfaces/bids_utils.py b/nipype/interfaces/bids_utils.py
deleted file mode 100644
--- a/nipype/interfaces/bids_utils.py
+++ /dev/null
@@ -1,149 +0,0 @@
-# -*- coding: utf-8 -*-
-# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
-# vi: set ft=python sts=4 ts=4 sw=4 et:
-""" Set of interfaces that allow interaction with BIDS data. Currently
-available interfaces are:
-
-BIDSDataGrabber: Query data from BIDS dataset using pybids grabbids.
-
-
-    Change directory to provide relative paths for doctests
-    >>> import os
-    >>> filepath = os.path.dirname( os.path.realpath( __file__ ) )
-    >>> datadir = os.path.realpath(os.path.join(filepath, '../testing/data'))
-    >>> os.chdir(datadir)
-"""
-from os.path import join, dirname
-import json
-from .. import logging
-from .base import (traits, DynamicTraitedSpec, Directory, BaseInterface,
-                   isdefined, Str, Undefined)
-
-have_pybids = True
-try:
-    from bids import grabbids as gb
-except ImportError:
-    have_pybids = False
-
-LOGGER = logging.getLogger('workflows')
-
-
-class BIDSDataGrabberInputSpec(DynamicTraitedSpec):
-    base_dir = Directory(
-        exists=True, desc='Path to BIDS Directory.', mandatory=True)
-    output_query = traits.Dict(
-        key_trait=Str,
-        value_trait=traits.Dict,
-        desc='Queries for outfield outputs')
-    raise_on_empty = traits.Bool(
-        True,
-        usedefault=True,
-        desc='Generate exception if list is empty '
-        'for a given field')
-    return_type = traits.Enum('file', 'namedtuple', usedefault=True)
-
-
-class BIDSDataGrabber(BaseInterface):
-    """ BIDS datagrabber module that wraps around pybids to allow arbitrary
-    querying of BIDS datasets.
-
-    Examples
-    --------
-
-    By default, the BIDSDataGrabber fetches anatomical and functional images
-    from a project, and makes BIDS entities (e.g. subject) available for
-    filtering outputs.
-
-    >>> bg = BIDSDataGrabber()
-    >>> bg.inputs.base_dir = 'ds005/'
-    >>> bg.inputs.subject = '01'
-    >>> results = bg.run() # doctest: +SKIP
-
-
-    Dynamically created, user-defined output fields can also be defined to
-    return different types of outputs from the same project. All outputs
-    are filtered on common entities, which can be explicitly defined as
-    infields.
-
-    >>> bg = BIDSDataGrabber(infields = ['subject'], outfields = ['dwi'])
-    >>> bg.inputs.base_dir = 'ds005/'
-    >>> bg.inputs.subject = '01'
-    >>> bg.inputs.output_query['dwi'] = dict(modality='dwi')
-    >>> results = bg.run() # doctest: +SKIP
-
-    """
-    input_spec = BIDSDataGrabberInputSpec
-    output_spec = DynamicTraitedSpec
-    _always_run = True
-
-    def __init__(self, infields=None, **kwargs):
-        """
-        Parameters
-        ----------
-        infields : list of str
-            Indicates the input fields to be dynamically created
-
-        outfields: list of str
-            Indicates output fields to be dynamically created.
-            If no matching items, returns Undefined.
-        """
-        super(BIDSDataGrabber, self).__init__(**kwargs)
-
-        if not isdefined(self.inputs.output_query):
-            self.inputs.output_query = {
-                "func": {
-                    "modality": "func"
-                },
-                "anat": {
-                    "modality": "anat"
-                }
-            }
-
-        # If infields is empty, use all BIDS entities
-        if infields is not None and have_pybids:
-            bids_config = join(dirname(gb.__file__), 'config', 'bids.json')
-            bids_config = json.load(open(bids_config, 'r'))
-            infields = [i['name'] for i in bids_config['entities']]
-
-        self._infields = infields or []
-
-        # used for mandatory inputs check
-        undefined_traits = {}
-        for key in self._infields:
-            self.inputs.add_trait(key, traits.Any)
-            undefined_traits[key] = kwargs[key] if key in kwargs else Undefined
-
-        self.inputs.trait_set(trait_change_notify=False, **undefined_traits)
-
-    def _run_interface(self, runtime):
-        if not have_pybids:
-            raise ImportError(
-                "The BIDSEventsGrabber interface requires pybids."
-                " Please make sure it is installed.")
-        return runtime
-
-    def _list_outputs(self):
-        layout = gb.BIDSLayout(self.inputs.base_dir)
-
-        # If infield is not given nm input value, silently ignore
-        filters = {}
-        for key in self._infields:
-            value = getattr(self.inputs, key)
-            if isdefined(value):
-                filters[key] = value
-
-        outputs = {}
-        for key, query in self.inputs.output_query.items():
-            args = query.copy()
-            args.update(filters)
-            filelist = layout.get(return_type=self.inputs.return_type, **args)
-            if len(filelist) == 0:
-                msg = 'Output key: %s returned no files' % key
-                if self.inputs.raise_on_empty:
-                    raise IOError(msg)
-                else:
-                    LOGGER.warning(msg)
-                    filelist = Undefined
-
-            outputs[key] = filelist
-        return outputs
diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py
--- a/nipype/interfaces/io.py
+++ b/nipype/interfaces/io.py
@@ -25,12 +25,14 @@
 import glob
 import fnmatch
 import string
+import json
 import os
 import os.path as op
 import shutil
 import subprocess
 import re
 import tempfile
+from os.path import join, dirname
 from warnings import warn
 
 import sqlite3
@@ -38,10 +40,15 @@
 from .. import config, logging
 from ..utils.filemanip import copyfile, list_to_filename, filename_to_list
 from ..utils.misc import human_order_sorted, str2bool
-from .base import (TraitedSpec, traits, Str, File, Directory, BaseInterface,
-                   InputMultiPath, isdefined, OutputMultiPath,
-                   DynamicTraitedSpec, Undefined, BaseInterfaceInputSpec)
-from .bids_utils import BIDSDataGrabber
+from .base import (
+    TraitedSpec, traits, Str, File, Directory, BaseInterface, InputMultiPath,
+    isdefined, OutputMultiPath, DynamicTraitedSpec, Undefined, BaseInterfaceInputSpec)
+
+have_pybids = True
+try:
+    from bids import grabbids as gb
+except ImportError:
+    have_pybids = False
 
 try:
     import pyxnat
@@ -2717,3 +2724,116 @@ def _list_outputs(self):
         outputs = self.output_spec().get()
         outputs['out_file'] = out_file
         return outputs
+
+
+class BIDSDataGrabberInputSpec(DynamicTraitedSpec):
+    base_dir = Directory(exists=True,
+                         desc='Path to BIDS Directory.',
+                         mandatory=True)
+    output_query = traits.Dict(key_trait=Str,
+                               value_trait=traits.Dict,
+                               desc='Queries for outfield outputs')
+    raise_on_empty = traits.Bool(True, usedefault=True,
+                                 desc='Generate exception if list is empty '
+                                 'for a given field')
+    return_type = traits.Enum('file', 'namedtuple', usedefault=True)
+
+
+class BIDSDataGrabber(IOBase):
+
+    """ BIDS datagrabber module that wraps around pybids to allow arbitrary
+    querying of BIDS datasets.
+
+    Examples
+    --------
+
+    By default, the BIDSDataGrabber fetches anatomical and functional images
+    from a project, and makes BIDS entities (e.g. subject) available for
+    filtering outputs.
+
+    >>> bg = BIDSDataGrabber()
+    >>> bg.inputs.base_dir = 'ds005/'
+    >>> bg.inputs.subject = '01'
+    >>> results = bg.run() # doctest: +SKIP
+
+
+    Dynamically created, user-defined output fields can also be defined to
+    return different types of outputs from the same project. All outputs
+    are filtered on common entities, which can be explicitly defined as
+    infields.
+
+    >>> bg = BIDSDataGrabber(infields = ['subject'], outfields = ['dwi'])
+    >>> bg.inputs.base_dir = 'ds005/'
+    >>> bg.inputs.subject = '01'
+    >>> bg.inputs.output_query['dwi'] = dict(modality='dwi')
+    >>> results = bg.run() # doctest: +SKIP
+
+    """
+    input_spec = BIDSDataGrabberInputSpec
+    output_spec = DynamicTraitedSpec
+    _always_run = True
+
+    def __init__(self, infields=None, **kwargs):
+        """
+        Parameters
+        ----------
+        infields : list of str
+            Indicates the input fields to be dynamically created
+        """
+        super(BIDSDataGrabber, self).__init__(**kwargs)
+
+        if not isdefined(self.inputs.output_query):
+            self.inputs.output_query = {"func": {"modality": "func"},
+                                        "anat": {"modality": "anat"}}
+
+        # If infields is empty, use all BIDS entities
+        if infields is None and have_pybids:
+            bids_config = join(dirname(gb.__file__), 'config', 'bids.json')
+            bids_config = json.load(open(bids_config, 'r'))
+            infields = [i['name'] for i in bids_config['entities']]
+
+        self._infields = infields or []
+
+        # used for mandatory inputs check
+        undefined_traits = {}
+        for key in self._infields:
+            self.inputs.add_trait(key, traits.Any)
+            undefined_traits[key] = kwargs[key] if key in kwargs else Undefined
+
+        self.inputs.trait_set(trait_change_notify=False, **undefined_traits)
+
+    def _run_interface(self, runtime):
+        if not have_pybids:
+            raise ImportError(
+                "The BIDSEventsGrabber interface requires pybids."
+                " Please make sure it is installed.")
+        return runtime
+
+    def _list_outputs(self):
+        layout = gb.BIDSLayout(self.inputs.base_dir)
+
+        # If infield is not given nm input value, silently ignore
+        filters = {}
+        for key in self._infields:
+            value = getattr(self.inputs, key)
+            if isdefined(value):
+                filters[key] = value
+
+        outputs = {}
+        for key, query in self.inputs.output_query.items():
+            args = query.copy()
+            args.update(filters)
+            filelist = layout.get(return_type=self.inputs.return_type, **args)
+            if len(filelist) == 0:
+                msg = 'Output key: %s returned no files' % key
+                if self.inputs.raise_on_empty:
+                    raise IOError(msg)
+                else:
+                    iflogger.warning(msg)
+                    filelist = Undefined
+
+            outputs[key] = filelist
+        return outputs
+
+    def _add_output_traits(self, base):
+        return add_traits(base, list(self.inputs.output_query.keys()))


Test patch:
diff --git a/nipype/interfaces/tests/test_auto_BIDSDataGrabber.py b/nipype/interfaces/tests/test_auto_BIDSDataGrabber.py
--- a/nipype/interfaces/tests/test_auto_BIDSDataGrabber.py
+++ b/nipype/interfaces/tests/test_auto_BIDSDataGrabber.py
@@ -1,6 +1,6 @@
 # AUTO-GENERATED by tools/checkspecs.py - DO NOT EDIT
 from __future__ import unicode_literals
-from ..bids_utils import BIDSDataGrabber
+from ..io import BIDSDataGrabber
 
 
 def test_BIDSDataGrabber_inputs():
diff --git a/nipype/interfaces/tests/test_bids.py b/nipype/interfaces/tests/test_bids.py
deleted file mode 100644
--- a/nipype/interfaces/tests/test_bids.py
+++ /dev/null
@@ -1,50 +0,0 @@
-import os
-import json
-import sys
-
-import pytest
-from nipype.interfaces.bids_utils import BIDSDataGrabber
-from nipype.utils.filemanip import dist_is_editable
-
-have_pybids = True
-try:
-    import bids
-    from bids import grabbids as gb
-    filepath = os.path.realpath(os.path.dirname(bids.__file__))
-    datadir = os.path.realpath(os.path.join(filepath, 'grabbids/tests/data/'))
-except ImportError:
-    have_pybids = False
-
-
-# There are three reasons these tests will be skipped:
-@pytest.mark.skipif(not have_pybids, reason="Pybids is not installed")
-@pytest.mark.skipif(
-    sys.version_info < (3, 0), reason="Pybids no longer supports Python 2")
-@pytest.mark.skipif(
-    not dist_is_editable('pybids'),
-    reason="Pybids is not installed in editable mode")
-def test_bids_grabber(tmpdir):
-    tmpdir.chdir()
-    bg = BIDSDataGrabber()
-    bg.inputs.base_dir = os.path.join(datadir, 'ds005')
-    bg.inputs.subject = '01'
-    results = bg.run()
-    assert os.path.basename(results.outputs.anat[0]) == 'sub-01_T1w.nii.gz'
-    assert os.path.basename(results.outputs.func[0]) == (
-        'sub-01_task-mixedgamblestask_run-01_bold.nii.gz')
-
-
-@pytest.mark.skipif(not have_pybids, reason="Pybids is not installed")
-@pytest.mark.skipif(
-    sys.version_info < (3, 0), reason="Pybids no longer supports Python 2")
-@pytest.mark.skipif(
-    not dist_is_editable('pybids'),
-    reason="Pybids is not installed in editable mode")
-def test_bids_fields(tmpdir):
-    tmpdir.chdir()
-    bg = BIDSDataGrabber(infields=['subject'], outfields=['dwi'])
-    bg.inputs.base_dir = os.path.join(datadir, 'ds005')
-    bg.inputs.subject = '01'
-    bg.inputs.output_query['dwi'] = dict(modality='dwi')
-    results = bg.run()
-    assert os.path.basename(results.outputs.dwi[0]) == 'sub-01_dwi.nii.gz'
diff --git a/nipype/interfaces/tests/test_io.py b/nipype/interfaces/tests/test_io.py
--- a/nipype/interfaces/tests/test_io.py
+++ b/nipype/interfaces/tests/test_io.py
@@ -9,6 +9,7 @@
 import glob
 import shutil
 import os.path as op
+import sys
 from subprocess import Popen
 import hashlib
 from collections import namedtuple
@@ -16,7 +17,9 @@
 import pytest
 import nipype
 import nipype.interfaces.io as nio
+from nipype.interfaces.base.traits_extension import isdefined
 from nipype.interfaces.base import Undefined, TraitError
+from nipype.utils.filemanip import dist_is_editable
 
 # Check for boto
 noboto = False
@@ -43,6 +46,16 @@
 except CalledProcessError:
     fakes3 = False
 
+# check for bids
+have_pybids = True
+try:
+    import bids
+    from bids import grabbids as gb
+    filepath = os.path.realpath(os.path.dirname(bids.__file__))
+    datadir = os.path.realpath(os.path.join(filepath, 'grabbids/tests/data/'))
+except ImportError:
+    have_pybids = False
+
 
 def test_datagrabber():
     dg = nio.DataGrabber()
@@ -537,3 +550,64 @@ def test_jsonsink(tmpdir, inputs_attributes):
         data = simplejson.load(f)
 
     assert data == expected_data
+
+
+# There are three reasons these tests will be skipped:
+@pytest.mark.skipif(not have_pybids,
+                    reason="Pybids is not installed")
+@pytest.mark.skipif(sys.version_info < (3, 0),
+                    reason="Pybids no longer supports Python 2")
+@pytest.mark.skipif(not dist_is_editable('pybids'),
+                    reason="Pybids is not installed in editable mode")
+def test_bids_grabber(tmpdir):
+    tmpdir.chdir()
+    bg = nio.BIDSDataGrabber()
+    bg.inputs.base_dir = os.path.join(datadir, 'ds005')
+    bg.inputs.subject = '01'
+    results = bg.run()
+    assert os.path.basename(results.outputs.anat[0]) == 'sub-01_T1w.nii.gz'
+    assert os.path.basename(results.outputs.func[0]) == (
+        'sub-01_task-mixedgamblestask_run-01_bold.nii.gz')
+
+
+@pytest.mark.skipif(not have_pybids,
+                    reason="Pybids is not installed")
+@pytest.mark.skipif(sys.version_info < (3, 0),
+                    reason="Pybids no longer supports Python 2")
+@pytest.mark.skipif(not dist_is_editable('pybids'),
+                    reason="Pybids is not installed in editable mode")
+def test_bids_fields(tmpdir):
+    tmpdir.chdir()
+    bg = nio.BIDSDataGrabber(infields = ['subject'], outfields = ['dwi'])
+    bg.inputs.base_dir = os.path.join(datadir, 'ds005')
+    bg.inputs.subject = '01'
+    bg.inputs.output_query['dwi'] = dict(modality='dwi')
+    results = bg.run()
+    assert os.path.basename(results.outputs.dwi[0]) == 'sub-01_dwi.nii.gz'
+
+
+@pytest.mark.skipif(not have_pybids,
+                    reason="Pybids is not installed")
+@pytest.mark.skipif(sys.version_info < (3, 0),
+                    reason="Pybids no longer supports Python 2")
+@pytest.mark.skipif(not dist_is_editable('pybids'),
+                    reason="Pybids is not installed in editable mode")
+def test_bids_infields_outfields(tmpdir):
+    tmpdir.chdir()
+    infields = ['infield1', 'infield2']
+    outfields = ['outfield1', 'outfield2']
+    bg = nio.BIDSDataGrabber(infields=infields)
+    for outfield in outfields:
+        bg.inputs.output_query[outfield] = {'key': 'value'}
+
+    for infield in infields:
+        assert(infield in bg.inputs.traits())
+        assert(not(isdefined(bg.inputs.get()[infield])))
+
+    for outfield in outfields:
+        assert(outfield in bg._outputs().traits())
+
+    # now try without defining outfields, we should get anat and func for free
+    bg = nio.BIDSDataGrabber()
+    for outfield in ['anat', 'func']:
+        assert outfield in bg._outputs().traits()