Repository: nipy/nipype

Issue:
PR #2479 has broken my package
### Summary

PR #2479 has broken my package (https://pypi.org/project/arcana/) 

I am not quite sure what the rationale behind the changes are so it is difficult to know how to debug or whether there is something I can change in my package.

### Actual behavior

Workflow exits with error

```
 File "/Users/tclose/git/ni/arcana/test/mwe/nipype_pr2479/test.py", line 71, in <module>
    study.data('out')
  File "/Users/tclose/git/ni/arcana/arcana/study/base.py", line 325, in data
    visit_ids=visit_ids)
  File "/Users/tclose/git/ni/arcana/arcana/runner/base.py", line 37, in run
    return workflow.run(plugin=self._plugin)
  File "/Users/tclose/git/ni/nipype/nipype/pipeline/engine/workflows.py", line 595, in run
    runner.run(execgraph, updatehash=updatehash, config=self.config)
  File "/Users/tclose/git/ni/nipype/nipype/pipeline/plugins/linear.py", line 44, in run
    node.run(updatehash=updatehash)
  File "/Users/tclose/git/ni/nipype/nipype/pipeline/engine/nodes.py", line 480, in run
    result = self._run_interface(execute=True)
  File "/Users/tclose/git/ni/nipype/nipype/pipeline/engine/nodes.py", line 564, in _run_interface
    return self._run_command(execute)
  File "/Users/tclose/git/ni/arcana/arcana/node.py", line 59, in _run_command
    result = self.nipype_cls._run_command(self, *args, **kwargs)
  File "/Users/tclose/git/ni/nipype/nipype/pipeline/engine/nodes.py", line 888, in _run_command
    self._collate_join_field_inputs()
  File "/Users/tclose/git/ni/nipype/nipype/pipeline/engine/nodes.py", line 898, in _collate_join_field_inputs
    val = self._collate_input_value(field)
  File "/Users/tclose/git/ni/nipype/nipype/pipeline/engine/nodes.py", line 928, in _collate_input_value
    for idx in range(self._next_slot_index)
  File "/Users/tclose/git/ni/nipype/nipype/pipeline/engine/nodes.py", line 947, in _slot_value
    field, index, e))
AttributeError: The join node pipeline1.pipeline1_subject_session_outputs does not have a slot field subject_session_pairsJ1 to hold the subject_session_pairs value at index 0: 'DynamicTraitedSpec' object has no attribute 'subject_session_pairsJ1'
```

### Expected behavior

The workflow runs without error

### How to replicate the behavior

See script below

### Script/Workflow details

I have tried to come up with a MWE that doesn't use my package but it was proving difficult. However, you can now install my package with pip

`pip install arcana`

and run the following

```
import os.path
import shutil
from nipype import config
config.enable_debug_mode()
import nipype  # @IgnorePep8
from nipype.interfaces.utility import IdentityInterface  # @IgnorePep8
from arcana.dataset import DatasetMatch, DatasetSpec  # @IgnorePep8
from arcana.data_format import text_format  # @IgnorePep8
from arcana.study.base import Study, StudyMetaClass  # @IgnorePep8
from arcana.archive.local import LocalArchive  # @IgnorePep8
from arcana.runner import LinearRunner  # @IgnorePep8


BASE_ARCHIVE_DIR = os.path.join(os.path.dirname(__file__), 'archives')
BASE_WORK_DIR = os.path.join(os.path.dirname(__file__), 'work')

print(nipype.get_info())
print(nipype.__version__)


class TestStudy(Study):

    __metaclass__ = StudyMetaClass

    add_data_specs = [
        DatasetSpec('in', text_format),
        DatasetSpec('out', text_format, 'pipeline')]

    def pipeline(self, **kwargs):
        pipeline = self.create_pipeline(
            name='pipeline1',
            inputs=[DatasetSpec('in', text_format)],
            outputs=[DatasetSpec('out', text_format)],
            desc="A dummy pipeline used to test 'run_pipeline' method",
            version=1,
            citations=[],
            **kwargs)
        ident = pipeline.create_node(IdentityInterface(['a']),
                                     name="ident")
        # Connect inputs
        pipeline.connect_input('in', ident, 'a')
        # Connect outputs
        pipeline.connect_output('out', ident, 'a')
        return pipeline


# Create archives
shutil.rmtree(BASE_ARCHIVE_DIR, ignore_errors=True)
shutil.rmtree(BASE_WORK_DIR, ignore_errors=True)
os.makedirs(BASE_ARCHIVE_DIR)

for sess in (['ARCHIVE1', 'SUBJECT', 'VISIT'],
             ['ARCHIVE2', 'SUBJECT1', 'VISIT1'],
             ['ARCHIVE2', 'SUBJECT1', 'VISIT2'],
             ['ARCHIVE2', 'SUBJECT2', 'VISIT1'],
             ['ARCHIVE2', 'SUBJECT2', 'VISIT2']):
    sess_dir = os.path.join(*([BASE_ARCHIVE_DIR] + sess))
    os.makedirs(sess_dir)
    with open(os.path.join(sess_dir, 'in.txt'), 'w') as f:
        f.write('in')

archive1_path = os.path.join(BASE_ARCHIVE_DIR, 'ARCHIVE1')
archive2_path = os.path.join(BASE_ARCHIVE_DIR, 'ARCHIVE2')

work1_path = os.path.join(BASE_WORK_DIR, 'WORK1')
work2_path = os.path.join(BASE_WORK_DIR, 'WORK2')

# Attempt to run with archive with 2 subjects and 2 visits
study = TestStudy('two',
                   LocalArchive(archive2_path),
                   LinearRunner(work2_path),
                   inputs=[DatasetMatch('in', text_format, 'in')])

# Fails here
study2.data('out')

print("Ran study 2")

# 
study1 = TestStudy('one',
                   LocalArchive(archive1_path),
                   LinearRunner(work1_path),
                   inputs=[DatasetMatch('in', text_format, 'in')])

study1.data('out')

print("Ran study 1")
```

to reproduce the error

### Platform details:

{'nibabel_version': '2.2.1', 'sys_executable': '/usr/local/opt/python@2/bin/python2.7', 'networkx_version': '1.9', 'numpy_version': '1.14.3', 'sys_platform': 'darwin', 'sys_version': '2.7.15 (default, May  1 2018, 16:44:08) \n[GCC 4.2.1 Compatible Apple LLVM 9.1.0 (clang-902.0.39.1)]', 'commit_source': 'repository', 'commit_hash': '5a96ea54a', 'pkg_path': '/Users/tclose/git/ni/nipype/nipype', 'nipype_version': '1.0.4-dev+g5a96ea54a', 'traits_version': '4.6.0', 'scipy_version': '1.1.0'}
1.0.4-dev+g5a96ea54a

(problem arose in 1.0.1)

### Execution environment

My Homebrew python 2 environment outside container


Comments:

Patch:
diff --git a/nipype/pipeline/engine/utils.py b/nipype/pipeline/engine/utils.py
index 08d357ff6..0a59aac26 100644
--- a/nipype/pipeline/engine/utils.py
+++ b/nipype/pipeline/engine/utils.py
@@ -1054,12 +1054,14 @@ def generate_expanded_graph(graph_in):
                 for src_id in list(old_edge_dict.keys()):
                     # Drop the original JoinNodes; only concerned with
                     # generated Nodes
-                    if hasattr(node, 'joinfield'):
+                    if hasattr(node, 'joinfield') and node.itername == src_id:
                         continue
                     # Patterns:
                     #   - src_id : Non-iterable node
-                    #   - src_id.[a-z]\d+ : IdentityInterface w/ iterables
-                    #   - src_id.[a-z]I.[a-z]\d+ : Non-IdentityInterface w/ iterables
+                    #   - src_id.[a-z]\d+ :
+                    #       IdentityInterface w/ iterables or nested JoinNode
+                    #   - src_id.[a-z]I.[a-z]\d+ :
+                    #       Non-IdentityInterface w/ iterables
                     #   - src_idJ\d+ : JoinNode(IdentityInterface)
                     if re.match(src_id + r'((\.[a-z](I\.[a-z])?|J)\d+)?$',
                                 node.itername):


Test patch:
diff --git a/nipype/pipeline/engine/tests/test_join.py b/nipype/pipeline/engine/tests/test_join.py
index 54ff15048..77fc0f2fd 100644
--- a/nipype/pipeline/engine/tests/test_join.py
+++ b/nipype/pipeline/engine/tests/test_join.py
@@ -627,3 +627,35 @@ def test_name_prefix_join(tmpdir):
                               joinfield=['in1'])
     wf.connect(square, 'out', square_join, "in1")
     wf.run()
+
+
+def test_join_nestediters(tmpdir):
+    tmpdir.chdir()
+
+    def exponent(x, p):
+        return x ** p
+
+    wf = pe.Workflow('wf', base_dir=tmpdir.strpath)
+
+    xs = pe.Node(IdentityInterface(['x']),
+                 iterables=[('x', [1, 2])],
+                 name='xs')
+    ps = pe.Node(IdentityInterface(['p']),
+                 iterables=[('p', [3, 4])],
+                 name='ps')
+    exp = pe.Node(Function(function=exponent), name='exp')
+    exp_joinx = pe.JoinNode(Merge(1, ravel_inputs=True),
+                            name='exp_joinx',
+                            joinsource='xs',
+                            joinfield=['in1'])
+    exp_joinp = pe.JoinNode(Merge(1, ravel_inputs=True),
+                            name='exp_joinp',
+                            joinsource='ps',
+                            joinfield=['in1'])
+    wf.connect([
+        (xs, exp, [('x', 'x')]),
+        (ps, exp, [('p', 'p')]),
+        (exp, exp_joinx, [('out', 'in1')]),
+        (exp_joinx, exp_joinp, [('out', 'in1')])])
+
+    wf.run()