Repository: nipy/nipype

Issue:
PR #2479 has broken my package
### Summary

PR #2479 has broken my package (https://pypi.org/project/arcana/) 

I am not quite sure what the rationale behind the changes are so it is difficult to know how to debug or whether there is something I can change in my package.

### Actual behavior

Workflow exits with error

```
 File "/Users/tclose/git/ni/arcana/test/mwe/nipype_pr2479/test.py", line 71, in <module>
    study.data('out')
  File "/Users/tclose/git/ni/arcana/arcana/study/base.py", line 325, in data
    visit_ids=visit_ids)
  File "/Users/tclose/git/ni/arcana/arcana/runner/base.py", line 37, in run
    return workflow.run(plugin=self._plugin)
  File "/Users/tclose/git/ni/nipype/nipype/pipeline/engine/workflows.py", line 595, in run
    runner.run(execgraph, updatehash=updatehash, config=self.config)
  File "/Users/tclose/git/ni/nipype/nipype/pipeline/plugins/linear.py", line 44, in run
    node.run(updatehash=updatehash)
  File "/Users/tclose/git/ni/nipype/nipype/pipeline/engine/nodes.py", line 480, in run
    result = self._run_interface(execute=True)
  File "/Users/tclose/git/ni/nipype/nipype/pipeline/engine/nodes.py", line 564, in _run_interface
    return self._run_command(execute)
  File "/Users/tclose/git/ni/arcana/arcana/node.py", line 59, in _run_command
    result = self.nipype_cls._run_command(self, *args, **kwargs)
  File "/Users/tclose/git/ni/nipype/nipype/pipeline/engine/nodes.py", line 888, in _run_command
    self._collate_join_field_inputs()
  File "/Users/tclose/git/ni/nipype/nipype/pipeline/engine/nodes.py", line 898, in _collate_join_field_inputs
    val = self._collate_input_value(field)
  File "/Users/tclose/git/ni/nipype/nipype/pipeline/engine/nodes.py", line 928, in _collate_input_value
    for idx in range(self._next_slot_index)
  File "/Users/tclose/git/ni/nipype/nipype/pipeline/engine/nodes.py", line 947, in _slot_value
    field, index, e))
AttributeError: The join node pipeline1.pipeline1_subject_session_outputs does not have a slot field subject_session_pairsJ1 to hold the subject_session_pairs value at index 0: 'DynamicTraitedSpec' object has no attribute 'subject_session_pairsJ1'
```

### Expected behavior

The workflow runs without error

### How to replicate the behavior

See script below

### Script/Workflow details

I have tried to come up with a MWE that doesn't use my package but it was proving difficult. However, you can now install my package with pip

`pip install arcana`

and run the following

```
import os.path
import shutil
from nipype import config
config.enable_debug_mode()
import nipype  # @IgnorePep8
from nipype.interfaces.utility import IdentityInterface  # @IgnorePep8
from arcana.dataset import DatasetMatch, DatasetSpec  # @IgnorePep8
from arcana.data_format import text_format  # @IgnorePep8
from arcana.study.base import Study, StudyMetaClass  # @IgnorePep8
from arcana.archive.local import LocalArchive  # @IgnorePep8
from arcana.runner import LinearRunner  # @IgnorePep8


BASE_ARCHIVE_DIR = os.path.join(os.path.dirname(__file__), 'archives')
BASE_WORK_DIR = os.path.join(os.path.dirname(__file__), 'work')

print(nipype.get_info())
print(nipype.__version__)


class TestStudy(Study):

    __metaclass__ = StudyMetaClass

    add_data_specs = [
        DatasetSpec('in', text_format),
        DatasetSpec('out', text_format, 'pipeline')]

    def pipeline(self, **kwargs):
        pipeline = self.create_pipeline(
            name='pipeline1',
            inputs=[DatasetSpec('in', text_format)],
            outputs=[DatasetSpec('out', text_format)],
            desc="A dummy pipeline used to test 'run_pipeline' method",
            version=1,
            citations=[],
            **kwargs)
        ident = pipeline.create_node(IdentityInterface(['a']),
                                     name="ident")
        # Connect inputs
        pipeline.connect_input('in', ident, 'a')
        # Connect outputs
        pipeline.connect_output('out', ident, 'a')
        return pipeline


# Create archives
shutil.rmtree(BASE_ARCHIVE_DIR, ignore_errors=True)
shutil.rmtree(BASE_WORK_DIR, ignore_errors=True)
os.makedirs(BASE_ARCHIVE_DIR)

for sess in (['ARCHIVE1', 'SUBJECT', 'VISIT'],
             ['ARCHIVE2', 'SUBJECT1', 'VISIT1'],
             ['ARCHIVE2', 'SUBJECT1', 'VISIT2'],
             ['ARCHIVE2', 'SUBJECT2', 'VISIT1'],
             ['ARCHIVE2', 'SUBJECT2', 'VISIT2']):
    sess_dir = os.path.join(*([BASE_ARCHIVE_DIR] + sess))
    os.makedirs(sess_dir)
    with open(os.path.join(sess_dir, 'in.txt'), 'w') as f:
        f.write('in')

archive1_path = os.path.join(BASE_ARCHIVE_DIR, 'ARCHIVE1')
archive2_path = os.path.join(BASE_ARCHIVE_DIR, 'ARCHIVE2')

work1_path = os.path.join(BASE_WORK_DIR, 'WORK1')
work2_path = os.path.join(BASE_WORK_DIR, 'WORK2')

# Attempt to run with archive with 2 subjects and 2 visits
study = TestStudy('two',
                   LocalArchive(archive2_path),
                   LinearRunner(work2_path),
                   inputs=[DatasetMatch('in', text_format, 'in')])

# Fails here
study2.data('out')

print("Ran study 2")

# 
study1 = TestStudy('one',
                   LocalArchive(archive1_path),
                   LinearRunner(work1_path),
                   inputs=[DatasetMatch('in', text_format, 'in')])

study1.data('out')

print("Ran study 1")
```

to reproduce the error

### Platform details:

{'nibabel_version': '2.2.1', 'sys_executable': '/usr/local/opt/python@2/bin/python2.7', 'networkx_version': '1.9', 'numpy_version': '1.14.3', 'sys_platform': 'darwin', 'sys_version': '2.7.15 (default, May  1 2018, 16:44:08) \n[GCC 4.2.1 Compatible Apple LLVM 9.1.0 (clang-902.0.39.1)]', 'commit_source': 'repository', 'commit_hash': '5a96ea54a', 'pkg_path': '/Users/tclose/git/ni/nipype/nipype', 'nipype_version': '1.0.4-dev+g5a96ea54a', 'traits_version': '4.6.0', 'scipy_version': '1.1.0'}
1.0.4-dev+g5a96ea54a

(problem arose in 1.0.1)

### Execution environment

My Homebrew python 2 environment outside container



Discussion:
I wonder if you can replace [the following](https://github.com/monashbiomedicalimaging/arcana/blob/8b2cc4266593123af10a53ddde0cd033332a1295/arcana/pipeline.py#L412-L416):

```Python
            subject_session_outputs = JoinNode(
                SubjectSessionReport(), joinfield='subject_session_pairs',
                joinsource=subjects,
                name=self.name + '_subject_session_outputs', wall_time=20,
                memory=4000)
```

with

```Python
subject_session_outputs = JoinNode(
    niu.Merge(1, ravel_inputs=True), joinfield=['in1'],
    joinsource='subjects',
    name=self.name + '_subject_session_outputs', wall_time=20,
    memory=4000)
```

(where `niu` is `nipype.interfaces.utility`).

From looking at your `SubjectSessionReport` interface, it looks like it serves the same purpose, and this is a familiar pattern. So if switching to this approach resolves the issue, then it may be something about the interface. If not, then there's more digging to do.
No luck

```
Traceback (most recent call last):
  File "/Applications/Eclipse.app/Contents/Eclipse/plugins/org.python.pydev.core_6.3.1.201802272029/pysrc/pydevd.py", line 1621, in <module>
    main()
  File "/Applications/Eclipse.app/Contents/Eclipse/plugins/org.python.pydev.core_6.3.1.201802272029/pysrc/pydevd.py", line 1615, in main
    globals = debugger.run(setup['file'], None, None, is_module)
  File "/Applications/Eclipse.app/Contents/Eclipse/plugins/org.python.pydev.core_6.3.1.201802272029/pysrc/pydevd.py", line 1022, in run
    pydev_imports.execfile(file, globals, locals)  # execute the script
  File "/Users/tclose/git/ni/arcana/test/mwe/nipype_pr2479/test.py", line 71, in <module>
    study.data('out')
  File "/Users/tclose/git/ni/arcana/arcana/study/base.py", line 325, in data
    visit_ids=visit_ids)
  File "/Users/tclose/git/ni/arcana/arcana/runner/base.py", line 37, in run
    return workflow.run(plugin=self._plugin)
  File "/Users/tclose/git/ni/nipype/nipype/pipeline/engine/workflows.py", line 595, in run
    runner.run(execgraph, updatehash=updatehash, config=self.config)
  File "/Users/tclose/git/ni/nipype/nipype/pipeline/plugins/linear.py", line 44, in run
    node.run(updatehash=updatehash)
  File "/Users/tclose/git/ni/nipype/nipype/pipeline/engine/nodes.py", line 480, in run
    result = self._run_interface(execute=True)
  File "/Users/tclose/git/ni/nipype/nipype/pipeline/engine/nodes.py", line 564, in _run_interface
    return self._run_command(execute)
  File "/Users/tclose/git/ni/arcana/arcana/node.py", line 59, in _run_command
    result = self.nipype_cls._run_command(self, *args, **kwargs)
  File "/Users/tclose/git/ni/nipype/nipype/pipeline/engine/nodes.py", line 888, in _run_command
    self._collate_join_field_inputs()
  File "/Users/tclose/git/ni/nipype/nipype/pipeline/engine/nodes.py", line 898, in _collate_join_field_inputs
    val = self._collate_input_value(field)
  File "/Users/tclose/git/ni/nipype/nipype/pipeline/engine/nodes.py", line 928, in _collate_input_value
    for idx in range(self._next_slot_index)
  File "/Users/tclose/git/ni/nipype/nipype/pipeline/engine/nodes.py", line 947, in _slot_value
    field, index, e))
AttributeError: The join node pipeline1.pipeline1_subject_session_outputs does not have a slot field in1J1 to hold the in1 value at index 0: 'DynamicTraitedSpec' object has no attribute 'in1J1'
```

See modified code at https://github.com/tclose/arcana/blob/nipype1_0_1_mwe/arcana/pipeline.py

Is it possible this is an issue with your custom `JoinNode`? Would you be able to switch to a nipype `JoinNode` for testing purposes?
Just tried that, and still no luck (the custom Node class
just adds handling for loading environment modules)



On 15 May 2018 at 11:48, Chris Markiewicz <notifications@github.com> wrote:

> Is it possible this is an issue with your custom JoinNode? Would you be
> able to switch to a nipype JoinNode for testing purposes?
>
> —
> You are receiving this because you authored the thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/nipy/nipype/issues/2578#issuecomment-389016500>, or mute
> the thread
> <https://github.com/notifications/unsubscribe-auth/ABAYlimdrnSTnZVbThzNWUFVTd3FcJziks5tyjPegaJpZM4T-van>
> .
>



-- 
*THOMAS G. CLOSE, PHD*
Senior Informatics Officer

*Monash Biomedical Imaging*
Monash University
Room 139, 770 Blackburn Rd
Clayton Campus, Clayton VIC 3800
Australia

T: +61 3 9902 9804
M: +61 491 141 390
E: tom.close@monash.edu
mbi.monash.edu.au


Resolving patch:
diff --git a/nipype/pipeline/engine/utils.py b/nipype/pipeline/engine/utils.py
--- a/nipype/pipeline/engine/utils.py
+++ b/nipype/pipeline/engine/utils.py
@@ -1054,12 +1054,14 @@ def make_field_func(*pair):
                 for src_id in list(old_edge_dict.keys()):
                     # Drop the original JoinNodes; only concerned with
                     # generated Nodes
-                    if hasattr(node, 'joinfield'):
+                    if hasattr(node, 'joinfield') and node.itername == src_id:
                         continue
                     # Patterns:
                     #   - src_id : Non-iterable node
-                    #   - src_id.[a-z]\d+ : IdentityInterface w/ iterables
-                    #   - src_id.[a-z]I.[a-z]\d+ : Non-IdentityInterface w/ iterables
+                    #   - src_id.[a-z]\d+ :
+                    #       IdentityInterface w/ iterables or nested JoinNode
+                    #   - src_id.[a-z]I.[a-z]\d+ :
+                    #       Non-IdentityInterface w/ iterables
                     #   - src_idJ\d+ : JoinNode(IdentityInterface)
                     if re.match(src_id + r'((\.[a-z](I\.[a-z])?|J)\d+)?$',
                                 node.itername):


Test patch:
diff --git a/nipype/pipeline/engine/tests/test_join.py b/nipype/pipeline/engine/tests/test_join.py
--- a/nipype/pipeline/engine/tests/test_join.py
+++ b/nipype/pipeline/engine/tests/test_join.py
@@ -627,3 +627,35 @@ def sq(x):
                               joinfield=['in1'])
     wf.connect(square, 'out', square_join, "in1")
     wf.run()
+
+
+def test_join_nestediters(tmpdir):
+    tmpdir.chdir()
+
+    def exponent(x, p):
+        return x ** p
+
+    wf = pe.Workflow('wf', base_dir=tmpdir.strpath)
+
+    xs = pe.Node(IdentityInterface(['x']),
+                 iterables=[('x', [1, 2])],
+                 name='xs')
+    ps = pe.Node(IdentityInterface(['p']),
+                 iterables=[('p', [3, 4])],
+                 name='ps')
+    exp = pe.Node(Function(function=exponent), name='exp')
+    exp_joinx = pe.JoinNode(Merge(1, ravel_inputs=True),
+                            name='exp_joinx',
+                            joinsource='xs',
+                            joinfield=['in1'])
+    exp_joinp = pe.JoinNode(Merge(1, ravel_inputs=True),
+                            name='exp_joinp',
+                            joinsource='ps',
+                            joinfield=['in1'])
+    wf.connect([
+        (xs, exp, [('x', 'x')]),
+        (ps, exp, [('p', 'p')]),
+        (exp, exp_joinx, [('out', 'in1')]),
+        (exp_joinx, exp_joinp, [('out', 'in1')])])
+
+    wf.run()