Repository: nipype/pydra

Issue:
More issues with numpy arrays
Still having errors with comparing arrays, now in `copyfile_workflow`.    It's somewhat confusing that `value` can be the actual output or the path to output file.  

Error:
```python
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-2-418998bcfc4f> in <module>
     27 
     28 with pydra.Submitter(plugin="cf") as sub:
---> 29     sub(wf2)
     30 
     31 wf2.result(return_inputs=True)

~/Desktop/nlo/pydra/pydra/engine/submitter.py in __call__(self, runnable, cache_locations, rerun)
     57             self.loop.run_until_complete(self.submit_workflow(runnable, rerun=rerun))
     58         else:
---> 59             self.loop.run_until_complete(self.submit(runnable, wait=True, rerun=rerun))
     60         if is_workflow(runnable):
     61             # resetting all connections with LazyFields

~/anaconda3/envs/pydra/lib/python3.7/asyncio/base_events.py in run_until_complete(self, future)
    581             raise RuntimeError('Event loop stopped before Future completed.')
    582 
--> 583         return future.result()
    584 
    585     def stop(self):

~/Desktop/nlo/pydra/pydra/engine/submitter.py in submit(self, runnable, wait, rerun)
    122             # run coroutines concurrently and wait for execution
    123             # wait until all states complete or error
--> 124             await asyncio.gather(*futures)
    125             return
    126         # pass along futures to be awaited independently

~/Desktop/nlo/pydra/pydra/engine/submitter.py in submit_workflow(self, workflow, rerun)
     69             await self.worker.run_el(workflow, rerun=rerun)
     70         else:
---> 71             await workflow._run(self, rerun=rerun)
     72 
     73     async def submit(self, runnable, wait=False, rerun=False):

~/Desktop/nlo/pydra/pydra/engine/core.py in _run(self, submitter, rerun, **kwargs)
    907                 self.hooks.post_run_task(self, result)
    908                 self.audit.finalize_audit(result=result)
--> 909                 save(odir, result=result, task=self)
    910                 os.chdir(cwd)
    911         self.hooks.post_run(self, result)

~/Desktop/nlo/pydra/pydra/engine/helpers.py in save(task_path, result, task)
    115         if Path(task_path).name.startswith("Workflow"):
    116             # copy files to the workflow directory
--> 117             result = copyfile_workflow(wf_path=task_path, result=result)
    118         with (task_path / "_result.pklz").open("wb") as fp:
    119             cp.dump(result, fp)

~/Desktop/nlo/pydra/pydra/engine/helpers.py in copyfile_workflow(wf_path, result)
    128         value = getattr(result.output, field.name)
    129         new_value = _copyfile_single_value(wf_path=wf_path, value=value)
--> 130         if new_value != value:
    131             setattr(result.output, field.name, new_value)
    132     return result

ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()
```



To replicate:
```python
import pydra
import typing as ty

@pydra.mark.task
@pydra.mark.annotate({"return": {'scores': ty.Any}})
def calc_metric(score):
    from sklearn import datasets
    from sklearn.model_selection import train_test_split
    from sklearn.svm import SVC
    from sklearn.model_selection import cross_val_score
    X, y = datasets.load_digits(n_class=10, return_X_y=True)
    X_tr, X_tt, y_tr, y_tt = train_test_split(X, y)
    
    clf = SVC()
    clf.fit(X_tr, y_tr)    
    cv_scores = cross_val_score(clf, X_tt, y_tt, scoring=f'{score}_macro')
    return(cv_scores)

# workflow
wf2 = pydra.Workflow(name="svm", 
                     input_spec=['score'], score=['precision', 'recall'])

wf2.split('score')
wf2.add(calc_metric(name='calc_metric', score=wf2.lzin.score))  
wf2.set_output([("scores", wf2.calc_metric.lzout.scores)])

with pydra.Submitter(plugin="cf") as sub:
    sub(wf2)
    
wf2.result(return_inputs=True)
```

Discussion:

Patch:
diff --git a/pydra/engine/core.py b/pydra/engine/core.py
index b04a50ea..90bd3d34 100644
--- a/pydra/engine/core.py
+++ b/pydra/engine/core.py
@@ -34,7 +34,6 @@ from .helpers import (
     ensure_list,
     record_error,
     hash_function,
-    output_from_inputfields,
 )
 from .helpers_file import copyfile_input, template_update
 from .graph import DiGraph
@@ -312,7 +311,7 @@ class TaskBase:
     @property
     def output_names(self):
         """Get the names of the outputs generated by the task."""
-        return output_from_inputfields(self.output_spec, self.inputs, names_only=True)
+        return [f.name for f in attr.fields(make_klass(self.output_spec))]
 
     @property
     def can_resume(self):
@@ -411,7 +410,7 @@ class TaskBase:
             try:
                 self.audit.monitor()
                 self._run_task()
-                result.output = self._collect_outputs()
+                result.output = self._collect_outputs(output_dir=odir)
             except Exception as e:
                 record_error(self.output_dir, e)
                 result.errored = True
@@ -429,12 +428,11 @@ class TaskBase:
         self.hooks.post_run(self, result)
         return result
 
-    def _collect_outputs(self):
+    def _collect_outputs(self, output_dir):
         run_output = self.output_
-        self.output_spec = output_from_inputfields(self.output_spec, self.inputs)
         output_klass = make_klass(self.output_spec)
         output = output_klass(**{f.name: None for f in attr.fields(output_klass)})
-        other_output = output.collect_additional_outputs(self.inputs, self.output_dir)
+        other_output = output.collect_additional_outputs(self.inputs, output_dir)
         return attr.evolve(output, **run_output, **other_output)
 
     def split(self, splitter, overwrite=False, **kwargs):
@@ -996,7 +994,22 @@ class Workflow(TaskBase):
             )
 
         self._connections += new_connections
-        fields = [(name, ty.Any) for name, _ in self._connections]
+        fields = []
+        for con in self._connections:
+            wf_out_nm, lf = con
+            task_nm, task_out_nm = lf.name, lf.field
+            if task_out_nm == "all_":
+                help_string = f"all outputs from {task_nm}"
+                fields.append((wf_out_nm, dict, {"help_string": help_string}))
+            else:
+                # getting information about the output field from the task output_spec
+                # providing proper type and some help string
+                task_output_spec = getattr(self, task_nm).output_spec
+                out_fld = attr.fields_dict(make_klass(task_output_spec))[task_out_nm]
+                help_string = (
+                    f"{out_fld.metadata.get('help_string', '')} (from {task_nm})"
+                )
+                fields.append((wf_out_nm, out_fld.type, {"help_string": help_string}))
         self.output_spec = SpecInfo(name="Output", fields=fields, bases=(BaseSpec,))
         logger.info("Added %s to %s", self.output_spec, self)
 
diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py
index c1a30bf8..975ef8e5 100644
--- a/pydra/engine/helpers.py
+++ b/pydra/engine/helpers.py
@@ -155,8 +155,14 @@ def copyfile_workflow(wf_path, result):
     """ if file in the wf results, the file will be copied to the workflow directory"""
     for field in attr_fields(result.output):
         value = getattr(result.output, field.name)
-        new_value = _copyfile_single_value(wf_path=wf_path, value=value)
-        if new_value != value:
+        # if the field is a path or it can contain a path _copyfile_single_value is run
+        # to move all files and directories to the workflow directory
+        if field.type in [File, Directory, MultiOutputObj] or type(value) in [
+            list,
+            tuple,
+            dict,
+        ]:
+            new_value = _copyfile_single_value(wf_path=wf_path, value=value)
             setattr(result.output, field.name, new_value)
     return result
 
@@ -672,7 +678,7 @@ def hash_value(value, tp=None, metadata=None):
             return value
 
 
-def output_from_inputfields(output_spec, inputs, names_only=False):
+def output_from_inputfields(output_spec, input_spec):
     """
     Collect values from output from input fields.
     If names_only is False, the output_spec is updated,
@@ -682,30 +688,26 @@ def output_from_inputfields(output_spec, inputs, names_only=False):
     ----------
     output_spec :
         TODO
-    inputs :
+    input_spec :
         TODO
 
     """
     current_output_spec_names = [f.name for f in attr.fields(make_klass(output_spec))]
     new_fields = []
-    for fld in attr_fields(inputs):
+    for fld in attr.fields(make_klass(input_spec)):
         if "output_file_template" in fld.metadata:
-            value = getattr(inputs, fld.name)
             if "output_field_name" in fld.metadata:
                 field_name = fld.metadata["output_field_name"]
             else:
                 field_name = fld.name
             # not adding if the field already in teh output_spec
             if field_name not in current_output_spec_names:
+                # TODO: should probably remove some of the keys
                 new_fields.append(
-                    (field_name, attr.ib(type=File, metadata={"value": value}))
+                    (field_name, attr.ib(type=File, metadata=fld.metadata))
                 )
-    if names_only:
-        new_names = [el[0] for el in new_fields]
-        return current_output_spec_names + new_names
-    else:
-        output_spec.fields += new_fields
-        return output_spec
+    output_spec.fields += new_fields
+    return output_spec
 
 
 def get_available_cpus():
diff --git a/pydra/engine/task.py b/pydra/engine/task.py
index 3a5f0d7d..f86c9172 100644
--- a/pydra/engine/task.py
+++ b/pydra/engine/task.py
@@ -57,7 +57,13 @@ from .specs import (
     SingularitySpec,
     attr_fields,
 )
-from .helpers import ensure_list, execute, position_adjustment, argstr_formatting
+from .helpers import (
+    ensure_list,
+    execute,
+    position_adjustment,
+    argstr_formatting,
+    output_from_inputfields,
+)
 from .helpers_file import template_update, is_local_file
 
 
@@ -276,6 +282,7 @@ class ShellCommandTask(TaskBase):
             output_spec = SpecInfo(name="Output", fields=[], bases=(ShellOutSpec,))
 
         self.output_spec = output_spec
+        self.output_spec = output_from_inputfields(self.output_spec, self.input_spec)
 
         super().__init__(
             name=name,


Test patch:
diff --git a/pydra/engine/tests/test_numpy_examples.py b/pydra/engine/tests/test_numpy_examples.py
index 55884ac2..35b89723 100644
--- a/pydra/engine/tests/test_numpy_examples.py
+++ b/pydra/engine/tests/test_numpy_examples.py
@@ -19,6 +19,22 @@ def arrayout(val):
 
 def test_multiout(plugin):
     """ testing a simple function that returns a numpy array"""
+    wf = Workflow("wf", input_spec=["val"], val=2)
+    wf.add(arrayout(name="mo", val=wf.lzin.val))
+
+    wf.set_output([("array", wf.mo.lzout.b)])
+
+    with Submitter(plugin=plugin, n_procs=2) as sub:
+        sub(runnable=wf)
+
+    results = wf.result(return_inputs=True)
+
+    assert results[0] == {"wf.val": 2}
+    assert np.array_equal(results[1].output.array, np.array([2, 2]))
+
+
+def test_multiout_st(plugin):
+    """ testing a simple function that returns a numpy array, adding splitter"""
     wf = Workflow("wf", input_spec=["val"], val=[0, 1, 2])
     wf.add(arrayout(name="mo", val=wf.lzin.val))
     wf.mo.split("val").combine("val")
diff --git a/pydra/engine/tests/utils.py b/pydra/engine/tests/utils.py
index ad087886..b2fbdab7 100644
--- a/pydra/engine/tests/utils.py
+++ b/pydra/engine/tests/utils.py
@@ -164,7 +164,7 @@ def fun_dict(d):
 
 
 @mark.task
-def fun_write_file(filename: ty.Union[str, File, Path], text="hello"):
+def fun_write_file(filename: ty.Union[str, File, Path], text="hello") -> File:
     with open(filename, "w") as f:
         f.write(text)
     return Path(filename).absolute()