Repository: Parsl/parsl

Issue:
Stage in files passed as args and kwargs
In `DataFlowKernel._add_input_deps`, we only stage stuff in if it is in `inputs`, while we resolve any futures that are passed as args or kwargs. We should iterate over args and kwargs looking for files to stage in as well as `inputs`. In my opinion the only advantage of `inputs` is that it allows you to pass an arbitrary number of inputs at call time. Unless a user needs to fulfill that specific (and somewhat unusual) need, they should use args and kwargs instead, because they are more readable and easily documented.


Discussion:

Resolving patch:
diff --git a/parsl/dataflow/dflow.py b/parsl/dataflow/dflow.py
--- a/parsl/dataflow/dflow.py
+++ b/parsl/dataflow/dflow.py
@@ -495,6 +495,17 @@ def _add_input_deps(self, executor, args, kwargs):
             if isinstance(f, File) and f.is_remote():
                 inputs[idx] = f.stage_in(executor)
 
+        for kwarg, potential_f in kwargs.items():
+            if isinstance(potential_f, File) and potential_f.is_remote():
+                kwargs[kwarg] = potential_f.stage_in(executor)
+
+        newargs = list(args)
+        for idx, f in enumerate(newargs):
+            if isinstance(f, File) and f.is_remote():
+                newargs[idx] = f.stage_in(executor)
+
+        return tuple(newargs), kwargs
+
     def _gather_all_deps(self, args, kwargs):
         """Count the number of unresolved futures on which a task depends.
 
@@ -626,6 +637,9 @@ def submit(self, func, *args, executors='all', fn_hash=None, cache=False, **kwar
             choices = executors
         executor = random.choice(choices)
 
+        # Transform remote input files to data futures
+        args, kwargs = self._add_input_deps(executor, args, kwargs)
+
         task_def = {'depends': None,
                     'executor': executor,
                     'func': func,
@@ -652,9 +666,6 @@ def submit(self, func, *args, executors='all', fn_hash=None, cache=False, **kwar
         else:
             self.tasks[task_id] = task_def
 
-        # Transform remote input files to data futures
-        self._add_input_deps(executor, args, kwargs)
-
         # Get the dep count and a list of dependencies for the task
         dep_cnt, depends = self._gather_all_deps(args, kwargs)
         self.tasks[task_id]['depends'] = depends


Test patch:
diff --git a/parsl/tests/test_staging/test_implicit_staging_https.py b/parsl/tests/test_staging/test_implicit_staging_https.py
--- a/parsl/tests/test_staging/test_implicit_staging_https.py
+++ b/parsl/tests/test_staging/test_implicit_staging_https.py
@@ -35,6 +35,50 @@ def test_implicit_staging_https():
     f.result()
 
 
+@App('python')
+def sort_strings_kw(x=None, outputs=[]):
+    with open(x.filepath, 'r') as u:
+        strs = u.readlines()
+        strs.sort()
+        with open(outputs[0].filepath, 'w') as s:
+            for e in strs:
+                s.write(e)
+
+
+@pytest.mark.local
+def test_implicit_staging_https_kwargs():
+
+    unsorted_file = File('https://testbed.petrel.host/test/public/unsorted.txt')
+
+    # Create a local file for output data
+    sorted_file = File('sorted.txt')
+
+    f = sort_strings_kw(x=unsorted_file, outputs=[sorted_file])
+    f.result()
+
+
+@App('python')
+def sort_strings_arg(x, outputs=[]):
+    with open(x.filepath, 'r') as u:
+        strs = u.readlines()
+        strs.sort()
+        with open(outputs[0].filepath, 'w') as s:
+            for e in strs:
+                s.write(e)
+
+
+@pytest.mark.local
+def test_implicit_staging_https_args():
+
+    unsorted_file = File('https://testbed.petrel.host/test/public/unsorted.txt')
+
+    # Create a local file for output data
+    sorted_file = File('sorted.txt')
+
+    f = sort_strings_arg(unsorted_file, outputs=[sorted_file])
+    f.result()
+
+
 if __name__ == "__main__":
 
     import argparse