Repository: Parsl/parsl

Issue:
Passing Files breaks over IPP
The new File class contains a dictionary that maps DataFutures for each site to which it is being staged and contains a reference to the DataManager. Neither of these are pickle-able.

So if we do something like this :+1: 
```
data = File("foo.txt")
fu = remote_app(inputs=[data])
fu.result() # <--- We'll get an error from here
```
Here's the relevant piece from the exception traceback :
```
File "/usr/local/lib/python3.5/dist-packages/ipyparallel/serialize/serialize.py", line 112, in serialize_object
   buffers.insert(0, pickle.dumps(cobj, PICKLE_PROTOCOL))
TypeError: can't pickle _thread.lock objects
```

I believe that the File object is the best place to hold the Future information about itself, and that would give us the opportunity to do smarter file staging in the future. So I propose that we fix this with a custom pickler for the File class.

This is blocker for 0.5.0.


Discussion:

Resolving patch:
diff --git a/parsl/data_provider/files.py b/parsl/data_provider/files.py
--- a/parsl/data_provider/files.py
+++ b/parsl/data_provider/files.py
@@ -95,6 +95,34 @@ def set_data_future(self, df, site=None):
     def get_data_future(self, site):
         return self.data_future.get(site)
 
+    def __getstate__(self):
+        """ Overriding the default pickling method.
+
+        The File object get's pickled and transmitted to remote sites during app
+        execution. This enables pickling while retaining the lockable resources
+        to the DFK/Client side.
+        """
+
+        state = self.__dict__.copy()
+
+        # We have already made a copy of the future objects, they are now no longer
+        # reliable as means to wait for the staging events
+        for site in state["data_future"]:
+            # This is assumed to be safe, since the data_future represents staging to a specific site
+            # and a site will only have one filepath.
+            state["data_future"][site] = state["data_future"][site].filepath
+
+        state["dman"] = None
+
+        return state
+
+    def __setstate__(self, state):
+        """ Overloading the default pickle method to reconstruct a File from serialized form
+
+        This might require knowledge of whethere a DataManager is already present in the context.
+        """
+        self.__dict__.update(state)
+
 
 if __name__ == '__main__':
 


Test patch:
diff --git a/parsl/tests/test_data/test_file_ipp.py b/parsl/tests/test_data/test_file_ipp.py
new file mode 100644
--- /dev/null
+++ b/parsl/tests/test_data/test_file_ipp.py
@@ -0,0 +1,84 @@
+from parsl import *
+import os
+
+from parsl.data_provider.files import File
+
+# parsl.set_stream_logger()
+from parsl.configs.local import localIPP as config
+dfk = DataFlowKernel(config=config)
+
+
+@App('bash', dfk)
+def cat(inputs=[], outputs=[], stdout=None, stderr=None):
+    infiles = ' '.join([i.filepath for i in inputs])
+    return """echo %s
+    cat %s &> {outputs[0]}
+    """ % (infiles, infiles)
+
+
+def test_files():
+
+    fs = [File('data/' + f) for f in os.listdir('data')]
+    x = cat(inputs=fs, outputs=['cat_out.txt'],
+            stdout='f_app.out', stderr='f_app.err')
+    d_x = x.outputs[0]
+    print(x.result())
+    print(d_x, type(d_x))
+
+
+@App('bash', dfk)
+def increment(inputs=[], outputs=[], stdout=None, stderr=None):
+    # Place double braces to avoid python complaining about missing keys for {item = $1}
+    return """
+    x=$(cat {inputs[0]})
+    echo $(($x+1)) > {outputs[0]}
+    """
+
+
+def test_regression_200():
+    """Regression test for #200. Pickleablility of Files"""
+
+    with open("test.txt", 'w') as f:
+        f.write("Hello World")
+
+    fu = cat(inputs=[File("test.txt")],
+             outputs=[File("test_output.txt")])
+    fu.result()
+
+    with open(fu.outputs[0].result(), 'r') as f:
+        data = f.readlines()
+        assert "Hello World" in data, "Missed data"
+
+
+def test_increment(depth=5):
+    """Test simple pipeline A->B...->N
+    """
+    # Create the first file
+    open("test0.txt", 'w').write('0\n')
+
+    # Create the first entry in the dictionary holding the futures
+    prev = File("test0.txt")
+    futs = {}
+    for i in range(1, depth):
+        print("Launching {0} with {1}".format(i, prev))
+        fu = increment(inputs=[prev],  # Depend on the future from previous call
+                       # Name the file to be created here
+                       outputs=[File("test{0}.txt".format(i))],
+                       stdout="incr{0}.out".format(i),
+                       stderr="incr{0}.err".format(i))
+        [prev] = fu.outputs
+        futs[i] = prev
+        print(prev.filepath)
+
+    for key in futs:
+        if key > 0:
+            fu = futs[key]
+            data = open(fu.result().filepath, 'r').read().strip()
+            assert data == str(
+                key), "[TEST] incr failed for key:{0} got:{1}".format(key, data)
+
+
+if __name__ == '__main__':
+
+    test_files()
+    test_increment()
diff --git a/parsl/tests/test_sites/configs/cc_in2p3.py b/parsl/tests/test_sites/configs/cc_in2p3.py
--- a/parsl/tests/test_sites/configs/cc_in2p3.py
+++ b/parsl/tests/test_sites/configs/cc_in2p3.py
@@ -14,6 +14,44 @@
 ==================
 """
 
+singleNode = {
+    "sites": [
+        {"site": "Local_CC-IN2P3",
+         "auth": {
+             "channel": "ssh-il",
+             "username": os.environ["CC_IN2P3_USERNAME"],
+             "scriptDir": "{}/parsl_scripts".format(os.environ["CC_IN2P3_HOME"])
+         },
+         "execution": {
+             "executor": "ipp",
+             "provider": "gridEngine",
+             "block": {  # Definition of a block
+                 "nodes": 1,            # of nodes in that block
+                 "taskBlocks": 1,       # total tasks in a block
+                 "initBlocks": 1,
+                 "maxBlocks": 1,
+                 "options": {
+                     "partition": "debug",
+                     "overrides": """export PATH=/pbs/throng/lsst/software/anaconda/anaconda3-5.0.1/bin:$PATH;
+source activate parsl_env_3.5"""
+                 }
+             }
+         }
+         }
+    ],
+    "globals": {"lazyErrors": True}
+}
+
+"""
+================== Block
+| ++++++++++++++ | Node
+| |            | |
+| |    Task    | |             . . .
+| |            | |
+| ++++++++++++++ |
+==================
+"""
+
 singleNodeLocal = {
     "sites": [
         {"site": "Local_CC-IN2P3",
diff --git a/parsl/tests/test_stress/test_python_simple.py b/parsl/tests/test_stress/test_python_simple.py
--- a/parsl/tests/test_stress/test_python_simple.py
+++ b/parsl/tests/test_stress/test_python_simple.py
@@ -28,8 +28,13 @@ def increment(x):
         parsl.set_stream_logger()
 
     start = time.time()
-    x = {}
+    x = []
     for i in range(int(args.count)):
-        x[i] = increment(i)
+        fu = increment(i)
+        x.append(fu)
     end = time.time()
     print("Launched {0} tasks in {1} s".format(args.count, end - start))
+
+    [fu.result() for fu in x]
+    end = time.time()
+    print("Completed {0} tasks in {1} s".format(args.count, end - start))