Repository: Parsl/parsl

Issue:
`repr` call after `AppException` instantiation raises `AttributeError`
The custom `str` method of `AppException` just reproduces the base class `str` method. The only thing the custom `repr` does is change the format, but I think we should consider sticking to the baseclass version for improved readability (it matches the built-in format); additionally, it introduces a bug because if `reason` is not set, the `repr` call will fail with an `AttributeError`.
```
In [28]: from parsl.app.errors import ParslError, AppException

In [29]: x = ParslError('baz')

In [30]: y = AppException('baz')

In [31]: x
Out[31]: parsl.app.errors.ParslError('baz')

In [32]: y
Out[32]: parsl.app.errors.AppException('baz')

In [33]: repr(x)
Out[33]: "ParslError('baz',)"

In [34]: repr(y)
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-34-2a48cbef3bc0> in <module>()
----> 1 repr(y)

~/ci/parsl/parsl/app/errors.py in __repr__(self)
     40
     41     def __repr__(self):
---> 42         return "{0} Reason:{1}".format(self.__class__, self.reason)
     43
     44     def __str__(self):

AttributeError: 'AppException' object has no attribute 'reason'

In [35]: y.reason = 'baz'

In [36]: repr(y)
Out[36]: "<class 'parsl.app.errors.AppException'> Reason:baz"
```
Exceptions raised on the IPP is wrapped by RemoteError
When running apps that raise an application, the exception is returned as is by Threads, however they 
are wrapped by IPP inside a RemoteError. This is confusing and adds inconsistencies in handling logic based on the executor being used. 

Here's a wrapped exception traceback 
```
Traceback (most recent call last):
  File "/home/yadu/src/parsl/parsl/dataflow/dflow.py", line 174, in handle_update
    future.result()
  File "/usr/lib/python3.5/concurrent/futures/_base.py", line 398, in result
    return self.__get_result()
  File "/usr/lib/python3.5/concurrent/futures/_base.py", line 357, in __get_result
    raise self._exception
  File "/usr/local/lib/python3.5/dist-packages/ipyparallel/client/asyncresult.py", line 224, in _resolve_result
    raise r
ipyparallel.error.RemoteError: TypeError(a float is required)
```


Discussion:

I think this should be very high-priority. Fixing this would speed up debugging on other items.
This is very hard. IPP converts the exceptions to a string, which makes it hard to reconstruct the original exception. Another confounding issue is that exceptions even if serialized might not have be deserializable on the local side due to missing (non-imported) libraries

Resolving patch:
diff --git a/parsl/app/errors.py b/parsl/app/errors.py
--- a/parsl/app/errors.py
+++ b/parsl/app/errors.py
@@ -1,4 +1,8 @@
 """Exceptions raise by Apps."""
+from functools import wraps
+
+import dill
+from six import reraise
 
 
 class ParslError(Exception):
@@ -25,12 +29,6 @@ class AppException(ParslError):
     What this exception contains depends entirely on context
     """
 
-    def __repr__(self):
-        return "{0} Reason:{1}".format(self.__class__, self.reason)
-
-    def __str__(self):
-        return self.__repr__()
-
 
 class AppBadFormatting(ParslError):
     """An error raised during formatting of a bash function.
@@ -159,3 +157,25 @@ def __repr__(self):
 
     def __str__(self):
         return "Reason:{0} Missing:{1}".format(self.reason, self.outputs)
+
+
+class RemoteException(ParslError):
+    def __init__(self, e_type, e_value, traceback):
+        self.e_type = dill.dumps(e_type)
+        self.e_value = dill.dumps(e_value)
+        self.traceback = dill.dumps(traceback)
+
+    def reraise(self):
+        reraise(dill.loads(self.e_type), dill.loads(self.e_value), dill.loads(self.traceback))
+
+
+def wrap_error(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        import sys
+        from parsl.app.errors import RemoteException
+        try:
+            return func(*args, **kwargs)
+        except Exception as e:
+            return RemoteException(*sys.exc_info())
+    return wrapper
diff --git a/parsl/app/python_app.py b/parsl/app/python_app.py
--- a/parsl/app/python_app.py
+++ b/parsl/app/python_app.py
@@ -1,7 +1,12 @@
 import logging
 
+import tblib.pickling_support
+tblib.pickling_support.install()
+
 from parsl.app.futures import DataFuture
 from parsl.app.app import AppBase
+from parsl.app.errors import wrap_error
+
 
 logger = logging.getLogger(__name__)
 
@@ -15,7 +20,7 @@ def __init__(self, func, executor, walltime=60, cache=False,
 
         This bit is the same for both bash & python apps.
         """
-        super().__init__(func, executor, walltime=walltime, sites=sites, exec_type="python")
+        super().__init__(wrap_error(func), executor, walltime=walltime, sites=sites, exec_type="python")
         self.fn_hash = fn_hash
         self.cache = cache
 
diff --git a/parsl/dataflow/futures.py b/parsl/dataflow/futures.py
--- a/parsl/dataflow/futures.py
+++ b/parsl/dataflow/futures.py
@@ -10,6 +10,8 @@
 import logging
 import threading
 
+from parsl.app.errors import RemoteException
+
 logger = logging.getLogger(__name__)
 
 # Possible future states (for internal use by the futures package).
@@ -112,20 +114,23 @@ def result(self, timeout=None):
         """
         try:
             if self.parent:
-                return self.parent.result(timeout=timeout)
+                res = self.parent.result(timeout=timeout)
             else:
-                return super().result(timeout=timeout)
+                res = super().result(timeout=timeout)
+            if isinstance(res, RemoteException):
+                res.reraise()
+            return res
 
         except Exception as e:
-            # logger.debug("[TODO] {} Retries:{}".format(self.parent,
-            #                                           self.parent.retries_left))
-
             if self.parent.retries_left > 0:
                 self._parent_update_event.wait()
                 self._parent_update_event.clear()
                 return self.result(timeout=timeout)
             else:
-                raise
+                if isinstance(e, RemoteException):
+                    e.reraise()
+                else:
+                    raise
 
     def cancel(self):
         if self.parent:


Test patch:
diff --git a/parsl/tests/test_ipp/test_python_apps.py b/parsl/tests/test_ipp/test_python_apps.py
--- a/parsl/tests/test_ipp/test_python_apps.py
+++ b/parsl/tests/test_ipp/test_python_apps.py
@@ -1,12 +1,13 @@
 """Testing bash apps
 """
-import parsl
-from parsl import *
-
 import os
 import time
 import argparse
-from nose.tools import nottest
+
+from nose.tools import nottest, assert_raises
+
+import parsl
+from parsl import *
 
 # parsl.set_stream_logger()
 workers = IPyParallelExecutor()
@@ -32,8 +33,13 @@ def import_echo(x, string, stdout=None):
     return x * 5
 
 
-def test_simple(n=10):
+@App('python', dfk)
+def custom_exception():
+    from globus_sdk import GlobusError
+    raise GlobusError('foobar')
+
 
+def test_simple(n=10):
     start = time.time()
     x = double(n)
     print("Result : ", x.result())
@@ -46,7 +52,6 @@ def test_simple(n=10):
 
 
 def test_imports(n=10):
-
     start = time.time()
     x = import_echo(n, "hello world")
     print("Result : ", x.result())
@@ -59,7 +64,6 @@ def test_imports(n=10):
 
 
 def test_parallel_for(n=10):
-
     d = {}
     start = time.time()
     for i in range(0, n):
@@ -90,6 +94,21 @@ def test_stdout():
     print("[TEST STATUS] test_stdout [SUCCESS]")
 
 
+def test_custom_exception():
+    from globus_sdk import GlobusError
+
+    def wrapper():
+        x = custom_exception()
+        return x.result()
+    assert_raises(GlobusError, wrapper)
+
+
+@nottest
+def demonstrate_custom_exception():
+    x = custom_exception()
+    print(x.result())
+
+
 if __name__ == '__main__':
 
     parser = argparse.ArgumentParser()
@@ -102,10 +121,9 @@ def test_stdout():
     if args.debug:
         parsl.set_stream_logger()
 
+    demonstrate_custom_exception()
     x = test_simple(int(args.count))
     x = test_imports()
     x = test_parallel_for()
-    # x = test_parallel_for(int(args.count))
-
-    # x = test_stdout()
-    # raise_error(0)
+    x = test_parallel_for(int(args.count))
+    x = test_stdout()
diff --git a/parsl/tests/test_threads/test_python_apps.py b/parsl/tests/test_threads/test_python_apps.py
--- a/parsl/tests/test_threads/test_python_apps.py
+++ b/parsl/tests/test_threads/test_python_apps.py
@@ -7,6 +7,8 @@
 import time
 import argparse
 
+from nose.tools import assert_raises
+
 # parsl.set_stream_logger()
 workers = ThreadPoolExecutor(max_workers=4)
 dfk = DataFlowKernel(executors=[workers])
@@ -23,6 +25,12 @@ def echo(x, string, stdout=None):
     return x * 5
 
 
+@App('python', dfk)
+def custom_exception():
+    from globus_sdk import GlobusError
+    raise GlobusError('foobar')
+
+
 def test_parallel_for(n=2):
 
     d = {}
@@ -55,6 +63,15 @@ def test_stdout():
     print("[TEST STATUS] test_stdout [SUCCESS]")
 
 
+def test_custom_exception():
+    from globus_sdk import GlobusError
+
+    def wrapper():
+        x = custom_exception()
+        return x.result()
+    assert_raises(GlobusError, wrapper)
+
+
 if __name__ == '__main__':
 
     parser = argparse.ArgumentParser()