Repository: dask/distributed

Issue:
Exception chaining (raise Exception() from ...)
distributed 2.6.0 discards the traceback of all but the latest exception in a chain:

```python
def f():
    try:
        raise Exception("foo")
    except Exception as e:
        raise Exception("bar") from e

f()
```
Output:
```
---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
<ipython-input-2-707a8f24b7c1> in f()
      2     try:
----> 3         raise Exception("foo")
      4     except Exception as e:

Exception: foo

The above exception was the direct cause of the following exception:

Exception                                 Traceback (most recent call last)
<ipython-input-2-707a8f24b7c1> in <module>
      5         raise Exception("bar") from e
      6 
----> 7 f()

<ipython-input-2-707a8f24b7c1> in f()
      3         raise Exception("foo")
      4     except Exception as e:
----> 5         raise Exception("bar") from e
      6 
      7 f()

Exception: bar
```
```python
import distributed
client = distributed.Client()
client.submit(f).result()
```
Output:
```
---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
<ipython-input-3-5d89c629add7> in <module>
----> 1 client.submit(f).result()

distributed/client.py in result(self, timeout)
    220         if self.status == "error":
    221             typ, exc, tb = result
--> 222             raise exc.with_traceback(tb)
    223         elif self.status == "cancelled":
    224             raise result

<ipython-input-2-707a8f24b7c1> in f()
      3         raise Exception("foo")
      4     except Exception as e:
----> 5         raise Exception("bar") from e
      6 
      7 f()

Exception: bar
```

Without looking at the source code of dask distributed, I can guess what the issue is:
```python
import pickle
import sys

try:
    f()
except Exception:
    exc_type, exc, tb = sys.exc_info()

exc, tb = pickle.loads(pickle.dumps((exc, tb)))
raise exc.with_traceback(tb)
```
Output:
```
---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
<ipython-input-76-1fc21a18f772> in <module>
      8 
      9 exc, tb = pickle.loads(pickle.dumps((exc, tb)))
---> 10 raise exc.with_traceback(tb)

<ipython-input-76-1fc21a18f772> in <module>
      3 
      4 try:
----> 5     f()
      6 except Exception:
      7     exc_type, exc, tb = sys.exc_info()

<ipython-input-2-707a8f24b7c1> in f()
      3         raise Exception("foo")
      4     except Exception as e:
----> 5         raise Exception("bar") from e
      6 
      7 f()

Exception: bar
```
Solution:
```python
class ExceptionChainPickler:
    def __init__(self, exc: BaseException):
        self.exc = exc

    def __reduce__(self):             
        cur_exc = self.exc
        exc_chain = []
        while cur_exc:
            exc_chain.append((cur_exc, cur_exc.__traceback__))
            cur_exc = cur_exc.__cause__
        return self.expand, tuple(exc_chain)

    @staticmethod
    def expand(*exc_chain) -> BaseException:
        for (exc, tb), (cause, _) in zip(exc_chain, exc_chain[1:]):
            exc.__traceback__ = tb
            exc.__cause__ = cause
        exc_chain[-1][0].__traceback__ = exc_chain[-1][1]
        return exc_chain[0][0]


try:
    f()
except Exception as exc:
    ep = ExceptionChainPickler(exc)

exc = pickle.loads(pickle.dumps(ep))
raise exc
```
Output:
```
---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
<ipython-input-2-707a8f24b7c1> in f()
      2     try:
----> 3         raise Exception("foo")
      4     except Exception as e:

Exception: foo

The above exception was the direct cause of the following exception:

Exception                                 Traceback (most recent call last)
<ipython-input-81-f7743f5d6c40> in <module>
     26 
     27 exc = pickle.loads(pickle.dumps(ep))
---> 28 raise exc

<ipython-input-81-f7743f5d6c40> in <module>
     21 
     22 try:
---> 23     f()
     24 except Exception as exc:
     25     ep = ExceptionChainPickler(exc)

<ipython-input-2-707a8f24b7c1> in f()
      3         raise Exception("foo")
      4     except Exception as e:
----> 5         raise Exception("bar") from e
      6 
      7 f()

Exception: bar
```


Discussion:
Interestingly (but, IMHO, not a wise choice), ``concurrent.futures.ProcessPoolExecutor`` instead copies over *the string representation* of the full remote traceback plus the ProcessPoolExecutor's own traceback:
```python
from concurrent.futures import ProcessPoolExecutor
with ProcessPoolExecutor(1) as ex:
    ex.submit(f).result()
```
Output:
```
---------------------------------------------------------------------------
_RemoteTraceback                          Traceback (most recent call last)
_RemoteTraceback: 
"""
Traceback (most recent call last):
  File "<ipython-input-2-707a8f24b7c1>", line 3, in f
    raise Exception("foo")
Exception: foo

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Users/crusaderky/miniconda3/envs/commrisk/lib/python3.7/concurrent/futures/process.py", line 249, in _process_worker
    r = call_item.fn(*call_item.args, **call_item.kwargs)
  File "<ipython-input-2-707a8f24b7c1>", line 5, in f
    raise Exception("bar") from e
Exception: bar
"""

The above exception was the direct cause of the following exception:

Exception                                 Traceback (most recent call last)
<ipython-input-84-72eab10c5381> in <module>
      1 from concurrent.futures import ProcessPoolExecutor
      2 with ProcessPoolExecutor(1) as ex:
----> 3     ex.submit(f).result()

~/miniconda3/envs/commrisk/lib/python3.7/concurrent/futures/_base.py in result(self, timeout)
    443                 raise CancelledError()
    444             elif self._state == FINISHED:
--> 445                 return self.__get_result()
    446             else:
    447                 raise TimeoutError()

~/miniconda3/envs/commrisk/lib/python3.7/concurrent/futures/_base.py in __get_result(self)
    392     def __get_result(self):
    393         if self._exception:
--> 394             raise self._exception
    395         else:
    396             return self._result

Exception: bar
```
dask.multprocessing suffers from the same problem:
```python
import dask
dask.delayed(f)().compute(scheduler='processes')
```
Output:
```
---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
<ipython-input-93-47ef2aa976cb> in <module>
      1 import dask
----> 2 dask.delayed(f)().compute(scheduler='processes')

~/miniconda3/envs/commrisk/lib/python3.7/site-packages/dask/base.py in compute(self, **kwargs)
    163         dask.base.compute
    164         """
--> 165         (result,) = compute(self, traverse=False, **kwargs)
    166         return result
    167 

~/miniconda3/envs/commrisk/lib/python3.7/site-packages/dask/base.py in compute(*args, **kwargs)
    434     keys = [x.__dask_keys__() for x in collections]
    435     postcomputes = [x.__dask_postcompute__() for x in collections]
--> 436     results = schedule(dsk, keys, **kwargs)
    437     return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
    438 

~/miniconda3/envs/commrisk/lib/python3.7/site-packages/dask/multiprocessing.py in get(dsk, keys, num_workers, func_loads, func_dumps, optimize_graph, pool, **kwargs)
    202             pack_exception=pack_exception,
    203             raise_exception=reraise,
--> 204             **kwargs
    205         )
    206     finally:

~/miniconda3/envs/commrisk/lib/python3.7/site-packages/dask/local.py in get_async(apply_async, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, **kwargs)
    484                         _execute_task(task, data)  # Re-execute locally
    485                     else:
--> 486                         raise_exception(exc, tb)
    487                 res, worker_id = loads(res_info)
    488                 state["cache"][key] = res

~/miniconda3/envs/commrisk/lib/python3.7/site-packages/dask/local.py in reraise(exc, tb)
    313 def reraise(exc, tb=None):
    314     if exc.__traceback__ is not tb:
--> 315         raise exc.with_traceback(tb)
    316     raise exc
    317 

~/miniconda3/envs/commrisk/lib/python3.7/site-packages/dask/local.py in execute_task()
    220     try:
    221         task, data = loads(task_info)
--> 222         result = _execute_task(task, data)
    223         id = get_id()
    224         result = dumps((result, id))

~/miniconda3/envs/commrisk/lib/python3.7/site-packages/dask/core.py in _execute_task()
    117         func, args = arg[0], arg[1:]
    118         args2 = [_execute_task(a, cache) for a in args]
--> 119         return func(*args2)
    120     elif not ishashable(arg):
    121         return arg

<ipython-input-2-707a8f24b7c1> in f()
      3         raise Exception("foo")
      4     except Exception as e:
----> 5         raise Exception("bar") from e
      6 
      7 f()

Exception: bar
```
Should we proceed as follows? Not sure what the usual approach is.

1. in dask, add ExceptionChainPickler and fix dask.multiprocessing; merge PR
2. in distributed, bump minimum dask version to the next dask version.
    Add ``from dask.multiprocessing import ExceptionChainPickler`` and fix distributed
3. before closing the distributed release, double check that the dask release has already been published

Alternatively, should ExceptionChainPickler be temporarily backported from dask into distributed? 
@crusaderky thanks for the write up but I'm confused about why the user would want the last exception.  Isn't this, as you've noted, the try/except around running the submit or the execution of some coroutine ?  If so, does this help the user debug ?  Maybe as the user is building on top of dask it is very useful to know where things broken -- but they will know.  The stack trace will also show the failing compute/persist/etc call
@quasiben I don't think I understand your question. Are you arguing about the usefulness of exception chaining in Python in general, regardless of pickling and dask? Otherwise, I don't get what you mean.

> The stack trace will also show the failing compute/persist/etc call

In dask.compute(), yes - and it is easy to clean up. But also a separate problem. distributed.Client.submit has no such problem, as you can see there is no distributed-specific stack information in the traceback.
@crusaderky no, not arguing, asking.  I was trying to better understand when this error reporting is especially useful.  I'm not arguing against exception chaining.
If you work on a 10k lines of code worth of project, and you rely on third party libraries worth another 200k lines of code, you very much don't want to have your whole program bomb out with a mysterious
``KeyError: "John Doe"``, because it's something that will take a lot of work to reproduce, even with a stack trace - because the stack trace says nothing about the *context* of the error, that is all the other local and global variables. And if the exception was dumped out to a log in a production server in semi-unknown conditions, good luck to you. You know exactly which line of code triggered the exception, but you have no idea whatsoever about how that unexpected key ended up there.

If instead you could read in the log:
```
KeyError: "John Doe"

The above exception was the direct cause of the following exception:

UserDatabaseError: failed to load user credentials for user="John Doe"

The above exception was the direct cause of the following exception:

PageRenderError: failed to render /foo/bar/baz.html?user=John%20Doe
```
all while retaining the full stack trace, you'd save A LOT of time reproducing and debugging the issue!

Sure, without exception chaining, you'll see from the stack that your ``KeyError: "John Doe"`` was caused by, say, htmlrender.py. So you know it's triggered by rendering a page. Which of the 500 pages of your site? With which rendering parameters? God knows.

A lot of software that doesn't use exception chaining either emits the very first, nebulous error ``KeyError: "John Doe"`` that says nothing about the context or the very last one ``PageRenderError: failed to render /foo/bar/baz.html?user=John%20Doe`` which won't have a meaningful stack trace and will likely not contain enough information to reproduce or debug the issue. The latter is, in fact, very close to the dreaded *This application has executed an invalid operation and will be terminated*.

Some software has intermediate checkpoints that emit to the log file and then re-raise, e.g.
```python
try:
    user = UserDatabase.get_user(username)
except Exception as e:
    logger.error(f"UserDatabase.get_user({username}) failed: {e}")
    raise
```
*which is better than nothing*, but you still have to hunt through the log file. And if your application was doing more unrelated things at the same time (e.g. a web server), you'll have to go through A LOT of noise.
Exception chaining, on the other hand, will carry the whole thing to the very bottom for you to read at once.
@crusaderky Those are excellent points!  Thank you for the thoughtful writeup.
I think that this should probably be resolved upstream.  Dask takes exceptions and tracebacks as given to us from user code and then moves those to the client using standard serialization systems.  If those serialization systems aren't working then they should be fixed, but I don't want dask to own that functionality.  
@mrocklin getting it upstream is problematic, because it would mean 
1. either adding tblib to the standard library (ugh) or writing from scratch a native serialization for traceback (much cleaner, more time consuming)
2. that an Exception pickled by a Python interpreter after the change won't be deserializable by a pre-change interpreter.

I'm not sure if (2) is an actual concern for the CPython developers. Anyway, it would be a very major (definitely PEP-worthy) amount of work.

----- 

distributed already relies on tblib to serialize the tracebacks. If I were to write a small module that patches the python interpreter, exactly in the same way that tblib does (with ``copyreg.pickle``) and allows transparently pickling/unpickling an exception, its cause, and its traceback, would you be happy to introduce it as a dependency in distributed instead of tblib? (the new module would use tblib internally).
There would be however the problem of having this new module, at least initially, in conda-forge only and not in pkgs/main, which would mean it can't be a mandatory dependency for distributed
Alternatively I can see if I can convince the tblib maintainers to accept a patch to pickle the whole Exception
Further discussion and POCs on https://github.com/ionelmc/python-tblib/issues/53
I ran into this exact issue (I had a chained exception being raised on a worker, and the real interesting context was contained in what would have been the `__cause__`) and decided to try to patch it before I saw this issue. My approach was to
- refactor `error_message` so that it adds a `"cause"` key, whose value is itself the result of `error_message(e.__cause__)`.
- add a `cause` slot to `FutureState` that gets populated in `set_error`
- forward the `"cause"` in the message sent from `transition_processing_erred`
- refactor `clean_exception` so that it reattaches the `__cause__` attribute to the exception, like it already does with the traceback

I have tested the changes to confirm they work. The changes only use the existing exception serialization machinery in distributed, extended slightly to serialize causes. I can submit the PR if there's interest.
Is it possible to fix this problem upstream in tblib?

On Wed, Jan 29, 2020, 8:14 AM Alex Adamson <notifications@github.com> wrote:

> I ran into this exact issue (I had a chained exception being raised on a
> worker, and the real interesting context was contained in what would have
> been the *cause*) and decided to try to patch it before I saw this issue.
> My approach was to
>
>    - refactor error_message so that it adds a "cause" key, whose value is
>    itself the result of error_message(e.__cause__).
>    - add a cause slot to FutureState that gets populated in set_error
>    - forward the "cause" in the message sent from
>    transition_processing_erred
>    - refactor clean_exception so that it reattaches the __cause__
>    attribute to the exception, like it already does with the traceback
>    I have tested the changes to confirm they work. The changes only use
>    the existing exception serialization machinery in distributed, extended
>    slightly to serialize causes. I can submit the PR if there's interest.
>
> —
> You are receiving this because you were mentioned.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/distributed/issues/3158?email_source=notifications&email_token=AACKZTGIHA6BP765Y5WSBFLRAGTMRA5CNFSM4JC3H2V2YY3PNVWWK3TUL52HS4DFVREXG43VMVBW63LNMVXHJKTDN5WW2ZLOORPWSZGOEKHY75A#issuecomment-579833844>,
> or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/AACKZTFIYGWXVDQSRU3SQHLRAGTMRANCNFSM4JC3H2VQ>
> .
>

Actually, it appears that it has been fixed upstream (https://github.com/ionelmc/python-tblib/pull/54). I will test with tblib >=1.6.0 and see if it just works.
It probably won't without a patch to distributed - read https://python-tblib.readthedocs.io/en/latest/readme.html#pickling-exceptions-together-with-their-traceback-and-chain-python-3-only

Try starting your remote functions with
```python
from tblib import pickling_support
pickling_support.install() 
```
this will guarantee that all custom exceptions are registered in tblib.
Reading the patch you submitted (@crusaderky), I think it would also work as long as we did
```python
def error_message(e, status="error"):
    ...
    if type(e) not in copyreg.dispatch_table:
         tblib.pickling_support.install(type(e))
    ....
    return {..., "exception": e4, ...}
```
Yeah, without the change above, as @crusaderky suggested, custom exception subclasses declared after distributed.utils is loaded do not get serialized properly, but they are serialized properly regardless of when they were declared if you add the change above.

Resolving patch:
diff --git a/distributed/core.py b/distributed/core.py
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -10,6 +10,7 @@
 import weakref
 
 import dask
+import tblib
 from toolz import merge
 from tornado import gen
 from tornado.ioloop import IOLoop
@@ -982,6 +983,14 @@ def coerce_to_address(o):
     return normalize_address(o)
 
 
+def collect_causes(e):
+    causes = []
+    while e.__cause__ is not None:
+        causes.append(e.__cause__)
+        e = e.__cause__
+    return causes
+
+
 def error_message(e, status="error"):
     """ Produce message to send back given an exception has occurred
 
@@ -998,6 +1007,7 @@ def error_message(e, status="error"):
     clean_exception: deserialize and unpack message into exception/traceback
     """
     MAX_ERROR_LEN = dask.config.get("distributed.admin.max-error-length")
+    tblib.pickling_support.install(e, *collect_causes(e))
     tb = get_traceback()
     e2 = truncate_exception(e, MAX_ERROR_LEN)
     try:


Test patch:
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -372,6 +372,32 @@ def __str__(self):
         assert len(msg["text"]) > 10100  # default + 100
 
 
+@gen_cluster(client=True)
+def test_chained_error_message(c, s, a, b):
+    def chained_exception_fn():
+        class MyException(Exception):
+            def __init__(self, msg):
+                self.msg = msg
+
+            def __str__(self):
+                return "MyException(%s)" % self.msg
+
+        exception = MyException("Foo")
+        inner_exception = MyException("Bar")
+
+        try:
+            raise inner_exception
+        except Exception as e:
+            raise exception from e
+
+    f = c.submit(chained_exception_fn)
+    try:
+        yield f
+    except Exception as e:
+        assert e.__cause__ is not None
+        assert "Bar" in str(e.__cause__)
+
+
 @gen_cluster()
 def test_gather(s, a, b):
     b.data["x"] = 1