Repository: dask/distributed

Issue:
Recreate remote tracebacks
Currently we send along a list of strings, which is less than pretty.  We should improve this because errors are important.  Some options:
1.  Construct a traceback-like object (the actual object appears to be inaccessible from pure Python) but hopefully a similarly duck-typed object suffices
2.  Use [tblib](https://pypi.python.org/pypi/tblib) to construct or to pickle the remote traceback and reraise locally.  This is CPython only, but I think I'm ok with that.

I'll probably go with option 2 in the near future unless anyone has other thoughts.



Discussion:

Resolving patch:
diff --git a/distributed/executor.py b/distributed/executor.py
--- a/distributed/executor.py
+++ b/distributed/executor.py
@@ -10,6 +10,7 @@
 from time import sleep
 import uuid
 from threading import Thread
+import six
 
 import dask
 from dask.base import tokenize, normalize_token, Base
@@ -87,7 +88,9 @@ def done(self):
     def result(self):
         """ Wait until computation completes. Gather result to local process """
         result = sync(self.executor.loop, self._result, raiseit=False)
-        if self.status in ('error', 'cancelled'):
+        if self.status == 'error':
+            six.reraise(*result)
+        if self.status == 'cancelled':
             raise result
         else:
             return result
@@ -106,11 +109,11 @@ def _result(self, raiseit=True):
         yield d['event'].wait()
         if self.status == 'error':
             exception = d['exception']
-            traceback = d['traceback']  # TODO: use me
+            traceback = d['traceback']
             if raiseit:
-                raise exception
+                six.reraise(type(exception), exception, traceback)
             else:
-                raise Return(exception)
+                raise Return([type(exception), exception, traceback])
         else:
             result = yield self.executor._gather([self])
             raise gen.Return(result[0])
@@ -150,7 +153,18 @@ def _traceback(self):
             raise Return(None)
 
     def traceback(self):
-        """ Return the exception of a failed task
+        """ Return the traceback of a failed task
+
+        This returns a traceback object.  You can inspect this object using the
+        ``traceback`` module.  Alternatively if you call ``future.result()``
+        this traceback will accompany the raised exception.
+
+        Examples
+        --------
+        >>> import traceback  # doctest: +SKIP
+        >>> tb = future.traceback()  # doctest: +SKIP
+        >>> traceback.export_tb(tb)  # doctest: +SKIP
+        [...]
 
         See Also
         --------
diff --git a/distributed/utils.py b/distributed/utils.py
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -7,6 +7,7 @@
 import re
 import socket
 import sys
+import tblib.pickling_support
 import tempfile
 import traceback
 
@@ -226,11 +227,18 @@ def ensure_ip(hostname):
         return socket.gethostbyname(hostname)
 
 
+tblib.pickling_support.install()
+
+
 def get_traceback():
     exc_type, exc_value, exc_traceback = sys.exc_info()
-    tb = traceback.format_tb(exc_traceback)
-    tb = [line[:10000] for line in tb]
-    return tb
+    bad = [os.path.join('distributed', 'worker'),
+           os.path.join('distributed', 'scheduler'),
+           os.path.join('tornado', 'gen.py'),
+           os.path.join('concurrent', 'futures')]
+    while any(b in exc_traceback.tb_frame.f_code.co_filename for b in bad):
+        exc_traceback = exc_traceback.tb_next
+    return exc_traceback
 
 
 def truncate_exception(e, n=10000):


Test patch:
diff --git a/distributed/tests/test_executor.py b/distributed/tests/test_executor.py
--- a/distributed/tests/test_executor.py
+++ b/distributed/tests/test_executor.py
@@ -9,9 +9,10 @@
 import sys
 from threading import Thread
 from time import sleep, time
+import traceback
 
 import pytest
-from toolz import identity, isdistinct, first
+from toolz import identity, isdistinct, first, concat, pluck
 from tornado.ioloop import IOLoop
 from tornado.iostream import IOStream
 from tornado import gen
@@ -1170,7 +1171,8 @@ def test_traceback(s, a, b):
     tb = yield x._traceback()
 
     if sys.version_info[0] >= 3:
-        assert any('x / y' in line for line in tb)
+        assert any('x / y' in line
+                   for line in pluck(3, traceback.extract_tb(tb)))
 
     yield e._shutdown()
 
@@ -1181,12 +1183,15 @@ def test_traceback_sync(loop):
             x = e.submit(div, 1, 0)
             tb = x.traceback()
             if sys.version_info[0] >= 3:
-                assert any('x / y' in line for line in tb)
+                assert any('x / y' in line
+                           for line in concat(traceback.extract_tb(tb))
+                           if isinstance(line, str))
 
             y = e.submit(inc, x)
             tb2 = y.traceback()
 
-            assert set(tb2).issuperset(set(tb))
+            assert set(pluck(3, traceback.extract_tb(tb2))).issuperset(
+                   set(pluck(3, traceback.extract_tb(tb))))
 
             z = e.submit(div, 1, 2)
             tb = z.traceback()
@@ -1682,7 +1687,9 @@ def bad(x):
         assert len(str(e)) < 100000
 
     tb = yield x._traceback()
-    assert all(len(line) < 100000 for line in tb)
+    assert all(len(line) < 100000
+               for line in concat(traceback.extract_tb(tb))
+               if isinstance(line, str))
 
 
 @gen_cluster()
@@ -2084,3 +2091,20 @@ def test_future_type(s, a, b):
     assert 'int' in str(x)
 
     yield e._shutdown()
+
+
+@gen_cluster()
+def test_traceback_clean(s, a, b):
+    e = Executor((s.ip, s.port), start=False)
+    yield e._start()
+
+    x = e.submit(div, 1, 0)
+    try:
+        yield x._result()
+    except Exception as e:
+        f = e
+        exc_type, exc_value, tb = sys.exc_info()
+        while tb:
+            assert 'scheduler' not in tb.tb_frame.f_code.co_filename
+            assert 'worker' not in tb.tb_frame.f_code.co_filename
+            tb = tb.tb_next
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -1,6 +1,6 @@
 from distributed.utils import (All, sync, is_kernel, ensure_ip,
-        truncate_exception)
-from distributed.utils_test import loop, inc, throws
+        truncate_exception, get_traceback)
+from distributed.utils_test import loop, inc, throws, div
 import pytest
 from threading import Thread
 import threading
@@ -111,3 +111,18 @@ def test_truncate_exception():
 
     e = ValueError('a')
     assert truncate_exception(e) is e
+
+
+def test_get_traceback():
+    def a(x):
+        return div(x, 0)
+    def b(x):
+        return a(x)
+    def c(x):
+        return b(x)
+
+    try:
+        c(x)
+    except Exception as e:
+        tb = get_traceback()
+        assert type(tb).__name__ == 'traceback'
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -2,16 +2,19 @@
 import os
 import shutil
 import sys
+import traceback
+
+import pytest
+from toolz import pluck
+from tornado import gen
+from tornado.ioloop import TimeoutError
 
 from distributed.center import Center
 from distributed.core import rpc
 from distributed.sizeof import sizeof
 from distributed.worker import Worker
 from distributed.utils_test import loop, _test_cluster, inc
-import pytest
 
-from tornado import gen
-from tornado.ioloop import TimeoutError
 
 
 def test_worker_ncores():
@@ -60,7 +63,9 @@ def bad_func():
         assert response == b'error'
         assert isinstance(content['exception'], ZeroDivisionError)
         if sys.version_info[0] >= 3:
-            assert any('1 / 0' in line for line in content['traceback'])
+            assert any('1 / 0' in line
+                      for line in pluck(3, traceback.extract_tb(content['traceback']))
+                      if line)
 
         aa.close_streams()
         yield a._close()