Repository: pallets/werkzeug

Issue:
Werkzeug does not properly log chained exceptions
When a chained exception is thrown, the Werkzeug server (used in development mode by Flask) logs only the "top" exception when it ought to log the whole chain.

To reproduce, run this Python script (with Flask installed):

```
from flask import Flask

app = Flask(__name__)


def do_something():
  raise ValueError("Foo")


@app.route('/exception_test', methods=['GET', 'POST'])
def process():
  try:
    do_something()
  except Exception as e:
    raise ValueError("Wrapper") from e

if __name__ == '__main__':
  app.config['PROPAGATE_EXCEPTIONS'] = True
  app.run(debug=False)
```

and then do `wget  http://127.0.0.1:5000/exception_test`. You should get something like:

```
 * Serving Flask app "chained_exception_mcve" (lazy loading)
 * Environment: production
   WARNING: Do not use the development server in a production environment.
   Use a production WSGI server instead.
 * Debug mode: off
 * Running on http://127.0.0.1:5000/ (Press CTRL+C to quit)
127.0.0.1 - - [30/Aug/2018 13:02:18] "GET /exception_test HTTP/1.1" 500 -
Error on request:
Traceback (most recent call last):
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/werkzeug/serving.py", line 270, in run_wsgi
    execute(self.server.app)
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/werkzeug/serving.py", line 258, in execute
    application_iter = app(environ, start_response)
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/app.py", line 2309, in __call__
    return self.wsgi_app(environ, start_response)
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/app.py", line 2295, in wsgi_app
    response = self.handle_exception(e)
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/app.py", line 1741, in handle_exception
    reraise(exc_type, exc_value, tb)
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/_compat.py", line 35, in reraise
    raise value
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/app.py", line 2292, in wsgi_app
    response = self.full_dispatch_request()
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/app.py", line 1815, in full_dispatch_request
    rv = self.handle_user_exception(e)
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/app.py", line 1718, in handle_user_exception
    reraise(exc_type, exc_value, tb)
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/_compat.py", line 35, in reraise
    raise value
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/app.py", line 1813, in full_dispatch_request
    rv = self.dispatch_request()
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/app.py", line 1799, in dispatch_request
    return self.view_functions[rule.endpoint](**req.view_args)
  File "/Users/gabbard/projects/gaia/gaia-alternatives/repos/vista-nlp-jvm/vista_nlp_serialization/chained_exception_mcve.py", line 20, in process
    raise ValueError("Wrapper") from e
ValueError: Wrapper
```

The desired output is:

```
 * Serving Flask app "chained_exception_mcve" (lazy loading)
 * Environment: production
   WARNING: Do not use the development server in a production environment.
   Use a production WSGI server instead.
 * Debug mode: off
 * Running on http://127.0.0.1:5000/ (Press CTRL+C to quit)
127.0.0.1 - - [30/Aug/2018 13:03:58] "GET /exception_test HTTP/1.1" 500 -
Error on request:
Traceback (most recent call last):
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/werkzeug/serving.py", line 270, in run_wsgi
    execute(self.server.app)
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/werkzeug/serving.py", line 258, in execute
    application_iter = app(environ, start_response)
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/app.py", line 2309, in __call__
    return self.wsgi_app(environ, start_response)
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/app.py", line 2295, in wsgi_app
    response = self.handle_exception(e)
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/app.py", line 1741, in handle_exception
    reraise(exc_type, exc_value, tb)
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/_compat.py", line 35, in reraise
    raise value
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/app.py", line 2292, in wsgi_app
    response = self.full_dispatch_request()
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/app.py", line 1815, in full_dispatch_request
    rv = self.handle_user_exception(e)
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/app.py", line 1718, in handle_user_exception
    reraise(exc_type, exc_value, tb)
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/_compat.py", line 35, in reraise
    raise value
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/app.py", line 1813, in full_dispatch_request
    rv = self.dispatch_request()
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/app.py", line 1799, in dispatch_request
    return self.view_functions[rule.endpoint](**req.view_args)
  File "/Users/gabbard/projects/gaia/gaia-alternatives/repos/vista-nlp-jvm/vista_nlp_serialization/chained_exception_mcve.py", line 20, in process
    raise ValueError("Wrapper") from e
Traceback (most recent call last):
  File "/Users/gabbard/projects/gaia/gaia-alternatives/repos/vista-nlp-jvm/vista_nlp_serialization/chained_exception_mcve.py", line 18, in process
    do_something()
  File "/Users/gabbard/projects/gaia/gaia-alternatives/repos/vista-nlp-jvm/vista_nlp_serialization/chained_exception_mcve.py", line 8, in do_something
    raise ValueError("Foo")
ValueError: Foo

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/werkzeug/serving.py", line 270, in run_wsgi
    execute(self.server.app)
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/werkzeug/serving.py", line 258, in execute
    application_iter = app(environ, start_response)
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/app.py", line 2309, in __call__
    return self.wsgi_app(environ, start_response)
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/app.py", line 2295, in wsgi_app
    response = self.handle_exception(e)
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/app.py", line 1741, in handle_exception
    reraise(exc_type, exc_value, tb)
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/_compat.py", line 35, in reraise
    raise value
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/app.py", line 2292, in wsgi_app
    response = self.full_dispatch_request()
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/app.py", line 1815, in full_dispatch_request
    rv = self.handle_user_exception(e)
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/app.py", line 1718, in handle_user_exception
    reraise(exc_type, exc_value, tb)
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/_compat.py", line 35, in reraise
    raise value
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/app.py", line 1813, in full_dispatch_request
    rv = self.dispatch_request()
  File "/Users/gabbard/anaconda3/envs/gaia-alternatives/lib/python3.6/site-packages/flask/app.py", line 1799, in dispatch_request
    return self.view_functions[rule.endpoint](**req.view_args)
  File "/Users/gabbard/projects/gaia/gaia-alternatives/repos/vista-nlp-jvm/vista_nlp_serialization/chained_exception_mcve.py", line 20, in process
    raise ValueError("Wrapper") from e
ValueError: Wrapper
```

The desired output can be achieved by editing the `Traceback` object from `tbtools.py` to
* save the `tb` object passed to its constructor
* changing https://github.com/pallets/werkzeug/blob/master/werkzeug/debug/tbtools.py#L280 to 
    ```
    buf = traceback.format_exception(self.exc_type, self.exc_value, self.tb,
                                 chain=True)
    ```

However, I am not sure whether or not this is a real solution because I don't know what consequences this might have elsewhere.

### Environment

* Python version: 3.6
* Flask version: 1.0.2
* Werkzeug version: 0.14.1

[This issue was originally reported on Flask](https://github.com/pallets/flask/issues/2894#issuecomment-417396297) before it became apparent it was a Werkzeug issue.


Discussion:

Resolving patch:
diff --git a/werkzeug/debug/tbtools.py b/werkzeug/debug/tbtools.py
--- a/werkzeug/debug/tbtools.py
+++ b/werkzeug/debug/tbtools.py
@@ -20,8 +20,10 @@
 
 from werkzeug.utils import cached_property, escape
 from werkzeug.debug.console import Console
-from werkzeug._compat import range_type, PY2, text_type, string_types, \
-    to_native, to_unicode
+from werkzeug._compat import (
+    range_type, PY2, text_type, string_types,
+    to_native, to_unicode, reraise,
+)
 from werkzeug.filesystem import get_filesystem_encoding
 
 
@@ -176,8 +178,8 @@ def get_current_traceback(ignore_system_exceptions=False,
     """
     exc_type, exc_value, tb = sys.exc_info()
     if ignore_system_exceptions and exc_type in system_exceptions:
-        raise
-    for x in range_type(skip):
+        reraise(exc_type, exc_value, tb)
+    for _ in range_type(skip):
         if tb.tb_next is None:
             break
         tb = tb.tb_next
@@ -197,6 +199,7 @@ def __init__(self, lineno, code):
         self.in_frame = False
         self.current = False
 
+    @property
     def classes(self):
         rv = ['line']
         if self.in_frame:
@@ -204,7 +207,6 @@ def classes(self):
         if self.current:
             rv.append('current')
         return rv
-    classes = property(classes)
 
     def render(self):
         return SOURCE_LINE_HTML % {
@@ -220,76 +222,49 @@ class Traceback(object):
     def __init__(self, exc_type, exc_value, tb):
         self.exc_type = exc_type
         self.exc_value = exc_value
-        if not isinstance(exc_type, str):
-            exception_type = exc_type.__name__
-            if exc_type.__module__ not in ('__builtin__', 'exceptions'):
-                exception_type = exc_type.__module__ + '.' + exception_type
-        else:
-            exception_type = exc_type
+        self.tb = tb
+
+        exception_type = exc_type.__name__
+        if exc_type.__module__ not in {"builtins", "__builtin__", "exceptions"}:
+            exception_type = exc_type.__module__ + '.' + exception_type
         self.exception_type = exception_type
 
-        # we only add frames to the list that are not hidden.  This follows
-        # the magic variables as defined by paste.exceptions.collector
-        self.frames = []
-        while tb:
-            self.frames.append(Frame(exc_type, exc_value, tb))
-            tb = tb.tb_next
+        self.groups = []
+        while True:
+            self.groups.append(Group(exc_type, exc_value, tb))
+            if PY2:
+                break
+            exc_value = exc_value.__cause__ or exc_value.__context__
+            if exc_value is None:
+                break
+            exc_type = type(exc_value)
+            tb = exc_value.__traceback__
+        self.groups.reverse()
+        self.frames = [frame for group in self.groups for frame in group.frames]
 
     def filter_hidden_frames(self):
         """Remove the frames according to the paste spec."""
-        if not self.frames:
-            return
-
-        new_frames = []
-        hidden = False
-        for frame in self.frames:
-            hide = frame.hide
-            if hide in ('before', 'before_and_this'):
-                new_frames = []
-                hidden = False
-                if hide == 'before_and_this':
-                    continue
-            elif hide in ('reset', 'reset_and_this'):
-                hidden = False
-                if hide == 'reset_and_this':
-                    continue
-            elif hide in ('after', 'after_and_this'):
-                hidden = True
-                if hide == 'after_and_this':
-                    continue
-            elif hide or hidden:
-                continue
-            new_frames.append(frame)
-
-        # if we only have one frame and that frame is from the codeop
-        # module, remove it.
-        if len(new_frames) == 1 and self.frames[0].module == 'codeop':
-            del self.frames[:]
+        for group in self.groups:
+            group.filter_hidden_frames()
 
-        # if the last frame is missing something went terrible wrong :(
-        elif self.frames[-1] in new_frames:
-            self.frames[:] = new_frames
+        self.frames[:] = [frame for group in self.groups for frame in group.frames]
 
+    @property
     def is_syntax_error(self):
         """Is it a syntax error?"""
         return isinstance(self.exc_value, SyntaxError)
-    is_syntax_error = property(is_syntax_error)
 
+    @property
     def exception(self):
-        """String representation of the exception."""
-        buf = traceback.format_exception_only(self.exc_type, self.exc_value)
-        rv = ''.join(buf).strip()
-        return rv.decode('utf-8', 'replace') if PY2 else rv
-    exception = property(exception)
+        """String representation of the final exception."""
+        return self.groups[-1].exception
 
     def log(self, logfile=None):
         """Log the ASCII traceback into a file object."""
         if logfile is None:
             logfile = sys.stderr
         tb = self.plaintext.rstrip() + u'\n'
-        if PY2:
-            tb = tb.encode('utf-8', 'replace')
-        logfile.write(tb)
+        logfile.write(to_native(tb, "utf-8", "replace"))
 
     def paste(self):
         """Create a paste and return the paste id."""
@@ -317,10 +292,12 @@ def paste(self):
     def render_summary(self, include_title=True):
         """Render the traceback for the interactive console."""
         title = ''
-        frames = []
         classes = ['traceback']
         if not self.frames:
             classes.append('noframe-traceback')
+            frames = []
+        else:
+            frames = [group.render() for group in self.groups]
 
         if include_title:
             if self.is_syntax_error:
@@ -328,12 +305,6 @@ def render_summary(self, include_title=True):
             else:
                 title = u'Traceback <em>(most recent call last)</em>:'
 
-        for frame in self.frames:
-            frames.append(u'<li%s>%s' % (
-                u' title="%s"' % escape(frame.info) if frame.info else u'',
-                frame.render()
-            ))
-
         if self.is_syntax_error:
             description_wrapper = u'<pre class=syntaxerror>%s</pre>'
         else:
@@ -364,23 +335,101 @@ def render_full(self, evalex=False, secret=None,
             'secret':           secret
         }
 
-    def generate_plaintext_traceback(self):
-        """Like the plaintext attribute but returns a generator"""
-        yield u'Traceback (most recent call last):'
+    @cached_property
+    def plaintext(self):
+        return u"\n".join([group.render_text() for group in self.groups])
+
+    @property
+    def id(self):
+        return id(self)
+
+
+class Group(object):
+    """A group of frames for an exception in a traceback. On Python 3,
+    if the exception has a ``__cause__`` or ``__context__``, there are
+    multiple exception groups.
+    """
+
+    def __init__(self, exc_type, exc_value, tb):
+        self.exc_type = exc_type
+        self.exc_value = exc_value
+        self.info = None
+        if not PY2:
+            if exc_value.__cause__ is not None:
+                self.info = (
+                    u"The above exception was the direct cause of the"
+                    u" following exception"
+                )
+            elif exc_value.__context__ is not None:
+                self.info = (
+                    u"During handling of the above exception, another"
+                    u" exception occurred"
+                )
+
+        self.frames = []
+        while tb is not None:
+            self.frames.append(Frame(exc_type, exc_value, tb))
+            tb = tb.tb_next
+
+    def filter_hidden_frames(self):
+        new_frames = []
+        hidden = False
+
         for frame in self.frames:
-            yield u'  File "%s", line %s, in %s' % (
-                frame.filename,
-                frame.lineno,
-                frame.function_name
-            )
-            yield u'    ' + frame.current_line.strip()
-        yield self.exception
+            hide = frame.hide
+            if hide in ("before", "before_and_this"):
+                new_frames = []
+                hidden = False
+                if hide == "before_and_this":
+                    continue
+            elif hide in ("reset", "reset_and_this"):
+                hidden = False
+                if hide == "reset_and_this":
+                    continue
+            elif hide in ("after", "after_and_this"):
+                hidden = True
+                if hide == "after_and_this":
+                    continue
+            elif hide or hidden:
+                continue
+            new_frames.append(frame)
 
-    def plaintext(self):
-        return u'\n'.join(self.generate_plaintext_traceback())
-    plaintext = cached_property(plaintext)
+        # if we only have one frame and that frame is from the codeop
+        # module, remove it.
+        if len(new_frames) == 1 and self.frames[0].module == "codeop":
+            del self.frames[:]
 
-    id = property(lambda x: id(x))
+        # if the last frame is missing something went terrible wrong :(
+        elif self.frames[-1] in new_frames:
+            self.frames[:] = new_frames
+
+    @property
+    def exception(self):
+        """String representation of the exception."""
+        buf = traceback.format_exception_only(self.exc_type, self.exc_value)
+        rv = "".join(buf).strip()
+        return to_unicode(rv, "utf-8", "replace")
+
+    def render(self):
+        out = []
+        if self.info is not None:
+            out.append(u'<li><div class="exc-divider">%s:</div>' % self.info)
+        for frame in self.frames:
+            out.append(u"<li%s>%s" % (
+                u' title="%s"' % escape(frame.info) if frame.info else u"",
+                frame.render()
+            ))
+        return u"\n".join(out)
+
+    def render_text(self):
+        out = []
+        if self.info is not None:
+            out.append(u"\n%s:\n" % self.info)
+        out.append(u"Traceback (most recent call last):")
+        for frame in self.frames:
+            out.append(frame.render_text())
+        out.append(self.exception)
+        return u"\n".join(out)
 
 
 class Frame(object):
@@ -408,10 +457,7 @@ def __init__(self, exc_type, exc_value, tb):
         self.hide = self.locals.get('__traceback_hide__', False)
         info = self.locals.get('__traceback_info__')
         if info is not None:
-            try:
-                info = text_type(info)
-            except UnicodeError:
-                info = str(info).decode('utf-8', 'replace')
+            info = to_unicode(info, "utf-8", "replace")
         self.info = info
 
     def render(self):
@@ -424,6 +470,14 @@ def render(self):
             'lines':            self.render_line_context(),
         }
 
+    def render_text(self):
+        return u'  File "%s", line %s, in %s\n    %s' % (
+            self.filename,
+            self.lineno,
+            self.function_name,
+            self.current_line.strip()
+        )
+
     def render_line_context(self):
         before, current, after = self.get_context_lines()
         rv = []
@@ -474,7 +528,7 @@ def get_annotated_lines(self):
     def eval(self, code, mode='single'):
         """Evaluate code in the context of the frame."""
         if isinstance(code, string_types):
-            if PY2 and isinstance(code, unicode):  # noqa
+            if PY2 and isinstance(code, text_type):  # noqa
                 code = UTF8_COOKIE + code.encode('utf-8')
             code = compile(code, '<interactive>', mode)
         return eval(code, self.globals, self.locals)
@@ -553,4 +607,6 @@ def current_line(self):
     def console(self):
         return Console(self.globals, self.locals)
 
-    id = property(lambda x: id(x))
+    @property
+    def id(self):
+        return id(self)


Test patch:
diff --git a/tests/test_debug.py b/tests/test_debug.py
--- a/tests/test_debug.py
+++ b/tests/test_debug.py
@@ -15,11 +15,13 @@
 import pytest
 import requests
 
-from werkzeug.debug import get_machine_id
+from werkzeug.debug import get_machine_id, DebuggedApplication
 from werkzeug.debug.repr import debug_repr, DebugReprGenerator, \
     dump, helper
 from werkzeug.debug.console import HTMLStringO
 from werkzeug.debug.tbtools import Traceback
+from werkzeug.test import Client
+from werkzeug.wrappers import Request, Response
 from werkzeug._compat import PY2
 
 
@@ -200,6 +202,25 @@ def test_debug_help(self):
         assert 'Help on list object' in x
         assert '__delitem__' in x
 
+    @pytest.mark.skipif(PY2, reason="Python 2 doesn't have chained exceptions.")
+    def test_exc_divider_found_on_chained_exception(self):
+        @Request.application
+        def app(request):
+            def do_something():
+                raise ValueError("inner")
+            try:
+                do_something()
+            except ValueError:
+                raise KeyError("outer")
+
+        debugged = DebuggedApplication(app)
+        client = Client(debugged, Response)
+        response = client.get("/")
+        data = response.get_data(as_text=True)
+        assert u'raise ValueError("inner")' in data
+        assert u'<div class="exc-divider">' in data
+        assert u'raise KeyError("outer")' in data
+
 
 class TestTraceback(object):