Repository: HypothesisWorks/hypothesis

Issue:
Further distinguish errors by exception `__cause__` or `__context__`
We have [an internal notion of `interesting_origin`](https://github.com/HypothesisWorks/hypothesis/search?q=interesting_origin), which is used to distinguish multiple failures in a single test.  Currently, that's a tuple of `(type, filename, line_number)` for each exception... which doesn't help much when the code does a general `except Exception: raise SomeGenericError`.

I therefore propose that we should also distinguish the `interesting_origin` by [the `__cause__` or `__context__` of the error](https://docs.python.org/3/library/exceptions.html), to allow for finer-grained reporting - and more importantly finer-grained caching of distinct failing examples.

There's some more design work to do on exactly how to distinguish this; I'm inclined to think that "the same way that Python would report them" is sensible but could see an argument to e.g. save finer distinctions than we bother reporting.  We'll also probably want to distinguish between cause and context - `except ValueError as err: raise AssertionError from err` is different from `except ValueError: assert state_is_ok`.

Comments, examples where this would be useful (or not), or suggestions are welcome 😀 


Discussion:

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/core.py b/hypothesis-python/src/hypothesis/core.py
--- a/hypothesis-python/src/hypothesis/core.py
+++ b/hypothesis-python/src/hypothesis/core.py
@@ -70,6 +70,7 @@
 from hypothesis.internal.entropy import deterministic_PRNG
 from hypothesis.internal.escalation import (
     escalate_hypothesis_internal_error,
+    get_interesting_origin,
     get_trimmed_traceback,
 )
 from hypothesis.internal.healthcheck import fail_health_check
@@ -721,10 +722,7 @@ def _execute_once_for_engine(self, data):
                 info.__expected_exception = e
                 verbose_report(info.__expected_traceback)
 
-                origin = traceback.extract_tb(tb)[-1]
-                filename = origin[0]
-                lineno = origin[1]
-                data.mark_interesting((type(e), filename, lineno))
+                data.mark_interesting(get_interesting_origin(e))
 
     def run_engine(self):
         """Run the test function many times, on database input and generated
diff --git a/hypothesis-python/src/hypothesis/internal/escalation.py b/hypothesis-python/src/hypothesis/internal/escalation.py
--- a/hypothesis-python/src/hypothesis/internal/escalation.py
+++ b/hypothesis-python/src/hypothesis/internal/escalation.py
@@ -81,14 +81,17 @@ def escalate_hypothesis_internal_error():
         raise
 
 
-def get_trimmed_traceback():
+def get_trimmed_traceback(exception=None):
     """Return the current traceback, minus any frames added by Hypothesis."""
-    error_type, _, tb = sys.exc_info()
+    if exception is None:
+        _, exception, tb = sys.exc_info()
+    else:
+        tb = exception.__traceback__
     # Avoid trimming the traceback if we're in verbose mode, or the error
     # was raised inside Hypothesis (and is not a MultipleFailures)
     if hypothesis.settings.default.verbosity >= hypothesis.Verbosity.debug or (
         is_hypothesis_file(traceback.extract_tb(tb)[-1][0])
-        and not isinstance(error_type, MultipleFailures)
+        and not isinstance(exception, MultipleFailures)
     ):
         return tb
     while tb is not None and (
@@ -100,3 +103,22 @@ def get_trimmed_traceback():
     ):
         tb = tb.tb_next
     return tb
+
+
+def get_interesting_origin(exception):
+    # The `interesting_origin` is how Hypothesis distinguishes between multiple
+    # failures, for reporting and also to replay from the example database (even
+    # if report_multiple_bugs=False).  We traditionally use the exception type and
+    # location, but have extracted this logic in order to see through `except ...:`
+    # blocks and understand the __cause__ (`raise x from y`) or __context__ that
+    # first raised an exception.
+    tb = get_trimmed_traceback(exception)
+    filename, lineno, *_ = traceback.extract_tb(tb)[-1]
+    return (
+        type(exception),
+        filename,
+        lineno,
+        # Note that if __cause__ is set it is always equal to __context__, explicitly
+        # to support introspection when debugging, so we can use that unconditionally.
+        get_interesting_origin(exception.__context__) if exception.__context__ else (),
+    )


Test patch:
diff --git a/hypothesis-python/tests/nocover/test_interesting_origin.py b/hypothesis-python/tests/nocover/test_interesting_origin.py
new file mode 100644
--- /dev/null
+++ b/hypothesis-python/tests/nocover/test_interesting_origin.py
@@ -0,0 +1,62 @@
+# This file is part of Hypothesis, which may be found at
+# https://github.com/HypothesisWorks/hypothesis/
+#
+# Most of this work is copyright (C) 2013-2021 David R. MacIver
+# (david@drmaciver.com), but it contains contributions by others. See
+# CONTRIBUTING.rst for a full list of people who may hold copyright, and
+# consult the git log if you need to determine who owns an individual
+# contribution.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at https://mozilla.org/MPL/2.0/.
+#
+# END HEADER
+
+import pytest
+
+from hypothesis import given, strategies as st
+from hypothesis.errors import MultipleFailures
+
+
+def go_wrong_naive(a, b):
+    try:
+        assert a + b < 100
+        a / b
+    except Exception:
+        # Hiding the actual problem is terrible, but this pattern can make sense
+        # if you're raising a library-specific or semantically meaningful error.
+        raise ValueError("Something went wrong")
+
+
+def go_wrong_with_cause(a, b):
+    try:
+        assert a + b < 100
+        a / b
+    except Exception as err:
+        # Explicit chaining is the *right way* to change exception type.
+        raise ValueError("Something went wrong") from err
+
+
+def go_wrong_coverup(a, b):
+    try:
+        assert a + b < 100
+        a / b
+    except Exception:
+        # This pattern SHOULD be local enough that it never distinguishes
+        # errors in practice... but if it does, we're ready.
+        raise ValueError("Something went wrong") from None
+
+
+@pytest.mark.parametrize(
+    "function",
+    [go_wrong_naive, go_wrong_with_cause, go_wrong_coverup],
+    ids=lambda f: f.__name__,
+)
+def test_can_generate_specified_version(function):
+    try:
+        given(st.integers(), st.integers())(function)()
+    except MultipleFailures:
+        pass
+    else:
+        raise AssertionError("Expected MultipleFailures")