Repository: HypothesisWorks/hypothesis

Issue:
Multiple-error reporting for explicit `@example` phase
Thanks to #2455, I'm writing up a demo test case which has `@example(...)`s which trigger *three* distinct bugs.

Currently, Hypothesis will stop at the first failure.  I'd like to instead report them all, at least when the `report_multiple_bugs` setting is `True`, as we would for generated test cases.  This will involve a bit of poking around in `core.py`, but shouldn't be too bad.


Discussion:

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/core.py b/hypothesis-python/src/hypothesis/core.py
--- a/hypothesis-python/src/hypothesis/core.py
+++ b/hypothesis-python/src/hypothesis/core.py
@@ -334,12 +334,6 @@ def execute_explicit_examples(state, wrapped_test, arguments, kwargs):
 
         with local_settings(state.settings):
             fragments_reported = []
-
-            def report_buffered():
-                for f in fragments_reported:
-                    report(f)
-                del fragments_reported[:]
-
             try:
                 with with_reporter(fragments_reported.append):
                     state.execute_once(
@@ -354,17 +348,20 @@ def report_buffered():
                 # a saved failure when other arguments are supplied by e.g. pytest.
                 # See https://github.com/HypothesisWorks/hypothesis/issues/2125
                 pass
-            except BaseException:
-                report_buffered()
-                raise
-
-            if current_verbosity() >= Verbosity.verbose:
-                prefix = "Falsifying example"
-                assert fragments_reported[0].startswith(prefix)
-                fragments_reported[0] = (
-                    "Trying example" + fragments_reported[0][len(prefix) :]
-                )
-                report_buffered()
+            except BaseException as err:
+                # In order to support reporting of multiple failing examples, we yield
+                # each of the (report text, error) pairs we find back to the top-level
+                # runner.  This also ensures that user-facing stack traces have as few
+                # frames of Hypothesis internals as possible.
+                yield (fragments_reported, err.with_traceback(get_trimmed_traceback()))
+                if state.settings.report_multiple_bugs:
+                    continue
+                break
+
+            assert fragments_reported[0].startswith("Falsifying example")
+            verbose_report(fragments_reported[0].replace("Falsifying", "Trying", 1))
+            for f in fragments_reported[1:]:
+                verbose_report(f)
 
 
 def get_random_for_wrapped_test(test, wrapped_test):
@@ -1069,8 +1066,28 @@ def wrapped_test(*arguments, **kwargs):
 
             # There was no @reproduce_failure, so start by running any explicit
             # examples from @example decorators.
-
-            execute_explicit_examples(state, wrapped_test, arguments, kwargs)
+            errors = list(
+                execute_explicit_examples(state, wrapped_test, arguments, kwargs)
+            )
+            with local_settings(state.settings):
+                if len(errors) > 1:
+                    # If we're not going to report multiple bugs, we would have
+                    # stopped running explicit examples at the first failure.
+                    assert state.settings.report_multiple_bugs
+                    for fragments, err in errors:
+                        for f in fragments:
+                            report(f)
+                        tb_lines = traceback.format_exception(
+                            type(err), err, err.__traceback__
+                        )
+                        report("".join(tb_lines))
+                    msg = "Hypothesis found %d failures in explicit examples."
+                    raise MultipleFailures(msg % (len(errors)))
+                elif errors:
+                    fragments, the_error_hypothesis_found = errors[0]
+                    for f in fragments:
+                        report(f)
+                    raise the_error_hypothesis_found
 
             # If there were any explicit examples, they all ran successfully.
             # The next step is to use the Conjecture engine to run the test on


Test patch:
diff --git a/hypothesis-python/tests/cover/test_explicit_examples.py b/hypothesis-python/tests/cover/test_explicit_examples.py
--- a/hypothesis-python/tests/cover/test_explicit_examples.py
+++ b/hypothesis-python/tests/cover/test_explicit_examples.py
@@ -28,7 +28,7 @@
     reporting,
     settings,
 )
-from hypothesis.errors import DeadlineExceeded, InvalidArgument
+from hypothesis.errors import DeadlineExceeded, InvalidArgument, MultipleFailures
 from hypothesis.strategies import floats, integers, nothing, text
 from tests.common.utils import assert_falsifying_output, capture_out
 
@@ -246,3 +246,17 @@ def test_unsatisfied_assumption_during_explicit_example(threshold, value):
     # Regression test, expected to pass / skip depending on parametrize.
     # See https://github.com/HypothesisWorks/hypothesis/issues/2125
     assume(value < threshold)
+
+
+@pytest.mark.parametrize("exc", [MultipleFailures, AssertionError])
+def test_multiple_example_reporting(exc):
+    @example(1)
+    @example(2)
+    @settings(report_multiple_bugs=exc is MultipleFailures, phases=[Phase.explicit])
+    @given(integers())
+    def inner_test_multiple_failing_examples(x):
+        assert x < 2
+        assert x < 1
+
+    with pytest.raises(exc):
+        inner_test_multiple_failing_examples()