Repository: HypothesisWorks/hypothesis

Issue:
Stateful testing: steps not printed on pytest.fail()
Via [a longer case posted on stackoverflow](https://stackoverflow.com/questions/51068206), I found out that BaseException - or subclasses like `pytest.Failed` - skips the printing of the steps taken to produce the error.

```python
from hypothesis.stateful import RuleBasedStateMachine, rule

class RaisesProblem(RuleBasedStateMachine):

    @rule()
    def oops(self):
        raise BaseException('The steps should be printed too!')

Test = RaisesProblem.TestCase
```

Suggested resolution: there is probably somewhere which catches `Exception` but should catch `BaseException` (perhaps re-raising `HypothesisBaseException`).  Finding and fixing that might be enough, along with a regression test of course!


Discussion:

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/core.py b/hypothesis-python/src/hypothesis/core.py
--- a/hypothesis-python/src/hypothesis/core.py
+++ b/hypothesis-python/src/hypothesis/core.py
@@ -384,7 +384,25 @@ def skip_exceptions_to_reraise():
     return tuple(sorted(exceptions, key=str))
 
 
-exceptions_to_reraise = skip_exceptions_to_reraise()
+EXCEPTIONS_TO_RERAISE = skip_exceptions_to_reraise()
+
+
+def failure_exceptions_to_catch():
+    """Return a tuple of exceptions meaning 'this test has failed', to catch.
+
+    This is intended to cover most common test runners; if you would
+    like another to be added please open an issue or pull request.
+    """
+    exceptions = [Exception]
+    try:  # pragma: no cover
+        from _pytest.outcomes import Failed
+        exceptions.append(Failed)
+    except ImportError:
+        pass
+    return tuple(exceptions)
+
+
+EXCEPTIONS_TO_FAIL = failure_exceptions_to_catch()
 
 
 def new_given_argspec(original_argspec, generator_kwargs):
@@ -692,7 +710,7 @@ def evaluate_test_data(self, data):
         except (
             HypothesisDeprecationWarning, FailedHealthCheck,
             StopTest,
-        ) + exceptions_to_reraise:
+        ) + EXCEPTIONS_TO_RERAISE:
             raise
         except Exception as e:
             escalate_hypothesis_internal_error()
diff --git a/hypothesis-python/src/hypothesis/stateful.py b/hypothesis-python/src/hypothesis/stateful.py
--- a/hypothesis-python/src/hypothesis/stateful.py
+++ b/hypothesis-python/src/hypothesis/stateful.py
@@ -34,7 +34,7 @@
 import attr
 
 import hypothesis.internal.conjecture.utils as cu
-from hypothesis.core import find
+from hypothesis.core import EXCEPTIONS_TO_FAIL, find
 from hypothesis.errors import Flaky, NoSuchExample, InvalidDefinition, \
     HypothesisException
 from hypothesis.control import BuildContext
@@ -77,7 +77,7 @@ def is_breaking_run(runner):
             return False
         except HypothesisException:
             raise
-        except Exception:
+        except EXCEPTIONS_TO_FAIL:
             verbose_report(traceback.format_exc)
             return True
     if settings is None:


Test patch:
diff --git a/hypothesis-python/tests/cover/test_skipping.py b/hypothesis-python/tests/cover/test_skipping.py
--- a/hypothesis-python/tests/cover/test_skipping.py
+++ b/hypothesis-python/tests/cover/test_skipping.py
@@ -22,12 +22,12 @@
 import pytest
 
 from hypothesis import given
-from hypothesis.core import exceptions_to_reraise
+from hypothesis.core import EXCEPTIONS_TO_RERAISE
 from tests.common.utils import capture_out
 from hypothesis.strategies import integers
 
 
-@pytest.mark.parametrize('skip_exception', exceptions_to_reraise)
+@pytest.mark.parametrize('skip_exception', EXCEPTIONS_TO_RERAISE)
 def test_no_falsifying_example_if_unittest_skip(skip_exception):
     """If a ``SkipTest`` exception is raised during a test, Hypothesis should
     not continue running the test and shrink process, nor should it print
diff --git a/hypothesis-python/tests/cover/test_stateful.py b/hypothesis-python/tests/cover/test_stateful.py
--- a/hypothesis-python/tests/cover/test_stateful.py
+++ b/hypothesis-python/tests/cover/test_stateful.py
@@ -21,6 +21,7 @@
 from collections import namedtuple, defaultdict
 
 import pytest
+from _pytest.outcomes import Failed, Skipped
 
 from hypothesis import assume
 from hypothesis import settings as Settings
@@ -958,3 +959,30 @@ class Foo(RuleBasedStateMachine):
         targets=(), function=lambda self: 2, arguments={}
     )
     assert len(Foo.initialize_rules()) == 2
+
+
+def test_steps_printed_despite_pytest_fail(capsys):
+    # Test for https://github.com/HypothesisWorks/hypothesis/issues/1372
+    class RaisesProblem(RuleBasedStateMachine):
+
+        @rule()
+        def oops(self):
+            pytest.fail()
+
+    with pytest.raises(Failed):
+        run_state_machine_as_test(RaisesProblem)
+    out, _ = capsys.readouterr()
+    assert 'state = RaisesProblem()\nstate.oops()\nstate.teardown()\n' == out
+
+
+def test_steps_not_printed_with_pytest_skip(capsys):
+    class RaisesProblem(RuleBasedStateMachine):
+
+        @rule()
+        def skip_whole_test(self):
+            pytest.skip()
+
+    with pytest.raises(Skipped):
+        run_state_machine_as_test(RaisesProblem)
+    out, _ = capsys.readouterr()
+    assert '' == out