Repository: HypothesisWorks/hypothesis
Pull request: Fix stateful _print_step for invariants
fix #3185

Problem statement:
stateful _print_step doesn't display invariant
minimal example:
```python
from hypothesis.stateful import *

class StateMachine(RuleBasedStateMachine):
    @invariant()
    def invariant1(self):
        assert False

    @rule()
    def rule1(self):
        assert True

run_state_machine_as_test(StateMachine)
```

Running this code gives:
```
Falsifying example:
state = StateMachine()
state.teardown()
Traceback (most recent call last):
  File "test42.py", line 33, in <module>
    run_state_machine_as_test(StateMachine)
  File "/home/emmanuel/projects/hypothesis-trio/venv/lib/python3.8/site-packages/hypothesis/stateful.py", line 75, in run_state_machine_as_test
    def run_state_machine_as_test(state_machine_factory, *, settings=None):
  File "/home/emmanuel/projects/hypothesis-trio/venv/lib/python3.8/site-packages/hypothesis/internal/reflection.py", line 655, in accept
    return func(*bound.args, **bound.kwargs)
  File "/home/emmanuel/projects/hypothesis-trio/venv/lib/python3.8/site-packages/hypothesis/stateful.py", line 200, in run_state_machine_as_test
    run_state_machine(state_machine_factory)
  File "/home/emmanuel/projects/hypothesis-trio/venv/lib/python3.8/site-packages/hypothesis/stateful.py", line 92, in run_state_machine
    @given(st.data())
  File "/home/emmanuel/projects/hypothesis-trio/venv/lib/python3.8/site-packages/hypothesis/core.py", line 1164, in wrapped_test
    raise the_error_hypothesis_found
  File "test42.py", line 27, in invariant1
    assert False
AssertionError
```

However the falsifying example displayed doesn't trigger the error.
The falsifying should be:
```python
state = StateMachine()
state.invariant1()
state.teardown()
```

Hints from issue discussion:

Commit message: Add RELEASE.rst for issue #3185

Patch to review:
diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst
new file mode 100644
index 0000000000..fb53656140
--- /dev/null
+++ b/hypothesis-python/RELEASE.rst
@@ -0,0 +1,3 @@
+RELEASE_TYPE: patch
+
+This patch fix invariants display in stateful falsifying examples (:issue:`3185`).
\ No newline at end of file
diff --git a/hypothesis-python/src/hypothesis/stateful.py b/hypothesis-python/src/hypothesis/stateful.py
index ea39977dfa..0fbb517333 100644
--- a/hypothesis-python/src/hypothesis/stateful.py
+++ b/hypothesis-python/src/hypothesis/stateful.py
@@ -122,7 +122,11 @@ def run_state_machine(factory, data):
         try:
             if print_steps:
                 report(f"state = {machine.__class__.__name__}()")
-            machine.check_invariants(settings)
+            try:
+                machine.check_invariants(settings)
+            finally:
+                if print_steps:
+                    machine._print_invariants()
             max_steps = settings.stateful_step_count
             steps_run = 0
 
@@ -204,7 +208,11 @@ def run_state_machine(factory, data):
                         # If it does, and the result is a 'MultipleResult',
                         # then 'print_step' prints a multi-variable assignment.
                         machine._print_step(rule, data_to_print, result)
-                machine.check_invariants(settings)
+                try:
+                    machine.check_invariants(settings)
+                finally:
+                    if print_steps:
+                        machine._print_invariants()
                 cd.stop_example()
         finally:
             if print_steps:
@@ -356,6 +364,15 @@ def _print_step(self, rule, data, result):
             )
         )
 
+    def _print_invariants(self):
+        for invar in self.invariants():
+            if self._initialize_rules_to_run and not invar.check_during_init:
+                continue
+            if not all(precond(self) for precond in invar.preconditions):
+                continue
+            report("state.{}()".format(invar.function.__name__)
+            )
+
     def _add_result_to_targets(self, targets, result):
         name = self._new_name()
         self.__printer.singleton_pprinters.setdefault(
diff --git a/hypothesis-python/tests/cover/test_stateful.py b/hypothesis-python/tests/cover/test_stateful.py
index e2ea83a88d..a353a4611d 100644
--- a/hypothesis-python/tests/cover/test_stateful.py
+++ b/hypothesis-python/tests/cover/test_stateful.py
@@ -597,6 +597,90 @@ def test_foo(self):
         run_state_machine_as_test(BadPrecondition)
 
 
+def test_invariant_failling_present_in_falsifying_example():
+    class BadInvariant(RuleBasedStateMachine):
+        @initialize()
+        def initialize_1(self):
+            pass
+
+        @invariant()
+        def invariant_1(self):
+            raise ValueError()
+
+        @rule()
+        def rule_1(self):
+            pass
+
+    with capture_out() as o:
+        with pytest.raises(ValueError):
+            run_state_machine_as_test(BadInvariant)
+
+    result = o.getvalue()
+    assert (
+        result
+        == """\
+Falsifying example:
+state = BadInvariant()
+state.initialize_1()
+state.invariant_1()
+state.teardown()
+"""
+    )
+
+
+def test_invariant_present_in_falsifying_example():
+    class BadRuleWithGoodInvariants(RuleBasedStateMachine):
+        def __init__(self):
+            super().__init__()
+            self.num = 0
+
+        @initialize()
+        def initialize_1(self):
+            pass
+
+        @invariant(check_during_init=True)
+        def invariant_1(self):
+            pass
+
+        @invariant(check_during_init=False)
+        def invariant_2(self):
+            pass
+
+        @precondition(lambda self: self.num > 0)
+        @invariant()
+        def invariant_3(self):
+            pass
+
+        @rule()
+        def rule_1(self):
+            self.num += 1
+            if self.num == 2:
+                raise ValueError()
+
+    with capture_out() as o:
+        with pytest.raises(ValueError):
+            run_state_machine_as_test(BadRuleWithGoodInvariants)
+
+    result = o.getvalue()
+    assert (
+        result
+        == """\
+Falsifying example:
+state = BadRuleWithGoodInvariants()
+state.invariant_1()
+state.initialize_1()
+state.invariant_1()
+state.invariant_2()
+state.rule_1()
+state.invariant_1()
+state.invariant_2()
+state.invariant_3()
+state.rule_1()
+state.teardown()
+"""
+    )
+
+
 def test_always_runs_at_least_one_step():
     class CountSteps(RuleBasedStateMachine):
         def __init__(self):


Code review comments:
diff_hunk: @@ -204,7 +208,11 @@ def run_state_machine(factory, data):
                         # If it does, and the result is a 'MultipleResult',
                         # then 'print_step' prints a multi-variable assignment.
                         machine._print_step(rule, data_to_print, result)
-                machine.check_invariants(settings)
+                try:
+                    machine.check_invariants(settings)
+                finally:
+                    if print_steps:
+                        machine._print_invariants()
path: hypothesis-python/src/hypothesis/stateful.py
text: @user1:
Instead of adding a new `_print_invariants()` method, I'd rather just add a `report()` line inside `check_invariants()` - just before `result = invar.function(self)`, so that the precondition checks etc. are all handled in one place.

That'll involve duplicating the `print_steps` condition, but I'm OK with that.