Repository: HypothesisWorks/hypothesis

Issue:
Easier way to check `RuleBasedStateMachine` invariants only when fully initialized
When using [stateful testing](https://hypothesis.readthedocs.io/en/latest/stateful.html), `@invariant()` methods are currently checked between every `@initialize()` method as well as between `@rule()` methods - which is awkward if your invariants require the machine to be initialized (see #2860 for discussion).

Instead of requiring users to use `@precondition()` decorators and track the "fully initialized" state themselves, I propose that we should add a new argument to the invariant decorator:

```python
def invariant(*, check_before_initialized=True):
```

to make disabling this behaviour on a per-invariant basis more convenient.

Contra #2860, I think we should retain the existing (always check invariants) behaviour by default - our [API style guide](https://github.com/HypothesisWorks/hypothesis/blob/master/guides/api-style.rst) emphasises avoiding things that can lead to "unexpectedly weak tests".  I'm open to suggestions for a better parameter name, with the goal that it should be very obvious than `@invariant(argname=True)` is checked both before and after while `False` is only checked after.


Discussion:
If anything I’m leaning the other way at the moment.

I’m struggling to imagine a realistic use case for checking invariants during partial init. Even in cases where it *would* be useful, we’d still check invariants after all initializers complete successfully, so the worst case is that we fail slightly later than we otherwise would have.
After #2883 is merged, I'll open a PR for this issue from https://github.com/Zac-HD/hypothesis/compare/multiple-preconditions...init-then-invariants.

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/stateful.py b/hypothesis-python/src/hypothesis/stateful.py
--- a/hypothesis-python/src/hypothesis/stateful.py
+++ b/hypothesis-python/src/hypothesis/stateful.py
@@ -333,6 +333,8 @@ def _add_result_to_targets(self, targets, result):
 
     def check_invariants(self):
         for invar in self.invariants():
+            if self._initialize_rules_to_run and not invar.check_during_init:
+                continue
             if not all(precond(self) for precond in invar.preconditions):
                 continue
             invar.function(self)
@@ -670,9 +672,10 @@ def precondition_wrapper(*args, **kwargs):
 class Invariant:
     function = attr.ib()
     preconditions = attr.ib()
+    check_during_init = attr.ib()
 
 
-def invariant():
+def invariant(*, check_during_init=False):
     """Decorator to apply an invariant for rules in a RuleBasedStateMachine.
     The decorated function will be run after every rule and can raise an
     exception to indicate failed invariants.
@@ -685,7 +688,13 @@ class MyTestMachine(RuleBasedStateMachine):
             @invariant()
             def is_nonzero(self):
                 assert self.state != 0
+
+    By default, invariants are only checked after all
+    :func:`@initialize() <hypothesis.stateful.initialize>` rules have been run.
+    Pass ``check_during_init=True`` for invariants which can also be checked
+    during initialization.
     """
+    check_type(bool, check_during_init, "check_during_init")
 
     def accept(f):
         existing_invariant = getattr(f, INVARIANT_MARKER, None)
@@ -695,13 +704,17 @@ def accept(f):
                 Settings.default,
             )
         preconditions = getattr(f, PRECONDITIONS_MARKER, ())
-        rule = Invariant(function=f, preconditions=preconditions)
+        invar = Invariant(
+            function=f,
+            preconditions=preconditions,
+            check_during_init=check_during_init,
+        )
 
         @proxies(f)
         def invariant_wrapper(*args, **kwargs):
             return f(*args, **kwargs)
 
-        setattr(invariant_wrapper, INVARIANT_MARKER, rule)
+        setattr(invariant_wrapper, INVARIANT_MARKER, invar)
         return invariant_wrapper
 
     return accept


Test patch:
diff --git a/hypothesis-python/tests/cover/test_stateful.py b/hypothesis-python/tests/cover/test_stateful.py
--- a/hypothesis-python/tests/cover/test_stateful.py
+++ b/hypothesis-python/tests/cover/test_stateful.py
@@ -720,7 +720,7 @@ def test_foo(self):
         run_state_machine_as_test(BadPrecondition)
 
 
-def test_invariant_checks_initial_state():
+def test_invariant_checks_initial_state_if_no_initialize_rules():
     """Invariants are checked before any rules run."""
 
     class BadPrecondition(RuleBasedStateMachine):
@@ -1190,3 +1190,41 @@ def bad_invariant_d(self):
             raise AssertionError("This invariant runs, even though it shouldn't.")
 
     run_state_machine_as_test(MultiplePreconditionMachine)
+
+
+class TrickyInitMachine(RuleBasedStateMachine):
+    @initialize()
+    def init_a(self):
+        self.a = 0
+
+    @rule()
+    def inc(self):
+        self.a += 1
+
+    @invariant()
+    def check_a_positive(self):
+        # This will fail if run before the init_a method, but without
+        # @invariant(check_during_init=True) it will only run afterwards.
+        assert self.a >= 0
+
+
+def test_invariants_are_checked_after_init_steps():
+    run_state_machine_as_test(TrickyInitMachine)
+
+
+def test_invariants_can_be_checked_during_init_steps():
+    class UndefinedMachine(TrickyInitMachine):
+        @invariant(check_during_init=True)
+        def check_a_defined(self):
+            # This will fail because `a` is undefined before the init rule.
+            self.a
+
+    with pytest.raises(AttributeError):
+        run_state_machine_as_test(UndefinedMachine)
+
+
+def test_check_during_init_must_be_boolean():
+    invariant(check_during_init=False)
+    invariant(check_during_init=True)
+    with pytest.raises(InvalidArgument):
+        invariant(check_during_init="not a bool")