Repository: HypothesisWorks/hypothesis

Issue:
Stateful: sometimes no steps are run?
I've noticed sometimes my stateful tests run zero rules - the rule-based state machine's `__init__` gets run and that's it. This counts against the 200 valid examples limit.

Here's a braindead stateful test to demonstrate: https://gist.github.com/Tinche/e1339718ff9d5b56e1c50773f3a7a8a2 (if it's a little quaint, it's because it's modeled after what I actually use).

```
> set -x HYPOTHESIS_VERBOSITY_LEVEL debug
> pytest t.py -s | grep "init" | wc -l
200
> pytest t.py -s | grep "Step #1:" | wc -l
157
```

The second number goes from ~130 to ~160. So a quarter of the tests just initialize the class. That's a little weird, and not that useful IMO. The default number of steps is 50, but I find very few runs actually get to 50. (5-15 out of 200, so 5%?)

I think it'd be better if all runs ran at least one rule, or at least if Hypothesis didn't count runs with no rules hit against `max_examples`, simply because I'm betting just running `__init__` doesn't actually test anything for most people.



Discussion:
#498 may help with this; we should check again once it's been merged.

Update: nope, looks like it didn't do much - I get 200 inits, 175 to 185 step one, and eventually 10 to 15 step fifty.  While it's clearly improved, I wouldn't call this issue closed.
<details>
  <summary>code with exact reporting</summary>

```python
from collections import Counter
from hypothesis.stateful import RuleBasedStateMachine, \
    run_state_machine_as_test, rule, precondition

cache = dict()  # id(test): num_of_steps(test)

class StatefulTest(RuleBasedStateMachine):
    def increment(self):
        if not id(self) in cache:
             cache[id(self)] = 0
        else:
            cache[id(self)] += 1

    def __init__(self):
        super().__init__()
        self.a = False
        self.b = False
        self.c = False
        self.increment()

    @rule()
    def open_a(self):
        self.a = True
        self.increment()

    @rule()
    @precondition(lambda self: self.a)
    def open_b(self):
        self.b = True
        self.increment()

    @rule()
    @precondition(lambda self: self.b)
    def open_c(self):
        self.c = True
        self.increment()

    @rule()
    @precondition(lambda self: self.c)
    def final(self):
        self.increment()

run_state_machine_as_test(StatefulTest)
Counter(cache.values())
```
</details>

As of Hypothesis 3.31.2 the situation is still basically the same, with 15-20% of runs ending before running any steps at all, and ~40% running two or fewer steps.  

As a solution, I suggest that we run *all* stateful tests until either `stateful_step_count` is reached or a bug is found.

The *other* problem is that without some memory, we are likely to retread the same sections of the tree of possible actions - especially when there are few transition rules.  This may be handled by coverage-guided tests, but I'm unsure how they interact with stateful testing in practice.  




Resolving patch:
diff --git a/src/hypothesis/internal/conjecture/utils.py b/src/hypothesis/internal/conjecture/utils.py
--- a/src/hypothesis/internal/conjecture/utils.py
+++ b/src/hypothesis/internal/conjecture/utils.py
@@ -60,6 +60,11 @@ def combine_labels(*labels):
 def integer_range(data, lower, upper, center=None):
     assert lower <= upper
     if lower == upper:
+        # Write a value even when this is trival so that when a bound depends
+        # on other values we don't suddenly disappear when the gap shrinks to
+        # zero - if that happens then often the data stream becomes misaligned
+        # and we fail to shrink in cases where we really should be able to.
+        data.write(hbytes([0]))
         return int(lower)
 
     if center is None:
diff --git a/src/hypothesis/stateful.py b/src/hypothesis/stateful.py
--- a/src/hypothesis/stateful.py
+++ b/src/hypothesis/stateful.py
@@ -40,8 +40,8 @@
 from hypothesis._settings import Verbosity
 from hypothesis._settings import settings as Settings
 from hypothesis.reporting import report, verbose_report, current_verbosity
-from hypothesis.strategies import just, lists, builds, one_of, runner, \
-    tuples, integers, fixed_dictionaries
+from hypothesis.strategies import just, one_of, runner, tuples, \
+    fixed_dictionaries
 from hypothesis.vendor.pretty import CUnicodeIO, RepresentationPrinter
 from hypothesis.internal.reflection import proxies, nicerepr
 from hypothesis.internal.conjecture.data import StopTest
@@ -221,25 +221,19 @@ def run(self, state_machine, print_steps=None):
             print_steps = current_verbosity() >= Verbosity.debug
         self.data.hypothesis_runner = state_machine
 
-        stopping_value = 1 - 1.0 / (1 + self.n_steps * 0.5)
+        should_continue = cu.many(
+            self.data, min_size=1, max_size=self.n_steps,
+            average_size=self.n_steps,
+        )
+
         try:
             state_machine.check_invariants()
 
-            steps = 0
-            while True:
-                if steps >= self.n_steps:
-                    stopping_value = 0
-                self.data.start_example(STATE_MACHINE_RUN_LABEL)
-                if not cu.biased_coin(self.data, stopping_value):
-                    self.data.stop_example()
-                    break
-                assert steps < self.n_steps
+            while should_continue.more():
                 value = self.data.draw(state_machine.steps())
-                steps += 1
                 if print_steps:
                     state_machine.print_step(value)
                 state_machine.execute_step(value)
-                self.data.stop_example()
                 state_machine.check_invariants()
         finally:
             state_machine.teardown()
@@ -275,8 +269,8 @@ def do_draw(self, data):
         bundle = machine.bundle(self.name)
         if not bundle:
             data.mark_invalid()
-        reference = bundle.pop()
-        bundle.insert(integer_range(data, 0, len(bundle)), reference)
+        i = integer_range(data, 0, len(bundle) - 1)
+        reference = bundle[i]
         return machine.names_to_values[reference.name]
 
 
@@ -413,12 +407,6 @@ def invariant_wrapper(*args, **kwargs):
     return accept
 
 
-@attr.s()
-class ShuffleBundle(object):
-    bundle = attr.ib()
-    swaps = attr.ib()
-
-
 class RuleBasedStateMachine(GenericStateMachine):
     """A RuleBasedStateMachine gives you a more structured way to define state
     machines.
@@ -542,18 +530,9 @@ def steps(self):
                 u'No progress can be made from state %r' % (self,)
             )
 
-        for name, bundle in self.bundles.items():
-            if len(bundle) > 1:
-                strategies.append(
-                    builds(
-                        ShuffleBundle, just(name),
-                        lists(integers(0, len(bundle) - 1))))
-
         return one_of(strategies)
 
     def print_step(self, step):
-        if isinstance(step, ShuffleBundle):
-            return
         rule, data = step
         data_repr = {}
         for k, v in data.items():
@@ -567,11 +546,6 @@ def print_step(self, step):
         ))
 
     def execute_step(self, step):
-        if isinstance(step, ShuffleBundle):
-            bundle = self.bundle(step.bundle)
-            for i in step.swaps:
-                bundle.insert(i, bundle.pop())
-            return
         rule, data = step
         data = dict(data)
         result = rule.function(self, **data)


Test patch:
diff --git a/tests/cover/test_conjecture_utils.py b/tests/cover/test_conjecture_utils.py
--- a/tests/cover/test_conjecture_utils.py
+++ b/tests/cover/test_conjecture_utils.py
@@ -28,8 +28,11 @@
 from hypothesis.internal.conjecture.data import ConjectureData
 
 
-def test_does_not_draw_data_for_empty_range():
-    assert cu.integer_range(ConjectureData.for_buffer(b''), 1, 1) == 1
+def test_does_draw_data_for_empty_range():
+    data = ConjectureData.for_buffer(b'\1')
+    assert cu.integer_range(data, 1, 1) == 1
+    data.freeze()
+    assert data.buffer == hbytes(b'\0')
 
 
 def test_uniform_float_shrinks_to_zero():
diff --git a/tests/cover/test_stateful.py b/tests/cover/test_stateful.py
--- a/tests/cover/test_stateful.py
+++ b/tests/cover/test_stateful.py
@@ -18,7 +18,7 @@
 from __future__ import division, print_function, absolute_import
 
 import inspect
-from collections import namedtuple
+from collections import namedtuple, defaultdict
 
 import pytest
 
@@ -679,3 +679,68 @@ def test_foo(self):
 
     with pytest.raises(ValueError):
         run_state_machine_as_test(BadPrecondition)
+
+
+def test_always_runs_at_least_one_step():
+    class CountSteps(RuleBasedStateMachine):
+        def __init__(self):
+            super(CountSteps, self).__init__()
+            self.count = 0
+
+        @rule()
+        def do_something(self):
+            self.count += 1
+
+        def teardown(self):
+            assert self.count > 0
+
+    run_state_machine_as_test(CountSteps)
+
+
+def test_removes_needless_steps():
+    """Regression test from an example based on
+    tests/nocover/test_database_agreement.py, but without the expensive bits.
+    Comparing two database implementations in which deletion is broken, so as
+    soon as a key/value pair is successfully deleted the test will now fail if
+    you ever check that key.
+
+    The main interesting feature of this is that it has a lot of
+    opportunities to generate keys and values before it actually fails,
+    but will still fail with very high probability.
+    """
+    class IncorrectDeletion(RuleBasedStateMachine):
+        def __init__(self):
+            super(IncorrectDeletion, self).__init__()
+            self.__saved = defaultdict(set)
+            self.__deleted = defaultdict(set)
+
+        keys = Bundle('keys')
+        values = Bundle('values')
+
+        @rule(target=keys, k=binary())
+        def k(self, k):
+            return k
+
+        @rule(target=values, v=binary())
+        def v(self, v):
+            return v
+
+        @rule(k=keys, v=values)
+        def save(self, k, v):
+            self.__saved[k].add(v)
+
+        @rule(k=keys, v=values)
+        def delete(self, k, v):
+            if v in self.__saved[k]:
+                self.__deleted[k].add(v)
+
+        @rule(k=keys)
+        def values_agree(self, k):
+            assert not self.__deleted[k]
+
+    with capture_out() as o:
+        with pytest.raises(AssertionError):
+            run_state_machine_as_test(IncorrectDeletion)
+
+    assert o.getvalue().count(' = k(') == 1
+    assert o.getvalue().count(' = v(') == 1