Repository: HypothesisWorks/hypothesis

Issue:
Internal assertion error thrown in builds() strategy
Hypothesis 3.55.4, Python 3.6.1. The following test reliably hits an internal assertion:

```python
import hypothesis
from hypothesis.strategies import (
    builds, fixed_dictionaries, lists, sampled_from, sets, text
)


strategy = builds(
    lambda x, y: dict((list(x.items()) + list(y.items()))),
    fixed_dictionaries({'0': text()}),
    builds(
        lambda dictionary, keys: {key: dictionary[key] for key in keys},
        fixed_dictionaries({
            '1': lists(elements=sampled_from(['a']))
        }),
        sets(elements=sampled_from(['1']))
    )
)

@hypothesis.given(strategy)
def test_false_is_false(params):
    hypothesis.assume(params.get('0') not in ('', '\x00'))
    assert False


test_false_is_false()
```

```console
$ python minimal.py
You can add @seed(271280762001646821722234716191843127890) to this test to reproduce this failure.
Traceback (most recent call last):
  File "minimal.py", line 28, in <module>
    test_false_is_false()
  File "minimal.py", line 23, in test_false_is_false
    def test_false_is_false(params):
  File "/Users/chana/.virtualenvs/tempenv-598466573bc3/lib/python3.6/site-packages/hypothesis/core.py", line 1054, in wrapped_test
    state.run()
  File "/Users/chana/.virtualenvs/tempenv-598466573bc3/lib/python3.6/site-packages/hypothesis/core.py", line 730, in run
    runner.run()
  File "/Users/chana/.virtualenvs/tempenv-598466573bc3/lib/python3.6/site-packages/hypothesis/internal/conjecture/engine.py", line 435, in run
    self._run()
  File "/Users/chana/.virtualenvs/tempenv-598466573bc3/lib/python3.6/site-packages/hypothesis/internal/conjecture/engine.py", line 771, in _run
    self.shrink_interesting_examples()
  File "/Users/chana/.virtualenvs/tempenv-598466573bc3/lib/python3.6/site-packages/hypothesis/internal/conjecture/engine.py", line 813, in shrink_interesting_examples
    self.shrink(example, predicate)
  File "/Users/chana/.virtualenvs/tempenv-598466573bc3/lib/python3.6/site-packages/hypothesis/internal/conjecture/engine.py", line 849, in shrink
    s.shrink()
  File "/Users/chana/.virtualenvs/tempenv-598466573bc3/lib/python3.6/site-packages/hypothesis/internal/conjecture/engine.py", line 1274, in shrink
    self.greedy_shrink()
  File "/Users/chana/.virtualenvs/tempenv-598466573bc3/lib/python3.6/site-packages/hypothesis/internal/conjecture/engine.py", line 1295, in greedy_shrink
    self.remove_discarded()
  File "/Users/chana/.virtualenvs/tempenv-598466573bc3/lib/python3.6/site-packages/hypothesis/internal/conjecture/engine.py", line 1787, in remove_discarded
    self.__discarding_failed = not self.incorporate_new_buffer(attempt)
  File "/Users/chana/.virtualenvs/tempenv-598466573bc3/lib/python3.6/site-packages/hypothesis/internal/conjecture/engine.py", line 1221, in incorporate_new_buffer
    assert sort_key(buffer) < sort_key(self.shrink_target.buffer)
AssertionError
```

How did I find this? I was testing the API at work with https://github.com/olipratt/swagger-conformance, and on my second run I hit this internal assertion. Quite a big of digging later, this strategy popped out.

It’s _incredibly_ fragile. I tried quite a few changes to simplify the example, all of which led it to finding the more boring `assert False`.

PS Having useful repr’s for strategies was really useful here – it allowed me to strip away a whole bunch of the Swagger-parsing code, and trim down to a pure Hypothesis bug.


Discussion:
❤️ ❤️ ❤️ ❤️ ❤️ 

I've been aware that it was possible to trigger this assertion for a while, but nobody has been able to come up with a good repro for it (I've even seen it once myself and then immediately lost it and couldn't reproduce it).

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/data.py b/hypothesis-python/src/hypothesis/internal/conjecture/data.py
--- a/hypothesis-python/src/hypothesis/internal/conjecture/data.py
+++ b/hypothesis-python/src/hypothesis/internal/conjecture/data.py
@@ -201,6 +201,15 @@ def stop_example(self, discard=False):
         k = self.example_stack.pop()
         ex = self.examples[k]
         ex.end = self.index
+
+        # We don't want to count empty examples as discards even if the flag
+        # says we should. This leads to situations like
+        # https://github.com/HypothesisWorks/hypothesis/issues/1230
+        # where it can look like we should discard data but there's nothing
+        # useful for us to do.
+        if self.index == ex.start:
+            discard = False
+
         ex.discarded = discard
 
         if discard:
diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py
--- a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py
+++ b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py
@@ -1773,6 +1773,8 @@ def remove_discarded(self):
             ):
                 discarded.append((ex.start, ex.end))
 
+        assert discarded
+
         attempt = bytearray(self.shrink_target.buffer)
         for u, v in reversed(discarded):
             del attempt[u:v]


Test patch:
diff --git a/hypothesis-python/tests/cover/test_conjecture_test_data.py b/hypothesis-python/tests/cover/test_conjecture_test_data.py
--- a/hypothesis-python/tests/cover/test_conjecture_test_data.py
+++ b/hypothesis-python/tests/cover/test_conjecture_test_data.py
@@ -118,3 +118,11 @@ def test_does_not_double_freeze_in_interval_close():
         x.draw(BigStrategy())
     assert x.frozen
     assert not any(eg.end is None for eg in x.examples)
+
+
+def test_empty_discards_do_not_count():
+    x = ConjectureData.for_buffer(b'')
+    x.start_example(label=1)
+    x.stop_example(discard=True)
+    x.freeze()
+    assert not x.has_discards
diff --git a/hypothesis-python/tests/cover/test_regressions.py b/hypothesis-python/tests/cover/test_regressions.py
--- a/hypothesis-python/tests/cover/test_regressions.py
+++ b/hypothesis-python/tests/cover/test_regressions.py
@@ -19,7 +19,7 @@
 
 import pytest
 
-from hypothesis import Verbosity, given, settings
+from hypothesis import Verbosity, seed, given, assume, settings
 from hypothesis import strategies as st
 
 
@@ -77,3 +77,27 @@ def test_foo_spec(bar, inp):
     test_foo_spec(Bar())
     test_foo_spec(Mock(Bar))
     test_foo_spec(Mock())
+
+
+def test_regression_issue_1230():
+    strategy = st.builds(
+        lambda x, y: dict((list(x.items()) + list(y.items()))),
+        st.fixed_dictionaries({'0': st.text()}),
+        st.builds(
+            lambda dictionary, keys: {key: dictionary[key] for key in keys},
+            st.fixed_dictionaries({
+                '1': st.lists(elements=st.sampled_from(['a']))
+            }),
+            st.sets(elements=st.sampled_from(['1']))
+        )
+    )
+
+    @seed(81581571036124932593143257836081491416)
+    @settings(database=None)
+    @given(strategy)
+    def test_false_is_false(params):
+        assume(params.get('0') not in ('', '\x00'))
+        raise ValueError()
+
+    with pytest.raises(ValueError):
+        test_false_is_false()