Repository: HypothesisWorks/hypothesis

Issue:
Off-by-one in step generation for block programs
This is the code that generates indices to run each block program at:

https://github.com/HypothesisWorks/hypothesis/blob/931796887bfb2f498d5e017215a9c755670efa1c/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py#L1322-L1326

For example, if our shrink target has 100 blocks (indices 0–99), and the block program has 2 steps, then we'll generate indices `hrange(98)`, i.e. 0–97.

This means that we will never try running the block program against at block 98, covering blocks [98,&nbsp;99]. In theory, that's a bug.

For `block_program("XX")` this isn't a problem, because deleting a suffix of the current target is guaranteed to be useless (and will be caught by the tree cache anyway).

For `block_program("-XX")` we could conceivably miss a legal shrink, though I'm not sure how likely that is in practice.


Discussion:

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/data.py b/hypothesis-python/src/hypothesis/internal/conjecture/data.py
--- a/hypothesis-python/src/hypothesis/internal/conjecture/data.py
+++ b/hypothesis-python/src/hypothesis/internal/conjecture/data.py
@@ -99,6 +99,9 @@ class Example(object):
     start = attr.ib()
     end = attr.ib(default=None)
 
+    # Index of example in parent's children, or None if this is the root.
+    child_index = attr.ib(default=None)
+
     # An example is "trivial" if it only contains forced bytes and zero bytes.
     # All examples start out as trivial, and then get marked non-trivial when
     # we see a byte that is neither forced nor zero.
@@ -223,7 +226,9 @@ def calc_examples(self):
                 examples.append(ex)
                 if example_stack:
                     p = example_stack[-1]
-                    examples[p].children.append(ex)
+                    children = examples[p].children
+                    ex.child_index = len(children)
+                    children.append(ex)
                 example_stack.append(i)
     for ex in examples:
         assert ex.end is not None
diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py b/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py
--- a/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py
+++ b/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py
@@ -538,8 +538,7 @@ def fixate_shrink_passes(self, passes):
         while initial is not self.shrink_target:
             initial = self.shrink_target
             for sp in passes:
-                if sp.arguments:
-                    sp.runs += 1
+                sp.runs += 1
 
             passes_with_steps = [
                 (sp, step) for sp in passes for step in sp.generate_steps()
@@ -610,11 +609,17 @@ def calculate_descents(self):
                         hi = mid
                     else:
                         lo = mid
-                result.extend([(ex, ls[j]) for j in hrange(i + 1, hi)])
+                id1 = self.stable_identifier_for_example(ex)
+                result.extend(
+                    [
+                        (id1, self.stable_identifier_for_example(ls[j]))
+                        for j in hrange(i + 1, hi)
+                    ]
+                )
         return result
 
     @defines_shrink_pass(calculate_descents)
-    def pass_to_descendant(self, ancestor, descendant):
+    def pass_to_descendant(self, ancestor_id, descendant_id):
         """Attempt to replace each example with a descendant example.
 
         This is designed to deal with strategies that call themselves
@@ -633,6 +638,12 @@ def pass_to_descendant(self, ancestor, descendant):
         as possible.
         """
 
+        ancestor = self.example_for_stable_identifier(ancestor_id)
+        descendant = self.example_for_stable_identifier(descendant_id)
+        if descendant is None:
+            return
+        assert ancestor is not None
+
         self.incorporate_new_buffer(
             self.buffer[: ancestor.start]
             + self.buffer[descendant.start : descendant.end]
@@ -919,36 +930,11 @@ def remove_discarded(self):
         return True
 
     @derived_value
-    def endpoints_by_depth(self):
-        """Defines a series of increasingly fine grained boundaries
-        to partition the current buffer, based on the depth of examples.
-        Each element of the result is the set of endpoints of examples
-        less than or equal to some depth (less than or equal to account
-        for the fact that there might be blocks below that depth and if
-        we ignore those we'll get bad boundaries).
-
-        This is primarily useful for adaptive_example_deletion."""
-        endpoints_at_depth = defaultdict(set)
-        max_depth = 0
-        for ex in self.examples:
-            endpoints_at_depth[ex.depth].add(ex.start)
-            endpoints_at_depth[ex.depth].add(ex.end)
-            max_depth = max(max_depth, ex.depth)
-        distinct_partitions = [{0, len(self.buffer)}]
-        for d in hrange(max_depth + 1):
-            prev = distinct_partitions[-1]
-            if not endpoints_at_depth[d].issubset(prev):
-                distinct_partitions.append(prev | endpoints_at_depth[d])
-        return [sorted(endpoints) for endpoints in distinct_partitions[1:]]
+    def stable_identifier_arguments(self):
+        return [(self.stable_identifier_for_example(ex),) for ex in self.examples]
 
-    @defines_shrink_pass(
-        lambda self: [
-            (i, j)
-            for i, ls in enumerate(self.endpoints_by_depth)
-            for j in hrange(len(ls))
-        ]
-    )
-    def adaptive_example_deletion(self, i, j):
+    @defines_shrink_pass(lambda self: self.stable_identifier_arguments)
+    def adaptive_example_deletion(self, example_id):
         """Attempts to delete every example from the test case.
 
         That is, it is logically equivalent to trying ``self.buffer[:ex.start] +
@@ -956,18 +942,32 @@ def adaptive_example_deletion(self, i, j):
         examples are tried is randomized, and when deletion is successful it
         will attempt to adapt to delete more than one example at a time.
         """
-        partition = self.endpoints_by_depth[i]
-        # No point in trying to delete the last element because that will always
-        # give us a prefix.
-        if j >= len(partition) - 1:
+        example = self.example_for_stable_identifier(example_id)
+
+        if example is None or not self.incorporate_new_buffer(
+            self.buffer[: example.start] + self.buffer[example.end :]
+        ):
             return
 
+        # If we successfully deleted one example there may be a useful
+        # deletable region around here.
+
+        original = self.shrink_target
+        endpoints = set()
+        for ex in original.examples:
+            if ex.depth <= example.depth:
+                endpoints.add(ex.start)
+                endpoints.add(ex.end)
+
+        partition = sorted(endpoints)
+        j = partition.index(example.start)
+
         def delete_region(a, b):
             assert a <= j <= b
             if a < 0 or b >= len(partition) - 1:
                 return False
             return self.consider_new_buffer(
-                self.buffer[: partition[a]] + self.buffer[partition[b] :]
+                original.buffer[: partition[a]] + original.buffer[partition[b] :]
             )
 
         to_right = find_integer(lambda n: delete_region(j, j + n))
@@ -975,8 +975,7 @@ def delete_region(a, b):
         if to_right > 0:
             find_integer(lambda n: delete_region(j - n, j + to_right))
 
-    def try_zero_example(self, i):
-        ex = self.examples[i]
+    def try_zero_example(self, ex):
         u = ex.start
         v = ex.end
         attempt = self.cached_test_function(
@@ -994,22 +993,25 @@ def try_zero_example(self, i):
                 self.incorporate_new_buffer(
                     self.buffer[:u] + hbytes(used) + self.buffer[v:]
                 )
-        return self.examples[i].trivial
+        return self.examples[ex.index].trivial
 
-    @defines_shrink_pass(lambda self: [(e.index,) for e in self.examples])
-    def zero_examples(self, ex_index):
+    @defines_shrink_pass(lambda self: self.stable_identifier_arguments)
+    def zero_examples(self, ex_id):
+        """Attempt to replace each example with a minimal version of itself."""
+        ex = self.example_for_stable_identifier(ex_id)
         """Attempt to replace each example with a minimal version of itself."""
         # If the example is already trivial, assume there's nothing to do here.
         # We could attempt to use it as an adaptive replacement for other
         # similar examples, but that seems to be ineffective, resulting mostly
         # in redundant work rather than helping.
-        if self.examples[ex_index].trivial:
+        if ex is None or ex.trivial:
             return
 
-        if not self.try_zero_example(ex_index):
+        if not self.try_zero_example(ex):
             return
 
-        ex = self.examples[ex_index]
+        # If we zeroed the example we need to get the new one that replaced it.
+        ex = self.examples[ex.index]
 
         original = self.shrink_target
         group = self.examples_by_label[ex.label]
@@ -1045,7 +1047,8 @@ def can_zero(a, b):
 
         expand_region(can_zero, start, end)
 
-    def duplicated_block_suffixes(self):
+    @derived_value
+    def blocks_by_non_zero_suffix(self):
         """Returns a list of blocks grouped by their non-zero suffix,
         as a list of (suffix, indices) pairs, skipping all groupings
         where there is only one index.
@@ -1057,16 +1060,12 @@ def duplicated_block_suffixes(self):
             duplicates[non_zero_suffix(self.buffer[block.start : block.end])].append(
                 block.index
             )
-        return sorted(
-            [
-                (suffix, indices)
-                for suffix, indices in duplicates.items()
-                if len(suffix) > 0 and len(indices) > 1
-            ]
-        )
+        return duplicates
 
-    @defines_shrink_pass(duplicated_block_suffixes)
-    def minimize_duplicated_blocks(self, block, targets):
+    @defines_shrink_pass(
+        lambda self: [(b,) for b in sorted(self.blocks_by_non_zero_suffix)]
+    )
+    def minimize_duplicated_blocks(self, block):
         """Find blocks that have been duplicated in multiple places and attempt
         to minimize all of the duplicates simultaneously.
 
@@ -1086,6 +1085,9 @@ def minimize_duplicated_blocks(self, block, targets):
         of the blocks doesn't matter very much because it allows us to replace
         more values at once.
         """
+        targets = self.blocks_by_non_zero_suffix[block]
+        if len(targets) <= 1:
+            return
         Lexical.shrink(
             block,
             lambda b: self.try_shrinking_blocks(targets, b),
@@ -1093,18 +1095,8 @@ def minimize_duplicated_blocks(self, block, targets):
             full=False,
         )
 
-    @defines_shrink_pass(
-        lambda self: [
-            (ex,)
-            for ex in self.shrink_target.examples
-            if (
-                ex.label == DRAW_FLOAT_LABEL
-                and len(ex.children) == 2
-                and ex.children[0].length == 8
-            )
-        ]
-    )
-    def minimize_floats(self, ex):
+    @defines_shrink_pass(lambda self: self.stable_identifier_arguments)
+    def minimize_floats(self, ex_id):
         """Some shrinks that we employ that only really make sense for our
         specific floating point encoding that are hard to discover from any
         sort of reasonable general principle. This allows us to make
@@ -1120,6 +1112,13 @@ def minimize_floats(self, ex):
         transformations to make, they just don't necessarily correspond to
         anything particularly meaningful for non-float values.
         """
+        ex = self.example_for_stable_identifier(ex_id)
+        if ex is None or not (
+            ex.label == DRAW_FLOAT_LABEL
+            and len(ex.children) == 2
+            and ex.children[0].length == 8
+        ):
+            return
         u = ex.children[0].start
         v = ex.children[0].end
         buf = self.shrink_target.buffer
@@ -1137,8 +1136,8 @@ def minimize_floats(self, ex):
                 random=self.random,
             )
 
-    @defines_shrink_pass(lambda self: [(b,) for b in self.blocks])
-    def minimize_individual_blocks(self, block):
+    @defines_shrink_pass(lambda self: [(i,) for i in hrange(len(self.blocks))])
+    def minimize_individual_blocks(self, i):
         """Attempt to minimize each block in sequence.
 
         This is the pass that ensures that e.g. each integer we draw is a
@@ -1149,6 +1148,9 @@ def minimize_individual_blocks(self, block):
 
         then in our shrunk example, x = 10 rather than say 97.
         """
+        if i >= len(self.blocks):
+            return
+        block = self.blocks[i]
         u, v = block.bounds
         i = block.index
         Lexical.shrink(
@@ -1160,14 +1162,14 @@ def minimize_individual_blocks(self, block):
 
     @defines_shrink_pass(
         lambda self: [
-            (block, ex)
-            for block in self.blocks
-            if self.is_shrinking_block(block.index)
+            (i, self.stable_identifier_for_example(ex))
+            for i in hrange(len(self.blocks))
+            if self.is_shrinking_block(i)
             for ex in self.examples
-            if ex.start >= block.end and ex.length > 0
+            if ex.length > 0 and ex.start >= self.blocks[i].end
         ]
     )
-    def example_deletion_with_block_lowering(self, block, ex):
+    def example_deletion_with_block_lowering(self, block_i, ex_id):
         """Sometimes we get stuck where there is data that we could easily
         delete, but it changes the number of examples generated, so we have to
         change that at the same time.
@@ -1181,6 +1183,12 @@ def example_deletion_with_block_lowering(self, block, ex):
         example and every block not inside that example it tries deleting the
         example and modifying the block's value by one in either direction.
         """
+        if block_i >= len(self.blocks):
+            return
+        block = self.blocks[block_i]
+        ex = self.example_for_stable_identifier(ex_id)
+        if ex is None:
+            return
         u, v = block.bounds
 
         n = int_from_bytes(self.shrink_target.buffer[u:v])
@@ -1192,19 +1200,14 @@ def example_deletion_with_block_lowering(self, block, ex):
         del buf[ex.start : ex.end]
         self.incorporate_new_buffer(buf)
 
-    @derived_value
-    def groups_for_reordering(self):
-        result = defaultdict(list)
-        for e in self.examples:
-            result[(e.parent, e.label)].append(e)
-        groups = [v for v in result.values() if len(v) > 1]
-        groups.sort(key=lambda ls: ls[0].index)
-        return groups
-
     @defines_shrink_pass(
-        lambda self: [(i,) for i in hrange(len(self.groups_for_reordering))]
+        lambda self: [
+            (self.stable_identifier_for_example(e), l)
+            for e in self.examples
+            for l in sorted({c.label for c in e.children}, key=str)
+        ]
     )
-    def reorder_examples(self, i):
+    def reorder_examples(self, ex_id, label):
         """This pass allows us to reorder the children of each example.
 
         For example, consider the following:
@@ -1222,7 +1225,13 @@ def test_not_equal(x, y):
         ``x=""``, ``y="0"``, or the other way around. With reordering it will
         reliably fail with ``x=""``, ``y="0"``.
         """
-        group = self.groups_for_reordering[i]
+        ex = self.example_for_stable_identifier(ex_id)
+        if ex is None:
+            return
+        group = [c for c in ex.children if c.label == label]
+        if len(group) <= 1:
+            return
+
         st = self.shrink_target
         pieces = [st.buffer[ex.start : ex.end] for ex in group]
         endpoints = [(ex.start, ex.end) for ex in group]
@@ -1309,6 +1318,63 @@ def should_replace_byte(b):
             else:
                 lo = mid
 
+    @derived_value
+    def example_to_stable_identifier_cache(self):
+        return []
+
+    @derived_value
+    def stable_identifier_to_example_cache(self):
+        return {}
+
+    def stable_identifier_for_example(self, example):
+        """A stable identifier is one that we can reasonably reliably
+        count on referring to "logically the same" example between two
+        different test runs. It is currently represented as a path from
+        the root."""
+
+        i = example.index
+        n = len(self.example_to_stable_identifier_cache)
+        if i >= n:
+            self.example_to_stable_identifier_cache.extend([None] * (i - n + 1))
+            assert i < len(self.example_to_stable_identifier_cache)
+        result = self.example_to_stable_identifier_cache[i]
+        if result is None:
+            if i == 0:
+                result = ""
+            else:
+                parent = self.stable_identifier_for_example(
+                    self.examples[example.parent]
+                )
+                child = str(example.child_index)
+                if parent:
+                    result = parent + "," + child
+                else:
+                    result = child
+            self.example_to_stable_identifier_cache[i] = result
+        return result
+
+    def example_for_stable_identifier(self, identifier):
+        """Returns the example in the current shrink target corresponding
+        to this stable identifier, or None if no such example exists."""
+        if not identifier:
+            return self.examples[0]
+
+        cache = self.stable_identifier_to_example_cache
+        try:
+            return cache[identifier]
+        except KeyError:
+            pass
+        path = list(map(int, identifier.split(",")))
+        ex = self.examples[0]
+        for i in path:
+            try:
+                ex = ex.children[i]
+            except IndexError:
+                ex = None
+                break
+        cache[identifier] = ex
+        return ex
+
     def run_block_program(self, i, description, original, repeats=1):
         """Block programs are a mini-DSL for block rewriting, defined as a sequence
         of commands that can be run at some index into the blocks
@@ -1372,13 +1438,14 @@ def block_program(description):
         n = len(description)
 
         def generate_arguments(self):
-            return [(i,) for i in hrange(len(self.blocks) - n)]
+            return [(i,) for i in hrange(len(self.blocks))]
 
         def run(self, i):
             """Adaptively attempt to run the block program at the current
             index. If this successfully applies the block program ``k`` times
             then this runs in ``O(log(k))`` test function calls."""
-            assert i + n < len(self.shrink_target.blocks)
+            if i + n >= len(self.shrink_target.blocks):
+                return
 
             # First, run the block program at the chosen index. If this fails,
             # don't do any extra work, so that failure is as cheap as possible.
@@ -1423,9 +1490,6 @@ class ShrinkPass(object):
     index = attr.ib()
     shrinker = attr.ib()
 
-    __arguments = attr.ib(default=None, init=False)
-    __target_at_argument_calculation = attr.ib(default=None, init=False)
-
     fixed_point_at = attr.ib(default=None)
     successes = attr.ib(default=0)
     runs = attr.ib(default=0)
@@ -1433,22 +1497,12 @@ class ShrinkPass(object):
     shrinks = attr.ib(default=0)
     deletions = attr.ib(default=0)
 
-    @property
-    def arguments(self):
-        if self.__target_at_argument_calculation is not self.shrinker.shrink_target:
-            self.__arguments = self.generate_arguments(self.shrinker)
-            self.__target_at_argument_calculation = self.shrinker.shrink_target
-        return self.__arguments
-
     def generate_steps(self):
         if self.fixed_point_at is self.shrinker.shrink_target:
             return []
-        return list(hrange(len(self.arguments)))
+        return list(self.generate_arguments(self.shrinker))
 
-    def run_step(self, i):
-        if i >= len(self.arguments):
-            return
-        args = self.arguments[i]
+    def run_step(self, args):
         self.shrinker.debug("%s(%s)" % (self.name, ", ".join(map(repr, args))))
         initial_shrinks = self.shrinker.shrinks
         initial_calls = self.shrinker.calls


Test patch:
diff --git a/hypothesis-python/tests/cover/test_conjecture_engine.py b/hypothesis-python/tests/cover/test_conjecture_engine.py
--- a/hypothesis-python/tests/cover/test_conjecture_engine.py
+++ b/hypothesis-python/tests/cover/test_conjecture_engine.py
@@ -646,16 +646,9 @@ def f(data):
     assert len(list(db.fetch(runner.database_key))) == 1
 
 
-def test_can_delete_intervals(monkeypatch):
-    def generate_new_examples(self):
-        self.test_function(ConjectureData.for_buffer(hbytes([255] * 10 + [1, 3])))
-
-    monkeypatch.setattr(
-        ConjectureRunner, "generate_new_examples", generate_new_examples
-    )
-    monkeypatch.setattr(Shrinker, "shrink", fixate(Shrinker.adaptive_example_deletion))
-
-    def f(data):
+def test_can_delete_intervals():
+    @shrinking_from([255] * 10 + [1, 3])
+    def shrinker(data):
         while True:
             n = data.draw_bits(8)
             if n == 255:
@@ -667,9 +660,9 @@ def f(data):
         if data.draw_bits(8) == 3:
             data.mark_interesting()
 
-    runner = ConjectureRunner(f, settings=settings(database=None))
-    runner.run()
-    x, = runner.interesting_examples.values()
+    shrinker.fixate_shrink_passes(["adaptive_example_deletion"])
+
+    x = shrinker.shrink_target
     assert x.buffer == hbytes([1, 3])
 
 
@@ -710,18 +703,9 @@ def f(data):
     assert runner.interesting_examples[1].buffer == hbytes([1])
 
 
-def test_duplicate_blocks_that_go_away(monkeypatch):
-    monkeypatch.setattr(Shrinker, "shrink", Shrinker.minimize_duplicated_blocks)
-    monkeypatch.setattr(
-        ConjectureRunner,
-        "generate_new_examples",
-        lambda runner: runner.test_function(
-            ConjectureData.for_buffer(hbytes([1, 1, 1, 2] * 2 + [5] * 2))
-        ),
-    )
-
-    @run_to_buffer
-    def x(data):
+def test_duplicate_blocks_that_go_away():
+    @shrinking_from([1, 1, 1, 2] * 2 + [5] * 2)
+    def shrinker(data):
         x = data.draw_bits(32)
         y = data.draw_bits(32)
         if x != y:
@@ -730,7 +714,8 @@ def x(data):
         if len(set(b)) <= 1:
             data.mark_interesting()
 
-    assert x == hbytes([0] * 8)
+    shrinker.fixate_shrink_passes(["minimize_duplicated_blocks"])
+    assert shrinker.shrink_target.buffer == hbytes([0] * 8)
 
 
 def test_accidental_duplication(monkeypatch):
@@ -1426,7 +1411,7 @@ def shrinker(data):
         if data.draw_bits(8) == 7:
             data.mark_interesting()
 
-    shrinker.run_shrink_pass("block_program('-XX')")
+    shrinker.run_shrink_pass(block_program("-XX"))
     assert list(shrinker.shrink_target.buffer) == [1, 7]
 
 
@@ -1604,3 +1589,26 @@ def shrinker(data):
     # single-bit block. Did not try to expand regions into the trivial two-byte
     # blocks on each side.
     assert shrinker.calls == initial + 12
+
+
+def test_stable_identifiers_match_their_examples():
+    def tree(data):
+        data.start_example(1)
+        n = data.draw_bits(1)
+        label = data.draw_bits(8)
+        if n:
+            tree(data)
+            tree(data)
+        data.stop_example(1)
+        return label
+
+    initial = hbytes([1, 10, 0, 0, 1, 0, 0, 10, 0, 0])
+
+    @shrinking_from(initial)
+    def shrinker(data):
+        tree(data)
+        data.mark_interesting()
+
+    for ex in shrinker.examples:
+        id = shrinker.stable_identifier_for_example(ex)
+        assert shrinker.example_for_stable_identifier(id) is ex