Repository: HypothesisWorks/hypothesis

Issue:
Reuse up to `max_examples` examples when generation is disabled
https://github.com/HypothesisWorks/hypothesis/blob/caccabbd107757ee6e9de1de7e3cff5474d9ccc5/hypothesis-python/src/hypothesis/internal/conjecture/engine.py#L544

No matter how many examples are in the database, we'll use at most 10% of our example budget replaying them - even if the generation phase is disabled.  It would be nice if we could replay up to `max_examples` when `Phase.generate not in self.settings.phases`.


Discussion:
@Zac-HD I'm interested to take a look at this.
Hi @oxfordhalfblood!  https://hypothesis.readthedocs.io/en/latest/settings.html#phases explains how phases work in Hypothesis.

While a run with default settings will reuse (up to) 10 examples and then generate 90 examples, if we've disabled the generate phase Hypothesis should reuse `max_examples=100` examples from the database instead.  So the calculation for `desired_size` should multiply by `(0.1 if Phase.generate in self.settings.phases else 1)`.

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py
--- a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py
+++ b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py
@@ -541,7 +541,8 @@ def reuse_existing_examples(self):
             corpus = sorted(
                 self.settings.database.fetch(self.database_key), key=sort_key
             )
-            desired_size = max(2, ceil(0.1 * self.settings.max_examples))
+            factor = 0.1 if (Phase.generate in self.settings.phases) else 1
+            desired_size = max(2, ceil(factor * self.settings.max_examples))
 
             if len(corpus) < desired_size:
                 extra_corpus = list(self.settings.database.fetch(self.secondary_key))


Test patch:
diff --git a/hypothesis-python/tests/conjecture/test_engine.py b/hypothesis-python/tests/conjecture/test_engine.py
--- a/hypothesis-python/tests/conjecture/test_engine.py
+++ b/hypothesis-python/tests/conjecture/test_engine.py
@@ -311,6 +311,25 @@ def f(data):
     assert len(seen) == MIN_TEST_CALLS
 
 
+def test_reuse_phase_runs_for_max_examples_if_generation_is_disabled():
+    with deterministic_PRNG():
+        db = InMemoryExampleDatabase()
+        for i in range(256):
+            db.save(b"key", bytes([i]))
+        seen = set()
+
+        def test(data):
+            seen.add(data.draw_bits(8))
+
+        ConjectureRunner(
+            test,
+            settings=settings(max_examples=100, database=db, phases=[Phase.reuse]),
+            database_key=b"key",
+        ).run()
+
+        assert len(seen) == 100
+
+
 def test_erratic_draws():
     n = [0]
 
diff --git a/hypothesis-python/tests/conjecture/test_pareto.py b/hypothesis-python/tests/conjecture/test_pareto.py
--- a/hypothesis-python/tests/conjecture/test_pareto.py
+++ b/hypothesis-python/tests/conjecture/test_pareto.py
@@ -13,11 +13,13 @@
 #
 # END HEADER
 
+import pytest
+
 from hypothesis import HealthCheck, Phase, settings
 from hypothesis.database import InMemoryExampleDatabase
 from hypothesis.internal.compat import int_to_bytes
 from hypothesis.internal.conjecture.data import Status
-from hypothesis.internal.conjecture.engine import ConjectureRunner
+from hypothesis.internal.conjecture.engine import ConjectureRunner, RunIsComplete
 from hypothesis.internal.entropy import deterministic_PRNG
 
 
@@ -133,9 +135,10 @@ def test(data):
         for i in range(10000):
             db.save(runner.pareto_key, int_to_bytes(i, 2))
 
-        runner.reuse_existing_examples()
+        with pytest.raises(RunIsComplete):
+            runner.reuse_existing_examples()
 
-        assert 0 < runner.valid_examples <= 100
+        assert runner.valid_examples == 1000
 
 
 def test_stops_loading_pareto_front_if_interesting():