Repository: HypothesisWorks/hypothesis

Issue:
Deprecate the buffer_size setting
As discussed in #535, this setting is entirely about Hypothesis internals, and the way it actually works is totally undocumented. Hypothesis should figure out something sensible to do here and then do it.

We have plans to do a certain amount of auto-expanding of the buffer at some point, so that the size can start *low*.  However the fixed buffer size is currently carrying a lot of weight in terms of how various cases are handled (deeply nested and recursive structures in particular), so this will take some care.


Discussion:

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/_settings.py b/hypothesis-python/src/hypothesis/_settings.py
--- a/hypothesis-python/src/hypothesis/_settings.py
+++ b/hypothesis-python/src/hypothesis/_settings.py
@@ -456,14 +456,11 @@ def _max_examples_validator(x):
 
 settings._define_setting(
     "buffer_size",
-    default=8 * 1024,
+    default=not_set,
     validator=lambda x: _ensure_positive_int(x, "buffer_size", since="2019-03-06"),
-    show_default=False,
-    description="""
-The size of the underlying data used to generate examples. If you need to
-generate really large examples you may want to increase this, but it will make
-your tests slower.
-""",
+    description="The buffer_size setting has been deprecated and no longer does anything.",
+    deprecation_message="The buffer_size setting can safely be removed with no effect.",
+    deprecated_since="RELEASEDAY",
 )
 
 
diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py
--- a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py
+++ b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py
@@ -57,6 +57,7 @@
 CACHE_SIZE = 10000
 MUTATION_POOL_SIZE = 100
 MIN_TEST_CALLS = 10
+BUFFER_SIZE = 8 * 1024
 
 
 @attr.s
@@ -237,7 +238,7 @@ def generate_novel_prefix(self):
 
     @property
     def cap(self):
-        return self.settings.buffer_size // 2
+        return BUFFER_SIZE // 2
 
     def record_for_health_check(self, data):
         # Once we've actually found a bug, there's no point in trying to run
@@ -602,13 +603,12 @@ def generate_new_examples(self):
             # the time to look for additional failures.
             return
 
-        zero_data = self.cached_test_function(hbytes(self.settings.buffer_size))
+        zero_data = self.cached_test_function(hbytes(BUFFER_SIZE))
         if zero_data.status > Status.OVERRUN:
             self.__data_cache.pin(zero_data.buffer)
 
         if zero_data.status == Status.OVERRUN or (
-            zero_data.status == Status.VALID
-            and len(zero_data.buffer) * 2 > self.settings.buffer_size
+            zero_data.status == Status.VALID and len(zero_data.buffer) * 2 > BUFFER_SIZE
         ):
             fail_health_check(
                 self.settings,
@@ -770,7 +770,7 @@ def _run(self):
     def new_conjecture_data(self, draw_bytes):
         return ConjectureData(
             draw_bytes=draw_bytes,
-            max_length=self.settings.buffer_size,
+            max_length=BUFFER_SIZE,
             observer=self.tree.new_observer(),
         )
 


Test patch:
diff --git a/hypothesis-python/tests/cover/test_conjecture_engine.py b/hypothesis-python/tests/cover/test_conjecture_engine.py
--- a/hypothesis-python/tests/cover/test_conjecture_engine.py
+++ b/hypothesis-python/tests/cover/test_conjecture_engine.py
@@ -18,6 +18,7 @@
 from __future__ import absolute_import, division, print_function
 
 import re
+from contextlib import contextmanager
 from random import Random
 
 import attr
@@ -52,10 +53,7 @@
 
 
 TEST_SETTINGS = settings(
-    max_examples=5000,
-    buffer_size=1024,
-    database=None,
-    suppress_health_check=HealthCheck.all(),
+    max_examples=5000, database=None, suppress_health_check=HealthCheck.all()
 )
 
 
@@ -72,6 +70,16 @@ def run_to_buffer(f):
     return hbytes(run_to_data(f).buffer)
 
 
+@contextmanager
+def buffer_size_limit(n):
+    original = engine_module.BUFFER_SIZE
+    try:
+        engine_module.BUFFER_SIZE = n
+        yield
+    finally:
+        engine_module.BUFFER_SIZE = original
+
+
 def test_can_index_results():
     @run_to_buffer
     def f(data):
@@ -235,10 +243,9 @@ def f(data):
         data.draw_bytes(i)
         data.mark_interesting()
 
-    runner = ConjectureRunner(
-        f, settings=settings(max_examples=5000, buffer_size=2, database=None)
-    )
-    runner.run()
+    runner = ConjectureRunner(f, settings=settings(max_examples=5000, database=None))
+    with buffer_size_limit(2):
+        runner.run()
     assert runner.interesting_examples
 
 
@@ -408,11 +415,7 @@ def accept(f):
         runner = ConjectureRunner(
             f,
             settings=settings(
-                max_examples=100,
-                phases=no_shrink,
-                buffer_size=1024,
-                database=None,
-                **kwargs
+                max_examples=100, phases=no_shrink, database=None, **kwargs
             ),
         )
 
@@ -546,7 +549,6 @@ def f(data):
         f,
         settings=settings(
             max_examples=5000,
-            buffer_size=1024,
             database=None,
             suppress_health_check=HealthCheck.all(),
             verbosity=Verbosity.debug,
@@ -566,7 +568,8 @@ def f(data):
             x = data.draw_bytes(9)
             assert not any(x[4:8])
 
-    ConjectureRunner(f, settings=settings(buffer_size=10)).run()
+    with buffer_size_limit(10):
+        ConjectureRunner(f).run()
 
 
 def test_can_write_bytes_towards_the_end():
@@ -578,7 +581,8 @@ def f(data):
             data.write(hbytes(buf))
             assert hbytes(data.buffer[-len(buf) :]) == buf
 
-    ConjectureRunner(f, settings=settings(buffer_size=10)).run()
+    with buffer_size_limit(10):
+        ConjectureRunner(f).run()
 
 
 def test_can_increase_number_of_bytes_drawn_in_tail():
@@ -592,13 +596,11 @@ def f(data):
         assert not any(b)
 
     runner = ConjectureRunner(
-        f,
-        settings=settings(
-            max_examples=100, buffer_size=11, suppress_health_check=HealthCheck.all()
-        ),
+        f, settings=settings(max_examples=100, suppress_health_check=HealthCheck.all())
     )
 
-    runner.run()
+    with buffer_size_limit(11):
+        runner.run()
 
 
 def test_uniqueness_is_preserved_when_writing_at_beginning():
@@ -995,12 +997,12 @@ def f(data):
             f,
             settings=settings(
                 max_examples=5000,
-                buffer_size=1024,
                 database=None,
                 suppress_health_check=HealthCheck.all(),
             ),
         )
-        runner.run()
+        with buffer_size_limit(1024):
+            runner.run()
 
     assert runner.exit_reason == ExitReason.finished
 
@@ -1096,7 +1098,6 @@ def accept(f):
                 f,
                 settings=settings(
                     max_examples=5000,
-                    buffer_size=1024,
                     database=None,
                     suppress_health_check=HealthCheck.all(),
                 ),
@@ -1175,10 +1176,7 @@ def f(data):
     runner = ConjectureRunner(
         f,
         settings=settings(
-            max_examples=1,
-            buffer_size=1024,
-            database=database,
-            suppress_health_check=HealthCheck.all(),
+            max_examples=1, database=database, suppress_health_check=HealthCheck.all()
         ),
         database_key=key,
     )
@@ -1211,10 +1209,7 @@ def f(data):
     runner = ConjectureRunner(
         f,
         settings=settings(
-            max_examples=1,
-            buffer_size=1024,
-            database=database,
-            suppress_health_check=HealthCheck.all(),
+            max_examples=1, database=database, suppress_health_check=HealthCheck.all()
         ),
         database_key=key,
     )
@@ -1248,10 +1243,7 @@ def f(data):
     runner = ConjectureRunner(
         f,
         settings=settings(
-            max_examples=1,
-            buffer_size=1024,
-            database=None,
-            suppress_health_check=HealthCheck.all(),
+            max_examples=1, database=None, suppress_health_check=HealthCheck.all()
         ),
     )
 
diff --git a/hypothesis-python/tests/cover/test_health_checks.py b/hypothesis-python/tests/cover/test_health_checks.py
--- a/hypothesis-python/tests/cover/test_health_checks.py
+++ b/hypothesis-python/tests/cover/test_health_checks.py
@@ -117,8 +117,8 @@ def test(x):
 
 
 def test_large_data_will_fail_a_health_check():
-    @given(st.none() | st.binary(min_size=1024))
-    @settings(database=None, buffer_size=1000)
+    @given(st.none() | st.binary(min_size=10 ** 5))
+    @settings(database=None)
     def test(x):
         pass
 
diff --git a/hypothesis-python/tests/cover/test_settings.py b/hypothesis-python/tests/cover/test_settings.py
--- a/hypothesis-python/tests/cover/test_settings.py
+++ b/hypothesis-python/tests/cover/test_settings.py
@@ -479,6 +479,11 @@ def test_dubious_settings_deprecations(name):
         settings(**{name: "2.5"})  # deprecation warning, then type cast error.
 
 
+@checks_deprecated_behaviour
+def test_buffer_size_deprecated():
+    assert settings(buffer_size=100).buffer_size == 100
+
+
 @checks_deprecated_behaviour
 def test_max_example_eq_0_warns_and_disables_generation():
     # Terrible way to disable generation, but did predate the phases setting
diff --git a/hypothesis-python/tests/nocover/test_explore_arbitrary_languages.py b/hypothesis-python/tests/nocover/test_explore_arbitrary_languages.py
--- a/hypothesis-python/tests/nocover/test_explore_arbitrary_languages.py
+++ b/hypothesis-python/tests/nocover/test_explore_arbitrary_languages.py
@@ -100,7 +100,6 @@ def test(local_data):
         test,
         settings=settings(
             max_examples=1,
-            buffer_size=512,
             database=None,
             suppress_health_check=HealthCheck.all(),
             verbosity=Verbosity.quiet,
diff --git a/hypothesis-python/tests/numpy/test_gen_data.py b/hypothesis-python/tests/numpy/test_gen_data.py
--- a/hypothesis-python/tests/numpy/test_gen_data.py
+++ b/hypothesis-python/tests/numpy/test_gen_data.py
@@ -199,7 +199,7 @@ def test_can_generate_compound_dtypes(dtype):
     assert isinstance(dtype, np.dtype)
 
 
-@given(nps.nested_dtypes(max_itemsize=settings.default.buffer_size // 10), st.data())
+@given(nps.nested_dtypes(max_itemsize=400), st.data())
 def test_infer_strategy_from_dtype(dtype, data):
     # Given a dtype
     assert isinstance(dtype, np.dtype)
@@ -226,7 +226,7 @@ def test_minimise_nested_types():
 def test_minimise_array_strategy():
     smallest = minimal(
         nps.arrays(
-            nps.nested_dtypes(max_itemsize=settings.default.buffer_size // 3 ** 3),
+            nps.nested_dtypes(max_itemsize=200),
             nps.array_shapes(max_dims=3, max_side=3),
         )
     )
diff --git a/hypothesis-python/tests/quality/test_poisoned_trees.py b/hypothesis-python/tests/quality/test_poisoned_trees.py
--- a/hypothesis-python/tests/quality/test_poisoned_trees.py
+++ b/hypothesis-python/tests/quality/test_poisoned_trees.py
@@ -94,9 +94,7 @@ def test_function(data):
         if len(v) >= size:
             data.mark_interesting()
 
-    runner = ConjectureRunner(
-        test_function, random=random, settings=settings(TEST_SETTINGS, buffer_size=LOTS)
-    )
+    runner = ConjectureRunner(test_function, random=random, settings=TEST_SETTINGS)
 
     while not runner.interesting_examples:
         runner.test_function(