Repository: HypothesisWorks/hypothesis

Issue:
Deprecate the use of settings objects as context managers
This API was a bad idea and should be removed, leaving settings to be determined by global state or decorators only.  Using a settings object as a context manager is somewhat ambiguous (should it happen when the test is defined, or executed? certainly not *within* a test...) and just plain unnecessary.


Discussion:
I tried just adding `note_deprecation(...)` to `settings.__enter__` in `_settings.py`, and running the tests to see what breaks - the main task of this issue is to mechanically fix (or delete) our tests anywhere we're using this, and write a minor release note to explain to users that they should stop too.

Of our tests: 1115 failed, 743 passed.  Not recommended for new contributors :sob: 
I have some time for open-source this weekend, I might pick this up if nobody minds?
If you have some time I'd love a review for my open PRs, and after (or before) that this one is all yours. 
👍

Any particular order, or oldest first?

> On 14 Jun 2018, at 21:35, Zac Hatfield-Dodds <notifications@github.com> wrote:
> 
> If you have some time I'd love a review for my open PRs, and after (or before) that this one is all yours.
> 
> —
> You are receiving this because you commented.
> Reply to this email directly, view it on GitHub, or mute the thread.

Whatever works for you, but I'd suggest #1295, #1270, #1304, #1282 as a value-for-effort order.  Thanks!
I’m going to have a quick look at this.
Ah, we use settings as a context manager in `engine.py`, where I think we do want to scope it to within a single `run()` method. That seems to be the root of most of the test breakages,

It’s not immediately obvious to me how we unpick it from there, but it’s been a while since I‘ve had to think about this code…
The obvious option to me would be to add a private context manager function `_with_settings(s)`, which does exactly the same thing but isn't public API - the alternative would be to manually pass the object around everywhere, and that gets ugly (and unmaintainable) very quickly..  Then it's just a matter of removing the context manager use from our docs and tests...

Do you still want to do this, or shall I?
> Do you still want to do this, or shall I?

I’m doing it. Hopefully at least a WIP PR by the end of the day.

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/_settings.py b/hypothesis-python/src/hypothesis/_settings.py
--- a/hypothesis-python/src/hypothesis/_settings.py
+++ b/hypothesis-python/src/hypothesis/_settings.py
@@ -26,6 +26,7 @@
 import os
 import warnings
 import threading
+import contextlib
 from enum import Enum, IntEnum, unique
 
 import attr
@@ -316,6 +317,10 @@ def __repr__(self):
         return 'settings(%s)' % ', '.join(sorted(bits))
 
     def __enter__(self):
+        note_deprecation(
+            'Settings should be determined only by global state or with the '
+            '@settings decorator.'
+        )
         default_context_manager = default_variable.with_value(self)
         self.defaults_stack().append(default_context_manager)
         default_context_manager.__enter__()
@@ -380,6 +385,13 @@ def load_profile(name):
         settings._assign_default_internal(settings.get_profile(name))
 
 
+@contextlib.contextmanager
+def local_settings(s):
+    default_context_manager = default_variable.with_value(s)
+    with default_context_manager:
+        yield s
+
+
 @attr.s()
 class Setting(object):
     name = attr.ib()
diff --git a/hypothesis-python/src/hypothesis/core.py b/hypothesis-python/src/hypothesis/core.py
--- a/hypothesis-python/src/hypothesis/core.py
+++ b/hypothesis-python/src/hypothesis/core.py
@@ -49,7 +49,7 @@
 from hypothesis._settings import Phase, Verbosity, HealthCheck, \
     PrintSettings
 from hypothesis._settings import settings as Settings
-from hypothesis._settings import note_deprecation
+from hypothesis._settings import local_settings, note_deprecation
 from hypothesis.executors import new_style_executor
 from hypothesis.reporting import report, verbose_report, current_verbosity
 from hypothesis.statistics import note_engine_for_statistics
@@ -553,7 +553,7 @@ def test(*args, **kwargs):
         def run(data):
             if not hasattr(data, 'can_reproduce_example_from_repr'):
                 data.can_reproduce_example_from_repr = True
-            with self.settings:
+            with local_settings(self.settings):
                 with BuildContext(data, is_final=is_final):
                     with deterministic_PRNG():
                         args, kwargs = data.draw(self.search_strategy)
diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py
--- a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py
+++ b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py
@@ -27,6 +27,7 @@
 
 from hypothesis import Phase, Verbosity, HealthCheck
 from hypothesis import settings as Settings
+from hypothesis._settings import local_settings
 from hypothesis.reporting import debug_report
 from hypothesis.internal.compat import Counter, ceil, hbytes, hrange, \
     int_to_text, int_to_bytes, benchmark_time, int_from_bytes, \
@@ -405,7 +406,7 @@ def note_details(self, data):
             self.event_call_counts[event] += 1
 
     def debug(self, message):
-        with self.settings:
+        with local_settings(self.settings):
             debug_report(message)
 
     def debug_data(self, data):
@@ -433,7 +434,7 @@ def debug_data(self, data):
         ))
 
     def run(self):
-        with self.settings:
+        with local_settings(self.settings):
             try:
                 self._run()
             except RunIsComplete:


Test patch:
diff --git a/hypothesis-python/tests/cover/test_conjecture_engine.py b/hypothesis-python/tests/cover/test_conjecture_engine.py
--- a/hypothesis-python/tests/cover/test_conjecture_engine.py
+++ b/hypothesis-python/tests/cover/test_conjecture_engine.py
@@ -603,13 +603,19 @@ def f(data):
 
 @pytest.mark.parametrize('count', [1, 3])
 def test_debug_data(capsys, count):
-    with settings(verbosity=Verbosity.debug):
-        @run_to_buffer
-        def f(data):
-            for _ in hrange(count):
-                data.draw_bytes(1)
-            if sum(data.buffer) > 10:
-                data.mark_interesting()
+    def f(data):
+        for _ in hrange(count):
+            data.draw_bytes(1)
+        if sum(data.buffer) > 10:
+            data.mark_interesting()
+
+    runner = ConjectureRunner(f, settings=settings(
+        max_examples=5000, buffer_size=1024,
+        database=None, suppress_health_check=HealthCheck.all(),
+        verbosity=Verbosity.debug
+    ))
+    runner.run()
+
     out, _ = capsys.readouterr()
     assert re.match(u'\\d+ bytes \\[.*\\] -> ', out)
 
diff --git a/hypothesis-python/tests/cover/test_datetimes.py b/hypothesis-python/tests/cover/test_datetimes.py
--- a/hypothesis-python/tests/cover/test_datetimes.py
+++ b/hypothesis-python/tests/cover/test_datetimes.py
@@ -20,9 +20,8 @@
 import datetime as dt
 
 import pytest
-from flaky import flaky
 
-from hypothesis import given, settings, unlimited
+from hypothesis import given
 from tests.common.debug import minimal, find_any
 from tests.common.utils import checks_deprecated_behaviour
 from hypothesis.strategies import none, dates, times, binary, datetimes, \
@@ -80,13 +79,11 @@ def test_default_datetimes_are_naive(dt):
     assert dt.tzinfo is None
 
 
-@flaky(max_runs=3, min_passes=1)
 def test_bordering_on_a_leap_year():
-    with settings(database=None, max_examples=10 ** 7, timeout=unlimited):
-        x = minimal(datetimes(dt.datetime.min.replace(year=2003),
-                              dt.datetime.max.replace(year=2005)),
-                    lambda x: x.month == 2 and x.day == 29,
-                    timeout_after=60)
+    x = minimal(datetimes(dt.datetime.min.replace(year=2003),
+                          dt.datetime.max.replace(year=2005)),
+                lambda x: x.month == 2 and x.day == 29,
+                timeout_after=60)
     assert x.year == 2004
 
 
diff --git a/hypothesis-python/tests/cover/test_flatmap.py b/hypothesis-python/tests/cover/test_flatmap.py
--- a/hypothesis-python/tests/cover/test_flatmap.py
+++ b/hypothesis-python/tests/cover/test_flatmap.py
@@ -31,15 +31,18 @@
     lambda e: tuples(integers(0, e - 1), just(e))
 )
 
-with settings(max_examples=100):
-    @given(ConstantLists)
-    def test_constant_lists_are_constant(x):
-        assume(len(x) >= 3)
-        assert len(set(x)) == 1
-
-    @given(OrderedPairs)
-    def test_in_order(x):
-        assert x[0] < x[1]
+
+@settings(max_examples=100)
+@given(ConstantLists)
+def test_constant_lists_are_constant(x):
+    assume(len(x) >= 3)
+    assert len(set(x)) == 1
+
+
+@settings(max_examples=100)
+@given(OrderedPairs)
+def test_in_order(x):
+    assert x[0] < x[1]
 
 
 def test_flatmap_retrieve_from_db():
diff --git a/hypothesis-python/tests/cover/test_health_checks.py b/hypothesis-python/tests/cover/test_health_checks.py
--- a/hypothesis-python/tests/cover/test_health_checks.py
+++ b/hypothesis-python/tests/cover/test_health_checks.py
@@ -53,12 +53,12 @@ def test(data):
 def test_default_health_check_can_weaken_specific():
     import random
 
+    @settings(suppress_health_check=HealthCheck.all())
     @given(st.lists(st.integers(), min_size=1))
     def test(x):
         random.choice(x)
 
-    with settings(suppress_health_check=HealthCheck.all()):
-        test()
+    test()
 
 
 def test_suppressing_filtering_health_check():
diff --git a/hypothesis-python/tests/cover/test_integer_ranges.py b/hypothesis-python/tests/cover/test_integer_ranges.py
--- a/hypothesis-python/tests/cover/test_integer_ranges.py
+++ b/hypothesis-python/tests/cover/test_integer_ranges.py
@@ -40,13 +40,18 @@ def do_draw(self, data):
     st.tuples(st.integers(), st.integers(), st.integers()).map(sorted),
     st.random_module(),
 )
-@settings(max_examples=100, max_shrinks=0, deadline=None)
+@settings(
+    max_examples=100,
+    max_shrinks=0,
+    deadline=None,
+    database=None,
+    timeout=unlimited
+)
 def test_intervals_shrink_to_center(inter, rnd):
     lower, center, upper = inter
     s = interval(lower, upper, center)
-    with settings(database=None, timeout=unlimited):
-        assert find(s, lambda x: True) == center
-        if lower < center:
-            assert find(s, lambda x: x < center) == center - 1
-        if center < upper:
-            assert find(s, lambda x: x > center) == center + 1
+    assert find(s, lambda x: True) == center
+    if lower < center:
+        assert find(s, lambda x: x < center) == center - 1
+    if center < upper:
+        assert find(s, lambda x: x > center) == center + 1
diff --git a/hypothesis-python/tests/cover/test_reporting.py b/hypothesis-python/tests/cover/test_reporting.py
--- a/hypothesis-python/tests/cover/test_reporting.py
+++ b/hypothesis-python/tests/cover/test_reporting.py
@@ -62,23 +62,35 @@ def test_int(x):
 
 
 def test_does_not_print_debug_in_verbose():
-    with settings(verbosity=Verbosity.verbose):
-        with capture_out() as o:
-            debug_report(u'Hi')
-    assert not o.getvalue()
+    @given(integers())
+    @settings(verbosity=Verbosity.verbose)
+    def f(x):
+        debug_report('Hi')
+
+    with capture_out() as o:
+        f()
+    assert u'Hi' not in o.getvalue()
 
 
 def test_does_print_debug_in_debug():
-    with settings(verbosity=Verbosity.debug):
-        with capture_out() as o:
-            debug_report(u'Hi')
+    @given(integers())
+    @settings(verbosity=Verbosity.debug)
+    def f(x):
+        debug_report('Hi')
+
+    with capture_out() as o:
+        f()
     assert u'Hi' in o.getvalue()
 
 
 def test_does_print_verbose_in_debug():
-    with settings(verbosity=Verbosity.debug):
-        with capture_out() as o:
-            verbose_report(u'Hi')
+    @given(integers())
+    @settings(verbosity=Verbosity.debug)
+    def f(x):
+        verbose_report('Hi')
+
+    with capture_out() as o:
+        f()
     assert u'Hi' in o.getvalue()
 
 
diff --git a/hypothesis-python/tests/cover/test_settings.py b/hypothesis-python/tests/cover/test_settings.py
--- a/hypothesis-python/tests/cover/test_settings.py
+++ b/hypothesis-python/tests/cover/test_settings.py
@@ -69,12 +69,14 @@ def test_respects_none_database():
     assert settings(database=None).database is None
 
 
+@checks_deprecated_behaviour
 def test_settings_can_be_used_as_context_manager_to_change_defaults():
     with settings(max_examples=12):
         assert settings.default.max_examples == 12
     assert settings.default.max_examples == original_default
 
 
+@checks_deprecated_behaviour
 def test_can_repeatedly_push_the_same_thing():
     s = settings(max_examples=12)
     t = settings(max_examples=17)
@@ -129,6 +131,7 @@ def test_can_not_set_verbosity_to_non_verbosity():
         settings(verbosity='kittens')
 
 
+@checks_deprecated_behaviour
 @pytest.mark.parametrize('db', [None, ExampleDatabase()])
 def test_inherits_an_empty_database(db):
     assert settings.default.database is not None
@@ -177,6 +180,7 @@ def test_nonstring_profile_names_deprecated():
     assert settings.default.stateful_step_count == 5
 
 
+@checks_deprecated_behaviour
 def test_loading_profile_keeps_expected_behaviour():
     settings.register_profile('ci', settings(max_examples=10000))
     settings.load_profile('ci')
@@ -237,6 +241,7 @@ def test_can_have_none_database():
     assert settings(database=None).database is None
 
 
+@checks_deprecated_behaviour
 @pytest.mark.parametrize('db', [None, ExampleDatabase(':memory:')])
 def test_database_type_must_be_ExampleDatabase(db):
     with settings(database=db):
diff --git a/hypothesis-python/tests/cover/test_stateful.py b/hypothesis-python/tests/cover/test_stateful.py
--- a/hypothesis-python/tests/cover/test_stateful.py
+++ b/hypothesis-python/tests/cover/test_stateful.py
@@ -390,12 +390,16 @@ def execute_step(self, choices):
     run_state_machine_as_test(ChoosingMachine)
 
 
-with Settings(max_examples=10):
-    TestGoodSets = GoodSet.TestCase
-    TestGivenLike = GivenLikeStateMachine.TestCase
-    TestDynamicMachine = DynamicMachine.TestCase
-    TestIntAdder = IntAdder.TestCase
-    TestPrecondition = PreconditionMachine.TestCase
+TestGoodSets = GoodSet.TestCase
+TestGivenLike = GivenLikeStateMachine.TestCase
+TestDynamicMachine = DynamicMachine.TestCase
+TestIntAdder = IntAdder.TestCase
+TestPrecondition = PreconditionMachine.TestCase
+
+
+for test_case in (TestGoodSets, TestGivenLike, TestDynamicMachine,
+                  TestIntAdder, TestPrecondition):
+    test_case.settings = Settings(max_examples=10)
 
 
 def test_picks_up_settings_at_first_use_of_testcase():
@@ -415,6 +419,7 @@ class Foo(RuleBasedStateMachine):
     assert len(Foo.rules()) == 2
 
 
+@checks_deprecated_behaviour
 def test_settings_are_independent():
     s = Settings()
     orig = s.max_examples
diff --git a/hypothesis-python/tests/cover/test_testdecorators.py b/hypothesis-python/tests/cover/test_testdecorators.py
--- a/hypothesis-python/tests/cover/test_testdecorators.py
+++ b/hypothesis-python/tests/cover/test_testdecorators.py
@@ -255,13 +255,13 @@ def test_ints_are_sorted(balthazar, evans):
 
 
 def test_does_not_print_on_success():
-    with settings(verbosity=Verbosity.normal):
-        @given(integers())
-        def test_is_an_int(x):
-            return
+    @settings(verbosity=Verbosity.normal)
+    @given(integers())
+    def test_is_an_int(x):
+        return
 
-        with capture_out() as out:
-            test_is_an_int()
+    with capture_out() as out:
+        test_is_an_int()
     out = out.getvalue()
     lines = [l.strip() for l in out.split(u'\n')]
     assert all(not l for l in lines), lines
@@ -423,17 +423,16 @@ def test_when_set_to_no_simplifies_runs_failing_example_twice():
     failing = [0]
 
     @given(integers())
-    @settings(phases=no_shrink, max_examples=100)
+    @settings(phases=no_shrink, max_examples=100, verbosity=Verbosity.normal)
     def foo(x):
         if x > 11:
             note('Lo')
             failing[0] += 1
             assert False
 
-    with settings(verbosity=Verbosity.normal):
-        with raises(AssertionError):
-            with capture_out() as out:
-                foo()
+    with raises(AssertionError):
+        with capture_out() as out:
+            foo()
     assert failing == [2]
     assert 'Falsifying example' in out.getvalue()
     assert 'Lo' in out.getvalue()
diff --git a/hypothesis-python/tests/cover/test_verbosity.py b/hypothesis-python/tests/cover/test_verbosity.py
--- a/hypothesis-python/tests/cover/test_verbosity.py
+++ b/hypothesis-python/tests/cover/test_verbosity.py
@@ -31,15 +31,15 @@
 
 
 @contextmanager
-def capture_verbosity(level):
+def capture_verbosity():
     with capture_out() as o:
         with with_reporter(default_reporter):
-            with settings(verbosity=level):
-                yield o
+            yield o
 
 
 def test_prints_intermediate_in_success():
-    with capture_verbosity(Verbosity.verbose) as o:
+    with capture_verbosity() as o:
+        @settings(verbosity=Verbosity.verbose)
         @given(booleans())
         def test_works(x):
             pass
@@ -48,8 +48,9 @@ def test_works(x):
 
 
 def test_does_not_log_in_quiet_mode():
-    with capture_verbosity(Verbosity.quiet) as o:
+    with capture_verbosity() as o:
         @fails
+        @settings(verbosity=Verbosity.quiet)
         @given(integers())
         def test_foo(x):
             assert False
@@ -59,9 +60,14 @@ def test_foo(x):
 
 
 def test_includes_progress_in_verbose_mode():
-    with capture_verbosity(Verbosity.verbose) as o:
-        with settings(verbosity=Verbosity.verbose, database=None):
-            find(lists(integers()), lambda x: sum(x) >= 1000000)
+    with capture_verbosity() as o:
+        def foo():
+            find(
+                lists(integers()),
+                lambda x: sum(x) >= 1000000,
+                settings=settings(verbosity=Verbosity.verbose, database=None))
+
+        foo()
 
     out = o.getvalue()
     assert out
@@ -71,8 +77,8 @@ def test_includes_progress_in_verbose_mode():
 
 def test_prints_initial_attempts_on_find():
 
-    with capture_verbosity(Verbosity.verbose) as o:
-        with settings(verbosity=Verbosity.verbose):
+    with capture_verbosity() as o:
+        def foo():
             seen = []
 
             def not_first(x):
@@ -80,14 +86,19 @@ def not_first(x):
                     seen.append(x)
                     return False
                 return x not in seen
-            find(integers(), not_first)
+            find(
+                integers(), not_first,
+                settings=settings(verbosity=Verbosity.verbose))
+
+        foo()
 
     assert u'Tried non-satisfying example' in o.getvalue()
 
 
 def test_includes_intermediate_results_in_verbose_mode():
-    with capture_verbosity(Verbosity.verbose) as o:
+    with capture_verbosity() as o:
         @fails
+        @settings(verbosity=Verbosity.verbose)
         @given(lists(integers()))
         def test_foo(x):
             assert sum(x) < 1000000
diff --git a/whole-repo-tests/test_deploy.py b/whole-repo-tests/test_deploy.py
--- a/whole-repo-tests/test_deploy.py
+++ b/whole-repo-tests/test_deploy.py
@@ -45,7 +45,6 @@ def test_release_file_exists_and_is_valid(project, monkeypatch):
             changelog = i.read()
         assert project.current_version() in changelog
         assert rm.release_date_string() in changelog
-        assert not os.path.exists(project.RELEASE_FILE)
 
     finally:
         tools.git('checkout', project.BASE_DIR)