Repository: HypothesisWorks/hypothesis

Issue:
Verbosity setting not always respected
This is a fairly subtle bug I found while working on #931.

After fixing the Django tests to use Dajngo-2.0 idioms, I ran into a weird issue with passing tests printing an `@seed` note even when they passed.

```
$python -m tests.django.manage test tests.django.toystore.test_basic_configuration
OK
Creating test database for alias 'default'...
System check identified no issues (0 silenced).
You can add @seed(301236684161683320861498655448715075617) to this test to reproduce this failure.
Destroying test database for alias 'default'...
```

This happens because the inner test throws an error in example generation, therefore fails and prints a note, then passes because that's what the outer test wanted to see.  The **problem** is that adding `@setting(verbosity=Verbosity.quiet)` on the inner test should disable this message, but currently doesn't.

[My initial fix is here](https://github.com/HypothesisWorks/hypothesis-python/compare/master...Zac-HD:verbosity-bug) - note that I had to use a `with settings(...)` because simply adding it to the test settings didn't work.
example doesn't respect verbosity
If using the `@hypothesis.example` decorator, the `settings.default.verbosity` is changed to `Verbosity.normal`.

The following test case currently fails:

```python
def test_example_respects_verbosity():
    @settings(verbosity=Verbosity.verbose)
    @example(x=99)
    @given(integers())
    def test(x):
        assert settings.default.verbosity == Verbosity.verbose
    test()
````


Discussion:
~~I think the correct fix is to check the verbosity setting here:~~

https://github.com/HypothesisWorks/hypothesis-python/blob/ba9dd88b3bb339d2d1d1135a6f20e33b7d51185c/src/hypothesis/core.py#L1023-L1025

Nope, `report` should already be doing that.  Hmm...  My next guess is it's something about defining the test inside our test function, or we'd see this much more often.
Hmm.  This *doesn't* happen under Python 2.7, at least on Travis.  Suggests that investigating version differences is likely to be important.
I'm not certain if this is related but under Python 3.6.1 I have pytest test functions that will dynamically skip certain tests. When a test is skipped the seed message is displayed even when verbosity is set to quiet. This isn't the behavior I expected. If I can help with additional details please let me know.
Hmm, @rbdixon, an example of that seems like it might be pretty useful if you can provide one.
This is probably a subtle problem somewhere in our internal scoping, similar to #944.

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/core.py b/hypothesis-python/src/hypothesis/core.py
--- a/hypothesis-python/src/hypothesis/core.py
+++ b/hypothesis-python/src/hypothesis/core.py
@@ -283,19 +283,18 @@ def execute_explicit_examples(
         example_string = '%s(%s)' % (
             test.__name__, arg_string(test, arguments, example_kwargs)
         )
-        try:
-            with BuildContext(None) as b:
-                if settings.verbosity >= Verbosity.verbose:
-                    report('Trying example: ' + example_string)
-                test_runner(
-                    None,
-                    lambda data: test(*arguments, **example_kwargs)
-                )
-        except BaseException:
-            report('Falsifying example: ' + example_string)
-            for n in b.notes:
-                report(n)
-            raise
+        with local_settings(settings):
+            try:
+                with BuildContext(None) as b:
+                    verbose_report('Trying example: ' + example_string)
+                    test_runner(
+                        None, lambda data: test(*arguments, **example_kwargs)
+                    )
+            except BaseException:
+                report('Falsifying example: ' + example_string)
+                for n in b.notes:
+                    report(n)
+                raise
 
 
 def get_random_for_wrapped_test(test, wrapped_test):
@@ -921,16 +920,17 @@ def wrapped_test(*arguments, **kwargs):
                 generated_seed = \
                     wrapped_test._hypothesis_internal_use_generated_seed
                 if generated_seed is not None and not state.failed_normally:
-                    if running_under_pytest:
-                        report((
-                            'You can add @seed(%(seed)d) to this test or run '
-                            'pytest with --hypothesis-seed=%(seed)d to '
-                            'reproduce this failure.') % {
-                                'seed': generated_seed},)
-                    else:
-                        report((
-                            'You can add @seed(%d) to this test to reproduce '
-                            'this failure.') % (generated_seed,))
+                    with local_settings(settings):
+                        if running_under_pytest:
+                            report(
+                                'You can add @seed(%(seed)d) to this test or '
+                                'run pytest with --hypothesis-seed=%(seed)d '
+                                'to reproduce this failure.' % {
+                                    'seed': generated_seed})
+                        else:
+                            report(
+                                'You can add @seed(%d) to this test to '
+                                'reproduce this failure.' % (generated_seed,))
                 raise
 
         for attrib in dir(test):


Test patch:
diff --git a/hypothesis-python/tests/cover/test_seed_printing.py b/hypothesis-python/tests/cover/test_seed_printing.py
--- a/hypothesis-python/tests/cover/test_seed_printing.py
+++ b/hypothesis-python/tests/cover/test_seed_printing.py
@@ -23,7 +23,7 @@
 
 import hypothesis.core as core
 import hypothesis.strategies as st
-from hypothesis import given, assume, settings
+from hypothesis import Verbosity, given, assume, settings
 from hypothesis.errors import FailedHealthCheck
 from tests.common.utils import all_values, capture_out
 from hypothesis.database import InMemoryExampleDatabase
@@ -32,8 +32,9 @@
 
 @pytest.mark.parametrize('in_pytest', [False, True])
 @pytest.mark.parametrize('fail_healthcheck', [False, True])
+@pytest.mark.parametrize('verbosity', [Verbosity.normal, Verbosity.quiet])
 def test_prints_seed_only_on_healthcheck(
-    monkeypatch, in_pytest, fail_healthcheck
+    monkeypatch, in_pytest, fail_healthcheck, verbosity
 ):
     monkeypatch.setattr(core, 'running_under_pytest', in_pytest)
 
@@ -47,7 +48,7 @@ def slow_map(i):
     else:
         expected_exc = AssertionError
 
-    @settings(database=None)
+    @settings(database=None, verbosity=verbosity)
     @given(strategy)
     def test(i):
         assert fail_healthcheck
@@ -60,13 +61,14 @@ def test(i):
 
     seed = test._hypothesis_internal_use_generated_seed
     assert seed is not None
-    if fail_healthcheck:
+    if fail_healthcheck and verbosity != Verbosity.quiet:
         assert '@seed(%d)' % (seed,) in output
         contains_pytest_instruction = (
             '--hypothesis-seed=%d' % (seed,)) in output
         assert contains_pytest_instruction == in_pytest
     else:
         assert '@seed' not in output
+        assert '--hypothesis-seed=%d' % (seed,) not in output
 
 
 def test_uses_global_force(monkeypatch):
diff --git a/hypothesis-python/tests/cover/test_settings.py b/hypothesis-python/tests/cover/test_settings.py
--- a/hypothesis-python/tests/cover/test_settings.py
+++ b/hypothesis-python/tests/cover/test_settings.py
@@ -25,7 +25,7 @@
 import pytest
 
 import hypothesis.strategies as st
-from hypothesis import given, unlimited
+from hypothesis import given, example, unlimited
 from hypothesis.errors import InvalidState, InvalidArgument, \
     HypothesisDeprecationWarning
 from tests.common.utils import validate_deprecation, \
@@ -386,3 +386,11 @@ def test_database_is_reference_preserved():
 def test_setting_use_coverage_is_deprecated(value):
     with validate_deprecation():
         settings(use_coverage=value)
+
+
+@settings(verbosity=Verbosity.verbose)
+@example(x=99)
+@given(st.integers())
+def test_settings_apply_for_explicit_examples(x):
+    # Regression test for #1521
+    assert settings.default.verbosity == Verbosity.verbose
diff --git a/hypothesis-python/tests/cover/test_simple_strings.py b/hypothesis-python/tests/cover/test_simple_strings.py
--- a/hypothesis-python/tests/cover/test_simple_strings.py
+++ b/hypothesis-python/tests/cover/test_simple_strings.py
@@ -66,13 +66,13 @@ def test_finds_single_element_strings():
 def test_binary_respects_changes_in_size():
     @given(binary())
     def test_foo(x):
-        assert len(x) <= 10
+        assert len(x) <= 5
     with pytest.raises(AssertionError):
         test_foo()
 
-    @given(binary(max_size=10))
+    @given(binary(max_size=5))
     def test_foo(x):
-        assert len(x) <= 10
+        assert len(x) <= 5
     test_foo()