Repository: HypothesisWorks/hypothesis

Issue:
Order of `given` and `example` decorators matters
In the [documentation of `hypothesis.given`](https://hypothesis.readthedocs.io/en/latest/reproducing.html#hypothesis.example), it is explicitly stated:

> It doesn’t matter whether you put the example decorator before or after given. Any permutation of the decorators in the above will do the same thing.

However, `example` **must** come before `given`. The below test should fail, however it passes without running the explicit example:

```python
# test_bad.py

from hypothesis import Phase, Verbosity, example, given, settings, strategies

@given(strategies.booleans())
@example(False)
@settings(phases=[Phase.explicit], verbosity=Verbosity.debug)
def test_booleans(boolean: bool) -> None:
    assert boolean

```

Output:

```log
$ pytest -vvv -s
========================================================= test session starts ==========================================================
platform linux -- Python 3.6.8, pytest-5.2.1, py-1.8.0, pluggy-0.13.0 -- /home/tom/reinfer/env/bin/python3.6
cachedir: .pytest_cache
benchmark: 3.2.2 (defaults: timer=time.perf_counter disable_gc=False min_rounds=5 min_time=0.000005 max_time=1.0 calibration_precision=10 warmup=False warmup_iterations=100000)
hypothesis profile 'default' -> database=DirectoryBasedExampleDatabase('/home/tom/Documents/personal/hypothesis-example-not-run/.hypothesis/examples')
rootdir: /home/tom/Documents/personal/hypothesis-example-not-run
plugins: cov-2.8.1, forked-1.1.1, benchmark-3.2.2, xdist-1.30.0, hypothesis-4.42.3, jobserver-1.0.0, flaky-3.6.1, timeout-1.3.3
collected 1 item

test_bad.py::test_booleans PASSED

========================================================== 1 passed in 0.06s ===========================================================
```

The following change makes the test fail, as expected:

```diff
diff --git a/test_bad.py b/test_bad.py
index 2fd0521..7027831 100644
--- a/test_bad.py
+++ b/test_bad.py
@@ -1,8 +1,8 @@
 from hypothesis import Phase, Verbosity, example, given, settings, strategies


-@given(strategies.booleans())
 @example(False)
+@given(strategies.booleans())
 @settings(phases=[Phase.explicit], verbosity=Verbosity.debug)
 def test_booleans(boolean: bool) -> None:
     assert boolean
```

---

Platform: Ubuntu 18.04
Python: 3.6.8
Hypothesis: 4.42.3


Discussion:
Happy to work on a fix for this, if you can point me in the right direction of where to get started.
This was introduced in hypothesis `3.52.0`, in commit  92b2dfd8
Ah, I think I know what's going on: when `@settings` is the innermost decorator we return a wrapper that raises the error if called; and `@given` knows how to retrieve the original test function. If you apply `@example` in between, it adds the data to that shim... and we forget to copy it back later.

So if you look at how those bits work in core.py a solution should emerge... Happy to give more detailed tips if that would help. 

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/_settings.py b/hypothesis-python/src/hypothesis/_settings.py
--- a/hypothesis-python/src/hypothesis/_settings.py
+++ b/hypothesis-python/src/hypothesis/_settings.py
@@ -38,7 +38,7 @@
     InvalidState,
 )
 from hypothesis.internal.compat import integer_types, quiet_raise, string_types
-from hypothesis.internal.reflection import get_pretty_function_description, proxies
+from hypothesis.internal.reflection import get_pretty_function_description
 from hypothesis.internal.validation import check_type, try_convert
 from hypothesis.utils.conventions import UniqueIdentifier, not_set
 from hypothesis.utils.dynamicvariables import DynamicVariable
@@ -238,25 +238,7 @@ def __call__(self, test):
 
         test._hypothesis_internal_use_settings = self
         test._hypothesis_internal_settings_applied = True
-        if getattr(test, "is_hypothesis_test", False):
-            return test
-
-        @proxies(test)
-        def new_test(*args, **kwargs):
-            """@given has not been applied to `test`, so we replace it with this
-            wrapper so that using *only* @settings is an error.
-
-            We then attach the actual test as an attribute of this function, so
-            that we can unwrap it if @given is applied after the settings decorator.
-            """
-            raise InvalidArgument(
-                "Using `@settings` on a test without `@given` is completely pointless."
-            )
-
-        new_test._hypothesis_internal_test_function_without_warning = test
-        new_test._hypothesis_internal_use_settings = self
-        new_test._hypothesis_internal_settings_applied = True
-        return new_test
+        return test
 
     @classmethod
     def _define_setting(
diff --git a/hypothesis-python/src/hypothesis/core.py b/hypothesis-python/src/hypothesis/core.py
--- a/hypothesis-python/src/hypothesis/core.py
+++ b/hypothesis-python/src/hypothesis/core.py
@@ -804,10 +804,6 @@ def given(
     """
 
     def run_test_with_generator(test):
-        if hasattr(test, "_hypothesis_internal_test_function_without_warning"):
-            # Pull out the original test function to avoid the warning we
-            # stuck in about using @settings without @given.
-            test = test._hypothesis_internal_test_function_without_warning
         if inspect.isclass(test):
             # Provide a meaningful error to users, instead of exceptions from
             # internals that assume we're dealing with a function.
diff --git a/hypothesis-python/src/hypothesis/extra/pytestplugin.py b/hypothesis-python/src/hypothesis/extra/pytestplugin.py
--- a/hypothesis-python/src/hypothesis/extra/pytestplugin.py
+++ b/hypothesis-python/src/hypothesis/extra/pytestplugin.py
@@ -23,6 +23,7 @@
 
 from hypothesis import Verbosity, core, settings
 from hypothesis._settings import note_deprecation
+from hypothesis.errors import InvalidArgument
 from hypothesis.internal.compat import OrderedDict, text_type
 from hypothesis.internal.detection import is_hypothesis_test
 from hypothesis.reporting import default as default_reporter, with_reporter
@@ -112,7 +113,15 @@ def pytest_configure(config):
 
 @pytest.hookimpl(hookwrapper=True)
 def pytest_runtest_call(item):
-    if not (hasattr(item, "obj") and is_hypothesis_test(item.obj)):
+    if not hasattr(item, "obj"):
+        yield
+    elif not is_hypothesis_test(item.obj):
+        # If @given was not applied, check whether other hypothesis decorators
+        # were applied, and fail if so
+        if getattr(item.obj, "_hypothesis_internal_settings_applied", False):
+            raise InvalidArgument(
+                "Using `@settings` on a test without `@given` is completely pointless."
+            )
         yield
     else:
         if item.get_closest_marker("parametrize") is not None:


Test patch:
diff --git a/hypothesis-python/tests/cover/test_debug_information.py b/hypothesis-python/tests/cover/test_debug_information.py
--- a/hypothesis-python/tests/cover/test_debug_information.py
+++ b/hypothesis-python/tests/cover/test_debug_information.py
@@ -28,7 +28,7 @@
 
 def test_reports_passes():
     @given(st.integers())
-    @settings(verbosity=Verbosity.debug)
+    @settings(verbosity=Verbosity.debug, max_examples=1000)
     def test(i):
         assert i < 10
 
diff --git a/hypothesis-python/tests/cover/test_settings.py b/hypothesis-python/tests/cover/test_settings.py
--- a/hypothesis-python/tests/cover/test_settings.py
+++ b/hypothesis-python/tests/cover/test_settings.py
@@ -20,12 +20,14 @@
 import datetime
 import subprocess
 import sys
+from unittest import TestCase
 
 import pytest
 
 import hypothesis.strategies as st
 from hypothesis import example, given, unlimited
 from hypothesis._settings import (
+    Phase,
     Verbosity,
     default_variable,
     local_settings,
@@ -36,7 +38,7 @@
 from hypothesis.errors import InvalidArgument, InvalidState
 from hypothesis.stateful import GenericStateMachine, RuleBasedStateMachine, rule
 from hypothesis.utils.conventions import not_set
-from tests.common.utils import checks_deprecated_behaviour, fails_with
+from tests.common.utils import checks_deprecated_behaviour, counts_calls, fails_with
 
 
 def test_has_docstrings():
@@ -239,13 +241,25 @@ def test_settings_in_strategies_are_from_test_scope(s):
     assert s.max_examples == 7
 
 
+TEST_SETTINGS_ALONE = """
+from hypothesis import settings
+from hypothesis.strategies import integers
+
+@settings()
 def test_settings_alone():
-    @settings()
-    def test_nothing():
-        pass
+    pass
+"""
 
-    with pytest.raises(InvalidArgument):
-        test_nothing()
+
+def test_settings_alone(testdir):
+    script = testdir.makepyfile(TEST_SETTINGS_ALONE)
+    result = testdir.runpytest(script)
+    out = "\n".join(result.stdout.lines)
+    assert (
+        "Using `@settings` on a test without `@given` is completely pointless." in out
+    )
+    assert "InvalidArgument" in out
+    assert result.ret == 1
 
 
 @fails_with(InvalidArgument)
@@ -312,6 +326,32 @@ def test_settings_apply_for_explicit_examples(x):
     assert settings.default.verbosity == Verbosity.verbose
 
 
+class TestGivenExampleSettingsExplicitCalled(TestCase):
+    """Real nasty edge case here.
+
+    in #2160, if ``example`` is after ``given`` but before ``settings``,
+    it will be completely ignored.
+
+    If we set phases to only ``explicit``, the test case will never be called!
+
+    We have to run an assertion outside of the test case itself.
+    """
+
+    @counts_calls
+    def call_target(self):
+        pass
+
+    @given(st.booleans())
+    @example(True)
+    @settings(phases=[Phase.explicit])
+    def test_example_explicit(self, x):
+        self.call_target()
+
+    def tearDown(self):
+        # In #2160, this is 0.
+        assert self.call_target.calls == 1
+
+
 def test_setattr_on_settings_singleton_is_error():
     # https://github.com/pandas-dev/pandas/pull/22679#issuecomment-420750921
     # Should be setting attributes on settings.default, not settings!