Repository: HypothesisWorks/hypothesis

Issue:
Deprecate use of other decorators without `@given`
This was first suggested in #1135 for `@settings()`, but the approach taken did not scale as each decorator would have to know about all of the others.  Fortunately, #2162 gave us a much nicer option: we can just check for other decorators in our pytest plugin, when we already know that `@given` was never applied!

That means adding a deprecation warning for each of `@example`, `@seed`, and `@reproduce_failure` based on the special attributes they attach to the test function.


Discussion:
I'm happy to take this, as it'll be identical to my last PR in most ways.
Going to give this a try 😄 

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/extra/pytestplugin.py b/hypothesis-python/src/hypothesis/extra/pytestplugin.py
--- a/hypothesis-python/src/hypothesis/extra/pytestplugin.py
+++ b/hypothesis-python/src/hypothesis/extra/pytestplugin.py
@@ -132,10 +132,24 @@ def pytest_runtest_call(item):
         elif not is_hypothesis_test(item.obj):
             # If @given was not applied, check whether other hypothesis
             # decorators were applied, and raise an error if they were.
+            message = "Using `@%s` on a test without `@given` is completely pointless."
             if getattr(item.obj, "_hypothesis_internal_settings_applied", False):
-                raise InvalidArgument(
-                    "Using `@settings` on a test without `@given` is completely pointless."
+                raise InvalidArgument(message % ("settings",))
+            if getattr(item.obj, "is_hypothesis_strategy_function", False):
+                note_deprecation(
+                    "%s is a function that returns a Hypothesis strategy, but pytest "
+                    "has collected it as a test function.  This is useless as the "
+                    "function body will never be executed.  To define a test "
+                    "function, use @given instead of @composite." % (item.nodeid,),
+                    since="2018-11-02",
                 )
+            for name, attribute in [
+                ("example", "hypothesis_explicit_examples"),
+                ("seed", "_hypothesis_internal_use_seed"),
+                ("reproduce_example", "_hypothesis_internal_use_reproduce_failure"),
+            ]:
+                if hasattr(item.obj, attribute):
+                    note_deprecation(message % (name,), since="RELEASEDAY")
             yield
         else:
             if item.get_closest_marker("parametrize") is not None:
@@ -181,23 +195,8 @@ def pytest_terminal_summary(terminalreporter):
 
     def pytest_collection_modifyitems(items):
         for item in items:
-            if not isinstance(item, pytest.Function):
-                continue
-            if is_hypothesis_test(item.obj):
+            if isinstance(item, pytest.Function) and is_hypothesis_test(item.obj):
                 item.add_marker("hypothesis")
-            if getattr(item.obj, "is_hypothesis_strategy_function", False):
-
-                def note_strategy_is_not_test(*args, **kwargs):
-                    note_deprecation(
-                        "%s is a function that returns a Hypothesis strategy, "
-                        "but pytest has collected it as a test function.  This "
-                        "is useless as the function body will never be executed.  "
-                        "To define a test function, use @given instead of "
-                        "@composite." % (item.nodeid,),
-                        since="2018-11-02",
-                    )
-
-                item.obj = note_strategy_is_not_test
 
 
 def load():


Test patch:
diff --git a/hypothesis-python/tests/cover/test_reproduce_failure.py b/hypothesis-python/tests/cover/test_reproduce_failure.py
--- a/hypothesis-python/tests/cover/test_reproduce_failure.py
+++ b/hypothesis-python/tests/cover/test_reproduce_failure.py
@@ -75,11 +75,18 @@ def test_reproduces_the_failure():
 
     @reproduce_failure(__version__, encode_failure(b))
     @given(st.binary(min_size=n, max_size=n))
-    def test(x):
+    def test_outer(x):
+        assert x != b
+
+    @given(st.binary(min_size=n, max_size=n))
+    @reproduce_failure(__version__, encode_failure(b))
+    def test_inner(x):
         assert x != b
 
     with pytest.raises(AssertionError):
-        test()
+        test_outer()
+    with pytest.raises(AssertionError):
+        test_inner()
 
 
 def test_errors_if_provided_example_does_not_reproduce_failure():
diff --git a/hypothesis-python/tests/pytest/test_checks.py b/hypothesis-python/tests/pytest/test_checks.py
new file mode 100644
--- /dev/null
+++ b/hypothesis-python/tests/pytest/test_checks.py
@@ -0,0 +1,46 @@
+# coding=utf-8
+#
+# This file is part of Hypothesis, which may be found at
+# https://github.com/HypothesisWorks/hypothesis/
+#
+# Most of this work is copyright (C) 2013-2019 David R. MacIver
+# (david@drmaciver.com), but it contains contributions by others. See
+# CONTRIBUTING.rst for a full list of people who may hold copyright, and
+# consult the git log if you need to determine who owns an individual
+# contribution.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at https://mozilla.org/MPL/2.0/.
+#
+# END HEADER
+
+from __future__ import absolute_import, division, print_function
+
+TEST_DECORATORS_ALONE = """
+import hypothesis
+from hypothesis.strategies import composite
+
+@composite
+def test_composite_is_not_a_test(draw):
+    # This strategy will be instantiated, but no draws == no calls.
+    assert False
+
+@hypothesis.seed(0)
+def test_seed_without_given_fails():
+    pass
+
+@hypothesis.example(x=None)
+def test_example_without_given_fails():
+    pass
+
+@hypothesis.reproduce_failure(hypothesis.__version__, b"AA==")
+def test_repro_without_given_fails():
+    pass
+"""
+
+
+def test_decorators_without_given_should_fail(testdir):
+    script = testdir.makepyfile(TEST_DECORATORS_ALONE)
+    testdir.runpytest(script).assert_outcomes(passed=4)
+    testdir.runpytest(script, "-Werror").assert_outcomes(failed=4)