Repository: HypothesisWorks/hypothesis

Issue:
Make @composite strategies collected by pytest act as failing tests
From #1658, occasionally users use `@composite` when they should be using `@given(st.data())`.  Unfortunately, this silently goes wrong:

```python
from hypothesis.strategies import composite

@composite
def test_data_factory(draw):
    foo = draw(custom_objects)
    assert foo.check_bar()
```

Pytest will happily collect `test_data_factory`, execute it, and report it as a passing test... *even if you replace the function body with `assert False`!*  This is because calling the function returns a `LazyStrategy` object, and the function isn't actually executed until later.  More generally, it's because `@composite` should not be used to define test functions!

So it would be good to add a check to our pytest plugin, in [`pytest_collection_modifyitems`](https://github.com/HypothesisWorks/hypothesis/blob/da43eba90c1fbfa436c02e621f56dfa3a153a093/hypothesis-python/src/hypothesis/extra/pytestplugin.py#L178-L183), that replaces the collected `item.obj` with a function that will (a) fail, and (b) do so with a useful error suggesting that the user either rename the strategy or if intending to execute it as a test use `@given(st.data())` instead.


Discussion:

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/extra/pytestplugin.py b/hypothesis-python/src/hypothesis/extra/pytestplugin.py
--- a/hypothesis-python/src/hypothesis/extra/pytestplugin.py
+++ b/hypothesis-python/src/hypothesis/extra/pytestplugin.py
@@ -20,6 +20,7 @@
 import pytest
 
 from hypothesis import Verbosity, core, settings
+from hypothesis._settings import note_deprecation
 from hypothesis.reporting import default as default_reporter
 from hypothesis.reporting import with_reporter
 from hypothesis.statistics import collector
@@ -179,8 +180,19 @@ def pytest_collection_modifyitems(items):
     for item in items:
         if not isinstance(item, pytest.Function):
             continue
-        if getattr(item.function, 'is_hypothesis_test', False):
+        if is_hypothesis_test(item.obj):
             item.add_marker('hypothesis')
+        if getattr(item.obj, 'is_hypothesis_strategy_function', False):
+            def note_strategy_is_not_test(*args, **kwargs):
+                note_deprecation(
+                    '%s is a function that returns a Hypothesis strategy, '
+                    'but pytest has collected it as a test function.  This '
+                    'is useless as the function body will never be executed.'
+                    'To define a test function, use @given instead of '
+                    '@composite.' % (item.nodeid,)
+                )
+
+            item.obj = note_strategy_is_not_test
 
 
 def load():
diff --git a/hypothesis-python/src/hypothesis/strategies.py b/hypothesis-python/src/hypothesis/strategies.py
--- a/hypothesis-python/src/hypothesis/strategies.py
+++ b/hypothesis-python/src/hypothesis/strategies.py
@@ -196,6 +196,8 @@ def accept(*args, **kwargs):
                 result.force_has_reusable_values = True
                 assert result.has_reusable_values
             return result
+
+        accept.is_hypothesis_strategy_function = True
         return accept
     return decorator
 


Test patch:
diff --git a/hypothesis-python/tests/pytest/test_capture.py b/hypothesis-python/tests/pytest/test_capture.py
--- a/hypothesis-python/tests/pytest/test_capture.py
+++ b/hypothesis-python/tests/pytest/test_capture.py
@@ -152,3 +152,20 @@ def test_healthcheck_traceback_is_hidden(testdir):
     def_line = get_line_num(def_token, result)
     timeout_line = get_line_num(timeout_token, result)
     assert timeout_line - def_line == 6
+
+
+COMPOSITE_IS_NOT_A_TEST = """
+from hypothesis.strategies import composite
+@composite
+def test_data_factory(draw):
+    assert False, 'Unreachable due to lazy construction'
+"""
+
+
+@pytest.mark.skipif(pytest.__version__[:3] == '3.0', reason='very very old')
+def test_deprecation_of_strategies_as_tests(testdir):
+    script = testdir.makepyfile(COMPOSITE_IS_NOT_A_TEST)
+    testdir.runpytest(script, '-Werror').assert_outcomes(failed=1)
+    result = testdir.runpytest(script)
+    result.assert_outcomes(passed=1)
+    result.stdout.fnmatch_lines(['*HypothesisDeprecationWarning*'])