Repository: HypothesisWorks/hypothesis

Issue:
Warn when an `@st.composite` strategy doesn't use the `draw()` function
I've seen a few cases where users have missed the point of `@st.composite`, and ended up using it as a substitute for `st.just()`:

```python
@st.composite
def foo(_):
    return "bar"
```

I propose to fix this by using `inspect.getsource()`+`ast.parse()`+a custom AST visitor, which will raise an explanatory error (initially a deprecation warning) if the `draw` argument is not referenced in the function body.  Note that the name of said argument is determined by the AST of the function signature, and the `ctx` of the node must be `Load()`.


Discussion:
Me, @gzaxel, and @bomtall are going to be tackling this. Wish us luck 😜 

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/internal/reflection.py b/hypothesis-python/src/hypothesis/internal/reflection.py
--- a/hypothesis-python/src/hypothesis/internal/reflection.py
+++ b/hypothesis-python/src/hypothesis/internal/reflection.py
@@ -17,6 +17,7 @@
 import os
 import re
 import sys
+import textwrap
 import types
 from functools import wraps
 from keyword import iskeyword
@@ -219,6 +220,21 @@ def ast_arguments_matches_signature(args, sig):
     return expected == [(p.name, p.kind) for p in sig.parameters.values()]
 
 
+def is_first_param_referenced_in_function(f):
+    """Is the given name referenced within f?"""
+    try:
+        tree = ast.parse(textwrap.dedent(inspect.getsource(f)))
+    except Exception:
+        return True  # Assume it's OK unless we know otherwise
+    name = list(get_signature(f).parameters)[0]
+    return any(
+        isinstance(node, ast.Name)
+        and node.id == name
+        and isinstance(node.ctx, ast.Load)
+        for node in ast.walk(tree)
+    )
+
+
 def extract_all_lambdas(tree, matching_signature):
     lambdas = []
 
diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/core.py b/hypothesis-python/src/hypothesis/strategies/_internal/core.py
--- a/hypothesis-python/src/hypothesis/strategies/_internal/core.py
+++ b/hypothesis-python/src/hypothesis/strategies/_internal/core.py
@@ -44,6 +44,7 @@
 
 import attr
 
+from hypothesis._settings import note_deprecation
 from hypothesis.control import cleanup, note
 from hypothesis.errors import InvalidArgument, ResolutionFailed
 from hypothesis.internal.cathetus import cathetus
@@ -59,6 +60,7 @@
     define_function_signature,
     get_pretty_function_description,
     get_signature,
+    is_first_param_referenced_in_function,
     nicerepr,
     required_args,
 )
@@ -1531,6 +1533,13 @@ def _composite(f):
         )
     if params[0].default is not sig.empty:
         raise InvalidArgument("A default value for initial argument will never be used")
+    if not is_first_param_referenced_in_function(f):
+        note_deprecation(
+            "There is no reason to use @st.composite on a function which "
+            + "does not call the provided draw() function internally.",
+            since="RELEASEDAY",
+            has_codemod=False,
+        )
     if params[0].kind.name != "VAR_POSITIONAL":
         params = params[1:]
     newsig = sig.replace(


Test patch:
diff --git a/hypothesis-python/tests/cover/test_annotations.py b/hypothesis-python/tests/cover/test_annotations.py
--- a/hypothesis-python/tests/cover/test_annotations.py
+++ b/hypothesis-python/tests/cover/test_annotations.py
@@ -83,8 +83,8 @@ def f(*, a, b=2):
         assert convert_positional_arguments(f, (), {})
 
 
-def pointless_composite(draw: None, strat: bool, nothing: list) -> int:
-    return 3
+def to_wrap_with_composite(draw: None, strat: bool, nothing: list) -> int:
+    return draw(st.none())
 
 
 def return_annot() -> int:
@@ -96,7 +96,7 @@ def first_annot(draw: None):
 
 
 def test_composite_edits_annotations():
-    sig_comp = signature(st.composite(pointless_composite))
+    sig_comp = signature(st.composite(to_wrap_with_composite))
     assert sig_comp.return_annotation == st.SearchStrategy[int]
     assert sig_comp.parameters["nothing"].annotation is not P.empty
     assert "draw" not in sig_comp.parameters
@@ -104,7 +104,7 @@ def test_composite_edits_annotations():
 
 @pytest.mark.parametrize("nargs", [1, 2, 3])
 def test_given_edits_annotations(nargs):
-    sig_given = signature(given(*(nargs * [st.none()]))(pointless_composite))
+    sig_given = signature(given(*(nargs * [st.none()]))(to_wrap_with_composite))
     assert sig_given.return_annotation is None
     assert len(sig_given.parameters) == 3 - nargs
     assert all(p.annotation is not P.empty for p in sig_given.parameters.values())
diff --git a/hypothesis-python/tests/cover/test_composite.py b/hypothesis-python/tests/cover/test_composite.py
--- a/hypothesis-python/tests/cover/test_composite.py
+++ b/hypothesis-python/tests/cover/test_composite.py
@@ -11,7 +11,7 @@
 import pytest
 
 from hypothesis import assume, given, strategies as st
-from hypothesis.errors import InvalidArgument
+from hypothesis.errors import HypothesisDeprecationWarning, InvalidArgument
 
 from tests.common.debug import minimal
 from tests.common.utils import flaky
@@ -75,6 +75,14 @@ def foo(**kwargs):
             pass
 
 
+def test_warning_given_no_drawfn_call():
+    with pytest.warns(HypothesisDeprecationWarning):
+
+        @st.composite
+        def foo(_):
+            return "bar"
+
+
 def test_can_use_pure_args():
     @st.composite
     def stuff(*args):
@@ -122,6 +130,7 @@ def test_does_not_change_arguments(data, ls):
     # regression test for issue #1017 or other argument mutation
     @st.composite
     def strat(draw, arg):
+        draw(st.none())
         return arg
 
     ex = data.draw(strat(ls))
diff --git a/hypothesis-python/tests/cover/test_reflection.py b/hypothesis-python/tests/cover/test_reflection.py
--- a/hypothesis-python/tests/cover/test_reflection.py
+++ b/hypothesis-python/tests/cover/test_reflection.py
@@ -27,6 +27,7 @@
     function_digest,
     get_pretty_function_description,
     get_signature,
+    is_first_param_referenced_in_function,
     is_mock,
     proxies,
     repr_call,
@@ -622,3 +623,31 @@ def f(source):
 
     with pytest.raises(ValueError, match="SyntaxError because `from` is a keyword"):
         get_signature(f)
+
+
+def test_param_is_called_within_func():
+    def f(any_name):
+        any_name()
+
+    assert is_first_param_referenced_in_function(f)
+
+
+def test_param_is_called_within_subfunc():
+    def f(any_name):
+        def f2():
+            any_name()
+
+    assert is_first_param_referenced_in_function(f)
+
+
+def test_param_is_not_called_within_func():
+    def f(any_name):
+        pass
+
+    assert not is_first_param_referenced_in_function(f)
+
+
+def test_param_called_within_defaults_on_error():
+    # Create a function object for which we cannot retrieve the source.
+    f = compile("lambda: ...", "_.py", "eval")
+    assert is_first_param_referenced_in_function(f)
diff --git a/hypothesis-python/tests/cover/test_statistical_events.py b/hypothesis-python/tests/cover/test_statistical_events.py
--- a/hypothesis-python/tests/cover/test_statistical_events.py
+++ b/hypothesis-python/tests/cover/test_statistical_events.py
@@ -152,6 +152,7 @@ def test_draw_time_percentage(draw_delay, test_delay):
     def s(draw):
         if draw_delay:
             time.sleep(0.05)
+        draw(st.integers())
 
     @given(s())
     def test(_):
diff --git a/hypothesis-python/tests/nocover/test_baseexception.py b/hypothesis-python/tests/nocover/test_baseexception.py
--- a/hypothesis-python/tests/nocover/test_baseexception.py
+++ b/hypothesis-python/tests/nocover/test_baseexception.py
@@ -34,6 +34,9 @@ def test_exception_propagates_fine_from_strategy(e):
     @composite
     def interrupt_eventually(draw):
         raise e
+        # this line will not be executed, but must be here
+        # to pass draw function static reference check
+        return draw(st.none())
 
     @given(interrupt_eventually())
     def test_do_nothing(x):
@@ -69,13 +72,14 @@ def test_raise_baseexception(x):
     "e", [KeyboardInterrupt, SystemExit, GeneratorExit, ValueError]
 )
 def test_baseexception_in_strategy_no_rerun_no_flaky(e):
-    runs = [0]
+    runs = 0
     interrupt = 3
 
     @composite
     def interrupt_eventually(draw):
-        runs[0] += 1
-        if runs[0] == interrupt:
+        nonlocal runs
+        runs += 1
+        if runs == interrupt:
             raise e
         return draw(integers())
 
@@ -87,7 +91,7 @@ def test_do_nothing(x):
         with pytest.raises(e):
             test_do_nothing()
 
-        assert runs[0] == interrupt
+        assert runs == interrupt
 
     else:
         # Now SystemExit and GeneratorExit are caught like other exceptions
@@ -101,6 +105,9 @@ def test_do_nothing(x):
 @st.composite
 def things(draw):
     raise {exception}
+    # this line will not be executed, but must be here 
+    # to pass draw function static reference check
+    return draw(st.none())
 
 
 @given(st.data(), st.integers())
diff --git a/hypothesis-python/tests/nocover/test_labels.py b/hypothesis-python/tests/nocover/test_labels.py
--- a/hypothesis-python/tests/nocover/test_labels.py
+++ b/hypothesis-python/tests/nocover/test_labels.py
@@ -22,12 +22,12 @@ def test_labels_are_distinct():
 
 @st.composite
 def foo(draw):
-    pass
+    return draw(st.none())
 
 
 @st.composite
 def bar(draw):
-    pass
+    return draw(st.none())
 
 
 def test_different_composites_have_different_labels():
diff --git a/hypothesis-python/tests/pytest/test_capture.py b/hypothesis-python/tests/pytest/test_capture.py
--- a/hypothesis-python/tests/pytest/test_capture.py
+++ b/hypothesis-python/tests/pytest/test_capture.py
@@ -108,10 +108,10 @@ def test_healthcheck_traceback_is_hidden(testdir):
 
 
 COMPOSITE_IS_NOT_A_TEST = """
-from hypothesis.strategies import composite
+from hypothesis.strategies import composite, none
 @composite
 def test_data_factory(draw):
-    pass
+    return draw(none())
 """
 
 
diff --git a/hypothesis-python/tests/pytest/test_checks.py b/hypothesis-python/tests/pytest/test_checks.py
--- a/hypothesis-python/tests/pytest/test_checks.py
+++ b/hypothesis-python/tests/pytest/test_checks.py
@@ -10,12 +10,12 @@
 
 TEST_DECORATORS_ALONE = """
 import hypothesis
-from hypothesis.strategies import composite
+from hypothesis.strategies import composite, none
 
 @composite
 def test_composite_is_not_a_test(draw):
     # This strategy will be instantiated, but no draws == no calls.
-    assert False
+    return draw(none())
 
 @hypothesis.seed(0)
 def test_seed_without_given_fails():
diff --git a/hypothesis-python/tests/typing_extensions/test_backported_types.py b/hypothesis-python/tests/typing_extensions/test_backported_types.py
--- a/hypothesis-python/tests/typing_extensions/test_backported_types.py
+++ b/hypothesis-python/tests/typing_extensions/test_backported_types.py
@@ -10,19 +10,16 @@
 
 import collections
 import sys
-from typing import Callable, Dict, List, Union
+from typing import Callable, DefaultDict, Dict, List, NewType, Type, Union
 
 import pytest
 from typing_extensions import (
     Annotated,
     Concatenate,
-    DefaultDict,
     Literal,
-    NewType,
     NotRequired,
     ParamSpec,
     Required,
-    Type,
     TypedDict,
     TypeGuard,
 )