Repository: HypothesisWorks/hypothesis

Issue:
confusing errors when using hypothesis with pytest-asyncio
given the following invalid test as `foo` does not exist...

```python
import hypothesis
import pytest

@pytest.mark.asyncio
@hypothesis.given(foo=hypothesis.infer)
async def test_foo() -> None:
    pass
```

pytest will error the test with the following...

```python-traceback
item = <Function test_foo>

    def pytest_runtest_setup(item):
        if 'asyncio' in item.keywords:
            # inject an event loop fixture for all async tests
            if 'event_loop' in item.fixturenames:
                item.fixturenames.remove('event_loop')
            item.fixturenames.insert(0, 'event_loop')
        if item.get_closest_marker("asyncio") is not None \
            and not getattr(item.obj, 'hypothesis', False) \
            and getattr(item.obj, 'is_hypothesis_test', False):
>               pytest.fail(
                    'test function `%r` is using Hypothesis, but pytest-asyncio '
                    'only works with Hypothesis 3.64.0 or later.' % item
                )
E               Failed: test function `<Function test_foo>` is using Hypothesis, but pytest-asyncio only works with Hypothesis 3.64.0 or later.
```

without the async mark

```python
import hypothesis
import pytest

@hypothesis.given(foo=hypothesis.infer)
async def test_foo() -> None:
    pass
```

you get the real error back....

```python-traceback
arguments = (), kwargs = {}

    def wrapped_test(*arguments, **kwargs):
>       raise InvalidArgument(message)
E       hypothesis.errors.InvalidArgument: test_foo() got an unexpected keyword argument 'foo', from `foo=infer` in @given
```

I am assuming it fails because the `invalid()` test returned by given has `is_hypothesis_test` set, but does not also set `hypothesis` to `HypothesisHandle`. and the hypotheisis detection code in pytest-asyncio relies on both or neither being present.

```
platform linux -- Python 3.9.4, pytest-6.2.4, py-1.10.0, pluggy-0.13.1
rootdir: /workspace, configfile: pyproject.toml
plugins: asyncio-0.14.0, hypothesis-6.12.0
```


Discussion:
Yep, looks like you're exactly right.  Want to open a PR to fix it?  If not, we'll be happy to fix

https://github.com/HypothesisWorks/hypothesis/blob/06aa7b45e57ac90e3c66e24b0b8157d70af1aa08/hypothesis-python/src/hypothesis/core.py#L238-L243
I am happy to make a PR, but what would the preferred fix be? Deleting the `is_hypothesis_test` property or adding the `hypothesis` property?
Adding the `.hypothesis` property would be best! 

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/core.py b/hypothesis-python/src/hypothesis/core.py
--- a/hypothesis-python/src/hypothesis/core.py
+++ b/hypothesis-python/src/hypothesis/core.py
@@ -226,7 +226,7 @@ def __repr__(self):
         return f"WithRunner({self.mapped_strategy!r}, runner={self.runner!r})"
 
 
-def is_invalid_test(name, original_argspec, given_arguments, given_kwargs):
+def is_invalid_test(test, original_argspec, given_arguments, given_kwargs):
     """Check the arguments to ``@given`` for basic usage constraints.
 
     Most errors are not raised immediately; instead we return a dummy test
@@ -240,6 +240,11 @@ def wrapped_test(*arguments, **kwargs):
             raise InvalidArgument(message)
 
         wrapped_test.is_hypothesis_test = True
+        wrapped_test.hypothesis = HypothesisHandle(
+            inner_test=test,
+            get_fuzz_target=wrapped_test,
+            given_kwargs=given_kwargs,
+        )
         return wrapped_test
 
     if not (given_arguments or given_kwargs):
@@ -258,7 +263,7 @@ def wrapped_test(*arguments, **kwargs):
         return invalid(
             "Too many positional arguments for %s() were passed to @given "
             "- expected at most %d arguments, but got %d %r"
-            % (name, len(original_argspec.args), len(args), args)
+            % (test.__name__, len(original_argspec.args), len(args), args)
         )
 
     if infer in given_arguments:
@@ -278,7 +283,7 @@ def wrapped_test(*arguments, **kwargs):
         arg = extra_kwargs[0]
         return invalid(
             "%s() got an unexpected keyword argument %r, from `%s=%r` in @given"
-            % (name, arg, arg, given_kwargs[arg])
+            % (test.__name__, arg, arg, given_kwargs[arg])
         )
     if original_argspec.defaults or original_argspec.kwonlydefaults:
         return invalid("Cannot apply @given to a function with defaults.")
@@ -959,7 +964,7 @@ def run_test_as_given(test):
         original_argspec = getfullargspec(test)
 
         check_invalid = is_invalid_test(
-            test.__name__, original_argspec, given_arguments, given_kwargs
+            test, original_argspec, given_arguments, given_kwargs
         )
 
         # If the argument check found problems, return a dummy test function


Test patch:
diff --git a/hypothesis-python/tests/cover/test_fuzz_one_input.py b/hypothesis-python/tests/cover/test_fuzz_one_input.py
--- a/hypothesis-python/tests/cover/test_fuzz_one_input.py
+++ b/hypothesis-python/tests/cover/test_fuzz_one_input.py
@@ -21,6 +21,7 @@
 
 from hypothesis import Phase, given, settings, strategies as st
 from hypothesis.database import InMemoryExampleDatabase
+from hypothesis.errors import InvalidArgument
 from hypothesis.internal.conjecture.shrinker import sort_key
 
 
@@ -155,3 +156,14 @@ def test(s):
     (saved_examples,) = db.data.values()
     assert seen == buffers
     assert len(saved_examples) == db_size
+
+
+def test_fuzzing_invalid_test_raises_error():
+    # Invalid: @given with too many positional arguments
+    @given(st.integers(), st.integers())
+    def invalid_test(s):
+        pass
+
+    with pytest.raises(InvalidArgument, match="Too many positional arguments"):
+        # access the property to check error happens during setup
+        invalid_test.hypothesis.fuzz_one_input
diff --git a/hypothesis-python/tests/nocover/test_modify_inner_test.py b/hypothesis-python/tests/nocover/test_modify_inner_test.py
--- a/hypothesis-python/tests/nocover/test_modify_inner_test.py
+++ b/hypothesis-python/tests/nocover/test_modify_inner_test.py
@@ -18,6 +18,7 @@
 import pytest
 
 from hypothesis import given, strategies as st
+from hypothesis.errors import InvalidArgument
 
 
 def always_passes(*args, **kwargs):
@@ -62,3 +63,36 @@ def test_can_replace_when_parametrized(x, y):
 
 
 test_can_replace_when_parametrized.hypothesis.inner_test = always_passes
+
+
+def test_can_replace_when_original_is_invalid():
+    # Invalid: @given with too many positional arguments
+    @given(st.integers(), st.integers())
+    def invalid_test(x):
+        assert False
+
+    invalid_test.hypothesis.inner_test = always_passes
+
+    # Even after replacing the inner test, calling the wrapper should still
+    # fail.
+    with pytest.raises(InvalidArgument, match="Too many positional arguments"):
+        invalid_test()
+
+
+def test_inner_is_original_even_when_invalid():
+    def invalid_test(x):
+        assert False
+
+    original = invalid_test
+
+    # Invalid: @given with no arguments
+    invalid_test = given()(invalid_test)
+
+    # Verify that the test is actually invalid
+    with pytest.raises(
+        InvalidArgument,
+        match="given must be called with at least one argument",
+    ):
+        invalid_test()
+
+    assert invalid_test.hypothesis.inner_test == original