Repository: HypothesisWorks/hypothesis

Issue:
Can not run hypothesis as async test with 6.39.0

I use pytest to run the tests:
```python
pytest==7.0.1
pytest-cov==3.0.0
pytest-runner==6.0.0
pytest-asyncio==0.18.1
```

Following test runs fine with 6.38.0:
```python
@given(json_array_gen)
@pytest.mark.asyncio
async def test_foo(elements: List[JsonElement]) -> None:
    pass
```

And fails with 6.39.0:
```
tests/resotocore/web/content_renderer_test.py::test_foo - TypeError: test_foo() takes 0 positional arguments but 1 was given
```

<details>
  <summary>Complete Log</summary>
  
```
============================================================================ test session starts ============================================================================
platform darwin -- Python 3.10.0, pytest-7.0.1, pluggy-1.0.0
rootdir: /Users/matthias/Documents/Work/someeng/resoto/resotocore, configfile: tox.ini
plugins: typeguard-2.13.3, asyncio-0.18.1, hypothesis-6.39.0, cov-3.0.0
asyncio: mode=auto
collected 1 item                                                                                                                                                            

tests/resotocore/web/content_renderer_test.py You can add @seed(187717537307108665848683760785146775420) to this test or run pytest with --hypothesis-seed=187717537307108665848683760785146775420 to reproduce this failure.
F

================================================================================= FAILURES ==================================================================================
_________________________________________________________________________________ test_foo __________________________________________________________________________________

    @given(json_array_gen)
>   @settings(max_examples=20, suppress_health_check=HealthCheck.all())

tests/resotocore/web/content_renderer_test.py:41: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
venv/lib/python3.10/site-packages/hypothesis/internal/conjecture/engine.py:470: in run
    self._run()
venv/lib/python3.10/site-packages/hypothesis/internal/conjecture/engine.py:870: in _run
    self.reuse_existing_examples()
venv/lib/python3.10/site-packages/hypothesis/internal/conjecture/engine.py:548: in reuse_existing_examples
    data = self.cached_test_function(existing)
venv/lib/python3.10/site-packages/hypothesis/internal/conjecture/engine.py:1051: in cached_test_function
    self.test_function(data)
venv/lib/python3.10/site-packages/hypothesis/internal/conjecture/engine.py:208: in test_function
    self.__stoppable_test_function(data)
venv/lib/python3.10/site-packages/hypothesis/internal/conjecture/engine.py:184: in __stoppable_test_function
    self._test_function(data)
venv/lib/python3.10/site-packages/hypothesis/core.py:725: in _execute_once_for_engine
    escalate_hypothesis_internal_error()
venv/lib/python3.10/site-packages/hypothesis/core.py:699: in _execute_once_for_engine
    result = self.execute_once(data)
venv/lib/python3.10/site-packages/hypothesis/core.py:637: in execute_once
    result = self.test_runner(data, run)
venv/lib/python3.10/site-packages/hypothesis/executors.py:47: in default_new_style_executor
    return function(data)
venv/lib/python3.10/site-packages/hypothesis/core.py:633: in run
    return test(*args, **kwargs)
venv/lib/python3.10/site-packages/pytest_asyncio/plugin.py:432: in test_foo
    def inner(**kwargs):
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

args = ([False],), kwargs = {}, initial_draws = 1, start = 540979.612708916

    @proxies(self.test)
    def test(*args, **kwargs):
        self.__test_runtime = None
        initial_draws = len(data.draw_times)
        start = time.perf_counter()
>       result = self.test(*args, **kwargs)
E       TypeError: test_foo() takes 0 positional arguments but 1 was given

venv/lib/python3.10/site-packages/hypothesis/core.py:575: TypeError
-------------------------------------------------------------------------------- Hypothesis ---------------------------------------------------------------------------------
You can add @seed(187717537307108665848683760785146775420) to this test or run pytest with --hypothesis-seed=187717537307108665848683760785146775420 to reproduce this failure.
========================================================================== short test summary info ==========================================================================
FAILED tests/resotocore/web/content_renderer_test.py::test_foo - TypeError: test_foo() takes 0 positional arguments but 1 was given
============================================================================= 1 failed in 0.54s =============================================================================
(resotocore env) 
someeng/resoto/resotocore  mv/bump ✗                                                                                           ⚑ (⎈ |do-fra1-posthog-k8s:posthog) 10:13:26 ⍉
▶ pytest tests/resotocore/web/content_renderer_test.py::test_foo -s
============================================================================ test session starts ============================================================================
platform darwin -- Python 3.10.0, pytest-7.0.1, pluggy-1.0.0
rootdir: /Users/matthias/Documents/Work/someeng/resoto/resotocore, configfile: tox.ini
plugins: typeguard-2.13.3, asyncio-0.18.1, hypothesis-6.39.0, cov-3.0.0
asyncio: mode=auto
collected 1 item                                                                                                                                                            

tests/resotocore/web/content_renderer_test.py You can add @seed(230021627888664504973027713392113020080) to this test or run pytest with --hypothesis-seed=230021627888664504973027713392113020080 to reproduce this failure.
F

================================================================================= FAILURES ==================================================================================
_________________________________________________________________________________ test_foo __________________________________________________________________________________

    @given(json_array_gen)
>   @pytest.mark.asyncio

tests/resotocore/web/content_renderer_test.py:41: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
venv/lib/python3.10/site-packages/hypothesis/internal/conjecture/engine.py:470: in run
    self._run()
venv/lib/python3.10/site-packages/hypothesis/internal/conjecture/engine.py:872: in _run
    self.generate_new_examples()
venv/lib/python3.10/site-packages/hypothesis/internal/conjecture/engine.py:604: in generate_new_examples
    zero_data = self.cached_test_function(bytes(BUFFER_SIZE))
venv/lib/python3.10/site-packages/hypothesis/internal/conjecture/engine.py:1051: in cached_test_function
    self.test_function(data)
venv/lib/python3.10/site-packages/hypothesis/internal/conjecture/engine.py:208: in test_function
    self.__stoppable_test_function(data)
venv/lib/python3.10/site-packages/hypothesis/internal/conjecture/engine.py:184: in __stoppable_test_function
    self._test_function(data)
venv/lib/python3.10/site-packages/hypothesis/core.py:725: in _execute_once_for_engine
    escalate_hypothesis_internal_error()
venv/lib/python3.10/site-packages/hypothesis/core.py:699: in _execute_once_for_engine
    result = self.execute_once(data)
venv/lib/python3.10/site-packages/hypothesis/core.py:637: in execute_once
    result = self.test_runner(data, run)
venv/lib/python3.10/site-packages/hypothesis/executors.py:47: in default_new_style_executor
    return function(data)
venv/lib/python3.10/site-packages/hypothesis/core.py:633: in run
    return test(*args, **kwargs)
venv/lib/python3.10/site-packages/pytest_asyncio/plugin.py:432: in test_foo
    def inner(**kwargs):
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

args = ([],), kwargs = {}, initial_draws = 1, start = 541046.91061275

    @proxies(self.test)
    def test(*args, **kwargs):
        self.__test_runtime = None
        initial_draws = len(data.draw_times)
        start = time.perf_counter()
>       result = self.test(*args, **kwargs)
E       TypeError: test_foo() takes 0 positional arguments but 1 was given

venv/lib/python3.10/site-packages/hypothesis/core.py:575: TypeError
-------------------------------------------------------------------------------- Hypothesis ---------------------------------------------------------------------------------
You can add @seed(230021627888664504973027713392113020080) to this test or run pytest with --hypothesis-seed=230021627888664504973027713392113020080 to reproduce this failure.
========================================================================== short test summary info ==========================================================================
FAILED tests/resotocore/web/content_renderer_test.py::test_foo - TypeError: test_foo() takes 0 positional arguments but 1 was given
============================================================================= 1 failed in 0.54s =============================================================================
(resotocore env) 
```
</details>

It looks like the combination with pytest-async leads to confusion.
This blocks us from upgrading to latest hypothesis.



Discussion:
Thanks for reporting - I'll aim to have a fix, or at worst revert, out without 24 hours. 
In case this is useful, I hit this problem as well and did a small amount of digging.

Locally, reverting the changes to [hypothesis.internal.reflection.proxies](https://github.com/HypothesisWorks/hypothesis/pull/3241/files#diff-5ea1bac91b1b87477f70e8944d06ce308204b2d6b2f42c617e49aaa150060351R686) is sufficient for tests to pass.

I've also noticed that the [inner](https://github.com/pytest-dev/pytest-asyncio/blob/master/pytest_asyncio/plugin.py#L435) function used by `pytest-asyncio` only accepts kwargs. Changing it to `def inner(*args, **kwargs)` also fixes tests locally.

Not super familiar with hypothesis internals, but my quick guess is that previously, when using `reflection.getfullargspec_except_self`, hypothesis did not detect positional args in the function after being wrapped by `inner` and passed it values as kwargs. 

Now, with `reflection.get_signature`, it _does_ detect positional args and then tries to pass values as positional args to `inner`, which fails.

Here's an example.

Definitions and imports:
```python
import asyncio

import hypothesis.internal.reflection
import pytest_asyncio

async def some_async_fn(a,b):
    return a==b

wrapped_async_fn = pytest_asyncio.plugin.wrap_in_sync(None, some_async_fn, asyncio.get_event_loop())
```
Output:
```python
# Raw fn can handle both args and kwargs.
In [33]: await some_async_fn(1,2)
Out[33]: False

In [34]: await some_async_fn(a=1,b=2)
Out[34]: False

# Wrapped fn can only handle kwargs.
In [35]: wrapped_async_fn(1,2)
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Input In [35], in <cell line: 1>()
----> 1 wrapped_async_fn(1,2)

TypeError: inner() takes 0 positional arguments but 2 were given

# This works.
In [36]: wrapped_async_fn(a=1,b=2)


# Old method of getting signature does not detect positional args for wrapped fn.
In [37]: hypothesis.internal.reflection.getfullargspec_except_self(some_async_fn)
Out[37]: FullArgSpec(args=['a', 'b'], varargs=None, varkw=None, defaults=None, kwonlyargs=[], kwonlydefaults=None, annotations={})

In [38]: hypothesis.internal.reflection.getfullargspec_except_self(wrapped_async_fn)
Out[38]: FullArgSpec(args=[], varargs=None, varkw='kwargs', defaults=None, kwonlyargs=[], kwonlydefaults=None, annotations={})

# New method has same output for both wrapped and raw.
In [39]: {k:v.kind for k,v in hypothesis.internal.reflection.get_signature(some_async_fn).parameters.items()}
Out[39]:
{'a': <_ParameterKind.POSITIONAL_OR_KEYWORD: 1>,
 'b': <_ParameterKind.POSITIONAL_OR_KEYWORD: 1>}

In [40]: {k:v.kind for k,v in hypothesis.internal.reflection.get_signature(wrapped_async_fn).parameters.items()}
Out[40]:
{'a': <_ParameterKind.POSITIONAL_OR_KEYWORD: 1>,
 'b': <_ParameterKind.POSITIONAL_OR_KEYWORD: 1>}
```
> I've also noticed that the [inner](https://github.com/pytest-dev/pytest-asyncio/blob/master/pytest_asyncio/plugin.py#L435) function used by `pytest-asyncio` only accepts kwargs. Changing it to `def inner(*args, **kwargs)` also fixes tests locally.

Ah, that's the bug then, they're ~~misusing our internal `@proxies` helper~~.

Nope, it's our fault, we changed from passing by name to passing positionally.  This is a bad breaking change - I'll revert tonight and then open an issue to fix it properly soonish.
Prior to https://github.com/HypothesisWorks/hypothesis/pull/3241, `@given` did not follow wrapper chains:

```python
def f(): ...

@given(st.integers())
@wraps(f)
def test(x): pass
```

Before #3241:

```python
>>> test()  # runs without problem
```

After #3241:

```python
>>> test() 
TypeError: f() got an unexpected keyword argument 'x'
```

I believe this is due to changing `getfullargspec(f)` to `signature(f)` . The former does *not* follow wrapper chains, whereas the latter does by default.
> I believe this is due to changing `getfullargspec(f)` to `signature(f)` . The former does not follow wrapper chains, whereas the latter does by default.

*That part* is fine, the bit we need to revert is where we were passing arguments by name and now we're doing it by position.

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/internal/reflection.py b/hypothesis-python/src/hypothesis/internal/reflection.py
--- a/hypothesis-python/src/hypothesis/internal/reflection.py
+++ b/hypothesis-python/src/hypothesis/internal/reflection.py
@@ -84,7 +84,7 @@ def function_digest(function):
     return hasher.digest()
 
 
-def get_signature(target):
+def get_signature(target, *, follow_wrapped=True):
     # Special case for use of `@unittest.mock.patch` decorator, mimicking the
     # behaviour of getfullargspec instead of reporting unusable arguments.
     patches = getattr(target, "patchings", None)
@@ -119,7 +119,7 @@ def get_signature(target):
             return sig.replace(
                 parameters=[v for k, v in sig.parameters.items() if k != "self"]
             )
-    return inspect.signature(target)
+    return inspect.signature(target, follow_wrapped=follow_wrapped)
 
 
 def arg_is_required(param):
@@ -605,7 +605,7 @@ def define_function_signature_from_signature(name, docstring, signature):
     ]
 
     def accept(f):
-        fsig = inspect.signature(f)
+        fsig = inspect.signature(f, follow_wrapped=False)
         must_pass_as_kwargs = []
         invocation_parts = []
         for p in pos_args:
@@ -687,7 +687,7 @@ def proxies(target: "T") -> Callable[[Callable], "T"]:
     replace_sig = define_function_signature_from_signature(
         target.__name__.replace("<lambda>", "_lambda_"),  # type: ignore
         target.__doc__,
-        get_signature(target),
+        get_signature(target, follow_wrapped=False),
     )
 
     def accept(proxy):


Test patch:
diff --git a/hypothesis-python/tests/nocover/test_modify_inner_test.py b/hypothesis-python/tests/nocover/test_modify_inner_test.py
--- a/hypothesis-python/tests/nocover/test_modify_inner_test.py
+++ b/hypothesis-python/tests/nocover/test_modify_inner_test.py
@@ -91,3 +91,14 @@ def invalid_test(x):
         invalid_test()
 
     assert invalid_test.hypothesis.inner_test == original
+
+
+def test_invokes_inner_function_with_args_by_name():
+    # Regression test for https://github.com/HypothesisWorks/hypothesis/issues/3245
+    @given(st.integers())
+    def test(x):
+        pass
+
+    f = test.hypothesis.inner_test
+    test.hypothesis.inner_test = wraps(f)(lambda **kw: f(**kw))
+    test()