Repository: HypothesisWorks/hypothesis

Issue:
Incorrect `TypeVar` + `ForwardRef` resolution
Here's a source code that produces an error:

```python
from hypothesis import strategies as st
from typing import TypeVar

_T = TypeVar('_T', bound='Test')

def example(t: _T) -> None:
    return t.arg

class Test(object):
    def __init__(self, arg: int) -> None:
        self.arg = arg

print(st.builds(example).example())
```

It generates this error:

```
» python ex.py
/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/strategies.py:268: NonInteractiveExampleWarning: The `.example()` method is good for exploring strategies, but should only be used interactively.  We recommend using `@given` for tests - it performs better, saves and replays failures to avoid flakiness, and reports minimal examples. (strategy: builds(example))
  warnings.warn(
Traceback (most recent call last):
  File "ex.py", line 13, in <module>
    print(st.builds(example).example())
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/strategies.py", line 314, in example
    example_generating_inner_function()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/strategies.py", line 302, in example_generating_inner_function
    @settings(
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/core.py", line 1001, in wrapped_test
    processed_args = process_arguments_to_given(
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/core.py", line 420, in process_arguments_to_given
    search_strategy.validate()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/strategies.py", line 377, in validate
    self.do_validate()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/collections.py", line 39, in do_validate
    s.validate()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/strategies.py", line 377, in validate
    self.do_validate()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/strategies.py", line 638, in do_validate
    self.mapped_strategy.validate()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/strategies.py", line 377, in validate
    self.do_validate()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/lazy.py", line 118, in do_validate
    w.validate()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/strategies.py", line 377, in validate
    self.do_validate()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/strategies.py", line 638, in do_validate
    self.mapped_strategy.validate()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/strategies.py", line 377, in validate
    self.do_validate()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/collections.py", line 39, in do_validate
    s.validate()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/strategies.py", line 377, in validate
    self.do_validate()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/lazy.py", line 118, in do_validate
    w.validate()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/strategies.py", line 377, in validate
    self.do_validate()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/strategies.py", line 638, in do_validate
    self.mapped_strategy.validate()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/strategies.py", line 377, in validate
    self.do_validate()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/lazy.py", line 118, in do_validate
    w.validate()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/strategies.py", line 377, in validate
    self.do_validate()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/collections.py", line 39, in do_validate
    s.validate()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/strategies.py", line 377, in validate
    self.do_validate()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/strategies.py", line 638, in do_validate
    self.mapped_strategy.validate()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/strategies.py", line 377, in validate
    self.do_validate()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/collections.py", line 39, in do_validate
    s.validate()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/strategies.py", line 377, in validate
    self.do_validate()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/lazy.py", line 118, in do_validate
    w.validate()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/strategies.py", line 378, in validate
    self.is_empty
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/strategies.py", line 125, in accept
    recur(self)
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/strategies.py", line 121, in recur
    mapping[strat] = getattr(strat, calculation)(recur)
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/deferred.py", line 80, in calc_is_empty
    return recur(self.wrapped_strategy)
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/deferred.py", line 43, in wrapped_strategy
    result = self.__definition()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/core.py", line 1385, in <lambda>
    lambda thing: deferred(lambda: _from_type(thing)),
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/core.py", line 1473, in _from_type
    return types.from_typing_type(thing)
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/types.py", line 152, in from_typing_type
    strat = unwrap_strategies(st.from_type(thing.__bound__))
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/lazy.py", line 45, in unwrap_strategies
    result = unwrap_strategies(s.wrapped_strategy)
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/lazy.py", line 45, in unwrap_strategies
    result = unwrap_strategies(s.wrapped_strategy)
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/deferred.py", line 43, in wrapped_strategy
    result = self.__definition()
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/core.py", line 1385, in <lambda>
    lambda thing: deferred(lambda: _from_type(thing)),
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/core.py", line 1473, in _from_type
    return types.from_typing_type(thing)
  File "/Users/sobolev/Documents/github/returns/.venv/lib/python3.8/site-packages/hypothesis/strategies/_internal/types.py", line 229, in from_typing_type
    raise ResolutionFailed(
hypothesis.errors.ResolutionFailed: Could not resolve ForwardRef('Test') to a strategy; consider using register_type_strategy
```

I guess that this happens due to lack of `ForwardRef` handling here: https://github.com/HypothesisWorks/hypothesis/blob/7d6d048ad5df1d01a4ddc90f1b8f2bb45fe2f0de/hypothesis-python/src/hypothesis/strategies/_internal/types.py#L104


Discussion:
As a workaround I use: `st.register_type_strategy(ForwardRef('SupportsEquality'), ...)`
You _might_ be able to work around it with the `from __future__ import annotations` in Python 3.7?  Unfortunately `typing.get_type_hints()` doesn't resolve this `ForwardRef`, and there's not much Hypothesis can do about it.  

Unless, as discussed, we try looking up the `.__module__` attribute and finding the referenced object in the module globals!  This would probably best (if that's the word) attempted in `hypothesis.internal.compat.get_type_hints()`.
@Zac-HD I have implemented this ugly hack we've discussed:

```python
def _try_import_forward_ref(thing, bound):
    module = __import__(thing.__module__)
    return st.from_type(getattr(module, bound.__forward_arg__))
```

And it actually works! 👍 

I will send a PR, I would love to hear you ideas about how we can actually test this. 
Because, well, there are lots of thing to go wrong!

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/types.py b/hypothesis-python/src/hypothesis/strategies/_internal/types.py
--- a/hypothesis-python/src/hypothesis/strategies/_internal/types.py
+++ b/hypothesis-python/src/hypothesis/strategies/_internal/types.py
@@ -131,6 +131,33 @@ def is_generic_type(type_):
     )
 
 
+def _try_import_forward_ref(thing, bound):  # pragma: no cover
+    """
+    Tries to import a real bound type from ``TypeVar`` bound to a ``ForwardRef``.
+
+    This function is very "magical" to say the least, please don't use it.
+    This function fully covered, but is excluded from coverage
+    because we can only cover each path in a separate python version.
+    """
+    try:
+        return typing._eval_type(bound, vars(sys.modules[thing.__module__]), None)
+    except (KeyError, AttributeError, NameError):
+        if (
+            isinstance(thing, typing.TypeVar)
+            and getattr(thing, "__module__", None) == "typing"
+        ):
+            raise ResolutionFailed(
+                "Looks like you are using python3.6 or any of the previous versions "
+                "together with a TypeVar bound to a ForwardRef. "
+                "It is not supported. Upgrading to python3.7+ will solve this problem."
+            ) from None
+        # We fallback to `ForwardRef` instance, you can register it as a type as well:
+        # >>> from typing import ForwardRef
+        # >>> from hypothesis import strategies as st
+        # >>> st.register_type_strategy(ForwardRef('YourType'), your_strategy)
+        return bound
+
+
 def from_typing_type(thing):
     # We start with special-case support for Union and Tuple - the latter
     # isn't actually a generic type. Then we handle Literal since it doesn't
@@ -179,7 +206,10 @@ def from_typing_type(thing):
         return st.sampled_from(literals)
     if isinstance(thing, typing.TypeVar):
         if getattr(thing, "__bound__", None) is not None:
-            strat = unwrap_strategies(st.from_type(thing.__bound__))
+            bound = thing.__bound__
+            if isinstance(bound, ForwardRef):
+                bound = _try_import_forward_ref(thing, bound)
+            strat = unwrap_strategies(st.from_type(bound))
             if not isinstance(strat, OneOfStrategy):
                 return strat
             # The bound was a union, or we resolved it as a union of subtypes,


Test patch:
diff --git a/hypothesis-python/tests/cover/test_type_lookup_forward_ref.py b/hypothesis-python/tests/cover/test_type_lookup_forward_ref.py
new file mode 100644
--- /dev/null
+++ b/hypothesis-python/tests/cover/test_type_lookup_forward_ref.py
@@ -0,0 +1,174 @@
+# This file is part of Hypothesis, which may be found at
+# https://github.com/HypothesisWorks/hypothesis/
+#
+# Most of this work is copyright (C) 2013-2020 David R. MacIver
+# (david@drmaciver.com), but it contains contributions by others. See
+# CONTRIBUTING.rst for a full list of people who may hold copyright, and
+# consult the git log if you need to determine who owns an individual
+# contribution.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at https://mozilla.org/MPL/2.0/.
+#
+# END HEADER
+
+"""
+We need these test to make sure ``TypeVar('X', bound='MyType')`` works correctly.
+
+There was a problem previously that ``bound='MyType'`` was resolved as ``ForwardRef('MyType')``
+which is not a real type. And ``hypothesis`` was not able to generate any meaningful values out of it.
+
+Right here we test different possible outcomes for different Python versions (excluding ``3.5``):
+- Regular case, when ``'MyType'`` can be imported
+- Alias case, when we use type aliases for ``'MyType'``
+- ``if TYPE_CHECKING:`` case, when ``'MyType'`` only exists during type checking and is not importable at all
+- Dot access case, like ``'module.MyType'``
+- Missing case, when there's no ``'MyType'`` at all
+
+We also separate how ``3.6`` works, because it has its limitations.
+Basically, ``TypeVar`` has no information about the module it was defined at.
+"""
+
+import sys
+from typing import TYPE_CHECKING, TypeVar
+
+import pytest
+
+from hypothesis import given, strategies as st
+from hypothesis.errors import ResolutionFailed
+from hypothesis.internal.compat import ForwardRef
+from tests.common import utils
+
+if TYPE_CHECKING:
+    from tests.common.utils import ExcInfo  # we just need any type  # noqa: F401
+
+skip_before_python37 = pytest.mark.skipif(
+    sys.version_info[:2] < (3, 7), reason="typing module was broken"
+)
+
+# Correct:
+
+_Correct = TypeVar("_Correct", bound="CustomType")
+
+
+def correct_fun(thing: _Correct) -> int:
+    return thing.arg
+
+
+class CustomType:
+    def __init__(self, arg: int) -> None:
+        self.arg = arg
+
+
+@skip_before_python37
+@given(st.builds(correct_fun))
+def test_bound_correct_forward_ref(built):
+    """Correct resolution of existing type in ``python3.7+`` codebase."""
+    assert isinstance(built, int)
+
+
+@pytest.mark.skipif(
+    sys.version_info[:2] != (3, 6), reason="typing in python3.6 is partially working",
+)
+def test_bound_correct_forward_ref_python36():
+    """
+    Very special case for ``python3.6`` where we have this feature partially suported.
+
+    Due to ``TypeVar`` module definition bug.
+    """
+    with pytest.raises(ResolutionFailed):
+        st.builds(correct_fun).example()
+
+
+# Alises:
+
+_Alias = TypeVar("_Alias ", bound="OurAlias")
+
+
+def alias_fun(thing: _Alias) -> int:
+    return thing.arg
+
+
+OurAlias = CustomType
+
+
+@skip_before_python37
+@given(st.builds(alias_fun))
+def test_bound_alias_forward_ref(built):
+    """Correct resolution of type aliases in ``python3.7+``."""
+    assert isinstance(built, int)
+
+
+# Dot access:
+
+_CorrectDotAccess = TypeVar("_CorrectDotAccess", bound="utils.ExcInfo")
+_WrongDotAccess = TypeVar("_WrongDotAccess", bound="wrong.ExcInfo")
+_MissingDotAccess = TypeVar("_MissingDotAccess", bound="utils.MissingType")
+
+
+def correct_dot_access_fun(thing: _CorrectDotAccess) -> int:
+    return 1
+
+
+def wrong_dot_access_fun(thing: _WrongDotAccess) -> int:
+    return 1
+
+
+def missing_dot_access_fun(thing: _MissingDotAccess) -> int:
+    return 1
+
+
+@skip_before_python37
+@given(st.builds(correct_dot_access_fun))
+def test_bound_correct_dot_access_forward_ref(built):
+    """Correct resolution of dot access types in ``python3.7+``."""
+    assert isinstance(built, int)
+
+
+@skip_before_python37
+@pytest.mark.parametrize("function", [wrong_dot_access_fun, missing_dot_access_fun])
+def test_bound_missing_dot_access_forward_ref(function):
+    """Resolution of missing type in dot access in ``python3.7+``."""
+    with pytest.raises(ResolutionFailed):
+        st.builds(function).example()
+
+
+# Missing:
+
+_Missing = TypeVar("_Missing", bound="MissingType")
+
+
+def missing_fun(thing: _Missing) -> int:
+    return 1
+
+
+@pytest.mark.skipif(sys.version_info[:2] < (3, 6), reason="typing module was strange")
+def test_bound_missing_forward_ref():
+    """We should raise proper errors on missing types."""
+    with pytest.raises(ResolutionFailed):
+        st.builds(missing_fun).example()
+
+
+# Type checking only:
+
+_TypeChecking = TypeVar("_TypeChecking", bound="ExcInfo")
+
+
+def typechecking_only_fun(thing: _TypeChecking) -> int:
+    return 1
+
+
+@skip_before_python37
+def test_bound_type_cheking_only_forward_ref():
+    """We should fallback to registering explicit ``ForwardRef`` when we have to."""
+    with utils.temp_registered(ForwardRef("ExcInfo"), st.just(1)):
+        st.builds(typechecking_only_fun).example()
+
+
+@skip_before_python37
+def test_bound_type_cheking_only_forward_ref_wrong_type():
+    """We should check ``ForwardRef`` parameter name correctly."""
+    with utils.temp_registered(ForwardRef("WrongType"), st.just(1)):
+        with pytest.raises(ResolutionFailed):
+            st.builds(typechecking_only_fun).example()