Repository: HypothesisWorks/hypothesis

Issue:
`st.builds()` generates only text for all `TypeVar`s
I rely on `.builds()` to generate test cases for my *very* annotated function. And I know that there's a bug in it.

Here's the source code: https://github.com/dry-python/returns/pull/543/files#diff-e06c48cc820df2c8762959046a397024R45-R55
Here's the bug context: https://github.com/dry-python/returns/pull/510
TLDR: I need at least one `None` with `Maybe` to catch this bug.

The sad thing is that `hypothesis` is unable to find this bug.
Let's see the simplest reproduction:

```python
from hypothesis import strategies as st, event, given, settings
from typing import Generic, TypeVar, Callable

T = TypeVar('T')
V = TypeVar('V')
U = TypeVar('U')

class MyGeneric(Generic[T]):
    def __init__(self, arg: T) -> T:
        self.arg = arg

def law(
    container: MyGeneric[T],
    first: Callable[[T], V],
    second: Callable[[V], U],
) -> None:
    f = first(...)
    s = second(...)
    assert f is not None  # I hope that at least one assert will fail!
    assert s is not None
    event('Generated first: {0} and second: {1}'.format(f, s))


@given(st.data())
@settings(max_examples=1000)
def test_draw_law(data):
    def factory(thing):
        like = (lambda: None) if len(
            thing.__args__,
        ) == 1 else (lambda *args, **kwargs: None)

        return st.functions(
            like=like,
            returns=st.from_type(thing.__args__[-1]),
            pure=True,
        )

    st.register_type_strategy(Callable, factory)
    st.register_type_strategy(MyGeneric, st.builds(MyGeneric))

    # I hope that his will fail!
    data.draw(st.builds(law))

# But, the test passes :(
```

Here's the full list of things that were generated: https://gist.github.com/sobolevn/593a10232550777fc222def248f647df
There are mostly text entries with just some ints. No `None`, no lists, etc.

Looks like this line is in charge: https://github.com/HypothesisWorks/hypothesis/blob/master/hypothesis-python/src/hypothesis/strategies/_internal/types.py#L228

I think that we can improve it! 
Is there any specific reason why it is `st.text()`?

Related https://github.com/dry-python/returns/pull/543


Discussion:
One more thing. I cannot even fix this with a custom strategy: `st.register_type_strategy(TypeVar, st.just(1))` does not work.
We can definitely improve this - `st.text()` is an arbitrary choice.  To avoid very slow tests we probably want to stick to builtin non-collection types, but that's easy enough: replace

```python
if getattr(thing, "__constraints__", None):
    return st.shared(
        st.sampled_from(thing.__constraints__), key="typevar=%r" % (thing,)
    ).flatmap(st.from_type)
# Constraints may be None or () on various Python versions.
return st.text()  # An arbitrary type for the typevar
```

with 

```python
# Sample from the constraints types if not None or (); or for unconstrained
# typevars limit ourselves to builtin scalars because generating many collections
# can cause substantial slowdowns.
builtin_scalar_types = [NoneType, bool, int, float, str, bytes]
type_strat = st.sampled_from(thing.__constraints__ or builtin_scalar_types)
return st.shared(type_strat, key="typevar=%r" % (thing,)).flatmap(st.from_type)
```

Plus `RELEASE.rst`, tests, you know the drill by now :wink: 
> To avoid very slow tests we probably want to stick to builtin non-collection types, but that's easy enough

I understand the reasoning, but I would love to see at least one mutable type there. Because that's where bugs happen!

> One more thing. I cannot even fix this with a custom strategy

@Zac-HD do you think that this is an issue too? It looks like one to me, but maybe there's something behind this decision I don't know.

Would work on it later today 👍 
> I understand the reasoning, but I would love to see at least one mutable type there. Because that's where bugs happen!

Makes sense - I think adding List and Dict would be OK with max_size=2.

> do you think that [not being able to fix it by registering a strategy] is an issue too? It looks like one to me, but maybe there's something behind this decision I don't know.

I think the rationale would have been if registering didn't work on some older version(s) of Python; and there's a good chance that we no longer need to worry about this.  If converting `TypeVar` handling from an if-clause to a registered generic type works, let's do it :+1: 

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/core.py b/hypothesis-python/src/hypothesis/strategies/_internal/core.py
--- a/hypothesis-python/src/hypothesis/strategies/_internal/core.py
+++ b/hypothesis-python/src/hypothesis/strategies/_internal/core.py
@@ -1468,6 +1468,11 @@ def as_strategy(strat_or_callable, thing, final=True):
     # convert empty results into an explicit error.
     if thing in types._global_type_lookup:
         return as_strategy(types._global_type_lookup[thing], thing)
+    # We also have a special case for TypeVars.
+    # They are represented as instances like `~T` when they come here.
+    # We need to work with their type instead.
+    if isinstance(thing, TypeVar) and type(thing) in types._global_type_lookup:  # type: ignore
+        return as_strategy(types._global_type_lookup[type(thing)], thing)
     # If there's no explicitly registered strategy, maybe a subtype of thing
     # is registered - if so, we can resolve it to the subclass strategy.
     # We'll start by checking if thing is from from the typing module,
diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/types.py b/hypothesis-python/src/hypothesis/strategies/_internal/types.py
--- a/hypothesis-python/src/hypothesis/strategies/_internal/types.py
+++ b/hypothesis-python/src/hypothesis/strategies/_internal/types.py
@@ -161,11 +161,11 @@ def _try_import_forward_ref(thing, bound):  # pragma: no cover
 def from_typing_type(thing):
     # We start with special-case support for Union and Tuple - the latter
     # isn't actually a generic type. Then we handle Literal since it doesn't
-    # support `isinstance`. Support for Callable may be added to this section
-    # later.
+    # support `isinstance`.
+    #
     # We then explicitly error on non-Generic types, which don't carry enough
     # information to sensibly resolve to strategies at runtime.
-    # Finally, we run a variation of the subclass lookup in st.from_type
+    # Finally, we run a variation of the subclass lookup in `st.from_type`
     # among generic types in the lookup.
     #
     # Under 3.6 Union is handled directly in st.from_type, as the argument is
@@ -204,28 +204,6 @@ def from_typing_type(thing):
             else:
                 literals.append(arg)
         return st.sampled_from(literals)
-    if isinstance(thing, typing.TypeVar):
-        if getattr(thing, "__bound__", None) is not None:
-            bound = thing.__bound__
-            if isinstance(bound, ForwardRef):
-                bound = _try_import_forward_ref(thing, bound)
-            strat = unwrap_strategies(st.from_type(bound))
-            if not isinstance(strat, OneOfStrategy):
-                return strat
-            # The bound was a union, or we resolved it as a union of subtypes,
-            # so we need to unpack the strategy to ensure consistency across uses.
-            # This incantation runs a sampled_from over the strategies inferred for
-            # each part of the union, wraps that in shared so that we only generate
-            # from one type per testcase, and flatmaps that back to instances.
-            return st.shared(
-                st.sampled_from(strat.original_strategies), key="typevar=%r" % (thing,)
-            ).flatmap(lambda s: s)
-        if getattr(thing, "__constraints__", None):
-            return st.shared(
-                st.sampled_from(thing.__constraints__), key="typevar=%r" % (thing,)
-            ).flatmap(st.from_type)
-        # Constraints may be None or () on various Python versions.
-        return st.text()  # An arbitrary type for the typevar
     # Now, confirm that we're dealing with a generic type as we expected
     if not isinstance(thing, typing_root_type):  # pragma: no cover
         raise ResolutionFailed("Cannot resolve %s to a strategy" % (thing,))
@@ -601,3 +579,34 @@ def resolve_Callable(thing):
         like=(lambda: None) if len(thing.__args__) == 1 else (lambda *a, **k: None),
         returns=st.from_type(thing.__args__[-1]),
     )
+
+
+@register(typing.TypeVar)
+def resolve_TypeVar(thing):
+    type_var_key = "typevar=%r" % (thing,)
+
+    if getattr(thing, "__bound__", None) is not None:
+        bound = thing.__bound__
+        if isinstance(bound, ForwardRef):
+            bound = _try_import_forward_ref(thing, bound)
+        strat = unwrap_strategies(st.from_type(bound))
+        if not isinstance(strat, OneOfStrategy):
+            return strat
+        # The bound was a union, or we resolved it as a union of subtypes,
+        # so we need to unpack the strategy to ensure consistency across uses.
+        # This incantation runs a sampled_from over the strategies inferred for
+        # each part of the union, wraps that in shared so that we only generate
+        # from one type per testcase, and flatmaps that back to instances.
+        return st.shared(
+            st.sampled_from(strat.original_strategies), key=type_var_key,
+        ).flatmap(lambda s: s)
+
+    builtin_scalar_types = [type(None), bool, int, float, str, bytes]
+    return st.shared(
+        st.sampled_from(
+            # Constraints may be None or () on various Python versions.
+            getattr(thing, "__constraints__", None)
+            or builtin_scalar_types,
+        ),
+        key=type_var_key,
+    ).flatmap(st.from_type)


Test patch:
diff --git a/hypothesis-python/tests/cover/test_lookup.py b/hypothesis-python/tests/cover/test_lookup.py
--- a/hypothesis-python/tests/cover/test_lookup.py
+++ b/hypothesis-python/tests/cover/test_lookup.py
@@ -35,7 +35,13 @@
 
 sentinel = object()
 generics = sorted(
-    (t for t in types._global_type_lookup if isinstance(t, typing_root_type)), key=str
+    (
+        t
+        for t in types._global_type_lookup
+        # We ignore TypeVar, because it is not a Generic type:
+        if isinstance(t, typing_root_type) and t != typing.TypeVar
+    ),
+    key=str,
 )
 xfail_on_39 = () if sys.version_info[:2] < (3, 9) else pytest.mark.xfail
 
@@ -284,6 +290,44 @@ def test_distinct_typevars_same_constraint():
     )
 
 
+def test_distinct_typevars_distinct_type():
+    """Ensures that two different type vars have at least one different type in their strategies."""
+    A = typing.TypeVar("A")
+    B = typing.TypeVar("B")
+    find_any(
+        st.tuples(st.from_type(A), st.from_type(B)),
+        lambda ab: type(ab[0]) != type(ab[1]),  # noqa
+    )
+
+
+@pytest.mark.skipif(sys.version_info < (3, 6), reason="python3.5 is weird")
+@given(st.data())
+def test_same_typevars_same_type(data):
+    """Ensures that single type argument will always have the same type in a single context."""
+    A = typing.TypeVar("A")
+
+    def same_type_args(a: A, b: A):
+        assert type(a) == type(b)
+
+    data.draw(st.builds(same_type_args))
+
+
+def test_typevars_can_be_redefined():
+    """We test that one can register a custom strategy for all type vars."""
+    A = typing.TypeVar("A")
+
+    with temp_registered(typing.TypeVar, st.just(1)):
+        assert_all_examples(st.from_type(A), lambda obj: obj == 1)
+
+
+def test_typevars_can_be_redefine_with_factory():
+    """We test that one can register a custom strategy for all type vars."""
+    A = typing.TypeVar("A")
+
+    with temp_registered(typing.TypeVar, lambda thing: st.just(thing.__name__)):
+        assert_all_examples(st.from_type(A), lambda obj: obj == "A")
+
+
 def annotated_func(a: int, b: int = 2, *, c: int, d: int = 4):
     return a + b + c + d