Repository: HypothesisWorks/hypothesis

Issue:
`hypothesis@6.14.8` stopped working for `dry-python/returns`
Dependabot PR, that updates version `6.14.7` to `6.14.8`, revealed that hypothesis has stopped working for us: https://github.com/dry-python/returns/pull/1028/checks?check_run_id=3351483158

Previous version did work fine: https://github.com/dry-python/returns/commit/50f06330c00c5c8a5538b192bb73bed109a052a0

Logs:
```
You can reproduce this example by temporarily adding @reproduce_failure('6.14.8', b'AA==') as a decorator on your test case
_______________________ test_pair_altablen_identity_law ________________________

    def factory(source: st.DataObject) -> None:
>       with type_vars():

returns/contrib/hypothesis/laws.py:204: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.venv/lib/python3.9/site-packages/hypothesis/core.py:624: in execute_once
    result = self.test_runner(data, run)
.venv/lib/python3.9/site-packages/hypothesis/executors.py:52: in default_new_style_executor
    return function(data)
.venv/lib/python3.9/site-packages/hypothesis/core.py:620: in run
    return test(*args, **kwargs)
returns/contrib/hypothesis/laws.py:204: in factory
    with type_vars():
.venv/lib/python3.9/site-packages/hypothesis/core.py:562: in test
    result = self.test(*args, **kwargs)
returns/contrib/hypothesis/laws.py:207: in factory
    source.draw(st.builds(law.definition))
.venv/lib/python3.9/site-packages/hypothesis/strategies/_internal/core.py:1639: in draw
    result = self.conjecture_data.draw(strategy)
.venv/lib/python3.9/site-packages/hypothesis/internal/conjecture/data.py:871: in draw
    strategy.validate()
.venv/lib/python3.9/site-packages/hypothesis/strategies/_internal/strategies.py:400: in validate
    self.do_validate()
.venv/lib/python3.9/site-packages/hypothesis/strategies/_internal/lazy.py:135: in do_validate
    w.validate()
.venv/lib/python3.9/site-packages/hypothesis/strategies/_internal/core.py:792: in validate
    fixed_dictionaries(self.kwargs).validate()
.venv/lib/python3.9/site-packages/hypothesis/strategies/_internal/strategies.py:400: in validate
    self.do_validate()
.venv/lib/python3.9/site-packages/hypothesis/strategies/_internal/lazy.py:135: in do_validate
    w.validate()
.venv/lib/python3.9/site-packages/hypothesis/strategies/_internal/strategies.py:400: in validate
    self.do_validate()
.venv/lib/python3.9/site-packages/hypothesis/strategies/_internal/strategies.py:800: in do_validate
    self.mapped_strategy.validate()
.venv/lib/python3.9/site-packages/hypothesis/strategies/_internal/strategies.py:400: in validate
    self.do_validate()
.venv/lib/python3.9/site-packages/hypothesis/strategies/_internal/collections.py:46: in do_validate
    s.validate()
.venv/lib/python3.9/site-packages/hypothesis/strategies/_internal/strategies.py:400: in validate
    self.do_validate()
.venv/lib/python3.9/site-packages/hypothesis/strategies/_internal/lazy.py:135: in do_validate
    w.validate()
.venv/lib/python3.9/site-packages/hypothesis/strategies/_internal/strategies.py:401: in validate
    self.is_empty
.venv/lib/python3.9/site-packages/hypothesis/strategies/_internal/strategies.py:135: in accept
    recur(self)
.venv/lib/python3.9/site-packages/hypothesis/strategies/_internal/strategies.py:131: in recur
    mapping[strat] = getattr(strat, calculation)(recur)
.venv/lib/python3.9/site-packages/hypothesis/strategies/_internal/deferred.py:70: in calc_is_empty
    return recur(self.wrapped_strategy)
            return st.from_type(annotated_type)
        # Now, confirm that we're dealing with a generic type as we expected
        if sys.version_info[:2] < (3, 9) and not isinstance(
            thing, typing_root_type
        ):  # pragma: no cover
            raise ResolutionFailed(f"Cannot resolve {thing} to a strategy")
    
        # Some "generic" classes are not generic *in* anything - for example both
        # Hashable and Sized have `__args__ == ()` on Python 3.7 or later.
        # (In 3.6 they're just aliases for the collections.abc classes)
        origin = getattr(thing, "__origin__", thing)
        if (
            typing.Hashable is not collections.abc.Hashable
            and origin in vars(collections.abc).values()
            and len(getattr(thing, "__args__", None) or []) == 0
        ):
            return st.from_type(origin)
    
        # Parametrised generic types have their __origin__ attribute set to the
        # un-parametrised version, which we need to use in the subclass checks.
        # e.g.:     typing.List[int].__origin__ == typing.List
        mapping = {
            k: v
            for k, v in _global_type_lookup.items()
            if is_generic_type(k) and try_issubclass(k, thing)
        }
        if typing.Dict in mapping or typing.Set in mapping:
            # ItemsView can cause test_lookup.py::test_specialised_collection_types
            # to fail, due to weird isinstance behaviour around the elements.
            mapping.pop(typing.ItemsView, None)
            if sys.version_info[:2] == (3, 6):  # pragma: no cover
                # `isinstance(dict().values(), Container) is False` on py36 only -_-
                mapping.pop(typing.ValuesView, None)
        if typing.Deque in mapping and len(mapping) > 1:
            # Resolving generic sequences to include a deque is more trouble for e.g.
            # the ghostwriter than it's worth, via undefined names in the repr.
            mapping.pop(typing.Deque)
        if len(mapping) > 1:
            # issubclass treats bytestring as a kind of sequence, which it is,
            # but treating it as such breaks everything else when it is presumed
            # to be a generic sequence or container that could hold any item.
            # Except for sequences of integers, or unions which include integer!
            # See https://github.com/HypothesisWorks/hypothesis/issues/2257
            #
            # This block drops ByteString from the types that can be generated
            # if there is more than one allowed type, and the element type is
            # not either `int` or a Union with `int` as one of its elements.
            elem_type = (getattr(thing, "__args__", None) or ["not int"])[0]
            if getattr(elem_type, "__origin__", None) is typing.Union:
                union_elems = elem_type.__args__
else:
                union_elems = ()
            if not any(
                isinstance(T, type) and issubclass(int, T)
                for T in list(union_elems) + [elem_type]
            ):
                mapping.pop(typing.ByteString, None)
        elif (
            (not mapping)
            and isinstance(thing, ForwardRef)
            and thing.__forward_arg__ in vars(builtins)
        ):
            return st.from_type(getattr(builtins, thing.__forward_arg__))
        strategies = [
            v if isinstance(v, st.SearchStrategy) else v(thing)
            for k, v in mapping.items()
            if sum(try_issubclass(k, T) for T in mapping) == 1
        ]
        empty = ", ".join(repr(s) for s in strategies if s.is_empty)
        if empty or not strategies:
>           raise ResolutionFailed(
                f"Could not resolve {empty or thing} to a strategy; "
                "consider using register_type_strategy"
            )
E           hypothesis.errors.ResolutionFailed: Could not resolve returns.interfaces.altable.AltableN[~_FirstType, ~_SecondType, ~_ThirdType] to a strategy; consider using register_type_strategy

.venv/lib/python3.9/site-packages/hypothesis/strategies/_internal/types.py:334: ResolutionFailed
---------------------------------- Hypothesis ----------------------------------
Falsifying example: factory(
    source=data(...),
)
```


Discussion:
Ah, 🤬, sorry.  Looks like fixing #2951 ([changelog](https://hypothesis.readthedocs.io/en/latest/changes.html#v6-14-8)) broke this for you.

I'm pretty sure I can get this working again without re-breaking @mristin's code, but it's going to be fiddly.  Could you provide a minimal independent reproducer, suitable for use in Hypothesis' test suite, so that I can confirm that the fix works before publishing?
Yes, sure! I will figure out what is the minimal example and submit it! 👍 
After further inspection, I've noticed several things:
1. Failures on different python versions differ. Here are just numbers of failed tests on different versions (all versions run the exact same test suite without any version-specific excludes):
    - 3.7 = 38 failed
    - 3.9 = 33 failed
    - 3.9 = 26 failed
2. There are different problems with tests (they have the same type on all versions, but they happen in different places on different versions):
    - First one (the most popular): `hypothesis.errors.ResolutionFailed: Could not resolve returns.interfaces.applicative.ApplicativeN[~_FirstType, ~_SecondType, ~_ThirdType] to a strategy; consider using register_type_strategy` It happens for different types, not just this one. Example: https://github.com/dry-python/returns/runs/3351874961?check_suite_focus=true#step:7:561
    - `TypeError` when it should not happen, example: https://github.com/dry-python/returns/runs/3351874819?check_suite_focus=true#step:7:5090 It looks like a result of invalid strategy resolution
    - `AssertionError`, when the internal test logic does not work. It happens from time to time in my local env: `AssertionError: <RequiresContextResult: <function RequiresContextResult.bind.<locals>.<lambda> at 0x10c306b80>> == <RequiresContextResult: <function RequiresContextResult.bind.<locals>.<lambda> at 0x10c1c3ca0>>` Before we didn't have any flaky tests.


Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/types.py b/hypothesis-python/src/hypothesis/strategies/_internal/types.py
--- a/hypothesis-python/src/hypothesis/strategies/_internal/types.py
+++ b/hypothesis-python/src/hypothesis/strategies/_internal/types.py
@@ -87,6 +87,20 @@ def type_sorting_key(t):
     return (int(issubclass(t, collections.abc.Container)), repr(t))
 
 
+def _compatible_args(args, superclass_args):
+    """Check that the args of two generic types are compatible for try_issubclass."""
+    assert superclass_args is not None
+    if args is None:
+        return True
+    return len(args) == len(superclass_args) and all(
+        # "a==b or either is a typevar" is a hacky approximation, but it's
+        # good enough for all the cases that I've seen so far and has the
+        # substantial virtue of (relative) simplicity.
+        a == b or isinstance(a, typing.TypeVar) or isinstance(b, typing.TypeVar)
+        for a, b in zip(args, superclass_args)
+    )
+
+
 def try_issubclass(thing, superclass):
     try:
         # In this case we're looking at two distinct classes - which might be generics.
@@ -106,26 +120,12 @@ def try_issubclass(thing, superclass):
             # generics, and so on.  If you need to change this code, read PEP-560
             # and Hypothesis issue #2951 closely first, and good luck.  The tests
             # will help you, I hope - good luck.
-            thing_args = getattr(thing, "__args__", None)
-            thing_orig_bases = getattr(thing, "__orig_bases__", None) or [None]
-            if thing_args is None and len(thing_orig_bases) != 1:  # pragma: no cover
-                raise ResolutionFailed(
-                    f"It looks like you're trying to resolve {thing!r} which "
-                    "inherits from more than one generic type, which isn't supported. "
-                    "Try using register_type_strategy()."
-                )
-            thing_orig_bases_args = getattr(thing_orig_bases[0], "__args__", None)
-            if thing_args is None and thing_orig_bases_args is not None:
-                return len(thing_orig_bases_args) == len(superclass_args) and all(
-                    # "a==b or either is a typevar" is a hacky approximation, but it's
-                    # good enough for all the cases that I've seen so far and has the
-                    # substantial virtue of (relative) simplicity.
-                    a == b
-                    or isinstance(a, typing.TypeVar)
-                    or isinstance(b, typing.TypeVar)
-                    for a, b in zip(thing_orig_bases_args, superclass_args)
-                )
-            return True
+            if getattr(thing, "__args__", None) is not None:
+                return True
+            for orig_base in getattr(thing, "__orig_bases__", None) or [None]:
+                args = getattr(orig_base, "__args__", None)
+                if _compatible_args(args, superclass_args):
+                    return True
         return False
     except (AttributeError, TypeError):
         # Some types can't be the subject or object of an instance or subclass check


Test patch:
diff --git a/hypothesis-python/tests/nocover/test_drypython_returns.py b/hypothesis-python/tests/nocover/test_drypython_returns.py
new file mode 100644
--- /dev/null
+++ b/hypothesis-python/tests/nocover/test_drypython_returns.py
@@ -0,0 +1,210 @@
+# This file is part of Hypothesis, which may be found at
+# https://github.com/HypothesisWorks/hypothesis/
+#
+# Most of this work is copyright (C) 2013-2021 David R. MacIver
+# (david@drmaciver.com), but it contains contributions by others. See
+# CONTRIBUTING.rst for a full list of people who may hold copyright, and
+# consult the git log if you need to determine who owns an individual
+# contribution.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at https://mozilla.org/MPL/2.0/.
+#
+# END HEADER
+
+import sys
+from typing import Generic, TypeVar
+
+import pytest
+
+from hypothesis import given, strategies as st
+from hypothesis.errors import ResolutionFailed
+from hypothesis.internal.compat import PYPY
+
+from tests.common.debug import find_any
+from tests.common.utils import temp_registered
+
+# Primitives:
+# ===========
+
+_InstanceType = TypeVar("_InstanceType", covariant=True)
+_TypeArgType1 = TypeVar("_TypeArgType1", covariant=True)
+_FirstType = TypeVar("_FirstType")
+_LawType = TypeVar("_LawType")
+
+
+class KindN(Generic[_InstanceType, _TypeArgType1]):
+    pass
+
+
+class Lawful(Generic[_LawType]):
+    """This type defines law-related operations."""
+
+
+class MappableN(Generic[_FirstType], Lawful["MappableN[_FirstType]"]):
+    """Behaves like a functor."""
+
+
+# End definition:
+# ===============
+
+_ValueType = TypeVar("_ValueType")
+
+
+class MyFunctor(KindN["MyFunctor", _ValueType], MappableN[_ValueType]):
+    def __init__(self, inner_value: _ValueType) -> None:
+        self.inner_value = inner_value
+
+
+# Testing part:
+# =============
+
+
+def target_func(mappable: "MappableN[_FirstType]") -> bool:
+    return isinstance(mappable, MappableN)
+
+
+@given(st.data())
+def test_my_mappable(source: st.DataObject) -> None:
+    """
+    Checks that complex types with multiple inheritance levels and strings are fine.
+
+    Regression test for https://github.com/HypothesisWorks/hypothesis/issues/3060
+    """
+    # In `returns` we register all types in `__mro__`
+    # to be this exact type at the moment. But here, we only need `Mappable`.
+    # Current `__mro__` is `MyFunctor / Kind / Mappable`:
+    assert MyFunctor.__mro__[2] is MappableN
+    with temp_registered(
+        MyFunctor.__mro__[2],
+        st.builds(MyFunctor),
+    ):
+        assert source.draw(st.builds(target_func)) is True
+
+
+A = TypeVar("A")
+B = TypeVar("B")
+C = TypeVar("C")
+D = TypeVar("D")
+
+
+class _FirstBase(Generic[A, B]):
+    pass
+
+
+class _SecondBase(Generic[C, D]):
+    pass
+
+
+# To be tested:
+
+
+class TwoGenericBases1(_FirstBase[A, B], _SecondBase[C, D]):
+    pass
+
+
+class TwoGenericBases2(_FirstBase[C, D], _SecondBase[A, B]):
+    pass
+
+
+class OneGenericOneConrete1(_FirstBase[int, str], _SecondBase[A, B]):
+    pass
+
+
+class OneGenericOneConrete2(_FirstBase[A, B], _SecondBase[float, bool]):
+    pass
+
+
+class MixedGenerics1(_FirstBase[int, B], _SecondBase[C, bool]):
+    pass
+
+
+class MixedGenerics2(_FirstBase[A, str], _SecondBase[float, D]):
+    pass
+
+
+class AllConcrete(_FirstBase[int, str], _SecondBase[float, bool]):
+    pass
+
+
+_generic_test_types = (
+    TwoGenericBases1,
+    TwoGenericBases2,
+    OneGenericOneConrete1,
+    OneGenericOneConrete2,
+    MixedGenerics1,
+    MixedGenerics2,
+    AllConcrete,
+)
+
+
+@pytest.mark.parametrize("type_", _generic_test_types)
+def test_several_generic_bases(type_):
+    with temp_registered(_FirstBase, st.builds(type_)):
+        find_any(st.builds(_FirstBase))
+
+    with temp_registered(_SecondBase, st.builds(type_)):
+        find_any(st.builds(_SecondBase))
+
+
+def var_generic_func1(obj: _FirstBase[A, B]):
+    pass
+
+
+def var_generic_func2(obj: _SecondBase[A, B]):
+    pass
+
+
+def concrete_generic_func1(obj: _FirstBase[int, str]):
+    pass
+
+
+def concrete_generic_func2(obj: _SecondBase[float, bool]):
+    pass
+
+
+def mixed_generic_func1(obj: _FirstBase[A, str]):
+    pass
+
+
+def mixed_generic_func2(obj: _SecondBase[float, D]):
+    pass
+
+
+@pytest.mark.parametrize("type_", _generic_test_types)
+@pytest.mark.parametrize(
+    "func",
+    [
+        var_generic_func1,
+        var_generic_func2,
+        concrete_generic_func1,
+        concrete_generic_func2,
+        mixed_generic_func1,
+        mixed_generic_func2,
+    ],
+)
+def test_several_generic_bases_functions(type_, func):
+    with temp_registered(_FirstBase, st.builds(type_)), temp_registered(
+        _SecondBase, st.builds(type_)
+    ):
+        find_any(st.builds(func))
+
+    with temp_registered(type_, st.builds(type_)):
+        find_any(st.builds(func))
+
+
+def wrong_generic_func1(obj: _FirstBase[A, None]):
+    pass
+
+
+def wrong_generic_func2(obj: _SecondBase[None, bool]):
+    pass
+
+
+@pytest.mark.skipif(PYPY or sys.version_info[:2] == (3, 6), reason="not supported")
+@pytest.mark.parametrize("func", [wrong_generic_func1, wrong_generic_func2])
+def test_several_generic_bases_wrong_functions(func):
+    with temp_registered(AllConcrete, st.builds(AllConcrete)):
+        with pytest.raises(ResolutionFailed):
+            st.builds(func).example()