Repository: scrapinghub/web-poet

Issue:
Returns doesn't work in some subclasses 
Similar to https://github.com/zytedata/zyte-common-items/issues/49, even though the code is different, it's also not recursive and fails on e.g. this:

```python
def test_returns_inheritance() -> None:
    @attrs.define
    class MyItem:
        name: str

    class BasePage(ItemPage[MyItem]):
        @field
        def name(self):
            return "hello"

    MetadataT = TypeVar("MetadataT")

    class HasMetadata(Generic[MetadataT]):
        pass

    class DummyMetadata:
        pass

    class Page(BasePage, HasMetadata[DummyMetadata]):
        pass

    page = Page()
    assert page.item_cls is MyItem
```


Comments:

Patch:
diff --git a/docs/page-objects/items.rst b/docs/page-objects/items.rst
index 4fc7980..0526b14 100644
--- a/docs/page-objects/items.rst
+++ b/docs/page-objects/items.rst
@@ -42,6 +42,11 @@ To keep your code maintainable, we recommend you to:
     :ref:`page object classes <page-object-classes>`, e.g. using :ref:`field
     processors <field-processors>`.
 
+    Having code that makes item field values different from their counterpart
+    page object field values can subvert the expectations of users of your
+    code, which might need to access page object fields directly, for example
+    for field subset selection.
+
 If you are looking for ready-made item classes, check out `zyte-common-items`_.
 
 .. _zyte-common-items: https://zyte-common-items.readthedocs.io/en/latest/index.html
diff --git a/web_poet/_typing.py b/web_poet/_typing.py
deleted file mode 100644
index d7a2c98..0000000
--- a/web_poet/_typing.py
+++ /dev/null
@@ -1,24 +0,0 @@
-"""Utilities for typing"""
-import typing
-
-
-def is_generic_alias(obj) -> bool:
-    for attr_name in ["GenericAlias", "_GenericAlias"]:
-        if hasattr(typing, attr_name):
-            if isinstance(obj, getattr(typing, attr_name)):
-                return True
-    return False
-
-
-def get_generic_parameter(cls):
-    for base in getattr(cls, "__orig_bases__", []):
-        if is_generic_alias(base):
-            args = typing.get_args(base)
-            return args[0]
-
-
-def get_item_cls(cls, default=None):
-    param = get_generic_parameter(cls)
-    if param is None or isinstance(param, typing.TypeVar):  # class is not parametrized
-        return default
-    return param
diff --git a/web_poet/pages.py b/web_poet/pages.py
index e53a585..e3cd3dc 100644
--- a/web_poet/pages.py
+++ b/web_poet/pages.py
@@ -1,17 +1,21 @@
 import abc
 import inspect
-import typing
 from contextlib import suppress
 from functools import wraps
+from typing import Any, Generic, Optional, TypeVar, overload
 
 import attr
 import parsel
 
-from web_poet._typing import get_item_cls
 from web_poet.fields import FieldsMixin, item_from_fields
 from web_poet.mixins import ResponseShortcutsMixin, SelectorShortcutsMixin
 from web_poet.page_inputs import HttpResponse
-from web_poet.utils import CallableT, _create_deprecated_class, cached_method
+from web_poet.utils import (
+    CallableT,
+    _create_deprecated_class,
+    cached_method,
+    get_generic_param,
+)
 
 
 class Injectable(abc.ABC, FieldsMixin):
@@ -35,25 +39,40 @@ class Injectable(abc.ABC, FieldsMixin):
 Injectable.register(type(None))
 
 
-def is_injectable(cls: typing.Any) -> bool:
+def is_injectable(cls: Any) -> bool:
     """Return True if ``cls`` is a class which inherits
     from :class:`~.Injectable`."""
     return isinstance(cls, type) and issubclass(cls, Injectable)
 
 
-ItemT = typing.TypeVar("ItemT")
+ItemT = TypeVar("ItemT")
 
 
-class Returns(typing.Generic[ItemT]):
+class Returns(Generic[ItemT]):
     """Inherit from this generic mixin to change the item class used by
     :class:`~.ItemPage`"""
 
     @property
-    def item_cls(self) -> typing.Type[ItemT]:
+    def item_cls(self) -> type:
         """Item class"""
         return get_item_cls(self.__class__, default=dict)
 
 
+@overload
+def get_item_cls(cls: type, default: type) -> type:
+    ...
+
+
+@overload
+def get_item_cls(cls: type, default: None) -> Optional[type]:
+    ...
+
+
+def get_item_cls(cls: type, default: Optional[type] = None) -> Optional[type]:
+    param = get_generic_param(cls, Returns)
+    return param or default
+
+
 _NOT_SET = object()
 
 
diff --git a/web_poet/rules.py b/web_poet/rules.py
index 53f3efe..3337bac 100644
--- a/web_poet/rules.py
+++ b/web_poet/rules.py
@@ -22,9 +22,8 @@ from typing import (
 import attrs
 from url_matcher import Patterns, URLMatcher
 
-from web_poet._typing import get_item_cls
 from web_poet.page_inputs.url import _Url
-from web_poet.pages import ItemPage
+from web_poet.pages import ItemPage, get_item_cls
 from web_poet.utils import _create_deprecated_class, as_list, str_to_pattern
 
 Strings = Union[str, Iterable[str]]
diff --git a/web_poet/utils.py b/web_poet/utils.py
index 4eccc99..49d7b8c 100644
--- a/web_poet/utils.py
+++ b/web_poet/utils.py
@@ -1,9 +1,10 @@
 import inspect
 import weakref
+from collections import deque
 from collections.abc import Iterable
 from functools import lru_cache, partial, wraps
 from types import MethodType
-from typing import Any, Callable, List, Optional, TypeVar, Union
+from typing import Any, Callable, List, Optional, Tuple, TypeVar, Union, get_args
 from warnings import warn
 
 import packaging.version
@@ -273,3 +274,25 @@ def str_to_pattern(url_pattern: Union[str, Patterns]) -> Patterns:
     if isinstance(url_pattern, Patterns):
         return url_pattern
     return Patterns([url_pattern])
+
+
+def get_generic_param(
+    cls: type, expected: Union[type, Tuple[type, ...]]
+) -> Optional[type]:
+    """Search the base classes recursively breadth-first for a generic class and return its param.
+
+    Returns the param of the first found class that is a subclass of ``expected``.
+    """
+    visited = set()
+    queue = deque([cls])
+    while queue:
+        node = queue.popleft()
+        visited.add(node)
+        for base in getattr(node, "__orig_bases__", []):
+            origin = getattr(base, "__origin__", None)
+            if origin and issubclass(origin, expected):
+                result = get_args(base)[0]
+                if not isinstance(result, TypeVar):
+                    return result
+            queue.append(base)
+    return None


Test patch:
diff --git a/tests/test_pages.py b/tests/test_pages.py
index 9e86583..315569b 100644
--- a/tests/test_pages.py
+++ b/tests/test_pages.py
@@ -1,4 +1,4 @@
-from typing import List, Optional
+from typing import Generic, List, Optional, TypeVar
 
 import attrs
 import pytest
@@ -232,6 +232,31 @@ async def test_item_page_change_item_type_remove_fields() -> None:
         await page2.to_item()
 
 
+def test_returns_inheritance() -> None:
+    @attrs.define
+    class MyItem:
+        name: str
+
+    class BasePage(ItemPage[MyItem]):
+        @field
+        def name(self):
+            return "hello"
+
+    MetadataT = TypeVar("MetadataT")
+
+    class HasMetadata(Generic[MetadataT]):
+        pass
+
+    class DummyMetadata:
+        pass
+
+    class Page(BasePage, HasMetadata[DummyMetadata]):
+        pass
+
+    page = Page()
+    assert page.item_cls is MyItem
+
+
 @pytest.mark.asyncio
 async def test_extractor(book_list_html_response) -> None:
     @attrs.define
diff --git a/tests/test_utils.py b/tests/test_utils.py
index c6f5273..9094fdb 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -2,12 +2,17 @@ import asyncio
 import inspect
 import random
 import warnings
-from typing import Any
+from typing import Any, Generic, TypeVar
 from unittest import mock
 
 import pytest
 
-from web_poet.utils import _create_deprecated_class, cached_method, ensure_awaitable
+from web_poet.utils import (
+    _create_deprecated_class,
+    cached_method,
+    ensure_awaitable,
+    get_generic_param,
+)
 
 
 class SomeBaseClass:
@@ -466,3 +471,72 @@ async def test_cached_method_async_race() -> None:
         foo.n_called(),
     )
     assert results == [1, 1, 1, 1, 1]
+
+
+ItemT = TypeVar("ItemT")
+
+
+class Item:
+    pass
+
+
+class Item2:
+    pass
+
+
+class MyGeneric(Generic[ItemT]):
+    pass
+
+
+class MyGeneric2(Generic[ItemT]):
+    pass
+
+
+class Base(MyGeneric[ItemT]):
+    pass
+
+
+class BaseSpecialized(MyGeneric[Item]):
+    pass
+
+
+class BaseAny(MyGeneric):
+    pass
+
+
+class Derived(Base):
+    pass
+
+
+class Specialized(BaseSpecialized):
+    pass
+
+
+class SpecializedAdditionalClass(BaseSpecialized, Item2):
+    pass
+
+
+class SpecializedTwice(BaseSpecialized, Base[Item2]):
+    pass
+
+
+class SpecializedTwoGenerics(MyGeneric2[Item2], BaseSpecialized):
+    pass
+
+
+@pytest.mark.parametrize(
+    ["cls", "param"],
+    [
+        (MyGeneric, None),
+        (Base, None),
+        (BaseAny, None),
+        (Derived, None),
+        (BaseSpecialized, Item),
+        (Specialized, Item),
+        (SpecializedAdditionalClass, Item),
+        (SpecializedTwice, Item2),
+        (SpecializedTwoGenerics, Item),
+    ],
+)
+def test_get_generic_param(cls, param) -> None:
+    assert get_generic_param(cls, expected=MyGeneric) == param