Repository: scrapy/scrapy

Issue:
`downloadermiddlewares.retry.BackwardsCompatibilityMetaclass` does not provide backward compatibility for middleware instances
# Description

Previously, `EXCEPTIONS_TO_RETRY` was an attribute of `RetryMiddleware`. This allows:
- `RetryMiddleware` subclasses could access `EXCEPTIONS_TO_RETRY` via `cls.EXCEPTIONS_TO_RETRY`.
- `RetryMiddleware` instances and instances of its subclasses could access `EXCEPTIONS_TO_RETRY` via `self.EXCEPTIONS_TO_RETRY`.

In 2.10 `EXCEPTIONS_TO_RETRY` was removed and added as a property to `BackwardsCompatibilityMetaclass`. This added compatibility only for the first point.

# Steps to Reproduce

```python
class MyRetryMiddleware(RetryMiddleware):

    def process_exception(self, request, exception, spider):
        if isinstance(exception, self.EXCEPTIONS_TO_RETRY) and not request.meta.get('dont_retry', False):
            # update request
            return self._retry(request, exception, spider)
```

# Expected behavior

A warning about `EXCEPTIONS_TO_RETRY` deprecation.

# Actual behavior

AttributeError: 'MyRetryMiddleware' object has no attribute 'EXCEPTIONS_TO_RETRY'

# Versions

```
Scrapy       : 2.10.1
lxml         : 4.9.3.0
libxml2      : 2.10.3
cssselect    : 1.2.0
parsel       : 1.8.1
w3lib        : 2.1.2
Twisted      : 22.10.0
Python       : 3.11.4 (tags/v3.11.4:d2340ef, Jun  7 2023, 05:45:37) [MSC v.1934 64 bit (AMD64)]
pyOpenSSL    : 23.2.0 (OpenSSL 3.1.2 1 Aug 2023)
cryptography : 41.0.3
Platform     : Windows-10-10.0.19044-SP0
```



Discussion:
The fix is actually very tricky. Here is a sample snippet:
```python
DEPRECATED_ATTRIBUTE = 'A'


def __getattr__(self, item):
    if item == DEPRECATED_ATTRIBUTE:
        return 'A does not exist'

    raise AttributeError(f'{self.__class__.__name__!r} object has no attribute {item!r}')


class Meta(type):
    __getattr__ = __getattr__


class Data(metaclass=Meta):
    def __init__(self):
        try:
            self.a = self.__getattribute__(DEPRECATED_ATTRIBUTE)
        except AttributeError:
            self.a = 'a here'

    __getattr__ = __getattr__


class DataDefineA(Data):
    A = 1


class DataNoA(Data): pass


print(Data.A)
print(Data().A)
print(Data().a)

print(DataDefineA.A)
print(DataDefineA().A)
print(DataDefineA().a)

print(DataNoA.A)
print(DataNoA().A)
print(DataNoA().a)
```

Output:
```
A does not exist
A does not exist
a here
1
1
1
A does not exist
A does not exist
a here
```

Such solution achieves both compatibility issues. In addition, if a subclass or its instance defines attribute `A`, then it will be assigned to attribute `a`.

Rewritten `retry.py`:

```python
DEPRECATED_ATTRIBUTE = 'EXCEPTIONS_TO_RETRY'


def backwards_compatibility_getattr(self, item):
    if item == DEPRECATED_ATTRIBUTE:
        warnings.warn(
            f"Attribute RetryMiddleware.{DEPRECATED_ATTRIBUTE} is deprecated. "
            "Use the RETRY_EXCEPTIONS setting instead.",
            ScrapyDeprecationWarning,
            stacklevel=2,
            )

        return tuple(
            load_object(x) if isinstance(x, str) else x
            for x in Settings().getlist("RETRY_EXCEPTIONS")
            )

    raise AttributeError(f'{self.__class__.__name__!r} object has no attribute {item!r}')


class BackwardsCompatibilityMetaclass(type):
    __getattr__ = backwards_compatibility_getattr


class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
    def __init__(self, settings):
        if not settings.getbool("RETRY_ENABLED"):
            raise NotConfigured
        self.max_retry_times = settings.getint("RETRY_TIMES")
        self.retry_http_codes = set(
            int(x) for x in settings.getlist("RETRY_HTTP_CODES")
        )
        self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")

        try:
            self.exceptions_to_retry = self.__getattribute__(DEPRECATED_ATTRIBUTE)
        except AttributeError:
            # If EXCEPTIONS_TO_RETRY is not "overridden"
            self.exceptions_to_retry = tuple(
                load_object(x) if isinstance(x, str) else x
                for x in settings.getlist("RETRY_EXCEPTIONS")
                )

    __getattr__ = backwards_compatibility_getattr
```

Resolving patch:
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -24,9 +24,8 @@
 retry_logger = getLogger(__name__)
 
 
-class BackwardsCompatibilityMetaclass(type):
-    @property
-    def EXCEPTIONS_TO_RETRY(cls):
+def backwards_compatibility_getattr(self, name):
+    if name == "EXCEPTIONS_TO_RETRY":
         warnings.warn(
             "Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. "
             "Use the RETRY_EXCEPTIONS setting instead.",
@@ -37,6 +36,13 @@ def EXCEPTIONS_TO_RETRY(cls):
             load_object(x) if isinstance(x, str) else x
             for x in Settings().getlist("RETRY_EXCEPTIONS")
         )
+    raise AttributeError(
+        f"{self.__class__.__name__!r} object has no attribute {name!r}"
+    )
+
+
+class BackwardsCompatibilityMetaclass(type):
+    __getattr__ = backwards_compatibility_getattr
 
 
 def get_retry_request(
@@ -137,15 +143,14 @@ def __init__(self, settings):
         )
         self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")
 
-        if not hasattr(
-            self, "EXCEPTIONS_TO_RETRY"
-        ):  # If EXCEPTIONS_TO_RETRY is not "overriden"
+        try:
+            self.exceptions_to_retry = self.__getattribute__("EXCEPTIONS_TO_RETRY")
+        except AttributeError:
+            # If EXCEPTIONS_TO_RETRY is not "overridden"
             self.exceptions_to_retry = tuple(
                 load_object(x) if isinstance(x, str) else x
                 for x in settings.getlist("RETRY_EXCEPTIONS")
             )
-        else:
-            self.exceptions_to_retry = self.EXCEPTIONS_TO_RETRY
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -175,3 +180,5 @@ def _retry(self, request, reason, spider):
             max_retry_times=max_retry_times,
             priority_adjust=priority_adjust,
         )
+
+    __getattr__ = backwards_compatibility_getattr


Test patch:
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -122,7 +122,7 @@ def test_exception_to_retry_added(self):
         req = Request(f"http://www.scrapytest.org/{exc.__name__}")
         self._test_retry_exception(req, exc("foo"), mw)
 
-    def test_exception_to_retry_customMiddleware(self):
+    def test_exception_to_retry_custom_middleware(self):
         exc = ValueError
 
         with warnings.catch_warnings(record=True) as warns:
@@ -138,6 +138,21 @@ class MyRetryMiddleware(RetryMiddleware):
         assert isinstance(req, Request)
         self.assertEqual(req.meta["retry_times"], 1)
 
+    def test_exception_to_retry_custom_middleware_self(self):
+        class MyRetryMiddleware(RetryMiddleware):
+            def process_exception(self, request, exception, spider):
+                if isinstance(exception, self.EXCEPTIONS_TO_RETRY):
+                    return self._retry(request, exception, spider)
+
+        exc = OSError
+        mw2 = MyRetryMiddleware.from_crawler(self.crawler)
+        req = Request(f"http://www.scrapytest.org/{exc.__name__}")
+        with warnings.catch_warnings(record=True) as warns:
+            req = mw2.process_exception(req, exc("foo"), self.spider)
+        assert isinstance(req, Request)
+        self.assertEqual(req.meta["retry_times"], 1)
+        self.assertEqual(len(warns), 1)
+
     def _test_retry_exception(self, req, exception, mw=None):
         if mw is None:
             mw = self.mw