Repository: scrapy/scrapy (python)

Issue:
`downloadermiddlewares.retry.BackwardsCompatibilityMetaclass` does not provide backward compatibility for middleware instances
# Description

Previously, `EXCEPTIONS_TO_RETRY` was an attribute of `RetryMiddleware`. This allows:
- `RetryMiddleware` subclasses could access `EXCEPTIONS_TO_RETRY` via `cls.EXCEPTIONS_TO_RETRY`.
- `RetryMiddleware` instances and instances of its subclasses could access `EXCEPTIONS_TO_RETRY` via `self.EXCEPTIONS_TO_RETRY`.

In 2.10 `EXCEPTIONS_TO_RETRY` was removed and added as a property to `BackwardsCompatibilityMetaclass`. This added compatibility only for the first point.

# Steps to Reproduce

```python
class MyRetryMiddleware(RetryMiddleware):

    def process_exception(self, request, exception, spider):
        if isinstance(exception, self.EXCEPTIONS_TO_RETRY) and not request.meta.get('dont_retry', False):
            # update request
            return self._retry(request, exception, spider)
```

# Expected behavior

A warning about `EXCEPTIONS_TO_RETRY` deprecation.

# Actual behavior

AttributeError: 'MyRetryMiddleware' object has no attribute 'EXCEPTIONS_TO_RETRY'

# Versions

```
Scrapy       : 2.10.1
lxml         : 4.9.3.0
libxml2      : 2.10.3
cssselect    : 1.2.0
parsel       : 1.8.1
w3lib        : 2.1.2
Twisted      : 22.10.0
Python       : 3.11.4 (tags/v3.11.4:d2340ef, Jun  7 2023, 05:45:37) [MSC v.1934 64 bit (AMD64)]
pyOpenSSL    : 23.2.0 (OpenSSL 3.1.2 1 Aug 2023)
cryptography : 41.0.3
Platform     : Windows-10-10.0.19044-SP0
```


Pull request description:
Improve the backwards compatibility for RetryMiddleware.EXCEPTIONS_TO_RETRY
Fixes #6049 (with the code provided by the reporter)

Patch:
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 205bb48b1..380623cea 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -24,9 +24,8 @@ from scrapy.utils.response import response_status_message
 retry_logger = getLogger(__name__)
 
 
-class BackwardsCompatibilityMetaclass(type):
-    @property
-    def EXCEPTIONS_TO_RETRY(cls):
+def backwards_compatibility_getattr(self, name):
+    if name == "EXCEPTIONS_TO_RETRY":
         warnings.warn(
             "Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. "
             "Use the RETRY_EXCEPTIONS setting instead.",
@@ -37,6 +36,13 @@ class BackwardsCompatibilityMetaclass(type):
             load_object(x) if isinstance(x, str) else x
             for x in Settings().getlist("RETRY_EXCEPTIONS")
         )
+    raise AttributeError(
+        f"{self.__class__.__name__!r} object has no attribute {name!r}"
+    )
+
+
+class BackwardsCompatibilityMetaclass(type):
+    __getattr__ = backwards_compatibility_getattr
 
 
 def get_retry_request(
@@ -137,15 +143,14 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
         )
         self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")
 
-        if not hasattr(
-            self, "EXCEPTIONS_TO_RETRY"
-        ):  # If EXCEPTIONS_TO_RETRY is not "overriden"
+        try:
+            self.exceptions_to_retry = self.__getattribute__("EXCEPTIONS_TO_RETRY")
+        except AttributeError:
+            # If EXCEPTIONS_TO_RETRY is not "overridden"
             self.exceptions_to_retry = tuple(
                 load_object(x) if isinstance(x, str) else x
                 for x in settings.getlist("RETRY_EXCEPTIONS")
             )
-        else:
-            self.exceptions_to_retry = self.EXCEPTIONS_TO_RETRY
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -175,3 +180,5 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
             max_retry_times=max_retry_times,
             priority_adjust=priority_adjust,
         )
+
+    __getattr__ = backwards_compatibility_getattr


Test patch:
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 97ae1e29a..661175840 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -122,7 +122,7 @@ class RetryTest(unittest.TestCase):
         req = Request(f"http://www.scrapytest.org/{exc.__name__}")
         self._test_retry_exception(req, exc("foo"), mw)
 
-    def test_exception_to_retry_customMiddleware(self):
+    def test_exception_to_retry_custom_middleware(self):
         exc = ValueError
 
         with warnings.catch_warnings(record=True) as warns:
@@ -138,6 +138,21 @@ class RetryTest(unittest.TestCase):
         assert isinstance(req, Request)
         self.assertEqual(req.meta["retry_times"], 1)
 
+    def test_exception_to_retry_custom_middleware_self(self):
+        class MyRetryMiddleware(RetryMiddleware):
+            def process_exception(self, request, exception, spider):
+                if isinstance(exception, self.EXCEPTIONS_TO_RETRY):
+                    return self._retry(request, exception, spider)
+
+        exc = OSError
+        mw2 = MyRetryMiddleware.from_crawler(self.crawler)
+        req = Request(f"http://www.scrapytest.org/{exc.__name__}")
+        with warnings.catch_warnings(record=True) as warns:
+            req = mw2.process_exception(req, exc("foo"), self.spider)
+        assert isinstance(req, Request)
+        self.assertEqual(req.meta["retry_times"], 1)
+        self.assertEqual(len(warns), 1)
+
     def _test_retry_exception(self, req, exception, mw=None):
         if mw is None:
             mw = self.mw