Repository: scrapy/scrapy

Issue:
Allow to add custom metadata to Response
Currently it is not possible to attach a custom attribute to Response in a middleware because `replace` method won't keep it. There are workarounds (e.g. putting objects to response.flags), but they are ugly.

What do you think about adding an attribute to Response where user can put anything, similar to Request.meta?



Discussion:
Almost the same issue: https://github.com/scrapy/scrapy/issues/1453

One workaround is to create custom Response subclasses and use `response.replace(cls=MyCustomResponse)` in a middleware. 

The problem with this solution is that Scrapy sometimes constructs responses by itself, e.g. when retreiving from HTTP cache or when handling decompression. In this case original response class gets forgotten.

I'd like to try to implement this, does anyone have any newer thoughts about it that could lead me in the right direction?
https://github.com/scrapy/scrapy/pull/5131 seems to address this when defining `Response.attributes` and using it from `replace`.

Resolving patch:
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -4,7 +4,7 @@
 
 See documentation in docs/topics/request-response.rst
 """
-from typing import Generator
+from typing import Generator, Tuple
 from urllib.parse import urljoin
 
 from scrapy.exceptions import NotSupported
@@ -16,6 +16,19 @@
 
 
 class Response(object_ref):
+    """An object that represents an HTTP response, which is usually
+    downloaded (by the Downloader) and fed to the Spiders for processing.
+    """
+
+    attributes: Tuple[str, ...] = (
+        "url", "status", "headers", "body", "flags", "request", "certificate", "ip_address", "protocol",
+    )
+    """A tuple of :class:`str` objects containing the name of all public
+    attributes of the class that are also keyword parameters of the
+    ``__init__`` method.
+
+    Currently used by :meth:`Response.replace`.
+    """
 
     def __init__(
         self,
@@ -97,12 +110,8 @@ def copy(self):
         return self.replace()
 
     def replace(self, *args, **kwargs):
-        """Create a new Response with the same attributes except for those
-        given new values.
-        """
-        for x in [
-            "url", "status", "headers", "body", "request", "flags", "certificate", "ip_address", "protocol",
-        ]:
+        """Create a new Response with the same attributes except for those given new values"""
+        for x in self.attributes:
             kwargs.setdefault(x, getattr(self, x))
         cls = kwargs.pop('cls', self.__class__)
         return cls(*args, **kwargs)
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -8,7 +8,7 @@
 import json
 import warnings
 from contextlib import suppress
-from typing import Generator
+from typing import Generator, Tuple
 from urllib.parse import urljoin
 
 import parsel
@@ -30,6 +30,8 @@ class TextResponse(Response):
     _DEFAULT_ENCODING = 'ascii'
     _cached_decoded_json = _NONE
 
+    attributes: Tuple[str, ...] = Response.attributes + ("encoding",)
+
     def __init__(self, *args, **kwargs):
         self._encoding = kwargs.pop('encoding', None)
         self._cached_benc = None
@@ -53,10 +55,6 @@ def _set_body(self, body):
         else:
             super()._set_body(body)
 
-    def replace(self, *args, **kwargs):
-        kwargs.setdefault('encoding', self.encoding)
-        return Response.replace(self, *args, **kwargs)
-
     @property
     def encoding(self):
         return self._declared_encoding() or self._body_inferred_encoding()


Test patch:
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -820,3 +820,62 @@ def test_selector_shortcuts_kwargs(self):
             response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).getall(),
             response.selector.xpath("//s2:elem/text()").getall(),
         )
+
+
+class CustomResponse(TextResponse):
+    attributes = TextResponse.attributes + ("foo", "bar")
+
+    def __init__(self, *args, **kwargs) -> None:
+        self.foo = kwargs.pop("foo", None)
+        self.bar = kwargs.pop("bar", None)
+        self.lost = kwargs.pop("lost", None)
+        super().__init__(*args, **kwargs)
+
+
+class CustomResponseTest(TextResponseTest):
+    response_class = CustomResponse
+
+    def test_copy(self):
+        super().test_copy()
+        r1 = self.response_class(url="https://example.org", status=200, foo="foo", bar="bar", lost="lost")
+        r2 = r1.copy()
+        self.assertIsInstance(r2, self.response_class)
+        self.assertEqual(r1.foo, r2.foo)
+        self.assertEqual(r1.bar, r2.bar)
+        self.assertEqual(r1.lost, "lost")
+        self.assertIsNone(r2.lost)
+
+    def test_replace(self):
+        super().test_replace()
+        r1 = self.response_class(url="https://example.org", status=200, foo="foo", bar="bar", lost="lost")
+
+        r2 = r1.replace(foo="new-foo", bar="new-bar", lost="new-lost")
+        self.assertIsInstance(r2, self.response_class)
+        self.assertEqual(r1.foo, "foo")
+        self.assertEqual(r1.bar, "bar")
+        self.assertEqual(r1.lost, "lost")
+        self.assertEqual(r2.foo, "new-foo")
+        self.assertEqual(r2.bar, "new-bar")
+        self.assertEqual(r2.lost, "new-lost")
+
+        r3 = r1.replace(foo="new-foo", bar="new-bar")
+        self.assertIsInstance(r3, self.response_class)
+        self.assertEqual(r1.foo, "foo")
+        self.assertEqual(r1.bar, "bar")
+        self.assertEqual(r1.lost, "lost")
+        self.assertEqual(r3.foo, "new-foo")
+        self.assertEqual(r3.bar, "new-bar")
+        self.assertIsNone(r3.lost)
+
+        r4 = r1.replace(foo="new-foo")
+        self.assertIsInstance(r4, self.response_class)
+        self.assertEqual(r1.foo, "foo")
+        self.assertEqual(r1.bar, "bar")
+        self.assertEqual(r1.lost, "lost")
+        self.assertEqual(r4.foo, "new-foo")
+        self.assertEqual(r4.bar, "bar")
+        self.assertIsNone(r4.lost)
+
+        with self.assertRaises(TypeError) as ctx:
+            r1.replace(unknown="unknown")
+        self.assertEqual(str(ctx.exception), "__init__() got an unexpected keyword argument 'unknown'")