Repository: scrapy/scrapy

Issue:
Response.headers loses data on multiple values
https://github.com/scrapy/scrapy/issues/1262 reported that by default `response.headers` would only expose the first value of a header e.g. when casted as a `dict`, acknowledging that `response.headers.getlist` could be used instead to get all values.

I have just found out that the latter is not true:

```python
>>> from scrapy.http import Response
>>> response = Response("https://example.com", headers=(("a", "b"), ("a", "c")))
>>> response.headers.getlist("a")
[b'c']
```

I could verify the issue happening as far back as Scrapy 1.6, so it does not look like a recent bug.

Discussion:

Patch:
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 1a2b99b0a..9c03fe54f 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -1,3 +1,5 @@
+from collections.abc import Mapping
+
 from w3lib.http import headers_dict_to_raw
 from scrapy.utils.datatypes import CaselessDict
 from scrapy.utils.python import to_unicode
@@ -10,6 +12,13 @@ class Headers(CaselessDict):
         self.encoding = encoding
         super().__init__(seq)
 
+    def update(self, seq):
+        seq = seq.items() if isinstance(seq, Mapping) else seq
+        iseq = {}
+        for k, v in seq:
+            iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v))
+        super().update(iseq)
+
     def normkey(self, key):
         """Normalize key to bytes"""
         return self._tobytes(key.title())
@@ -86,4 +95,5 @@ class Headers(CaselessDict):
 
     def __copy__(self):
         return self.__class__(self)
+
     copy = __copy__


Test patch:
diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py
index 64ff7a73d..1ca936247 100644
--- a/tests/test_http_headers.py
+++ b/tests/test_http_headers.py
@@ -38,6 +38,12 @@ class HeadersTest(unittest.TestCase):
         self.assertEqual(h.getlist('X-Forwarded-For'), [b'ip1', b'ip2'])
         assert h.getlist('X-Forwarded-For') is not hlist
 
+    def test_multivalue_for_one_header(self):
+        h = Headers((("a", "b"), ("a", "c")))
+        self.assertEqual(h["a"], b"c")
+        self.assertEqual(h.get("a"), b"c")
+        self.assertEqual(h.getlist("a"), [b"b", b"c"])
+
     def test_encode_utf8(self):
         h = Headers({'key': '\xa3'}, encoding='utf-8')
         key, val = dict(h).popitem()