Repository: scrapy/scrapy

Issue:
Exception with non-UTF-8 Content-Type
<!--

Thanks for taking an interest in Scrapy!

If you have a question that starts with "How to...", please see the Scrapy Community page: https://scrapy.org/community/.
The GitHub issue tracker's purpose is to deal with bug reports and feature requests for the project itself.

Keep in mind that by filing an issue, you are expected to comply with Scrapy's Code of Conduct, including treating everyone with respect: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md

The following is a suggested template to structure your issue, you can find more guidelines at https://doc.scrapy.org/en/latest/contributing.html#reporting-bugs

-->

### Description

I am trying to download a document from the link http://pravo.gov.ru/proxy/ips/?savertf=&link_id=0&nd=128284801&intelsearch=&firstDoc=1&page=all
Everything works fine in the browser, but when I try to automate this process through scrapy, everything break down.
### Steps to Reproduce

1.Create new spider `scrapy genspider test pravo.gov.ru`
2. Paste code
```
import scrapy


class TestSpider(scrapy.Spider):
    name = "test"
    allowed_domains = ["pravo.gov.ru"]
    start_urls = ["http://pravo.gov.ru/proxy/ips/"]

    def parse(self, response):
        yield scrapy.Request(
            self.start_urls[0] + f'?savertf=&link_id=0&nd=128284801&intelsearch=&firstDoc=1&page=all',
            callback=self.test_parse,
            encoding='cp1251') #The same behavior without this line
           

    def test_parse(self, response):
        pass
```

4. run
***OR***
run `scrapy fetch http://pravo.gov.ru/proxy/ips/?savertf=&link_id=0&nd=128284801&intelsearch=&firstDoc=1&page=all""`
**Expected behavior:** The document on the link is downloaded
**Actual behavior:** 
2023-04-28 00:07:35 [scrapy.core.scraper] ERROR: Error downloading <GET http://pravo.gov.ru/proxy/ips/?savertf=&link_id=0&nd=128284801&intelsearch=&fir
stDoc=1&page=all>
Traceback (most recent call last):
  File "C:\Users\Admin\AppData\Roaming\Python\Python310\site-packages\twisted\internet\defer.py", line 1693, in _inlineCallbacks
    result = context.run(
  File "C:\Users\Admin\AppData\Roaming\Python\Python310\site-packages\twisted\python\failure.py", line 518, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "C:\Users\Admin\AppData\Roaming\Python\Python310\site-packages\scrapy\core\downloader\middleware.py", line 52, in process_request
    return (yield download_func(request=request, spider=spider))
  File "C:\Users\Admin\AppData\Roaming\Python\Python310\site-packages\twisted\internet\defer.py", line 892, in _runCallbacks
    current.result = callback(  # type: ignore[misc]
  File "C:\Users\Admin\AppData\Roaming\Python\Python310\site-packages\scrapy\core\downloader\handlers\http11.py", line 501, in _cb_bodydone
    respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
  File "C:\Users\Admin\AppData\Roaming\Python\Python310\site-packages\scrapy\responsetypes.py", line 113, in from_args
    cls = self.from_headers(headers)
  File "C:\Users\Admin\AppData\Roaming\Python\Python310\site-packages\scrapy\responsetypes.py", line 75, in from_headers
    cls = self.from_content_type(
  File "C:\Users\Admin\AppData\Roaming\Python\Python310\site-packages\scrapy\responsetypes.py", line 55, in from_content_type
    mimetype = to_unicode(content_type).split(";")[0].strip().lower()
  File "C:\Users\Admin\AppData\Roaming\Python\Python310\site-packages\scrapy\utils\python.py", line 97, in to_unicode
    return text.decode(encoding, errors)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xcf in position 33: invalid continuation byte

**Reproduces how often:** 100% runs

### Versions

Scrapy       : 2.8.0
lxml         : 4.9.2.0
libxml2      : 2.9.12
cssselect    : 1.2.0
parsel       : 1.8.1
w3lib        : 2.1.1
Twisted      : 22.10.0
Python       : 3.10.9 | packaged by Anaconda, Inc. | (main, Mar  1 2023, 18:18:15) [MSC v.1916 64 bit (AMD64)]
pyOpenSSL    : 23.1.1 (OpenSSL 3.1.0 14 Mar 2023)
cryptography : 40.0.2
Platform     : Windows-10-10.0.19044-SP0
### Additional context

These documents are encoded in cp1251 encoding, which is clearly indicated in their headers :  
Content-Transfer-Encoding: quoted-printable
Content-Type: text/html; charset="windows-1251"
The same behavior when trying to save a file using FilesPipeline

Discussion:

Patch:
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 73bb811de..d580a7876 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -100,11 +100,13 @@ class TextResponse(Response):
     @memoizemethod_noargs
     def _headers_encoding(self):
         content_type = self.headers.get(b"Content-Type", b"")
-        return http_content_type_encoding(to_unicode(content_type))
+        return http_content_type_encoding(to_unicode(content_type, encoding="latin-1"))
 
     def _body_inferred_encoding(self):
         if self._cached_benc is None:
-            content_type = to_unicode(self.headers.get(b"Content-Type", b""))
+            content_type = to_unicode(
+                self.headers.get(b"Content-Type", b""), encoding="latin-1"
+            )
             benc, ubody = html_to_unicode(
                 content_type,
                 self.body,
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index f01e9096c..58884f21a 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -51,7 +51,9 @@ class ResponseTypes:
         header"""
         if content_encoding:
             return Response
-        mimetype = to_unicode(content_type).split(";")[0].strip().lower()
+        mimetype = (
+            to_unicode(content_type, encoding="latin-1").split(";")[0].strip().lower()
+        )
         return self.from_mimetype(mimetype)
 
     def from_content_disposition(self, content_disposition):


Test patch:
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index dbc9f1fef..a05b702aa 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -448,6 +448,13 @@ class TextResponseTest(BaseResponseTest):
             body=codecs.BOM_UTF8 + b"\xc2\xa3",
             headers={"Content-type": ["text/html; charset=cp1251"]},
         )
+        r9 = self.response_class(
+            "http://www.example.com",
+            body=b"\x80",
+            headers={
+                "Content-type": [b"application/x-download; filename=\x80dummy.txt"]
+            },
+        )
 
         self.assertEqual(r1._headers_encoding(), "utf-8")
         self.assertEqual(r2._headers_encoding(), None)
@@ -458,9 +465,12 @@ class TextResponseTest(BaseResponseTest):
         self.assertEqual(r4._headers_encoding(), None)
         self.assertEqual(r5._headers_encoding(), None)
         self.assertEqual(r8._headers_encoding(), "cp1251")
+        self.assertEqual(r9._headers_encoding(), None)
         self.assertEqual(r8._declared_encoding(), "utf-8")
+        self.assertEqual(r9._declared_encoding(), None)
         self._assert_response_encoding(r5, "utf-8")
         self._assert_response_encoding(r8, "utf-8")
+        self._assert_response_encoding(r9, "cp1252")
         assert (
             r4._body_inferred_encoding() is not None
             and r4._body_inferred_encoding() != "ascii"
@@ -470,6 +480,7 @@ class TextResponseTest(BaseResponseTest):
         self._assert_response_values(r3, "iso-8859-1", "\xa3")
         self._assert_response_values(r6, "gb18030", "\u2015")
         self._assert_response_values(r7, "gb18030", "\u2015")
+        self._assert_response_values(r9, "cp1252", "€")
 
         # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies
         self.assertRaises(
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index 859960518..6e1ed82f0 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -42,6 +42,7 @@ class ResponseTypesTest(unittest.TestCase):
             ("application/octet-stream", Response),
             ("application/x-json; encoding=UTF8;charset=UTF-8", TextResponse),
             ("application/json-amazonui-streaming;charset=UTF-8", TextResponse),
+            (b"application/x-download; filename=\x80dummy.txt", Response),
         ]
         for source, cls in mappings:
             retcls = responsetypes.from_content_type(source)