Repository: scrapy/scrapy

Issue:
BOM should take precedence over Content-Type header when detecting the encoding
Currently Scrapy uses headers first to detect the encoding. But browsers actually put a higher priority for BOM; this is also in WHATWG [standard](https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding). It can be checked e.g. by running this server, and opening URL in a browser - UTF-8 is used by browser, but cp1251 is used by Scrapy:

```py
import codecs
from http.server import BaseHTTPRequestHandler
from http.server import HTTPServer


class HttpGetHandler(BaseHTTPRequestHandler):
    def do_GET(self):
        self.send_response(200)
        self.send_header("Content-type", "text/html; charset=cp1251")
        self.end_headers()
        self.wfile.write(codecs.BOM_UTF8)
        self.wfile.write("<!DOCTYPE html>".encode('utf8'))
        self.wfile.write("Привет!".encode('utf8'))


if __name__ == '__main__':
    httpd = HTTPServer(('', 8000), HttpGetHandler)
    httpd.serve_forever()
```

When opening this page in a browser, it shows "Привет!".

Spider code to check it:

```py
import scrapy
from scrapy.crawler import CrawlerProcess


class MySpider(scrapy.Spider):
    name = "tst"

    start_urls = ["http://0.0.0.0:8000"]

    def parse(self, response):
        return {"encoding": response.encoding, "text": response.text}


if __name__ == '__main__':
    process = CrawlerProcess()
    process.crawl(MySpider)
    process.start()
```

Spider outputs

> {'encoding': 'cp1251', 'text': 'п»ї<!DOCTYPE html>РџСЂРёРІРµС‚!'}

See also: https://github.com/scrapy/w3lib/issues/189 - it's a similar issue, but fixing it in w3lib is not enough to make it working in Scrapy.

Discussion:

Patch:
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 89516b9b6..bfcde878d 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -11,8 +11,13 @@ from typing import Generator, Tuple
 from urllib.parse import urljoin
 
 import parsel
-from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
-                            http_content_type_encoding, resolve_encoding)
+from w3lib.encoding import (
+    html_body_declared_encoding,
+    html_to_unicode,
+    http_content_type_encoding,
+    resolve_encoding,
+    read_bom,
+)
 from w3lib.html import strip_html5_whitespace
 
 from scrapy.http import Request
@@ -60,6 +65,7 @@ class TextResponse(Response):
     def _declared_encoding(self):
         return (
             self._encoding
+            or self._bom_encoding()
             or self._headers_encoding()
             or self._body_declared_encoding()
         )
@@ -117,6 +123,10 @@ class TextResponse(Response):
     def _body_declared_encoding(self):
         return html_body_declared_encoding(self.body)
 
+    @memoizemethod_noargs
+    def _bom_encoding(self):
+        return read_bom(self.body)[0]
+
     @property
     def selector(self):
         from scrapy.selector import Selector


Test patch:
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 2986f884f..5d67a5e74 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -1,3 +1,4 @@
+import codecs
 import unittest
 from unittest import mock
 
@@ -358,6 +359,8 @@ class TextResponseTest(BaseResponseTest):
                                  headers={"Content-type": ["text/html; charset=gb2312"]})
         r7 = self.response_class("http://www.example.com", body=b"\xa8D",
                                  headers={"Content-type": ["text/html; charset=gbk"]})
+        r8 = self.response_class("http://www.example.com", body=codecs.BOM_UTF8 + b"\xc2\xa3",
+                                 headers={"Content-type": ["text/html; charset=cp1251"]})
 
         self.assertEqual(r1._headers_encoding(), "utf-8")
         self.assertEqual(r2._headers_encoding(), None)
@@ -367,7 +370,10 @@ class TextResponseTest(BaseResponseTest):
         self.assertEqual(r3._declared_encoding(), "cp1252")
         self.assertEqual(r4._headers_encoding(), None)
         self.assertEqual(r5._headers_encoding(), None)
+        self.assertEqual(r8._headers_encoding(), "cp1251")
+        self.assertEqual(r8._declared_encoding(), "utf-8")
         self._assert_response_encoding(r5, "utf-8")
+        self._assert_response_encoding(r8, "utf-8")
         assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii'
         self._assert_response_values(r1, 'utf-8', "\xa3")
         self._assert_response_values(r2, 'utf-8', "\xa3")