Repository: scrapy/scrapy
Pull request #5347 (2021-12-07 11:01:26)

correction of comma
 you do not need a comma if the clause comes at the end of the sentence. This applies to all subordinate clauses, not just. should i merg ?


Patch:
diff --git a/pylintrc b/pylintrc
index 699686e16..2cdd6321e 100644
--- a/pylintrc
+++ b/pylintrc
@@ -112,6 +112,7 @@ disable=abstract-method,
         unused-private-member,
         unused-variable,
         unused-wildcard-import,
+        use-implicit-booleaness-not-comparison,
         used-before-assignment,
         useless-object-inheritance,  # Required for Python 2 support
         useless-return,
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 6e77551c6..5f1dabd33 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -43,14 +43,14 @@ class ScrapyCommand:
 
     def long_desc(self):
         """A long description of the command. Return short description when not
-        available. It cannot contain newlines, since contents will be formatted
+        available. It cannot contain newlines since contents will be formatted
         by optparser which removes newlines and wraps text.
         """
         return self.short_desc()
 
     def help(self):
         """An extensive help for the command. It will be shown when using the
-        "help" command. It can contain newlines, since no post-formatting will
+        "help" command. It can contain newlines since no post-formatting will
         be applied to its contents.
         """
         return self.long_desc()
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index b3ef7b463..8b109dced 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -3,8 +3,9 @@ This module provides some useful functions for working with
 scrapy.http.Response objects
 """
 import os
-import webbrowser
+import re
 import tempfile
+import webbrowser
 from typing import Any, Callable, Iterable, Optional, Tuple, Union
 from weakref import WeakKeyDictionary
 
@@ -80,8 +81,9 @@ def open_in_browser(
     body = response.body
     if isinstance(response, HtmlResponse):
         if b'<base' not in body:
-            repl = f'<head><base href="{response.url}">'
-            body = body.replace(b'<head>', to_bytes(repl))
+            repl = fr'\1<base href="{response.url}">'
+            body = re.sub(b"<!--.*?-->", b"", body, flags=re.DOTALL)
+            body = re.sub(rb"(<head(?:>|\s.*?>))", to_bytes(repl), body)
         ext = '.html'
     elif isinstance(response, TextResponse):
         ext = '.txt'
diff --git a/tox.ini b/tox.ini
index e4514f512..2031a2d92 100644
--- a/tox.ini
+++ b/tox.ini
@@ -66,7 +66,7 @@ commands =
 basepython = python3
 deps =
     {[testenv:extra-deps]deps}
-    pylint
+    pylint==2.12.1
 commands =
     pylint conftest.py docs extras scrapy setup.py tests
 


Test patch:
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index d6f4c0bb5..0a09f6109 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -83,3 +83,56 @@ class ResponseUtilsTest(unittest.TestCase):
         self.assertEqual(response_status_message(200), '200 OK')
         self.assertEqual(response_status_message(404), '404 Not Found')
         self.assertEqual(response_status_message(573), "573 Unknown Status")
+
+    def test_inject_base_url(self):
+        url = "http://www.example.com"
+
+        def check_base_url(burl):
+            path = urlparse(burl).path
+            if not os.path.exists(path):
+                path = burl.replace('file://', '')
+            with open(path, "rb") as f:
+                bbody = f.read()
+            self.assertEqual(bbody.count(b'<base href="' + to_bytes(url) + b'">'), 1)
+            return True
+
+        r1 = HtmlResponse(url, body=b"""
+        <html>
+            <head><title>Dummy</title></head>
+            <body><p>Hello world.</p></body>
+        </html>""")
+        r2 = HtmlResponse(url, body=b"""
+        <html>
+            <head id="foo"><title>Dummy</title></head>
+            <body>Hello world.</body>
+        </html>""")
+        r3 = HtmlResponse(url, body=b"""
+        <html>
+            <head><title>Dummy</title></head>
+            <body>
+                <header>Hello header</header>
+                <p>Hello world.</p>
+            </body>
+        </html>""")
+        r4 = HtmlResponse(url, body=b"""
+        <html>
+            <!-- <head>Dummy comment</head> -->
+            <head><title>Dummy</title></head>
+            <body><p>Hello world.</p></body>
+        </html>""")
+        r5 = HtmlResponse(url, body=b"""
+        <html>
+            <!--[if IE]>
+            <head><title>IE head</title></head>
+            <![endif]-->
+            <!--[if !IE]>-->
+            <head><title>Standard head</title></head>
+            <!--<![endif]-->
+            <body><p>Hello world.</p></body>
+        </html>""")
+
+        assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url"
+        assert open_in_browser(r2, _openfunc=check_base_url), "Inject base url with argumented head"
+        assert open_in_browser(r3, _openfunc=check_base_url), "Inject unique base url with misleading tag"
+        assert open_in_browser(r4, _openfunc=check_base_url), "Inject unique base url with misleading comment"
+        assert open_in_browser(r5, _openfunc=check_base_url), "Inject unique base url with conditional comment"