Repository: scrapy/scrapy
Pull request #5332 (2021-11-26 07:11:37)

Remove the experimental note about asyncio.
I also added a line about `TWISTED_REACTOR` and `custom_settings`.


Patch:
diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst
index 71d027c86..0c3a7ed88 100644
--- a/docs/topics/architecture.rst
+++ b/docs/topics/architecture.rst
@@ -67,7 +67,7 @@ this:
    the :ref:`Scheduler <component-scheduler>` and asks for possible next Requests
    to crawl.
 
-9. The process repeats (from step 1) until there are no more requests from the
+9. The process repeats (from step 3) until there are no more requests from the
    :ref:`Scheduler <component-scheduler>`.
 
 Components
diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 28241ae24..402352721 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -10,11 +10,6 @@ Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the
 asyncio reactor <install-asyncio>`, you may use :mod:`asyncio` and
 :mod:`asyncio`-powered libraries in any :doc:`coroutine <coroutines>`.
 
-.. warning:: :mod:`asyncio` support in Scrapy is experimental, and not yet
-             recommended for production environments. Future Scrapy versions
-             may introduce related changes without a deprecation period or
-             warning.
-
 .. _install-asyncio:
 
 Installing the asyncio reactor
diff --git a/pylintrc b/pylintrc
index 699686e16..2cdd6321e 100644
--- a/pylintrc
+++ b/pylintrc
@@ -112,6 +112,7 @@ disable=abstract-method,
         unused-private-member,
         unused-variable,
         unused-wildcard-import,
+        use-implicit-booleaness-not-comparison,
         used-before-assignment,
         useless-object-inheritance,  # Required for Python 2 support
         useless-return,
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 6e77551c6..5f1dabd33 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -43,14 +43,14 @@ class ScrapyCommand:
 
     def long_desc(self):
         """A long description of the command. Return short description when not
-        available. It cannot contain newlines, since contents will be formatted
+        available. It cannot contain newlines since contents will be formatted
         by optparser which removes newlines and wraps text.
         """
         return self.short_desc()
 
     def help(self):
         """An extensive help for the command. It will be shown when using the
-        "help" command. It can contain newlines, since no post-formatting will
+        "help" command. It can contain newlines since no post-formatting will
         be applied to its contents.
         """
         return self.long_desc()
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 915cb5fe3..06cb96489 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -3,7 +3,7 @@ from time import time
 from urllib.parse import urlparse, urlunparse, urldefrag
 
 from twisted.web.http import HTTPClient
-from twisted.internet import defer, reactor
+from twisted.internet import defer
 from twisted.internet.protocol import ClientFactory
 
 from scrapy.http import Headers
@@ -170,6 +170,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
         p.followRedirect = self.followRedirect
         p.afterFoundGet = self.afterFoundGet
         if self.timeout:
+            from twisted.internet import reactor
             timeoutCall = reactor.callLater(self.timeout, p.timeout)
             self.deferred.addBoth(self._cancelTimeout, timeoutCall)
         return p
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 24873f75d..24a6187b9 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -164,7 +164,7 @@ def feed_process_params_from_cli(settings, output, output_format=None,
             message = (
                 'The -t command line option is deprecated in favor of '
                 'specifying the output format within the output URI. See the '
-                'documentation of the -o and -O options for more information.',
+                'documentation of the -o and -O options for more information.'
             )
             warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2)
             return {output[0]: {'format': output_format}}
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index b3ef7b463..8b109dced 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -3,8 +3,9 @@ This module provides some useful functions for working with
 scrapy.http.Response objects
 """
 import os
-import webbrowser
+import re
 import tempfile
+import webbrowser
 from typing import Any, Callable, Iterable, Optional, Tuple, Union
 from weakref import WeakKeyDictionary
 
@@ -80,8 +81,9 @@ def open_in_browser(
     body = response.body
     if isinstance(response, HtmlResponse):
         if b'<base' not in body:
-            repl = f'<head><base href="{response.url}">'
-            body = body.replace(b'<head>', to_bytes(repl))
+            repl = fr'\1<base href="{response.url}">'
+            body = re.sub(b"<!--.*?-->", b"", body, flags=re.DOTALL)
+            body = re.sub(rb"(<head(?:>|\s.*?>))", to_bytes(repl), body)
         ext = '.html'
     elif isinstance(response, TextResponse):
         ext = '.txt'
diff --git a/tox.ini b/tox.ini
index e4514f512..2031a2d92 100644
--- a/tox.ini
+++ b/tox.ini
@@ -66,7 +66,7 @@ commands =
 basepython = python3
 deps =
     {[testenv:extra-deps]deps}
-    pylint
+    pylint==2.12.1
 commands =
     pylint conftest.py docs extras scrapy setup.py tests
 


Test patch:
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index d6f4c0bb5..0a09f6109 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -83,3 +83,56 @@ class ResponseUtilsTest(unittest.TestCase):
         self.assertEqual(response_status_message(200), '200 OK')
         self.assertEqual(response_status_message(404), '404 Not Found')
         self.assertEqual(response_status_message(573), "573 Unknown Status")
+
+    def test_inject_base_url(self):
+        url = "http://www.example.com"
+
+        def check_base_url(burl):
+            path = urlparse(burl).path
+            if not os.path.exists(path):
+                path = burl.replace('file://', '')
+            with open(path, "rb") as f:
+                bbody = f.read()
+            self.assertEqual(bbody.count(b'<base href="' + to_bytes(url) + b'">'), 1)
+            return True
+
+        r1 = HtmlResponse(url, body=b"""
+        <html>
+            <head><title>Dummy</title></head>
+            <body><p>Hello world.</p></body>
+        </html>""")
+        r2 = HtmlResponse(url, body=b"""
+        <html>
+            <head id="foo"><title>Dummy</title></head>
+            <body>Hello world.</body>
+        </html>""")
+        r3 = HtmlResponse(url, body=b"""
+        <html>
+            <head><title>Dummy</title></head>
+            <body>
+                <header>Hello header</header>
+                <p>Hello world.</p>
+            </body>
+        </html>""")
+        r4 = HtmlResponse(url, body=b"""
+        <html>
+            <!-- <head>Dummy comment</head> -->
+            <head><title>Dummy</title></head>
+            <body><p>Hello world.</p></body>
+        </html>""")
+        r5 = HtmlResponse(url, body=b"""
+        <html>
+            <!--[if IE]>
+            <head><title>IE head</title></head>
+            <![endif]-->
+            <!--[if !IE]>-->
+            <head><title>Standard head</title></head>
+            <!--<![endif]-->
+            <body><p>Hello world.</p></body>
+        </html>""")
+
+        assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url"
+        assert open_in_browser(r2, _openfunc=check_base_url), "Inject base url with argumented head"
+        assert open_in_browser(r3, _openfunc=check_base_url), "Inject unique base url with misleading tag"
+        assert open_in_browser(r4, _openfunc=check_base_url), "Inject unique base url with misleading comment"
+        assert open_in_browser(r5, _openfunc=check_base_url), "Inject unique base url with conditional comment"