Repository: scrapy/scrapy

Issue:
DeprecationWarning: There is no current event loop
```
/home/runner/work/scrapy/scrapy/scrapy/utils/reactor.py:70: DeprecationWarning: There is no current event loop
  event_loop = asyncio.get_event_loop()
tests/test_downloadermiddleware.py: 2 warnings
tests/test_utils_asyncgen.py: 2 warnings
tests/test_utils_defer.py: 4 warnings
tests/test_utils_python.py: 2 warnings
tests/test_utils_signal.py: 2 warnings
  /home/runner/work/scrapy/scrapy/scrapy/utils/defer.py:272: DeprecationWarning: There is no current event loop
    return Deferred.fromFuture(asyncio.ensure_future(o))
tests/test_utils_asyncio.py::AsyncioTest::test_install_asyncio_reactor
  /home/runner/work/scrapy/scrapy/scrapy/utils/reactor.py:70: DeprecationWarning: There is no current event loop
    event_loop = asyncio.get_event_loop()
```

Not sure yet what does this imply and when can this break (this is from tests on 3.10 and 3.11).

Discussion:

Patch:
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index 439dfee51..8c1ae4bd3 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -8,10 +8,10 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: "3.10"
+        - python-version: "3.11"
           env:
             TOXENV: security
-        - python-version: "3.10"
+        - python-version: "3.11"
           env:
             TOXENV: flake8
         # Pylint requires installing reppy, which does not support Python 3.9
@@ -22,10 +22,10 @@ jobs:
         - python-version: 3.7
           env:
             TOXENV: typing
-        - python-version: "3.10"  # Keep in sync with .readthedocs.yml
+        - python-version: "3.11"  # Keep in sync with .readthedocs.yml
           env:
             TOXENV: docs
-        - python-version: "3.10"
+        - python-version: "3.11"
           env:
             TOXENV: twinecheck
 
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index f6b098b80..991b0b6e8 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -12,7 +12,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.10"
+        python-version: "3.11"
 
     - name: Check Tag
       id: check-release-tag
diff --git a/.readthedocs.yml b/.readthedocs.yml
index 390be3749..e71d34f3a 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -9,7 +9,7 @@ build:
   tools:
     # For available versions, see:
     # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
-    python: "3.10"  # Keep in sync with .github/workflows/checks.yml
+    python: "3.11"  # Keep in sync with .github/workflows/checks.yml
 
 python:
   install:
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 9ab479edd..2c2079f68 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -52,7 +52,7 @@ Scrapy is written in pure Python and depends on a few key Python packages (among
 * `twisted`_, an asynchronous networking framework
 * `cryptography`_ and `pyOpenSSL`_, to deal with various network-level security needs
 
-Some of these packages themselves depends on non-Python packages
+Some of these packages themselves depend on non-Python packages
 that might require additional installation steps depending on your platform.
 Please check :ref:`platform-specific guides below <intro-install-platform-notes>`.
 
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 6602f661d..1228e78da 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -257,9 +257,7 @@ class ExecutionEngine:
 
     def download(self, request: Request, spider: Optional[Spider] = None) -> Deferred:
         """Return a Deferred which fires with a Response as result, only downloader middlewares are applied"""
-        if spider is None:
-            spider = self.spider
-        else:
+        if spider is not None:
             warnings.warn(
                 "Passing a 'spider' argument to ExecutionEngine.download is deprecated",
                 category=ScrapyDeprecationWarning,
@@ -267,7 +265,7 @@ class ExecutionEngine:
             )
             if spider is not self.spider:
                 logger.warning("The spider '%s' does not match the open spider", spider.name)
-        if spider is None:
+        if self.spider is None:
             raise RuntimeError(f"No open spider to crawl: {request}")
         return self._download(request, spider).addBoth(self._downloaded, request, spider)
 
@@ -278,11 +276,14 @@ class ExecutionEngine:
         self.slot.remove_request(request)
         return self.download(result, spider) if isinstance(result, Request) else result
 
-    def _download(self, request: Request, spider: Spider) -> Deferred:
+    def _download(self, request: Request, spider: Optional[Spider]) -> Deferred:
         assert self.slot is not None  # typing
 
         self.slot.add_request(request)
 
+        if spider is None:
+            spider = self.spider
+
         def _on_success(result: Union[Response, Request]) -> Union[Response, Request]:
             if not isinstance(result, (Response, Request)):
                 raise TypeError(f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}")
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 1deda42bd..dd8a7e797 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -78,4 +78,7 @@ class HttpProxyMiddleware:
                     del request.headers[b'Proxy-Authorization']
                 del request.meta['_auth_proxy']
         elif b'Proxy-Authorization' in request.headers:
-            del request.headers[b'Proxy-Authorization']
+            if proxy_url:
+                request.meta['_auth_proxy'] = proxy_url
+            else:
+                del request.headers[b'Proxy-Authorization']
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 8fcf31cab..38aefd6d0 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -26,7 +26,7 @@ from twisted.python import failure
 from twisted.python.failure import Failure
 
 from scrapy.exceptions import IgnoreRequest
-from scrapy.utils.reactor import is_asyncio_reactor_installed
+from scrapy.utils.reactor import is_asyncio_reactor_installed, get_asyncio_event_loop_policy
 
 
 def defer_fail(_failure: Failure) -> Deferred:
@@ -269,7 +269,8 @@ def deferred_from_coro(o) -> Any:
             return ensureDeferred(o)
         else:
             # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
-            return Deferred.fromFuture(asyncio.ensure_future(o))
+            event_loop = get_asyncio_event_loop_policy().get_event_loop()
+            return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop))
     return o
 
 
@@ -320,7 +321,8 @@ def deferred_to_future(d: Deferred) -> Future:
                 d = treq.get('https://example.com/additional')
                 additional_response = await deferred_to_future(d)
     """
-    return d.asFuture(asyncio.get_event_loop())
+    policy = get_asyncio_event_loop_policy()
+    return d.asFuture(policy.get_event_loop())
 
 
 def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]:
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 652733ce8..ddf354d88 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -51,6 +51,19 @@ class CallLaterOnce:
         return self._func(*self._a, **self._kw)
 
 
+def get_asyncio_event_loop_policy():
+    policy = asyncio.get_event_loop_policy()
+    if (
+        sys.version_info >= (3, 8)
+        and sys.platform == "win32"
+        and not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy)
+    ):
+        policy = asyncio.WindowsSelectorEventLoopPolicy()
+        asyncio.set_event_loop_policy(policy)
+
+    return policy
+
+
 def install_reactor(reactor_path, event_loop_path=None):
     """Installs the :mod:`~twisted.internet.reactor` with the specified
     import path. Also installs the asyncio event loop with the specified import
@@ -58,16 +71,14 @@ def install_reactor(reactor_path, event_loop_path=None):
     reactor_class = load_object(reactor_path)
     if reactor_class is asyncioreactor.AsyncioSelectorReactor:
         with suppress(error.ReactorAlreadyInstalledError):
-            if sys.version_info >= (3, 8) and sys.platform == "win32":
-                policy = asyncio.get_event_loop_policy()
-                if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy):
-                    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+            policy = get_asyncio_event_loop_policy()
             if event_loop_path is not None:
                 event_loop_class = load_object(event_loop_path)
                 event_loop = event_loop_class()
                 asyncio.set_event_loop(event_loop)
             else:
-                event_loop = asyncio.get_event_loop()
+                event_loop = policy.get_event_loop()
+
             asyncioreactor.install(eventloop=event_loop)
     else:
         *module, _ = reactor_path.split(".")


Test patch:
diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
index fdb9f4980..174d245ca 100644
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@@ -3,11 +3,11 @@ on: [push, pull_request]
 
 jobs:
   tests:
-    runs-on: macos-10.15
+    runs-on: macos-11
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
 
     steps:
     - uses: actions/checkout@v3
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index d2bfe4a5f..9c3ce8115 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -17,13 +17,10 @@ jobs:
         - python-version: "3.10"
           env:
             TOXENV: py
-        - python-version: "3.10"
-          env:
-            TOXENV: asyncio
-        - python-version: "3.11.0-rc.2"
+        - python-version: "3.11"
           env:
             TOXENV: py
-        - python-version: "3.11.0-rc.2"
+        - python-version: "3.11"
           env:
             TOXENV: asyncio
         - python-version: pypy3.9
@@ -57,7 +54,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
 
     - name: Install system libraries
-      if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.11.0-rc.2'
+      if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned')
       run: |
         sudo apt-get update
         sudo apt-get install libxml2-dev libxslt-dev
diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
index 14683fd53..f60c48841 100644
--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@@ -23,6 +23,13 @@ jobs:
         - python-version: "3.10"
           env:
             TOXENV: asyncio
+# no binary package for lxml for 3.11 yet
+#        - python-version: "3.11"
+#          env:
+#            TOXENV: py
+#        - python-version: "3.11"
+#          env:
+#            TOXENV: asyncio
 
     steps:
     - uses: actions/checkout@v3
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 70eb94d77..44434f90e 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -400,6 +400,9 @@ class TestHttpProxyMiddleware(TestCase):
         self.assertNotIn(b'Proxy-Authorization', request.headers)
 
     def test_proxy_authentication_header_proxy_without_credentials(self):
+        """As long as the proxy URL in request metadata remains the same, the
+        Proxy-Authorization header is used and kept, and may even be
+        changed."""
         middleware = HttpProxyMiddleware()
         request = Request(
             'https://example.com',
@@ -408,7 +411,16 @@ class TestHttpProxyMiddleware(TestCase):
         )
         assert middleware.process_request(request, spider) is None
         self.assertEqual(request.meta['proxy'], 'https://example.com')
-        self.assertNotIn(b'Proxy-Authorization', request.headers)
+        self.assertEqual(request.headers['Proxy-Authorization'], b'Basic foo')
+
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        self.assertEqual(request.headers['Proxy-Authorization'], b'Basic foo')
+
+        request.headers['Proxy-Authorization'] = b'Basic bar'
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        self.assertEqual(request.headers['Proxy-Authorization'], b'Basic bar')
 
     def test_proxy_authentication_header_proxy_with_same_credentials(self):
         middleware = HttpProxyMiddleware()
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index 295323e4d..741c6a505 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -1,3 +1,4 @@
+import warnings
 from unittest import TestCase
 
 from pytest import mark
@@ -13,5 +14,6 @@ class AsyncioTest(TestCase):
         self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio')
 
     def test_install_asyncio_reactor(self):
-        # this should do nothing
-        install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+        with warnings.catch_warnings(record=True) as w:
+            install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+            self.assertEqual(len(w), 0)