Repository: scrapy/scrapy
Pull request #5929 (2023-05-09 15:17:38)

Configurable exception list retry middleware 2701
Resolves https://github.com/scrapy/scrapy/issues/2701, closes https://github.com/scrapy/scrapy/pull/2984, closes https://github.com/scrapy/scrapy/pull/3334.


Patch:
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 3916bd295..a65bab3ca 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -134,6 +134,63 @@ Common use cases for asynchronous code include:
 .. _aio-libs: https://github.com/aio-libs
 
 
+.. _inline-requests:
+
+Inline requests
+===============
+
+The spider below shows how to send a request and await its response all from
+within a spider callback:
+
+.. code-block:: python
+
+    from scrapy import Spider, Request
+    from scrapy.utils.defer import maybe_deferred_to_future
+
+
+    class SingleRequestSpider(Spider):
+        name = "single"
+        start_urls = ["https://example.org/product"]
+
+        async def parse(self, response, **kwargs):
+            additional_request = Request("https://example.org/price")
+            deferred = self.crawler.engine.download(additional_request)
+            additional_response = await maybe_deferred_to_future(deferred)
+            yield {
+                "h1": response.css("h1").get(),
+                "price": additional_response.css("#price").get(),
+            }
+
+You can also send multiple requests in parallel:
+
+.. code-block:: python
+
+    from scrapy import Spider, Request
+    from scrapy.utils.defer import maybe_deferred_to_future
+    from twisted.internet.defer import DeferredList
+
+
+    class MultipleRequestsSpider(Spider):
+        name = "multiple"
+        start_urls = ["https://example.com/product"]
+
+        async def parse(self, response, **kwargs):
+            additional_requests = [
+                Request("https://example.com/price"),
+                Request("https://example.com/color"),
+            ]
+            deferreds = []
+            for r in additional_requests:
+                deferred = self.crawler.engine.download(r)
+                deferreds.append(deferred)
+            responses = await maybe_deferred_to_future(DeferredList(deferreds))
+            yield {
+                "h1": response.css("h1::text").get(),
+                "price": responses[0][1].css(".price::text").get(),
+                "price2": responses[1][1].css(".color::text").get(),
+            }
+
+
 .. _sync-async-spider-middleware:
 
 Mixing synchronous and asynchronous spider middlewares
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 7665a901a..a8e5b23bf 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -915,6 +915,7 @@ settings (see the settings documentation for more info):
 * :setting:`RETRY_ENABLED`
 * :setting:`RETRY_TIMES`
 * :setting:`RETRY_HTTP_CODES`
+* :setting:`RETRY_EXCEPTIONS`
 
 .. reqmeta:: dont_retry
 
@@ -966,6 +967,37 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because
 it is a common code used to indicate server overload. It is not included by
 default because HTTP specs say so.
 
+.. setting:: RETRY_EXCEPTIONS
+
+RETRY_EXCEPTIONS
+^^^^^^^^^^^^^^^^
+
+Default::
+
+    [
+        'twisted.internet.defer.TimeoutError',
+        'twisted.internet.error.TimeoutError',
+        'twisted.internet.error.DNSLookupError',
+        'twisted.internet.error.ConnectionRefusedError',
+        'twisted.internet.error.ConnectionDone',
+        'twisted.internet.error.ConnectError',
+        'twisted.internet.error.ConnectionLost',
+        'twisted.internet.error.TCPTimedOutError',
+        'twisted.web.client.ResponseFailed',
+        IOError,
+        'scrapy.core.downloader.handlers.http11.TunnelError',
+    ]
+
+List of exceptions to retry.
+
+Each list entry may be an exception type or its import path as a string.
+
+An exception will not be caught when the exception type is not in
+:setting:`RETRY_EXCEPTIONS` or when the maximum number of retries for a request
+has been exceeded (see :setting:`RETRY_TIMES`). To learn about uncaught
+exception propagation, see
+:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`.
+
 .. setting:: RETRY_PRIORITY_ADJUST
 
 RETRY_PRIORITY_ADJUST
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index eef0bb5ca..b31dc069e 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -101,12 +101,12 @@ The storages backends supported out of the box are:
 
 -   :ref:`topics-feed-storage-fs`
 -   :ref:`topics-feed-storage-ftp`
--   :ref:`topics-feed-storage-s3` (requires botocore_)
+-   :ref:`topics-feed-storage-s3` (requires boto3_)
 -   :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
 -   :ref:`topics-feed-storage-stdout`
 
 Some storage backends may be unavailable if the required external libraries are
-not available. For example, the S3 backend is only available if the botocore_
+not available. For example, the S3 backend is only available if the boto3_
 library is installed.
 
 
@@ -175,6 +175,12 @@ FTP supports two different connection modes: `active or passive
 mode by default. To use the active connection mode instead, set the
 :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
 
+The default value for the ``overwrite`` key in the :setting:`FEEDS` for this 
+storage backend is: ``True``.
+
+.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the
+     previous version of your data.
+
 This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
 
 
@@ -193,7 +199,7 @@ The feeds are stored on `Amazon S3`_.
 
     -   ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
 
--   Required external libraries: `botocore`_ >= 1.4.87
+-   Required external libraries: `boto3`_ >= 1.20.0
 
 The AWS credentials can be passed as user/password in the URI, or they can be
 passed through the following settings:
@@ -209,6 +215,12 @@ You can also define a custom ACL and custom endpoint for exported feeds using th
 -   :setting:`FEED_STORAGE_S3_ACL`
 -   :setting:`AWS_ENDPOINT_URL`
 
+The default value for the ``overwrite`` key in the :setting:`FEEDS` for this 
+storage backend is: ``True``.
+
+.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the
+     previous version of your data.
+
 This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
 
 
@@ -236,6 +248,12 @@ You can set a *Project ID* and *Access Control List (ACL)* through the following
 -   :setting:`FEED_STORAGE_GCS_ACL`
 -   :setting:`GCS_PROJECT_ID`
 
+The default value for the ``overwrite`` key in the :setting:`FEEDS` for this 
+storage backend is: ``True``.
+
+.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the
+     previous version of your data.
+
 This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
 
 .. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
@@ -488,6 +506,8 @@ as a fallback value if that key is not provided for a specific feed definition:
     -   :ref:`topics-feed-storage-s3`: ``True`` (appending `is not supported
         <https://forums.aws.amazon.com/message.jspa?messageID=540395>`_)
 
+    -   :ref:`topics-feed-storage-gcs`: ``True`` (appending is not supported)
+
     -   :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported)
 
     .. versionadded:: 2.4.0
@@ -779,6 +799,6 @@ source spider in the feed URI:
 
 .. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
 .. _Amazon S3: https://aws.amazon.com/s3/
-.. _botocore: https://github.com/boto/botocore
+.. _boto3: https://github.com/boto/boto3
 .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 .. _Google Cloud Storage: https://cloud.google.com/storage/
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 081642a4b..50cbc3111 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -9,31 +9,36 @@ RETRY_HTTP_CODES - which HTTP response codes to retry
 Failed pages are collected on the scraping process and rescheduled at the end,
 once the spider has finished crawling all regular (non failed) pages.
 """
+import warnings
 from logging import Logger, getLogger
 from typing import Optional, Union
 
-from twisted.internet import defer
-from twisted.internet.error import (
-    ConnectError,
-    ConnectionDone,
-    ConnectionLost,
-    ConnectionRefusedError,
-    DNSLookupError,
-    TCPTimedOutError,
-    TimeoutError,
-)
-from twisted.web.client import ResponseFailed
-
-from scrapy.core.downloader.handlers.http11 import TunnelError
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.http.request import Request
+from scrapy.settings import Settings
 from scrapy.spiders import Spider
+from scrapy.utils.misc import load_object
 from scrapy.utils.python import global_object_name
 from scrapy.utils.response import response_status_message
 
 retry_logger = getLogger(__name__)
 
 
+class BackwardsCompatibilityMetaclass(type):
+    @property
+    def EXCEPTIONS_TO_RETRY(cls):
+        warnings.warn(
+            "Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. "
+            "Use the RETRY_EXCEPTIONS setting instead.",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        return tuple(
+            load_object(x) if isinstance(x, str) else x
+            for x in Settings().getlist("RETRY_EXCEPTIONS")
+        )
+
+
 def get_retry_request(
     request: Request,
     *,
@@ -121,23 +126,7 @@ def get_retry_request(
     return None
 
 
-class RetryMiddleware:
-    # IOError is raised by the HttpCompression middleware when trying to
-    # decompress an empty response
-    EXCEPTIONS_TO_RETRY = (
-        defer.TimeoutError,
-        TimeoutError,
-        DNSLookupError,
-        ConnectionRefusedError,
-        ConnectionDone,
-        ConnectError,
-        ConnectionLost,
-        TCPTimedOutError,
-        ResponseFailed,
-        IOError,
-        TunnelError,
-    )
-
+class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
     def __init__(self, settings):
         if not settings.getbool("RETRY_ENABLED"):
             raise NotConfigured
@@ -147,6 +136,16 @@ class RetryMiddleware:
         )
         self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")
 
+        if not hasattr(
+            self, "EXCEPTIONS_TO_RETRY"
+        ):  # If EXCEPTIONS_TO_RETRY is not "overriden"
+            self.exceptions_to_retry = tuple(
+                load_object(x) if isinstance(x, str) else x
+                for x in settings.getlist("RETRY_EXCEPTIONS")
+            )
+        else:
+            self.exceptions_to_retry = self.EXCEPTIONS_TO_RETRY
+
     @classmethod
     def from_crawler(cls, crawler):
         return cls(crawler.settings)
@@ -160,7 +159,7 @@ class RetryMiddleware:
         return response
 
     def process_exception(self, request, exception, spider):
-        if isinstance(exception, self.EXCEPTIONS_TO_RETRY) and not request.meta.get(
+        if isinstance(exception, self.exceptions_to_retry) and not request.meta.get(
             "dont_retry", False
         ):
             return self._retry(request, exception, spider)
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index bb3e3c662..4538c9ee1 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -255,6 +255,9 @@ class CsvItemExporter(BaseItemExporter):
         values = list(self._build_row(x for _, x in fields))
         self.csv_writer.writerow(values)
 
+    def finish_exporting(self):
+        self.stream.detach()  # Avoid closing the wrapped file.
+
     def _build_row(self, values):
         for s in values:
             try:
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index bcf0b779a..39934cbf3 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -33,6 +33,13 @@ from scrapy.utils.python import get_func_args, without_none_values
 
 logger = logging.getLogger(__name__)
 
+try:
+    import boto3  # noqa: F401
+
+    IS_BOTO3_AVAILABLE = True
+except ImportError:
+    IS_BOTO3_AVAILABLE = False
+
 
 def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
     argument_names = get_func_args(builder)
@@ -176,16 +183,38 @@ class S3FeedStorage(BlockingFeedStorage):
         self.keyname = u.path[1:]  # remove first "/"
         self.acl = acl
         self.endpoint_url = endpoint_url
-        import botocore.session
-
-        session = botocore.session.get_session()
-        self.s3_client = session.create_client(
-            "s3",
-            aws_access_key_id=self.access_key,
-            aws_secret_access_key=self.secret_key,
-            aws_session_token=self.session_token,
-            endpoint_url=self.endpoint_url,
-        )
+
+        if IS_BOTO3_AVAILABLE:
+            import boto3.session
+
+            session = boto3.session.Session()
+
+            self.s3_client = session.client(
+                "s3",
+                aws_access_key_id=self.access_key,
+                aws_secret_access_key=self.secret_key,
+                aws_session_token=self.session_token,
+                endpoint_url=self.endpoint_url,
+            )
+        else:
+            warnings.warn(
+                "`botocore` usage has been deprecated for S3 feed "
+                "export, please use `boto3` to avoid problems",
+                category=ScrapyDeprecationWarning,
+            )
+
+            import botocore.session
+
+            session = botocore.session.get_session()
+
+            self.s3_client = session.create_client(
+                "s3",
+                aws_access_key_id=self.access_key,
+                aws_secret_access_key=self.secret_key,
+                aws_session_token=self.session_token,
+                endpoint_url=self.endpoint_url,
+            )
+
         if feed_options and feed_options.get("overwrite", True) is False:
             logger.warning(
                 "S3 does not support appending to files. To "
@@ -208,10 +237,16 @@ class S3FeedStorage(BlockingFeedStorage):
 
     def _store_in_thread(self, file):
         file.seek(0)
-        kwargs = {"ACL": self.acl} if self.acl else {}
-        self.s3_client.put_object(
-            Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs
-        )
+        if IS_BOTO3_AVAILABLE:
+            kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {}
+            self.s3_client.upload_fileobj(
+                Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs
+            )
+        else:
+            kwargs = {"ACL": self.acl} if self.acl else {}
+            self.s3_client.put_object(
+                Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs
+            )
         file.close()
 
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 260ec1701..89837b4ab 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -258,6 +258,21 @@ RETRY_ENABLED = True
 RETRY_TIMES = 2  # initial response + 2 retries = 3 requests
 RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
 RETRY_PRIORITY_ADJUST = -1
+RETRY_EXCEPTIONS = [
+    "twisted.internet.defer.TimeoutError",
+    "twisted.internet.error.TimeoutError",
+    "twisted.internet.error.DNSLookupError",
+    "twisted.internet.error.ConnectionRefusedError",
+    "twisted.internet.error.ConnectionDone",
+    "twisted.internet.error.ConnectError",
+    "twisted.internet.error.ConnectionLost",
+    "twisted.internet.error.TCPTimedOutError",
+    "twisted.web.client.ResponseFailed",
+    # IOError is raised by the HttpCompression middleware when trying to
+    # decompress an empty response
+    IOError,
+    "scrapy.core.downloader.handlers.http11.TunnelError",
+]
 
 ROBOTSTXT_OBEY = False
 ROBOTSTXT_PARSER = "scrapy.robotstxt.ProtegoRobotParser"
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index d25ebbdf4..a46274fef 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -340,8 +340,9 @@ def deferred_to_future(d: Deferred) -> Future:
         class MySpider(Spider):
             ...
             async def parse(self, response):
-                d = treq.get('https://example.com/additional')
-                additional_response = await deferred_to_future(d)
+                additional_request = scrapy.Request('https://example.org/price')
+                deferred = self.crawler.engine.download(additional_request)
+                additional_response = await deferred_to_future(deferred)
     """
     return d.asFuture(_get_asyncio_event_loop())
 
@@ -368,8 +369,9 @@ def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]:
         class MySpider(Spider):
             ...
             async def parse(self, response):
-                d = treq.get('https://example.com/additional')
-                extra_response = await maybe_deferred_to_future(d)
+                additional_request = scrapy.Request('https://example.org/price')
+                deferred = self.crawler.engine.download(additional_request)
+                additional_response = await maybe_deferred_to_future(deferred)
     """
     if not is_asyncio_reactor_installed():
         return d
diff --git a/tox.ini b/tox.ini
index af8f1f57a..5f8bf85f2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -18,8 +18,6 @@ deps =
     mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy'
     # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower)
     markupsafe < 2.1.0; python_version < '3.8' and implementation_name != 'pypy'
-    # Extras
-    botocore>=1.4.87
 passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID
@@ -90,11 +88,6 @@ deps =
 
     # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies
     # above, hence we do not install it in pinned environments at the moment
-
-    # Extras
-    botocore==1.4.87
-    google-cloud-storage==1.29.0
-    Pillow==7.1.0
 setenv =
     _SCRAPY_PINNED=true
 install_command =
@@ -126,14 +119,27 @@ commands = {[pinned]commands}
 basepython = python3
 deps =
     {[testenv]deps}
-    boto
+    boto3
     google-cloud-storage
     # Twisted[http2] currently forces old mitmproxy because of h2 version
     # restrictions in their deps, so we need to pin old markupsafe here too.
     markupsafe < 2.1.0
     robotexclusionrulesparser
-    Pillow>=4.0.0
-    Twisted[http2]>=17.9.0
+    Pillow
+    Twisted[http2]
+
+[testenv:extra-deps-pinned]
+basepython = python3.7
+deps =
+    {[pinned]deps}
+    boto3==1.20.0
+    google-cloud-storage==1.29.0
+    Pillow==7.1.0
+    robotexclusionrulesparser==1.6.2
+install_command = {[pinned]install_command}
+setenv =
+    {[pinned]setenv}
+commands = {[pinned]commands}
 
 [testenv:asyncio]
 commands =
@@ -193,3 +199,24 @@ deps = {[docs]deps}
 setenv = {[docs]setenv}
 commands =
     sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
+
+
+# Run S3 tests with botocore installed but without boto3.
+
+[testenv:botocore]
+deps =
+    {[testenv]deps}
+    botocore>=1.4.87
+commands =
+    pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}
+
+[testenv:botocore-pinned]
+basepython = python3.7
+deps =
+    {[pinned]deps}
+    botocore==1.4.87
+install_command = {[pinned]install_command}
+setenv =
+    {[pinned]setenv}
+commands =
+    pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}


Test patch:
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 8fcf90a18..96b26a1f8 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -37,10 +37,19 @@ jobs:
         - python-version: pypy3.7
           env:
             TOXENV: pypy3-pinned
+        - python-version: 3.7.13
+          env:
+            TOXENV: extra-deps-pinned
+        - python-version: 3.7.13
+          env:
+            TOXENV: botocore-pinned
 
         - python-version: "3.11"
           env:
             TOXENV: extra-deps
+        - python-version: "3.11"
+          env:
+            TOXENV: botocore
 
     steps:
     - uses: actions/checkout@v3
diff --git a/tests/mockserver.py b/tests/mockserver.py
index eb4f8334e..647b0682e 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -1,11 +1,13 @@
 import argparse
 import json
+import os
 import random
 import sys
 from pathlib import Path
 from shutil import rmtree
 from subprocess import PIPE, Popen
 from tempfile import mkdtemp
+from typing import Dict
 from urllib.parse import urlencode
 
 from OpenSSL import SSL
@@ -20,7 +22,6 @@ from twisted.web.static import File
 from twisted.web.util import redirectTo
 
 from scrapy.utils.python import to_bytes, to_unicode
-from scrapy.utils.test import get_testenv
 
 
 def getarg(request, name, default=None, type=None):
@@ -32,6 +33,16 @@ def getarg(request, name, default=None, type=None):
     return default
 
 
+def get_mockserver_env() -> Dict[str, str]:
+    """Return a OS environment dict suitable to run mockserver processes."""
+
+    tests_path = Path(__file__).parent.parent
+    pythonpath = str(tests_path) + os.pathsep + os.environ.get("PYTHONPATH", "")
+    env = os.environ.copy()
+    env["PYTHONPATH"] = pythonpath
+    return env
+
+
 # most of the following resources are copied from twisted.web.test.test_webclient
 class ForeverTakingResource(resource.Resource):
     """
@@ -264,7 +275,7 @@ class MockServer:
         self.proc = Popen(
             [sys.executable, "-u", "-m", "tests.mockserver", "-t", "http"],
             stdout=PIPE,
-            env=get_testenv(),
+            env=get_mockserver_env(),
         )
         http_address = self.proc.stdout.readline().strip().decode("ascii")
         https_address = self.proc.stdout.readline().strip().decode("ascii")
@@ -308,7 +319,7 @@ class MockDNSServer:
         self.proc = Popen(
             [sys.executable, "-u", "-m", "tests.mockserver", "-t", "dns"],
             stdout=PIPE,
-            env=get_testenv(),
+            env=get_mockserver_env(),
         )
         self.host = "127.0.0.1"
         self.port = int(
@@ -331,7 +342,7 @@ class MockFTPServer:
         self.proc = Popen(
             [sys.executable, "-u", "-m", "tests.ftpserver", "-d", str(self.path)],
             stderr=PIPE,
-            env=get_testenv(),
+            env=get_mockserver_env(),
         )
         for line in self.proc.stderr:
             if b"starting FTP server" in line:
diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index 15833cd19..25ded143c 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -1,4 +1,5 @@
 import json
+import os
 import pstats
 import shutil
 import sys
@@ -14,6 +15,8 @@ from scrapy.utils.test import get_testenv
 class CmdlineTest(unittest.TestCase):
     def setUp(self):
         self.env = get_testenv()
+        tests_path = Path(__file__).parent.parent
+        self.env["PYTHONPATH"] += os.pathsep + str(tests_path.parent)
         self.env["SCRAPY_SETTINGS_MODULE"] = "tests.test_cmdline.settings"
 
     def _execute(self, *new_args, **kwargs):
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 706bfbaa9..ecb9c9b62 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,4 +1,5 @@
 import logging
+import os
 import platform
 import subprocess
 import sys
@@ -23,8 +24,8 @@ from scrapy.spiderloader import SpiderLoader
 from scrapy.utils.log import configure_logging, get_scrapy_root_handler
 from scrapy.utils.misc import load_object
 from scrapy.utils.spider import DefaultSpider
-from scrapy.utils.test import get_crawler, get_testenv
-from tests.mockserver import MockServer
+from scrapy.utils.test import get_crawler
+from tests.mockserver import MockServer, get_mockserver_env
 
 
 class BaseCrawlerTest(unittest.TestCase):
@@ -289,12 +290,16 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
 
 class ScriptRunnerMixin:
     script_dir: Path
+    cwd = os.getcwd()
 
     def run_script(self, script_name: str, *script_args):
         script_path = self.script_dir / script_name
         args = [sys.executable, str(script_path)] + list(script_args)
         p = subprocess.Popen(
-            args, env=get_testenv(), stdout=subprocess.PIPE, stderr=subprocess.PIPE
+            args,
+            env=get_mockserver_env(),
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
         )
         stdout, stderr = p.communicate()
         return stderr.decode("utf-8")
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 63bd61848..97ae1e29a 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -1,5 +1,6 @@
 import logging
 import unittest
+import warnings
 
 from testfixtures import LogCapture
 from twisted.internet import defer
@@ -15,6 +16,7 @@ from twisted.web.client import ResponseFailed
 from scrapy.downloadermiddlewares.retry import RetryMiddleware, get_retry_request
 from scrapy.exceptions import IgnoreRequest
 from scrapy.http import Request, Response
+from scrapy.settings.default_settings import RETRY_EXCEPTIONS
 from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
 
@@ -110,19 +112,48 @@ class RetryTest(unittest.TestCase):
             == 2
         )
 
-    def _test_retry_exception(self, req, exception):
+    def test_exception_to_retry_added(self):
+        exc = ValueError
+        settings_dict = {
+            "RETRY_EXCEPTIONS": list(RETRY_EXCEPTIONS) + [exc],
+        }
+        crawler = get_crawler(Spider, settings_dict=settings_dict)
+        mw = RetryMiddleware.from_crawler(crawler)
+        req = Request(f"http://www.scrapytest.org/{exc.__name__}")
+        self._test_retry_exception(req, exc("foo"), mw)
+
+    def test_exception_to_retry_customMiddleware(self):
+        exc = ValueError
+
+        with warnings.catch_warnings(record=True) as warns:
+
+            class MyRetryMiddleware(RetryMiddleware):
+                EXCEPTIONS_TO_RETRY = RetryMiddleware.EXCEPTIONS_TO_RETRY + (exc,)
+
+            self.assertEqual(len(warns), 1)
+
+        mw2 = MyRetryMiddleware.from_crawler(self.crawler)
+        req = Request(f"http://www.scrapytest.org/{exc.__name__}")
+        req = mw2.process_exception(req, exc("foo"), self.spider)
+        assert isinstance(req, Request)
+        self.assertEqual(req.meta["retry_times"], 1)
+
+    def _test_retry_exception(self, req, exception, mw=None):
+        if mw is None:
+            mw = self.mw
+
         # first retry
-        req = self.mw.process_exception(req, exception, self.spider)
+        req = mw.process_exception(req, exception, self.spider)
         assert isinstance(req, Request)
         self.assertEqual(req.meta["retry_times"], 1)
 
         # second retry
-        req = self.mw.process_exception(req, exception, self.spider)
+        req = mw.process_exception(req, exception, self.spider)
         assert isinstance(req, Request)
         self.assertEqual(req.meta["retry_times"], 2)
 
         # discard it
-        req = self.mw.process_exception(req, exception, self.spider)
+        req = mw.process_exception(req, exception, self.spider)
         self.assertEqual(req, None)
 
 
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 1491e788e..63bebcf7a 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -90,6 +90,10 @@ class BaseItemExporterTest(unittest.TestCase):
             if self.ie.__class__ is not BaseItemExporter:
                 raise
         self.ie.finish_exporting()
+        # Delete the item exporter object, so that if it causes the output
+        # file handle to be closed, which should not be the case, follow-up
+        # interactions with the output file handle will surface the issue.
+        del self.ie
         self._check_output()
 
     def test_export_item(self):
@@ -243,6 +247,7 @@ class PickleItemExporterTest(BaseItemExporterTest):
         ie.export_item(i1)
         ie.export_item(i2)
         ie.finish_exporting()
+        del ie  # See the first “del self.ie” in this file for context.
         f.seek(0)
         self.assertEqual(self.item_class(**pickle.load(f)), i1)
         self.assertEqual(self.item_class(**pickle.load(f)), i2)
@@ -254,6 +259,7 @@ class PickleItemExporterTest(BaseItemExporterTest):
         ie.start_exporting()
         ie.export_item(item)
         ie.finish_exporting()
+        del ie  # See the first “del self.ie” in this file for context.
         self.assertEqual(pickle.loads(fp.getvalue()), item)
 
 
@@ -279,6 +285,7 @@ class MarshalItemExporterTest(BaseItemExporterTest):
         ie.start_exporting()
         ie.export_item(item)
         ie.finish_exporting()
+        del ie  # See the first “del self.ie” in this file for context.
         fp.seek(0)
         self.assertEqual(marshal.load(fp), item)
 
@@ -314,6 +321,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
         ie.start_exporting()
         ie.export_item(item)
         ie.finish_exporting()
+        del ie  # See the first “del self.ie” in this file for context.
         self.assertCsvEqual(fp.getvalue(), expected)
 
     def test_header_export_all(self):
@@ -345,6 +353,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
             ie.export_item(item)
             ie.export_item(item)
             ie.finish_exporting()
+            del ie  # See the first “del self.ie” in this file for context.
             self.assertCsvEqual(
                 output.getvalue(), b"age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n"
             )
@@ -429,6 +438,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
         ie.start_exporting()
         ie.export_item(item)
         ie.finish_exporting()
+        del ie  # See the first “del self.ie” in this file for context.
         self.assertXmlEquivalent(fp.getvalue(), expected_value)
 
     def _check_output(self):
@@ -536,6 +546,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
+        del self.ie  # See the first “del self.ie” in this file for context.
         exported = json.loads(to_unicode(self.output.getvalue()))
         self.assertEqual(exported, self._expected_nested)
 
@@ -550,6 +561,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
         self.ie.start_exporting()
         self.ie.export_item(item)
         self.ie.finish_exporting()
+        del self.ie  # See the first “del self.ie” in this file for context.
         exported = json.loads(to_unicode(self.output.getvalue()))
         item["time"] = str(item["time"])
         self.assertEqual(exported, item)
@@ -575,6 +587,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         self.ie.export_item(item)
         self.ie.export_item(item)
         self.ie.finish_exporting()
+        del self.ie  # See the first “del self.ie” in this file for context.
         exported = json.loads(to_unicode(self.output.getvalue()))
         self.assertEqual(
             exported, [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()]
@@ -593,6 +606,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
+        del self.ie  # See the first “del self.ie” in this file for context.
         exported = json.loads(to_unicode(self.output.getvalue()))
         expected = {
             "name": "Jesus",
@@ -607,6 +621,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
+        del self.ie  # See the first “del self.ie” in this file for context.
         exported = json.loads(to_unicode(self.output.getvalue()))
         expected = {"name": "Jesus", "age": {"name": "Maria", "age": i1}}
         self.assertEqual(exported, [expected])
@@ -616,6 +631,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         self.ie.start_exporting()
         self.ie.export_item(item)
         self.ie.finish_exporting()
+        del self.ie  # See the first “del self.ie” in this file for context.
         exported = json.loads(to_unicode(self.output.getvalue()))
         item["time"] = str(item["time"])
         self.assertEqual(exported, [item])
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index b1059099a..62a5697cd 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -36,6 +36,7 @@ from scrapy import signals
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.exporters import CsvItemExporter, JsonItemExporter
 from scrapy.extensions.feedexport import (
+    IS_BOTO3_AVAILABLE,
     BlockingFeedStorage,
     FeedExporter,
     FeedSlot,
@@ -236,8 +237,10 @@ class BlockingFeedStorageTest(unittest.TestCase):
 
 
 class S3FeedStorageTest(unittest.TestCase):
-    def test_parse_credentials(self):
+    def setUp(self):
         skip_if_no_boto()
+
+    def test_parse_credentials(self):
         aws_credentials = {
             "AWS_ACCESS_KEY_ID": "settings_key",
             "AWS_SECRET_ACCESS_KEY": "settings_secret",
@@ -273,8 +276,6 @@ class S3FeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_store(self):
-        skip_if_no_boto()
-
         settings = {
             "AWS_ACCESS_KEY_ID": "access_key",
             "AWS_SECRET_ACCESS_KEY": "secret_key",
@@ -286,30 +287,39 @@ class S3FeedStorageTest(unittest.TestCase):
         verifyObject(IFeedStorage, storage)
 
         file = mock.MagicMock()
-        from botocore.stub import Stubber
-
-        with Stubber(storage.s3_client) as stub:
-            stub.add_response(
-                "put_object",
-                expected_params={
-                    "Body": file,
-                    "Bucket": bucket,
-                    "Key": key,
-                },
-                service_response={},
-            )
 
+        if IS_BOTO3_AVAILABLE:
+            storage.s3_client = mock.MagicMock()
             yield storage.store(file)
-
-            stub.assert_no_pending_responses()
             self.assertEqual(
-                file.method_calls,
-                [
-                    mock.call.seek(0),
-                    # The call to read does not happen with Stubber
-                    mock.call.close(),
-                ],
+                storage.s3_client.upload_fileobj.call_args,
+                mock.call(Bucket=bucket, Key=key, Fileobj=file),
             )
+        else:
+            from botocore.stub import Stubber
+
+            with Stubber(storage.s3_client) as stub:
+                stub.add_response(
+                    "put_object",
+                    expected_params={
+                        "Body": file,
+                        "Bucket": bucket,
+                        "Key": key,
+                    },
+                    service_response={},
+                )
+
+                yield storage.store(file)
+
+                stub.assert_no_pending_responses()
+                self.assertEqual(
+                    file.method_calls,
+                    [
+                        mock.call.seek(0),
+                        # The call to read does not happen with Stubber
+                        mock.call.close(),
+                    ],
+                )
 
     def test_init_without_acl(self):
         storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key")
@@ -392,8 +402,7 @@ class S3FeedStorageTest(unittest.TestCase):
         self.assertEqual(storage.endpoint_url, "https://example.com")
 
     @defer.inlineCallbacks
-    def test_store_botocore_without_acl(self):
-        skip_if_no_boto()
+    def test_store_without_acl(self):
         storage = S3FeedStorage(
             "s3://mybucket/export.csv",
             "access_key",
@@ -405,11 +414,18 @@ class S3FeedStorageTest(unittest.TestCase):
 
         storage.s3_client = mock.MagicMock()
         yield storage.store(BytesIO(b"test file"))
-        self.assertNotIn("ACL", storage.s3_client.put_object.call_args[1])
+        if IS_BOTO3_AVAILABLE:
+            acl = (
+                storage.s3_client.upload_fileobj.call_args[1]
+                .get("ExtraArgs", {})
+                .get("ACL")
+            )
+        else:
+            acl = storage.s3_client.put_object.call_args[1].get("ACL")
+        self.assertIsNone(acl)
 
     @defer.inlineCallbacks
-    def test_store_botocore_with_acl(self):
-        skip_if_no_boto()
+    def test_store_with_acl(self):
         storage = S3FeedStorage(
             "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl"
         )
@@ -419,9 +435,11 @@ class S3FeedStorageTest(unittest.TestCase):
 
         storage.s3_client = mock.MagicMock()
         yield storage.store(BytesIO(b"test file"))
-        self.assertEqual(
-            storage.s3_client.put_object.call_args[1].get("ACL"), "custom-acl"
-        )
+        if IS_BOTO3_AVAILABLE:
+            acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"]
+        else:
+            acl = storage.s3_client.put_object.call_args[1]["ACL"]
+        self.assertEqual(acl, "custom-acl")
 
     def test_overwrite_default(self):
         with LogCapture() as log:
@@ -889,15 +907,10 @@ class FeedExportTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def test_stats_multiple_file(self):
         settings = {
-            "AWS_ACCESS_KEY_ID": "access_key",
-            "AWS_SECRET_ACCESS_KEY": "secret_key",
             "FEEDS": {
                 printf_escape(path_to_url(str(self._random_temp_filename()))): {
                     "format": "json",
                 },
-                "s3://bucket/key/foo.csv": {
-                    "format": "csv",
-                },
                 "stdout:": {
                     "format": "xml",
                 },
@@ -909,18 +922,12 @@ class FeedExportTest(FeedExportTestBase):
         self.assertIn(
             "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats()
         )
-        self.assertIn(
-            "feedexport/success_count/S3FeedStorage", crawler.stats.get_stats()
-        )
         self.assertIn(
             "feedexport/success_count/StdoutFeedStorage", crawler.stats.get_stats()
         )
         self.assertEqual(
             crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1
         )
-        self.assertEqual(
-            crawler.stats.get_value("feedexport/success_count/S3FeedStorage"), 1
-        )
         self.assertEqual(
             crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage"), 1
         )
@@ -2587,7 +2594,6 @@ class BatchDeliveriesTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def test_s3_export(self):
         skip_if_no_boto()
-
         bucket = "mybucket"
         items = [
             self.MyItem({"foo": "bar1", "egg": "spam1"}),
@@ -2836,6 +2842,9 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase):
 
     maxDiff = None
 
+    def setUp(self):
+        skip_if_no_boto()
+
     def test_init(self):
         settings_dict = {
             "FEED_URI": "file:///tmp/foobar",
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index c80666586..8f9f165b6 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -225,12 +225,16 @@ class FilesPipelineTestCase(unittest.TestCase):
 
 
 class FilesPipelineTestCaseFieldsMixin:
+    def setUp(self):
+        self.tempdir = mkdtemp()
+
+    def tearDown(self):
+        rmtree(self.tempdir)
+
     def test_item_fields_default(self):
         url = "http://www.example.com/files/1.txt"
         item = self.item_class(name="item1", file_urls=[url])
-        pipeline = FilesPipeline.from_settings(
-            Settings({"FILES_STORE": "s3://example/files/"})
-        )
+        pipeline = FilesPipeline.from_settings(Settings({"FILES_STORE": self.tempdir}))
         requests = list(pipeline.get_media_requests(item, None))
         self.assertEqual(requests[0].url, url)
         results = [(True, {"url": url})]
@@ -245,7 +249,7 @@ class FilesPipelineTestCaseFieldsMixin:
         pipeline = FilesPipeline.from_settings(
             Settings(
                 {
-                    "FILES_STORE": "s3://example/files/",
+                    "FILES_STORE": self.tempdir,
                     "FILES_URLS_FIELD": "custom_file_urls",
                     "FILES_RESULT_FIELD": "custom_files",
                 }
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index c05f4da91..dc0a82086 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -21,8 +21,6 @@ class MitmProxy:
     auth_pass = "scrapy"
 
     def start(self):
-        from scrapy.utils.test import get_testenv
-
         script = """
 import sys
 from mitmproxy.tools.main import mitmdump
@@ -46,7 +44,6 @@ sys.exit(mitmdump())
                 "--ssl-insecure",
             ],
             stdout=PIPE,
-            env=get_testenv(),
         )
         line = self.proc.stdout.readline().decode("utf-8")
         host_port = re.search(r"listening at http://([^:]+:\d+)", line).group(1)
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index 01d0ee043..65e352053 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -4,6 +4,7 @@ from unittest import TestCase
 
 from pytest import mark
 
+from scrapy.utils.defer import deferred_f_from_coro_f
 from scrapy.utils.reactor import (
     install_reactor,
     is_asyncio_reactor_installed,
@@ -29,6 +30,8 @@ class AsyncioTest(TestCase):
 
         assert original_reactor == reactor
 
+    @mark.only_asyncio()
+    @deferred_f_from_coro_f
     async def test_set_asyncio_event_loop(self):
         install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
-        assert set_asyncio_event_loop() is asyncio.get_running_loop()
+        assert set_asyncio_event_loop(None) is asyncio.get_running_loop()