Repository: scrapy/scrapy
Pull request #6112 (2023-10-17 20:44:28)

Make shutdown tests more robust.


Patch:
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 1abbc4968..a8e5b23bf 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1039,8 +1039,8 @@ RobotsTxtMiddleware
 
     * :ref:`Protego <protego-parser>` (default)
     * :ref:`RobotFileParser <python-robotfileparser>`
+    * :ref:`Reppy <reppy-parser>`
     * :ref:`Robotexclusionrulesparser <rerp-parser>`
-    * :ref:`Reppy <reppy-parser>` (deprecated)
 
     You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`
     setting. Or you can also :ref:`implement support for a new parser <support-for-new-robots-parser>`.
@@ -1133,7 +1133,6 @@ In order to use this parser:
 
     .. warning:: `Upstream issue #122
         <https://github.com/seomoz/reppy/issues/122>`_ prevents reppy usage in Python 3.9+.
-        Because of this the Reppy parser is deprecated.
 
 * Set :setting:`ROBOTSTXT_PARSER` setting to
   ``scrapy.robotstxt.ReppyRobotParser``
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 1d3a11208..6f54e62e9 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -12,6 +12,7 @@ from twisted.internet.defer import (
     inlineCallbacks,
     maybeDeferred,
 )
+from zope.interface.exceptions import DoesNotImplement
 
 try:
     # zope >= 5.0 only supports MultipleInvalid
@@ -204,7 +205,19 @@ class CrawlerRunner:
         """Get SpiderLoader instance from settings"""
         cls_path = settings.get("SPIDER_LOADER_CLASS")
         loader_cls = load_object(cls_path)
-        verifyClass(ISpiderLoader, loader_cls)
+        excs = (
+            (DoesNotImplement, MultipleInvalid) if MultipleInvalid else DoesNotImplement
+        )
+        try:
+            verifyClass(ISpiderLoader, loader_cls)
+        except excs:
+            warnings.warn(
+                "SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does "
+                "not fully implement scrapy.interfaces.ISpiderLoader interface. "
+                "Please add all missing methods to avoid unexpected runtime errors.",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
         return loader_cls.from_settings(settings.frozencopy())
 
     def __init__(self, settings: Union[Dict[str, Any], Settings, None] = None):
diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py
new file mode 100644
index 000000000..3b8702419
--- /dev/null
+++ b/scrapy/downloadermiddlewares/decompression.py
@@ -0,0 +1,94 @@
+""" This module implements the DecompressionMiddleware which tries to recognise
+and extract the potentially compressed responses that may arrive.
+"""
+
+import bz2
+import gzip
+import logging
+import tarfile
+import zipfile
+from io import BytesIO
+from tempfile import mktemp
+from warnings import warn
+
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.responsetypes import responsetypes
+
+warn(
+    "scrapy.downloadermiddlewares.decompression is deprecated",
+    ScrapyDeprecationWarning,
+    stacklevel=2,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+class DecompressionMiddleware:
+    """This middleware tries to recognise and extract the possibly compressed
+    responses that may arrive."""
+
+    def __init__(self):
+        self._formats = {
+            "tar": self._is_tar,
+            "zip": self._is_zip,
+            "gz": self._is_gzip,
+            "bz2": self._is_bzip2,
+        }
+
+    def _is_tar(self, response):
+        archive = BytesIO(response.body)
+        try:
+            tar_file = tarfile.open(name=mktemp(), fileobj=archive)
+        except tarfile.ReadError:
+            return
+
+        body = tar_file.extractfile(tar_file.members[0]).read()
+        respcls = responsetypes.from_args(filename=tar_file.members[0].name, body=body)
+        return response.replace(body=body, cls=respcls)
+
+    def _is_zip(self, response):
+        archive = BytesIO(response.body)
+        try:
+            zip_file = zipfile.ZipFile(archive)
+        except zipfile.BadZipFile:
+            return
+
+        namelist = zip_file.namelist()
+        body = zip_file.read(namelist[0])
+        respcls = responsetypes.from_args(filename=namelist[0], body=body)
+        return response.replace(body=body, cls=respcls)
+
+    def _is_gzip(self, response):
+        archive = BytesIO(response.body)
+        try:
+            body = gzip.GzipFile(fileobj=archive).read()
+        except OSError:
+            return
+
+        respcls = responsetypes.from_args(body=body)
+        return response.replace(body=body, cls=respcls)
+
+    def _is_bzip2(self, response):
+        try:
+            body = bz2.decompress(response.body)
+        except OSError:
+            return
+
+        respcls = responsetypes.from_args(body=body)
+        return response.replace(body=body, cls=respcls)
+
+    def process_response(self, request, response, spider):
+        if not response.body:
+            return response
+
+        for fmt, func in self._formats.items():
+            new_response = func(response)
+            if new_response:
+                logger.debug(
+                    "Decompressed response with format: %(responsefmt)s",
+                    {"responsefmt": fmt},
+                    extra={"spider": spider},
+                )
+                return new_response
+        return response
diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py
index 5228db786..de5a81388 100644
--- a/scrapy/downloadermiddlewares/httpauth.py
+++ b/scrapy/downloadermiddlewares/httpauth.py
@@ -3,10 +3,13 @@ HTTP basic auth downloader middleware
 
 See documentation in docs/topics/downloader-middleware.rst
 """
+import warnings
 
 from w3lib.http import basic_auth_header
 
 from scrapy import signals
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.url import url_is_from_any_domain
 
 
@@ -25,10 +28,25 @@ class HttpAuthMiddleware:
         pwd = getattr(spider, "http_pass", "")
         if usr or pwd:
             self.auth = basic_auth_header(usr, pwd)
-            self.domain = spider.http_auth_domain
+            if not hasattr(spider, "http_auth_domain"):
+                warnings.warn(
+                    "Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security "
+                    "problems if the spider makes requests to several different domains. http_auth_domain "
+                    "will be set to the domain of the first request, please set it to the correct value "
+                    "explicitly.",
+                    category=ScrapyDeprecationWarning,
+                )
+                self.domain_unset = True
+            else:
+                self.domain = spider.http_auth_domain
+                self.domain_unset = False
 
     def process_request(self, request, spider):
         auth = getattr(self, "auth", None)
         if auth and b"Authorization" not in request.headers:
+            domain = urlparse_cached(request).hostname
+            if self.domain_unset:
+                self.domain = domain
+                self.domain_unset = False
             if not self.domain or url_is_from_any_domain(request.url, [self.domain]):
                 request.headers[b"Authorization"] = auth
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 7b1d3f829..ead426951 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -1,9 +1,11 @@
 import io
+import warnings
 import zlib
 
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Response, TextResponse
 from scrapy.responsetypes import responsetypes
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.gz import gunzip
 
 ACCEPTED_ENCODINGS = [b"gzip", b"deflate"]
@@ -34,7 +36,18 @@ class HttpCompressionMiddleware:
     def from_crawler(cls, crawler):
         if not crawler.settings.getbool("COMPRESSION_ENABLED"):
             raise NotConfigured
-        return cls(stats=crawler.stats)
+        try:
+            return cls(stats=crawler.stats)
+        except TypeError:
+            warnings.warn(
+                "HttpCompressionMiddleware subclasses must either modify "
+                "their '__init__' method to support a 'stats' parameter or "
+                "reimplement the 'from_crawler' method.",
+                ScrapyDeprecationWarning,
+            )
+            result = cls()
+            result.stats = crawler.stats
+            return result
 
     def process_request(self, request, spider):
         request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index dd2420e98..0b20f53b9 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -3,12 +3,14 @@ from __future__ import annotations
 import logging
 from pathlib import Path
 from typing import TYPE_CHECKING, Optional, Set
+from warnings import warn
 
 from twisted.internet.defer import Deferred
 
 from scrapy.http.request import Request
 from scrapy.settings import BaseSettings
 from scrapy.spiders import Spider
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.job import job_dir
 from scrapy.utils.request import (
     RequestFingerprinter,
@@ -73,15 +75,38 @@ class RFPDupeFilter(BaseDupeFilter):
         fingerprinter: Optional[RequestFingerprinterProtocol] = None,
     ) -> Self:
         debug = settings.getbool("DUPEFILTER_DEBUG")
-        return cls(job_dir(settings), debug, fingerprinter=fingerprinter)
+        try:
+            return cls(job_dir(settings), debug, fingerprinter=fingerprinter)
+        except TypeError:
+            warn(
+                "RFPDupeFilter subclasses must either modify their '__init__' "
+                "method to support a 'fingerprinter' parameter or reimplement "
+                "the 'from_settings' class method.",
+                ScrapyDeprecationWarning,
+            )
+            result = cls(job_dir(settings), debug)
+            result.fingerprinter = fingerprinter or RequestFingerprinter()
+            return result
 
     @classmethod
     def from_crawler(cls, crawler: Crawler) -> Self:
         assert crawler.request_fingerprinter
-        return cls.from_settings(
-            crawler.settings,
-            fingerprinter=crawler.request_fingerprinter,
-        )
+        try:
+            return cls.from_settings(
+                crawler.settings,
+                fingerprinter=crawler.request_fingerprinter,
+            )
+        except TypeError:
+            warn(
+                "RFPDupeFilter subclasses must either modify their overridden "
+                "'__init__' method and 'from_settings' class method to "
+                "support a 'fingerprinter' parameter, or reimplement the "
+                "'from_crawler' class method.",
+                ScrapyDeprecationWarning,
+            )
+            result = cls.from_settings(crawler.settings)
+            result.fingerprinter = crawler.request_fingerprinter
+            return result
 
     def request_seen(self, request: Request) -> bool:
         fp = self.request_fingerprint(request)
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index fadbbb582..4e846d1bd 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -29,7 +29,7 @@ from scrapy.utils.deprecate import create_deprecated_class
 from scrapy.utils.ftp import ftp_store_file
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import create_instance, load_object
-from scrapy.utils.python import without_none_values
+from scrapy.utils.python import get_func_args, without_none_values
 
 logger = logging.getLogger(__name__)
 
@@ -42,7 +42,17 @@ except ImportError:
 
 
 def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
-    kwargs["feed_options"] = feed_options
+    argument_names = get_func_args(builder)
+    if "feed_options" in argument_names:
+        kwargs["feed_options"] = feed_options
+    else:
+        warnings.warn(
+            f"{builder.__qualname__} does not support the 'feed_options' keyword argument. Add a "
+            "'feed_options' parameter to its signature to remove this "
+            "warning. This parameter will become mandatory in a future "
+            "version of Scrapy.",
+            category=ScrapyDeprecationWarning,
+        )
     return builder(*preargs, uri, *args, **kwargs)
 
 
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 06a52f5fd..0b443c7d0 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -200,7 +200,7 @@ class Request(object_ref):
         ``ignore_unknown_options=False``.
 
         .. caution:: Using :meth:`from_curl` from :class:`~scrapy.http.Request`
-                     subclasses, such as :class:`~scrapy.http.JsonRequest`, or
+                     subclasses, such as :class:`~scrapy.http.JSONRequest`, or
                      :class:`~scrapy.http.XmlRpcRequest`, as well as having
                      :ref:`downloader middlewares <topics-downloader-middleware>`
                      and
diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index f52c0401d..510c903db 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -11,6 +11,7 @@ import warnings
 from typing import Optional, Tuple
 
 from scrapy.http.request import Request
+from scrapy.utils.deprecate import create_deprecated_class
 
 
 class JsonRequest(Request):
@@ -57,3 +58,6 @@ class JsonRequest(Request):
     def _dumps(self, data: dict) -> str:
         """Convert to JSON"""
         return json.dumps(data, **self._dumps_kwargs)
+
+
+JSONRequest = create_deprecated_class("JSONRequest", JsonRequest)
diff --git a/scrapy/loader/common.py b/scrapy/loader/common.py
new file mode 100644
index 000000000..3e8644e0c
--- /dev/null
+++ b/scrapy/loader/common.py
@@ -0,0 +1,21 @@
+"""Common functions used in Item Loaders code"""
+
+import warnings
+
+from itemloaders import common
+
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
+
+
+def wrap_loader_context(function, context):
+    """Wrap functions that receive loader_context to contain the context
+    "pre-loaded" and expose a interface that receives only one argument
+    """
+    warnings.warn(
+        "scrapy.loader.common.wrap_loader_context has moved to a new library."
+        "Please update your reference to itemloaders.common.wrap_loader_context",
+        ScrapyDeprecationWarning,
+        stacklevel=2,
+    )
+
+    return common.wrap_loader_context(function, context)
diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py
new file mode 100644
index 000000000..b82c6d5c7
--- /dev/null
+++ b/scrapy/loader/processors.py
@@ -0,0 +1,20 @@
+"""
+This module provides some commonly used processors for Item Loaders.
+
+See documentation in docs/topics/loaders.rst
+"""
+from itemloaders import processors
+
+from scrapy.utils.deprecate import create_deprecated_class
+
+MapCompose = create_deprecated_class("MapCompose", processors.MapCompose)
+
+Compose = create_deprecated_class("Compose", processors.Compose)
+
+TakeFirst = create_deprecated_class("TakeFirst", processors.TakeFirst)
+
+Identity = create_deprecated_class("Identity", processors.Identity)
+
+SelectJmes = create_deprecated_class("SelectJmes", processors.SelectJmes)
+
+Join = create_deprecated_class("Join", processors.Join)
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 75532034a..401029439 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -1,6 +1,8 @@
 import functools
 import logging
 from collections import defaultdict
+from inspect import signature
+from warnings import warn
 
 from twisted.internet.defer import Deferred, DeferredList
 from twisted.python.failure import Failure
@@ -9,6 +11,7 @@ from scrapy.http.request import NO_CALLBACK
 from scrapy.settings import Settings
 from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.defer import defer_result, mustbe_deferred
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import arg_to_iter
 
@@ -41,6 +44,9 @@ class MediaPipeline:
         self.allow_redirects = settings.getbool(resolve("MEDIA_ALLOW_REDIRECTS"), False)
         self._handle_statuses(self.allow_redirects)
 
+        # Check if deprecated methods are being used and make them compatible
+        self._make_compatible()
+
     def _handle_statuses(self, allow_redirects):
         self.handle_httpstatus_list = None
         if allow_redirects:
@@ -120,6 +126,52 @@ class MediaPipeline:
         )
         return dfd.addBoth(lambda _: wad)  # it must return wad at last
 
+    def _make_compatible(self):
+        """Make overridable methods of MediaPipeline and subclasses backwards compatible"""
+        methods = [
+            "file_path",
+            "thumb_path",
+            "media_to_download",
+            "media_downloaded",
+            "file_downloaded",
+            "image_downloaded",
+            "get_images",
+        ]
+
+        for method_name in methods:
+            method = getattr(self, method_name, None)
+            if callable(method):
+                setattr(self, method_name, self._compatible(method))
+
+    def _compatible(self, func):
+        """Wrapper for overridable methods to allow backwards compatibility"""
+        self._check_signature(func)
+
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            if self._expects_item[func.__name__]:
+                return func(*args, **kwargs)
+
+            kwargs.pop("item", None)
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    def _check_signature(self, func):
+        sig = signature(func)
+        self._expects_item[func.__name__] = True
+
+        if "item" not in sig.parameters:
+            old_params = str(sig)[1:-1]
+            new_params = old_params + ", *, item=None"
+            warn(
+                f"{func.__name__}(self, {old_params}) is deprecated, "
+                f"please use {func.__name__}(self, {new_params})",
+                ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+            self._expects_item[func.__name__] = False
+
     def _modify_media_request(self, request):
         if self.handle_httpstatus_list:
             request.meta["handle_httpstatus_list"] = self.handle_httpstatus_list
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index 5c5ac4e41..604b5e314 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -1,9 +1,7 @@
 import logging
 import sys
 from abc import ABCMeta, abstractmethod
-from warnings import warn
 
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.python import to_unicode
 
 logger = logging.getLogger(__name__)
@@ -81,7 +79,6 @@ class PythonRobotParser(RobotParser):
 
 class ReppyRobotParser(RobotParser):
     def __init__(self, robotstxt_body, spider):
-        warn("ReppyRobotParser is deprecated.", ScrapyDeprecationWarning, stacklevel=2)
         from reppy.robots import Robots
 
         self.spider = spider
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index a9364bea2..b3c28da92 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -21,12 +21,17 @@ from typing import (
     Iterable,
     List,
     Optional,
+    Pattern,
     Union,
     cast,
 )
 
+from w3lib.html import replace_entities
+
 from scrapy.item import Item
 from scrapy.utils.datatypes import LocalWeakReferencedCache
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
+from scrapy.utils.python import flatten, to_unicode
 
 if TYPE_CHECKING:
     from scrapy import Spider
@@ -103,6 +108,39 @@ def walk_modules(path: str) -> List[ModuleType]:
     return mods
 
 
+def extract_regex(
+    regex: Union[str, Pattern], text: str, encoding: str = "utf-8"
+) -> List[str]:
+    """Extract a list of unicode strings from the given text/encoding using the following policies:
+
+    * if the regex contains a named group called "extract" that will be returned
+    * if the regex contains multiple numbered groups, all those will be returned (flattened)
+    * if the regex doesn't contain any group the entire regex matching is returned
+    """
+    warnings.warn(
+        "scrapy.utils.misc.extract_regex has moved to parsel.utils.extract_regex.",
+        ScrapyDeprecationWarning,
+        stacklevel=2,
+    )
+
+    if isinstance(regex, str):
+        regex = re.compile(regex, re.UNICODE)
+
+    try:
+        # named group
+        strings = [regex.search(text).group("extract")]  # type: ignore[union-attr]
+    except Exception:
+        # full regex or numbered groups
+        strings = regex.findall(text)
+    strings = flatten(strings)
+
+    if isinstance(text, str):
+        return [replace_entities(s, keep=["lt", "amp"]) for s in strings]
+    return [
+        replace_entities(to_unicode(s, encoding), keep=["lt", "amp"]) for s in strings
+    ]
+
+
 def md5sum(file: IO) -> str:
     """Calculate the md5 checksum of a file-like object without reading its
     whole content in memory.


Test patch:
diff --git a/tests/CrawlerProcess/sleeping.py b/tests/CrawlerProcess/sleeping.py
index 420d9d328..45479ea4f 100644
--- a/tests/CrawlerProcess/sleeping.py
+++ b/tests/CrawlerProcess/sleeping.py
@@ -14,7 +14,7 @@ class SleepingSpider(scrapy.Spider):
         from twisted.internet import reactor
 
         d = Deferred()
-        reactor.callLater(3, d.callback, None)
+        reactor.callLater(int(self.sleep), d.callback, None)
         await maybe_deferred_to_future(d)
 
 
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 7d87eb62c..7918d94b2 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -1,3 +1,4 @@
+import os
 import sys
 from io import BytesIO
 from pathlib import Path
@@ -147,8 +148,10 @@ class InteractiveShellTest(unittest.TestCase):
             "scrapy.cmdline",
             "shell",
         )
+        env = os.environ.copy()
+        env["SCRAPY_PYTHON_SHELL"] = "python"
         logfile = BytesIO()
-        p = PopenSpawn(args, timeout=5)
+        p = PopenSpawn(args, env=env, timeout=5)
         p.logfile_read = logfile
         p.expect_exact("Available Scrapy objects")
         with MockServer() as mockserver:
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index a71b9c2b7..0a7f9bac8 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -15,7 +15,6 @@ from pytest import mark, raises
 from twisted.internet import defer
 from twisted.trial import unittest
 from w3lib import __version__ as w3lib_version
-from zope.interface.exceptions import MultipleInvalid
 
 import scrapy
 from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner
@@ -180,7 +179,11 @@ class CrawlerRunnerTestCase(BaseCrawlerTest):
                 "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
             }
         )
-        self.assertRaises(MultipleInvalid, CrawlerRunner, settings)
+        with warnings.catch_warnings(record=True) as w:
+            self.assertRaises(AttributeError, CrawlerRunner, settings)
+            self.assertEqual(len(w), 1)
+            self.assertIn("SPIDER_LOADER_CLASS", str(w[0].message))
+            self.assertIn("scrapy.interfaces.ISpiderLoader", str(w[0].message))
 
     def test_crawler_runner_accepts_dict(self):
         runner = CrawlerRunner({"foo": "bar"})
@@ -522,7 +525,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
 
     def test_shutdown_graceful(self):
         sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK
-        args = self.get_script_args("sleeping.py")
+        args = self.get_script_args("sleeping.py", "-a", "sleep=3")
         p = PopenSpawn(args, timeout=5)
         p.expect_exact("Spider opened")
         p.expect_exact("Crawled (200)")
@@ -531,14 +534,21 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         p.expect_exact("Spider closed (shutdown)")
         p.wait()
 
+    @defer.inlineCallbacks
     def test_shutdown_forced(self):
+        from twisted.internet import reactor
+
         sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK
-        args = self.get_script_args("sleeping.py")
+        args = self.get_script_args("sleeping.py", "-a", "sleep=10")
         p = PopenSpawn(args, timeout=5)
         p.expect_exact("Spider opened")
         p.expect_exact("Crawled (200)")
         p.kill(sig)
         p.expect_exact("shutting down gracefully")
+        # sending the second signal too fast often causes problems
+        d = defer.Deferred()
+        reactor.callLater(0.1, d.callback, None)
+        yield d
         p.kill(sig)
         p.expect_exact("forcing unclean shutdown")
         p.wait()
diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py
new file mode 100644
index 000000000..95739414e
--- /dev/null
+++ b/tests/test_downloadermiddleware_decompression.py
@@ -0,0 +1,53 @@
+from unittest import TestCase, main
+
+from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware
+from scrapy.http import Response, XmlResponse
+from scrapy.spiders import Spider
+from scrapy.utils.test import assert_samelines
+from tests import get_testdata
+
+
+def _test_data(formats):
+    uncompressed_body = get_testdata("compressed", "feed-sample1.xml")
+    test_responses = {}
+    for format in formats:
+        body = get_testdata("compressed", "feed-sample1." + format)
+        test_responses[format] = Response("http://foo.com/bar", body=body)
+    return uncompressed_body, test_responses
+
+
+class DecompressionMiddlewareTest(TestCase):
+    test_formats = ["tar", "xml.bz2", "xml.gz", "zip"]
+    uncompressed_body, test_responses = _test_data(test_formats)
+
+    def setUp(self):
+        self.mw = DecompressionMiddleware()
+        self.spider = Spider("foo")
+
+    def test_known_compression_formats(self):
+        for fmt in self.test_formats:
+            rsp = self.test_responses[fmt]
+            new = self.mw.process_response(None, rsp, self.spider)
+            error_msg = f"Failed {fmt}, response type {type(new).__name__}"
+            assert isinstance(new, XmlResponse), error_msg
+            assert_samelines(self, new.body, self.uncompressed_body, fmt)
+
+    def test_plain_response(self):
+        rsp = Response(url="http://test.com", body=self.uncompressed_body)
+        new = self.mw.process_response(None, rsp, self.spider)
+        assert new is rsp
+        assert_samelines(self, new.body, rsp.body)
+
+    def test_empty_response(self):
+        rsp = Response(url="http://test.com", body=b"")
+        new = self.mw.process_response(None, rsp, self.spider)
+        assert new is rsp
+        assert not rsp.body
+        assert not new.body
+
+    def tearDown(self):
+        del self.mw
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py
index fc110e6cc..6b79234d0 100644
--- a/tests/test_downloadermiddleware_httpauth.py
+++ b/tests/test_downloadermiddleware_httpauth.py
@@ -1,8 +1,10 @@
 import unittest
 
+import pytest
 from w3lib.http import basic_auth_header
 
 from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request
 from scrapy.spiders import Spider
 
@@ -29,10 +31,39 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase):
         self.spider = TestSpiderLegacy("foo")
 
     def test_auth(self):
-        with self.assertRaises(AttributeError):
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="Using HttpAuthMiddleware without http_auth_domain is deprecated",
+        ):
             mw = HttpAuthMiddleware()
             mw.spider_opened(self.spider)
 
+        # initial request, sets the domain and sends the header
+        req = Request("http://example.com/")
+        assert mw.process_request(req, self.spider) is None
+        self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar"))
+
+        # subsequent request to the same domain, should send the header
+        req = Request("http://example.com/")
+        assert mw.process_request(req, self.spider) is None
+        self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar"))
+
+        # subsequent request to a different domain, shouldn't send the header
+        req = Request("http://example-noauth.com/")
+        assert mw.process_request(req, self.spider) is None
+        self.assertNotIn("Authorization", req.headers)
+
+    def test_auth_already_set(self):
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="Using HttpAuthMiddleware without http_auth_domain is deprecated",
+        ):
+            mw = HttpAuthMiddleware()
+            mw.spider_opened(self.spider)
+        req = Request("http://example.com/", headers=dict(Authorization="Digest 123"))
+        assert mw.process_request(req, self.spider) is None
+        self.assertEqual(req.headers["Authorization"], b"Digest 123")
+
 
 class HttpAuthMiddlewareTest(unittest.TestCase):
     def setUp(self):
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index a96b710f3..9dad056de 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -2,6 +2,7 @@ from gzip import GzipFile
 from io import BytesIO
 from pathlib import Path
 from unittest import SkipTest, TestCase
+from warnings import catch_warnings
 
 from w3lib.encoding import resolve_encoding
 
@@ -9,7 +10,7 @@ from scrapy.downloadermiddlewares.httpcompression import (
     ACCEPTED_ENCODINGS,
     HttpCompressionMiddleware,
 )
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import HtmlResponse, Request, Response
 from scrapy.responsetypes import responsetypes
 from scrapy.spiders import Spider
@@ -371,3 +372,29 @@ class HttpCompressionTest(TestCase):
         self.assertEqual(response.body, b"")
         self.assertStatsEqual("httpcompression/response_count", None)
         self.assertStatsEqual("httpcompression/response_bytes", None)
+
+
+class HttpCompressionSubclassTest(TestCase):
+    def test_init_missing_stats(self):
+        class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware):
+            def __init__(self):
+                super().__init__()
+
+        crawler = get_crawler(Spider)
+        with catch_warnings(record=True) as caught_warnings:
+            HttpCompressionMiddlewareSubclass.from_crawler(crawler)
+        messages = tuple(
+            str(warning.message)
+            for warning in caught_warnings
+            if warning.category is ScrapyDeprecationWarning
+        )
+        self.assertEqual(
+            messages,
+            (
+                (
+                    "HttpCompressionMiddleware subclasses must either modify "
+                    "their '__init__' method to support a 'stats' parameter "
+                    "or reimplement the 'from_crawler' method."
+                ),
+            ),
+        )
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 89169fd7c..56967c0d5 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -2835,6 +2835,200 @@ class FeedExportInitTest(unittest.TestCase):
         self.assertIsInstance(exporter, FeedExporter)
 
 
+class StdoutFeedStorageWithoutFeedOptions(StdoutFeedStorage):
+    def __init__(self, uri):
+        super().__init__(uri)
+
+
+class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase):
+    """Make sure that any feed exporter created by users before the
+    introduction of the ``feed_options`` parameter continues to work as
+    expected, and simply issues a warning."""
+
+    def test_init(self):
+        settings_dict = {
+            "FEED_URI": "file:///tmp/foobar",
+            "FEED_STORAGES": {"file": StdoutFeedStorageWithoutFeedOptions},
+        }
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
+        ):
+            crawler = get_crawler(settings_dict=settings_dict)
+            feed_exporter = FeedExporter.from_crawler(crawler)
+
+        spider = scrapy.Spider("default")
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="StdoutFeedStorageWithoutFeedOptions does not support "
+            "the 'feed_options' keyword argument.",
+        ):
+            feed_exporter.open_spider(spider)
+
+
+class FileFeedStorageWithoutFeedOptions(FileFeedStorage):
+    def __init__(self, uri):
+        super().__init__(uri)
+
+
+class FileFeedStoragePreFeedOptionsTest(unittest.TestCase):
+    """Make sure that any feed exporter created by users before the
+    introduction of the ``feed_options`` parameter continues to work as
+    expected, and simply issues a warning."""
+
+    maxDiff = None
+
+    def test_init(self):
+        with tempfile.NamedTemporaryFile() as temp:
+            settings_dict = {
+                "FEED_URI": f"file:///{temp.name}",
+                "FEED_STORAGES": {"file": FileFeedStorageWithoutFeedOptions},
+            }
+            with pytest.warns(
+                ScrapyDeprecationWarning,
+                match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
+            ):
+                crawler = get_crawler(settings_dict=settings_dict)
+                feed_exporter = FeedExporter.from_crawler(crawler)
+        spider = scrapy.Spider("default")
+
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="FileFeedStorageWithoutFeedOptions does not support "
+            "the 'feed_options' keyword argument.",
+        ):
+            feed_exporter.open_spider(spider)
+
+
+class S3FeedStorageWithoutFeedOptions(S3FeedStorage):
+    def __init__(self, uri, access_key, secret_key, acl, endpoint_url, **kwargs):
+        super().__init__(uri, access_key, secret_key, acl, endpoint_url, **kwargs)
+
+
+class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage):
+    @classmethod
+    def from_crawler(cls, crawler, uri):
+        return super().from_crawler(crawler, uri)
+
+
+class S3FeedStoragePreFeedOptionsTest(unittest.TestCase):
+    """Make sure that any feed exporter created by users before the
+    introduction of the ``feed_options`` parameter continues to work as
+    expected, and simply issues a warning."""
+
+    maxDiff = None
+
+    def setUp(self):
+        skip_if_no_boto()
+
+    def test_init(self):
+        settings_dict = {
+            "FEED_URI": "file:///tmp/foobar",
+            "FEED_STORAGES": {"file": S3FeedStorageWithoutFeedOptions},
+        }
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
+        ):
+            crawler = get_crawler(settings_dict=settings_dict)
+            feed_exporter = FeedExporter.from_crawler(crawler)
+
+        spider = scrapy.Spider("default")
+        spider.crawler = crawler
+
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="S3FeedStorageWithoutFeedOptions does not support "
+            "the 'feed_options' keyword argument.",
+        ):
+            feed_exporter.open_spider(spider)
+
+    def test_from_crawler(self):
+        settings_dict = {
+            "FEED_URI": "file:///tmp/foobar",
+            "FEED_STORAGES": {"file": S3FeedStorageWithoutFeedOptionsWithFromCrawler},
+        }
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
+        ):
+            crawler = get_crawler(settings_dict=settings_dict)
+            feed_exporter = FeedExporter.from_crawler(crawler)
+
+        spider = scrapy.Spider("default")
+        spider.crawler = crawler
+
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support "
+            "the 'feed_options' keyword argument.",
+        ):
+            feed_exporter.open_spider(spider)
+
+
+class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage):
+    def __init__(self, uri, use_active_mode=False):
+        super().__init__(uri)
+
+
+class FTPFeedStorageWithoutFeedOptionsWithFromCrawler(FTPFeedStorage):
+    @classmethod
+    def from_crawler(cls, crawler, uri):
+        return super().from_crawler(crawler, uri)
+
+
+class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
+    """Make sure that any feed exporter created by users before the
+    introduction of the ``feed_options`` parameter continues to work as
+    expected, and simply issues a warning."""
+
+    maxDiff = None
+
+    def test_init(self):
+        settings_dict = {
+            "FEED_URI": "ftp://localhost/foo",
+            "FEED_STORAGES": {"ftp": FTPFeedStorageWithoutFeedOptions},
+        }
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
+        ):
+            crawler = get_crawler(settings_dict=settings_dict)
+            feed_exporter = FeedExporter.from_crawler(crawler)
+
+        spider = scrapy.Spider("default")
+        spider.crawler = crawler
+
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="FTPFeedStorageWithoutFeedOptions does not support "
+            "the 'feed_options' keyword argument.",
+        ):
+            feed_exporter.open_spider(spider)
+
+    def test_from_crawler(self):
+        settings_dict = {
+            "FEED_URI": "ftp://localhost/foo",
+            "FEED_STORAGES": {"ftp": FTPFeedStorageWithoutFeedOptionsWithFromCrawler},
+        }
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
+        ):
+            crawler = get_crawler(settings_dict=settings_dict)
+            feed_exporter = FeedExporter.from_crawler(crawler)
+
+        spider = scrapy.Spider("default")
+        spider.crawler = crawler
+
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support "
+            "the 'feed_options' keyword argument.",
+        ):
+            feed_exporter.open_spider(spider)
+
+
 class URIParamsTest:
     spider_name = "uri_params_spider"
     deprecated_options = False
diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py
index d7f773d5c..3eae87599 100644
--- a/tests/test_loader_deprecated.py
+++ b/tests/test_loader_deprecated.py
@@ -4,6 +4,7 @@ Once we remove the references from scrapy, we can remove these tests.
 """
 
 import unittest
+import warnings
 from functools import partial
 
 from itemloaders.processors import (
@@ -17,6 +18,9 @@ from itemloaders.processors import (
 
 from scrapy.item import Field, Item
 from scrapy.loader import ItemLoader
+from scrapy.loader.common import wrap_loader_context
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
+from scrapy.utils.misc import extract_regex
 
 
 # test items
@@ -718,5 +722,24 @@ class FunctionProcessorTestCase(unittest.TestCase):
         self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]})
 
 
+class DeprecatedUtilityFunctionsTestCase(unittest.TestCase):
+    def test_deprecated_wrap_loader_context(self):
+        def function(*args):
+            return None
+
+        with warnings.catch_warnings(record=True) as w:
+            wrap_loader_context(function, context={})
+
+            assert len(w) == 1
+            assert issubclass(w[0].category, ScrapyDeprecationWarning)
+
+    def test_deprecated_extract_regex(self):
+        with warnings.catch_warnings(record=True) as w:
+            extract_regex(r"\w+", "this is a test")
+
+            assert len(w) == 1
+            assert issubclass(w[0].category, ScrapyDeprecationWarning)
+
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 820484565..d655eb128 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -15,6 +15,7 @@ from scrapy.pipelines.images import ImagesPipeline
 from scrapy.pipelines.media import MediaPipeline
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.signal import disconnect_all
 from scrapy.utils.test import get_crawler
@@ -426,6 +427,102 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
         return super().image_downloaded(response, request, info)
 
 
+class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):
+    skip = skip_pillow
+
+    def setUp(self):
+        settings_dict = {
+            "IMAGES_STORE": "store-uri",
+            "IMAGES_THUMBS": {"small": (50, 50)},
+        }
+        crawler = get_crawler(spidercls=None, settings_dict=settings_dict)
+        self.pipe = MockedMediaPipelineDeprecatedMethods.from_crawler(crawler)
+        self.pipe.download_func = _mocked_download_func
+        self.pipe.open_spider(None)
+        self.item = dict(image_urls=["http://picsum.photos/id/1014/200/300"], images=[])
+
+    def _assert_method_called_with_warnings(self, method, message, warnings):
+        self.assertIn(method, self.pipe._mockcalled)
+        warningShown = False
+        for warning in warnings:
+            if (
+                warning["message"] == message
+                and warning["category"] == ScrapyDeprecationWarning
+            ):
+                warningShown = True
+        self.assertTrue(warningShown)
+
+    @inlineCallbacks
+    def test_media_to_download_called(self):
+        yield self.pipe.process_item(self.item, None)
+        warnings = self.flushWarnings([MediaPipeline._compatible])
+        message = (
+            "media_to_download(self, request, info) is deprecated, "
+            "please use media_to_download(self, request, info, *, item=None)"
+        )
+        self._assert_method_called_with_warnings("media_to_download", message, warnings)
+
+    @inlineCallbacks
+    def test_media_downloaded_called(self):
+        yield self.pipe.process_item(self.item, None)
+        warnings = self.flushWarnings([MediaPipeline._compatible])
+        message = (
+            "media_downloaded(self, response, request, info) is deprecated, "
+            "please use media_downloaded(self, response, request, info, *, item=None)"
+        )
+        self._assert_method_called_with_warnings("media_downloaded", message, warnings)
+
+    @inlineCallbacks
+    def test_file_downloaded_called(self):
+        yield self.pipe.process_item(self.item, None)
+        warnings = self.flushWarnings([MediaPipeline._compatible])
+        message = (
+            "file_downloaded(self, response, request, info) is deprecated, "
+            "please use file_downloaded(self, response, request, info, *, item=None)"
+        )
+        self._assert_method_called_with_warnings("file_downloaded", message, warnings)
+
+    @inlineCallbacks
+    def test_file_path_called(self):
+        yield self.pipe.process_item(self.item, None)
+        warnings = self.flushWarnings([MediaPipeline._compatible])
+        message = (
+            "file_path(self, request, response=None, info=None) is deprecated, "
+            "please use file_path(self, request, response=None, info=None, *, item=None)"
+        )
+        self._assert_method_called_with_warnings("file_path", message, warnings)
+
+    @inlineCallbacks
+    def test_thumb_path_called(self):
+        yield self.pipe.process_item(self.item, None)
+        warnings = self.flushWarnings([MediaPipeline._compatible])
+        message = (
+            "thumb_path(self, request, thumb_id, response=None, info=None) is deprecated, "
+            "please use thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)"
+        )
+        self._assert_method_called_with_warnings("thumb_path", message, warnings)
+
+    @inlineCallbacks
+    def test_get_images_called(self):
+        yield self.pipe.process_item(self.item, None)
+        warnings = self.flushWarnings([MediaPipeline._compatible])
+        message = (
+            "get_images(self, response, request, info) is deprecated, "
+            "please use get_images(self, response, request, info, *, item=None)"
+        )
+        self._assert_method_called_with_warnings("get_images", message, warnings)
+
+    @inlineCallbacks
+    def test_image_downloaded_called(self):
+        yield self.pipe.process_item(self.item, None)
+        warnings = self.flushWarnings([MediaPipeline._compatible])
+        message = (
+            "image_downloaded(self, response, request, info) is deprecated, "
+            "please use image_downloaded(self, response, request, info, *, item=None)"
+        )
+        self._assert_method_called_with_warnings("image_downloaded", message, warnings)
+
+
 class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase):
     def _assert_request_no3xx(self, pipeline_class, settings):
         pipe = pipeline_class(settings=Settings(settings))