Repository: scrapy/scrapy
Pull request #6182 (2023-12-20 11:02:38)

scrapy parse: fix the signature of callbacks from the CLI
Fixes https://github.com/scrapinghub/scrapy-poet/issues/39.

It feels kind of hacky, specially the fact that it will not work for “rules”, but I see no way about it given rules depend on the response.

codecov[bot]: ## [Codecov](https://app.codecov.io/gh/scrapy/scrapy/pull/6182?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scrapy) Report
> Merging [#6182](https://app.codecov.io/gh/scrapy/scrapy/pull/6182?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scrapy) (6c96949) into [master](https://app.codecov.io/gh/scrapy/scrapy/commit/c67f73069570dd6dbe8427f55d6f9e65af07132a?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scrapy) (c67f730) will **decrease** coverage by `45.54%`.
> The diff coverage is `8.69%`.

<details><summary>Additional details and impacted files</summary>


```diff
@@             Coverage Diff             @@
##           master    #6182       +/-   ##
===========================================
- Coverage   88.61%   43.08%   -45.54%     
===========================================
  Files         159      159               
  Lines       11607    11614        +7     
  Branches     1891     1893        +2     
===========================================
- Hits        10286     5004     -5282     
- Misses        994     6218     +5224     
- Partials      327      392       +65     
```

| [Files](https://app.codecov.io/gh/scrapy/scrapy/pull/6182?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scrapy) | Coverage Δ | |
|---|---|---|
| [scrapy/commands/parse.py](https://app.codecov.io/gh/scrapy/scrapy/pull/6182?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scrapy#diff-c2NyYXB5L2NvbW1hbmRzL3BhcnNlLnB5) | `22.64% <8.69%> (-6.63%)` | :arrow_down: |

... and [139 files with indirect coverage changes](https://app.codecov.io/gh/scrapy/scrapy/pull/6182/indirect-changes?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scrapy)

</details>

Patch:
diff --git a/docs/contributing.rst b/docs/contributing.rst
index 2b3249601..d728338da 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -178,7 +178,7 @@ Scrapy:
 * We use `black <https://black.readthedocs.io/en/stable/>`_ for code formatting.
   There is a hook in the pre-commit config
   that will automatically format your code before every commit. You can also
-  run black manually with ``tox -e black``.
+  run black manually with ``tox -e pre-commit``.
 
 * Don't put your name in the code you contribute; git provides enough
   metadata to identify author of the code.
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 700775e4b..f64bbac06 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -13,6 +13,11 @@ Scrapy provides this functionality out of the box with the Feed Exports, which
 allows you to generate feeds with the scraped items, using multiple
 serialization formats and storage backends.
 
+This page provides detailed documentation for all feed export features. If you
+are looking for a step-by-step guide, check out `Zyte’s export guides`_.
+
+.. _Zyte’s export guides: https://docs.zyte.com/web-scraping/guides/export/index.html#exporting-scraped-data
+
 .. _topics-feed-format:
 
 Serialization formats
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 8edf710bc..9d64eee45 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -1357,3 +1357,13 @@ XmlResponse objects
     line.  See :attr:`TextResponse.encoding`.
 
 .. _bug in lxml: https://bugs.launchpad.net/lxml/+bug/1665241
+
+JsonResponse objects
+--------------------
+
+.. class:: JsonResponse(url[, ...])
+
+    The :class:`JsonResponse` class is a subclass of :class:`TextResponse` 
+    that is used when the response has a `JSON MIME type 
+    <https://mimesniff.spec.whatwg.org/#json-mime-type>`_ in its `Content-Type` 
+    header.
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index ac937e464..c9f8586d3 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -1,3 +1,4 @@
+import functools
 import inspect
 import json
 import logging
@@ -251,39 +252,40 @@ class Command(BaseRunSpiderCommand):
 
         return scraped_data
 
+    def _get_callback(self, *, spider, opts, response=None):
+        cb = None
+        if response:
+            cb = response.meta["_callback"]
+        if not cb:
+            if opts.callback:
+                cb = opts.callback
+            elif response and opts.rules and self.first_response == response:
+                cb = self.get_callback_from_rules(spider, response)
+                if not cb:
+                    raise ValueError(
+                        f"Cannot find a rule that matches {response.url!r} in spider: "
+                        f"{spider.name}"
+                    )
+            else:
+                cb = "parse"
+
+        if not callable(cb):
+            cb_method = getattr(spider, cb, None)
+            if callable(cb_method):
+                cb = cb_method
+            else:
+                raise ValueError(
+                    f"Cannot find callback {cb!r} in spider: {spider.name}"
+                )
+        return cb
+
     def prepare_request(self, spider, request, opts):
         def callback(response, **cb_kwargs):
             # memorize first request
             if not self.first_response:
                 self.first_response = response
 
-            # determine real callback
-            cb = response.meta["_callback"]
-            if not cb:
-                if opts.callback:
-                    cb = opts.callback
-                elif opts.rules and self.first_response == response:
-                    cb = self.get_callback_from_rules(spider, response)
-
-                    if not cb:
-                        logger.error(
-                            "Cannot find a rule that matches %(url)r in spider: %(spider)s",
-                            {"url": response.url, "spider": spider.name},
-                        )
-                        return
-                else:
-                    cb = "parse"
-
-            if not callable(cb):
-                cb_method = getattr(spider, cb, None)
-                if callable(cb_method):
-                    cb = cb_method
-                else:
-                    logger.error(
-                        "Cannot find callback %(callback)r in spider: %(spider)s",
-                        {"callback": cb, "spider": spider.name},
-                    )
-                    return
+            cb = self._get_callback(spider=spider, opts=opts, response=response)
 
             # parse items and requests
             depth = response.meta["_depth"]
@@ -303,6 +305,9 @@ class Command(BaseRunSpiderCommand):
 
         request.meta["_depth"] = 1
         request.meta["_callback"] = request.callback
+        if not request.callback and not opts.rules:
+            cb = self._get_callback(spider=spider, opts=opts)
+            functools.update_wrapper(callback, cb)
         request.callback = callback
         return request
 
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index dcf1a6dbc..031a0be36 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -103,8 +103,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
     ) -> Union[Generator, AsyncGenerator]:
         def process_sync(iterable: Iterable) -> Generator:
             try:
-                for r in iterable:
-                    yield r
+                yield from iterable
             except Exception as ex:
                 exception_result = self._process_spider_exception(
                     response, spider, Failure(ex), exception_processor_index
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 844d5f759..1db9ace28 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -178,6 +178,48 @@ class Crawler:
             assert self.engine
             yield maybeDeferred(self.engine.stop)
 
+    @staticmethod
+    def _get_component(component_class, components):
+        for component in components:
+            if isinstance(component, component_class):
+                return component
+        return None
+
+    def get_addon(self, cls):
+        return self._get_component(cls, self.addons.addons)
+
+    def get_downloader_middleware(self, cls):
+        if not self.engine:
+            raise RuntimeError(
+                "Crawler.get_downloader_middleware() can only be called after "
+                "the crawl engine has been created."
+            )
+        return self._get_component(cls, self.engine.downloader.middleware.middlewares)
+
+    def get_extension(self, cls):
+        if not self.extensions:
+            raise RuntimeError(
+                "Crawler.get_extension() can only be called after the "
+                "extension manager has been created."
+            )
+        return self._get_component(cls, self.extensions.middlewares)
+
+    def get_item_pipeline(self, cls):
+        if not self.engine:
+            raise RuntimeError(
+                "Crawler.get_item_pipeline() can only be called after the "
+                "crawl engine has been created."
+            )
+        return self._get_component(cls, self.engine.scraper.itemproc.middlewares)
+
+    def get_spider_middleware(self, cls):
+        if not self.engine:
+            raise RuntimeError(
+                "Crawler.get_spider_middleware() can only be called after the "
+                "crawl engine has been created."
+            )
+        return self._get_component(cls, self.engine.scraper.spidermw.middlewares)
+
 
 class CrawlerRunner:
     """
diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index 221967bda..ca766c938 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -128,9 +128,9 @@ class MemoryUsage:
     def _send_report(self, rcpts, subject):
         """send notification mail with some additional useful info"""
         stats = self.crawler.stats
-        s = f"Memory usage at engine startup : {stats.get_value('memusage/startup')/1024/1024}M\r\n"
-        s += f"Maximum memory usage          : {stats.get_value('memusage/max')/1024/1024}M\r\n"
-        s += f"Current memory usage          : {self.get_virtual_size()/1024/1024}M\r\n"
+        s = f"Memory usage at engine startup : {stats.get_value('memusage/startup') / 1024 / 1024}M\r\n"
+        s += f"Maximum memory usage          : {stats.get_value('memusage/max') / 1024 / 1024}M\r\n"
+        s += f"Current memory usage          : {self.get_virtual_size() / 1024 / 1024}M\r\n"
 
         s += (
             "ENGINE STATUS ------------------------------------------------------- \r\n"
diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py
index ac3946302..d0b726bad 100644
--- a/scrapy/http/__init__.py
+++ b/scrapy/http/__init__.py
@@ -12,5 +12,6 @@ from scrapy.http.request.json_request import JsonRequest
 from scrapy.http.request.rpc import XmlRpcRequest
 from scrapy.http.response import Response
 from scrapy.http.response.html import HtmlResponse
+from scrapy.http.response.json import JsonResponse
 from scrapy.http.response.text import TextResponse
 from scrapy.http.response.xml import XmlResponse
diff --git a/scrapy/http/response/json.py b/scrapy/http/response/json.py
new file mode 100644
index 000000000..219691094
--- /dev/null
+++ b/scrapy/http/response/json.py
@@ -0,0 +1,12 @@
+"""
+This module implements the JsonResponse class that is used when the response
+has a JSON MIME type in its Content-Type header.
+
+See documentation in docs/topics/request-response.rst
+"""
+
+from scrapy.http.response.text import TextResponse
+
+
+class JsonResponse(TextResponse):
+    pass
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 5c09ab37e..73064ad10 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -8,7 +8,6 @@ import functools
 import hashlib
 import logging
 import mimetypes
-import os
 import time
 from collections import defaultdict
 from contextlib import suppress
@@ -66,7 +65,7 @@ class FSFilesStore:
         absolute_path = self._get_filesystem_path(path)
         try:
             last_modified = absolute_path.stat().st_mtime
-        except os.error:
+        except OSError:
             return {}
 
         with absolute_path.open("rb") as f:
@@ -340,7 +339,9 @@ class FilesPipeline(MediaPipeline):
     DEFAULT_FILES_URLS_FIELD = "file_urls"
     DEFAULT_FILES_RESULT_FIELD = "files"
 
-    def __init__(self, store_uri, download_func=None, settings=None):
+    def __init__(
+        self, store_uri: Union[str, PathLike], download_func=None, settings=None
+    ):
         store_uri = _to_string(store_uri)
         if not store_uri:
             raise NotConfigured
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 1bd9832a8..02c4b1361 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -8,7 +8,8 @@ import hashlib
 import warnings
 from contextlib import suppress
 from io import BytesIO
-from typing import Dict, Tuple
+from os import PathLike
+from typing import Dict, Tuple, Union
 
 from itemadapter import ItemAdapter
 
@@ -53,7 +54,9 @@ class ImagesPipeline(FilesPipeline):
     DEFAULT_IMAGES_URLS_FIELD = "image_urls"
     DEFAULT_IMAGES_RESULT_FIELD = "images"
 
-    def __init__(self, store_uri, download_func=None, settings=None):
+    def __init__(
+        self, store_uri: Union[str, PathLike], download_func=None, settings=None
+    ):
         try:
             from PIL import Image
 
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 75532034a..fc156ab41 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -112,14 +112,14 @@ class MediaPipeline:
         info.downloading.add(fp)
         dfd = mustbe_deferred(self.media_to_download, request, info, item=item)
         dfd.addCallback(self._check_media_to_download, request, info, item=item)
+        dfd.addErrback(self._log_exception)
         dfd.addBoth(self._cache_result_and_execute_waiters, fp, info)
-        dfd.addErrback(
-            lambda f: logger.error(
-                f.value, exc_info=failure_to_exc_info(f), extra={"spider": info.spider}
-            )
-        )
         return dfd.addBoth(lambda _: wad)  # it must return wad at last
 
+    def _log_exception(self, result):
+        logger.exception(result)
+        return result
+
     def _modify_media_request(self, request):
         if self.handle_httpstatus_list:
             request.meta["handle_httpstatus_list"] = self.handle_httpstatus_list
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 9e411d4aa..0d127d851 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -21,9 +21,9 @@ class ResponseTypes:
         "application/xhtml+xml": "scrapy.http.HtmlResponse",
         "application/vnd.wap.xhtml+xml": "scrapy.http.HtmlResponse",
         "application/xml": "scrapy.http.XmlResponse",
-        "application/json": "scrapy.http.TextResponse",
-        "application/x-json": "scrapy.http.TextResponse",
-        "application/json-amazonui-streaming": "scrapy.http.TextResponse",
+        "application/json": "scrapy.http.JsonResponse",
+        "application/x-json": "scrapy.http.JsonResponse",
+        "application/json-amazonui-streaming": "scrapy.http.JsonResponse",
         "application/javascript": "scrapy.http.TextResponse",
         "application/x-javascript": "scrapy.http.TextResponse",
         "text/xml": "scrapy.http.XmlResponse",
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 31e845716..ebb4f5984 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -131,8 +131,7 @@ class CrawlSpider(Spider):
     def _handle_failure(self, failure, errback):
         if errback:
             results = errback(failure) or ()
-            for request_or_item in iterate_spider_output(results):
-                yield request_or_item
+            yield from iterate_spider_output(results)
 
     def _compile_rules(self):
         self._rules = []
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index 6afadc577..47827e442 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -58,8 +58,7 @@ class XMLFeedSpider(Spider):
 
         for selector in nodes:
             ret = iterate_spider_output(self.parse_node(response, selector))
-            for result_item in self.process_results(response, ret):
-                yield result_item
+            yield from self.process_results(response, ret)
 
     def _parse(self, response, **kwargs):
         if not hasattr(self, "parse_node"):
@@ -133,8 +132,7 @@ class CSVFeedSpider(Spider):
             response, self.delimiter, self.headers, quotechar=self.quotechar
         ):
             ret = iterate_spider_output(self.parse_row(response, row))
-            for result_item in self.process_results(response, ret):
-                yield result_item
+            yield from self.process_results(response, ret)
 
     def _parse(self, response, **kwargs):
         if not hasattr(self, "parse_row"):
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index aaf75a519..974665fe0 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -33,8 +33,7 @@ class SitemapSpider(Spider):
         attributes, for example, you can filter locs with lastmod greater
         than a given date (see docs).
         """
-        for entry in entries:
-            yield entry
+        yield from entries
 
     def _parse_sitemap(self, response):
         if response.url.endswith("/robots.txt"):
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 0b5dc324f..68ca96b69 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -57,8 +57,7 @@ def iflatten(x: Iterable) -> Iterable:
     Similar to ``.flatten()``, but returns iterator instead"""
     for el in x:
         if is_listlike(el):
-            for el_ in iflatten(el):
-                yield el_
+            yield from iflatten(el)
         else:
             yield el
 
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 24fcbd85e..cea1bc727 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -44,8 +44,7 @@ def _serialize_headers(
     for header in headers:
         if header in request.headers:
             yield header
-            for value in request.headers.getlist(header):
-                yield value
+            yield from request.headers.getlist(header)
 
 
 def request_fingerprint(
diff --git a/tox.ini b/tox.ini
index 932c0b805..d9dcacc01 100644
--- a/tox.ini
+++ b/tox.ini
@@ -11,11 +11,7 @@ minversion = 1.7.0
 deps =
     -rtests/requirements.txt
     # mitmproxy does not support PyPy
-    # Python 3.9+ requires mitmproxy >= 5.3.0
-    # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0
-    #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy'
-    # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454
-    mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy'
+    mitmproxy; implementation_name != 'pypy'
 passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID
@@ -87,7 +83,7 @@ deps =
     lxml==4.4.1
     -rtests/requirements.txt
 
-    # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies
+    # mitmproxy 8.0.0 requires upgrading some of the pinned dependencies
     # above, hence we do not install it in pinned environments at the moment
 setenv =
     _SCRAPY_PINNED=true


Test patch:
diff --git a/docs/_tests/quotes.html b/docs/_tests/quotes.html
index 71aff8847..f4002ecd1 100644
--- a/docs/_tests/quotes.html
+++ b/docs/_tests/quotes.html
@@ -273,7 +273,7 @@
                 Quotes by: <a href="https://www.goodreads.com/quotes">GoodReads.com</a>
             </p>
             <p class="copyright">
-                Made with <span class='sh-red'>❤</span> by <a href="https://scrapinghub.com">Scrapinghub</a>
+                Made with <span class='sh-red'>❤</span> by <a href="https://www.zyte.com">Zyte</a>
             </p>
         </div>
     </footer>
diff --git a/docs/_tests/quotes1.html b/docs/_tests/quotes1.html
index 71aff8847..f4002ecd1 100644
--- a/docs/_tests/quotes1.html
+++ b/docs/_tests/quotes1.html
@@ -273,7 +273,7 @@
                 Quotes by: <a href="https://www.goodreads.com/quotes">GoodReads.com</a>
             </p>
             <p class="copyright">
-                Made with <span class='sh-red'>❤</span> by <a href="https://scrapinghub.com">Scrapinghub</a>
+                Made with <span class='sh-red'>❤</span> by <a href="https://www.zyte.com">Zyte</a>
             </p>
         </div>
     </footer>
diff --git a/tests/spiders.py b/tests/spiders.py
index f29dea2a1..3df153a12 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -301,8 +301,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
 
     def parse(self, response):
         self.seedsseen.append(response.meta.get("seed"))
-        for req in super().parse(response):
-            yield req
+        yield from super().parse(response)
 
 
 class SingleRequestSpider(MetaSpider):
diff --git a/tests/test_command_check.py b/tests/test_command_check.py
index 129ef0121..592494aba 100644
--- a/tests/test_command_check.py
+++ b/tests/test_command_check.py
@@ -16,11 +16,11 @@ import scrapy
 
 class CheckSpider(scrapy.Spider):
     name = '{self.spider_name}'
-    start_urls = ['http://toscrape.com']
+    start_urls = ['data:,']
 
     def parse(self, response, **cb_kwargs):
         \"\"\"
-        @url http://toscrape.com
+        @url data:,
         {contracts}
         \"\"\"
         {parse_def}
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 037333c03..9356d6b79 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -78,9 +78,21 @@ class AsyncDefAsyncioGenExcSpider(scrapy.Spider):
             if i > 5:
                 raise ValueError("Stopping the processing")
 
+class CallbackSignatureDownloaderMiddleware:
+    def process_request(self, request, spider):
+        from inspect import signature
+        spider.logger.debug(f"request.callback signature: {{signature(request.callback)}}")
+
+
 class MySpider(scrapy.Spider):
     name = '{self.spider_name}'
 
+    custom_settings = {{
+        "DOWNLOADER_MIDDLEWARES": {{
+            CallbackSignatureDownloaderMiddleware: 0,
+        }}
+    }}
+
     def parse(self, response):
         if getattr(self, 'test_arg', None):
             self.logger.debug('It Works!')
@@ -220,7 +232,11 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
                 self.url("/html"),
             ]
         )
-        self.assertIn("DEBUG: It Works!", _textmode(stderr))
+        log = _textmode(stderr)
+        self.assertIn("DEBUG: It Works!", log)
+        self.assertIn(
+            "DEBUG: request.callback signature: (response, foo=None, key=None)", log
+        )
 
     @defer.inlineCallbacks
     def test_request_without_meta(self):
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 92bd5f38f..989208694 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -12,12 +12,13 @@ import pytest
 from packaging.version import parse as parse_version
 from pexpect.popen_spawn import PopenSpawn
 from pytest import mark, raises
-from twisted.internet import defer
+from twisted.internet.defer import Deferred, inlineCallbacks
 from twisted.trial import unittest
 from w3lib import __version__ as w3lib_version
 from zope.interface.exceptions import MultipleInvalid
 
 import scrapy
+from scrapy import Spider
 from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.extensions import telnet
@@ -29,6 +30,19 @@ from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler
 from tests.mockserver import MockServer, get_mockserver_env
 
+# To prevent warnings.
+BASE_SETTINGS = {
+    "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
+}
+
+
+def get_raw_crawler(spidercls=None, settings_dict=None):
+    """get_crawler alternative that only calls the __init__ method of the
+    crawler."""
+    settings = Settings()
+    settings.setdict(settings_dict or {})
+    return Crawler(spidercls or DefaultSpider, settings)
+
 
 class BaseCrawlerTest(unittest.TestCase):
     def assertOptionIsDefault(self, settings, key):
@@ -39,7 +53,7 @@ class BaseCrawlerTest(unittest.TestCase):
 class CrawlerTestCase(BaseCrawlerTest):
     def test_populate_spidercls_settings(self):
         spider_settings = {"TEST1": "spider", "TEST2": "spider"}
-        project_settings = {"TEST1": "project", "TEST3": "project"}
+        project_settings = {**BASE_SETTINGS, "TEST1": "project", "TEST3": "project"}
 
         class CustomSettingsSpider(DefaultSpider):
             custom_settings = spider_settings
@@ -71,9 +85,9 @@ class CrawlerTestCase(BaseCrawlerTest):
         with raises(ValueError):
             Crawler(DefaultSpider())
 
-    @defer.inlineCallbacks
+    @inlineCallbacks
     def test_crawler_crawl_twice_deprecated(self):
-        crawler = Crawler(NoRequestsSpider)
+        crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
         yield crawler.crawl()
         with pytest.warns(
             ScrapyDeprecationWarning,
@@ -81,6 +95,358 @@ class CrawlerTestCase(BaseCrawlerTest):
         ):
             yield crawler.crawl()
 
+    def test_get_addon(self):
+        class ParentAddon:
+            pass
+
+        class TrackingAddon(ParentAddon):
+            instances = []
+
+            def __init__(self):
+                TrackingAddon.instances.append(self)
+
+            def update_settings(self, settings):
+                pass
+
+        settings = {
+            **BASE_SETTINGS,
+            "ADDONS": {
+                TrackingAddon: 0,
+            },
+        }
+        crawler = get_crawler(settings_dict=settings)
+        self.assertEqual(len(TrackingAddon.instances), 1)
+        expected = TrackingAddon.instances[-1]
+
+        addon = crawler.get_addon(TrackingAddon)
+        self.assertEqual(addon, expected)
+
+        addon = crawler.get_addon(DefaultSpider)
+        self.assertIsNone(addon)
+
+        addon = crawler.get_addon(ParentAddon)
+        self.assertEqual(addon, expected)
+
+        class ChildAddon(TrackingAddon):
+            pass
+
+        addon = crawler.get_addon(ChildAddon)
+        self.assertIsNone(addon)
+
+    @inlineCallbacks
+    def test_get_downloader_middleware(self):
+        class ParentDownloaderMiddleware:
+            pass
+
+        class TrackingDownloaderMiddleware(ParentDownloaderMiddleware):
+            instances = []
+
+            def __init__(self):
+                TrackingDownloaderMiddleware.instances.append(self)
+
+        class MySpider(Spider):
+            name = "myspider"
+
+            @classmethod
+            def from_crawler(cls, crawler):
+                return cls(crawler=crawler)
+
+            def __init__(self, crawler):
+                self.crawler = crawler
+
+            def start_requests(self):
+                MySpider.result = crawler.get_downloader_middleware(MySpider.cls)
+                return
+                yield
+
+        settings = {
+            **BASE_SETTINGS,
+            "DOWNLOADER_MIDDLEWARES": {
+                TrackingDownloaderMiddleware: 0,
+            },
+        }
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = TrackingDownloaderMiddleware
+        yield crawler.crawl()
+        self.assertEqual(len(TrackingDownloaderMiddleware.instances), 1)
+        self.assertEqual(MySpider.result, TrackingDownloaderMiddleware.instances[-1])
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = DefaultSpider
+        yield crawler.crawl()
+        self.assertIsNone(MySpider.result)
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = ParentDownloaderMiddleware
+        yield crawler.crawl()
+        self.assertEqual(MySpider.result, TrackingDownloaderMiddleware.instances[-1])
+
+        class ChildDownloaderMiddleware(TrackingDownloaderMiddleware):
+            pass
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = ChildDownloaderMiddleware
+        yield crawler.crawl()
+        self.assertIsNone(MySpider.result)
+
+    def test_get_downloader_middleware_not_crawling(self):
+        crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
+        self.assertRaises(
+            RuntimeError, crawler.get_downloader_middleware, DefaultSpider
+        )
+
+    @inlineCallbacks
+    def test_get_downloader_middleware_no_engine(self):
+        class MySpider(Spider):
+            name = "myspider"
+
+            @classmethod
+            def from_crawler(cls, crawler):
+                try:
+                    crawler.get_downloader_middleware(DefaultSpider)
+                except Exception as e:
+                    MySpider.result = e
+                    raise
+
+        crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
+        with raises(RuntimeError):
+            yield crawler.crawl()
+
+    @inlineCallbacks
+    def test_get_extension(self):
+        class ParentExtension:
+            pass
+
+        class TrackingExtension(ParentExtension):
+            instances = []
+
+            def __init__(self):
+                TrackingExtension.instances.append(self)
+
+        class MySpider(Spider):
+            name = "myspider"
+
+            @classmethod
+            def from_crawler(cls, crawler):
+                return cls(crawler=crawler)
+
+            def __init__(self, crawler):
+                self.crawler = crawler
+
+            def start_requests(self):
+                MySpider.result = crawler.get_extension(MySpider.cls)
+                return
+                yield
+
+        settings = {
+            **BASE_SETTINGS,
+            "EXTENSIONS": {
+                TrackingExtension: 0,
+            },
+        }
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = TrackingExtension
+        yield crawler.crawl()
+        self.assertEqual(len(TrackingExtension.instances), 1)
+        self.assertEqual(MySpider.result, TrackingExtension.instances[-1])
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = DefaultSpider
+        yield crawler.crawl()
+        self.assertIsNone(MySpider.result)
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = ParentExtension
+        yield crawler.crawl()
+        self.assertEqual(MySpider.result, TrackingExtension.instances[-1])
+
+        class ChildExtension(TrackingExtension):
+            pass
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = ChildExtension
+        yield crawler.crawl()
+        self.assertIsNone(MySpider.result)
+
+    def test_get_extension_not_crawling(self):
+        crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
+        self.assertRaises(RuntimeError, crawler.get_extension, DefaultSpider)
+
+    @inlineCallbacks
+    def test_get_extension_no_engine(self):
+        class MySpider(Spider):
+            name = "myspider"
+
+            @classmethod
+            def from_crawler(cls, crawler):
+                try:
+                    crawler.get_extension(DefaultSpider)
+                except Exception as e:
+                    MySpider.result = e
+                    raise
+
+        crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
+        with raises(RuntimeError):
+            yield crawler.crawl()
+
+    @inlineCallbacks
+    def test_get_item_pipeline(self):
+        class ParentItemPipeline:
+            pass
+
+        class TrackingItemPipeline(ParentItemPipeline):
+            instances = []
+
+            def __init__(self):
+                TrackingItemPipeline.instances.append(self)
+
+        class MySpider(Spider):
+            name = "myspider"
+
+            @classmethod
+            def from_crawler(cls, crawler):
+                return cls(crawler=crawler)
+
+            def __init__(self, crawler):
+                self.crawler = crawler
+
+            def start_requests(self):
+                MySpider.result = crawler.get_item_pipeline(MySpider.cls)
+                return
+                yield
+
+        settings = {
+            **BASE_SETTINGS,
+            "ITEM_PIPELINES": {
+                TrackingItemPipeline: 0,
+            },
+        }
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = TrackingItemPipeline
+        yield crawler.crawl()
+        self.assertEqual(len(TrackingItemPipeline.instances), 1)
+        self.assertEqual(MySpider.result, TrackingItemPipeline.instances[-1])
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = DefaultSpider
+        yield crawler.crawl()
+        self.assertIsNone(MySpider.result)
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = ParentItemPipeline
+        yield crawler.crawl()
+        self.assertEqual(MySpider.result, TrackingItemPipeline.instances[-1])
+
+        class ChildItemPipeline(TrackingItemPipeline):
+            pass
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = ChildItemPipeline
+        yield crawler.crawl()
+        self.assertIsNone(MySpider.result)
+
+    def test_get_item_pipeline_not_crawling(self):
+        crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
+        self.assertRaises(RuntimeError, crawler.get_item_pipeline, DefaultSpider)
+
+    @inlineCallbacks
+    def test_get_item_pipeline_no_engine(self):
+        class MySpider(Spider):
+            name = "myspider"
+
+            @classmethod
+            def from_crawler(cls, crawler):
+                try:
+                    crawler.get_item_pipeline(DefaultSpider)
+                except Exception as e:
+                    MySpider.result = e
+                    raise
+
+        crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
+        with raises(RuntimeError):
+            yield crawler.crawl()
+
+    @inlineCallbacks
+    def test_get_spider_middleware(self):
+        class ParentSpiderMiddleware:
+            pass
+
+        class TrackingSpiderMiddleware(ParentSpiderMiddleware):
+            instances = []
+
+            def __init__(self):
+                TrackingSpiderMiddleware.instances.append(self)
+
+        class MySpider(Spider):
+            name = "myspider"
+
+            @classmethod
+            def from_crawler(cls, crawler):
+                return cls(crawler=crawler)
+
+            def __init__(self, crawler):
+                self.crawler = crawler
+
+            def start_requests(self):
+                MySpider.result = crawler.get_spider_middleware(MySpider.cls)
+                return
+                yield
+
+        settings = {
+            **BASE_SETTINGS,
+            "SPIDER_MIDDLEWARES": {
+                TrackingSpiderMiddleware: 0,
+            },
+        }
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = TrackingSpiderMiddleware
+        yield crawler.crawl()
+        self.assertEqual(len(TrackingSpiderMiddleware.instances), 1)
+        self.assertEqual(MySpider.result, TrackingSpiderMiddleware.instances[-1])
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = DefaultSpider
+        yield crawler.crawl()
+        self.assertIsNone(MySpider.result)
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = ParentSpiderMiddleware
+        yield crawler.crawl()
+        self.assertEqual(MySpider.result, TrackingSpiderMiddleware.instances[-1])
+
+        class ChildSpiderMiddleware(TrackingSpiderMiddleware):
+            pass
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = ChildSpiderMiddleware
+        yield crawler.crawl()
+        self.assertIsNone(MySpider.result)
+
+    def test_get_spider_middleware_not_crawling(self):
+        crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
+        self.assertRaises(RuntimeError, crawler.get_spider_middleware, DefaultSpider)
+
+    @inlineCallbacks
+    def test_get_spider_middleware_no_engine(self):
+        class MySpider(Spider):
+            name = "myspider"
+
+            @classmethod
+            def from_crawler(cls, crawler):
+                try:
+                    crawler.get_spider_middleware(DefaultSpider)
+                except Exception as e:
+                    MySpider.result = e
+                    raise
+
+        crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
+        with raises(RuntimeError):
+            yield crawler.crawl()
+
 
 class SpiderSettingsTestCase(unittest.TestCase):
     def test_spider_custom_settings(self):
@@ -223,20 +589,20 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
     def _runner(self):
         return CrawlerRunner({"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7"})
 
-    @defer.inlineCallbacks
+    @inlineCallbacks
     def test_crawler_runner_bootstrap_successful(self):
         runner = self._runner()
         yield runner.crawl(NoRequestsSpider)
         self.assertFalse(runner.bootstrap_failed)
 
-    @defer.inlineCallbacks
+    @inlineCallbacks
     def test_crawler_runner_bootstrap_successful_for_several(self):
         runner = self._runner()
         yield runner.crawl(NoRequestsSpider)
         yield runner.crawl(NoRequestsSpider)
         self.assertFalse(runner.bootstrap_failed)
 
-    @defer.inlineCallbacks
+    @inlineCallbacks
     def test_crawler_runner_bootstrap_failed(self):
         runner = self._runner()
 
@@ -249,7 +615,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
 
         self.assertTrue(runner.bootstrap_failed)
 
-    @defer.inlineCallbacks
+    @inlineCallbacks
     def test_crawler_runner_bootstrap_failed_for_several(self):
         runner = self._runner()
 
@@ -264,7 +630,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
 
         self.assertTrue(runner.bootstrap_failed)
 
-    @defer.inlineCallbacks
+    @inlineCallbacks
     def test_crawler_runner_asyncio_enabled_true(self):
         if self.reactor_pytest == "asyncio":
             CrawlerRunner(
@@ -531,7 +897,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         p.expect_exact("Spider closed (shutdown)")
         p.wait()
 
-    @defer.inlineCallbacks
+    @inlineCallbacks
     def test_shutdown_forced(self):
         from twisted.internet import reactor
 
@@ -543,7 +909,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         p.kill(sig)
         p.expect_exact("shutting down gracefully")
         # sending the second signal too fast often causes problems
-        d = defer.Deferred()
+        d = Deferred()
         reactor.callLater(0.1, d.callback, None)
         yield d
         p.kill(sig)
diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py
index 502ada6be..b7312bbcd 100644
--- a/tests/test_extension_periodic_log.py
+++ b/tests/test_extension_periodic_log.py
@@ -67,7 +67,7 @@ def extension(settings=None):
 
 class TestPeriodicLog(unittest.TestCase):
     def test_extension_enabled(self):
-        # Expected that settings for this extension loaded succesfully
+        # Expected that settings for this extension loaded successfully
         # And on certain conditions - extension raising NotConfigured
 
         # "PERIODIC_LOG_STATS": True -> set to {"enabled": True}
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 89169fd7c..277555608 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -673,8 +673,7 @@ class FeedExportTestBase(ABC, unittest.TestCase):
             name = "testspider"
 
             def parse(self, response):
-                for item in items:
-                    yield item
+                yield from items
 
         data = yield self.run_and_export(TestSpider, settings)
         return data
@@ -2300,7 +2299,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                     content[feed["format"]].append(file.read_bytes())
         finally:
             self.tearDown()
-        defer.returnValue(content)
+        return content
 
     @defer.inlineCallbacks
     def assertExportedJsonLines(self, items, rows, settings=None):
@@ -2696,8 +2695,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
             name = "testspider"
 
             def parse(self, response):
-                for item in items:
-                    yield item
+                yield from items
 
         with MockServer() as server:
             TestSpider.start_urls = [server.url("/")]
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index ed8483483..c41ab483f 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -9,6 +9,7 @@ from w3lib.url import add_or_replace_parameter
 
 from scrapy import signals
 from scrapy.crawler import CrawlerRunner
+from scrapy.utils.misc import load_object
 from tests.mockserver import MockServer
 from tests.spiders import SimpleSpider
 
@@ -193,6 +194,29 @@ class FileDownloadCrawlTestCase(TestCase):
             crawler.stats.get_value("downloader/response_status_count/302"), 3
         )
 
+    @defer.inlineCallbacks
+    def test_download_media_file_path_error(self):
+        cls = load_object(self.pipeline_class)
+
+        class ExceptionRaisingMediaPipeline(cls):
+            def file_path(self, request, response=None, info=None, *, item=None):
+                return 1 / 0
+
+        settings = {
+            **self.settings,
+            "ITEM_PIPELINES": {ExceptionRaisingMediaPipeline: 1},
+        }
+        runner = CrawlerRunner(settings)
+        crawler = self._create_crawler(MediaDownloadSpider, runner=runner)
+        with LogCapture() as log:
+            yield crawler.crawl(
+                self.mockserver.url("/files/images/"),
+                media_key=self.media_key,
+                media_urls_key=self.media_urls_key,
+                mockserver=self.mockserver,
+            )
+        self.assertIn("ZeroDivisionError", str(log))
+
 
 skip_pillow: Optional[str]
 try:
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index dc0a82086..46d42e9f6 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -31,6 +31,7 @@ sys.exit(mitmdump())
         self.proc = Popen(
             [
                 sys.executable,
+                "-u",
                 "-c",
                 script,
                 "--listen-host",
@@ -46,7 +47,7 @@ sys.exit(mitmdump())
             stdout=PIPE,
         )
         line = self.proc.stdout.readline().decode("utf-8")
-        host_port = re.search(r"listening at http://([^:]+:\d+)", line).group(1)
+        host_port = re.search(r"listening at (?:http://)?([^:]+:\d+)", line).group(1)
         address = f"http://{self.auth_user}:{self.auth_pass}@{host_port}"
         return address
 
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index 6e1ed82f0..713a83d52 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -1,6 +1,13 @@
 import unittest
 
-from scrapy.http import Headers, HtmlResponse, Response, TextResponse, XmlResponse
+from scrapy.http import (
+    Headers,
+    HtmlResponse,
+    JsonResponse,
+    Response,
+    TextResponse,
+    XmlResponse,
+)
 from scrapy.responsetypes import responsetypes
 
 
@@ -40,8 +47,9 @@ class ResponseTypesTest(unittest.TestCase):
             ("application/vnd.wap.xhtml+xml; charset=utf-8", HtmlResponse),
             ("application/xml; charset=UTF-8", XmlResponse),
             ("application/octet-stream", Response),
-            ("application/x-json; encoding=UTF8;charset=UTF-8", TextResponse),
-            ("application/json-amazonui-streaming;charset=UTF-8", TextResponse),
+            ("application/json; encoding=UTF8;charset=UTF-8", JsonResponse),
+            ("application/x-json; encoding=UTF8;charset=UTF-8", JsonResponse),
+            ("application/json-amazonui-streaming;charset=UTF-8", JsonResponse),
             (b"application/x-download; filename=\x80dummy.txt", Response),
         ]
         for source, cls in mappings:
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 00da3d485..9ce40f921 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -317,8 +317,7 @@ class CrawlSpiderTest(SpiderTest):
             rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),)
 
             def dummy_process_links(self, links):
-                for link in links:
-                    yield link
+                yield from links
 
         spider = _CrawlSpider()
         output = list(spider._requests_to_follow(response))
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index d167adbb7..38ca8d950 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -170,8 +170,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
 
 class ProcessSpiderOutputSimpleMiddleware:
     def process_spider_output(self, response, result, spider):
-        for r in result:
-            yield r
+        yield from result
 
 
 class ProcessSpiderOutputAsyncGenMiddleware:
@@ -182,8 +181,7 @@ class ProcessSpiderOutputAsyncGenMiddleware:
 
 class ProcessSpiderOutputUniversalMiddleware:
     def process_spider_output(self, response, result, spider):
-        for r in result:
-            yield r
+        yield from result
 
     async def process_spider_output_async(self, response, result, spider):
         async for r in result:
@@ -324,8 +322,7 @@ class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):
 
 class ProcessStartRequestsSimpleMiddleware:
     def process_start_requests(self, start_requests, spider):
-        for r in start_requests:
-            yield r
+        yield from start_requests
 
 
 class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase):
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index bb0ebc2a4..a7d54b565 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -107,8 +107,7 @@ class DeferUtilsTest(unittest.TestCase):
 class IterErrbackTest(unittest.TestCase):
     def test_iter_errback_good(self):
         def itergood():
-            for x in range(10):
-                yield x
+            yield from range(10)
 
         errors = []
         out = list(iter_errback(itergood(), errors.append))
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 3598fa0bb..4a0c34d82 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -524,6 +524,6 @@ class TestHelper(unittest.TestCase):
 
     def _assert_type_and_value(self, a, b, obj):
         self.assertTrue(
-            type(a) is type(b), f"Got {type(a)}, expected {type(b)} for { obj!r}"
+            type(a) is type(b), f"Got {type(a)}, expected {type(b)} for {obj!r}"
         )
         self.assertEqual(a, b)