Repository: scrapy/scrapy (python)

Issue:
MediaPipeline exceptions passed silently
### Description

MediaPipeline exceptions passed silently both for method body or method signature errors.

### Steps to Reproduce

```
from scrapy.pipelines.files import FilesPipeline
class BuggyFilesPipeline(FilesPipeline):
    def file_path(self, request, response=None, info=None, *, item=None):
        return 1 / 0
```
**Expected behavior:** Exception logged

**Actual behavior:** Exception passed silently

**Reproduces how often:** 100%

### Versions

Scrapy 2.4


Pull request description:
MediaPipeline: log media_to_download errors before stripping them
Fixes #5067

To do:

- [x] Make sure that existing tests pass
- [x] Add a test for the issue being fixed here

Patch:
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 700775e4b..f64bbac06 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -13,6 +13,11 @@ Scrapy provides this functionality out of the box with the Feed Exports, which
 allows you to generate feeds with the scraped items, using multiple
 serialization formats and storage backends.
 
+This page provides detailed documentation for all feed export features. If you
+are looking for a step-by-step guide, check out `Zyte’s export guides`_.
+
+.. _Zyte’s export guides: https://docs.zyte.com/web-scraping/guides/export/index.html#exporting-scraped-data
+
 .. _topics-feed-format:
 
 Serialization formats
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 8edf710bc..9d64eee45 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -1357,3 +1357,13 @@ XmlResponse objects
     line.  See :attr:`TextResponse.encoding`.
 
 .. _bug in lxml: https://bugs.launchpad.net/lxml/+bug/1665241
+
+JsonResponse objects
+--------------------
+
+.. class:: JsonResponse(url[, ...])
+
+    The :class:`JsonResponse` class is a subclass of :class:`TextResponse` 
+    that is used when the response has a `JSON MIME type 
+    <https://mimesniff.spec.whatwg.org/#json-mime-type>`_ in its `Content-Type` 
+    header.
diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py
index ac3946302..d0b726bad 100644
--- a/scrapy/http/__init__.py
+++ b/scrapy/http/__init__.py
@@ -12,5 +12,6 @@ from scrapy.http.request.json_request import JsonRequest
 from scrapy.http.request.rpc import XmlRpcRequest
 from scrapy.http.response import Response
 from scrapy.http.response.html import HtmlResponse
+from scrapy.http.response.json import JsonResponse
 from scrapy.http.response.text import TextResponse
 from scrapy.http.response.xml import XmlResponse
diff --git a/scrapy/http/response/json.py b/scrapy/http/response/json.py
new file mode 100644
index 000000000..219691094
--- /dev/null
+++ b/scrapy/http/response/json.py
@@ -0,0 +1,12 @@
+"""
+This module implements the JsonResponse class that is used when the response
+has a JSON MIME type in its Content-Type header.
+
+See documentation in docs/topics/request-response.rst
+"""
+
+from scrapy.http.response.text import TextResponse
+
+
+class JsonResponse(TextResponse):
+    pass
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 75532034a..fc156ab41 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -112,14 +112,14 @@ class MediaPipeline:
         info.downloading.add(fp)
         dfd = mustbe_deferred(self.media_to_download, request, info, item=item)
         dfd.addCallback(self._check_media_to_download, request, info, item=item)
+        dfd.addErrback(self._log_exception)
         dfd.addBoth(self._cache_result_and_execute_waiters, fp, info)
-        dfd.addErrback(
-            lambda f: logger.error(
-                f.value, exc_info=failure_to_exc_info(f), extra={"spider": info.spider}
-            )
-        )
         return dfd.addBoth(lambda _: wad)  # it must return wad at last
 
+    def _log_exception(self, result):
+        logger.exception(result)
+        return result
+
     def _modify_media_request(self, request):
         if self.handle_httpstatus_list:
             request.meta["handle_httpstatus_list"] = self.handle_httpstatus_list
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 9e411d4aa..0d127d851 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -21,9 +21,9 @@ class ResponseTypes:
         "application/xhtml+xml": "scrapy.http.HtmlResponse",
         "application/vnd.wap.xhtml+xml": "scrapy.http.HtmlResponse",
         "application/xml": "scrapy.http.XmlResponse",
-        "application/json": "scrapy.http.TextResponse",
-        "application/x-json": "scrapy.http.TextResponse",
-        "application/json-amazonui-streaming": "scrapy.http.TextResponse",
+        "application/json": "scrapy.http.JsonResponse",
+        "application/x-json": "scrapy.http.JsonResponse",
+        "application/json-amazonui-streaming": "scrapy.http.JsonResponse",
         "application/javascript": "scrapy.http.TextResponse",
         "application/x-javascript": "scrapy.http.TextResponse",
         "text/xml": "scrapy.http.XmlResponse",


Test patch:
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index ed8483483..c41ab483f 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -9,6 +9,7 @@ from w3lib.url import add_or_replace_parameter
 
 from scrapy import signals
 from scrapy.crawler import CrawlerRunner
+from scrapy.utils.misc import load_object
 from tests.mockserver import MockServer
 from tests.spiders import SimpleSpider
 
@@ -193,6 +194,29 @@ class FileDownloadCrawlTestCase(TestCase):
             crawler.stats.get_value("downloader/response_status_count/302"), 3
         )
 
+    @defer.inlineCallbacks
+    def test_download_media_file_path_error(self):
+        cls = load_object(self.pipeline_class)
+
+        class ExceptionRaisingMediaPipeline(cls):
+            def file_path(self, request, response=None, info=None, *, item=None):
+                return 1 / 0
+
+        settings = {
+            **self.settings,
+            "ITEM_PIPELINES": {ExceptionRaisingMediaPipeline: 1},
+        }
+        runner = CrawlerRunner(settings)
+        crawler = self._create_crawler(MediaDownloadSpider, runner=runner)
+        with LogCapture() as log:
+            yield crawler.crawl(
+                self.mockserver.url("/files/images/"),
+                media_key=self.media_key,
+                media_urls_key=self.media_urls_key,
+                mockserver=self.mockserver,
+            )
+        self.assertIn("ZeroDivisionError", str(log))
+
 
 skip_pillow: Optional[str]
 try:
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index 6e1ed82f0..713a83d52 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -1,6 +1,13 @@
 import unittest
 
-from scrapy.http import Headers, HtmlResponse, Response, TextResponse, XmlResponse
+from scrapy.http import (
+    Headers,
+    HtmlResponse,
+    JsonResponse,
+    Response,
+    TextResponse,
+    XmlResponse,
+)
 from scrapy.responsetypes import responsetypes
 
 
@@ -40,8 +47,9 @@ class ResponseTypesTest(unittest.TestCase):
             ("application/vnd.wap.xhtml+xml; charset=utf-8", HtmlResponse),
             ("application/xml; charset=UTF-8", XmlResponse),
             ("application/octet-stream", Response),
-            ("application/x-json; encoding=UTF8;charset=UTF-8", TextResponse),
-            ("application/json-amazonui-streaming;charset=UTF-8", TextResponse),
+            ("application/json; encoding=UTF8;charset=UTF-8", JsonResponse),
+            ("application/x-json; encoding=UTF8;charset=UTF-8", JsonResponse),
+            ("application/json-amazonui-streaming;charset=UTF-8", JsonResponse),
             (b"application/x-download; filename=\x80dummy.txt", Response),
         ]
         for source, cls in mappings: