Repository: scrapy/scrapy (python)

Issue:
Results of errbacks for downloader errors are discarded
<!--

Thanks for taking an interest in Scrapy!

If you have a question that starts with "How to...", please see the Scrapy Community page: https://scrapy.org/community/.
The GitHub issue tracker's purpose is to deal with bug reports and feature requests for the project itself.

Keep in mind that by filing an issue, you are expected to comply with Scrapy's Code of Conduct, including treating everyone with respect: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md

The following is a suggested template to structure your issue, you can find more guidelines at https://doc.scrapy.org/en/latest/contributing.html#reporting-bugs

-->

### Description

In Scrapy 2.13, errback functions defined as generator functions are not being executed when errors occur, while this worked correctly in Scrapy 2.12

### Steps to Reproduce

Spider:
```python
import scrapy


class QuotesSpider(scrapy.Spider):
    name = "quotes"

    def start_requests(self):
        urls = [
            "https://quotes.toscrape.com/page/1/",
        ]
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse, errback=self.errback)

    def errback(self, failure):
        print("errback", failure)
        yield  # <-- This works fine if it's not a generator function

    def parse(self, response):
        print(response)

```

Middleware:
```python
class TutorialDownloaderMiddleware:
    def process_request(self, request, spider):
        raise Exception("process request error")
```

**Expected behavior:** [What you expect to happen]
The errback function is called

**Actual behavior:** [What actually happens]
The errback function was not called

**Reproduces how often:** [What percentage of the time does it reproduce?]

### Versions

```
Scrapy       : 2.13.1
lxml         : 5.4.0
libxml2      : 2.13.8
cssselect    : 1.3.0
parsel       : 1.10.0
w3lib        : 2.3.1
Twisted      : 24.11.0
Python       : 3.13.4 (main, Jun  4 2025, 17:44:50) [Clang 20.1.4 ]
pyOpenSSL    : 25.1.0 (OpenSSL 3.5.0 8 Apr 2025)
cryptography : 45.0.3
Platform     : macOS-15.5-x86_64-i386-64bit-Mach-O
```


Pull request:
Fix a regression in errback result handling.
Fixes #6861 

Patch:
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 9fd68bce5..1f0d57c63 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -189,6 +189,7 @@ class Scraper:
             )
 
         assert self.crawler.spider
+        output: Iterable[Any] | AsyncIterator[Any]
         if isinstance(result, Response):
             try:
                 # call the spider middlewares and the request callback with the response
@@ -203,7 +204,7 @@ class Scraper:
 
         try:
             # call the request errback with the downloader error
-            await self.call_spider_async(result, request)
+            output = await self.call_spider_async(result, request)
         except Exception as spider_exc:
             # the errback didn't silence the exception
             if not result.check(IgnoreRequest):
@@ -218,6 +219,8 @@ class Scraper:
             if spider_exc is not result.value:
                 # the errback raised a different exception, handle it
                 self.handle_spider_error(Failure(), request, result)
+        else:
+            await self.handle_spider_output_async(output, request, result)
 
     def call_spider(
         self, result: Response | Failure, request: Request, spider: Spider | None = None
@@ -308,7 +311,7 @@ class Scraper:
         self,
         result: Iterable[_T] | AsyncIterator[_T],
         request: Request,
-        response: Response,
+        response: Response | Failure,
         spider: Spider | None = None,
     ) -> Deferred[None]:
         """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel."""
@@ -326,7 +329,7 @@ class Scraper:
         self,
         result: Iterable[_T] | AsyncIterator[_T],
         request: Request,
-        response: Response,
+        response: Response | Failure,
     ) -> None:
         """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel."""
         it: Iterable[_T] | AsyncIterator[_T]
@@ -361,7 +364,7 @@ class Scraper:
         )
 
     def _process_spidermw_output(
-        self, output: Any, response: Response
+        self, output: Any, response: Response | Failure
     ) -> Deferred[None]:
         """Process each Request/Item (given in the output parameter) returned
         from the given spider.
@@ -371,7 +374,7 @@ class Scraper:
         return deferred_from_coro(self._process_spidermw_output_async(output, response))
 
     async def _process_spidermw_output_async(
-        self, output: Any, response: Response
+        self, output: Any, response: Response | Failure
     ) -> None:
         """Process each Request/Item (given in the output parameter) returned
         from the given spider.
@@ -385,7 +388,9 @@ class Scraper:
         if output is not None:
             await self.start_itemproc_async(output, response=response)
 
-    def start_itemproc(self, item: Any, *, response: Response | None) -> Deferred[None]:
+    def start_itemproc(
+        self, item: Any, *, response: Response | Failure | None
+    ) -> Deferred[None]:
         """Send *item* to the item pipelines for processing.
 
         *response* is the source of the item data. If the item does not come
@@ -394,7 +399,7 @@ class Scraper:
         return deferred_from_coro(self.start_itemproc_async(item, response=response))
 
     async def start_itemproc_async(
-        self, item: Any, *, response: Response | None
+        self, item: Any, *, response: Response | Failure | None
     ) -> None:
         """Send *item* to the item pipelines for processing.
 
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 4f08918ae..e81a9ec93 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -116,7 +116,7 @@ class LogFormatter:
         self,
         item: Any,
         exception: BaseException,
-        response: Response | None,
+        response: Response | Failure | None,
         spider: Spider,
     ) -> LogFormatterResult:
         """Logs a message when an item is dropped while it is passing through the item pipeline."""
@@ -137,7 +137,7 @@ class LogFormatter:
         self,
         item: Any,
         exception: BaseException,
-        response: Response | None,
+        response: Response | Failure | None,
         spider: Spider,
     ) -> LogFormatterResult:
         """Logs a message when an item causes an error while it is passing


Test patch:
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index b90706027..8289b2243 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -5,6 +5,7 @@ import logging
 import unittest
 from ipaddress import IPv4Address
 from socket import gethostbyname
+from typing import Any
 from urllib.parse import urlparse
 
 import pytest
@@ -419,6 +420,8 @@ with multiples lines
 
 
 class TestCrawlSpider(TestCase):
+    mockserver: MockServer
+
     @classmethod
     def setUpClass(cls):
         cls.mockserver = MockServer()
@@ -756,6 +759,34 @@ class TestCrawlSpider(TestCase):
             )
         assert "Spider error processing" in str(log)
 
+    @defer.inlineCallbacks
+    def test_spider_errback_item(self):
+        def eb(failure: Failure) -> Any:
+            return {"foo": "bar"}
+
+        crawler = get_crawler(SingleRequestSpider)
+        with LogCapture() as log:
+            yield crawler.crawl(
+                seed=self.mockserver.url("/status?n=400"), errback_func=eb
+            )
+        assert "HTTP status code is not handled or not allowed" not in str(log)
+        assert "Spider error processing" not in str(log)
+        assert "'item_scraped_count': 1" in str(log)
+
+    @defer.inlineCallbacks
+    def test_spider_errback_request(self):
+        def eb(failure: Failure) -> Request:
+            return Request(self.mockserver.url("/"))
+
+        crawler = get_crawler(SingleRequestSpider)
+        with LogCapture() as log:
+            yield crawler.crawl(
+                seed=self.mockserver.url("/status?n=400"), errback_func=eb
+            )
+        assert "HTTP status code is not handled or not allowed" not in str(log)
+        assert "Spider error processing" not in str(log)
+        assert "Crawled (200)" in str(log)
+
     @defer.inlineCallbacks
     def test_spider_errback_downloader_error(self):
         failures = []
@@ -774,7 +805,7 @@ class TestCrawlSpider(TestCase):
         assert "Spider error processing" not in str(log)
 
     @defer.inlineCallbacks
-    def test_spider_errback_exception_downloader_error(self):
+    def test_spider_errback_downloader_error_exception(self):
         def eb(failure: Failure) -> None:
             raise ValueError("foo")
 
@@ -786,6 +817,34 @@ class TestCrawlSpider(TestCase):
         assert "Error downloading" in str(log)
         assert "Spider error processing" in str(log)
 
+    @defer.inlineCallbacks
+    def test_spider_errback_downloader_error_item(self):
+        def eb(failure: Failure) -> Any:
+            return {"foo": "bar"}
+
+        crawler = get_crawler(SingleRequestSpider)
+        with LogCapture() as log:
+            yield crawler.crawl(
+                seed=self.mockserver.url("/drop?abort=1"), errback_func=eb
+            )
+        assert "HTTP status code is not handled or not allowed" not in str(log)
+        assert "Spider error processing" not in str(log)
+        assert "'item_scraped_count': 1" in str(log)
+
+    @defer.inlineCallbacks
+    def test_spider_errback_downloader_error_request(self):
+        def eb(failure: Failure) -> Request:
+            return Request(self.mockserver.url("/"))
+
+        crawler = get_crawler(SingleRequestSpider)
+        with LogCapture() as log:
+            yield crawler.crawl(
+                seed=self.mockserver.url("/drop?abort=1"), errback_func=eb
+            )
+        assert "HTTP status code is not handled or not allowed" not in str(log)
+        assert "Spider error processing" not in str(log)
+        assert "Crawled (200)" in str(log)
+
     @defer.inlineCallbacks
     def test_raise_closespider(self):
         def cb(response):