Repository: scrapy/scrapy
Pull request #5889 (2023-04-08 17:47:30)

Typing improvements for some core components


Patch:
diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst
index d01e0a8d4..a0f4b4411 100644
--- a/docs/topics/dynamic-content.rst
+++ b/docs/topics/dynamic-content.rst
@@ -276,7 +276,7 @@ The following is a simple snippet to illustrate its usage within a Scrapy spider
             async with async_playwright() as pw:
                 browser = await pw.chromium.launch()
                 page = await browser.new_page()
-                await page.goto("https:/example.org")
+                await page.goto("https://example.org")
                 title = await page.title()
                 return {"title": title}
 
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 1b83c3a8a..dde76a547 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -4,8 +4,9 @@ from datetime import datetime
 from time import time
 
 from twisted.internet import defer, task
+from twisted.internet.defer import Deferred
 
-from scrapy import signals
+from scrapy import Request, Spider, signals
 from scrapy.core.downloader.handlers import DownloadHandlers
 from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
 from scrapy.resolver import dnscache
@@ -86,7 +87,7 @@ class Downloader:
         self._slot_gc_loop.start(60)
         self.per_slot_settings = self.settings.getdict("DOWNLOAD_SLOTS", {})
 
-    def fetch(self, request, spider):
+    def fetch(self, request: Request, spider: Spider) -> Deferred:
         def _deactivate(response):
             self.active.remove(request)
             return response
@@ -206,12 +207,12 @@ class Downloader:
 
         return dfd.addBoth(finish_transferring)
 
-    def close(self):
+    def close(self) -> None:
         self._slot_gc_loop.stop()
         for slot in self.slots.values():
             slot.close()
 
-    def _slot_gc(self, age=60):
+    def _slot_gc(self, age: float = 60) -> None:
         mintime = time() - age
         for key, slot in list(self.slots.items()):
             if not slot.active and slot.lastseen + slot.delay < mintime:
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index c6738b531..3e5a281b2 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -8,13 +8,16 @@ import logging
 import warnings
 from time import time
 from typing import (
+    TYPE_CHECKING,
     Any,
     Callable,
     Generator,
     Iterable,
     Iterator,
+    List,
     Optional,
     Set,
+    Type,
     Union,
     cast,
 )
@@ -28,30 +31,36 @@ from scrapy.core.downloader import Downloader
 from scrapy.core.scraper import Scraper
 from scrapy.exceptions import CloseSpider, DontCloseSpider, ScrapyDeprecationWarning
 from scrapy.http import Request, Response
-from scrapy.settings import BaseSettings
+from scrapy.logformatter import LogFormatter
+from scrapy.settings import BaseSettings, Settings
+from scrapy.signalmanager import SignalManager
 from scrapy.spiders import Spider
 from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.reactor import CallLaterOnce
 
+if TYPE_CHECKING:
+    from scrapy.core.scheduler import BaseScheduler
+    from scrapy.crawler import Crawler
+
 logger = logging.getLogger(__name__)
 
 
 class Slot:
     def __init__(
         self,
-        start_requests: Iterable,
+        start_requests: Iterable[Request],
         close_if_idle: bool,
         nextcall: CallLaterOnce,
-        scheduler,
+        scheduler: "BaseScheduler",
     ) -> None:
         self.closing: Optional[Deferred] = None
         self.inprogress: Set[Request] = set()
-        self.start_requests: Optional[Iterator] = iter(start_requests)
-        self.close_if_idle = close_if_idle
-        self.nextcall = nextcall
-        self.scheduler = scheduler
-        self.heartbeat = LoopingCall(nextcall.schedule)
+        self.start_requests: Optional[Iterator[Request]] = iter(start_requests)
+        self.close_if_idle: bool = close_if_idle
+        self.nextcall: CallLaterOnce = nextcall
+        self.scheduler: "BaseScheduler" = scheduler
+        self.heartbeat: LoopingCall = LoopingCall(nextcall.schedule)
 
     def add_request(self, request: Request) -> None:
         self.inprogress.add(request)
@@ -75,25 +84,28 @@ class Slot:
 
 
 class ExecutionEngine:
-    def __init__(self, crawler, spider_closed_callback: Callable) -> None:
-        self.crawler = crawler
-        self.settings = crawler.settings
-        self.signals = crawler.signals
-        self.logformatter = crawler.logformatter
+    def __init__(self, crawler: "Crawler", spider_closed_callback: Callable) -> None:
+        self.crawler: "Crawler" = crawler
+        self.settings: Settings = crawler.settings
+        self.signals: SignalManager = crawler.signals
+        self.logformatter: LogFormatter = crawler.logformatter
         self.slot: Optional[Slot] = None
         self.spider: Optional[Spider] = None
-        self.running = False
-        self.paused = False
-        self.scheduler_cls = self._get_scheduler_class(crawler.settings)
-        downloader_cls = load_object(self.settings["DOWNLOADER"])
+        self.running: bool = False
+        self.paused: bool = False
+        self.scheduler_cls: Type["BaseScheduler"] = self._get_scheduler_class(
+            crawler.settings
+        )
+        downloader_cls: Type[Downloader] = load_object(self.settings["DOWNLOADER"])
         self.downloader: Downloader = downloader_cls(crawler)
         self.scraper = Scraper(crawler)
-        self._spider_closed_callback = spider_closed_callback
+        self._spider_closed_callback: Callable = spider_closed_callback
+        self.start_time: Optional[float] = None
 
-    def _get_scheduler_class(self, settings: BaseSettings) -> type:
+    def _get_scheduler_class(self, settings: BaseSettings) -> Type["BaseScheduler"]:
         from scrapy.core.scheduler import BaseScheduler
 
-        scheduler_cls = load_object(settings["SCHEDULER"])
+        scheduler_cls: Type = load_object(settings["SCHEDULER"])
         if not issubclass(scheduler_cls, BaseScheduler):
             raise TypeError(
                 f"The provided scheduler class ({settings['SCHEDULER']})"
@@ -115,7 +127,7 @@ class ExecutionEngine:
         """Gracefully stop the execution engine"""
 
         @inlineCallbacks
-        def _finish_stopping_engine(_) -> Generator[Deferred, Any, None]:
+        def _finish_stopping_engine(_: Any) -> Generator[Deferred, Any, None]:
             yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped)
             self._closewait.callback(None)
 
@@ -141,7 +153,8 @@ class ExecutionEngine:
             return self.close_spider(
                 self.spider, reason="shutdown"
             )  # will also close downloader
-        return succeed(self.downloader.close())
+        self.downloader.close()
+        return succeed(None)
 
     def pause(self) -> None:
         self.paused = True
@@ -209,7 +222,7 @@ class ExecutionEngine:
                 extra={"spider": self.spider},
             )
         )
-        d.addBoth(lambda _: cast(Slot, self.slot).remove_request(request))
+        d.addBoth(lambda _: cast(Slot, self.slot).remove_request(request))  # type: ignore[arg-type]
         d.addErrback(
             lambda f: logger.info(
                 "Error while removing request from slot",
@@ -339,6 +352,7 @@ class ExecutionEngine:
             if isinstance(result, Response):
                 if result.request is None:
                     result.request = request
+                assert spider is not None
                 logkws = self.logformatter.crawled(result.request, result, spider)
                 if logkws is not None:
                     logger.log(*logformatter_adapter(logkws), extra={"spider": spider})
@@ -350,10 +364,12 @@ class ExecutionEngine:
                 )
             return result
 
-        def _on_complete(_):
+        def _on_complete(_: Any) -> Any:
+            assert self.slot is not None
             self.slot.nextcall.schedule()
             return _
 
+        assert spider is not None
         dwld = self.downloader.fetch(request, spider)
         dwld.addCallbacks(_on_success)
         dwld.addBoth(_on_complete)
@@ -362,7 +378,7 @@ class ExecutionEngine:
     @inlineCallbacks
     def open_spider(
         self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True
-    ):
+    ) -> Generator[Deferred, Any, None]:
         if self.slot is not None:
             raise RuntimeError(f"No free spider slot when opening {spider.name!r}")
         logger.info("Spider opened", extra={"spider": spider})
@@ -471,7 +487,7 @@ class ExecutionEngine:
         return dfd
 
     @property
-    def open_spiders(self) -> list:
+    def open_spiders(self) -> List[Spider]:
         warnings.warn(
             "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead",
             category=ScrapyDeprecationWarning,
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index 3c46e3a5f..3fb0bbaff 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -2,13 +2,15 @@ import json
 import logging
 from abc import abstractmethod
 from pathlib import Path
-from typing import Optional, Type, TypeVar
+from typing import Any, Optional, Type, TypeVar, cast
 
 from twisted.internet.defer import Deferred
 
 from scrapy.crawler import Crawler
+from scrapy.dupefilters import BaseDupeFilter
 from scrapy.http.request import Request
 from scrapy.spiders import Spider
+from scrapy.statscollectors import StatsCollector
 from scrapy.utils.job import job_dir
 from scrapy.utils.misc import create_instance, load_object
 
@@ -20,10 +22,10 @@ class BaseSchedulerMeta(type):
     Metaclass to check scheduler classes against the necessary interface
     """
 
-    def __instancecheck__(cls, instance):
+    def __instancecheck__(cls, instance: Any) -> bool:
         return cls.__subclasscheck__(type(instance))
 
-    def __subclasscheck__(cls, subclass):
+    def __subclasscheck__(cls, subclass: type) -> bool:
         return (
             hasattr(subclass, "has_pending_requests")
             and callable(subclass.has_pending_requests)
@@ -168,26 +170,26 @@ class Scheduler(BaseScheduler):
 
     def __init__(
         self,
-        dupefilter,
+        dupefilter: BaseDupeFilter,
         jobdir: Optional[str] = None,
         dqclass=None,
         mqclass=None,
         logunser: bool = False,
-        stats=None,
+        stats: Optional[StatsCollector] = None,
         pqclass=None,
         crawler: Optional[Crawler] = None,
     ):
-        self.df = dupefilter
-        self.dqdir = self._dqdir(jobdir)
+        self.df: BaseDupeFilter = dupefilter
+        self.dqdir: Optional[str] = self._dqdir(jobdir)
         self.pqclass = pqclass
         self.dqclass = dqclass
         self.mqclass = mqclass
-        self.logunser = logunser
-        self.stats = stats
-        self.crawler = crawler
+        self.logunser: bool = logunser
+        self.stats: Optional[StatsCollector] = stats
+        self.crawler: Optional[Crawler] = crawler
 
     @classmethod
-    def from_crawler(cls: Type[SchedulerTV], crawler) -> SchedulerTV:
+    def from_crawler(cls: Type[SchedulerTV], crawler: Crawler) -> SchedulerTV:
         """
         Factory method, initializes the scheduler with arguments taken from the crawl settings
         """
@@ -242,6 +244,7 @@ class Scheduler(BaseScheduler):
             self.df.log(request, self.spider)
             return False
         dqok = self._dqpush(request)
+        assert self.stats is not None
         if dqok:
             self.stats.inc_value("scheduler/enqueued/disk", spider=self.spider)
         else:
@@ -259,7 +262,8 @@ class Scheduler(BaseScheduler):
         Increment the appropriate stats, such as: ``scheduler/dequeued``,
         ``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``.
         """
-        request = self.mqs.pop()
+        request: Optional[Request] = self.mqs.pop()
+        assert self.stats is not None
         if request is not None:
             self.stats.inc_value("scheduler/dequeued/memory", spider=self.spider)
         else:
@@ -295,6 +299,7 @@ class Scheduler(BaseScheduler):
                     extra={"spider": self.spider},
                 )
                 self.logunser = False
+            assert self.stats is not None
             self.stats.inc_value("scheduler/unserializable", spider=self.spider)
             return False
         else:
@@ -351,7 +356,7 @@ class Scheduler(BaseScheduler):
         if not path.exists():
             return []
         with path.open(encoding="utf-8") as f:
-            return json.load(f)
+            return cast(list, json.load(f))
 
     def _write_dqs_state(self, dqdir: str, state: list) -> None:
         with Path(dqdir, "active.json").open("w", encoding="utf-8") as f:
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 8468b8419..a85f6a661 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -15,6 +15,7 @@ from typing import (
     Optional,
     Set,
     Tuple,
+    Type,
     Union,
 )
 
@@ -26,6 +27,9 @@ from scrapy import Spider, signals
 from scrapy.core.spidermw import SpiderMiddlewareManager
 from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
 from scrapy.http import Request, Response
+from scrapy.logformatter import LogFormatter
+from scrapy.pipelines import ItemPipelineManager
+from scrapy.signalmanager import SignalManager
 from scrapy.utils.defer import (
     aiter_errback,
     defer_fail,
@@ -96,16 +100,20 @@ class Slot:
 class Scraper:
     def __init__(self, crawler: Crawler) -> None:
         self.slot: Optional[Slot] = None
-        self.spidermw = SpiderMiddlewareManager.from_crawler(crawler)
-        itemproc_cls = load_object(crawler.settings["ITEM_PROCESSOR"])
-        self.itemproc = itemproc_cls.from_crawler(crawler)
-        self.concurrent_items = crawler.settings.getint("CONCURRENT_ITEMS")
-        self.crawler = crawler
-        self.signals = crawler.signals
-        self.logformatter = crawler.logformatter
+        self.spidermw: SpiderMiddlewareManager = SpiderMiddlewareManager.from_crawler(
+            crawler
+        )
+        itemproc_cls: Type[ItemPipelineManager] = load_object(
+            crawler.settings["ITEM_PROCESSOR"]
+        )
+        self.itemproc: ItemPipelineManager = itemproc_cls.from_crawler(crawler)
+        self.concurrent_items: int = crawler.settings.getint("CONCURRENT_ITEMS")
+        self.crawler: Crawler = crawler
+        self.signals: SignalManager = crawler.signals
+        self.logformatter: LogFormatter = crawler.logformatter
 
     @inlineCallbacks
-    def open_spider(self, spider: Spider):
+    def open_spider(self, spider: Spider) -> Generator[Deferred, Any, None]:
         """Open the given spider for scraping and allocate resources for it"""
         self.slot = Slot(self.crawler.settings.getint("SCRAPER_SLOT_MAX_ACTIVE_SIZE"))
         yield self.itemproc.open_spider(spider)
@@ -135,7 +143,8 @@ class Scraper:
             raise RuntimeError("Scraper slot not assigned")
         dfd = self.slot.add_response_request(result, request)
 
-        def finish_scraping(_):
+        def finish_scraping(_: Any) -> Any:
+            assert self.slot is not None
             self.slot.finish_response(result, request)
             self._check_if_closing(spider)
             self._scrape_next(spider)
@@ -205,9 +214,9 @@ class Scraper:
         else:  # result is a Failure
             # TODO: properly type adding this attribute to a Failure
             result.request = request  # type: ignore[attr-defined]
-            warn_on_generator_with_return_value(spider, request.errback)
             dfd = defer_fail(result)
             if request.errback:
+                warn_on_generator_with_return_value(spider, request.errback)
                 dfd.addErrback(request.errback)
         return dfd.addCallback(iterate_spider_output)
 
@@ -338,7 +347,7 @@ class Scraper:
 
     def _itemproc_finished(
         self, output: Any, item: Any, response: Response, spider: Spider
-    ) -> None:
+    ) -> Deferred:
         """ItemProcessor finished for the given ``item`` and returned ``output``"""
         assert self.slot is not None  # typing
         self.slot.itemproc_size -= 1
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index c21985b18..dcf1a6dbc 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -13,6 +13,8 @@ from typing import (
     Callable,
     Generator,
     Iterable,
+    List,
+    Optional,
     Tuple,
     Union,
     cast,
@@ -25,6 +27,7 @@ from scrapy import Request, Spider
 from scrapy.exceptions import _InvalidOutput
 from scrapy.http import Response
 from scrapy.middleware import MiddlewareManager
+from scrapy.settings import BaseSettings
 from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
 from scrapy.utils.conf import build_component_list
 from scrapy.utils.defer import (
@@ -41,22 +44,22 @@ logger = logging.getLogger(__name__)
 ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]
 
 
-def _isiterable(o) -> bool:
+def _isiterable(o: Any) -> bool:
     return isinstance(o, (Iterable, AsyncIterable))
 
 
 class SpiderMiddlewareManager(MiddlewareManager):
     component_name = "spider middleware"
 
-    def __init__(self, *middlewares):
+    def __init__(self, *middlewares: Any):
         super().__init__(*middlewares)
         self.downgrade_warning_done = False
 
     @classmethod
-    def _get_mwlist_from_settings(cls, settings):
+    def _get_mwlist_from_settings(cls, settings: BaseSettings) -> List[Any]:
         return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES"))
 
-    def _add_middleware(self, mw):
+    def _add_middleware(self, mw: Any) -> None:
         super()._add_middleware(mw)
         if hasattr(mw, "process_spider_input"):
             self.methods["process_spider_input"].append(mw.process_spider_input)
@@ -98,7 +101,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
         exception_processor_index: int,
         recover_to: Union[MutableChain, MutableAsyncChain],
     ) -> Union[Generator, AsyncGenerator]:
-        def process_sync(iterable: Iterable):
+        def process_sync(iterable: Iterable) -> Generator:
             try:
                 for r in iterable:
                     yield r
@@ -110,7 +113,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                     raise
                 recover_to.extend(exception_result)
 
-        async def process_async(iterable: AsyncIterable):
+        async def process_async(iterable: AsyncIterable) -> AsyncGenerator:
             try:
                 async for r in iterable:
                     yield r
@@ -280,7 +283,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
         if isinstance(recovered, AsyncIterable):
             recovered_collected = await collect_asyncgen(recovered)
             recovered = MutableChain(recovered_collected)
-        return MutableChain(result, recovered)  # type: ignore[arg-type]
+        return MutableChain(result, recovered)
 
     def scrape_response(
         self,
@@ -306,7 +309,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
         )
         return dfd
 
-    def process_start_requests(self, start_requests, spider: Spider) -> Deferred:
+    def process_start_requests(
+        self, start_requests: Iterable[Request], spider: Spider
+    ) -> Deferred:
         return self._process_chain("process_start_requests", start_requests, spider)
 
     # This method is only needed until _async compatibility methods are removed.
@@ -314,9 +319,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
     def _get_async_method_pair(
         mw: Any, methodname: str
     ) -> Union[None, Callable, Tuple[Callable, Callable]]:
-        normal_method = getattr(mw, methodname, None)
+        normal_method: Optional[Callable] = getattr(mw, methodname, None)
         methodname_async = methodname + "_async"
-        async_method = getattr(mw, methodname_async, None)
+        async_method: Optional[Callable] = getattr(mw, methodname_async, None)
         if not async_method:
             return normal_method
         if not normal_method:
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 397817d6f..256f6e2c5 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -4,7 +4,7 @@ import logging
 import pprint
 import signal
 import warnings
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING, Optional, Type, Union
 
 from twisted.internet import defer
 from zope.interface.exceptions import DoesNotImplement
@@ -22,8 +22,10 @@ from scrapy.core.engine import ExecutionEngine
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.extension import ExtensionManager
 from scrapy.interfaces import ISpiderLoader
+from scrapy.logformatter import LogFormatter
 from scrapy.settings import Settings, overridden_settings
 from scrapy.signalmanager import SignalManager
+from scrapy.statscollectors import StatsCollector
 from scrapy.utils.log import (
     LogCounterHandler,
     configure_logging,
@@ -49,20 +51,25 @@ logger = logging.getLogger(__name__)
 
 
 class Crawler:
-    def __init__(self, spidercls, settings=None, init_reactor: bool = False):
+    def __init__(
+        self,
+        spidercls: Type[Spider],
+        settings: Union[None, dict, Settings] = None,
+        init_reactor: bool = False,
+    ):
         if isinstance(spidercls, Spider):
             raise ValueError("The spidercls argument must be a class, not an object")
 
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
 
-        self.spidercls = spidercls
-        self.settings = settings.copy()
+        self.spidercls: Type[Spider] = spidercls
+        self.settings: Settings = settings.copy()
         self.spidercls.update_settings(self.settings)
 
-        self.signals = SignalManager(self)
+        self.signals: SignalManager = SignalManager(self)
 
-        self.stats = load_object(self.settings["STATS_CLASS"])(self)
+        self.stats: StatsCollector = load_object(self.settings["STATS_CLASS"])(self)
 
         handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL"))
         logging.root.addHandler(handler)
@@ -80,8 +87,8 @@ class Crawler:
         self.__remove_handler = lambda: logging.root.removeHandler(handler)
         self.signals.connect(self.__remove_handler, signals.engine_stopped)
 
-        lf_cls = load_object(self.settings["LOG_FORMATTER"])
-        self.logformatter = lf_cls.from_crawler(self)
+        lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])
+        self.logformatter: LogFormatter = lf_cls.from_crawler(self)
 
         self.request_fingerprinter: RequestFingerprinter = create_instance(
             load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]),
@@ -89,8 +96,8 @@ class Crawler:
             crawler=self,
         )
 
-        reactor_class = self.settings["TWISTED_REACTOR"]
-        event_loop = self.settings["ASYNCIO_EVENT_LOOP"]
+        reactor_class: str = self.settings["TWISTED_REACTOR"]
+        event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"]
         if init_reactor:
             # this needs to be done after the spider settings are merged,
             # but before something imports twisted.internet.reactor
@@ -104,11 +111,11 @@ class Crawler:
             if is_asyncio_reactor_installed() and event_loop:
                 verify_installed_asyncio_event_loop(event_loop)
 
-        self.extensions = ExtensionManager.from_crawler(self)
+        self.extensions: ExtensionManager = ExtensionManager.from_crawler(self)
 
         self.settings.freeze()
-        self.crawling = False
-        self.spider = None
+        self.crawling: bool = False
+        self.spider: Optional[Spider] = None
         self.engine: Optional[ExecutionEngine] = None
 
     @defer.inlineCallbacks
diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py
index 7f202b8b8..fedd02805 100644
--- a/scrapy/exceptions.py
+++ b/scrapy/exceptions.py
@@ -39,7 +39,7 @@ class DontCloseSpider(Exception):
 class CloseSpider(Exception):
     """Raise this from callbacks to request the spider to be closed"""
 
-    def __init__(self, reason="cancelled"):
+    def __init__(self, reason: str = "cancelled"):
         super().__init__()
         self.reason = reason
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index cd26b5778..da1a88299 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -383,13 +383,20 @@ class FeedExporter:
         return defer.DeferredList(deferred_list) if deferred_list else None
 
     def _close_slot(self, slot, spider):
+        def get_file(slot_):
+            if isinstance(slot_.file, PostProcessingManager):
+                slot_.file.close()
+                return slot_.file.file
+            return slot_.file
+
         slot.finish_exporting()
         if not slot.itemcount and not slot.store_empty:
             # We need to call slot.storage.store nonetheless to get the file
             # properly closed.
-            return defer.maybeDeferred(slot.storage.store, slot.file)
+            return defer.maybeDeferred(slot.storage.store, get_file(slot))
+
         logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"
-        d = defer.maybeDeferred(slot.storage.store, slot.file)
+        d = defer.maybeDeferred(slot.storage.store, get_file(slot))
 
         d.addCallback(
             self._handle_store_success, logmsg, spider, type(slot.storage).__name__
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 560006c95..7cb379b46 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -1,8 +1,11 @@
 import logging
 import os
+from typing import Any, Dict, Optional, Union
 
 from twisted.python.failure import Failure
 
+from scrapy import Request, Spider
+from scrapy.http import Response
 from scrapy.utils.request import referer_str
 
 SCRAPEDMSG = "Scraped from %(src)s" + os.linesep + "%(item)s"
@@ -52,7 +55,7 @@ class LogFormatter:
                     }
     """
 
-    def crawled(self, request, response, spider):
+    def crawled(self, request: Request, response: Response, spider: Spider) -> dict:
         """Logs a message when the crawler finds a webpage."""
         request_flags = f" {str(request.flags)}" if request.flags else ""
         response_flags = f" {str(response.flags)}" if response.flags else ""
@@ -70,8 +73,11 @@ class LogFormatter:
             },
         }
 
-    def scraped(self, item, response, spider):
+    def scraped(
+        self, item: Any, response: Union[Response, Failure], spider: Spider
+    ) -> dict:
         """Logs a message when an item is scraped by a spider."""
+        src: Any
         if isinstance(response, Failure):
             src = response.getErrorMessage()
         else:
@@ -85,7 +91,9 @@ class LogFormatter:
             },
         }
 
-    def dropped(self, item, exception, response, spider):
+    def dropped(
+        self, item: Any, exception: BaseException, response: Response, spider: Spider
+    ) -> dict:
         """Logs a message when an item is dropped while it is passing through the item pipeline."""
         return {
             "level": logging.WARNING,
@@ -96,7 +104,9 @@ class LogFormatter:
             },
         }
 
-    def item_error(self, item, exception, response, spider):
+    def item_error(
+        self, item: Any, exception, response: Response, spider: Spider
+    ) -> dict:
         """Logs a message when an item causes an error while it is passing
         through the item pipeline.
 
@@ -110,7 +120,9 @@ class LogFormatter:
             },
         }
 
-    def spider_error(self, failure, request, response, spider):
+    def spider_error(
+        self, failure: Failure, request: Request, response: Response, spider: Spider
+    ) -> dict:
         """Logs an error message from a spider.
 
         .. versionadded:: 2.0
@@ -124,13 +136,19 @@ class LogFormatter:
             },
         }
 
-    def download_error(self, failure, request, spider, errmsg=None):
+    def download_error(
+        self,
+        failure: Failure,
+        request: Request,
+        spider: Spider,
+        errmsg: Optional[str] = None,
+    ) -> dict:
         """Logs a download error message from a spider (typically coming from
         the engine).
 
         .. versionadded:: 2.0
         """
-        args = {"request": request}
+        args: Dict[str, Any] = {"request": request}
         if errmsg:
             msg = DOWNLOADERRORMSG_LONG
             args["errmsg"] = errmsg
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index f82d722fa..03e92b565 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -1,7 +1,7 @@
 import logging
 import pprint
 from collections import defaultdict, deque
-from typing import Any, Callable, Deque, Dict, Iterable, Tuple, Union, cast
+from typing import Any, Callable, Deque, Dict, Iterable, List, Tuple, Union, cast
 
 from twisted.internet.defer import Deferred
 
@@ -30,7 +30,7 @@ class MiddlewareManager:
             self._add_middleware(mw)
 
     @classmethod
-    def _get_mwlist_from_settings(cls, settings: Settings) -> list:
+    def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]:
         raise NotImplementedError
 
     @classmethod
@@ -67,17 +67,17 @@ class MiddlewareManager:
     def from_crawler(cls, crawler):
         return cls.from_settings(crawler.settings, crawler)
 
-    def _add_middleware(self, mw) -> None:
+    def _add_middleware(self, mw: Any) -> None:
         if hasattr(mw, "open_spider"):
             self.methods["open_spider"].append(mw.open_spider)
         if hasattr(mw, "close_spider"):
             self.methods["close_spider"].appendleft(mw.close_spider)
 
-    def _process_parallel(self, methodname: str, obj, *args) -> Deferred:
+    def _process_parallel(self, methodname: str, obj: Any, *args: Any) -> Deferred:
         methods = cast(Iterable[Callable], self.methods[methodname])
         return process_parallel(methods, obj, *args)
 
-    def _process_chain(self, methodname: str, obj, *args) -> Deferred:
+    def _process_chain(self, methodname: str, obj: Any, *args: Any) -> Deferred:
         methods = cast(Iterable[Callable], self.methods[methodname])
         return process_chain(methods, obj, *args)
 
diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py
index df574a0a1..c97d71fb6 100644
--- a/scrapy/pipelines/__init__.py
+++ b/scrapy/pipelines/__init__.py
@@ -3,7 +3,11 @@ Item pipeline
 
 See documentation in docs/item-pipeline.rst
 """
+from typing import Any, List
 
+from twisted.internet.defer import Deferred
+
+from scrapy import Spider
 from scrapy.middleware import MiddlewareManager
 from scrapy.utils.conf import build_component_list
 from scrapy.utils.defer import deferred_f_from_coro_f
@@ -13,15 +17,15 @@ class ItemPipelineManager(MiddlewareManager):
     component_name = "item pipeline"
 
     @classmethod
-    def _get_mwlist_from_settings(cls, settings):
+    def _get_mwlist_from_settings(cls, settings) -> List[Any]:
         return build_component_list(settings.getwithbase("ITEM_PIPELINES"))
 
-    def _add_middleware(self, pipe):
+    def _add_middleware(self, pipe: Any) -> None:
         super()._add_middleware(pipe)
         if hasattr(pipe, "process_item"):
             self.methods["process_item"].append(
                 deferred_f_from_coro_f(pipe.process_item)
             )
 
-    def process_item(self, item, spider):
+    def process_item(self, item: Any, spider: Spider) -> Deferred:
         return self._process_chain("process_item", item, spider)
diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py
index d7e3bce91..f6df191d8 100644
--- a/scrapy/signalmanager.py
+++ b/scrapy/signalmanager.py
@@ -1,13 +1,16 @@
+from typing import Any, List, Tuple
+
 from pydispatch import dispatcher
+from twisted.internet.defer import Deferred
 
 from scrapy.utils import signal as _signal
 
 
 class SignalManager:
-    def __init__(self, sender=dispatcher.Anonymous):
-        self.sender = sender
+    def __init__(self, sender: Any = dispatcher.Anonymous):
+        self.sender: Any = sender
 
-    def connect(self, receiver, signal, **kwargs):
+    def connect(self, receiver: Any, signal: Any, **kwargs: Any) -> None:
         """
         Connect a receiver function to a signal.
 
@@ -22,18 +25,18 @@ class SignalManager:
         :type signal: object
         """
         kwargs.setdefault("sender", self.sender)
-        return dispatcher.connect(receiver, signal, **kwargs)
+        dispatcher.connect(receiver, signal, **kwargs)
 
-    def disconnect(self, receiver, signal, **kwargs):
+    def disconnect(self, receiver: Any, signal: Any, **kwargs: Any) -> None:
         """
         Disconnect a receiver function from a signal. This has the
         opposite effect of the :meth:`connect` method, and the arguments
         are the same.
         """
         kwargs.setdefault("sender", self.sender)
-        return dispatcher.disconnect(receiver, signal, **kwargs)
+        dispatcher.disconnect(receiver, signal, **kwargs)
 
-    def send_catch_log(self, signal, **kwargs):
+    def send_catch_log(self, signal: Any, **kwargs: Any) -> List[Tuple[Any, Any]]:
         """
         Send a signal, catch exceptions and log them.
 
@@ -43,7 +46,7 @@ class SignalManager:
         kwargs.setdefault("sender", self.sender)
         return _signal.send_catch_log(signal, **kwargs)
 
-    def send_catch_log_deferred(self, signal, **kwargs):
+    def send_catch_log_deferred(self, signal: Any, **kwargs: Any) -> Deferred:
         """
         Like :meth:`send_catch_log` but supports returning
         :class:`~twisted.internet.defer.Deferred` objects from signal handlers.
@@ -57,7 +60,7 @@ class SignalManager:
         kwargs.setdefault("sender", self.sender)
         return _signal.send_catch_log_deferred(signal, **kwargs)
 
-    def disconnect_all(self, signal, **kwargs):
+    def disconnect_all(self, signal: Any, **kwargs: Any) -> None:
         """
         Disconnect all receivers from the given signal.
 
@@ -65,4 +68,4 @@ class SignalManager:
         :type signal: object
         """
         kwargs.setdefault("sender", self.sender)
-        return _signal.disconnect_all(signal, **kwargs)
+        _signal.disconnect_all(signal, **kwargs)
diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py
index dd3c32737..15193aac5 100644
--- a/scrapy/statscollectors.py
+++ b/scrapy/statscollectors.py
@@ -3,44 +3,57 @@ Scrapy extension for collecting scraping stats
 """
 import logging
 import pprint
+from typing import TYPE_CHECKING, Any, Dict, Optional
+
+from scrapy import Spider
+
+if TYPE_CHECKING:
+    from scrapy.crawler import Crawler
 
 logger = logging.getLogger(__name__)
 
 
+StatsT = Dict[str, Any]
+
+
 class StatsCollector:
-    def __init__(self, crawler):
-        self._dump = crawler.settings.getbool("STATS_DUMP")
-        self._stats = {}
+    def __init__(self, crawler: "Crawler"):
+        self._dump: bool = crawler.settings.getbool("STATS_DUMP")
+        self._stats: StatsT = {}
 
-    def get_value(self, key, default=None, spider=None):
+    def get_value(
+        self, key: str, default: Any = None, spider: Optional[Spider] = None
+    ) -> Any:
         return self._stats.get(key, default)
 
-    def get_stats(self, spider=None):
+    def get_stats(self, spider: Optional[Spider] = None) -> StatsT:
         return self._stats
 
-    def set_value(self, key, value, spider=None):
+    def set_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None:
         self._stats[key] = value
 
-    def set_stats(self, stats, spider=None):
+    def set_stats(self, stats: StatsT, spider: Optional[Spider] = None) -> None:
         self._stats = stats
 
-    def inc_value(self, key, count=1, start=0, spider=None):
+    def inc_value(
+        self, key: str, count: int = 1, start: int = 0, spider: Optional[Spider] = None
+    ) -> None:
         d = self._stats
         d[key] = d.setdefault(key, start) + count
 
-    def max_value(self, key, value, spider=None):
+    def max_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None:
         self._stats[key] = max(self._stats.setdefault(key, value), value)
 
-    def min_value(self, key, value, spider=None):
+    def min_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None:
         self._stats[key] = min(self._stats.setdefault(key, value), value)
 
-    def clear_stats(self, spider=None):
+    def clear_stats(self, spider: Optional[Spider] = None) -> None:
         self._stats.clear()
 
-    def open_spider(self, spider):
+    def open_spider(self, spider: Spider) -> None:
         pass
 
-    def close_spider(self, spider, reason):
+    def close_spider(self, spider: Spider, reason: str) -> None:
         if self._dump:
             logger.info(
                 "Dumping Scrapy stats:\n" + pprint.pformat(self._stats),
@@ -48,34 +61,38 @@ class StatsCollector:
             )
         self._persist_stats(self._stats, spider)
 
-    def _persist_stats(self, stats, spider):
+    def _persist_stats(self, stats: StatsT, spider: Spider) -> None:
         pass
 
 
 class MemoryStatsCollector(StatsCollector):
-    def __init__(self, crawler):
+    def __init__(self, crawler: "Crawler"):
         super().__init__(crawler)
-        self.spider_stats = {}
+        self.spider_stats: Dict[str, StatsT] = {}
 
-    def _persist_stats(self, stats, spider):
+    def _persist_stats(self, stats: StatsT, spider: Spider) -> None:
         self.spider_stats[spider.name] = stats
 
 
 class DummyStatsCollector(StatsCollector):
-    def get_value(self, key, default=None, spider=None):
+    def get_value(
+        self, key: str, default: Any = None, spider: Optional[Spider] = None
+    ) -> Any:
         return default
 
-    def set_value(self, key, value, spider=None):
+    def set_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None:
         pass
 
-    def set_stats(self, stats, spider=None):
+    def set_stats(self, stats: StatsT, spider: Optional[Spider] = None) -> None:
         pass
 
-    def inc_value(self, key, count=1, start=0, spider=None):
+    def inc_value(
+        self, key: str, count: int = 1, start: int = 0, spider: Optional[Spider] = None
+    ) -> None:
         pass
 
-    def max_value(self, key, value, spider=None):
+    def max_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None:
         pass
 
-    def min_value(self, key, value, spider=None):
+    def min_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None:
         pass
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 6ae27dc29..2ce4725f4 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -2,6 +2,7 @@ import logging
 import sys
 import warnings
 from logging.config import dictConfig
+from typing import Tuple
 
 from twisted.python import log as twisted_log
 from twisted.python.failure import Failure
@@ -14,7 +15,7 @@ from scrapy.utils.versions import scrapy_components_versions
 logger = logging.getLogger(__name__)
 
 
-def failure_to_exc_info(failure):
+def failure_to_exc_info(failure: Failure):
     """Extract exc_info from Failure instances"""
     if isinstance(failure, Failure):
         return (failure.type, failure.value, failure.getTracebackObject())
@@ -206,7 +207,7 @@ class LogCounterHandler(logging.Handler):
         self.crawler.stats.inc_value(sname)
 
 
-def logformatter_adapter(logkws):
+def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]:
     """
     Helper that takes the dictionary output from the methods in LogFormatter
     and adapts it into a tuple of positional arguments for logger.log calls,
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index f9f9c0d5b..d861c9ab6 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -10,6 +10,7 @@ from contextlib import contextmanager
 from functools import partial
 from importlib import import_module
 from pkgutil import iter_modules
+from typing import TYPE_CHECKING, Any, Callable, Union
 
 from w3lib.html import replace_entities
 
@@ -18,6 +19,10 @@ from scrapy.utils.datatypes import LocalWeakReferencedCache
 from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.python import flatten, to_unicode
 
+if TYPE_CHECKING:
+    from scrapy import Spider
+
+
 _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes
 
 
@@ -34,7 +39,7 @@ def arg_to_iter(arg):
     return [arg]
 
 
-def load_object(path):
+def load_object(path: Union[str, Callable]) -> Any:
     """Load an object given its absolute object path, and return it.
 
     The object can be the import path of a class, function, variable or an
@@ -249,7 +254,7 @@ def is_generator_with_return_value(callable):
     return _generator_callbacks_cache[callable]
 
 
-def warn_on_generator_with_return_value(spider, callable):
+def warn_on_generator_with_return_value(spider: "Spider", callable: Callable) -> None:
     """
     Logs a warning if a callable is a generator function and includes
     a 'return' statement with a value different than None
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 7f67d036a..f1b9239e6 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -1,9 +1,11 @@
 import asyncio
 import sys
 from contextlib import suppress
+from typing import Any, Callable, Dict, Optional, Sequence
 from warnings import catch_warnings, filterwarnings, warn
 
 from twisted.internet import asyncioreactor, error
+from twisted.internet.base import DelayedCall
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.misc import load_object
@@ -34,23 +36,23 @@ class CallLaterOnce:
     it hasn't been already scheduled since the last time it ran.
     """
 
-    def __init__(self, func, *a, **kw):
-        self._func = func
-        self._a = a
-        self._kw = kw
-        self._call = None
+    def __init__(self, func: Callable, *a: Any, **kw: Any):
+        self._func: Callable = func
+        self._a: Sequence[Any] = a
+        self._kw: Dict[str, Any] = kw
+        self._call: Optional[DelayedCall] = None
 
-    def schedule(self, delay=0):
+    def schedule(self, delay: float = 0) -> None:
         from twisted.internet import reactor
 
         if self._call is None:
             self._call = reactor.callLater(delay, self)
 
-    def cancel(self):
+    def cancel(self) -> None:
         if self._call:
             self._call.cancel()
 
-    def __call__(self):
+    def __call__(self) -> Any:
         self._call = None
         return self._func(*self._a, **self._kw)
 
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index b95786d35..9e7ddd827 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -1,6 +1,8 @@
 """Helper functions for working with signals"""
 import collections.abc
 import logging
+from typing import Any as TypingAny
+from typing import List, Tuple
 
 from pydispatch.dispatcher import (
     Anonymous,
@@ -20,7 +22,9 @@ from scrapy.utils.log import failure_to_exc_info
 logger = logging.getLogger(__name__)
 
 
-def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
+def send_catch_log(
+    signal=Any, sender=Anonymous, *arguments, **named
+) -> List[Tuple[TypingAny, TypingAny]]:
     """Like pydispatcher.robust.sendRobust but it also logs errors and returns
     Failures instead of exceptions.
     """
@@ -32,8 +36,9 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
     )
     dont_log += (StopDownload,)
     spider = named.get("spider", None)
-    responses = []
+    responses: List[Tuple[TypingAny, TypingAny]] = []
     for receiver in liveReceivers(getAllReceivers(sender, signal)):
+        result: TypingAny
         try:
             response = robustApply(
                 receiver, signal=signal, sender=sender, *arguments, **named
diff --git a/tox.ini b/tox.ini
index 5a9d9cf29..d96a278ea 100644
--- a/tox.ini
+++ b/tox.ini
@@ -37,13 +37,13 @@ install_command =
 [testenv:typing]
 basepython = python3
 deps =
-    mypy==1.0.1
+    mypy==1.2.0
     types-attrs==19.1.0
-    types-lxml==2023.2.11
-    types-Pillow==9.4.0.16
-    types-Pygments==2.14.0.5
-    types-pyOpenSSL==23.0.0.4
-    types-setuptools==67.4.0.1
+    types-lxml==2023.3.28
+    types-Pillow==9.4.0.19
+    types-Pygments==2.14.0.7
+    types-pyOpenSSL==23.1.0.1
+    types-setuptools==67.6.0.7
 commands =
     mypy --show-error-codes {posargs: scrapy tests}
 


Test patch:
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index eafe1b334..3124d9d67 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1638,6 +1638,57 @@ class FeedExportTest(FeedExportTestBase):
             data = yield self.exported_data(items, settings)
             self.assertEqual(row["expected"], data[feed_options["format"]])
 
+    @defer.inlineCallbacks
+    def test_storage_file_no_postprocessing(self):
+        @implementer(IFeedStorage)
+        class Storage:
+            def __init__(self, uri, *, feed_options=None):
+                pass
+
+            def open(self, spider):
+                Storage.open_file = tempfile.NamedTemporaryFile(prefix="feed-")
+                return Storage.open_file
+
+            def store(self, file):
+                Storage.store_file = file
+                file.close()
+
+        settings = {
+            "FEEDS": {self._random_temp_filename(): {"format": "jsonlines"}},
+            "FEED_STORAGES": {"file": Storage},
+        }
+        yield self.exported_no_data(settings)
+        self.assertIs(Storage.open_file, Storage.store_file)
+
+    @defer.inlineCallbacks
+    def test_storage_file_postprocessing(self):
+        @implementer(IFeedStorage)
+        class Storage:
+            def __init__(self, uri, *, feed_options=None):
+                pass
+
+            def open(self, spider):
+                Storage.open_file = tempfile.NamedTemporaryFile(prefix="feed-")
+                return Storage.open_file
+
+            def store(self, file):
+                Storage.store_file = file
+                file.close()
+
+        settings = {
+            "FEEDS": {
+                self._random_temp_filename(): {
+                    "format": "jsonlines",
+                    "postprocessing": [
+                        "scrapy.extensions.postprocessing.GzipPlugin",
+                    ],
+                },
+            },
+            "FEED_STORAGES": {"file": Storage},
+        }
+        yield self.exported_no_data(settings)
+        self.assertIs(Storage.open_file, Storage.store_file)
+
 
 class FeedPostProcessedExportsTest(FeedExportTestBase):
     __test__ = True