Repository: scrapy/scrapy
Pull request #5832 (2023-02-16 07:22:57)

Prevent edge case that creates extra event loop
Fixes https://github.com/scrapy/scrapy/issues/5831.

`asyncio.get_event_loop_policy().get_event_loop() is asyncio.get_running_loop()` is not always true, as mentioned in https://github.com/python/cpython/issues/96377#issuecomment-1260006538. The AsyncioSelectorReactor runs in the second thread and uses the already initialised event loop in the main thread to run the crawler, so a single event loop will be running code from two threads.

According to https://github.com/python/cpython/issues/100160, we might need to replace `asyncio.get_event_loop()` with `asyncio.get_running_loop()` if semantics change. Currently, some low level aspects of asyncio are in flux.


Patch:
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index aa79cbc0d..ee0cb4b1e 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -11,7 +11,7 @@ jobs:
         - python-version: "3.11"
           env:
             TOXENV: pylint
-        - python-version: 3.7
+        - python-version: 3.8
           env:
             TOXENV: typing
         - python-version: "3.11"  # Keep in sync with .readthedocs.yml
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 54fd5d663..106045fc0 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -238,9 +238,6 @@ genspider
 
 Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain or URL>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
 
-.. note:: Even if an HTTPS URL is specified, the protocol used in
-          ``start_urls`` is always HTTP. This is a known issue: :issue:`3553`.
-
 Usage example::
 
     $ scrapy genspider -l
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 420e85d37..4412b5c1c 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -776,6 +776,31 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2:
 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption
 .. _server pushes: https://tools.ietf.org/html/rfc7540#section-8.2
 
+.. setting:: DOWNLOAD_SLOTS
+
+DOWNLOAD_SLOTS
+----------------
+
+Default: ``{}``
+
+Allows to define concurrency/delay parameters on per slot(domain) basis:
+
+    .. code-block:: python
+
+        DOWNLOAD_SLOTS = {
+            "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False},
+            "books.toscrape.com": {"delay": 3, "randomize_delay": False},
+        }
+
+.. note::
+
+    For other downloader slots default settings values will be used:
+
+    -   :setting:`DOWNLOAD_DELAY`: ``delay``
+    -   :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency``
+    -   :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay``
+
+
 .. setting:: DOWNLOAD_TIMEOUT
 
 DOWNLOAD_TIMEOUT
@@ -1090,7 +1115,7 @@ LOG_FORMAT
 Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'``
 
 String for formatting log messages. Refer to the
-:ref:`Python logging documentation <logrecord-attributes>` for the qwhole
+:ref:`Python logging documentation <logrecord-attributes>` for the whole
 list of available placeholders.
 
 .. setting:: LOG_DATEFORMAT
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index c1565a138..68cbe8ff6 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -31,6 +31,14 @@ def extract_domain(url):
     return o.netloc
 
 
+def verify_url_scheme(url):
+    """Check url for scheme and insert https if none found."""
+    parsed = urlparse(url)
+    if parsed.scheme == "" and parsed.netloc == "":
+        parsed = urlparse("//" + url)._replace(scheme="https")
+    return parsed.geturl()
+
+
 class Command(ScrapyCommand):
     requires_project = False
     default_settings = {"LOG_ENABLED": False}
@@ -91,7 +99,7 @@ class Command(ScrapyCommand):
             raise UsageError()
 
         name, url = args[0:2]
-        domain = extract_domain(url)
+        url = verify_url_scheme(url)
         module = sanitize_module_name(name)
 
         if self.settings.get("BOT_NAME") == module:
@@ -103,18 +111,20 @@ class Command(ScrapyCommand):
 
         template_file = self._find_template(opts.template)
         if template_file:
-            self._genspider(module, name, domain, opts.template, template_file)
+            self._genspider(module, name, url, opts.template, template_file)
             if opts.edit:
                 self.exitcode = os.system(f'scrapy edit "{name}"')
 
-    def _genspider(self, module, name, domain, template_name, template_file):
+    def _genspider(self, module, name, url, template_name, template_file):
         """Generate the spider module, based on the given template"""
         capitalized_module = "".join(s.capitalize() for s in module.split("_"))
+        domain = extract_domain(url)
         tvars = {
             "project_name": self.settings.get("BOT_NAME"),
             "ProjectName": string_camelcase(self.settings.get("BOT_NAME")),
             "module": module,
             "name": name,
+            "url": url,
             "domain": domain,
             "classname": f"{capitalized_module}Spider",
         }
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 9c3fc86d4..ac937e464 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -1,3 +1,4 @@
+import inspect
 import json
 import logging
 from typing import Dict
@@ -10,7 +11,11 @@ from scrapy.commands import BaseRunSpiderCommand
 from scrapy.exceptions import UsageError
 from scrapy.http import Request
 from scrapy.utils import display
-from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
+from scrapy.utils.asyncgen import collect_asyncgen
+from scrapy.utils.defer import aiter_errback, deferred_from_coro
+from scrapy.utils.log import failure_to_exc_info
+from scrapy.utils.misc import arg_to_iter
+from scrapy.utils.spider import spidercls_for_request
 
 logger = logging.getLogger(__name__)
 
@@ -108,6 +113,25 @@ class Command(BaseRunSpiderCommand):
             max_requests = max(self.requests)
         return max(max_items, max_requests)
 
+    def handle_exception(self, _failure):
+        logger.error(
+            "An error is caught while iterating the async iterable",
+            exc_info=failure_to_exc_info(_failure),
+        )
+
+    def iterate_spider_output(self, result):
+        if inspect.isasyncgen(result):
+            d = deferred_from_coro(
+                collect_asyncgen(aiter_errback(result, self.handle_exception))
+            )
+            d.addCallback(self.iterate_spider_output)
+            return d
+        if inspect.iscoroutine(result):
+            d = deferred_from_coro(result)
+            d.addCallback(self.iterate_spider_output)
+            return d
+        return arg_to_iter(deferred_from_coro(result))
+
     def add_items(self, lvl, new_items):
         old_items = self.items.get(lvl, [])
         self.items[lvl] = old_items + new_items
@@ -165,7 +189,7 @@ class Command(BaseRunSpiderCommand):
 
     def run_callback(self, response, callback, cb_kwargs=None):
         cb_kwargs = cb_kwargs or {}
-        d = maybeDeferred(iterate_spider_output, callback(response, **cb_kwargs))
+        d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs))
         return d
 
     def get_callback_from_rules(self, spider, response):
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 7d27a5193..1b83c3a8a 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -84,6 +84,7 @@ class Downloader:
         self.middleware = DownloaderMiddlewareManager.from_crawler(crawler)
         self._slot_gc_loop = task.LoopingCall(self._slot_gc)
         self._slot_gc_loop.start(60)
+        self.per_slot_settings = self.settings.getdict("DOWNLOAD_SLOTS", {})
 
     def fetch(self, request, spider):
         def _deactivate(response):
@@ -94,17 +95,24 @@ class Downloader:
         dfd = self.middleware.download(self._enqueue_request, request, spider)
         return dfd.addBoth(_deactivate)
 
-    def needs_backout(self):
+    def needs_backout(self) -> bool:
         return len(self.active) >= self.total_concurrency
 
     def _get_slot(self, request, spider):
         key = self._get_slot_key(request, spider)
         if key not in self.slots:
+            slot_settings = self.per_slot_settings.get(key, {})
             conc = (
                 self.ip_concurrency if self.ip_concurrency else self.domain_concurrency
             )
             conc, delay = _get_concurrency_delay(conc, spider, self.settings)
-            self.slots[key] = Slot(conc, delay, self.randomize_delay)
+            conc, delay = (
+                slot_settings.get("concurrency", conc),
+                slot_settings.get("delay", delay),
+            )
+            randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay)
+            new_slot = Slot(conc, delay, randomize_delay)
+            self.slots[key] = new_slot
 
         return key, self.slots[key]
 
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 1efbdb271..c6738b531 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -7,13 +7,24 @@ For more information see docs/topics/architecture.rst
 import logging
 import warnings
 from time import time
-from typing import Callable, Iterable, Iterator, Optional, Set, Union
+from typing import (
+    Any,
+    Callable,
+    Generator,
+    Iterable,
+    Iterator,
+    Optional,
+    Set,
+    Union,
+    cast,
+)
 
 from twisted.internet.defer import Deferred, inlineCallbacks, succeed
 from twisted.internet.task import LoopingCall
 from twisted.python.failure import Failure
 
 from scrapy import signals
+from scrapy.core.downloader import Downloader
 from scrapy.core.scraper import Scraper
 from scrapy.exceptions import CloseSpider, DontCloseSpider, ScrapyDeprecationWarning
 from scrapy.http import Request, Response
@@ -75,7 +86,7 @@ class ExecutionEngine:
         self.paused = False
         self.scheduler_cls = self._get_scheduler_class(crawler.settings)
         downloader_cls = load_object(self.settings["DOWNLOADER"])
-        self.downloader = downloader_cls(crawler)
+        self.downloader: Downloader = downloader_cls(crawler)
         self.scraper = Scraper(crawler)
         self._spider_closed_callback = spider_closed_callback
 
@@ -91,20 +102,20 @@ class ExecutionEngine:
         return scheduler_cls
 
     @inlineCallbacks
-    def start(self) -> Deferred:
+    def start(self) -> Generator[Deferred, Any, None]:
         if self.running:
             raise RuntimeError("Engine already running")
         self.start_time = time()
         yield self.signals.send_catch_log_deferred(signal=signals.engine_started)
         self.running = True
-        self._closewait = Deferred()
+        self._closewait: Deferred = Deferred()
         yield self._closewait
 
     def stop(self) -> Deferred:
         """Gracefully stop the execution engine"""
 
         @inlineCallbacks
-        def _finish_stopping_engine(_) -> Deferred:
+        def _finish_stopping_engine(_) -> Generator[Deferred, Any, None]:
             yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped)
             self._closewait.callback(None)
 
@@ -172,11 +183,13 @@ class ExecutionEngine:
             self._spider_idle()
 
     def _needs_backout(self) -> bool:
+        assert self.slot is not None  # typing
+        assert self.scraper.slot is not None  # typing
         return (
             not self.running
-            or self.slot.closing  # type: ignore[union-attr]
+            or bool(self.slot.closing)
             or self.downloader.needs_backout()
-            or self.scraper.slot.needs_backout()  # type: ignore[union-attr]
+            or self.scraper.slot.needs_backout()
         )
 
     def _next_request_from_scheduler(self) -> Optional[Deferred]:
@@ -196,7 +209,7 @@ class ExecutionEngine:
                 extra={"spider": self.spider},
             )
         )
-        d.addBoth(lambda _: self.slot.remove_request(request))
+        d.addBoth(lambda _: cast(Slot, self.slot).remove_request(request))
         d.addErrback(
             lambda f: logger.info(
                 "Error while removing request from slot",
@@ -424,7 +437,7 @@ class ExecutionEngine:
         dfd.addErrback(log_failure("Scraper close failure"))
 
         if hasattr(self.slot.scheduler, "close"):
-            dfd.addBoth(lambda _: self.slot.scheduler.close(reason))
+            dfd.addBoth(lambda _: cast(Slot, self.slot).scheduler.close(reason))
             dfd.addErrback(log_failure("Scheduler close failure"))
 
         dfd.addBoth(
diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py
index 1c43d241c..215ea9716 100644
--- a/scrapy/core/http2/agent.py
+++ b/scrapy/core/http2/agent.py
@@ -6,7 +6,12 @@ from twisted.internet.base import ReactorBase
 from twisted.internet.defer import Deferred
 from twisted.internet.endpoints import HostnameEndpoint
 from twisted.python.failure import Failure
-from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory
+from twisted.web.client import (
+    URI,
+    BrowserLikePolicyForHTTPS,
+    ResponseFailed,
+    _StandardEndpointFactory,
+)
 from twisted.web.error import SchemeNotSupported
 
 from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory
@@ -35,7 +40,7 @@ class H2ConnectionPool:
             # Received a request while connecting to remote
             # Create a deferred which will fire with the H2ClientProtocol
             # instance
-            d = Deferred()
+            d: Deferred = Deferred()
             self._pending_requests[key].append(d)
             return d
 
@@ -53,14 +58,14 @@ class H2ConnectionPool:
     ) -> Deferred:
         self._pending_requests[key] = deque()
 
-        conn_lost_deferred = Deferred()
+        conn_lost_deferred: Deferred = Deferred()
         conn_lost_deferred.addCallback(self._remove_connection, key)
 
         factory = H2ClientFactory(uri, self.settings, conn_lost_deferred)
         conn_d = endpoint.connect(factory)
         conn_d.addCallback(self.put_connection, key)
 
-        d = Deferred()
+        d: Deferred = Deferred()
         self._pending_requests[key].append(d)
         return d
 
@@ -83,7 +88,7 @@ class H2ConnectionPool:
         pending_requests = self._pending_requests.pop(key, None)
         while pending_requests:
             d = pending_requests.popleft()
-            d.errback(errors)
+            d.errback(ResponseFailed(errors))
 
     def close_connections(self) -> None:
         """Close all the HTTP/2 connections and remove them from pool
@@ -92,6 +97,7 @@ class H2ConnectionPool:
             Deferred that fires when all connections have been closed
         """
         for conn in self._connections.values():
+            assert conn.transport is not None  # typing
             conn.transport.abortConnection()
 
 
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 0bf69e513..bc8da50d7 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -140,6 +140,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         This is used while initiating pending streams to make sure
         that we initiate stream only during active HTTP/2 Connection
         """
+        assert self.transport is not None  # typing
         return bool(self.transport.connected) and self.metadata["settings_acknowledged"]
 
     @property
@@ -196,6 +197,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         """Write data to the underlying transport connection
         from the HTTP2 connection instance if any
         """
+        assert self.transport is not None  # typing
         # Reset the idle timeout as connection is still actively sending data
         self.resetTimeout()
 
@@ -226,6 +228,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         # Initialize the timeout
         self.setTimeout(self.IDLE_TIMEOUT)
 
+        assert self.transport is not None  # typing
         destination = self.transport.getPeer()
         self.metadata["ip_address"] = ipaddress.ip_address(destination.host)
 
@@ -237,12 +240,14 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         """Helper function to lose the connection with the error sent as a
         reason"""
         self._conn_lost_errors += errors
+        assert self.transport is not None  # typing
         self.transport.loseConnection()
 
     def handshakeCompleted(self) -> None:
         """
         Close the connection if it's not made via the expected protocol
         """
+        assert self.transport is not None  # typing
         if (
             self.transport.negotiatedProtocol is not None
             and self.transport.negotiatedProtocol != PROTOCOL_NAME
@@ -275,6 +280,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
                 # hyper-h2 does not drop the connection in this scenario, we
                 # need to abort the connection manually.
                 self._conn_lost_errors += [e]
+                assert self.transport is not None  # typing
                 self.transport.abortConnection()
                 return
 
@@ -388,6 +394,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         self._send_pending_requests()
 
         # Update certificate when our HTTP/2 connection is established
+        assert self.transport is not None  # typing
         self.metadata["certificate"] = Certificate(self.transport.getPeerCertificate())
 
     def stream_ended(self, event: StreamEnded) -> None:
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 87beb41e5..6c6ed6f9b 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -151,7 +151,7 @@ class Stream:
             else:
                 self.close(StreamCloseReason.CANCELLED)
 
-        self._deferred_response = Deferred(_cancel)
+        self._deferred_response: Deferred = Deferred(_cancel)
 
     def __repr__(self) -> str:
         return f"Stream(id={self.stream_id!r})"
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 1a09f22f7..8468b8419 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -64,7 +64,7 @@ class Slot:
     def add_response_request(
         self, result: Union[Response, Failure], request: Request
     ) -> Deferred:
-        deferred = Deferred()
+        deferred: Deferred = Deferred()
         self.queue.append((result, request, deferred))
         if isinstance(result, Response):
             self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE)
@@ -203,10 +203,12 @@ class Scraper:
                 callback=callback, callbackKeywords=result.request.cb_kwargs
             )
         else:  # result is a Failure
-            result.request = request
+            # TODO: properly type adding this attribute to a Failure
+            result.request = request  # type: ignore[attr-defined]
             warn_on_generator_with_return_value(spider, request.errback)
             dfd = defer_fail(result)
-            dfd.addErrback(request.errback)
+            if request.errback:
+                dfd.addErrback(request.errback)
         return dfd.addCallback(iterate_spider_output)
 
     def handle_spider_error(
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 971cf52e4..c21985b18 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -155,7 +155,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 # simplified when downgrading is removed.
                 if dfd.called:
                     # the result is available immediately if _process_spider_output didn't do downgrading
-                    return dfd.result
+                    return cast(MutableChain, dfd.result)
                 # we forbid waiting here because otherwise we would need to return a deferred from
                 # _process_spider_exception too, which complicates the architecture
                 msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded"
@@ -180,7 +180,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
         spider: Spider,
         result: Union[Iterable, AsyncIterable],
         start_index: int = 0,
-    ) -> Deferred:
+    ) -> Generator[Deferred, Any, Union[MutableChain, MutableAsyncChain]]:
         # items in this iterable do not need to go through the process_spider_output
         # chain, they went through it already from the process_spider_exception method
         recovered: Union[MutableChain, MutableAsyncChain]
diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py
index 368ca60f7..5839dc243 100644
--- a/scrapy/downloadermiddlewares/decompression.py
+++ b/scrapy/downloadermiddlewares/decompression.py
@@ -51,7 +51,7 @@ class DecompressionMiddleware:
         archive = BytesIO(response.body)
         try:
             zip_file = zipfile.ZipFile(archive)
-        except zipfile.BadZipfile:
+        except zipfile.BadZipFile:
             return
 
         namelist = zip_file.namelist()
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index bdc6a3e39..8da779b7c 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -5,10 +5,17 @@ This module implements the FormRequest class which is a more convenient class
 See documentation in docs/topics/request-response.rst
 """
 
-from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union
+from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union, cast
 from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
 
-from lxml.html import FormElement, HtmlElement, HTMLParser, SelectElement
+from lxml.html import (
+    FormElement,
+    HTMLParser,
+    InputElement,
+    MultipleSelectOptions,
+    SelectElement,
+    TextareaElement,
+)
 from parsel.selector import create_root_node
 from w3lib.html import strip_html5_whitespace
 
@@ -19,7 +26,8 @@ from scrapy.utils.response import get_base_url
 
 FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest")
 
-FormdataType = Optional[Union[dict, List[Tuple[str, str]]]]
+FormdataKVType = Tuple[str, Union[str, Iterable[str]]]
+FormdataType = Optional[Union[dict, List[FormdataKVType]]]
 
 
 class FormRequest(Request):
@@ -50,7 +58,7 @@ class FormRequest(Request):
         response: TextResponse,
         formname: Optional[str] = None,
         formid: Optional[str] = None,
-        formnumber: Optional[int] = 0,
+        formnumber: int = 0,
         formdata: FormdataType = None,
         clickdata: Optional[dict] = None,
         dont_click: bool = False,
@@ -79,6 +87,7 @@ class FormRequest(Request):
 
 
 def _get_form_url(form: FormElement, url: Optional[str]) -> str:
+    assert form.base_url is not None  # typing
     if url is None:
         action = form.get("action")
         if action is None:
@@ -87,11 +96,11 @@ def _get_form_url(form: FormElement, url: Optional[str]) -> str:
     return urljoin(form.base_url, url)
 
 
-def _urlencode(seq: Iterable, enc: str) -> str:
+def _urlencode(seq: Iterable[FormdataKVType], enc: str) -> str:
     values = [
         (to_bytes(k, enc), to_bytes(v, enc))
         for k, vs in seq
-        for v in (vs if is_listlike(vs) else [vs])
+        for v in (cast(Iterable[str], vs) if is_listlike(vs) else [cast(str, vs)])
     ]
     return urlencode(values, doseq=True)
 
@@ -100,7 +109,7 @@ def _get_form(
     response: TextResponse,
     formname: Optional[str],
     formid: Optional[str],
-    formnumber: Optional[int],
+    formnumber: int,
     formxpath: Optional[str],
 ) -> FormElement:
     """Find the wanted form element within the given response."""
@@ -133,13 +142,12 @@ def _get_form(
         raise ValueError(f"No <form> element found with {formxpath}")
 
     # If we get here, it means that either formname was None or invalid
-    if formnumber is not None:
-        try:
-            form = forms[formnumber]
-        except IndexError:
-            raise IndexError(f"Form number {formnumber} not found in {response}")
-        else:
-            return form
+    try:
+        form = forms[formnumber]
+    except IndexError:
+        raise IndexError(f"Form number {formnumber} not found in {response}")
+    else:
+        return form
 
 
 def _get_inputs(
@@ -147,7 +155,7 @@ def _get_inputs(
     formdata: FormdataType,
     dont_click: bool,
     clickdata: Optional[dict],
-) -> List[Tuple[str, str]]:
+) -> List[FormdataKVType]:
     """Return a list of key-value pairs for the inputs found in the given form."""
     try:
         formdata_keys = dict(formdata or ()).keys()
@@ -165,7 +173,7 @@ def _get_inputs(
         '  not(re:test(., "^(?:checkbox|radio)$", "i")))]]',
         namespaces={"re": "http://exslt.org/regular-expressions"},
     )
-    values = [
+    values: List[FormdataKVType] = [
         (k, "" if v is None else v)
         for k, v in (_value(e) for e in inputs)
         if k and k not in formdata_keys
@@ -183,27 +191,25 @@ def _get_inputs(
     return values
 
 
-def _value(ele: HtmlElement):
+def _value(
+    ele: Union[InputElement, SelectElement, TextareaElement]
+) -> Tuple[Optional[str], Union[None, str, MultipleSelectOptions]]:
     n = ele.name
     v = ele.value
     if ele.tag == "select":
-        return _select_value(ele, n, v)
+        return _select_value(cast(SelectElement, ele), n, v)
     return n, v
 
 
-def _select_value(ele: SelectElement, n: str, v: str):
+def _select_value(
+    ele: SelectElement, n: Optional[str], v: Union[None, str, MultipleSelectOptions]
+) -> Tuple[Optional[str], Union[None, str, MultipleSelectOptions]]:
     multiple = ele.multiple
     if v is None and not multiple:
         # Match browser behaviour on simple select tag without options selected
         # And for select tags without options
         o = ele.value_options
         return (n, o[0]) if o else (None, None)
-    if v is not None and multiple:
-        # This is a workaround to bug in lxml fixed 2.3.1
-        # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139
-        selected_options = ele.xpath(".//option[@selected]")
-        values = [(o.get("value") or o.text or "").strip() for o in selected_options]
-        return n, values
     return n, v
 
 
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 0be28933e..6e9f661e5 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -13,8 +13,9 @@ from collections import defaultdict
 from contextlib import suppress
 from ftplib import FTP
 from io import BytesIO
+from os import PathLike
 from pathlib import Path
-from typing import DefaultDict, Optional, Set
+from typing import DefaultDict, Optional, Set, Union
 from urllib.parse import urlparse
 
 from itemadapter import ItemAdapter
@@ -36,24 +37,31 @@ from scrapy.utils.request import referer_str
 logger = logging.getLogger(__name__)
 
 
+def _to_string(path: Union[str, PathLike]) -> str:
+    return str(path)  # convert a Path object to string
+
+
 class FileException(Exception):
     """General media error exception"""
 
 
 class FSFilesStore:
-    def __init__(self, basedir: str):
+    def __init__(self, basedir: Union[str, PathLike]):
+        basedir = _to_string(basedir)
         if "://" in basedir:
             basedir = basedir.split("://", 1)[1]
         self.basedir = basedir
         self._mkdir(Path(self.basedir))
         self.created_directories: DefaultDict[str, Set[str]] = defaultdict(set)
 
-    def persist_file(self, path: str, buf, info, meta=None, headers=None):
+    def persist_file(
+        self, path: Union[str, PathLike], buf, info, meta=None, headers=None
+    ):
         absolute_path = self._get_filesystem_path(path)
         self._mkdir(absolute_path.parent, info)
         absolute_path.write_bytes(buf.getvalue())
 
-    def stat_file(self, path: str, info):
+    def stat_file(self, path: Union[str, PathLike], info):
         absolute_path = self._get_filesystem_path(path)
         try:
             last_modified = absolute_path.stat().st_mtime
@@ -65,8 +73,8 @@ class FSFilesStore:
 
         return {"last_modified": last_modified, "checksum": checksum}
 
-    def _get_filesystem_path(self, path: str) -> Path:
-        path_comps = path.split("/")
+    def _get_filesystem_path(self, path: Union[str, PathLike]) -> Path:
+        path_comps = _to_string(path).split("/")
         return Path(self.basedir, *path_comps)
 
     def _mkdir(self, dirname: Path, domain: Optional[str] = None):
@@ -332,12 +340,12 @@ class FilesPipeline(MediaPipeline):
     DEFAULT_FILES_RESULT_FIELD = "files"
 
     def __init__(self, store_uri, download_func=None, settings=None):
+        store_uri = _to_string(store_uri)
         if not store_uri:
             raise NotConfigured
 
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
-
         cls_name = "FilesPipeline"
         self.store = self._get_store(store_uri)
         resolve = functools.partial(
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index fde8fdde4..a3b849f7b 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -293,6 +293,13 @@ class BaseSettings(MutableMapping):
         else:
             self.attributes[name].set(value, priority)
 
+    def setdefault(self, name, default=None, priority="project"):
+        if name not in self:
+            self.set(name, default, priority)
+            return default
+
+        return self.attributes[name].value
+
     def setdict(self, values, priority="project"):
         self.update(values, priority)
 
diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl
index d3ba19553..20e777271 100644
--- a/scrapy/templates/spiders/basic.tmpl
+++ b/scrapy/templates/spiders/basic.tmpl
@@ -4,7 +4,7 @@ import scrapy
 class $classname(scrapy.Spider):
     name = "$name"
     allowed_domains = ["$domain"]
-    start_urls = ["http://$domain/"]
+    start_urls = ["$url"]
 
     def parse(self, response):
         pass
diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl
index 2e467e632..36d05e43a 100644
--- a/scrapy/templates/spiders/crawl.tmpl
+++ b/scrapy/templates/spiders/crawl.tmpl
@@ -6,7 +6,7 @@ from scrapy.spiders import CrawlSpider, Rule
 class $classname(CrawlSpider):
     name = "$name"
     allowed_domains = ["$domain"]
-    start_urls = ["http://$domain/"]
+    start_urls = ["$url"]
 
     rules = (Rule(LinkExtractor(allow=r"Items/"), callback="parse_item", follow=True),)
 
diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl
index ce9c1dd20..fe96878dc 100644
--- a/scrapy/templates/spiders/csvfeed.tmpl
+++ b/scrapy/templates/spiders/csvfeed.tmpl
@@ -4,7 +4,7 @@ from scrapy.spiders import CSVFeedSpider
 class $classname(CSVFeedSpider):
     name = "$name"
     allowed_domains = ["$domain"]
-    start_urls = ["http://$domain/feed.csv"]
+    start_urls = ["$url"]
     #headers = ["id", "name", "description", "image_link"]
     #delimiter = "\t"
 
diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl
index 6b50e4cf4..ac62d78d1 100644
--- a/scrapy/templates/spiders/xmlfeed.tmpl
+++ b/scrapy/templates/spiders/xmlfeed.tmpl
@@ -4,7 +4,7 @@ from scrapy.spiders import XMLFeedSpider
 class $classname(XMLFeedSpider):
     name = "$name"
     allowed_domains = ["$domain"]
-    start_urls = ["http://$domain/feed.xml"]
+    start_urls = ["$url"]
     iterator = "iternodes"  # you can change this; see the docs
     itertag = "item"  # change it accordingly
 
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index ec130d685..d25ebbdf4 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -17,6 +17,7 @@ from typing import (
     List,
     Optional,
     Union,
+    cast,
 )
 
 from twisted.internet import defer
@@ -38,7 +39,7 @@ def defer_fail(_failure: Failure) -> Deferred:
     """
     from twisted.internet import reactor
 
-    d = Deferred()
+    d: Deferred = Deferred()
     reactor.callLater(0.1, d.errback, _failure)
     return d
 
@@ -52,7 +53,7 @@ def defer_succeed(result) -> Deferred:
     """
     from twisted.internet import reactor
 
-    d = Deferred()
+    d: Deferred = Deferred()
     reactor.callLater(0.1, d.callback, result)
     return d
 
@@ -84,7 +85,7 @@ def mustbe_deferred(f: Callable, *args, **kw) -> Deferred:
 
 def parallel(
     iterable: Iterable, count: int, callable: Callable, *args, **named
-) -> DeferredList:
+) -> Deferred:
     """Execute a callable over the objects in the given iterable, in parallel,
     using no more than ``count`` concurrent calls.
 
@@ -182,7 +183,9 @@ class _AsyncCooperatorAdapter(Iterator):
     def _call_anext(self) -> None:
         # This starts waiting for the next result from aiterator.
         # If aiterator is exhausted, _errback will be called.
-        self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())
+        self.anext_deferred = cast(
+            Deferred, deferred_from_coro(self.aiterator.__anext__())
+        )
         self.anext_deferred.addCallbacks(self._callback, self._errback)
 
     def __next__(self) -> Deferred:
@@ -190,7 +193,7 @@ class _AsyncCooperatorAdapter(Iterator):
         # It also calls __anext__() if needed.
         if self.finished:
             raise StopIteration
-        d = Deferred()
+        d: Deferred = Deferred()
         self.waiting_deferreds.append(d)
         if not self.anext_deferred:
             self._call_anext()
@@ -199,17 +202,17 @@ class _AsyncCooperatorAdapter(Iterator):
 
 def parallel_async(
     async_iterable: AsyncIterable, count: int, callable: Callable, *args, **named
-) -> DeferredList:
+) -> Deferred:
     """Like parallel but for async iterators"""
     coop = Cooperator()
     work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named)
-    dl = DeferredList([coop.coiterate(work) for _ in range(count)])
+    dl: Deferred = DeferredList([coop.coiterate(work) for _ in range(count)])
     return dl
 
 
 def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred:
     """Return a Deferred built by chaining the given callbacks"""
-    d = Deferred()
+    d: Deferred = Deferred()
     for x in callbacks:
         d.addCallback(x, *a, **kw)
     d.callback(input)
@@ -220,7 +223,7 @@ def process_chain_both(
     callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw
 ) -> Deferred:
     """Return a Deferred built by chaining the given callbacks and errbacks"""
-    d = Deferred()
+    d: Deferred = Deferred()
     for cb, eb in zip(callbacks, errbacks):
         d.addCallbacks(
             callback=cb,
@@ -242,7 +245,7 @@ def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred
     callbacks
     """
     dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks]
-    d = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True)
+    d: Deferred = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True)
     d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure)
     return d
 
@@ -285,7 +288,7 @@ def deferred_from_coro(o) -> Any:
         if not is_asyncio_reactor_installed():
             # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines
             # that use asyncio, e.g. "await asyncio.sleep(1)"
-            return ensureDeferred(o)
+            return ensureDeferred(cast(Coroutine[Deferred, Any, Any], o))
         # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
         event_loop = _get_asyncio_event_loop()
         return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop))
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 1771e5550..fc50e0f12 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -8,7 +8,7 @@ import sys
 import weakref
 from functools import partial, wraps
 from itertools import chain
-from typing import AsyncGenerator, AsyncIterable, Iterable, Union
+from typing import Any, AsyncGenerator, AsyncIterable, Iterable, Union
 
 from scrapy.utils.asyncgen import as_async_generator
 
@@ -45,7 +45,7 @@ def iflatten(x):
             yield el
 
 
-def is_listlike(x):
+def is_listlike(x: Any) -> bool:
     """
     >>> is_listlike("foo")
     False
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index c20948fd3..7f67d036a 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -1,10 +1,11 @@
 import asyncio
 import sys
 from contextlib import suppress
-from warnings import catch_warnings, filterwarnings
+from warnings import catch_warnings, filterwarnings, warn
 
 from twisted.internet import asyncioreactor, error
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.misc import load_object
 
 
@@ -54,7 +55,31 @@ class CallLaterOnce:
         return self._func(*self._a, **self._kw)
 
 
+def set_asyncio_event_loop_policy():
+    """The policy functions from asyncio often behave unexpectedly,
+    so we restrict their use to the absolutely essential case.
+    This should only be used to install the reactor.
+    """
+    _get_asyncio_event_loop_policy()
+
+
 def get_asyncio_event_loop_policy():
+    warn(
+        "Call to deprecated function "
+        "scrapy.utils.reactor.get_asyncio_event_loop_policy().\n"
+        "\n"
+        "Please use get_event_loop, new_event_loop and set_event_loop"
+        " from asyncio instead, as the corresponding policy methods may lead"
+        " to unexpected behaviour.\n"
+        "This function is replaced by set_asyncio_event_loop_policy and"
+        " is meant to be used only when the reactor is being installed.",
+        category=ScrapyDeprecationWarning,
+        stacklevel=2,
+    )
+    return _get_asyncio_event_loop_policy()
+
+
+def _get_asyncio_event_loop_policy():
     policy = asyncio.get_event_loop_policy()
     if (
         sys.version_info >= (3, 8)
@@ -63,7 +88,6 @@ def get_asyncio_event_loop_policy():
     ):
         policy = asyncio.WindowsSelectorEventLoopPolicy()
         asyncio.set_event_loop_policy(policy)
-
     return policy
 
 
@@ -73,6 +97,7 @@ def install_reactor(reactor_path, event_loop_path=None):
     path if the asyncio reactor is enabled"""
     reactor_class = load_object(reactor_path)
     if reactor_class is asyncioreactor.AsyncioSelectorReactor:
+        set_asyncio_event_loop_policy()
         with suppress(error.ReactorAlreadyInstalledError):
             event_loop = set_asyncio_event_loop(event_loop_path)
             asyncioreactor.install(eventloop=event_loop)
@@ -90,7 +115,6 @@ def _get_asyncio_event_loop():
 
 def set_asyncio_event_loop(event_loop_path):
     """Sets and returns the event loop with specified import path."""
-    policy = get_asyncio_event_loop_policy()
     if event_loop_path is not None:
         event_loop_class = load_object(event_loop_path)
         event_loop = event_loop_class()
@@ -109,15 +133,13 @@ def set_asyncio_event_loop(event_loop_path):
                     message="There is no current event loop",
                     category=DeprecationWarning,
                 )
-                event_loop = policy.get_event_loop()
+                event_loop = asyncio.get_event_loop()
         except RuntimeError:
             # `get_event_loop` raises RuntimeError when called with no asyncio
             # event loop yet installed in the following scenarios:
-            # - From a thread other than the main thread. For example, when
-            #   using ``scrapy shell``.
             # - Previsibly on Python 3.14 and later.
             #   https://github.com/python/cpython/issues/100160#issuecomment-1345581902
-            event_loop = policy.new_event_loop()
+            event_loop = asyncio.new_event_loop()
             asyncio.set_event_loop(event_loop)
     return event_loop
 
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 730d005e8..794678c48 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -6,7 +6,7 @@ import os
 import re
 import tempfile
 import webbrowser
-from typing import Any, Callable, Iterable, Optional, Tuple, Union
+from typing import Any, Callable, Iterable, Tuple, Union
 from weakref import WeakKeyDictionary
 
 from twisted.web import http
@@ -37,12 +37,13 @@ _metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[floa
 
 def get_meta_refresh(
     response: "scrapy.http.response.text.TextResponse",
-    ignore_tags: Optional[Iterable[str]] = ("script", "noscript"),
+    ignore_tags: Iterable[str] = ("script", "noscript"),
 ) -> Union[Tuple[None, None], Tuple[float, str]]:
     """Parse the http-equiv refresh parameter from the given response"""
     if response not in _metaref_cache:
         text = response.text[0:4096]
-        _metaref_cache[response] = html.get_meta_refresh(
+        # a w3lib typing bug here, fixed in https://github.com/scrapy/w3lib/pull/211
+        _metaref_cache[response] = html.get_meta_refresh(  # type: ignore[assignment]
             text, response.url, response.encoding, ignore_tags=ignore_tags
         )
     return _metaref_cache[response]
diff --git a/setup.cfg b/setup.cfg
index 1fab6fe22..db79c5821 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -6,6 +6,12 @@ universal=1
 
 [mypy]
 ignore_missing_imports = true
+
+# Interface classes are hard to support
+[mypy-twisted.internet.interfaces]
+follow_imports = skip
+
+[mypy-twisted.internet.reactor]
 follow_imports = skip
 
 # FIXME: remove the following sections once the issues are solved
diff --git a/tox.ini b/tox.ini
index 77385968e..5a9d9cf29 100644
--- a/tox.ini
+++ b/tox.ini
@@ -37,11 +37,13 @@ install_command =
 [testenv:typing]
 basepython = python3
 deps =
-    lxml-stubs==0.4.0
-    mypy==0.991
+    mypy==1.0.1
     types-attrs==19.1.0
-    types-pyOpenSSL==23.0.0.2
-    types-setuptools==65.7.0.3
+    types-lxml==2023.2.11
+    types-Pillow==9.4.0.16
+    types-Pygments==2.14.0.5
+    types-pyOpenSSL==23.0.0.4
+    types-setuptools==67.4.0.1
 commands =
     mypy --show-error-codes {posargs: scrapy tests}
 


Test patch:
diff --git a/tests/mockserver.py b/tests/mockserver.py
index d541f3f7c..eb4f8334e 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -10,6 +10,7 @@ from urllib.parse import urlencode
 
 from OpenSSL import SSL
 from twisted.internet import defer, reactor, ssl
+from twisted.internet.protocol import ServerFactory
 from twisted.internet.task import deferLater
 from twisted.names import dns, error
 from twisted.names.server import DNSServerFactory
@@ -368,6 +369,8 @@ if __name__ == "__main__":
     )
     args = parser.parse_args()
 
+    factory: ServerFactory
+
     if args.type == "http":
         root = Root()
         factory = Site(root)
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index b0fb978e9..037333c03 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -30,14 +30,53 @@ import scrapy
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import CrawlSpider, Rule
 from scrapy.utils.test import get_from_asyncio_queue
+import asyncio
 
-class AsyncDefAsyncioSpider(scrapy.Spider):
 
-    name = 'asyncdef{self.spider_name}'
+class AsyncDefAsyncioReturnSpider(scrapy.Spider):
+    name = "asyncdef_asyncio_return"
 
     async def parse(self, response):
+        await asyncio.sleep(0.2)
         status = await get_from_asyncio_queue(response.status)
-        return [scrapy.Item(), dict(foo='bar')]
+        self.logger.info(f"Got response {{status}}")
+        return [{{'id': 1}}, {{'id': 2}}]
+
+class AsyncDefAsyncioReturnSingleElementSpider(scrapy.Spider):
+    name = "asyncdef_asyncio_return_single_element"
+
+    async def parse(self, response):
+        await asyncio.sleep(0.1)
+        status = await get_from_asyncio_queue(response.status)
+        self.logger.info(f"Got response {{status}}")
+        return {{'foo': 42}}
+
+class AsyncDefAsyncioGenLoopSpider(scrapy.Spider):
+    name = "asyncdef_asyncio_gen_loop"
+
+    async def parse(self, response):
+        for i in range(10):
+            await asyncio.sleep(0.1)
+            yield {{'foo': i}}
+        self.logger.info(f"Got response {{response.status}}")
+
+class AsyncDefAsyncioSpider(scrapy.Spider):
+    name = "asyncdef_asyncio"
+
+    async def parse(self, response):
+        await asyncio.sleep(0.2)
+        status = await get_from_asyncio_queue(response.status)
+        self.logger.debug(f"Got response {{status}}")
+
+class AsyncDefAsyncioGenExcSpider(scrapy.Spider):
+    name = "asyncdef_asyncio_gen_exc"
+
+    async def parse(self, response):
+        for i in range(10):
+            await asyncio.sleep(0.1)
+            yield {{'foo': i}}
+            if i > 5:
+                raise ValueError("Stopping the processing")
 
 class MySpider(scrapy.Spider):
     name = '{self.spider_name}'
@@ -213,17 +252,76 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
         self.assertIn("INFO: It Works!", _textmode(stderr))
 
     @defer.inlineCallbacks
-    def test_asyncio_parse_items(self):
+    def test_async_def_asyncio_parse_items_list(self):
         status, out, stderr = yield self.execute(
             [
                 "--spider",
-                "asyncdef" + self.spider_name,
+                "asyncdef_asyncio_return",
                 "-c",
                 "parse",
                 self.url("/html"),
             ]
         )
-        self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out))
+        self.assertIn("INFO: Got response 200", _textmode(stderr))
+        self.assertIn("{'id': 1}", _textmode(out))
+        self.assertIn("{'id': 2}", _textmode(out))
+
+    @defer.inlineCallbacks
+    def test_async_def_asyncio_parse_items_single_element(self):
+        status, out, stderr = yield self.execute(
+            [
+                "--spider",
+                "asyncdef_asyncio_return_single_element",
+                "-c",
+                "parse",
+                self.url("/html"),
+            ]
+        )
+        self.assertIn("INFO: Got response 200", _textmode(stderr))
+        self.assertIn("{'foo': 42}", _textmode(out))
+
+    @defer.inlineCallbacks
+    def test_async_def_asyncgen_parse_loop(self):
+        status, out, stderr = yield self.execute(
+            [
+                "--spider",
+                "asyncdef_asyncio_gen_loop",
+                "-c",
+                "parse",
+                self.url("/html"),
+            ]
+        )
+        self.assertIn("INFO: Got response 200", _textmode(stderr))
+        for i in range(10):
+            self.assertIn(f"{{'foo': {i}}}", _textmode(out))
+
+    @defer.inlineCallbacks
+    def test_async_def_asyncgen_parse_exc(self):
+        status, out, stderr = yield self.execute(
+            [
+                "--spider",
+                "asyncdef_asyncio_gen_exc",
+                "-c",
+                "parse",
+                self.url("/html"),
+            ]
+        )
+        self.assertIn("ValueError", _textmode(stderr))
+        for i in range(7):
+            self.assertIn(f"{{'foo': {i}}}", _textmode(out))
+
+    @defer.inlineCallbacks
+    def test_async_def_asyncio_parse(self):
+        _, _, stderr = yield self.execute(
+            [
+                "--spider",
+                "asyncdef_asyncio",
+                "-c",
+                "parse",
+                self.url("/html"),
+            ]
+        )
+        self.assertIn("DEBUG: Got response 200", _textmode(stderr))
 
     @defer.inlineCallbacks
     def test_parse_items(self):
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 00ddcdd3e..014f50e92 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -541,7 +541,7 @@ class GenspiderCommandTest(CommandTest):
             ).group(1),
         )
         self.assertEqual(
-            f"http://{domain}/",
+            f"https://{domain}",
             self.find_in_file(
                 Path(self.proj_mod_path, "spiders", "test_name.py"),
                 r"start_urls\s*=\s*\[['\"](.+)['\"]\]",
@@ -549,13 +549,64 @@ class GenspiderCommandTest(CommandTest):
         )
 
     def test_url_schema(self):
-        self.test_url("http://test.com", "test.com")
+        self.test_url("https://test.com", "test.com")
 
-    def test_url_path(self):
-        self.test_url("test.com/some/other/page", "test.com")
+    def test_template_start_urls(
+        self, url="test.com", expected="https://test.com", template="basic"
+    ):
+        self.assertEqual(
+            0, self.call("genspider", "-t", template, "--force", "test_name", url)
+        )
+        self.assertEqual(
+            expected,
+            self.find_in_file(
+                Path(self.proj_mod_path, "spiders", "test_name.py"),
+                r"start_urls\s*=\s*\[['\"](.+)['\"]\]",
+            ).group(1),
+        )
+
+    def test_genspider_basic_start_urls(self):
+        self.test_template_start_urls("https://test.com", "https://test.com", "basic")
+        self.test_template_start_urls("http://test.com", "http://test.com", "basic")
+        self.test_template_start_urls(
+            "http://test.com/other/path", "http://test.com/other/path", "basic"
+        )
+        self.test_template_start_urls(
+            "test.com/other/path", "https://test.com/other/path", "basic"
+        )
 
-    def test_url_schema_path(self):
-        self.test_url("https://test.com/some/other/page", "test.com")
+    def test_genspider_crawl_start_urls(self):
+        self.test_template_start_urls("https://test.com", "https://test.com", "crawl")
+        self.test_template_start_urls("http://test.com", "http://test.com", "crawl")
+        self.test_template_start_urls(
+            "http://test.com/other/path", "http://test.com/other/path", "crawl"
+        )
+        self.test_template_start_urls(
+            "test.com/other/path", "https://test.com/other/path", "crawl"
+        )
+        self.test_template_start_urls("test.com", "https://test.com", "crawl")
+
+    def test_genspider_xmlfeed_start_urls(self):
+        self.test_template_start_urls(
+            "https://test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"
+        )
+        self.test_template_start_urls(
+            "http://test.com/feed.xml", "http://test.com/feed.xml", "xmlfeed"
+        )
+        self.test_template_start_urls(
+            "test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"
+        )
+
+    def test_genspider_csvfeed_start_urls(self):
+        self.test_template_start_urls(
+            "https://test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"
+        )
+        self.test_template_start_urls(
+            "http://test.com/feed.xml", "http://test.com/feed.xml", "csvfeed"
+        )
+        self.test_template_start_urls(
+            "test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"
+        )
 
 
 class GenspiderStandaloneCommandTest(ProjectTest):
diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py
index f9b2d853c..a39ed0694 100644
--- a/tests/test_dependencies.py
+++ b/tests/test_dependencies.py
@@ -37,7 +37,3 @@ class ScrapyUtilsTest(unittest.TestCase):
         pinned_twisted_version_string = match[1]
 
         self.assertEqual(twisted_version.short(), pinned_twisted_version_string)
-
-
-if __name__ == "__main__":
-    unittest.main()
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 9ec51f75b..0ea1ef5eb 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -13,7 +13,7 @@ spider = Spider("foo")
 
 
 class TestHttpProxyMiddleware(TestCase):
-    failureException = AssertionError
+    failureException = AssertionError  # type: ignore[assignment]
 
     def setUp(self):
         self._oldenv = os.environ.copy()
diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py
new file mode 100644
index 000000000..9d4072d19
--- /dev/null
+++ b/tests/test_downloaderslotssettings.py
@@ -0,0 +1,72 @@
+import time
+
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase
+
+from scrapy.crawler import CrawlerRunner
+from scrapy.http import Request
+from tests.mockserver import MockServer
+from tests.spiders import MetaSpider
+
+
+class DownloaderSlotsSettingsTestSpider(MetaSpider):
+    name = "downloader_slots"
+
+    custom_settings = {
+        "DOWNLOAD_DELAY": 1,
+        "RANDOMIZE_DOWNLOAD_DELAY": False,
+        "DOWNLOAD_SLOTS": {
+            "quotes.toscrape.com": {
+                "concurrency": 1,
+                "delay": 2,
+                "randomize_delay": False,
+            },
+            "books.toscrape.com": {"delay": 3, "randomize_delay": False},
+        },
+    }
+
+    def start_requests(self):
+        self.times = {None: []}
+
+        slots = list(self.custom_settings.get("DOWNLOAD_SLOTS", {}).keys()) + [None]
+
+        for slot in slots:
+            url = self.mockserver.url(f"/?downloader_slot={slot}")
+            self.times[slot] = []
+            yield Request(url, callback=self.parse, meta={"download_slot": slot})
+
+    def parse(self, response):
+        slot = response.meta.get("download_slot", None)
+        self.times[slot].append(time.time())
+        url = self.mockserver.url(f"/?downloader_slot={slot}&req=2")
+        yield Request(url, callback=self.not_parse, meta={"download_slot": slot})
+
+    def not_parse(self, response):
+        slot = response.meta.get("download_slot", None)
+        self.times[slot].append(time.time())
+
+
+class CrawlTestCase(TestCase):
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+        self.runner = CrawlerRunner()
+
+    def tearDown(self):
+        self.mockserver.__exit__(None, None, None)
+
+    @defer.inlineCallbacks
+    def test_delay(self):
+        crawler = CrawlerRunner().create_crawler(DownloaderSlotsSettingsTestSpider)
+        yield crawler.crawl(mockserver=self.mockserver)
+        slots = crawler.engine.downloader.slots
+        times = crawler.spider.times
+        tolerance = 0.3
+
+        delays_real = {k: v[1] - v[0] for k, v in times.items()}
+        error_delta = {
+            k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay)
+            for k, v in slots.items()
+        }
+
+        self.assertTrue(max(list(error_delta.values())) < tolerance)
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 17a94f036..4709007a8 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -5,6 +5,7 @@ import shutil
 import string
 from ipaddress import IPv4Address
 from pathlib import Path
+from typing import Dict
 from unittest import mock, skipIf
 from urllib.parse import urlencode
 
@@ -87,6 +88,7 @@ class GetDataHtmlLarge(LeafResource):
 class PostDataJsonMixin:
     @staticmethod
     def make_response(request: TxRequest, extra_data: str):
+        assert request.content is not None
         response = {
             "request-headers": {},
             "request-body": json.loads(request.content.read()),
@@ -145,7 +147,8 @@ class QueryParams(LeafResource):
         request.setHeader("Content-Type", "application/json; charset=UTF-8")
         request.setHeader("Content-Encoding", "UTF-8")
 
-        query_params = {}
+        query_params: Dict[str, str] = {}
+        assert request.args is not None
         for k, v in request.args.items():
             query_params[str(k, "utf-8")] = str(v[0], "utf-8")
 
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index c414e582d..9701e5d4e 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -480,6 +480,22 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
             expected_value = settings.get(settings_attr)
             self.assertEqual(getattr(pipeline_cls, pipe_inst_attr), expected_value)
 
+    def test_file_pipeline_using_pathlike_objects(self):
+        class CustomFilesPipelineWithPathLikeDir(FilesPipeline):
+            def file_path(self, request, response=None, info=None, *, item=None):
+                return Path("subdir") / Path(request.url).name
+
+        pipeline = CustomFilesPipelineWithPathLikeDir.from_settings(
+            Settings({"FILES_STORE": Path("./Temp")})
+        )
+        request = Request("http://example.com/image01.jpg")
+        self.assertEqual(pipeline.file_path(request), Path("subdir/image01.jpg"))
+
+    def test_files_store_constructor_with_pathlike_object(self):
+        path = Path("./FileDir")
+        fs_store = FSFilesStore(path)
+        self.assertEqual(fs_store.basedir, str(path))
+
 
 class TestS3FilesStore(unittest.TestCase):
     @defer.inlineCallbacks
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 3e8187096..4a577cd8c 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -65,6 +65,19 @@ class BaseSettingsTest(unittest.TestCase):
     def setUp(self):
         self.settings = BaseSettings()
 
+    def test_setdefault_not_existing_value(self):
+        settings = BaseSettings()
+        value = settings.setdefault("TEST_OPTION", "value")
+        self.assertEqual(settings["TEST_OPTION"], "value")
+        self.assertEqual(value, "value")
+        self.assertIsNotNone(value)
+
+    def test_setdefault_existing_value(self):
+        settings = BaseSettings({"TEST_OPTION": "value"})
+        value = settings.setdefault("TEST_OPTION", None)
+        self.assertEqual(settings["TEST_OPTION"], "value")
+        self.assertEqual(value, "value")
+
     def test_set_new_attribute(self):
         self.settings.set("TEST_OPTION", "value", 0)
         self.assertIn("TEST_OPTION", self.settings.attributes)
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 974a0023d..d167adbb7 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -112,6 +112,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
     Should work for process_spider_output and, when it's supported, process_start_requests.
     """
 
+    ITEM_TYPE: type
     RESULT_COUNT = 3  # to simplify checks, let everything return 3 objects
 
     @staticmethod
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index 746731a2e..01d0ee043 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -1,9 +1,14 @@
+import asyncio
 import warnings
 from unittest import TestCase
 
 from pytest import mark
 
-from scrapy.utils.reactor import install_reactor, is_asyncio_reactor_installed
+from scrapy.utils.reactor import (
+    install_reactor,
+    is_asyncio_reactor_installed,
+    set_asyncio_event_loop,
+)
 
 
 @mark.usefixtures("reactor_pytest")
@@ -23,3 +28,7 @@ class AsyncioTest(TestCase):
         from twisted.internet import reactor
 
         assert original_reactor == reactor
+
+    async def test_set_asyncio_event_loop(self):
+        install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+        assert set_asyncio_event_loop() is asyncio.get_running_loop()
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index bab6e9b68..faf7d2709 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -527,7 +527,3 @@ class TestHelper(unittest.TestCase):
             type(a) is type(b), f"Got {type(a)}, expected {type(b)} for { obj!r}"
         )
         self.assertEqual(a, b)
-
-
-if __name__ == "__main__":
-    unittest.main()
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index fbf60ca71..57f40c2e5 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -257,7 +257,3 @@ class UtilsPythonTestCase(unittest.TestCase):
             without_none_values({"one": 1, "none": None, "three": 3, "four": 4}),
             {"one": 1, "three": 3, "four": 4},
         )
-
-
-if __name__ == "__main__":
-    unittest.main()