Repository: scrapy/scrapy
Pull request #6824 (2025-05-24 09:45:07)

Don't use CrawlerProcess in the commands that don't need it.
This allows commands to opt out of `self.crawler_process` by setting a class attribute. It also extracts `CrawlerRunnerBase._get_spider_loader()` as many commands only need a SpiderLoader instance out of `self.crawler_process`.

Initializing a `CrawlerProcess` instance can be expensive, though most of that cost is initializing a `SpiderLoader` instance.

The following commands only need `SpiderLoader`:

* `edit`
* `genspider`
* `list`

The following commands don't need anything:

* `settings`
* `startproject`
* `version`

This should be compatible both forward and backward for custom user commands.

@GeorgeA92 can you please review this and check that this doesn't break any of your SpiderLoader-related code and ideas?


Patch:
diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index b11de2914..d90eb0bad 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -213,6 +213,8 @@ SpiderLoader API
        :param request: queried request
        :type request: :class:`~scrapy.Request` instance
 
+.. autoclass:: DummySpiderLoader
+
 .. _topics-api-signals:
 
 Signals API
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 68c5079cf..65f2e5ebd 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1868,15 +1868,6 @@ it will fail loudly if there is any ``ImportError`` or ``SyntaxError`` exception
 But you can choose to silence this exception and turn it into a simple
 warning by setting ``SPIDER_LOADER_WARN_ONLY = True``.
 
-.. note::
-    Some :ref:`scrapy commands <topics-commands>` run with this setting to ``True``
-    already (i.e. they will only issue a warning and will not fail)
-    since they do not actually need to load spider classes to work:
-    :command:`scrapy runspider <runspider>`,
-    :command:`scrapy settings <settings>`,
-    :command:`scrapy startproject <startproject>`,
-    :command:`scrapy version <version>`.
-
 .. setting:: SPIDER_MIDDLEWARES
 
 SPIDER_MIDDLEWARES
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index b08fd3409..81e507a4e 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -201,7 +201,8 @@ def execute(argv: list[str] | None = None, settings: Settings | None = None) ->
     opts, args = parser.parse_known_args(args=argv[1:])
     _run_print_help(parser, cmd.process_options, args, opts)
 
-    cmd.crawler_process = CrawlerProcess(settings)
+    if cmd.requires_crawler_process:
+        cmd.crawler_process = CrawlerProcess(settings)
     _run_print_help(parser, _run_command, cmd, args, opts)
     sys.exit(cmd.exitcode)
 
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 56199cc01..2818ead77 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -19,11 +19,13 @@ if TYPE_CHECKING:
     from collections.abc import Iterable
 
     from scrapy.crawler import Crawler, CrawlerProcess
+    from scrapy.settings import Settings
 
 
 class ScrapyCommand:
     requires_project: bool = False
-    crawler_process: CrawlerProcess | None = None
+    requires_crawler_process: bool = True
+    crawler_process: CrawlerProcess | None = None  # set in scrapy.cmdline
 
     # default settings to be used for this command instead of global defaults
     default_settings: dict[str, Any] = {}
@@ -31,7 +33,7 @@ class ScrapyCommand:
     exitcode: int = 0
 
     def __init__(self) -> None:
-        self.settings: Any = None  # set in scrapy.cmdline
+        self.settings: Settings | None = None  # set in scrapy.cmdline
 
     def set_crawler(self, crawler: Crawler) -> None:
         if hasattr(self, "_crawler"):
@@ -68,6 +70,7 @@ class ScrapyCommand:
         """
         Populate option parse with options available for this command
         """
+        assert self.settings is not None
         group = parser.add_argument_group(title="Global Options")
         group.add_argument(
             "--logfile", metavar="FILE", help="log file. if omitted stderr will be used"
@@ -100,6 +103,7 @@ class ScrapyCommand:
         group.add_argument("--pdb", action="store_true", help="enable pdb on failure")
 
     def process_options(self, args: list[str], opts: argparse.Namespace) -> None:
+        assert self.settings is not None
         try:
             self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline")
         except ValueError:
@@ -170,6 +174,7 @@ class BaseRunSpiderCommand(ScrapyCommand):
         except ValueError:
             raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
         if opts.output or opts.overwrite_output:
+            assert self.settings is not None
             feeds = feed_process_params_from_cli(
                 self.settings,
                 opts.output,
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index 56dc1ea55..e9ada0fb6 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -69,6 +69,7 @@ class Command(ScrapyCommand):
 
     def run(self, args: list[str], opts: argparse.Namespace) -> None:
         # load contracts
+        assert self.settings is not None
         contracts = build_component_list(self.settings.getwithbase("SPIDER_CONTRACTS"))
         conman = ContractsManager(load_object(c) for c in contracts)
         runner = TextTestRunner(verbosity=2 if opts.verbose else 1)
diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py
index d153a5271..f2d52673a 100644
--- a/scrapy/commands/edit.py
+++ b/scrapy/commands/edit.py
@@ -4,10 +4,12 @@ import sys
 
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
+from scrapy.spiderloader import get_spider_loader
 
 
 class Command(ScrapyCommand):
     requires_project = True
+    requires_crawler_process = False
     default_settings = {"LOG_ENABLED": False}
 
     def syntax(self) -> str:
@@ -30,10 +32,11 @@ class Command(ScrapyCommand):
         if len(args) != 1:
             raise UsageError
 
+        assert self.settings is not None
         editor = self.settings["EDITOR"]
-        assert self.crawler_process
+        spider_loader = get_spider_loader(self.settings)
         try:
-            spidercls = self.crawler_process.spider_loader.load(args[0])
+            spidercls = spider_loader.load(args[0])
         except KeyError:
             self._err(f"Spider not found: {args[0]}")
             return
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index 6d4aec3d8..c4abfc4c9 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -11,6 +11,7 @@ from urllib.parse import urlparse
 import scrapy
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
+from scrapy.spiderloader import get_spider_loader
 from scrapy.utils.template import render_templatefile, string_camelcase
 
 if TYPE_CHECKING:
@@ -46,6 +47,7 @@ def verify_url_scheme(url: str) -> str:
 
 class Command(ScrapyCommand):
     requires_project = False
+    requires_crawler_process = False
     default_settings = {"LOG_ENABLED": False}
 
     def syntax(self) -> str:
@@ -92,6 +94,7 @@ class Command(ScrapyCommand):
         )
 
     def run(self, args: list[str], opts: argparse.Namespace) -> None:
+        assert self.settings is not None
         if opts.list:
             self._list_templates()
             return
@@ -127,6 +130,7 @@ class Command(ScrapyCommand):
         url: str,
         template_name: str,
     ) -> dict[str, Any]:
+        assert self.settings is not None
         capitalized_module = "".join(s.capitalize() for s in module.split("_"))
         return {
             "project_name": self.settings.get("BOT_NAME"),
@@ -147,6 +151,7 @@ class Command(ScrapyCommand):
         template_file: str | os.PathLike,
     ) -> None:
         """Generate the spider module, based on the given template"""
+        assert self.settings is not None
         tvars = self._generate_template_variables(module, name, url, template_name)
         if self.settings.get("NEWSPIDER_MODULE"):
             spiders_module = import_module(self.settings["NEWSPIDER_MODULE"])
@@ -180,6 +185,7 @@ class Command(ScrapyCommand):
                 print(f"  {file.stem}")
 
     def _spider_exists(self, name: str) -> bool:
+        assert self.settings is not None
         if not self.settings.get("NEWSPIDER_MODULE"):
             # if run as a standalone command and file with same filename already exists
             path = Path(name + ".py")
@@ -188,12 +194,9 @@ class Command(ScrapyCommand):
                 return True
             return False
 
-        assert self.crawler_process is not None, (
-            "crawler_process must be set before calling run"
-        )
-
+        spider_loader = get_spider_loader(self.settings)
         try:
-            spidercls = self.crawler_process.spider_loader.load(name)
+            spidercls = spider_loader.load(name)
         except KeyError:
             pass
         else:
@@ -215,6 +218,7 @@ class Command(ScrapyCommand):
 
     @property
     def templates_dir(self) -> str:
+        assert self.settings is not None
         return str(
             Path(
                 self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"),
diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py
index 3b2f127c2..b4dc97f3d 100644
--- a/scrapy/commands/list.py
+++ b/scrapy/commands/list.py
@@ -3,6 +3,7 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 
 from scrapy.commands import ScrapyCommand
+from scrapy.spiderloader import get_spider_loader
 
 if TYPE_CHECKING:
     import argparse
@@ -10,12 +11,14 @@ if TYPE_CHECKING:
 
 class Command(ScrapyCommand):
     requires_project = True
+    requires_crawler_process = False
     default_settings = {"LOG_ENABLED": False}
 
     def short_desc(self) -> str:
         return "List available spiders"
 
     def run(self, args: list[str], opts: argparse.Namespace) -> None:
-        assert self.crawler_process
-        for s in sorted(self.crawler_process.spider_loader.list()):
+        assert self.settings is not None
+        spider_loader = get_spider_loader(self.settings)
+        for s in sorted(spider_loader.list()):
             print(s)
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 357ca8b37..3e826456e 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING
 
 from scrapy.commands import BaseRunSpiderCommand
 from scrapy.exceptions import UsageError
+from scrapy.spiderloader import DummySpiderLoader
 from scrapy.utils.spider import iter_spider_classes
 
 if TYPE_CHECKING:
@@ -30,7 +31,7 @@ def _import_file(filepath: str | PathLike[str]) -> ModuleType:
 
 class Command(BaseRunSpiderCommand):
     requires_project = False
-    default_settings = {"SPIDER_LOADER_WARN_ONLY": True}
+    default_settings = {"SPIDER_LOADER_CLASS": DummySpiderLoader}
 
     def syntax(self) -> str:
         return "[options] <spider_file>"
diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py
index 59f86b9a7..e63031f2d 100644
--- a/scrapy/commands/settings.py
+++ b/scrapy/commands/settings.py
@@ -7,7 +7,8 @@ from scrapy.settings import BaseSettings
 
 class Command(ScrapyCommand):
     requires_project = False
-    default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
+    requires_crawler_process = False
+    default_settings = {"LOG_ENABLED": False}
 
     def syntax(self) -> str:
         return "[options]"
@@ -46,8 +47,8 @@ class Command(ScrapyCommand):
         )
 
     def run(self, args: list[str], opts: argparse.Namespace) -> None:
-        assert self.crawler_process
-        settings = self.crawler_process.settings
+        assert self.settings is not None
+        settings = self.settings
         if opts.get:
             s = settings.get(opts.get)
             if isinstance(s, BaseSettings):
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 1adc1530f..323979193 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -34,7 +34,8 @@ def _make_writable(path: Path) -> None:
 
 class Command(ScrapyCommand):
     requires_project = False
-    default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
+    requires_crawler_process = False
+    default_settings = {"LOG_ENABLED": False}
 
     def syntax(self) -> str:
         return "<project_name> [project_dir]"
@@ -132,6 +133,7 @@ class Command(ScrapyCommand):
 
     @property
     def templates_dir(self) -> str:
+        assert self.settings is not None
         return str(
             Path(
                 self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"),
diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index 713a78ad9..30b0e9fd7 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -6,7 +6,8 @@ from scrapy.utils.versions import get_versions
 
 
 class Command(ScrapyCommand):
-    default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
+    requires_crawler_process = False
+    default_settings = {"LOG_ENABLED": False}
 
     def syntax(self) -> str:
         return "[-v]"
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index c22b8603b..8e3223a5c 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -5,22 +5,21 @@ import contextlib
 import logging
 import pprint
 import signal
-from typing import TYPE_CHECKING, Any, TypeVar, cast
+from typing import TYPE_CHECKING, Any, TypeVar
 
 from twisted.internet.defer import (
     Deferred,
     DeferredList,
     inlineCallbacks,
 )
-from zope.interface.verify import verifyClass
 
 from scrapy import Spider, signals
 from scrapy.addons import AddonManager
 from scrapy.core.engine import ExecutionEngine
 from scrapy.extension import ExtensionManager
-from scrapy.interfaces import ISpiderLoader
-from scrapy.settings import BaseSettings, Settings, overridden_settings
+from scrapy.settings import Settings, overridden_settings
 from scrapy.signalmanager import SignalManager
+from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader
 from scrapy.utils.asyncio import is_asyncio_available
 from scrapy.utils.defer import deferred_from_coro, deferred_to_future
 from scrapy.utils.log import (
@@ -46,7 +45,6 @@ if TYPE_CHECKING:
     from collections.abc import Generator, Iterable
 
     from scrapy.logformatter import LogFormatter
-    from scrapy.spiderloader import SpiderLoaderProtocol
     from scrapy.statscollectors import StatsCollector
     from scrapy.utils.request import RequestFingerprinterProtocol
 
@@ -324,22 +322,12 @@ class Crawler:
 
 
 class CrawlerRunnerBase:
-    @staticmethod
-    def _get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol:
-        """Get SpiderLoader instance from settings"""
-        cls_path = settings.get("SPIDER_LOADER_CLASS")
-        loader_cls = load_object(cls_path)
-        verifyClass(ISpiderLoader, loader_cls)
-        return cast(
-            "SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy())
-        )
-
     def __init__(self, settings: dict[str, Any] | Settings | None = None):
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
         AddonManager.load_pre_crawler_settings(settings)
         self.settings: Settings = settings
-        self.spider_loader: SpiderLoaderProtocol = self._get_spider_loader(settings)
+        self.spider_loader: SpiderLoaderProtocol = get_spider_loader(settings)
         self._crawlers: set[Crawler] = set()
         self.bootstrap_failed = False
 
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index f537e0593..8eac188c8 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -3,12 +3,13 @@ from __future__ import annotations
 import traceback
 import warnings
 from collections import defaultdict
-from typing import TYPE_CHECKING, Protocol
+from typing import TYPE_CHECKING, Protocol, cast
 
 from zope.interface import implementer
+from zope.interface.verify import verifyClass
 
 from scrapy.interfaces import ISpiderLoader
-from scrapy.utils.misc import walk_modules
+from scrapy.utils.misc import load_object, walk_modules
 from scrapy.utils.spider import iter_spider_classes
 
 if TYPE_CHECKING:
@@ -21,6 +22,14 @@ if TYPE_CHECKING:
     from scrapy.settings import BaseSettings
 
 
+def get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol:
+    """Get SpiderLoader instance from settings"""
+    cls_path = settings.get("SPIDER_LOADER_CLASS")
+    loader_cls = load_object(cls_path)
+    verifyClass(ISpiderLoader, loader_cls)
+    return cast("SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy()))
+
+
 class SpiderLoaderProtocol(Protocol):
     @classmethod
     def from_settings(cls, settings: BaseSettings) -> Self:
@@ -120,3 +129,21 @@ class SpiderLoader:
         Return a list with the names of all spiders available in the project.
         """
         return list(self._spiders.keys())
+
+
+@implementer(ISpiderLoader)
+class DummySpiderLoader:
+    """A dummy spider loader that does not load any spiders."""
+
+    @classmethod
+    def from_settings(cls, settings: BaseSettings) -> Self:
+        return cls()
+
+    def load(self, spider_name: str) -> type[Spider]:
+        raise KeyError("DummySpiderLoader doesn't load any spiders")
+
+    def list(self) -> list[str]:
+        return []
+
+    def find_by_request(self, request: Request) -> __builtins__.list[str]:
+        return []


Test patch:
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index ce5963cd7..56cb21650 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -30,7 +30,6 @@ from scrapy.crawler import (
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.extensions.throttle import AutoThrottle
 from scrapy.settings import Settings, default_settings
-from scrapy.spiderloader import SpiderLoader
 from scrapy.utils.defer import deferred_f_from_coro_f, deferred_from_coro
 from scrapy.utils.log import configure_logging, get_scrapy_root_handler
 from scrapy.utils.spider import DefaultSpider
@@ -570,10 +569,6 @@ class SpiderLoaderWithWrongInterface:
         pass
 
 
-class CustomSpiderLoader(SpiderLoader):
-    pass
-
-
 class TestCrawlerRunner(TestBaseCrawler):
     def test_spider_manager_verify_interface(self):
         settings = Settings(
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 476487a04..245507c0b 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -1,10 +1,8 @@
 import contextlib
 import shutil
 import sys
-import tempfile
 import warnings
 from pathlib import Path
-from tempfile import mkdtemp
 from unittest import mock
 
 import pytest
@@ -17,7 +15,7 @@ from scrapy.crawler import CrawlerRunner
 from scrapy.http import Request
 from scrapy.interfaces import ISpiderLoader
 from scrapy.settings import Settings
-from scrapy.spiderloader import SpiderLoader
+from scrapy.spiderloader import DummySpiderLoader, SpiderLoader, get_spider_loader
 
 module_dir = Path(__file__).resolve().parent
 
@@ -27,73 +25,76 @@ def _copytree(source: Path, target: Path):
         shutil.copytree(source, target)
 
 
+@pytest.fixture
+def spider_loader_env(tmp_path):
+    orig_spiders_dir = module_dir / "test_spiders"
+    spiders_dir = tmp_path / "test_spiders_xxx"
+    _copytree(orig_spiders_dir, spiders_dir)
+    sys.path.append(str(tmp_path))
+    settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})
+
+    yield settings, spiders_dir
+
+    sys.modules.pop("test_spiders_xxx", None)
+    sys.path.remove(str(tmp_path))
+
+
+@pytest.fixture
+def spider_loader(spider_loader_env):
+    settings, _ = spider_loader_env
+    return SpiderLoader.from_settings(settings)
+
+
 class TestSpiderLoader:
-    def setup_method(self):
-        orig_spiders_dir = module_dir / "test_spiders"
-        self.tmpdir = Path(tempfile.mkdtemp())
-        self.spiders_dir = self.tmpdir / "test_spiders_xxx"
-        _copytree(orig_spiders_dir, self.spiders_dir)
-        sys.path.append(str(self.tmpdir))
-        settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})
-        self.spider_loader = SpiderLoader.from_settings(settings)
-
-    def teardown_method(self):
-        del self.spider_loader
-        del sys.modules["test_spiders_xxx"]
-        sys.path.remove(str(self.tmpdir))
-
-    def test_interface(self):
-        verifyObject(ISpiderLoader, self.spider_loader)
-
-    def test_list(self):
-        assert set(self.spider_loader.list()) == {
+    def test_interface(self, spider_loader):
+        verifyObject(ISpiderLoader, spider_loader)
+
+    def test_list(self, spider_loader):
+        assert set(spider_loader.list()) == {
             "spider1",
             "spider2",
             "spider3",
             "spider4",
         }
 
-    def test_load(self):
-        spider1 = self.spider_loader.load("spider1")
+    def test_load(self, spider_loader):
+        spider1 = spider_loader.load("spider1")
         assert spider1.__name__ == "Spider1"
 
-    def test_find_by_request(self):
-        assert self.spider_loader.find_by_request(
-            Request("http://scrapy1.org/test")
-        ) == ["spider1"]
-        assert self.spider_loader.find_by_request(
-            Request("http://scrapy2.org/test")
-        ) == ["spider2"]
+    def test_find_by_request(self, spider_loader):
+        assert spider_loader.find_by_request(Request("http://scrapy1.org/test")) == [
+            "spider1"
+        ]
+        assert spider_loader.find_by_request(Request("http://scrapy2.org/test")) == [
+            "spider2"
+        ]
         assert set(
-            self.spider_loader.find_by_request(Request("http://scrapy3.org/test"))
+            spider_loader.find_by_request(Request("http://scrapy3.org/test"))
         ) == {"spider1", "spider2"}
-        assert (
-            self.spider_loader.find_by_request(Request("http://scrapy999.org/test"))
-            == []
-        )
-        assert self.spider_loader.find_by_request(Request("http://spider3.com")) == []
-        assert self.spider_loader.find_by_request(
+        assert spider_loader.find_by_request(Request("http://scrapy999.org/test")) == []
+        assert spider_loader.find_by_request(Request("http://spider3.com")) == []
+        assert spider_loader.find_by_request(
             Request("http://spider3.com/onlythis")
         ) == ["spider3"]
 
     def test_load_spider_module(self):
         module = "tests.test_spiderloader.test_spiders.spider1"
         settings = Settings({"SPIDER_MODULES": [module]})
-        self.spider_loader = SpiderLoader.from_settings(settings)
-        assert len(self.spider_loader._spiders) == 1
+        spider_loader = SpiderLoader.from_settings(settings)
+        assert len(spider_loader._spiders) == 1
 
     def test_load_spider_module_multiple(self):
         prefix = "tests.test_spiderloader.test_spiders."
         module = ",".join(prefix + s for s in ("spider1", "spider2"))
         settings = Settings({"SPIDER_MODULES": module})
-        self.spider_loader = SpiderLoader.from_settings(settings)
-        assert len(self.spider_loader._spiders) == 2
+        spider_loader = SpiderLoader.from_settings(settings)
+        assert len(spider_loader._spiders) == 2
 
     def test_load_base_spider(self):
         module = "tests.test_spiderloader.test_spiders.spider0"
         settings = Settings({"SPIDER_MODULES": [module]})
-        self.spider_loader = SpiderLoader.from_settings(settings)
-        assert len(self.spider_loader._spiders) == 0
+        spider_loader = SpiderLoader.from_settings(settings)
+        assert len(spider_loader._spiders) == 0
 
     def test_load_spider_module_from_addons(self):
         module = "tests.test_spiderloader.spiders_from_addons.spider0"
@@ -183,27 +184,14 @@ class TestSpiderLoader:
 
 
 class TestDuplicateSpiderNameLoader:
-    def setup_method(self):
-        orig_spiders_dir = module_dir / "test_spiders"
-        self.tmpdir = Path(mkdtemp())
-        self.spiders_dir = self.tmpdir / "test_spiders_xxx"
-        _copytree(orig_spiders_dir, self.spiders_dir)
-        sys.path.append(str(self.tmpdir))
-        self.settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})
-
-    def teardown_method(self):
-        del sys.modules["test_spiders_xxx"]
-        sys.path.remove(str(self.tmpdir))
-
-    def test_dupename_warning(self):
+    def test_dupename_warning(self, spider_loader_env):
+        settings, spiders_dir = spider_loader_env
+
         # copy 1 spider module so as to have duplicate spider name
-        shutil.copyfile(
-            self.tmpdir / "test_spiders_xxx" / "spider3.py",
-            self.tmpdir / "test_spiders_xxx" / "spider3dupe.py",
-        )
+        shutil.copyfile(spiders_dir / "spider3.py", spiders_dir / "spider3dupe.py")
 
         with warnings.catch_warnings(record=True) as w:
-            spider_loader = SpiderLoader.from_settings(self.settings)
+            spider_loader = SpiderLoader.from_settings(settings)
 
             assert len(w) == 1
             msg = str(w[0].message)
@@ -218,20 +206,15 @@ class TestDuplicateSpiderNameLoader:
             spiders = set(spider_loader.list())
             assert spiders == {"spider1", "spider2", "spider3", "spider4"}
 
-    def test_multiple_dupename_warning(self):
+    def test_multiple_dupename_warning(self, spider_loader_env):
+        settings, spiders_dir = spider_loader_env
         # copy 2 spider modules so as to have duplicate spider name
         # This should issue 2 warning, 1 for each duplicate spider name
-        shutil.copyfile(
-            self.tmpdir / "test_spiders_xxx" / "spider1.py",
-            self.tmpdir / "test_spiders_xxx" / "spider1dupe.py",
-        )
-        shutil.copyfile(
-            self.tmpdir / "test_spiders_xxx" / "spider2.py",
-            self.tmpdir / "test_spiders_xxx" / "spider2dupe.py",
-        )
+        shutil.copyfile(spiders_dir / "spider1.py", spiders_dir / "spider1dupe.py")
+        shutil.copyfile(spiders_dir / "spider2.py", spiders_dir / "spider2dupe.py")
 
         with warnings.catch_warnings(record=True) as w:
-            spider_loader = SpiderLoader.from_settings(self.settings)
+            spider_loader = SpiderLoader.from_settings(settings)
 
             assert len(w) == 1
             msg = str(w[0].message)
@@ -247,3 +230,25 @@ class TestDuplicateSpiderNameLoader:
 
             spiders = set(spider_loader.list())
             assert spiders == {"spider1", "spider2", "spider3", "spider4"}
+
+
+class CustomSpiderLoader(SpiderLoader):
+    pass
+
+
+def test_custom_spider_loader():
+    settings = Settings(
+        {
+            "SPIDER_LOADER_CLASS": CustomSpiderLoader,
+        }
+    )
+    spider_loader = get_spider_loader(settings)
+    assert isinstance(spider_loader, CustomSpiderLoader)
+
+
+def test_dummy_spider_loader(spider_loader_env):
+    settings, _ = spider_loader_env
+    spider_loader = DummySpiderLoader.from_settings(settings)
+    assert not spider_loader.list()
+    with pytest.raises(KeyError):
+        spider_loader.load("spider1")