Repository: scrapy/scrapy
Pull request #6582 (2024-12-13 17:03:55)

Allow customizing logged software versions
Resolves: #4203

wRAR: Related: #4203 

Patch:
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index cce4a7b3e..76904a26e 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1228,6 +1228,25 @@ Default: ``False``
 If ``True``, the logs will just contain the root path. If it is set to ``False``
 then it displays the component responsible for the log output
 
+.. setting:: LOG_VERSIONS
+
+LOG_VERSIONS
+------------
+
+Default: ``["lxml", "libxml2", "cssselect", "parsel", "w3lib", "Twisted", "Python", "pyOpenSSL", "cryptography", "Platform"]``
+
+Logs the installed versions of the specified items.
+
+An item can be any installed Python package.
+
+The following special items are also supported:
+
+-   ``libxml2``
+
+-   ``Platform`` (:func:`platform.platform`)
+
+-   ``Python``
+
 .. setting:: LOGSTATS_INTERVAL
 
 LOGSTATS_INTERVAL
diff --git a/pyproject.toml b/pyproject.toml
index c0297e192..7dc1f6ec3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -122,6 +122,9 @@ profile = "black"
 [tool.pylint.MASTER]
 persistent = "no"
 jobs = 1          # >1 hides results
+extension-pkg-allow-list=[
+    "lxml",
+]
 
 [tool.pylint."MESSAGES CONTROL"]
 disable = [
@@ -129,11 +132,7 @@ disable = [
     "arguments-differ",
     "arguments-renamed",
     "attribute-defined-outside-init",
-    "bad-classmethod-argument",
-    "bare-except",
-    "broad-except",
-    "broad-exception-raised",
-    "c-extension-no-member",
+    "broad-exception-caught",
     "consider-using-with",
     "cyclic-import",
     "dangerous-default-value",
@@ -141,9 +140,6 @@ disable = [
     "duplicate-code",                            # https://github.com/PyCQA/pylint/issues/214
     "eval-used",
     "fixme",
-    "function-redefined",
-    "global-statement",
-    "implicit-str-concat",
     "import-error",
     "import-outside-toplevel",
     "inherit-non-class",
@@ -155,7 +151,6 @@ disable = [
     "logging-format-interpolation",
     "logging-fstring-interpolation",
     "logging-not-lazy",
-    "lost-exception",
     "missing-docstring",
     "no-member",
     "no-method-argument",
@@ -169,13 +164,11 @@ disable = [
     "raise-missing-from",
     "redefined-builtin",
     "redefined-outer-name",
-    "reimported",
     "signature-differs",
     "too-few-public-methods",
     "too-many-ancestors",
     "too-many-arguments",
     "too-many-branches",
-    "too-many-format-args",
     "too-many-function-args",
     "too-many-instance-attributes",
     "too-many-lines",
@@ -184,14 +177,11 @@ disable = [
     "too-many-return-statements",
     "unbalanced-tuple-unpacking",
     "unnecessary-dunder-call",
-    "unnecessary-pass",
-    "unreachable",
     "unused-argument",
     "unused-import",
     "unused-variable",
     "used-before-assignment",
     "useless-return",
-    "wildcard-import",
     "wrong-import-position",
 ]
 
@@ -246,6 +236,8 @@ extend-select = [
     "PGH",
     # flake8-pie
     "PIE",
+    # pylint
+    "PL",
     # flake8-pyi
     "PYI",
     # flake8-quotes
@@ -320,6 +312,18 @@ ignore = [
     "D403",
     # `try`-`except` within a loop incurs performance overhead
     "PERF203",
+    # Too many return statements
+    "PLR0911",
+    # Too many branches
+    "PLR0912",
+    # Too many arguments in function definition
+    "PLR0913",
+    # Too many statements
+    "PLR0915",
+    # Magic value used in comparison
+    "PLR2004",
+    # `for` loop variable overwritten by assignment target
+    "PLW2901",
     # Use of `assert` detected; needed for mypy
     "S101",
     # FTP-related functions are being called; https://github.com/scrapy/scrapy/issues/4180
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 921296502..c19710a6a 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -31,7 +31,7 @@ version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split("."
 
 def __getattr__(name: str):
     if name == "twisted_version":
-        import warnings
+        import warnings  # pylint: disable=reimported
 
         from twisted import version as _txv
 
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index b6f19a37f..9a24871de 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -74,7 +74,7 @@ def _get_commands_from_entry_points(
         if inspect.isclass(obj):
             cmds[entry_point.name] = obj()
         else:
-            raise Exception(f"Invalid entry point {entry_point.name}")
+            raise ValueError(f"Invalid entry point {entry_point.name}")
     return cmds
 
 
diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index 571f4fda8..713a78ad9 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -2,7 +2,7 @@ import argparse
 
 import scrapy
 from scrapy.commands import ScrapyCommand
-from scrapy.utils.versions import scrapy_components_versions
+from scrapy.utils.versions import get_versions
 
 
 class Command(ScrapyCommand):
@@ -26,7 +26,7 @@ class Command(ScrapyCommand):
 
     def run(self, args: list[str], opts: argparse.Namespace) -> None:
         if opts.verbose:
-            versions = scrapy_components_versions()
+            versions = get_versions()
             width = max(len(n) for (n, _) in versions)
             for name, version in versions:
                 print(f"{name:<{width}} : {version}")
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index b6997ef67..cdb7ac159 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -92,11 +92,10 @@ class BaseItemExporter:
                 field_iter = (
                     (x, y) for x, y in self.fields_to_export.items() if x in item
                 )
+        elif include_empty:
+            field_iter = self.fields_to_export
         else:
-            if include_empty:
-                field_iter = self.fields_to_export
-            else:
-                field_iter = (x for x in self.fields_to_export if x in item)
+            field_iter = (x for x in self.fields_to_export if x in item)
 
         for field_name in field_iter:
             if isinstance(field_name, str):
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 89ab21fbe..0bbde118e 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -219,6 +219,18 @@ LOG_LEVEL = "DEBUG"
 LOG_FILE = None
 LOG_FILE_APPEND = True
 LOG_SHORT_NAMES = False
+LOG_VERSIONS = [
+    "lxml",
+    "libxml2",
+    "cssselect",
+    "parsel",
+    "w3lib",
+    "Twisted",
+    "Python",
+    "pyOpenSSL",
+    "cryptography",
+    "Platform",
+]
 
 SCHEDULER_DEBUG = False
 
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 9f1b816c8..000ab5c65 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -398,7 +398,7 @@ def maybeDeferred_coro(
     """Copy of defer.maybeDeferred that also converts coroutines to Deferreds."""
     try:
         result = f(*args, **kw)
-    except:  # noqa: E722,B001
+    except:  # noqa: E722  # pylint: disable=bare-except
         return defer.fail(failure.Failure(captureVars=Deferred.debug))
 
     if isinstance(result, Deferred):
diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py
index 32430cd6c..0a0acc742 100644
--- a/scrapy/utils/deprecate.py
+++ b/scrapy/utils/deprecate.py
@@ -60,7 +60,7 @@ def create_deprecated_class(
         deprecated_class: type | None = None
         warned_on_subclass: bool = False
 
-        def __new__(
+        def __new__(  # pylint: disable=bad-classmethod-argument
             metacls, name: str, bases: tuple[type, ...], clsdict_: dict[str, Any]
         ) -> type:
             cls = super().__new__(metacls, name, bases, clsdict_)
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index a40b835cd..d51231b82 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import pprint
 import sys
 from collections.abc import MutableMapping
 from logging.config import dictConfig
@@ -12,7 +13,7 @@ from twisted.python.failure import Failure
 
 import scrapy
 from scrapy.settings import Settings, _SettingsKeyT
-from scrapy.utils.versions import scrapy_components_versions
+from scrapy.utils.versions import get_versions
 
 if TYPE_CHECKING:
 
@@ -130,7 +131,7 @@ _scrapy_root_handler: logging.Handler | None = None
 
 
 def install_scrapy_root_handler(settings: Settings) -> None:
-    global _scrapy_root_handler
+    global _scrapy_root_handler  # noqa: PLW0603  # pylint: disable=global-statement
 
     if (
         _scrapy_root_handler is not None
@@ -174,12 +175,11 @@ def log_scrapy_info(settings: Settings) -> None:
         "Scrapy %(version)s started (bot: %(bot)s)",
         {"version": scrapy.__version__, "bot": settings["BOT_NAME"]},
     )
-    versions = [
-        f"{name} {version}"
-        for name, version in scrapy_components_versions()
-        if name != "Scrapy"
-    ]
-    logger.info("Versions: %(versions)s", {"versions": ", ".join(versions)})
+    software = settings.getlist("LOG_VERSIONS")
+    if not software:
+        return
+    versions = pprint.pformat(dict(get_versions(software)), sort_dicts=False)
+    logger.info(f"Versions:\n{versions}")
 
 
 def log_reactor_info() -> None:
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 2d781cc27..66a06a9f0 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -149,12 +149,11 @@ def verify_installed_reactor(reactor_path: str) -> None:
 
     reactor_class = load_object(reactor_path)
     if not reactor.__class__ == reactor_class:
-        msg = (
+        raise RuntimeError(
             "The installed reactor "
             f"({reactor.__module__}.{reactor.__class__.__name__}) does not "
             f"match the requested one ({reactor_path})"
         )
-        raise Exception(msg)
 
 
 def verify_installed_asyncio_event_loop(loop_path: str) -> None:
@@ -168,7 +167,7 @@ def verify_installed_asyncio_event_loop(loop_path: str) -> None:
         f".{reactor._asyncioEventloop.__class__.__qualname__}"
     )
     specified = f"{loop_class.__module__}.{loop_class.__qualname__}"
-    raise Exception(
+    raise RuntimeError(
         "Scrapy found an asyncio Twisted reactor already "
         f"installed, and its event loop class ({installed}) does "
         "not match the one specified in the ASYNCIO_EVENT_LOOP "
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index e58eb8134..5277a292c 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -52,10 +52,6 @@ def iter_spider_classes(module: ModuleType) -> Iterable[type[Spider]]:
     """Return an iterator over all spider classes defined in the given module
     that can be instantiated (i.e. which have name)
     """
-    # this needs to be imported here until get rid of the spider manager
-    # singleton in scrapy.spider.spiders
-    from scrapy.spiders import Spider
-
     for obj in vars(module).values():
         if (
             inspect.isclass(obj)
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 2539f30c7..1cbfbfd99 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -14,7 +14,7 @@ from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse
 
 # scrapy.utils.url was moved to w3lib.url and import * ensures this
 # move doesn't break old code
-from w3lib.url import *  # pylint: disable=unused-wildcard-import
+from w3lib.url import *  # pylint: disable=unused-wildcard-import,wildcard-import
 from w3lib.url import _safe_chars, _unquotepath  # noqa: F401
 
 from scrapy.utils.python import to_unicode
@@ -50,7 +50,9 @@ def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool:
     return any(lowercase_path.endswith(ext) for ext in extensions)
 
 
-def parse_url(url: UrlT, encoding: str | None = None) -> ParseResult:
+def parse_url(  # pylint: disable=function-redefined
+    url: UrlT, encoding: str | None = None
+) -> ParseResult:
     """Return urlparsed url from the given argument (which could be an already
     parsed url)
     """
diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py
index 996a5cdb3..ff1f9b346 100644
--- a/scrapy/utils/versions.py
+++ b/scrapy/utils/versions.py
@@ -1,31 +1,46 @@
+from __future__ import annotations
+
 import platform
 import sys
+from importlib.metadata import version
+from warnings import warn
 
-import cryptography
-import cssselect
 import lxml.etree
-import parsel
-import twisted
-import w3lib
 
-import scrapy
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.settings.default_settings import LOG_VERSIONS
 from scrapy.utils.ssl import get_openssl_version
 
+_DEFAULT_SOFTWARE = ["Scrapy"] + LOG_VERSIONS
+
+
+def _version(item):
+    lowercase_item = item.lower()
+    if lowercase_item == "libxml2":
+        return ".".join(map(str, lxml.etree.LIBXML_VERSION))
+    if lowercase_item == "platform":
+        return platform.platform()
+    if lowercase_item == "pyopenssl":
+        return get_openssl_version()
+    if lowercase_item == "python":
+        return sys.version.replace("\n", "- ")
+    return version(item)
+
+
+def get_versions(
+    software: list | None = None,
+) -> list[tuple[str, str]]:
+    software = software or _DEFAULT_SOFTWARE
+    return [(item, _version(item)) for item in software]
+
 
 def scrapy_components_versions() -> list[tuple[str, str]]:
-    lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION))
-    libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION))
-
-    return [
-        ("Scrapy", scrapy.__version__),
-        ("lxml", lxml_version),
-        ("libxml2", libxml2_version),
-        ("cssselect", cssselect.__version__),
-        ("parsel", parsel.__version__),
-        ("w3lib", w3lib.__version__),
-        ("Twisted", twisted.version.short()),
-        ("Python", sys.version.replace("\n", "- ")),
-        ("pyOpenSSL", get_openssl_version()),
-        ("cryptography", cryptography.__version__),
-        ("Platform", platform.platform()),
-    ]
+    warn(
+        (
+            "scrapy.utils.versions.scrapy_components_versions() is deprecated, "
+            "use scrapy.utils.versions.get_versions() instead."
+        ),
+        ScrapyDeprecationWarning,
+        stacklevel=2,
+    )
+    return get_versions()


Test patch:
diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py
index af15cac68..3e29c70ed 100644
--- a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py
+++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py
@@ -8,7 +8,7 @@ class TestSpiderPipeline:
 
 class TestSpiderExceptionPipeline:
     def open_spider(self, spider):
-        raise Exception("exception")
+        raise RuntimeError("exception")
 
     def process_item(self, item, spider):
         return item
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index a77531f62..f3e5ebf5d 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,6 +1,7 @@
 import logging
 import os
 import platform
+import re
 import signal
 import subprocess
 import sys
@@ -923,3 +924,28 @@ class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
             log,
         )
         self.assertIn("DEBUG: Using asyncio event loop", log)
+
+
+@mark.parametrize(
+    ["settings", "items"],
+    (
+        ({}, default_settings.LOG_VERSIONS),
+        ({"LOG_VERSIONS": ["itemadapter"]}, ["itemadapter"]),
+        ({"LOG_VERSIONS": []}, None),
+    ),
+)
+def test_log_scrapy_info(settings, items, caplog):
+    with caplog.at_level("INFO"):
+        CrawlerProcess(settings)
+    assert (
+        caplog.records[0].getMessage()
+        == f"Scrapy {scrapy.__version__} started (bot: scrapybot)"
+    ), repr(caplog.records[0].msg)
+    if not items:
+        assert len(caplog.records) == 1
+        return
+    version_string = caplog.records[1].getMessage()
+    expected_items_pattern = "',\n '".join(
+        f"{item}': '[^']+('\n +'[^']+)*" for item in items
+    )
+    assert re.search(r"^Versions:\n{'" + expected_items_pattern + "'}$", version_string)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 3fcba4ef2..8ecba41bf 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -349,11 +349,6 @@ class HttpTestCase(unittest.TestCase):
         request = Request(self.getURL("host"), headers={"Host": host})
         return self.download_request(request, Spider("foo")).addCallback(_test)
 
-        d = self.download_request(request, Spider("foo"))
-        d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEqual, b"localhost")
-        return d
-
     def test_content_length_zero_bodyless_post_request_headers(self):
         """Tests if "Content-Length: 0" is sent for bodyless POST requests.
 
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index e650b4936..c581e7596 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -178,7 +178,7 @@ class ProcessExceptionInvalidOutput(ManagerTestCase):
 
         class InvalidProcessExceptionMiddleware:
             def process_request(self, request, spider):
-                raise Exception
+                raise RuntimeError
 
             def process_exception(self, request, exception, spider):
                 return 1
diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py
index b7312bbcd..15129e31f 100644
--- a/tests/test_extension_periodic_log.py
+++ b/tests/test_extension_periodic_log.py
@@ -192,4 +192,3 @@ class TestPeriodicLog(unittest.TestCase):
             {"PERIODIC_LOG_STATS": {"include": ["downloader/"], "exclude": ["bytes"]}},
             lambda k, v: "downloader/" in k and "bytes" not in k,
         )
-        #
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 2debbe0d7..b087aaab1 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -727,7 +727,7 @@ class ExceptionJsonItemExporter(JsonItemExporter):
     """JsonItemExporter that throws an exception every time export_item is called."""
 
     def export_item(self, _):
-        raise Exception("foo")
+        raise RuntimeError("foo")
 
 
 class FeedExportTest(FeedExportTestBase):
diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py
index f9b841a61..4bf22f6a0 100644
--- a/tests/test_loader_deprecated.py
+++ b/tests/test_loader_deprecated.py
@@ -330,10 +330,10 @@ class BasicItemLoaderTest(unittest.TestCase):
         il.add_value("name", ["mar", "ta"])
         self.assertEqual(il.get_output_value("name"), "Mar Ta")
 
-        class TakeFirstItemLoader(TestItemLoader):
+        class TakeFirstItemLoader2(TestItemLoader):
             name_out = Join("<br>")
 
-        il = TakeFirstItemLoader()
+        il = TakeFirstItemLoader2()
         il.add_value("name", ["mar", "ta"])
         self.assertEqual(il.get_output_value("name"), "Mar<br>Ta")
 
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index f2a57bd88..ba64ba721 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -78,7 +78,7 @@ class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase):
 
         class RaiseExceptionProcessSpiderOutputMiddleware:
             def process_spider_output(self, response, result, spider):
-                raise Exception
+                raise RuntimeError
 
         self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware())
         self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 4c19d167f..e51957497 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -247,7 +247,6 @@ class NotGeneratorFailMiddleware:
             r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
             out.append(r)
         raise ReferenceError
-        return out
 
     def process_spider_exception(self, response, exception, spider):
         method = f"{self.__class__.__name__}.process_spider_exception"
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index 1c93829e9..e00f69573 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -26,7 +26,7 @@ class AsyncioTest(TestCase):
         with warnings.catch_warnings(record=True) as w:
             install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
             self.assertEqual(len(w), 0)
-        from twisted.internet import reactor
+        from twisted.internet import reactor  # pylint: disable=reimported
 
         assert original_reactor == reactor
 
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 3f153bdc0..e4ab97e5d 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -153,7 +153,7 @@ class AsyncDefTestsuiteTest(unittest.TestCase):
     @mark.xfail(reason="Checks that the test is actually executed", strict=True)
     @deferred_f_from_coro_f
     async def test_deferred_f_from_coro_f_xfail(self):
-        raise Exception("This is expected to be raised")
+        raise RuntimeError("This is expected to be raised")
 
 
 class AsyncCooperatorTest(unittest.TestCase):
diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py
index c7774751e..480729d11 100644
--- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py
+++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py
@@ -32,7 +32,6 @@ def top_level_return_none():
 https://example.org
 """
     yield url
-    return
 
 
 def generator_that_returns_stuff():
@@ -103,11 +102,9 @@ https://example.org
     def test_generators_return_none(self):
         def f2():
             yield 1
-            return None
 
         def g2():
             yield 1
-            return
 
         def h2():
             yield 1
@@ -132,7 +129,6 @@ https://example.org
 https://example.org
         """
             yield url
-            return
 
         def l2():
             return
@@ -181,12 +177,10 @@ https://example.org
         @decorator
         def f3():
             yield 1
-            return None
 
         @decorator
         def g3():
             yield 1
-            return
 
         @decorator
         def h3():
@@ -215,7 +209,6 @@ https://example.org
 https://example.org
         """
             yield url
-            return
 
         @decorator
         def l3():
diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py
index dd1d26448..ae59d0137 100644
--- a/tests/test_utils_spider.py
+++ b/tests/test_utils_spider.py
@@ -26,7 +26,7 @@ class UtilsSpidersTestCase(unittest.TestCase):
         self.assertEqual(list(iterate_spider_output([r, i, o])), [r, i, o])
 
     def test_iter_spider_classes(self):
-        import tests.test_utils_spider  # pylint: disable=import-self
+        import tests.test_utils_spider  # noqa: PLW0406  # pylint: disable=import-self
 
         it = iter_spider_classes(tests.test_utils_spider)
         self.assertEqual(set(it), {MySpider1, MySpider2})
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 59a95b0e2..a15ad749d 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -327,8 +327,6 @@ def create_guess_scheme_t(args):
 def create_skipped_scheme_t(args):
     def do_expected(self):
         raise unittest.SkipTest(args[2])
-        url = guess_scheme(args[0])
-        assert url.startswith(args[1])
 
     return do_expected