Repository: scrapy/scrapy (python)

Issue:
Allow changing the log severity of DropItem
By default, when an item is dropped, a `WARNING` message is logged. [It would be nice to make that more flexible](https://github.com/zytedata/zyte-common-items/pull/126#discussion_r1900692613).

Pull request description:
 Flexible severity of logging level when items are dropped
This PR is created to resolve #6603. I have added a new default setting called `DEFAULT_DROPITEM_LOG_LEVEL`, set to `WARNING` by default. I have modified the exception for `DropItem` to use the default log value unless severity is provided by the component. Modified the documentation under docs/topics/settings.rst.

This PR is currently drafted because tests are not written yet. Please suggest whether I should create a new test file or can I use any existing test file to include the tests?

Patch:
diff --git a/docs/conf.py b/docs/conf.py
index fd8165db3..8196b6934 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -258,6 +258,10 @@ coverage_ignore_pyobjects = [
     # Base classes of downloader middlewares are implementation details that
     # are not meant for users.
     r"^scrapy\.downloadermiddlewares\.\w*?\.Base\w*?Middleware",
+    # The interface methods of duplicate request filtering classes are already
+    # covered in the interface documentation part of the DUPEFILTER_CLASS
+    # setting documentation.
+    r"^scrapy\.dupefilters\.[A-Z]\w*?\.(from_settings|request_seen|open|close|log)$",
     # Private exception used by the command-line interface implementation.
     r"^scrapy\.exceptions\.UsageError",
     # Methods of BaseItemExporter subclasses are only documented in
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index af7885a45..1ab8f588f 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -797,9 +797,12 @@ OffsiteMiddleware
    :attr:`~scrapy.Spider.allowed_domains` attribute, or the
    attribute is empty, the offsite middleware will allow all requests.
 
-   If the request has the :attr:`~scrapy.Request.dont_filter` attribute
-   set, the offsite middleware will allow the request even if its domain is not
-   listed in allowed domains.
+   .. reqmeta:: allow_offsite
+
+   If the request has the :attr:`~scrapy.Request.dont_filter` attribute set to
+   ``True`` or :attr:`Request.meta` has ``allow_offsite`` set to ``True``, then
+   the OffsiteMiddleware will allow the request even if its domain is not listed
+   in allowed domains.
 
 RedirectMiddleware
 ------------------
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 1bb1a10a4..b187f3aaf 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -145,9 +145,9 @@ Request objects
     :type priority: int
 
     :param dont_filter: indicates that this request should not be filtered by
-       the scheduler. This is used when you want to perform an identical
-       request multiple times, to ignore the duplicates filter. Use it with
-       care, or you will get into crawling loops. Default to ``False``.
+       the scheduler or some middlewares. This is used when you want to perform
+       an identical request multiple times, to ignore the duplicates filter.
+       Use it with care, or you will get into crawling loops. Default to ``False``.
     :type dont_filter: bool
 
     :param errback: a function that will be called if any exception was
@@ -660,6 +660,7 @@ are some special keys recognized by Scrapy and its built-in extensions.
 
 Those are:
 
+* :reqmeta:`allow_offsite`
 * :reqmeta:`autothrottle_dont_adjust_delay`
 * :reqmeta:`bindaddress`
 * :reqmeta:`cookiejar`
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 76904a26e..8801434d8 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -418,6 +418,38 @@ This setting also affects :setting:`DOWNLOAD_DELAY` and
 :ref:`topics-autothrottle`: if :setting:`CONCURRENT_REQUESTS_PER_IP`
 is non-zero, download delay is enforced per IP, not per domain.
 
+.. setting:: DEFAULT_DROPITEM_LOG_LEVEL
+
+DEFAULT_DROPITEM_LOG_LEVEL
+--------------------------
+
+Default: ``"WARNING"``
+
+Default :ref:`log level <levels>` of messages about dropped items.
+
+When an item is dropped by raising :exc:`scrapy.exceptions.DropItem` from the
+:func:`process_item` method of an :ref:`item pipeline <topics-item-pipeline>`,
+a message is logged, and by default its log level is the one configured in this
+setting.
+
+You may specify this log level as an integer (e.g. ``20``), as a log level
+constant (e.g. ``logging.INFO``) or as a string with the name of a log level
+constant (e.g. ``"INFO"``).
+
+When writing an item pipeline, you can force a different log level by setting
+:attr:`scrapy.exceptions.DropItem.log_level` in your
+:exc:`scrapy.exceptions.DropItem` exception. For example:
+
+.. code-block:: python
+
+   from scrapy.exceptions import DropItem
+
+
+   class MyPipeline:
+       def process_item(self, item, spider):
+           if not item.get("price"):
+               raise DropItem("Missing price data", log_level="INFO")
+           return item
 
 .. setting:: DEFAULT_ITEM_CLASS
 
@@ -955,15 +987,79 @@ Default: ``'scrapy.dupefilters.RFPDupeFilter'``
 
 The class used to detect and filter duplicate requests.
 
-The default (``RFPDupeFilter``) filters based on the
+The default, :class:`~scrapy.dupefilters.RFPDupeFilter`, filters based on the
 :setting:`REQUEST_FINGERPRINTER_CLASS` setting.
 
-You can disable filtering of duplicate requests by setting
-:setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``.
-Be very careful about this however, because you can get into crawling loops.
-It's usually a better idea to set the ``dont_filter`` parameter to
-``True`` on the specific :class:`~scrapy.Request` that should not be
-filtered.
+To change how duplicates are checked, you can point :setting:`DUPEFILTER_CLASS`
+to a custom subclass of :class:`~scrapy.dupefilters.RFPDupeFilter` that
+overrides its ``__init__`` method to use a :ref:`different request
+fingerprinting class <custom-request-fingerprinter>`. For example:
+
+.. code-block:: python
+
+    from scrapy.dupefilters import RFPDupeFilter
+    from scrapy.utils.request import fingerprint
+
+
+    class CustomRequestFingerprinter:
+        def fingerprint(self, request):
+            return fingerprint(request, include_headers=["X-ID"])
+
+
+    class CustomDupeFilter(RFPDupeFilter):
+
+        def __init__(self, path=None, debug=False, *, fingerprinter=None):
+            super().__init__(
+                path=path, debug=debug, fingerprinter=CustomRequestFingerprinter()
+            )
+
+To disable duplicate request filtering set :setting:`DUPEFILTER_CLASS` to
+``'scrapy.dupefilters.BaseDupeFilter'``. Note that not filtering out duplicate
+requests may cause crawling loops. It is usually better to set
+the ``dont_filter`` parameter to ``True`` on the ``__init__`` method of a
+specific :class:`~scrapy.Request` object that should not be filtered out.
+
+A class assigned to :setting:`DUPEFILTER_CLASS` must implement the following
+interface::
+
+    class MyDupeFilter:
+
+        @classmethod
+        def from_settings(cls, settings):
+            """Returns an instance of this duplicate request filtering class
+            based on the current crawl settings."""
+            return cls()
+
+        def request_seen(self, request):
+            """Returns ``True`` if *request* is a duplicate of another request
+            seen in a previous call to :meth:`request_seen`, or ``False``
+            otherwise."""
+            return False
+
+        def open(self):
+            """Called before the spider opens. It may return a deferred."""
+            pass
+
+        def close(self, reason):
+            """Called before the spider closes. It may return a deferred."""
+            pass
+
+        def log(self, request, spider):
+            """Logs that a request has been filtered out.
+
+            It is called right after a call to :meth:`request_seen` that
+            returns ``True``.
+
+            If :meth:`request_seen` always returns ``False``, such as in the
+            case of :class:`~scrapy.dupefilters.BaseDupeFilter`, this method
+            may be omitted.
+            """
+            pass
+
+.. autoclass:: scrapy.dupefilters.BaseDupeFilter
+
+.. autoclass:: scrapy.dupefilters.RFPDupeFilter
+
 
 .. setting:: DUPEFILTER_DEBUG
 
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 48f462c65..b08fd3409 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -89,11 +89,16 @@ def _get_commands_dict(
     return cmds
 
 
+def _get_project_only_cmds(settings: BaseSettings) -> set[str]:
+    return set(_get_commands_dict(settings, inproject=True)) - set(
+        _get_commands_dict(settings, inproject=False)
+    )
+
+
 def _pop_command_name(argv: list[str]) -> str | None:
-    for i, arg in enumerate(argv[1:]):
-        if not arg.startswith("-"):
-            del argv[i]
-            return arg
+    for i in range(1, len(argv)):
+        if not argv[i].startswith("-"):
+            return argv.pop(i)
     return None
 
 
@@ -121,11 +126,25 @@ def _print_commands(settings: BaseSettings, inproject: bool) -> None:
     print('Use "scrapy <command> -h" to see more info about a command')
 
 
+def _print_unknown_command_msg(
+    settings: BaseSettings, cmdname: str, inproject: bool
+) -> None:
+    proj_only_cmds = _get_project_only_cmds(settings)
+    if cmdname in proj_only_cmds and not inproject:
+        cmd_list = ", ".join(sorted(proj_only_cmds))
+        print(
+            f"The {cmdname} command is not available from this location.\n"
+            f"These commands are only available from within a project: {cmd_list}.\n"
+        )
+    else:
+        print(f"Unknown command: {cmdname}\n")
+
+
 def _print_unknown_command(
     settings: BaseSettings, cmdname: str, inproject: bool
 ) -> None:
     _print_header(settings, inproject)
-    print(f"Unknown command: {cmdname}\n")
+    _print_unknown_command_msg(settings, cmdname, inproject)
     print('Use "scrapy" to see available commands')
 
 
diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py
index a69f531a7..a2cff65e7 100644
--- a/scrapy/downloadermiddlewares/offsite.py
+++ b/scrapy/downloadermiddlewares/offsite.py
@@ -40,7 +40,11 @@ class OffsiteMiddleware:
         self.process_request(request, spider)
 
     def process_request(self, request: Request, spider: Spider) -> None:
-        if request.dont_filter or self.should_follow(request, spider):
+        if (
+            request.dont_filter
+            or request.meta.get("allow_offsite")
+            or self.should_follow(request, spider)
+        ):
             return
         domain = urlparse_cached(request).hostname
         if domain and domain not in self.domains_seen:
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index caf69daf4..a3e2c5eb4 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -4,6 +4,7 @@ import logging
 import warnings
 from pathlib import Path
 from typing import TYPE_CHECKING
+from warnings import warn
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.job import job_dir
@@ -26,6 +27,9 @@ if TYPE_CHECKING:
 
 
 class BaseDupeFilter:
+    """Dummy duplicate request filtering class (:setting:`DUPEFILTER_CLASS`)
+    that does not filter out any request."""
+
     @classmethod
     def from_settings(cls, settings: BaseSettings) -> Self:
         warnings.warn(
@@ -50,10 +54,19 @@ class BaseDupeFilter:
 
     def log(self, request: Request, spider: Spider) -> None:
         """Log that a request has been filtered"""
+        warn(
+            "Calling BaseDupeFilter.log() is deprecated.",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
 
 
 class RFPDupeFilter(BaseDupeFilter):
-    """Request Fingerprint duplicates filter"""
+    """Duplicate request filtering class (:setting:`DUPEFILTER_CLASS`) that
+    filters out requests with the canonical
+    (:func:`w3lib.url.canonicalize_url`) :attr:`~scrapy.http.Request.url`,
+    :attr:`~scrapy.http.Request.method` and :attr:`~scrapy.http.Request.body`.
+    """
 
     def __init__(
         self,
@@ -117,6 +130,7 @@ class RFPDupeFilter(BaseDupeFilter):
         return False
 
     def request_fingerprint(self, request: Request) -> str:
+        """Returns a string that uniquely identifies the specified request."""
         return self.fingerprinter.fingerprint(request).hex()
 
     def close(self, reason: str) -> None:
diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py
index 96566ba86..f37f881a7 100644
--- a/scrapy/exceptions.py
+++ b/scrapy/exceptions.py
@@ -5,6 +5,8 @@ These exceptions are documented in docs/topics/exceptions.rst. Please don't add
 new exceptions here without documenting them there.
 """
 
+from __future__ import annotations
+
 from typing import Any
 
 # Internal
@@ -58,6 +60,10 @@ class StopDownload(Exception):
 class DropItem(Exception):
     """Drop item from the item pipeline"""
 
+    def __init__(self, message: str, log_level: str | None = None):
+        super().__init__(message)
+        self.log_level = log_level
+
 
 class NotSupported(Exception):
     """Indicates a feature or method is not supported"""
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 76f9c7856..f10e91beb 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -120,8 +120,12 @@ class LogFormatter:
         spider: Spider,
     ) -> LogFormatterResult:
         """Logs a message when an item is dropped while it is passing through the item pipeline."""
+        if (level := getattr(exception, "log_level", None)) is None:
+            level = spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"]
+        if isinstance(level, str):
+            level = getattr(logging, level)
         return {
-            "level": logging.WARNING,
+            "level": level,
             "msg": DROPPEDMSG,
             "args": {
                 "exception": exception,
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 0bbde118e..7ef365f68 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -49,6 +49,8 @@ CONCURRENT_REQUESTS_PER_IP = 0
 COOKIES_ENABLED = True
 COOKIES_DEBUG = False
 
+DEFAULT_DROPITEM_LOG_LEVEL = "WARNING"
+
 DEFAULT_ITEM_CLASS = "scrapy.item.Item"
 
 DEFAULT_REQUEST_HEADERS = {
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index d3ed64ef5..95e753830 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -61,7 +61,11 @@ class OffsiteMiddleware:
     def _filter(self, request: Any, spider: Spider) -> bool:
         if not isinstance(request, Request):
             return True
-        if request.dont_filter or self.should_follow(request, spider):
+        if (
+            request.dont_filter
+            or request.meta.get("allow_offsite")
+            or self.should_follow(request, spider)
+        ):
             return True
         domain = urlparse_cached(request).hostname
         if domain and domain not in self.domains_seen:


Test patch:
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 9d5720b98..50f093043 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -9,6 +9,7 @@ import re
 import subprocess
 import sys
 from contextlib import contextmanager
+from io import StringIO
 from itertools import chain
 from pathlib import Path
 from shutil import copytree, rmtree
@@ -16,12 +17,13 @@ from stat import S_IWRITE as ANYONE_WRITE_PERMISSION
 from tempfile import TemporaryFile, mkdtemp
 from threading import Timer
 from typing import TYPE_CHECKING
-from unittest import skipIf
+from unittest import mock, skipIf
 
 from pytest import mark
 from twisted.trial import unittest
 
 import scrapy
+from scrapy.cmdline import _pop_command_name, _print_unknown_command_msg
 from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view
 from scrapy.commands.startproject import IGNORE
 from scrapy.settings import Settings
@@ -652,6 +654,24 @@ class MiscCommandsTest(CommandTest):
     def test_list(self):
         self.assertEqual(0, self.call("list"))
 
+    def test_command_not_found(self):
+        na_msg = """
+The list command is not available from this location.
+These commands are only available from within a project: check, crawl, edit, list, parse.
+"""
+        not_found_msg = """
+Unknown command: abc
+"""
+        params = [
+            ("list", 0, na_msg),
+            ("abc", 0, not_found_msg),
+            ("abc", 1, not_found_msg),
+        ]
+        for cmdname, inproject, message in params:
+            with mock.patch("sys.stdout", new=StringIO()) as out:
+                _print_unknown_command_msg(Settings(), cmdname, inproject)
+                self.assertEqual(out.getvalue().strip(), message.strip())
+
 
 class RunSpiderCommandTest(CommandTest):
     spider_filename = "myspider.py"
@@ -1143,3 +1163,29 @@ class HelpMessageTest(CommandTest):
         for command in self.commands:
             _, out, _ = self.proc(command, "-h")
             self.assertIn("Usage", out)
+
+
+class PopCommandNameTest(unittest.TestCase):
+    def test_valid_command(self):
+        argv = ["scrapy", "crawl", "my_spider"]
+        command = _pop_command_name(argv)
+        self.assertEqual(command, "crawl")
+        self.assertEqual(argv, ["scrapy", "my_spider"])
+
+    def test_no_command(self):
+        argv = ["scrapy"]
+        command = _pop_command_name(argv)
+        self.assertIsNone(command)
+        self.assertEqual(argv, ["scrapy"])
+
+    def test_option_before_command(self):
+        argv = ["scrapy", "-h", "crawl"]
+        command = _pop_command_name(argv)
+        self.assertEqual(command, "crawl")
+        self.assertEqual(argv, ["scrapy", "-h"])
+
+    def test_option_after_command(self):
+        argv = ["scrapy", "crawl", "-h"]
+        command = _pop_command_name(argv)
+        self.assertEqual(command, "crawl")
+        self.assertEqual(argv, ["scrapy", "-h"])
diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py
index fec56a39f..23a1d06da 100644
--- a/tests/test_downloadermiddleware_offsite.py
+++ b/tests/test_downloadermiddleware_offsite.py
@@ -64,6 +64,37 @@ def test_process_request_dont_filter(value, filtered):
         assert mw.process_request(request, spider) is None
 
 
+@pytest.mark.parametrize(
+    ("allow_offsite", "dont_filter", "filtered"),
+    (
+        (True, UNSET, False),
+        (True, None, False),
+        (True, False, False),
+        (True, True, False),
+        (False, UNSET, True),
+        (False, None, True),
+        (False, False, True),
+        (False, True, False),
+    ),
+)
+def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered):
+    crawler = get_crawler(Spider)
+    spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
+    mw = OffsiteMiddleware.from_crawler(crawler)
+    mw.spider_opened(spider)
+    kwargs = {"meta": {}}
+    if allow_offsite is not UNSET:
+        kwargs["meta"]["allow_offsite"] = allow_offsite
+    if dont_filter is not UNSET:
+        kwargs["dont_filter"] = dont_filter
+    request = Request("https://b.example", **kwargs)
+    if filtered:
+        with pytest.raises(IgnoreRequest):
+            mw.process_request(request, spider)
+    else:
+        assert mw.process_request(request, spider) is None
+
+
 @pytest.mark.parametrize(
     "value",
     (
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 4fd648f48..703c23529 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -4,11 +4,13 @@ import sys
 import tempfile
 import unittest
 from pathlib import Path
+from warnings import catch_warnings
 
 from testfixtures import LogCapture
 
 from scrapy.core.scheduler import Scheduler
-from scrapy.dupefilters import RFPDupeFilter
+from scrapy.dupefilters import BaseDupeFilter, RFPDupeFilter
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request
 from scrapy.utils.python import to_bytes
 from scrapy.utils.test import get_crawler
@@ -252,3 +254,18 @@ class RFPDupeFilterTest(unittest.TestCase):
             )
 
             dupefilter.close("finished")
+
+
+class BaseDupeFilterTestCase(unittest.TestCase):
+    def test_log_deprecation(self):
+        dupefilter = _get_dupefilter(
+            settings={"DUPEFILTER_CLASS": BaseDupeFilter},
+        )
+        with catch_warnings(record=True) as warning_list:
+            dupefilter.log(None, None)
+        self.assertEqual(len(warning_list), 1)
+        self.assertEqual(
+            str(warning_list[0].message),
+            "Calling BaseDupeFilter.log() is deprecated.",
+        )
+        self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 61a9f3f8d..e5d077858 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -1,5 +1,7 @@
+import logging
 import unittest
 
+import pytest
 from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.python.failure import Failure
@@ -26,6 +28,7 @@ class LogFormatterTestCase(unittest.TestCase):
     def setUp(self):
         self.formatter = LogFormatter()
         self.spider = Spider("default")
+        self.spider.crawler = get_crawler()
 
     def test_crawled_with_referer(self):
         req = Request("http://www.example.com")
@@ -68,6 +71,62 @@ class LogFormatterTestCase(unittest.TestCase):
         assert all(isinstance(x, str) for x in lines)
         self.assertEqual(lines, ["Dropped: \u2018", "{}"])
 
+    def test_dropitem_default_log_level(self):
+        item = {}
+        exception = DropItem("Test drop")
+        response = Response("http://www.example.com")
+        spider = Spider("foo")
+        spider.crawler = get_crawler(Spider)
+
+        logkws = self.formatter.dropped(item, exception, response, spider)
+        self.assertEqual(logkws["level"], logging.WARNING)
+
+        spider.crawler.settings.frozen = False
+        spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = logging.INFO
+        spider.crawler.settings.frozen = True
+        logkws = self.formatter.dropped(item, exception, response, spider)
+        self.assertEqual(logkws["level"], logging.INFO)
+
+        spider.crawler.settings.frozen = False
+        spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = "INFO"
+        spider.crawler.settings.frozen = True
+        logkws = self.formatter.dropped(item, exception, response, spider)
+        self.assertEqual(logkws["level"], logging.INFO)
+
+        spider.crawler.settings.frozen = False
+        spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = 10
+        spider.crawler.settings.frozen = True
+        logkws = self.formatter.dropped(item, exception, response, spider)
+        self.assertEqual(logkws["level"], logging.DEBUG)
+
+        spider.crawler.settings.frozen = False
+        spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = 0
+        spider.crawler.settings.frozen = True
+        logkws = self.formatter.dropped(item, exception, response, spider)
+        self.assertEqual(logkws["level"], logging.NOTSET)
+
+        unsupported_value = object()
+        spider.crawler.settings.frozen = False
+        spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = unsupported_value
+        spider.crawler.settings.frozen = True
+        logkws = self.formatter.dropped(item, exception, response, spider)
+        self.assertEqual(logkws["level"], unsupported_value)
+
+        with pytest.raises(TypeError):
+            logging.log(logkws["level"], "message")
+
+    def test_dropitem_custom_log_level(self):
+        item = {}
+        response = Response("http://www.example.com")
+
+        exception = DropItem("Test drop", log_level="INFO")
+        logkws = self.formatter.dropped(item, exception, response, self.spider)
+        self.assertEqual(logkws["level"], logging.INFO)
+
+        exception = DropItem("Test drop", log_level="ERROR")
+        logkws = self.formatter.dropped(item, exception, response, self.spider)
+        self.assertEqual(logkws["level"], logging.ERROR)
+
     def test_item_error(self):
         # In practice, the complete traceback is shown by passing the
         # 'exc_info' argument to the logging function
@@ -145,6 +204,7 @@ class LogformatterSubclassTest(LogFormatterTestCase):
     def setUp(self):
         self.formatter = LogFormatterSubclass()
         self.spider = Spider("default")
+        self.spider.crawler = get_crawler(Spider)
 
     def test_crawled_with_referer(self):
         req = Request("http://www.example.com")
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index 837f1c2c8..906928e01 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -29,6 +29,7 @@ class TestOffsiteMiddleware(TestCase):
             Request("http://scrapy.org/1"),
             Request("http://sub.scrapy.org/1"),
             Request("http://offsite.tld/letmepass", dont_filter=True),
+            Request("http://offsite-2.tld/allow", meta={"allow_offsite": True}),
             Request("http://scrapy.test.org/"),
             Request("http://scrapy.test.org:8000/"),
         ]