Repository: scrapy/scrapy
Pull request #6527 (2024-11-05 18:46:25)

Replace Slot.throttle with Request.meta['dont_throttle']
https://github.com/scrapy/scrapy/pull/6246#discussion_r1827583470

cc @GeorgeA92, @VMRuiz 


Patch:
diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst
index 8a13b8976..5bd72fa15 100644
--- a/docs/topics/autothrottle.rst
+++ b/docs/topics/autothrottle.rst
@@ -21,9 +21,14 @@ Design goals
 How it works
 ============
 
-AutoThrottle extension adjusts download delays dynamically to make spider send
-:setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` concurrent requests on average
-to each remote website.
+Scrapy allows defining the concurrency and delay of different download slots,
+e.g. through the :setting:`DOWNLOAD_SLOTS` setting. By default requests are
+assigned to slots based on their URL domain, although it is possible to
+customize the download slot of any request.
+
+The AutoThrottle extension adjusts the delay of each download slot dynamically,
+to make your spider send :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` concurrent
+requests on average to each remote website.
 
 It uses download latency to compute the delays. The main idea is the
 following: if a server needs ``latency`` seconds to respond, a client
@@ -47,18 +52,6 @@ effect, but there are some important differences:
 
 AutoThrottle doesn't have these issues.
 
-Disabling throttling on a downloader slot
-=========================================
-
-It is possible to disable AutoThrottle for a specific download slot at run time
-by setting its ``throttle`` attribute to ``False``, e.g. using
-:setting:`DOWNLOAD_SLOTS`.
-
-Note, however, that AutoThrottle still determines the starting delay of every
-slot by setting the ``download_delay`` attribute on the running spider. You
-might want to set a custom value for the ``delay`` attribute of the slot, e.g.
-using :setting:`DOWNLOAD_SLOTS`.
-
 Throttling algorithm
 ====================
 
@@ -92,6 +85,33 @@ callback, for example, and unable to attend downloads. However, these latencies
 should still give a reasonable estimate of how busy Scrapy (and ultimately, the
 server) is, and this extension builds on that premise.
 
+.. reqmeta:: autothrottle_dont_adjust_delay
+
+Prevent specific requests from triggering slot delay adjustments
+================================================================
+
+AutoThrottle adjusts the delay of download slots based on the latencies of
+responses that belong to that download slot. The only exceptions are non-200
+responses, which are only taken into account to increase that delay, but
+ignored if they would decrease that delay.
+
+You can also set the ``autothrottle_dont_adjust_delay`` request metadata key to
+``True`` in any request to prevent its response latency from impacting the
+delay of its download slot:
+
+.. code-block:: python
+
+    from scrapy import Request
+
+    Request("https://example.com", meta={"autothrottle_dont_adjust_delay": True})
+
+Note, however, that AutoThrottle still determines the starting delay of every
+download slot by setting the ``download_delay`` attribute on the running
+spider. If you want AutoThrottle not to impact a download slot at all, in
+addition to setting this meta key in all requests that use that download slot,
+you might want to set a custom value for the ``delay`` attribute of that
+download slot, e.g. using :setting:`DOWNLOAD_SLOTS`.
+
 Settings
 ========
 
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 3c2843bc1..7c15b67e8 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -668,6 +668,7 @@ are some special keys recognized by Scrapy and its built-in extensions.
 
 Those are:
 
+* :reqmeta:`autothrottle_dont_adjust_delay`
 * :reqmeta:`bindaddress`
 * :reqmeta:`cookiejar`
 * :reqmeta:`dont_cache`
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 116e8226e..cce4a7b3e 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -845,12 +845,7 @@ Allows to define concurrency/delay parameters on per slot (domain) basis:
     .. code-block:: python
 
         DOWNLOAD_SLOTS = {
-            "quotes.toscrape.com": {
-                "concurrency": 1,
-                "delay": 2,
-                "randomize_delay": False,
-                "throttle": False,
-            },
+            "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False},
             "books.toscrape.com": {"delay": 3, "randomize_delay": False},
         }
 
@@ -862,9 +857,6 @@ Allows to define concurrency/delay parameters on per slot (domain) basis:
     -   :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency``
     -   :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay``
 
-    There is no global setting for ``throttle``, whose default value is
-    ``None``.
-
 
 .. setting:: DOWNLOAD_TIMEOUT
 
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 1cc0422b7..5040741e2 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -36,13 +36,10 @@ class Slot:
         concurrency: int,
         delay: float,
         randomize_delay: bool,
-        *,
-        throttle: bool | None = None,
     ):
         self.concurrency: int = concurrency
         self.delay: float = delay
         self.randomize_delay: bool = randomize_delay
-        self.throttle = throttle
 
         self.active: set[Request] = set()
         self.queue: deque[tuple[Request, Deferred[Response]]] = deque()
@@ -67,15 +64,13 @@ class Slot:
         return (
             f"{cls_name}(concurrency={self.concurrency!r}, "
             f"delay={self.delay:.2f}, "
-            f"randomize_delay={self.randomize_delay!r}, "
-            f"throttle={self.throttle!r})"
+            f"randomize_delay={self.randomize_delay!r})"
         )
 
     def __str__(self) -> str:
         return (
             f"<downloader.Slot concurrency={self.concurrency!r} "
             f"delay={self.delay:.2f} randomize_delay={self.randomize_delay!r} "
-            f"throttle={self.throttle!r} "
             f"len(active)={len(self.active)} len(queue)={len(self.queue)} "
             f"len(transferring)={len(self.transferring)} "
             f"lastseen={datetime.fromtimestamp(self.lastseen).isoformat()}>"
@@ -146,8 +141,7 @@ class Downloader:
                 slot_settings.get("delay", delay),
             )
             randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay)
-            throttle = slot_settings.get("throttle", None)
-            new_slot = Slot(conc, delay, randomize_delay, throttle=throttle)
+            new_slot = Slot(conc, delay, randomize_delay)
             self.slots[key] = new_slot
 
         return key, self.slots[key]
diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py
index d4b4f0e9d..cdb0671ae 100644
--- a/scrapy/extensions/throttle.py
+++ b/scrapy/extensions/throttle.py
@@ -64,7 +64,11 @@ class AutoThrottle:
     ) -> None:
         key, slot = self._get_slot(request, spider)
         latency = request.meta.get("download_latency")
-        if latency is None or slot is None or slot.throttle is False:
+        if (
+            latency is None
+            or slot is None
+            or request.meta.get("autothrottle_dont_adjust_delay", False) is True
+        ):
             return
 
         olddelay = slot.delay
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 12c09839f..516218347 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -111,7 +111,7 @@ def md5sum(file: IO[bytes]) -> str:
     """
     warnings.warn(
         (
-            "The scrapy.utils.misc.md5sum function is deprecated and will be "
+            "The scrapy.utils.misc.md5sum function is deprecated, and will be "
             "removed in a future version of Scrapy."
         ),
         ScrapyDeprecationWarning,
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index d970f5da5..6268af728 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -8,14 +8,12 @@ import gc
 import inspect
 import re
 import sys
-import warnings
 import weakref
 from collections.abc import AsyncIterable, Iterable, Mapping
 from functools import partial, wraps
 from itertools import chain
 from typing import TYPE_CHECKING, Any, TypeVar, overload
 
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.asyncgen import as_async_generator
 
 if TYPE_CHECKING:
@@ -49,11 +47,6 @@ def flatten(x: Iterable[Any]) -> list[Any]:
     >>> flatten(["foo", ["baz", 42], "bar"])
     ['foo', 'baz', 42, 'bar']
     """
-    warnings.warn(
-        "The flatten function is deprecated and will be removed in a future version of Scrapy.",
-        category=ScrapyDeprecationWarning,
-        stacklevel=2,
-    )
     return list(iflatten(x))
 
 
@@ -61,11 +54,6 @@ def iflatten(x: Iterable[Any]) -> Iterable[Any]:
     """iflatten(sequence) -> iterator
 
     Similar to ``.flatten()``, but returns iterator instead"""
-    warnings.warn(
-        "The iflatten function is deprecated and will be removed in a future version of Scrapy.",
-        category=ScrapyDeprecationWarning,
-        stacklevel=2,
-    )
     for el in x:
         if is_listlike(el):
             yield from iflatten(el)
@@ -284,11 +272,6 @@ def equal_attributes(
     obj1: Any, obj2: Any, attributes: list[str | Callable[[Any], Any]] | None
 ) -> bool:
     """Compare two objects attributes"""
-    warnings.warn(
-        "The equal_attributes function is deprecated and will be removed in a future version of Scrapy.",
-        category=ScrapyDeprecationWarning,
-        stacklevel=2,
-    )
     # not attributes given return False by default
     if not attributes:
         return False
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 20e3151da..7848b9318 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -130,7 +130,7 @@ class RequestFingerprinter:
         if implementation != "SENTINEL":
             message = (
                 "'REQUEST_FINGERPRINTER_IMPLEMENTATION' is a deprecated setting.\n"
-                "It will be removed in a future version of Scrapy."
+                "And it will be removed in future version of Scrapy."
             )
             warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
         self._fingerprint = fingerprint
@@ -147,11 +147,6 @@ def request_authenticate(
     """Authenticate the given request (in place) using the HTTP basic access
     authentication mechanism (RFC 2617) and the given username and password
     """
-    warnings.warn(
-        "The request_authenticate function is deprecated and will be removed in a future version of Scrapy.",
-        category=ScrapyDeprecationWarning,
-        stacklevel=2,
-    )
     request.headers["Authorization"] = basic_auth_header(username, password)
 
 
diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py
index 308e351c6..3b4f67f00 100644
--- a/scrapy/utils/serialize.py
+++ b/scrapy/utils/serialize.py
@@ -1,13 +1,11 @@
 import datetime
 import decimal
 import json
-import warnings
 from typing import Any
 
 from itemadapter import ItemAdapter, is_item
 from twisted.internet import defer
 
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 
 
@@ -38,10 +36,4 @@ class ScrapyJSONEncoder(json.JSONEncoder):
 
 
 class ScrapyJSONDecoder(json.JSONDecoder):
-    def __init__(self, *args, **kwargs):
-        warnings.warn(
-            "The ScrapyJSONDecoder class is deprecated and will be removed in a future version of Scrapy.",
-            category=ScrapyDeprecationWarning,
-            stacklevel=2,
-        )
-        super().__init__(*args, **kwargs)
+    pass


Test patch:
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 92b73a91a..d65f2a76d 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -6,7 +6,6 @@ from __future__ import annotations
 
 import asyncio
 import os
-import warnings
 from importlib import import_module
 from pathlib import Path
 from posixpath import split
@@ -17,7 +16,6 @@ from twisted.trial.unittest import SkipTest
 
 from scrapy import Spider
 from scrapy.crawler import Crawler
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.boto import is_botocore_available
 
 if TYPE_CHECKING:
@@ -127,11 +125,6 @@ def assert_samelines(
     """Asserts text1 and text2 have the same lines, ignoring differences in
     line endings between platforms
     """
-    warnings.warn(
-        "The assert_samelines function is deprecated and will be removed in a future version of Scrapy.",
-        category=ScrapyDeprecationWarning,
-        stacklevel=2,
-    )
     testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg)
 
 
diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py
index 81cff4947..d929a9369 100644
--- a/tests/test_core_downloader.py
+++ b/tests/test_core_downloader.py
@@ -8,5 +8,5 @@ class SlotTest(unittest.TestCase):
         slot = Slot(concurrency=8, delay=0.1, randomize_delay=True)
         self.assertEqual(
             repr(slot),
-            "Slot(concurrency=8, delay=0.10, randomize_delay=True, throttle=None)",
+            "Slot(concurrency=8, delay=0.10, randomize_delay=True)",
         )
diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py
index ea8c5b4f0..55f9ecac9 100644
--- a/tests/test_downloaderslotssettings.py
+++ b/tests/test_downloaderslotssettings.py
@@ -80,7 +80,6 @@ def test_params():
         "concurrency": 1,
         "delay": 2,
         "randomize_delay": False,
-        "throttle": False,
     }
     settings = {
         "DOWNLOAD_SLOTS": {
diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py
index 722a05c26..f2c9dc063 100644
--- a/tests/test_extension_throttle.py
+++ b/tests/test_extension_throttle.py
@@ -157,17 +157,24 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected)
 
 
 @pytest.mark.parametrize(
-    ("meta", "slot", "throttle"),
+    ("meta", "slot"),
     (
-        ({}, None, None),
-        ({"download_latency": 1.0}, None, None),
-        ({"download_slot": "foo"}, None, None),
-        ({"download_slot": "foo"}, "foo", None),
-        ({"download_latency": 1.0, "download_slot": "foo"}, None, None),
-        ({"download_latency": 1.0, "download_slot": "foo"}, "foo", False),
+        ({}, None),
+        ({"download_latency": 1.0}, None),
+        ({"download_slot": "foo"}, None),
+        ({"download_slot": "foo"}, "foo"),
+        ({"download_latency": 1.0, "download_slot": "foo"}, None),
+        (
+            {
+                "download_latency": 1.0,
+                "download_slot": "foo",
+                "autothrottle_dont_adjust_delay": True,
+            },
+            "foo",
+        ),
     ),
 )
-def test_skipped(meta, slot, throttle):
+def test_skipped(meta, slot):
     crawler = get_crawler()
     at = build_from_crawler(AutoThrottle, crawler)
     spider = TestSpider()
@@ -178,9 +185,7 @@ def test_skipped(meta, slot, throttle):
     crawler.engine.downloader = Mock()
     crawler.engine.downloader.slots = {}
     if slot is not None:
-        _slot = Mock()
-        _slot.throttle = throttle
-        crawler.engine.downloader.slots[slot] = _slot
+        crawler.engine.downloader.slots[slot] = object()
     at._adjust_delay = None  # Raise exception if called.
 
     at._response_downloaded(None, request, spider)
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index f80f2517a..5681ff9a4 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -3,7 +3,6 @@ import operator
 import platform
 import sys
 
-import pytest
 from twisted.trial import unittest
 
 from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
@@ -152,7 +151,6 @@ class BinaryIsTextTest(unittest.TestCase):
 
 
 class UtilsPythonTestCase(unittest.TestCase):
-    @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
     def test_equal_attributes(self):
         class Obj:
             pass
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 965d050a4..7156b13d0 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -6,8 +6,6 @@ import warnings
 from hashlib import sha1
 from weakref import WeakKeyDictionary
 
-import pytest
-
 from scrapy.http import Request
 from scrapy.utils.python import to_bytes
 from scrapy.utils.request import (
@@ -21,7 +19,6 @@ from scrapy.utils.test import get_crawler
 
 
 class UtilsRequestTest(unittest.TestCase):
-    @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
     def test_request_authenticate(self):
         r = Request("http://www.example.com")
         request_authenticate(r, "someuser", "somepass")