Repository: scrapy/scrapy

Issue:
Deprecate the `spider` argument to `Downloader._get_slot_key()`
The `spider` argument is not used inside the method since 2012, but we can't remove it as external code calls it (either because it subclasses `Downloader` or because it wants the slot name).

Actually maybe we want to promote it to a public method for the second reason? Not sure how often it's needed but see e.g. https://github.com/scrapy-plugins/scrapy-zyte-api/blob/a2284c8cdf157ef6d36c1cc413933761c5ed792b/scrapy_zyte_api/_middlewares.py#L32
Deprecate the `spider` argument to `Downloader._get_slot_key()`
The `spider` argument is not used inside the method since 2012, but we can't remove it as external code calls it (either because it subclasses `Downloader` or because it wants the slot name).

Actually maybe we want to promote it to a public method for the second reason? Not sure how often it's needed but see e.g. https://github.com/scrapy-plugins/scrapy-zyte-api/blob/a2284c8cdf157ef6d36c1cc413933761c5ed792b/scrapy_zyte_api/_middlewares.py#L32


Discussion:
+1 to make it public.

Although I wonder if, for examples like the linked one, the right approach could be to allow customizing how slots IDs are generated instead.
@wRAR do you want to promote this method to public and keep the old method with some Deprecate message ?
Yeah, let's make it public and without the extra argument and keep the old one with a deprecation warning.
+1 to make it public.

Although I wonder if, for examples like the linked one, the right approach could be to allow customizing how slots IDs are generated instead.
@wRAR do you want to promote this method to public and keep the old method with some Deprecate message ?
Yeah, let's make it public and without the extra argument and keep the old one with a deprecation warning.

Resolving patch:
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -1,4 +1,5 @@
 import random
+import warnings
 from collections import deque
 from datetime import datetime
 from time import time
@@ -10,6 +11,7 @@
 from scrapy import Request, Spider, signals
 from scrapy.core.downloader.handlers import DownloadHandlers
 from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Response
 from scrapy.resolver import dnscache
 from scrapy.settings import BaseSettings
@@ -125,7 +127,7 @@ def needs_backout(self) -> bool:
         return len(self.active) >= self.total_concurrency
 
     def _get_slot(self, request: Request, spider: Spider) -> Tuple[str, Slot]:
-        key = self._get_slot_key(request, spider)
+        key = self.get_slot_key(request)
         if key not in self.slots:
             slot_settings = self.per_slot_settings.get(key, {})
             conc = (
@@ -143,7 +145,7 @@ def _get_slot(self, request: Request, spider: Spider) -> Tuple[str, Slot]:
 
         return key, self.slots[key]
 
-    def _get_slot_key(self, request: Request, spider: Optional[Spider]) -> str:
+    def get_slot_key(self, request: Request) -> str:
         if self.DOWNLOAD_SLOT in request.meta:
             return cast(str, request.meta[self.DOWNLOAD_SLOT])
 
@@ -153,6 +155,14 @@ def _get_slot_key(self, request: Request, spider: Optional[Spider]) -> str:
 
         return key
 
+    def _get_slot_key(self, request: Request, spider: Optional[Spider]) -> str:
+        warnings.warn(
+            "Use of this protected method is deprecated. Consider using its corresponding public method get_slot_key() instead.",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        return self.get_slot_key(request)
+
     def _enqueue_request(self, request: Request, spider: Spider) -> Deferred:
         key, slot = self._get_slot(request, spider)
         request.meta[self.DOWNLOAD_SLOT] = key
diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -180,7 +180,7 @@ def stats(self, possible_slots: Iterable[str]) -> List[Tuple[int, str]]:
         return [(self._active_downloads(slot), slot) for slot in possible_slots]
 
     def get_slot_key(self, request: Request) -> str:
-        return self.downloader._get_slot_key(request, None)
+        return self.downloader.get_slot_key(request)
 
     def _active_downloads(self, slot: str) -> int:
         """Return a number of requests in a Downloader for a given slot"""


Test patch:
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -25,7 +25,7 @@ class MockDownloader:
     def __init__(self):
         self.slots = {}
 
-    def _get_slot_key(self, request, spider):
+    def get_slot_key(self, request):
         if Downloader.DOWNLOAD_SLOT in request.meta:
             return request.meta[Downloader.DOWNLOAD_SLOT]
 
@@ -273,14 +273,14 @@ def test_logic(self):
         while self.scheduler.has_pending_requests():
             request = self.scheduler.next_request()
             # pylint: disable=protected-access
-            slot = downloader._get_slot_key(request, None)
+            slot = downloader.get_slot_key(request)
             dequeued_slots.append(slot)
             downloader.increment(slot)
             requests.append(request)
 
         for request in requests:
             # pylint: disable=protected-access
-            slot = downloader._get_slot_key(request, None)
+            slot = downloader.get_slot_key(request)
             downloader.decrement(slot)
 
         self.assertTrue(