Repository: scrapy/scrapy
Pull request: Deprecate unused scrapy utils
Fixes #6517 

This PR deprecates following unused functions and classes in `scrapy.utils`:
- `flatten`
- `iflatten`
- `equal_attributes`
- `request_authenticate`
- `ScrapyJSONDecoder`
- `assert_samelines`

I have also changed some other warnings for a grammatical uniformity

pls review this PR

Problem statement:
Deprecate unused scrapy.utils code
As far as I can see the following functions and classes from `scrapy.utils` are not used anywhere (usually because the code using them is gone):

* `scrapy.utils.python.flatten()`
* `scrapy.utils.python.iflatten()`
* `scrapy.utils.python.equal_attributes()`
* `scrapy.utils.request.request_authenticate()`
* `scrapy.utils.serialize.ScrapyJSONDecoder`
* `scrapy.utils.test.assert_samelines()`

`flatten()` and `iflatten()` are also present in Parsel where they are used but I wouldn't recommend switching to those (and they are easy to reimplement anyway).

Hints from issue discussion:
I'd be happy to work on this issue. I understand that the goal is to identify and remove the unused functions and classes listed above

I'll check for any remaining references across the repository to confirm that these are indeed unused and proceed with removing them.

Also, could you please let me know which specific tests I should run to verify everything is functioning correctly after these removals? 


> I understand that the goal is to identify and remove the unused functions and classes listed above

They are already identified and the goal is to deprecate them, not remove them.
Sorry for the confusion so i have to just implement warnings about the deprecation

Commit message: Added ignore statements for deprecation warning in tests

Patch to review:
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 51621834730..12c09839f0f 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -111,7 +111,7 @@ def md5sum(file: IO[bytes]) -> str:
     """
     warnings.warn(
         (
-            "The scrapy.utils.misc.md5sum function is deprecated, and will be "
+            "The scrapy.utils.misc.md5sum function is deprecated and will be "
             "removed in a future version of Scrapy."
         ),
         ScrapyDeprecationWarning,
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 6268af72888..14b84e50e30 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -8,12 +8,14 @@
 import inspect
 import re
 import sys
+import warnings
 import weakref
 from collections.abc import AsyncIterable, Iterable, Mapping
 from functools import partial, wraps
 from itertools import chain
 from typing import TYPE_CHECKING, Any, TypeVar, overload
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.asyncgen import as_async_generator
 
 if TYPE_CHECKING:
@@ -47,6 +49,10 @@ def flatten(x: Iterable[Any]) -> list[Any]:
     >>> flatten(["foo", ["baz", 42], "bar"])
     ['foo', 'baz', 42, 'bar']
     """
+    warnings.warn(
+        "The 'flatten' function is deprecated and will be removed in a future version of Scrapy.",
+        category=ScrapyDeprecationWarning,
+    )
     return list(iflatten(x))
 
 
@@ -54,6 +60,10 @@ def iflatten(x: Iterable[Any]) -> Iterable[Any]:
     """iflatten(sequence) -> iterator
 
     Similar to ``.flatten()``, but returns iterator instead"""
+    warnings.warn(
+        "The 'iflatten' function is deprecated and will be removed in a future version of Scrapy.",
+        category=ScrapyDeprecationWarning,
+    )
     for el in x:
         if is_listlike(el):
             yield from iflatten(el)
@@ -272,6 +282,10 @@ def equal_attributes(
     obj1: Any, obj2: Any, attributes: list[str | Callable[[Any], Any]] | None
 ) -> bool:
     """Compare two objects attributes"""
+    warnings.warn(
+        "The 'equal_attributes' function is deprecated and will be removed in a future version of Scrapy.",
+        category=ScrapyDeprecationWarning,
+    )
     # not attributes given return False by default
     if not attributes:
         return False
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 82bdcb0f94a..11f45ff8b1b 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -140,7 +140,7 @@ def __init__(self, crawler: Crawler | None = None):
         if implementation != "SENTINEL":
             message = (
                 "'REQUEST_FINGERPRINTER_IMPLEMENTATION' is a deprecated setting.\n"
-                "And it will be removed in future version of Scrapy."
+                "It will be removed in a future version of Scrapy."
             )
             warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
         self._fingerprint = fingerprint
@@ -157,6 +157,10 @@ def request_authenticate(
     """Authenticate the given request (in place) using the HTTP basic access
     authentication mechanism (RFC 2617) and the given username and password
     """
+    warnings.warn(
+        "The 'request_authenticate' function is deprecated and will be removed in a future version of Scrapy.",
+        category=ScrapyDeprecationWarning,
+    )
     request.headers["Authorization"] = basic_auth_header(username, password)
 
 
diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py
index 3b4f67f000c..1bb3fb59c8e 100644
--- a/scrapy/utils/serialize.py
+++ b/scrapy/utils/serialize.py
@@ -1,11 +1,13 @@
 import datetime
 import decimal
 import json
+import warnings
 from typing import Any
 
 from itemadapter import ItemAdapter, is_item
 from twisted.internet import defer
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 
 
@@ -36,4 +38,10 @@ def default(self, o: Any) -> Any:
 
 
 class ScrapyJSONDecoder(json.JSONDecoder):
-    pass
+    def __init__(self, *args, **kwargs):
+        warnings.warn(
+            "'ScrapyJSONDecoder' class is deprecated and will be removed in a future version of Scrapy.",
+            category=ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        super().__init__(*args, **kwargs)
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index d65f2a76d7d..a0a407ad09d 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -6,6 +6,7 @@
 
 import asyncio
 import os
+import warnings
 from importlib import import_module
 from pathlib import Path
 from posixpath import split
@@ -16,6 +17,7 @@
 
 from scrapy import Spider
 from scrapy.crawler import Crawler
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.boto import is_botocore_available
 
 if TYPE_CHECKING:
@@ -125,6 +127,10 @@ def assert_samelines(
     """Asserts text1 and text2 have the same lines, ignoring differences in
     line endings between platforms
     """
+    warnings.warn(
+        "The 'assert_samelines' function is deprecated and will be removed in a future version of Scrapy.",
+        category=ScrapyDeprecationWarning,
+    )
     testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg)
 
 
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 5681ff9a4cc..f80f2517ac6 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -3,6 +3,7 @@
 import platform
 import sys
 
+import pytest
 from twisted.trial import unittest
 
 from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
@@ -151,6 +152,7 @@ def test_real_binary_bytes(self):
 
 
 class UtilsPythonTestCase(unittest.TestCase):
+    @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
     def test_equal_attributes(self):
         class Obj:
             pass
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 7156b13d0fc..965d050a4da 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -6,6 +6,8 @@
 from hashlib import sha1
 from weakref import WeakKeyDictionary
 
+import pytest
+
 from scrapy.http import Request
 from scrapy.utils.python import to_bytes
 from scrapy.utils.request import (
@@ -19,6 +21,7 @@
 
 
 class UtilsRequestTest(unittest.TestCase):
+    @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
     def test_request_authenticate(self):
         r = Request("http://www.example.com")
         request_authenticate(r, "someuser", "somepass")


Code review comments:
diff_hunk: @@ -40,7 +40,7 @@ def default(self, o: Any) -> Any:
 class ScrapyJSONDecoder(json.JSONDecoder):
     def __init__(self, *args, **kwargs):
         warnings.warn(
-            "'ScrapyJSONDecoder' is a deprecated setting.\nIt will be removed in a future version of Scrapy.",
+            "'ScrapyJSONDecoder' class is deprecated and will be removed in a future version of Scrapy.",
path: scrapy/utils/serialize.py
text: @user1:
```suggestion
            "The 'ScrapyJSONDecoder' class is deprecated and will be removed in a future version of Scrapy.",
```

You could also omit the single quotes, I think, but no strong opinion on that.