Repository: scrapy/scrapy
Pull request #6016 (2023-08-17 16:45:04)

Fix: response.json() call makes unnecessary memory allocation
Now `response.json()` uses `response.body` to retrieve the `dict` object, so the str response is no longer stored in RAM.


Closes https://github.com/scrapy/scrapy/issues/5968


Patch:
diff --git a/docs/news.rst b/docs/news.rst
index c55c0b222..0154e64ff 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,23 @@
 Release notes
 =============
 
+.. _release-2.11.0:
+
+Scrapy 2.11.0 (to be released)
+------------------------------
+
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+-   The :meth:`TextResponse.json <scrapy.http.TextResponse.json>` method now
+    requires the response to be in a valid JSON encoding (UTF-8, UTF-16, or
+    UTF-32).
+
+    If you need to deal with JSON documents in an invalid encoding, use
+    ``json.loads(response.text)`` instead.
+
+    (:issue:`5968`)
+
 .. _release-2.10.0:
 
 Scrapy 2.10.0 (2023-08-04)
diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py
index 302a615f2..30c987253 100644
--- a/scrapy/extensions/corestats.py
+++ b/scrapy/extensions/corestats.py
@@ -1,7 +1,7 @@
 """
 Extension for collecting core stats like items scraped and start/finish times
 """
-from datetime import datetime, timezone
+from datetime import datetime
 
 from scrapy import signals
 
@@ -22,11 +22,11 @@ class CoreStats:
         return o
 
     def spider_opened(self, spider):
-        self.start_time = datetime.now(tz=timezone.utc)
+        self.start_time = datetime.utcnow()
         self.stats.set_value("start_time", self.start_time, spider=spider)
 
     def spider_closed(self, spider, reason):
-        finish_time = datetime.now(tz=timezone.utc)
+        finish_time = datetime.utcnow()
         elapsed_time = finish_time - self.start_time
         elapsed_time_seconds = elapsed_time.total_seconds()
         self.stats.set_value(
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 4e846d1bd..c1b77f4fb 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -8,7 +8,7 @@ import logging
 import re
 import sys
 import warnings
-from datetime import datetime, timezone
+from datetime import datetime
 from pathlib import Path, PureWindowsPath
 from tempfile import NamedTemporaryFile
 from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Union
@@ -676,7 +676,7 @@ class FeedExporter:
         params = {}
         for k in dir(spider):
             params[k] = getattr(spider, k)
-        utc_now = datetime.now(tz=timezone.utc)
+        utc_now = datetime.utcnow()
         params["time"] = utc_now.replace(microsecond=0).isoformat().replace(":", "-")
         params["batch_time"] = utc_now.isoformat().replace(":", "-")
         params["batch_id"] = slot.batch_id + 1 if slot is not None else 1
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 7fc54b5d3..47d7bc10f 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -82,7 +82,7 @@ class TextResponse(Response):
         Deserialize a JSON document to a Python object.
         """
         if self._cached_decoded_json is _NONE:
-            self._cached_decoded_json = json.loads(self.text)
+            self._cached_decoded_json = json.loads(self.body)
         return self._cached_decoded_json
 
     @property


Test patch:
diff --git a/tests/keys/__init__.py b/tests/keys/__init__.py
index 9b73ca4f0..5cc65a903 100644
--- a/tests/keys/__init__.py
+++ b/tests/keys/__init__.py
@@ -1,4 +1,4 @@
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timedelta
 from pathlib import Path
 
 from cryptography.hazmat.backends import default_backend
@@ -50,8 +50,8 @@ def generate_keys():
         .issuer_name(issuer)
         .public_key(key.public_key())
         .serial_number(random_serial_number())
-        .not_valid_before(datetime.now(tz=timezone.utc))
-        .not_valid_after(datetime.now(tz=timezone.utc) + timedelta(days=10))
+        .not_valid_before(datetime.utcnow())
+        .not_valid_after(datetime.utcnow() + timedelta(days=10))
         .add_extension(
             SubjectAlternativeName([DNSName("localhost")]),
             critical=False,
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 54f0461e8..80d46274b 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -844,7 +844,7 @@ class TextResponseTest(BaseResponseTest):
             with mock.patch("json.loads") as mock_json:
                 for _ in range(2):
                     json_response.json()
-                mock_json.assert_called_once_with(json_body.decode())
+                mock_json.assert_called_once_with(json_body)
 
 
 class HtmlResponseTest(TextResponseTest):
diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py
index f97125b76..f645f4cce 100644
--- a/tests/test_spiderstate.py
+++ b/tests/test_spiderstate.py
@@ -1,5 +1,5 @@
 import shutil
-from datetime import datetime, timezone
+from datetime import datetime
 from pathlib import Path
 
 from twisted.trial import unittest
@@ -16,7 +16,7 @@ class SpiderStateTest(unittest.TestCase):
         Path(jobdir).mkdir()
         try:
             spider = Spider(name="default")
-            dt = datetime.now(tz=timezone.utc)
+            dt = datetime.now()
 
             ss = SpiderState(jobdir)
             ss.spider_opened(spider)
diff --git a/tests/test_stats.py b/tests/test_stats.py
index 3d4c7e88e..7a8adf638 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -16,7 +16,7 @@ class CoreStatsExtensionTest(unittest.TestCase):
     @mock.patch("scrapy.extensions.corestats.datetime")
     def test_core_stats_default_stats_collector(self, mock_datetime):
         fixed_datetime = datetime(2019, 12, 1, 11, 38)
-        mock_datetime.now = mock.Mock(return_value=fixed_datetime)
+        mock_datetime.utcnow = mock.Mock(return_value=fixed_datetime)
         self.crawler.stats = StatsCollector(self.crawler)
         ext = CoreStats.from_crawler(self.crawler)
         ext.spider_opened(self.spider)