Repository: scrapy/scrapy
Pull request #6014 (2023-08-14 23:15:03)

Remove datetime.utcnow() usage
Make our datetime calls timezone-aware, `utcnow()` is deprecated and scheduled for removal, check why in this [article.](https://blog.ganssle.io/articles/2019/11/utcnow.html)

codecov[bot]: ## [Codecov](https://app.codecov.io/gh/scrapy/scrapy/pull/6014?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scrapy) Report
> Merging [#6014](https://app.codecov.io/gh/scrapy/scrapy/pull/6014?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scrapy) (dd31b81) into [master](https://app.codecov.io/gh/scrapy/scrapy/commit/3318971512fd0314ffd6053b68e4aef2142c1a80?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scrapy) (3318971) will **decrease** coverage by `46.23%`.
> The diff coverage is `100.00%`.

> :exclamation: Current head dd31b81 differs from pull request most recent head ba0e558. Consider uploading reports for the commit ba0e558 to get more accurate results

```diff
@@             Coverage Diff             @@
##           master    #6014       +/-   ##
===========================================
- Coverage   88.85%   42.62%   -46.23%     
===========================================
  Files         162      162               
  Lines       11445    11445               
  Branches     1861     1859        -2     
===========================================
- Hits        10170     4879     -5291     
- Misses        968     6200     +5232     
- Partials      307      366       +59     
```

| [Files Changed](https://app.codecov.io/gh/scrapy/scrapy/pull/6014?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scrapy) | Coverage Δ | |
|---|---|---|
| [scrapy/extensions/corestats.py](https://app.codecov.io/gh/scrapy/scrapy/pull/6014?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scrapy#diff-c2NyYXB5L2V4dGVuc2lvbnMvY29yZXN0YXRzLnB5) | `90.90% <100.00%> (-9.10%)` | :arrow_down: |
| [scrapy/extensions/feedexport.py](https://app.codecov.io/gh/scrapy/scrapy/pull/6014?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scrapy#diff-c2NyYXB5L2V4dGVuc2lvbnMvZmVlZGV4cG9ydC5weQ==) | `70.63% <100.00%> (-22.10%)` | :arrow_down: |

... and [140 files with indirect coverage changes](https://app.codecov.io/gh/scrapy/scrapy/pull/6014/indirect-changes?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scrapy)


Patch:
diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py
index 30c987253..302a615f2 100644
--- a/scrapy/extensions/corestats.py
+++ b/scrapy/extensions/corestats.py
@@ -1,7 +1,7 @@
 """
 Extension for collecting core stats like items scraped and start/finish times
 """
-from datetime import datetime
+from datetime import datetime, timezone
 
 from scrapy import signals
 
@@ -22,11 +22,11 @@ class CoreStats:
         return o
 
     def spider_opened(self, spider):
-        self.start_time = datetime.utcnow()
+        self.start_time = datetime.now(tz=timezone.utc)
         self.stats.set_value("start_time", self.start_time, spider=spider)
 
     def spider_closed(self, spider, reason):
-        finish_time = datetime.utcnow()
+        finish_time = datetime.now(tz=timezone.utc)
         elapsed_time = finish_time - self.start_time
         elapsed_time_seconds = elapsed_time.total_seconds()
         self.stats.set_value(
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index c1b77f4fb..4e846d1bd 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -8,7 +8,7 @@ import logging
 import re
 import sys
 import warnings
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import Path, PureWindowsPath
 from tempfile import NamedTemporaryFile
 from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Union
@@ -676,7 +676,7 @@ class FeedExporter:
         params = {}
         for k in dir(spider):
             params[k] = getattr(spider, k)
-        utc_now = datetime.utcnow()
+        utc_now = datetime.now(tz=timezone.utc)
         params["time"] = utc_now.replace(microsecond=0).isoformat().replace(":", "-")
         params["batch_time"] = utc_now.isoformat().replace(":", "-")
         params["batch_id"] = slot.batch_id + 1 if slot is not None else 1


Test patch:
diff --git a/tests/keys/__init__.py b/tests/keys/__init__.py
index 5cc65a903..9b73ca4f0 100644
--- a/tests/keys/__init__.py
+++ b/tests/keys/__init__.py
@@ -1,4 +1,4 @@
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from pathlib import Path
 
 from cryptography.hazmat.backends import default_backend
@@ -50,8 +50,8 @@ def generate_keys():
         .issuer_name(issuer)
         .public_key(key.public_key())
         .serial_number(random_serial_number())
-        .not_valid_before(datetime.utcnow())
-        .not_valid_after(datetime.utcnow() + timedelta(days=10))
+        .not_valid_before(datetime.now(tz=timezone.utc))
+        .not_valid_after(datetime.now(tz=timezone.utc) + timedelta(days=10))
         .add_extension(
             SubjectAlternativeName([DNSName("localhost")]),
             critical=False,
diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py
index f645f4cce..f97125b76 100644
--- a/tests/test_spiderstate.py
+++ b/tests/test_spiderstate.py
@@ -1,5 +1,5 @@
 import shutil
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import Path
 
 from twisted.trial import unittest
@@ -16,7 +16,7 @@ class SpiderStateTest(unittest.TestCase):
         Path(jobdir).mkdir()
         try:
             spider = Spider(name="default")
-            dt = datetime.now()
+            dt = datetime.now(tz=timezone.utc)
 
             ss = SpiderState(jobdir)
             ss.spider_opened(spider)
diff --git a/tests/test_stats.py b/tests/test_stats.py
index 7a8adf638..3d4c7e88e 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -16,7 +16,7 @@ class CoreStatsExtensionTest(unittest.TestCase):
     @mock.patch("scrapy.extensions.corestats.datetime")
     def test_core_stats_default_stats_collector(self, mock_datetime):
         fixed_datetime = datetime(2019, 12, 1, 11, 38)
-        mock_datetime.utcnow = mock.Mock(return_value=fixed_datetime)
+        mock_datetime.now = mock.Mock(return_value=fixed_datetime)
         self.crawler.stats = StatsCollector(self.crawler)
         ext = CoreStats.from_crawler(self.crawler)
         ext.spider_opened(self.spider)