Repository: scrapy/scrapy
Pull request #4055 (2019-10-01 20:38:21)

[Doc] Use autoclass in topics/request-response.rst
As suggested in https://github.com/scrapy/scrapy/pull/4054#discussion_r330207869, plus other wording fixes.


Patch:
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 284d3479b..727c67482 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -24,7 +24,7 @@ below in :ref:`topics-request-response-ref-request-subclasses` and
 Request objects
 ===============
 
-.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags, cb_kwargs])
+.. autoclass:: Request
 
     A :class:`Request` object represents an HTTP request, which is usually
     generated in the Spider and executed by the Downloader, and thus generating
@@ -400,7 +400,7 @@ fields with form data from :class:`Response` objects.
 
 .. class:: FormRequest(url, [formdata, ...])
 
-    The :class:`FormRequest` class adds a new argument to the constructor. The
+    The :class:`FormRequest` class adds a new keyword parameter to the constructor. The
     remaining arguments are the same as for the :class:`Request` class and are
     not documented here.
 
@@ -547,7 +547,7 @@ dealing with JSON requests.
 
 .. class:: JsonRequest(url, [... data, dumps_kwargs])
 
-   The :class:`JsonRequest` class adds two new argument to the constructor. The
+   The :class:`JsonRequest` class adds two new keyword parameters to the constructor. The
    remaining arguments are the same as for the :class:`Request` class and are
    not documented here.
 
@@ -581,7 +581,7 @@ Sending a JSON POST request with a JSON payload::
 Response objects
 ================
 
-.. class:: Response(url, [status=200, headers=None, body=b'', flags=None, request=None])
+.. autoclass:: Response
 
     A :class:`Response` object represents an HTTP response, which is usually
     downloaded (by the Downloader) and fed to the Spiders for processing.
diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py
index 8cc5e18ac..20adfbe4b 100644
--- a/scrapy/extensions/corestats.py
+++ b/scrapy/extensions/corestats.py
@@ -1,14 +1,16 @@
 """
 Extension for collecting core stats like items scraped and start/finish times
 """
-import datetime
+from datetime import datetime
 
 from scrapy import signals
 
+
 class CoreStats(object):
 
     def __init__(self, stats):
         self.stats = stats
+        self.start_time = None
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -21,11 +23,12 @@ class CoreStats(object):
         return o
 
     def spider_opened(self, spider):
-        self.stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider)
+        self.start_time = datetime.utcnow()
+        self.stats.set_value('start_time', self.start_time, spider=spider)
 
     def spider_closed(self, spider, reason):
-        finish_time = datetime.datetime.utcnow()
-        elapsed_time = finish_time - self.stats.get_value('start_time')
+        finish_time = datetime.utcnow()
+        elapsed_time = finish_time - self.start_time
         elapsed_time_seconds = elapsed_time.total_seconds()
         self.stats.set_value('elapsed_time_seconds', elapsed_time_seconds, spider=spider)
         self.stats.set_value('finish_time', finish_time, spider=spider)


Test patch:
diff --git a/tests/test_stats.py b/tests/test_stats.py
index 9f950ebc9..2033dbe07 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -1,10 +1,59 @@
+from datetime import datetime
 import unittest
 
+try:
+    from unittest import mock
+except ImportError:
+    import mock
+
+from scrapy.extensions.corestats import CoreStats
 from scrapy.spiders import Spider
 from scrapy.statscollectors import StatsCollector, DummyStatsCollector
 from scrapy.utils.test import get_crawler
 
 
+class CoreStatsExtensionTest(unittest.TestCase):
+
+    def setUp(self):
+        self.crawler = get_crawler(Spider)
+        self.spider = self.crawler._create_spider('foo')
+
+    @mock.patch('scrapy.extensions.corestats.datetime')
+    def test_core_stats_default_stats_collector(self, mock_datetime):
+        fixed_datetime = datetime(2019, 12, 1, 11, 38)
+        mock_datetime.utcnow = mock.Mock(return_value=fixed_datetime)
+        self.crawler.stats = StatsCollector(self.crawler)
+        ext = CoreStats.from_crawler(self.crawler)
+        ext.spider_opened(self.spider)
+        ext.item_scraped({}, self.spider)
+        ext.response_received(self.spider)
+        ext.item_dropped({}, self.spider, ZeroDivisionError())
+        ext.spider_closed(self.spider, 'finished')
+        self.assertEqual(
+            ext.stats._stats,
+            {
+                'start_time': fixed_datetime,
+                'finish_time': fixed_datetime,
+                'item_scraped_count': 1,
+                'response_received_count': 1,
+                'item_dropped_count': 1,
+                'item_dropped_reasons_count/ZeroDivisionError': 1,
+                'finish_reason': 'finished',
+                'elapsed_time_seconds': 0.0,
+            }
+        )
+
+    def test_core_stats_dummy_stats_collector(self):
+        self.crawler.stats = DummyStatsCollector(self.crawler)
+        ext = CoreStats.from_crawler(self.crawler)
+        ext.spider_opened(self.spider)
+        ext.item_scraped({}, self.spider)
+        ext.response_received(self.spider)
+        ext.item_dropped({}, self.spider, ZeroDivisionError())
+        ext.spider_closed(self.spider, 'finished')
+        self.assertEqual(ext.stats._stats, {})
+
+
 class StatsCollectorTest(unittest.TestCase):
 
     def setUp(self):