Repository: scrapy/scrapy

Issue:
Exception when using DummyStatsCollector
### Description

Using the DummyStatsCollector results in an exception:

```
2019-09-09 13:51:23 [scrapy.utils.signal] ERROR: Error caught on signal handler: <bound method CoreStats.spider_closed of <scrapy.extensions.corestats.CoreStats object at 0x7f86269cac18>>
Traceback (most recent call last):
  File ".../lib/python3.6/site-packages/twisted/internet/defer.py", line 150, in maybeDeferred
    result = f(*args, **kw)
  File ".../lib/python3.6/site-packages/pydispatch/robustapply.py", line 55, in robustApply
    return receiver(*arguments, **named)
  File ".../lib/python3.6/site-packages/scrapy/extensions/corestats.py", line 28, in spider_closed
    elapsed_time = finish_time - self.stats.get_value('start_time')
TypeError: unsupported operand type(s) for -: 'datetime.datetime' and 'NoneType'
```

This problem has been introduced in aa46e1995cd5cb1099aba17535372b538bd656b3.

### Steps to Reproduce

Set `STATS_CLASS = "scrapy.statscollectors.DummyStatsCollector"` in the settings module as described in the documentation (https://docs.scrapy.org/en/latest/topics/stats.html#dummystatscollector).

**Expected behavior:** no exception
**Actual behavior:** exception thrown
**Reproduces how often:** always

### Versions

At least master as of 534de7395da3a53b5a2c89960db9ec5d8fdab60c

### Fix

A possible fix is to use the elapsed time as a default argument so that `get_value()` does not return None. I can prepare a PR if needed.

```diff
--- a/scrapy/extensions/corestats.py
+++ b/scrapy/extensions/corestats.py
@@ -25,7 +25,7 @@ class CoreStats(object):
 
     def spider_closed(self, spider, reason):
         finish_time = datetime.datetime.utcnow()
-        elapsed_time = finish_time - self.stats.get_value('start_time')
+        elapsed_time = finish_time - self.stats.get_value('start_time', finish_time)
         elapsed_time_seconds = elapsed_time.total_seconds()
         self.stats.set_value('elapsed_time_seconds', elapsed_time_seconds, spider=spider)
         self.stats.set_value('finish_time', finish_time, spider=spider)
```


Discussion:
Confirmed. `scrapy.extensions.corestats.CoreStats` sets the `start_time` value [here](https://github.com/scrapy/scrapy/blob/1.7.3/scrapy/extensions/corestats.py#L23-L24) but `scrapy.statscollectors.DummyStatsCollector` overrides `set_value` to do nothing [here](https://github.com/scrapy/scrapy/blob/1.7.3/scrapy/statscollectors.py#L69-L70).

However, I'm not sure I agree with the proposed fix, that would mean the `elapsed_time` variable could end up being `0`, which is probably not accurate. Given that the idea of the `DummyStatsCollector` is to improve performance by not collecting stats, I'm wondering if we even need to connect the signal handlers in the first place. A fix could look something like this:

```diff
diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py
index 8cc5e18a..5e4dbb05 100644
--- a/scrapy/extensions/corestats.py
+++ b/scrapy/extensions/corestats.py
@@ -4,6 +4,8 @@ Extension for collecting core stats like items scraped and start/finish times
 import datetime

 from scrapy import signals
+from scrapy.statscollectors import DummyStatsCollector
+

 class CoreStats(object):

@@ -13,11 +15,12 @@ class CoreStats(object):
     @classmethod
     def from_crawler(cls, crawler):
         o = cls(crawler.stats)
-        crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
-        crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
-        crawler.signals.connect(o.item_scraped, signal=signals.item_scraped)
-        crawler.signals.connect(o.item_dropped, signal=signals.item_dropped)
-        crawler.signals.connect(o.response_received, signal=signals.response_received)
+        if not isinstance(crawler.stats, DummyStatsCollector):
+            crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
+            crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
+            crawler.signals.connect(o.item_scraped, signal=signals.item_scraped)
+            crawler.signals.connect(o.item_dropped, signal=signals.item_dropped)
+            crawler.signals.connect(o.response_received, signal=signals.response_received)
         return o

     def spider_opened(self, spider):
```
Thanks for the quick response!

Yes, you are right that `elapsed_time` would be 0 which is not optimal. However, special casing the DummyStatsCollector also seems a bit odd to me because if I were to implement my own stats collector which exposed the same functionality as the DummyStatsCollector it would still fail.
Makes sense, the extension should be robust enough not to break depending on the Stats class implementation. In that case, something along the following lines should do the trick:

```diff
diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py
index 8cc5e18a..91d8558b 100644
--- a/scrapy/extensions/corestats.py
+++ b/scrapy/extensions/corestats.py
@@ -9,6 +9,7 @@ class CoreStats(object):

     def __init__(self, stats):
         self.stats = stats
+        self.start_time = None

     @classmethod
     def from_crawler(cls, crawler):
@@ -21,11 +22,12 @@ class CoreStats(object):
         return o

     def spider_opened(self, spider):
-        self.stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider)
+        self.start_time = datetime.datetime.utcnow()
+        self.stats.set_value('start_time', self.start_time, spider=spider)

     def spider_closed(self, spider, reason):
         finish_time = datetime.datetime.utcnow()
-        elapsed_time = finish_time - self.stats.get_value('start_time')
+        elapsed_time = finish_time - self.start_time
         elapsed_time_seconds = elapsed_time.total_seconds()
         self.stats.set_value('elapsed_time_seconds', elapsed_time_seconds, spider=spider)
         self.stats.set_value('finish_time', finish_time, spider=spider)
```

I still think it might be a good idea to skip connecting the handlers though 😅 
Let's see what other committers think.

Resolving patch:
diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py
--- a/scrapy/extensions/corestats.py
+++ b/scrapy/extensions/corestats.py
@@ -1,14 +1,16 @@
 """
 Extension for collecting core stats like items scraped and start/finish times
 """
-import datetime
+from datetime import datetime
 
 from scrapy import signals
 
+
 class CoreStats(object):
 
     def __init__(self, stats):
         self.stats = stats
+        self.start_time = None
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -21,11 +23,12 @@ def from_crawler(cls, crawler):
         return o
 
     def spider_opened(self, spider):
-        self.stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider)
+        self.start_time = datetime.utcnow()
+        self.stats.set_value('start_time', self.start_time, spider=spider)
 
     def spider_closed(self, spider, reason):
-        finish_time = datetime.datetime.utcnow()
-        elapsed_time = finish_time - self.stats.get_value('start_time')
+        finish_time = datetime.utcnow()
+        elapsed_time = finish_time - self.start_time
         elapsed_time_seconds = elapsed_time.total_seconds()
         self.stats.set_value('elapsed_time_seconds', elapsed_time_seconds, spider=spider)
         self.stats.set_value('finish_time', finish_time, spider=spider)


Test patch:
diff --git a/tests/test_stats.py b/tests/test_stats.py
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -1,10 +1,59 @@
+from datetime import datetime
 import unittest
 
+try:
+    from unittest import mock
+except ImportError:
+    import mock
+
+from scrapy.extensions.corestats import CoreStats
 from scrapy.spiders import Spider
 from scrapy.statscollectors import StatsCollector, DummyStatsCollector
 from scrapy.utils.test import get_crawler
 
 
+class CoreStatsExtensionTest(unittest.TestCase):
+
+    def setUp(self):
+        self.crawler = get_crawler(Spider)
+        self.spider = self.crawler._create_spider('foo')
+
+    @mock.patch('scrapy.extensions.corestats.datetime')
+    def test_core_stats_default_stats_collector(self, mock_datetime):
+        fixed_datetime = datetime(2019, 12, 1, 11, 38)
+        mock_datetime.utcnow = mock.Mock(return_value=fixed_datetime)
+        self.crawler.stats = StatsCollector(self.crawler)
+        ext = CoreStats.from_crawler(self.crawler)
+        ext.spider_opened(self.spider)
+        ext.item_scraped({}, self.spider)
+        ext.response_received(self.spider)
+        ext.item_dropped({}, self.spider, ZeroDivisionError())
+        ext.spider_closed(self.spider, 'finished')
+        self.assertEqual(
+            ext.stats._stats,
+            {
+                'start_time': fixed_datetime,
+                'finish_time': fixed_datetime,
+                'item_scraped_count': 1,
+                'response_received_count': 1,
+                'item_dropped_count': 1,
+                'item_dropped_reasons_count/ZeroDivisionError': 1,
+                'finish_reason': 'finished',
+                'elapsed_time_seconds': 0.0,
+            }
+        )
+
+    def test_core_stats_dummy_stats_collector(self):
+        self.crawler.stats = DummyStatsCollector(self.crawler)
+        ext = CoreStats.from_crawler(self.crawler)
+        ext.spider_opened(self.spider)
+        ext.item_scraped({}, self.spider)
+        ext.response_received(self.spider)
+        ext.item_dropped({}, self.spider, ZeroDivisionError())
+        ext.spider_closed(self.spider, 'finished')
+        self.assertEqual(ext.stats._stats, {})
+
+
 class StatsCollectorTest(unittest.TestCase):
 
     def setUp(self):