Repository: stummjr/scrapy-fieldstats

Issue:
add an option to skip None values in the counts
So far, if a field is `None`, it's gonna be counted as if it was populated.

The default behavior should be changed, so that `None` values are considered as if the field didn't exist. An option to allow `None` fields to be counted as not empty should be added as well.

Discussion:

Patch:
diff --git a/scrapy_fieldstats/fieldstats.py b/scrapy_fieldstats/fieldstats.py
index 114613c..2b41dd1 100644
--- a/scrapy_fieldstats/fieldstats.py
+++ b/scrapy_fieldstats/fieldstats.py
@@ -12,10 +12,11 @@ class FieldStatsExtension(object):
     """ When enabled, the FieldStats extension logs the percentage of
         items coverage for a crawl.
     """
-    def __init__(self, show_counts=False):
+    def __init__(self, show_counts=False, skip_none=True):
         self.item_count = 0
         self.field_counts = {}
         self.show_counts = show_counts
+        self.skip_none = skip_none
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -23,7 +24,8 @@ class FieldStatsExtension(object):
             raise NotConfigured
 
         show_counts = crawler.settings.getbool('FIELDSTATS_COUNTS_ONLY', False)
-        ext = cls(show_counts)
+        skip_none = crawler.settings.getbool('FIELDSTATS_SKIP_NONE', True)
+        ext = cls(show_counts, skip_none)
         crawler.signals.connect(ext.item_scraped, signal=signals.item_scraped)
         crawler.signals.connect(ext.spider_closed, signal=signals.spider_closed)
         return ext
@@ -56,6 +58,9 @@ class FieldStatsExtension(object):
                 self.count_item_fields(value, current_node=current_node[name])
                 continue
 
+            if self.skip_none and value is None:
+                continue
+
             if name not in current_node:
                 current_node[name] = 0
             current_node[name] += 1


Test patch:
diff --git a/tests/test_scrapy_fieldstats.py b/tests/test_scrapy_fieldstats.py
index 827bc44..5feda39 100644
--- a/tests/test_scrapy_fieldstats.py
+++ b/tests/test_scrapy_fieldstats.py
@@ -2,8 +2,8 @@
 from scrapy_fieldstats.fieldstats import FieldStatsExtension
 
 
-def extract_fake_items_and_compute_stats(fake_items, show_counts=False):
-    ext = FieldStatsExtension()
+def extract_fake_items_and_compute_stats(fake_items, show_counts=False, skip_none=True):
+    ext = FieldStatsExtension(skip_none=skip_none)
     for item in fake_items:
         ext.compute_item(item)
 
@@ -123,6 +123,20 @@ def test_many_items_many_fields_empty_field():
     assert field_stats['field2'] == 2
 
 
+def test_none_values():
+    fake_items = [
+        {"field1": None, "field2": ""},
+        {"field1": "value1", "field2": "value2"},
+    ]
+    field_stats = extract_fake_items_and_compute_stats(fake_items, show_counts=True)
+    assert field_stats['field1'] == 1
+    assert field_stats['field2'] == 2
+
+    field_stats = extract_fake_items_and_compute_stats(fake_items, show_counts=True, skip_none=False)
+    assert field_stats['field1'] == 2
+    assert field_stats['field2'] == 2
+
+
 def test_nested_items():
     fake_items = [
         {