Repository: stummjr/scrapy-fieldstats

Issue:
Support nested items
Currently, the extension only counts shallow items in dictionaries.

However, it's quite common to have nested items such as:

```json
{
    "title": "Animal Farm",
    "author": {
        "name": "George Orwell",
        "birth_location": "Motihari, India",
        "birth_date": "1903-06-25"
    },
    "ratings": 4.9
}
```

In this case, the extension doesn't generate coverage stats for the nested `name`, `birth_location` and `birth_date` fields.


Discussion:

Patch:
diff --git a/scrapy_fieldstats/fieldstats.py b/scrapy_fieldstats/fieldstats.py
index 703b2dc..83d9860 100644
--- a/scrapy_fieldstats/fieldstats.py
+++ b/scrapy_fieldstats/fieldstats.py
@@ -1,7 +1,6 @@
 # -*- coding:utf-8 -*-
 import logging
 import pprint
-from collections import defaultdict
 
 from scrapy import signals
 from scrapy.exceptions import NotConfigured
@@ -10,12 +9,12 @@ logger = logging.getLogger(__name__)
 
 
 class FieldStatsExtension(object):
-    """ When enabled, the FieldStats extensions logs the percentage of
+    """ When enabled, the FieldStats extension logs the percentage of
         items coverage for a crawl.
     """
     def __init__(self):
         self.item_count = 0
-        self.field_counts = defaultdict(int)
+        self.field_counts = {}
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -28,20 +27,46 @@ class FieldStatsExtension(object):
         return ext
 
     def item_scraped(self, item, spider):
+        self.compute_item(item)
+
+    def spider_closed(self, spider):
+        fields_summary = self.build_fields_summary()
+        logger.info('Field stats:\n{}'.format(pprint.pformat(fields_summary)))
+
+    def compute_item(self, item):
         self.item_count += 1
+        self.count_item_fields(item)
+
+    def count_item_fields(self, item, current_node=None):
+        if current_node is None:
+            current_node = self.field_counts
+
         for name, value in item.items():
             if not value:
                 continue
-            self.field_counts[name] += 1
 
-    def spider_closed(self, spider):
-        field_stats = self.compute_fieldstats()
-        logger.info('Field stats:\n{}'.format(pprint.pformat(field_stats)))
+            if isinstance(value, dict):
+                # recurse into nested items
+                if name not in current_node:
+                    current_node[name] = {}
+                self.count_item_fields(value, current_node=current_node[name])
+                continue
+
+            if name not in current_node:
+                current_node[name] = 0
+            current_node[name] += 1
+
+    def build_fields_summary(self, field_counts=None, fields_summary=None):
+        if field_counts is None:
+            field_counts = self.field_counts
+            fields_summary = {}
 
-    def compute_fieldstats(self):
-        field_stats = {}
-        for name, count in self.field_counts.items():
-            field_coverage = int(count) * 100 / self.item_count
-            field_stats[name] = "{}%".format(field_coverage)
+        for name, value in field_counts.items():
+            if isinstance(value, dict):
+                fields_summary[name] = {}
+                self.build_fields_summary(field_counts[name], fields_summary[name])
+            else:
+                field_percentage = int(value) * 100 / self.item_count
+                fields_summary[name] = "{}%".format(field_percentage)
 
-        return field_stats
+        return fields_summary


Test patch:
diff --git a/tests/test_scrapy_fieldstats.py b/tests/test_scrapy_fieldstats.py
index 2873777..24b749f 100644
--- a/tests/test_scrapy_fieldstats.py
+++ b/tests/test_scrapy_fieldstats.py
@@ -3,18 +3,20 @@
 from scrapy_fieldstats.fieldstats import FieldStatsExtension
 
 
-def fake_extract_items(fake_items, extension):
+def extract_fake_items_and_compute_stats(fake_items):
+    ext = FieldStatsExtension()
     for item in fake_items:
-        extension.item_scraped(item, None)
+        ext.compute_item(item)
+    field_stats = ext.build_fields_summary()
+    return field_stats
 
 
 def test_single_item():
     fake_items = [{"field1": "value1"}]
-    ext = FieldStatsExtension()
-    fake_extract_items(fake_items, ext)
-    field_stats = ext.compute_fieldstats()
+    field_stats = extract_fake_items_and_compute_stats(fake_items)
+
     assert len(field_stats) == 1
-    assert field_stats.get('field1') == '100.0%'
+    assert field_stats['field1'] == '100.0%'
 
 
 def test_single_item_many_fields():
@@ -24,21 +26,19 @@ def test_single_item_many_fields():
             "field2": "value2",
         }
     ]
-    ext = FieldStatsExtension()
-    fake_extract_items(fake_items, ext)
-    field_stats = ext.compute_fieldstats()
+    field_stats = extract_fake_items_and_compute_stats(fake_items)
+
     assert len(field_stats) == 2
-    assert field_stats.get('field1') == '100.0%'
-    assert field_stats.get('field2') == '100.0%'
+    assert field_stats['field1'] == '100.0%'
+    assert field_stats['field2'] == '100.0%'
 
 
 def test_many_items():
     fake_items = [{"field1": "value1"}, {"field1": "value1"}]
-    ext = FieldStatsExtension()
-    fake_extract_items(fake_items, ext)
-    field_stats = ext.compute_fieldstats()
+    field_stats = extract_fake_items_and_compute_stats(fake_items)
+
     assert len(field_stats) == 1
-    assert field_stats.get('field1') == '100.0%'
+    assert field_stats['field1'] == '100.0%'
 
 
 def test_many_items_many_fields():
@@ -52,12 +52,11 @@ def test_many_items_many_fields():
             "field2": "value2",
         }
     ]
-    ext = FieldStatsExtension()
-    fake_extract_items(fake_items, ext)
-    field_stats = ext.compute_fieldstats()
+    field_stats = extract_fake_items_and_compute_stats(fake_items)
+
     assert len(field_stats) == 2
-    assert field_stats.get('field1') == '100.0%'
-    assert field_stats.get('field2') == '100.0%'
+    assert field_stats['field1'] == '100.0%'
+    assert field_stats['field2'] == '100.0%'
 
 
 def test_many_items_many_fields_missing_field():
@@ -70,12 +69,11 @@ def test_many_items_many_fields_missing_field():
             "field2": "value2",
         }
     ]
-    ext = FieldStatsExtension()
-    fake_extract_items(fake_items, ext)
-    field_stats = ext.compute_fieldstats()
+    field_stats = extract_fake_items_and_compute_stats(fake_items)
+
     assert len(field_stats) == 2
-    assert field_stats.get('field1') == '100.0%'
-    assert field_stats.get('field2') == '50.0%'
+    assert field_stats['field1'] == '100.0%'
+    assert field_stats['field2'] == '50.0%'
 
 
 def test_many_items_many_fields_empty_field():
@@ -89,9 +87,41 @@ def test_many_items_many_fields_empty_field():
             "field2": "value2",
         }
     ]
-    ext = FieldStatsExtension()
-    fake_extract_items(fake_items, ext)
-    field_stats = ext.compute_fieldstats()
+    field_stats = extract_fake_items_and_compute_stats(fake_items)
+
     assert len(field_stats) == 2
-    assert field_stats.get('field1') == '100.0%'
-    assert field_stats.get('field2') == '50.0%'
+    assert field_stats['field1'] == '100.0%'
+    assert field_stats['field2'] == '50.0%'
+
+
+def test_nested_items():
+    fake_items = [
+        {
+            "field1": "value1",
+            "field2": {
+                "field2.1": "value2.1",
+                "field2.2": "value2.2",
+                "field2.3": {
+                    "field2.3.1": "value2.3.1",
+                    "field2.3.2": "value2.3.2",
+                },
+            }
+        },
+        {
+            "field1": "value1",
+            "field2": {
+                "field2.1": "value2.1",
+                "field2.3": {
+                    "field2.3.1": "value2.3.1",
+                    "field2.3.2": "",
+                },
+                "field2.4": "value2.2",
+            }
+        }
+    ]
+    field_stats = extract_fake_items_and_compute_stats(fake_items)
+    assert field_stats['field1'] == '100.0%'
+    assert field_stats['field2']['field2.1'] == '100.0%'
+    assert field_stats['field2']['field2.2'] == '50.0%'
+    assert field_stats['field2']['field2.2'] == '50.0%'
+    assert field_stats['field2']['field2.3']['field2.3.2'] == '50.0%'