Repository: scrapy/scrapy

Issue:
Issue with CsvItemExporter when used as Feed Exporter with a crawler that yields more than one type of Item
My project includes a spider that yields several different types of Items. All my items inherit from a 'DefaultItem' that has fields such as 'url', 'time_crawled' and 'group_id'.

When defining `FEED_FORMAT = 'csv'`, the feed exporter uses the fields of the first type of Item yielded as the csv's header/column names and then for subsequent yielded items (despite the Item's type) tries to export those items with the fields of the first yielded item.

Since all of the items have 'url', 'time_crawled', 'group_id' and fields that may share the same names, those columns in the CSV get exported but otherwise the column for particular field is empty.

`tim@lindev:~/kkc$ scrapy -v
Scrapy 0.24.4 - project: kkc`



Discussion:
That could be termed undefined behavior.
It's really not so possible to do what you want, the way the code works now, because if that would be handled the Exporter couldn't start exporting until the whole crawl is finished, to be certain it has "seen" all the possible Items that can be exported before building a CSV header column.

You should write a custom Pipeline to define your required behavior.

I think we can merge a PR of a csv headers setting.


Resolving patch:
diff --git a/scrapy/contrib/feedexport.py b/scrapy/contrib/feedexport.py
--- a/scrapy/contrib/feedexport.py
+++ b/scrapy/contrib/feedexport.py
@@ -146,6 +146,7 @@ def __init__(self, settings):
         if not self._exporter_supported(self.format):
             raise NotConfigured
         self.store_empty = settings.getbool('FEED_STORE_EMPTY')
+        self.export_fields = settings.getlist('FEED_EXPORT_FIELDS')
         uripar = settings['FEED_URI_PARAMS']
         self._uripar = load_object(uripar) if uripar else lambda x, y: None
 
@@ -169,7 +170,7 @@ def open_spider(self, spider):
         uri = self.urifmt % self._get_uri_params(spider)
         storage = self._get_storage(uri)
         file = storage.open(spider)
-        exporter = self._get_exporter(file)
+        exporter = self._get_exporter(file, fields_to_export=self.export_fields)
         exporter.start_exporting()
         self.slot = SpiderSlot(file, exporter, storage, uri)
 
@@ -218,8 +219,8 @@ def _storage_supported(self, uri):
         else:
             log.msg("Unknown feed storage scheme: %s" % scheme, log.ERROR)
 
-    def _get_exporter(self, *a, **kw):
-        return self.exporters[self.format](*a, **kw)
+    def _get_exporter(self, *args, **kwargs):
+        return self.exporters[self.format](*args, **kwargs)
 
     def _get_storage(self, uri):
         return self.storages[urlparse(uri).scheme](uri)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -126,6 +126,7 @@
 FEED_URI_PARAMS = None  # a function to extend uri arguments
 FEED_FORMAT = 'jsonlines'
 FEED_STORE_EMPTY = False
+FEED_EXPORT_FIELDS = None
 FEED_STORAGES = {}
 FEED_STORAGES_BASE = {
     '': 'scrapy.contrib.feedexport.FileFeedStorage',
diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py
--- a/scrapy/utils/testproc.py
+++ b/scrapy/utils/testproc.py
@@ -1,13 +1,15 @@
+from __future__ import absolute_import
 import sys
 import os
 
 from twisted.internet import reactor, defer, protocol
 
+
 class ProcessTest(object):
 
     command = None
     prefix = [sys.executable, '-m', 'scrapy.cmdline']
-    cwd = os.getcwd() # trial chdirs to temp dir
+    cwd = os.getcwd()  # trial chdirs to temp dir
 
     def execute(self, args, check_code=True, settings=None):
         env = os.environ.copy()


Test patch:
diff --git a/tests/mockserver.py b/tests/mockserver.py
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -191,6 +191,7 @@ def __enter__(self):
         self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver'],
                           stdout=PIPE, env=get_testenv())
         self.proc.stdout.readline()
+        return self
 
     def __exit__(self, exc_type, exc_value, traceback):
         self.proc.kill()
diff --git a/tests/test_commands.py b/tests/test_commands.py
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -95,11 +95,11 @@ def test_template(self, tplname='crawl'):
         spname = 'test_spider'
         p = self.proc('genspider', spname, 'test.com', *args)
         out = retry_on_eintr(p.stdout.read)
-        self.assert_("Created spider %r using template %r in module" % (spname, tplname) in out)
-        self.assert_(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')))
+        self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out)
+        self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')))
         p = self.proc('genspider', spname, 'test.com', *args)
         out = retry_on_eintr(p.stdout.read)
-        self.assert_("Spider %r already exists in module" % spname in out)
+        self.assertIn("Spider %r already exists in module" % spname, out)
 
     def test_template_basic(self):
         self.test_template('basic')
@@ -148,10 +148,10 @@ def start_requests(self):
 """)
         p = self.proc('runspider', fname)
         log = p.stderr.read()
-        self.assert_("[myspider] DEBUG: It Works!" in log, log)
-        self.assert_("[myspider] INFO: Spider opened" in log, log)
-        self.assert_("[myspider] INFO: Closing spider (finished)" in log, log)
-        self.assert_("[myspider] INFO: Spider closed (finished)" in log, log)
+        self.assertIn("[myspider] DEBUG: It Works!", log)
+        self.assertIn("[myspider] INFO: Spider opened", log)
+        self.assertIn("[myspider] INFO: Closing spider (finished)", log)
+        self.assertIn("[myspider] INFO: Spider closed (finished)", log)
 
     def test_runspider_no_spider_found(self):
         tmpdir = self.mktemp()
@@ -164,12 +164,12 @@ def test_runspider_no_spider_found(self):
 """)
         p = self.proc('runspider', fname)
         log = p.stderr.read()
-        self.assert_("No spider found in file" in log)
+        self.assertIn("No spider found in file", log)
 
     def test_runspider_file_not_found(self):
         p = self.proc('runspider', 'some_non_existent_file')
         log = p.stderr.read()
-        self.assert_("File not found: some_non_existent_file" in log)
+        self.assertIn("File not found: some_non_existent_file", log)
 
     def test_runspider_unable_to_load(self):
         tmpdir = self.mktemp()
@@ -179,7 +179,7 @@ def test_runspider_unable_to_load(self):
             f.write("")
         p = self.proc('runspider', fname)
         log = p.stderr.read()
-        self.assert_("Unable to load" in log)
+        self.assertIn("Unable to load", log)
 
 
 class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
@@ -229,7 +229,7 @@ def test_spider_arguments(self):
                                            '-a', 'test_arg=1',
                                            '-c', 'parse',
                                            self.url('/html')])
-        self.assert_("[parse_spider] DEBUG: It Works!" in stderr, stderr)
+        self.assertIn("[parse_spider] DEBUG: It Works!", stderr)
 
     @defer.inlineCallbacks
     def test_pipelines(self):
@@ -237,7 +237,7 @@ def test_pipelines(self):
                                            '--pipelines',
                                            '-c', 'parse',
                                            self.url('/html')])
-        self.assert_("[scrapy] INFO: It Works!" in stderr, stderr)
+        self.assertIn("[scrapy] INFO: It Works!", stderr)
 
     @defer.inlineCallbacks
     def test_parse_items(self):
@@ -254,4 +254,4 @@ def test_run(self):
         p = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001',
                 '-s', 'CLOSESPIDER_TIMEOUT=0.01')
         log = p.stderr.read()
-        self.assert_('INFO: Crawled' in log, log)
+        self.assertIn('INFO: Crawled', log)
diff --git a/tests/test_contrib_feedexport.py b/tests/test_contrib_feedexport.py
--- a/tests/test_contrib_feedexport.py
+++ b/tests/test_contrib_feedexport.py
@@ -1,16 +1,28 @@
+from __future__ import absolute_import
 import os
+import csv
 from io import BytesIO
+import tempfile
+import shutil
+import six
 from six.moves.urllib.parse import urlparse
 
 from zope.interface.verify import verifyObject
 from twisted.trial import unittest
 from twisted.internet import defer
+from scrapy.crawler import CrawlerRunner
+from scrapy.settings import Settings
+from tests.mockserver import MockServer
 from w3lib.url import path_to_file_uri
 
-from scrapy.spider import Spider
-from scrapy.contrib.feedexport import IFeedStorage, FileFeedStorage, FTPFeedStorage, S3FeedStorage, StdoutFeedStorage
+import scrapy
+from scrapy.contrib.feedexport import (
+    IFeedStorage, FileFeedStorage, FTPFeedStorage,
+    S3FeedStorage, StdoutFeedStorage
+)
 from scrapy.utils.test import assert_aws_environ
 
+
 class FileFeedStorageTest(unittest.TestCase):
 
     def test_store_file_uri(self):
@@ -39,7 +51,7 @@ def test_interface(self):
 
     @defer.inlineCallbacks
     def _assert_stores(self, storage, path):
-        spider = Spider("default")
+        spider = scrapy.Spider("default")
         file = storage.open(spider)
         file.write(b"content")
         yield storage.store(file)
@@ -61,7 +73,7 @@ def test_store(self):
 
     @defer.inlineCallbacks
     def _assert_stores(self, storage, path):
-        spider = Spider("default")
+        spider = scrapy.Spider("default")
         file = storage.open(spider)
         file.write(b"content")
         yield storage.store(file)
@@ -85,7 +97,7 @@ def test_store(self):
         from boto import connect_s3
         storage = S3FeedStorage(uri)
         verifyObject(IFeedStorage, storage)
-        file = storage.open(Spider("default"))
+        file = storage.open(scrapy.Spider("default"))
         file.write("content")
         yield storage.store(file)
         u = urlparse(uri)
@@ -99,7 +111,163 @@ class StdoutFeedStorageTest(unittest.TestCase):
     def test_store(self):
         out = BytesIO()
         storage = StdoutFeedStorage('stdout:', _stdout=out)
-        file = storage.open(Spider("default"))
+        file = storage.open(scrapy.Spider("default"))
         file.write(b"content")
         yield storage.store(file)
         self.assertEqual(out.getvalue(), b"content")
+
+
+class FeedExportTest(unittest.TestCase):
+
+    skip = not six.PY2
+
+    class MyItem(scrapy.Item):
+        foo = scrapy.Field()
+        egg = scrapy.Field()
+        baz = scrapy.Field()
+
+
+    @defer.inlineCallbacks
+    def run_and_export(self, spider_cls, settings=None):
+        """ Run spider with specified settings; return exported data. """
+        tmpdir = tempfile.mkdtemp()
+        res_name = tmpdir + '/res'
+        defaults = {
+            'FEED_URI': 'file://' + res_name,
+            'FEED_FORMAT': 'csv',
+        }
+        defaults.update(settings or {})
+        try:
+            with MockServer() as s:
+                runner = CrawlerRunner(Settings(defaults))
+                yield runner.crawl(spider_cls)
+
+            with open(res_name, 'rb') as f:
+                defer.returnValue(f.read())
+
+        finally:
+            shutil.rmtree(tmpdir)
+
+    @defer.inlineCallbacks
+    def exported_data(self, items, settings):
+        """
+        Return exported data which a spider yielding ``items`` would return.
+        """
+        class TestSpider(scrapy.Spider):
+            name = 'testspider'
+            start_urls = ['http://localhost:8998/']
+
+            def parse(self, response):
+                for item in items:
+                    yield item
+
+        data = yield self.run_and_export(TestSpider, settings)
+        defer.returnValue(data)
+
+    @defer.inlineCallbacks
+    def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
+        settings = settings or {}
+        settings.update({'FEED_FORMAT': 'csv'})
+        data = yield self.exported_data(items, settings)
+
+        reader = csv.DictReader(data.splitlines())
+        got_rows = list(reader)
+        if ordered:
+            self.assertEqual(reader.fieldnames, header)
+        else:
+            self.assertEqual(set(reader.fieldnames), set(header))
+
+        self.assertEqual(rows, got_rows)
+
+    @defer.inlineCallbacks
+    def test_export_csv_items(self):
+        # feed exporters use field names from Item
+        items = [
+            self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
+            self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
+        ]
+        rows = [
+            {'egg': 'spam1', 'foo': 'bar1', 'baz': ''},
+            {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
+        ]
+        header = self.MyItem.fields.keys()
+        yield self.assertExportedCsv(items, header, rows, ordered=False)
+
+    @defer.inlineCallbacks
+    def test_export_csv_multiple_item_classes(self):
+
+        class MyItem2(scrapy.Item):
+            foo = scrapy.Field()
+            hello = scrapy.Field()
+
+        items = [
+            self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
+            MyItem2({'hello': 'world2', 'foo': 'bar2'}),
+            self.MyItem({'foo': 'bar3', 'egg': 'spam3', 'baz': 'quux3'}),
+            {'hello': 'world4', 'egg': 'spam4'},
+        ]
+
+        # by default, Scrapy uses fields of the first Item
+        header = self.MyItem.fields.keys()
+        rows = [
+            {'egg': 'spam1', 'foo': 'bar1', 'baz': ''},
+            {'egg': '',      'foo': 'bar2', 'baz': ''},
+            {'egg': 'spam3', 'foo': 'bar3', 'baz': 'quux3'},
+            {'egg': 'spam4', 'foo': '',     'baz': ''},
+        ]
+        yield self.assertExportedCsv(items, header, rows, ordered=False)
+
+        # but it is possible to override fields using FEED_EXPORT_FIELDS
+        header = ["foo", "baz", "hello"]
+        settings = {'FEED_EXPORT_FIELDS': header}
+        rows = [
+            {'foo': 'bar1', 'baz': '',      'hello': ''},
+            {'foo': 'bar2', 'baz': '',      'hello': 'world2'},
+            {'foo': 'bar3', 'baz': 'quux3', 'hello': ''},
+            {'foo': '',     'baz': '',      'hello': 'world4'},
+        ]
+        yield self.assertExportedCsv(items, header, rows,
+                                     settings=settings, ordered=True)
+
+    @defer.inlineCallbacks
+    def test_export_csv_dicts(self):
+        # When dicts are used, only keys from the first row are used as
+        # a header.
+        items = [
+            {'foo': 'bar', 'egg': 'spam'},
+            {'foo': 'bar', 'egg': 'spam', 'baz': 'quux'},
+        ]
+        rows = [
+            {'egg': 'spam', 'foo': 'bar'},
+            {'egg': 'spam', 'foo': 'bar'}
+        ]
+        yield self.assertExportedCsv(items, ['egg', 'foo'], rows, ordered=False)
+
+    @defer.inlineCallbacks
+    def test_export_csv_feed_export_fields(self):
+        # FEED_EXPORT_FIELDS option allows to order export fields
+        # and to select a subset of fields to export, both for Items and dicts.
+
+        for item_cls in [self.MyItem, dict]:
+            items = [
+                item_cls({'foo': 'bar1', 'egg': 'spam1'}),
+                item_cls({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
+            ]
+
+            # export all columns
+            settings = {'FEED_EXPORT_FIELDS': 'foo,baz,egg'}
+            rows = [
+                {'egg': 'spam1', 'foo': 'bar1', 'baz': ''},
+                {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
+            ]
+            yield self.assertExportedCsv(items, ['foo', 'baz', 'egg'], rows,
+                                         settings=settings, ordered=True)
+
+            # export a subset of columns
+            settings = {'FEED_EXPORT_FIELDS': 'egg,baz'}
+            rows = [
+                {'egg': 'spam1', 'baz': ''},
+                {'egg': 'spam2', 'baz': 'quux2'}
+            ]
+            yield self.assertExportedCsv(items, ['egg', 'baz'], rows,
+                                         settings=settings, ordered=True)