Repository: scrapy/scrapy

Issue:
Non-ascii field names
In Python 2 variable names are limited by ascii character set.
CSV Scrapy exporter uses Item's field names as an csv title row.
I want to use unicode characters for csv title row.
Can I do that without writing my own exporter?

Allow defining field names that don't follow the restrictions of python variables
Sometimes we may require defining field names that are not valid Python variables (for example containing spaces). I believe the correct way of doing this right now is that the developer needs to add a Pipeline that maps the internal name to the name we wan't to export. Scrapy could ship with a Pipeline (maybe enabled by default) that does this mapping for fields that define a field parameter. For example:

```
class MyItem(Item):
   date_from = Field(export_name='Initial date')
```

This is a very simple pipeline. However, since this is a very common use case, I think it is something that could be shipped by default. I'm happy to provide a PR for this.


Discussion:
I think, the easiest way is - I often see this as the easiest way, so maybe I am wrong - to extend the existing class like this (untested, for reference):

``` python
class UnicodeCsvItemExporter(CsvItemExporter):
    def __init__(self, file, include_headers_line=True, join_multivalued=',', headers={}, **kwargs):
        super(UnicodeCsvItemExporter, self).__init__(file, include_headers_line, join_multivalued, **kwargs)

        self.header_mapping = headers

    def _write_headers_and_set_fields_to_export(self, item):
        if self.include_headers_line:
            if not self.fields_to_export:
                self.fields_to_export = item.fields.keys()
            # Maybe list(...) is required here to give writerow a list and not a lazy iterable
            self.csv_writer.writerow(map(lambda x: self.header_mapping[x], self.fields_to_export))
```

Then, you'd have to add your exporter to the settings:

``` python
FEED_FORMAT = 'csv_unicode'
FEED_EXPORTERS = {
    'csv_unicode': '.....',
}
```

Edit:
mmhh... or maybe not. You will not have access to the `headers` parameter through the string settings... If you find a solution for this, then it will be fine. Otherwise, it's not a solution.
My idea here would be to add another setting in `settings.py` called `CSV_UNICODE_HEADERS`, but it seems exporters are one of the few places not having a reference of `settings`.

I am now really wondering, where scrapy generates an instance of the selected exporter, because each class has a totally different constructor signature.

@Vanuan Extend [this method](https://github.com/scrapy/scrapy/blob/master/scrapy/contrib/exporter/__init__.py#L191) to pull the header name from the [item.Field](doc.scrapy.org/en/latest/topics/items.html#item-fields).

If you're using dicts instead of scrapy.Item instances, you can do `yield {'Initial date': my_date_from, ...}`. 

But I like the idea, I think we can it built-in.
I like that idea, but, wouldn't it be possible to reference the Item or the Item Loader by both the export_name or the var name instead of adding a new pipeline?

For example:

```
class MyItem(Item):
   date_from = Field(export_name='Initial date')

item = MyItem()
item['date_from'] = '01/01/1970' # This Works
item['Initial date'] = '01/01/1970' # This should work

loader = ItemLoader(item=MyItem()
loader.add_value('Initial date', '01/01/1970')  # This should work
```



Resolving patch:
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -8,6 +8,7 @@
 import pickle
 import pprint
 import warnings
+from collections.abc import Mapping
 from xml.sax.saxutils import XMLGenerator
 
 from itemadapter import is_item, ItemAdapter
@@ -68,6 +69,14 @@ def _get_serialized_fields(self, item, default_value=None, include_empty=None):
                 field_iter = item.field_names()
             else:
                 field_iter = item.keys()
+        elif isinstance(self.fields_to_export, Mapping):
+            if include_empty:
+                field_iter = self.fields_to_export.items()
+            else:
+                field_iter = (
+                    (x, y) for x, y in self.fields_to_export.items()
+                    if x in item
+                )
         else:
             if include_empty:
                 field_iter = self.fields_to_export
@@ -75,13 +84,17 @@ def _get_serialized_fields(self, item, default_value=None, include_empty=None):
                 field_iter = (x for x in self.fields_to_export if x in item)
 
         for field_name in field_iter:
-            if field_name in item:
-                field_meta = item.get_field_meta(field_name)
-                value = self.serialize_field(field_meta, field_name, item[field_name])
+            if isinstance(field_name, str):
+                item_field, output_field = field_name, field_name
+            else:
+                item_field, output_field = field_name
+            if item_field in item:
+                field_meta = item.get_field_meta(item_field)
+                value = self.serialize_field(field_meta, output_field, item[item_field])
             else:
                 value = default_value
 
-            yield field_name, value
+            yield output_field, value
 
 
 class JsonLinesItemExporter(BaseItemExporter):
@@ -246,7 +259,11 @@ def _write_headers_and_set_fields_to_export(self, item):
             if not self.fields_to_export:
                 # use declared field names, or keys if the item is a dict
                 self.fields_to_export = ItemAdapter(item).field_names()
-            row = list(self._build_row(self.fields_to_export))
+            if isinstance(self.fields_to_export, Mapping):
+                fields = self.fields_to_export.values()
+            else:
+                fields = self.fields_to_export
+            row = list(self._build_row(fields))
             self.csv_writer.writerow(row)
 
 
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -197,6 +197,38 @@ def getdict(self, name, default=None):
             value = json.loads(value)
         return dict(value)
 
+    def getdictorlist(self, name, default=None):
+        """Get a setting value as either a :class:`dict` or a :class:`list`.
+
+        If the setting is already a dict or a list, a copy of it will be
+        returned.
+
+        If it is a string it will be evaluated as JSON, or as a comma-separated
+        list of strings as a fallback.
+
+        For example, settings populated from the command line will return:
+
+        -   ``{'key1': 'value1', 'key2': 'value2'}`` if set to
+            ``'{"key1": "value1", "key2": "value2"}'``
+
+        -   ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'``
+
+        :param name: the setting name
+        :type name: string
+
+        :param default: the value to return if no setting is found
+        :type default: any
+        """
+        value = self.get(name, default)
+        if value is None:
+            return {}
+        if isinstance(value, str):
+            try:
+                return json.loads(value)
+            except ValueError:
+                return value.split(',')
+        return copy.deepcopy(value)
+
     def getwithbase(self, name):
         """Get a composition of a dictionary-like setting and its `_BASE`
         counterpart.
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -118,7 +118,7 @@ def feed_complete_default_values_from_settings(feed, settings):
     out = feed.copy()
     out.setdefault("batch_item_count", settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT'))
     out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
-    out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
+    out.setdefault("fields", settings.getdictorlist("FEED_EXPORT_FIELDS") or None)
     out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
     out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
     out.setdefault("item_export_kwargs", {})


Test patch:
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -112,6 +112,14 @@ def test_fields_to_export(self):
         assert isinstance(name, str)
         self.assertEqual(name, 'John\xa3')
 
+        ie = self._get_exporter(
+            fields_to_export={'name': '名稱'}
+        )
+        self.assertEqual(
+            list(ie._get_serialized_fields(self.i)),
+            [('名稱', 'John\xa3')]
+        )
+
     def test_field_custom_serializer(self):
         i = self.custom_field_item_class(name='John\xa3', age='22')
         a = ItemAdapter(i)
@@ -272,6 +280,7 @@ class MarshalItemExporterDataclassTest(MarshalItemExporterTest):
 
 class CsvItemExporterTest(BaseItemExporterTest):
     def _get_exporter(self, **kwargs):
+        self.output = tempfile.TemporaryFile()
         return CsvItemExporter(self.output, **kwargs)
 
     def assertCsvEqual(self, first, second, msg=None):
@@ -283,7 +292,8 @@ def split_csv(csv):
         return self.assertEqual(split_csv(first), split_csv(second), msg=msg)
 
     def _check_output(self):
-        self.assertCsvEqual(to_unicode(self.output.getvalue()), 'age,name\r\n22,John\xa3\r\n')
+        self.output.seek(0)
+        self.assertCsvEqual(to_unicode(self.output.read()), 'age,name\r\n22,John\xa3\r\n')
 
     def assertExportResult(self, item, expected, **kwargs):
         fp = BytesIO()
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -7,6 +7,7 @@
 import random
 import shutil
 import string
+import sys
 import tempfile
 import warnings
 from abc import ABC, abstractmethod
@@ -655,8 +656,8 @@ def parse(self, response):
         return data
 
     @defer.inlineCallbacks
-    def assertExported(self, items, header, rows, settings=None, ordered=True):
-        yield self.assertExportedCsv(items, header, rows, settings, ordered)
+    def assertExported(self, items, header, rows, settings=None):
+        yield self.assertExportedCsv(items, header, rows, settings)
         yield self.assertExportedJsonLines(items, rows, settings)
         yield self.assertExportedXml(items, rows, settings)
         yield self.assertExportedPickle(items, rows, settings)
@@ -717,7 +718,7 @@ def run_and_export(self, spider_cls, settings):
         return content
 
     @defer.inlineCallbacks
-    def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
+    def assertExportedCsv(self, items, header, rows, settings=None):
         settings = settings or {}
         settings.update({
             'FEEDS': {
@@ -725,15 +726,9 @@ def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
             },
         })
         data = yield self.exported_data(items, settings)
-
         reader = csv.DictReader(to_unicode(data['csv']).splitlines())
-        got_rows = list(reader)
-        if ordered:
-            self.assertEqual(reader.fieldnames, header)
-        else:
-            self.assertEqual(set(reader.fieldnames), set(header))
-
-        self.assertEqual(rows, got_rows)
+        self.assertEqual(reader.fieldnames, list(header))
+        self.assertEqual(rows, list(reader))
 
     @defer.inlineCallbacks
     def assertExportedJsonLines(self, items, rows, settings=None):
@@ -884,7 +879,7 @@ def test_export_items(self):
             {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
         ]
         header = self.MyItem.fields.keys()
-        yield self.assertExported(items, header, rows, ordered=False)
+        yield self.assertExported(items, header, rows)
 
     @defer.inlineCallbacks
     def test_export_no_items_not_store_empty(self):
@@ -956,25 +951,72 @@ def test_export_multiple_item_classes(self):
             {'egg': 'spam4', 'foo': '', 'baz': ''},
         ]
         rows_jl = [dict(row) for row in items]
-        yield self.assertExportedCsv(items, header, rows_csv, ordered=False)
+        yield self.assertExportedCsv(items, header, rows_csv)
         yield self.assertExportedJsonLines(items, rows_jl)
 
-        # edge case: FEED_EXPORT_FIELDS==[] means the same as default None
+    @defer.inlineCallbacks
+    def test_export_items_empty_field_list(self):
+        # FEED_EXPORT_FIELDS==[] means the same as default None
+        items = [{'foo': 'bar'}]
+        header = ["foo"]
+        rows = [{'foo': 'bar'}]
         settings = {'FEED_EXPORT_FIELDS': []}
-        yield self.assertExportedCsv(items, header, rows_csv, ordered=False)
-        yield self.assertExportedJsonLines(items, rows_jl, settings)
+        yield self.assertExportedCsv(items, header, rows)
+        yield self.assertExportedJsonLines(items, rows, settings)
 
-        # it is possible to override fields using FEED_EXPORT_FIELDS
-        header = ["foo", "baz", "hello"]
+    @defer.inlineCallbacks
+    def test_export_items_field_list(self):
+        items = [{'foo': 'bar'}]
+        header = ["foo", "baz"]
+        rows = [{'foo': 'bar', 'baz': ''}]
         settings = {'FEED_EXPORT_FIELDS': header}
-        rows = [
-            {'foo': 'bar1', 'baz': '', 'hello': ''},
-            {'foo': 'bar2', 'baz': '', 'hello': 'world2'},
-            {'foo': 'bar3', 'baz': 'quux3', 'hello': ''},
-            {'foo': '', 'baz': '', 'hello': 'world4'},
-        ]
-        yield self.assertExported(items, header, rows,
-                                  settings=settings, ordered=True)
+        yield self.assertExported(items, header, rows, settings=settings)
+
+    @defer.inlineCallbacks
+    def test_export_items_comma_separated_field_list(self):
+        items = [{'foo': 'bar'}]
+        header = ["foo", "baz"]
+        rows = [{'foo': 'bar', 'baz': ''}]
+        settings = {'FEED_EXPORT_FIELDS': ",".join(header)}
+        yield self.assertExported(items, header, rows, settings=settings)
+
+    @defer.inlineCallbacks
+    def test_export_items_json_field_list(self):
+        items = [{'foo': 'bar'}]
+        header = ["foo", "baz"]
+        rows = [{'foo': 'bar', 'baz': ''}]
+        settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
+        yield self.assertExported(items, header, rows, settings=settings)
+
+    @defer.inlineCallbacks
+    def test_export_items_field_names(self):
+        items = [{'foo': 'bar'}]
+        header = {'foo': 'Foo'}
+        rows = [{'Foo': 'bar'}]
+        settings = {'FEED_EXPORT_FIELDS': header}
+        yield self.assertExported(items, list(header.values()), rows,
+                                  settings=settings)
+
+    @defer.inlineCallbacks
+    def test_export_items_dict_field_names(self):
+        items = [{'foo': 'bar'}]
+        header = {
+            'baz': 'Baz',
+            'foo': 'Foo',
+        }
+        rows = [{'Baz': '', 'Foo': 'bar'}]
+        settings = {'FEED_EXPORT_FIELDS': header}
+        yield self.assertExported(items, ['Baz', 'Foo'], rows,
+                                  settings=settings)
+
+    @defer.inlineCallbacks
+    def test_export_items_json_field_names(self):
+        items = [{'foo': 'bar'}]
+        header = {'foo': 'Foo'}
+        rows = [{'Foo': 'bar'}]
+        settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
+        yield self.assertExported(items, list(header.values()), rows,
+                                  settings=settings)
 
     @defer.inlineCallbacks
     def test_export_based_on_item_classes(self):
@@ -1097,7 +1139,7 @@ def test_export_dicts(self):
             {'egg': 'spam', 'foo': 'bar'}
         ]
         rows_jl = items
-        yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv, ordered=False)
+        yield self.assertExportedCsv(items, ['foo', 'egg'], rows_csv)
         yield self.assertExportedJsonLines(items, rows_jl)
 
     @defer.inlineCallbacks
@@ -1118,7 +1160,7 @@ def test_export_feed_export_fields(self):
                 {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
             ]
             yield self.assertExported(items, ['foo', 'baz', 'egg'], rows,
-                                      settings=settings, ordered=True)
+                                      settings=settings)
 
             # export a subset of columns
             settings = {'FEED_EXPORT_FIELDS': 'egg,baz'}
@@ -1127,7 +1169,7 @@ def test_export_feed_export_fields(self):
                 {'egg': 'spam2', 'baz': 'quux2'}
             ]
             yield self.assertExported(items, ['egg', 'baz'], rows,
-                                      settings=settings, ordered=True)
+                                      settings=settings)
 
     @defer.inlineCallbacks
     def test_export_encoding(self):
@@ -1769,7 +1811,6 @@ def test_lzma_plugin_preset(self):
 
     @defer.inlineCallbacks
     def test_lzma_plugin_filters(self):
-        import sys
         if "PyPy" in sys.version:
             # https://foss.heptapod.net/pypy/pypy/-/issues/3527
             raise unittest.SkipTest("lzma filters doesn't work in PyPy")
@@ -2016,7 +2057,7 @@ def assertExportedJsonLines(self, items, rows, settings=None):
             self.assertEqual(expected_batch, got_batch)
 
     @defer.inlineCallbacks
-    def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
+    def assertExportedCsv(self, items, header, rows, settings=None):
         settings = settings or {}
         settings.update({
             'FEEDS': {