Repository: scrapy/scrapy

Issue:
JsonItemExporter puts lone comma in the output if encoder fails
If `JsonItemExporter` is unable to encode the item, it still writes a delimiter (comma) to the output file. Here is a sample spider:

```
# -*- coding: utf-8 -*-
import datetime
import scrapy

class DummySpider(scrapy.Spider):
    name = 'dummy'
    start_urls = ['http://example.org/']

    def parse(self, response):
        yield {'date': datetime.date(2018, 1, 1)}
        yield {'date': datetime.date(1234, 1, 1)}
        yield {'date': datetime.date(2019, 1, 1)})
```

Encoding the second items fails:

```
2018-01-25 09:05:57 [scrapy.utils.signal] ERROR: Error caught on signal handler: <bound method ?.item_scraped of <scrapy.extensions.feedexport.FeedExporter object at 0x7fcbfbd81250>>
Traceback (most recent call last):
  File "/home/pasmen/SW/Python/data-sci-env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 150, in maybeDeferred
    result = f(*args, **kw)
  File "/home/pasmen/SW/Python/data-sci-env/local/lib/python2.7/site-packages/pydispatch/robustapply.py", line 55, in robustApply
    return receiver(*arguments, **named)
  File "/home/pasmen/SW/Python/data-sci-env/local/lib/python2.7/site-packages/scrapy/extensions/feedexport.py", line 224, in item_scraped
    slot.exporter.export_item(item)
  File "/home/pasmen/SW/Python/data-sci-env/local/lib/python2.7/site-packages/scrapy/exporters.py", line 130, in export_item
    data = self.encoder.encode(itemdict)
  File "/usr/lib/python2.7/json/encoder.py", line 207, in encode
    chunks = self.iterencode(o, _one_shot=True)
  File "/usr/lib/python2.7/json/encoder.py", line 270, in iterencode
    return _iterencode(o, 0)
  File "/home/pasmen/SW/Python/data-sci-env/local/lib/python2.7/site-packages/scrapy/utils/serialize.py", line 22, in default
    return o.strftime(self.DATE_FORMAT)
ValueError: year=1234 is before 1900; the datetime strftime() methods require year >= 1900
```

The output looks like this:

```
[
{"date": "2018-01-01"},
,
{"date": "2019-01-01"}
]
```

This seems not to be a valid JSON file as e.g. `json.load()` and `jq` fail to parse it.

I think the problem is in [`export_item`](https://github.com/scrapy/scrapy/blob/108f8c4fd20a47bb94e010e9c6296f7ed9fdb2bd/scrapy/exporters.py#L123) method of `JsonItemExporter` class where it outputs the comma before decoding the item. The correct approach would be to try to decode the item (possibly with other needed operations) and perform the write atomically.
Partial fix for #3090 - only addresses JSON feeds.
Quick fix for #3090, but the general problem of failure-during-multistep-serialisation also probably affects our XML exporter. A more broad fix would be to buffer changes and then "commit" them once everything has succeeded, but that's a bigger change than I felt comfortable writing without some discussion. :)


Discussion:
Thanks @tlinhart - will take a look at making a fix to that right away. :)


Resolving patch:
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -133,6 +133,13 @@ def _beautify_newline(self):
         if self.indent is not None:
             self.file.write(b"\n")
 
+    def _add_comma_after_first(self):
+        if self.first_item:
+            self.first_item = False
+        else:
+            self.file.write(b",")
+            self._beautify_newline()
+
     def start_exporting(self):
         self.file.write(b"[")
         self._beautify_newline()
@@ -142,14 +149,10 @@ def finish_exporting(self):
         self.file.write(b"]")
 
     def export_item(self, item):
-        if self.first_item:
-            self.first_item = False
-        else:
-            self.file.write(b",")
-            self._beautify_newline()
         itemdict = dict(self._get_serialized_fields(item))
-        data = self.encoder.encode(itemdict)
-        self.file.write(to_bytes(data, self.encoding))
+        data = to_bytes(self.encoder.encode(itemdict), self.encoding)
+        self._add_comma_after_first()
+        self.file.write(data)
 
 
 class XmlItemExporter(BaseItemExporter):


Test patch:
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -599,6 +599,20 @@ def test_two_items(self):
     def test_two_dict_items(self):
         self.assertTwoItemsExported(ItemAdapter(self.i).asdict())
 
+    def test_two_items_with_failure_between(self):
+        i1 = TestItem(name="Joseph\xa3", age="22")
+        i2 = TestItem(
+            name="Maria", age=1j
+        )  # Invalid datetimes didn't consistently fail between Python versions
+        i3 = TestItem(name="Jesus", age="44")
+        self.ie.start_exporting()
+        self.ie.export_item(i1)
+        self.assertRaises(TypeError, self.ie.export_item, i2)
+        self.ie.export_item(i3)
+        self.ie.finish_exporting()
+        exported = json.loads(to_unicode(self.output.getvalue()))
+        self.assertEqual(exported, [dict(i1), dict(i3)])
+
     def test_nested_item(self):
         i1 = self.item_class(name="Joseph\xa3", age="22")
         i2 = self.item_class(name="Maria", age=i1)
@@ -637,6 +651,24 @@ def test_nonstring_types_item(self):
         self.assertEqual(exported, [item])
 
 
+class JsonItemExporterToBytesTest(BaseItemExporterTest):
+    def _get_exporter(self, **kwargs):
+        kwargs["encoding"] = "latin"
+        return JsonItemExporter(self.output, **kwargs)
+
+    def test_two_items_with_failure_between(self):
+        i1 = TestItem(name="Joseph", age="22")
+        i2 = TestItem(name="\u263a", age="11")
+        i3 = TestItem(name="Jesus", age="44")
+        self.ie.start_exporting()
+        self.ie.export_item(i1)
+        self.assertRaises(UnicodeEncodeError, self.ie.export_item, i2)
+        self.ie.export_item(i3)
+        self.ie.finish_exporting()
+        exported = json.loads(to_unicode(self.output.getvalue(), encoding="latin"))
+        self.assertEqual(exported, [dict(i1), dict(i3)])
+
+
 class JsonItemExporterDataclassTest(JsonItemExporterTest):
     item_class = TestDataClass
     custom_field_item_class = CustomFieldDataclass