Repository: scrapy/scrapy

Issue:
Storage.store is called only for the first feed when empty
### Description

When having `FEED_STORE_EMPTY=False` and multiple feeds, and the output is empty, `storage.store` is called until first empty feed slot is encountered.

### Steps to Reproduce

1. Create a `test.py` file with [this spider](https://gist.github.com/StasDeep/c7f0b775eb4897b1012f2c76ce6296f5)
2. `scrapy runspider test.py -L INFO`

**Expected behavior:**

```
2020-06-10 10:50:27 [test] INFO: Storing in thread: gs://bucket/output.json
2020-06-10 10:50:27 [test] INFO: Storing in thread: gs://bucket/output.csv
```

**Actual behavior:**

```
2020-06-10 10:50:27 [test] INFO: Storing in thread: gs://bucket/output.json
```

**Reproduces how often:** always

### Versions

```
Scrapy       : 2.1.0
lxml         : 4.5.1.0
libxml2      : 2.9.10
cssselect    : 1.1.0
parsel       : 1.6.0
w3lib        : 1.22.0
Twisted      : 20.3.0
Python       : 3.7.4 (default, Sep  4 2019, 15:20:53) - [Clang 10.0.0 (clang-1000.10.44.4)]
pyOpenSSL    : 19.1.0 (OpenSSL 1.1.1g  21 Apr 2020)
cryptography : 2.9.2
Platform     : Darwin-19.4.0-x86_64-i386-64bit
```

### Reason

In `FeedExporter.close_spider`, it **returns** a deferred from inside a loop instead of adding it to `deferred_list`

```
        deferred_list = []
        for slot in self.slots:
            if not slot.itemcount and not slot.store_empty:
                # We need to call slot.storage.store nonetheless to get the file
                # properly closed.
                return defer.maybeDeferred(slot.storage.store, slot.file)
```


Discussion:
Will provide a fix for this.

Resolving patch:
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -270,7 +270,9 @@ def close_spider(self, spider):
             if not slot.itemcount and not slot.store_empty:
                 # We need to call slot.storage.store nonetheless to get the file
                 # properly closed.
-                return defer.maybeDeferred(slot.storage.store, slot.file)
+                d = defer.maybeDeferred(slot.storage.store, slot.file)
+                deferred_list.append(d)
+                continue
             slot.finish_exporting()
             logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
             log_args = {'format': slot.format,


Test patch:
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -7,6 +7,7 @@
 import tempfile
 import warnings
 from io import BytesIO
+from logging import getLogger
 from pathlib import Path
 from string import ascii_letters, digits
 from unittest import mock
@@ -14,9 +15,11 @@
 from urllib.request import pathname2url
 
 import lxml.etree
+from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial import unittest
-from w3lib.url import path_to_file_uri
+from w3lib.url import file_uri_to_path, path_to_file_uri
+from zope.interface import implementer
 from zope.interface.verify import verifyObject
 
 import scrapy
@@ -390,6 +393,25 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin):
     pass
 
 
+@implementer(IFeedStorage)
+class LogOnStoreFileStorage:
+    """
+    This storage logs inside `store` method.
+    It can be used to make sure `store` method is invoked.
+    """
+
+    def __init__(self, uri):
+        self.path = file_uri_to_path(uri)
+        self.logger = getLogger()
+
+    def open(self, spider):
+        return tempfile.NamedTemporaryFile(prefix='feed-')
+
+    def store(self, file):
+        self.logger.info('Storage.store is called')
+        file.close()
+
+
 class FeedExportTest(unittest.TestCase):
 
     class MyItem(scrapy.Item):
@@ -426,11 +448,17 @@ def run_and_export(self, spider_cls, settings):
                 yield runner.crawl(spider_cls)
 
             for file_path, feed in FEEDS.items():
+                if not os.path.exists(str(file_path)):
+                    continue
+
                 with open(str(file_path), 'rb') as f:
                     content[feed['format']] = f.read()
 
         finally:
             for file_path in FEEDS.keys():
+                if not os.path.exists(str(file_path)):
+                    continue
+
                 os.remove(str(file_path))
 
         return content
@@ -623,6 +651,25 @@ def test_export_no_items_store_empty(self):
             data = yield self.exported_no_data(settings)
             self.assertEqual(data[fmt], expctd)
 
+    @defer.inlineCallbacks
+    def test_export_no_items_multiple_feeds(self):
+        """ Make sure that `storage.store` is called for every feed. """
+        settings = {
+            'FEEDS': {
+                self._random_temp_filename(): {'format': 'json'},
+                self._random_temp_filename(): {'format': 'xml'},
+                self._random_temp_filename(): {'format': 'csv'},
+            },
+            'FEED_STORAGES': {'file': 'tests.test_feedexport.LogOnStoreFileStorage'},
+            'FEED_STORE_EMPTY': False
+        }
+
+        with LogCapture() as log:
+            yield self.exported_no_data(settings)
+
+        print(log)
+        self.assertEqual(str(log).count('Storage.store is called'), 3)
+
     @defer.inlineCallbacks
     def test_export_multiple_item_classes(self):