Repository: scrapy/scrapy

Issue:
Google Cloud Storage Support (Storage backends)
Does it make sense to support [Google Cloud Storage](https://cloud.google.com/products/cloud-storage/) as storage backend? Boto already supports Cloud storage: http://boto.readthedocs.org/en/latest/ref/gs.html



Discussion:
It would be nice to have, can you implement it?

boto is ok for aw services, but for more generic cloud (storage) interaction, I'd look into [pyrax](https://github.com/rackspace/pyrax) to build on, a [python-cloudfiles](https://github.com/rackerlabs/python-cloudfiles) replacement.

boto is an optional dependency, it is fine to add backends based on other optional libraries.

does pyrax support Google Cloud Storage? it looks like a openstack only thing.

I haven't tried, since I'm not using google's stuff, but I thought it should - it's [s3-compat from what I can see](https://developers.google.com/storage/docs/migrating), though I understand there are small differences between s3 and swift api implementations.

I was seeing pyrax as the most generic solution here since openstack's becoming the baseline tools api, but if I'm wrong, [libcloud](https://libcloud.readthedocs.org/en/latest/supported_providers.html#object-storage) looks good as well and seems to support [google c.s.](https://libcloud.readthedocs.org/en/latest/supported_providers.html#object-storage)

That way it wouldn't be a google-only backend here, but hopefully useable with many [object storage providers](http://www.dragondisk.com/s3-storage-providers.html), like rackspace cloud, [ovh hubic](https://hubic.com/) ([cloudfiles-version](https://github.com/Gu1/python-cloudfiles-hubic)), and of course all openstack swift users, maybe [riak](http://docs.basho.com/riakcs/latest/references/apis/storage/s3/) and more.

I would like to try to implement it but I don't think I have the skills to do that on my own.
I implemented this feature. Please check my [PR](https://github.com/scrapy/scrapy/pull/2923) 🙏 
I think we could reopen this issue. The GCS support was added only as a File Store and not as a Feed Storage. 

Resolving patch:
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -153,6 +153,32 @@ def _store_in_thread(self, file):
             key.close()
 
 
+class GCSFeedStorage(BlockingFeedStorage):
+
+    def __init__(self, uri, project_id, acl):
+        self.project_id = project_id
+        self.acl = acl
+        u = urlparse(uri)
+        self.bucket_name = u.hostname
+        self.blob_name = u.path[1:]  # remove first "/"
+
+    @classmethod
+    def from_crawler(cls, crawler, uri):
+        return cls(
+            uri,
+            crawler.settings['GCS_PROJECT_ID'],
+            crawler.settings['FEED_STORAGE_GCS_ACL'] or None
+        )
+
+    def _store_in_thread(self, file):
+        file.seek(0)
+        from google.cloud.storage import Client
+        client = Client(project=self.project_id)
+        bucket = client.get_bucket(self.bucket_name)
+        blob = bucket.blob(self.blob_name)
+        blob.upload_from_file(file, predefined_acl=self.acl)
+
+
 class FTPFeedStorage(BlockingFeedStorage):
 
     def __init__(self, uri, use_active_mode=False):
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -142,9 +142,10 @@
 FEED_STORAGES_BASE = {
     '': 'scrapy.extensions.feedexport.FileFeedStorage',
     'file': 'scrapy.extensions.feedexport.FileFeedStorage',
-    'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
-    's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
+    'gs': 'scrapy.extensions.feedexport.GCSFeedStorage',
+    's3': 'scrapy.extensions.feedexport.S3FeedStorage',
+    'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
 }
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
@@ -159,6 +160,7 @@
 FEED_EXPORT_INDENT = 0
 
 FEED_STORAGE_FTP_ACTIVE = False
+FEED_STORAGE_GCS_ACL = ''
 FEED_STORAGE_S3_ACL = ''
 
 FILES_STORE_S3_ACL = 'private'
@@ -168,6 +170,8 @@
 FTP_PASSWORD = 'guest'
 FTP_PASSIVE_MODE = True
 
+GCS_PROJECT_ID = None
+
 HTTPCACHE_ENABLED = False
 HTTPCACHE_DIR = 'httpcache'
 HTTPCACHE_IGNORE_MISSING = False


Test patch:
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -2,10 +2,10 @@
 This module contains some assorted functions used in tests
 """
 
-from __future__ import absolute_import
-from posixpath import split
 import asyncio
 import os
+from posixpath import split
+from unittest import mock
 
 from importlib import import_module
 from twisted.trial.unittest import SkipTest
@@ -126,3 +126,19 @@ def get_from_asyncio_queue(value):
     getter = q.get()
     q.put_nowait(value)
     return getter
+
+
+def mock_google_cloud_storage():
+    """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
+    classes and set their proper return values.
+    """
+    from google.cloud.storage import Client, Bucket, Blob
+    client_mock = mock.create_autospec(Client)
+
+    bucket_mock = mock.create_autospec(Bucket)
+    client_mock.get_bucket.return_value = bucket_mock
+
+    blob_mock = mock.create_autospec(Blob)
+    bucket_mock.blob.return_value = blob_mock
+
+    return (client_mock, bucket_mock, blob_mock)
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -25,11 +25,23 @@
 import scrapy
 from scrapy.crawler import CrawlerRunner
 from scrapy.exporters import CsvItemExporter
-from scrapy.extensions.feedexport import (BlockingFeedStorage, FileFeedStorage, FTPFeedStorage,
-                                          IFeedStorage, S3FeedStorage, StdoutFeedStorage)
+from scrapy.extensions.feedexport import (
+    BlockingFeedStorage,
+    FileFeedStorage,
+    FTPFeedStorage,
+    GCSFeedStorage,
+    IFeedStorage,
+    S3FeedStorage,
+    StdoutFeedStorage,
+)
 from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
-from scrapy.utils.test import assert_aws_environ, get_crawler, get_s3_content_and_delete
+from scrapy.utils.test import (
+    assert_aws_environ,
+    get_s3_content_and_delete,
+    get_crawler,
+    mock_google_cloud_storage,
+)
 
 from tests.mockserver import MockServer
 
@@ -364,6 +376,63 @@ def test_store_not_botocore_with_acl(self):
         )
 
 
+class GCSFeedStorageTest(unittest.TestCase):
+
+    def test_parse_settings(self):
+        try:
+            from google.cloud.storage import Client  # noqa
+        except ImportError:
+            raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
+
+        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead'}
+        crawler = get_crawler(settings_dict=settings)
+        storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
+        assert storage.project_id == '123'
+        assert storage.acl == 'publicRead'
+        assert storage.bucket_name == 'mybucket'
+        assert storage.blob_name == 'export.csv'
+
+    def test_parse_empty_acl(self):
+        try:
+            from google.cloud.storage import Client  # noqa
+        except ImportError:
+            raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
+
+        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': ''}
+        crawler = get_crawler(settings_dict=settings)
+        storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
+        assert storage.acl is None
+
+        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': None}
+        crawler = get_crawler(settings_dict=settings)
+        storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
+        assert storage.acl is None
+
+    @defer.inlineCallbacks
+    def test_store(self):
+        try:
+            from google.cloud.storage import Client  # noqa
+        except ImportError:
+            raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
+
+        uri = 'gs://mybucket/export.csv'
+        project_id = 'myproject-123'
+        acl = 'publicRead'
+        (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage()
+        with mock.patch('google.cloud.storage.Client') as m:
+            m.return_value = client_mock
+
+            f = mock.Mock()
+            storage = GCSFeedStorage(uri, project_id, acl)
+            yield storage.store(f)
+
+            f.seek.assert_called_once_with(0)
+            m.assert_called_once_with(project=project_id)
+            client_mock.get_bucket.assert_called_once_with('mybucket')
+            bucket_mock.blob.assert_called_once_with('export.csv')
+            blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)
+
+
 class StdoutFeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks