Repository: scrapy/scrapy

Issue:
Google Cloud Storage Support (Storage backends)
Does it make sense to support [Google Cloud Storage](https://cloud.google.com/products/cloud-storage/) as storage backend? Boto already supports Cloud storage: http://boto.readthedocs.org/en/latest/ref/gs.html



Discussion:
It would be nice to have, can you implement it?

boto is ok for aw services, but for more generic cloud (storage) interaction, I'd look into [pyrax](https://github.com/rackspace/pyrax) to build on, a [python-cloudfiles](https://github.com/rackerlabs/python-cloudfiles) replacement.

boto is an optional dependency, it is fine to add backends based on other optional libraries.

does pyrax support Google Cloud Storage? it looks like a openstack only thing.

I haven't tried, since I'm not using google's stuff, but I thought it should - it's [s3-compat from what I can see](https://developers.google.com/storage/docs/migrating), though I understand there are small differences between s3 and swift api implementations.

I was seeing pyrax as the most generic solution here since openstack's becoming the baseline tools api, but if I'm wrong, [libcloud](https://libcloud.readthedocs.org/en/latest/supported_providers.html#object-storage) looks good as well and seems to support [google c.s.](https://libcloud.readthedocs.org/en/latest/supported_providers.html#object-storage)

That way it wouldn't be a google-only backend here, but hopefully useable with many [object storage providers](http://www.dragondisk.com/s3-storage-providers.html), like rackspace cloud, [ovh hubic](https://hubic.com/) ([cloudfiles-version](https://github.com/Gu1/python-cloudfiles-hubic)), and of course all openstack swift users, maybe [riak](http://docs.basho.com/riakcs/latest/references/apis/storage/s3/) and more.

I would like to try to implement it but I don't think I have the skills to do that on my own.

Resolving patch:
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -194,6 +194,47 @@ def _headers_to_botocore_kwargs(self, headers):
         return extra
 
 
+class GCSFilesStore(object):
+
+    GCS_PROJECT_ID = None
+
+    CACHE_CONTROL = 'max-age=172800'
+
+    def __init__(self, uri):
+        from google.cloud import storage
+        client = storage.Client(project=self.GCS_PROJECT_ID)
+        bucket, prefix = uri[5:].split('/', 1)
+        self.bucket = client.bucket(bucket)
+        self.prefix = prefix
+
+    def stat_file(self, path, info):
+        def _onsuccess(blob):
+            if blob:
+                checksum = blob.md5_hash
+                last_modified = time.mktime(blob.updated.timetuple())
+                return {'checksum': checksum, 'last_modified': last_modified}
+            else:
+                return {}
+
+        return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess)
+
+    def _get_content_type(self, headers):
+        if headers and 'Content-Type' in headers:
+            return headers['Content-Type']
+        else:
+            return 'application/octet-stream'
+
+    def persist_file(self, path, buf, info, meta=None, headers=None):
+        blob = self.bucket.blob(self.prefix + path)
+        blob.cache_control = self.CACHE_CONTROL
+        blob.metadata = {k: str(v) for k, v in six.iteritems(meta or {})}
+        return threads.deferToThread(
+            blob.upload_from_string,
+            data=buf.getvalue(),
+            content_type=self._get_content_type(headers)
+        )
+
+
 class FilesPipeline(MediaPipeline):
     """Abstract pipeline that implement the file downloading
 
@@ -219,6 +260,7 @@ class FilesPipeline(MediaPipeline):
         '': FSFilesStore,
         'file': FSFilesStore,
         's3': S3FilesStore,
+        'gs': GCSFilesStore,
     }
     DEFAULT_FILES_URLS_FIELD = 'file_urls'
     DEFAULT_FILES_RESULT_FIELD = 'files'
@@ -258,6 +300,9 @@ def from_settings(cls, settings):
         s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
         s3store.POLICY = settings['FILES_STORE_S3_ACL']
 
+        gcs_store = cls.STORE_SCHEMES['gs']
+        gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']
+
         store_uri = settings['FILES_STORE']
         return cls(store_uri, settings=settings)
 
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -91,6 +91,9 @@ def from_settings(cls, settings):
         s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
         s3store.POLICY = settings['IMAGES_STORE_S3_ACL']
 
+        gcs_store = cls.STORE_SCHEMES['gs']
+        gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']
+
         store_uri = settings['IMAGES_STORE']
         return cls(store_uri, settings=settings)
 


Test patch:
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -20,6 +20,12 @@ def assert_aws_environ():
     if 'AWS_ACCESS_KEY_ID' not in os.environ:
         raise SkipTest("AWS keys not found")
 
+
+def assert_gcs_environ():
+    if 'GCS_PROJECT_ID' not in os.environ:
+        raise SkipTest("GCS_PROJECT_ID not found")
+
+
 def skip_if_no_boto():
     try:
         is_botocore()
@@ -45,6 +51,15 @@ def get_s3_content_and_delete(bucket, path, with_key=False):
         bucket.delete_key(path)
     return (content, key) if with_key else content
 
+def get_gcs_content_and_delete(bucket, path):
+    from google.cloud import storage
+    client = storage.Client(project=os.environ.get('GCS_PROJECT_ID'))
+    bucket = client.get_bucket(bucket)
+    blob = bucket.get_blob(path)
+    content = blob.download_as_string()
+    bucket.delete_blob(path)
+    return content, blob
+
 def get_crawler(spidercls=None, settings_dict=None):
     """Return an unconfigured Crawler object. If settings_dict is given, it
     will be used to populate the crawler settings with a project level
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -11,12 +11,13 @@
 from twisted.trial import unittest
 from twisted.internet import defer
 
-from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore
+from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GCSFilesStore
 from scrapy.item import Item, Field
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
 from scrapy.utils.python import to_bytes
 from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
+from scrapy.utils.test import assert_gcs_environ, get_gcs_content_and_delete
 from scrapy.utils.boto import is_botocore
 
 from tests import mock
@@ -375,6 +376,31 @@ def test_persist(self):
             self.assertEqual(key.content_type, 'image/png')
 
 
+class TestGCSFilesStore(unittest.TestCase):
+    @defer.inlineCallbacks
+    def test_persist(self):
+        assert_gcs_environ()
+        uri = os.environ.get('GCS_TEST_FILE_URI')
+        if not uri:
+            raise unittest.SkipTest("No GCS URI available for testing")
+        data = b"TestGCSFilesStore: \xe2\x98\x83"
+        buf = BytesIO(data)
+        meta = {'foo': 'bar'}
+        path = 'full/filename'
+        store = GCSFilesStore(uri)
+        yield store.persist_file(path, buf, info=None, meta=meta, headers=None)
+        s = yield store.stat_file(path, info=None)
+        self.assertIn('last_modified', s)
+        self.assertIn('checksum', s)
+        self.assertEqual(s['checksum'], 'zc2oVgXkbQr2EQdSdw3OPA==')
+        u = urlparse(uri)
+        content, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path)
+        self.assertEqual(content, data)
+        self.assertEqual(blob.metadata, {'foo': 'bar'})
+        self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL)
+        self.assertEqual(blob.content_type, 'application/octet-stream')
+
+
 class ItemWithFiles(Item):
     file_urls = Field()
     files = Field()