Repository: googleapis/google-cloud-python

Issue:
Support custom encryption keys
Can we support custom encryption keys for files uploaded?

Spec is here: https://cloud.google.com/compute/docs/disks/customer-supplied-encryption#top_of_page



Discussion:
I think this might also be relevant.
https://cloud.google.com/storage/docs/encryption

@dhermes @tseaver any thoughts on how we should pass the key and hash?

```
bucket = client.create_bucket('my-bucket', key=key)
```

Then I think we can just hash the key.

```
bucket = client.get_bucket('my-bucket', key=key)
```

This is the python they have to help you generate the key and hash.

``` python
python -c 'import base64; import hashlib; import os; key = os.urandom(32); \
    print "Key: %sSHA256 hash: %s" % (base64.encodestring(key), \
    base64.encodestring(hashlib.sha256(key).digest()))'
```

https://cloud.google.com/storage/docs/encryption#customer-supplied

@jgeewax, @stephenplusplus mentioned being able to just supply a password when storing data.
I think that would be very useful, but it's counter to the other documentation. I'm game for doing that and trying to be really clear in our docs on how to use it.

I'll continue in the direction of providing a key unless someone says otherwise.

edit: s/password/key/g, I'll start with the key since adding a password later will be an added bonus I think.


Resolving patch:
diff --git a/gcloud/_helpers.py b/gcloud/_helpers.py
--- a/gcloud/_helpers.py
+++ b/gcloud/_helpers.py
@@ -394,6 +394,27 @@ def _to_bytes(value, encoding='ascii'):
         raise TypeError('%r could not be converted to bytes' % (value,))
 
 
+def _bytes_to_unicode(value):
+    """Converts bytes to a unicode value, if necessary.
+
+    :type value: bytes
+    :param value: bytes value to attempt string conversion on.
+
+    :rtype: str
+    :returns: The original value converted to unicode (if bytes) or as passed
+              in if it started out as unicode.
+
+    :raises: :class:`ValueError` if the value could not be converted to
+             unicode.
+    """
+    result = (value.decode('utf-8')
+              if isinstance(value, six.binary_type) else value)
+    if isinstance(result, six.text_type):
+        return result
+    else:
+        raise ValueError('%r could not be converted to unicode' % (value,))
+
+
 def _pb_timestamp_to_datetime(timestamp):
     """Convert a Timestamp protobuf to a datetime object.
 
diff --git a/gcloud/storage/blob.py b/gcloud/storage/blob.py
--- a/gcloud/storage/blob.py
+++ b/gcloud/storage/blob.py
@@ -14,7 +14,9 @@
 
 """Create / interact with Google Cloud Storage blobs."""
 
+import base64
 import copy
+import hashlib
 from io import BytesIO
 import json
 import mimetypes
@@ -26,6 +28,8 @@
 from six.moves.urllib.parse import quote
 
 from gcloud._helpers import _rfc3339_to_datetime
+from gcloud._helpers import _to_bytes
+from gcloud._helpers import _bytes_to_unicode
 from gcloud.credentials import generate_signed_url
 from gcloud.exceptions import NotFound
 from gcloud.exceptions import make_exception
@@ -276,7 +280,7 @@ def delete(self, client=None):
         """
         return self.bucket.delete_blob(self.name, client=client)
 
-    def download_to_file(self, file_obj, client=None):
+    def download_to_file(self, file_obj, encryption_key=None, client=None):
         """Download the contents of this blob into a file-like object.
 
         .. note::
@@ -284,9 +288,33 @@ def download_to_file(self, file_obj, client=None):
            If the server-set property, :attr:`media_link`, is not yet
            initialized, makes an additional API request to load it.
 
+         Downloading a file that has been encrypted with a `customer-supplied`_
+         encryption key::
+
+            >>> from gcloud import storage
+            >>> from gcloud.storage import Blob
+
+            >>> client = storage.Client(project='my-project')
+            >>> bucket = client.get_bucket('my-bucket')
+            >>> encryption_key = 'aa426195405adee2c8081bb9e7e74b19'
+            >>> blob = Blob('secure-data', bucket)
+            >>> with open('/tmp/my-secure-file', 'wb') as file_obj:
+            >>>     blob.download_to_file(file_obj,
+            ...                           encryption_key=encryption_key)
+
+        The ``encryption_key`` should be a str or bytes with a length of at
+        least 32.
+
+        .. _customer-supplied: https://cloud.google.com/storage/docs/\
+                               encryption#customer-supplied
+
         :type file_obj: file
         :param file_obj: A file handle to which to write the blob's data.
 
+        :type encryption_key: str or bytes
+        :param encryption_key: Optional 32 byte encryption key for
+                               customer-supplied encryption.
+
         :type client: :class:`gcloud.storage.client.Client` or ``NoneType``
         :param client: Optional. The client to use.  If not passed, falls back
                        to the ``client`` stored on the blob's bucket.
@@ -305,7 +333,11 @@ def download_to_file(self, file_obj, client=None):
         if self.chunk_size is not None:
             download.chunksize = self.chunk_size
 
-        request = Request(download_url, 'GET')
+        headers = {}
+        if encryption_key:
+            _set_encryption_headers(encryption_key, headers)
+
+        request = Request(download_url, 'GET', headers)
 
         # Use the private ``_connection`` rather than the public
         # ``.connection``, since the public connection may be a batch. A
@@ -315,12 +347,16 @@ def download_to_file(self, file_obj, client=None):
         # it has all three (http, API_BASE_URL and build_api_url).
         download.initialize_download(request, client._connection.http)
 
-    def download_to_filename(self, filename, client=None):
+    def download_to_filename(self, filename, encryption_key=None, client=None):
         """Download the contents of this blob into a named file.
 
         :type filename: string
         :param filename: A filename to be passed to ``open``.
 
+        :type encryption_key: str or bytes
+        :param encryption_key: Optional 32 byte encryption key for
+                               customer-supplied encryption.
+
         :type client: :class:`gcloud.storage.client.Client` or ``NoneType``
         :param client: Optional. The client to use.  If not passed, falls back
                        to the ``client`` stored on the blob's bucket.
@@ -328,14 +364,19 @@ def download_to_filename(self, filename, client=None):
         :raises: :class:`gcloud.exceptions.NotFound`
         """
         with open(filename, 'wb') as file_obj:
-            self.download_to_file(file_obj, client=client)
+            self.download_to_file(file_obj, encryption_key=encryption_key,
+                                  client=client)
 
         mtime = time.mktime(self.updated.timetuple())
         os.utime(file_obj.name, (mtime, mtime))
 
-    def download_as_string(self, client=None):
+    def download_as_string(self, encryption_key=None, client=None):
         """Download the contents of this blob as a string.
 
+        :type encryption_key: str or bytes
+        :param encryption_key: Optional 32 byte encryption key for
+                               customer-supplied encryption.
+
         :type client: :class:`gcloud.storage.client.Client` or ``NoneType``
         :param client: Optional. The client to use.  If not passed, falls back
                        to the ``client`` stored on the blob's bucket.
@@ -345,7 +386,8 @@ def download_as_string(self, client=None):
         :raises: :class:`gcloud.exceptions.NotFound`
         """
         string_buffer = BytesIO()
-        self.download_to_file(string_buffer, client=client)
+        self.download_to_file(string_buffer, encryption_key=encryption_key,
+                              client=client)
         return string_buffer.getvalue()
 
     @staticmethod
@@ -358,8 +400,10 @@ def _check_response_error(request, http_response):
             raise make_exception(faux_response, http_response.content,
                                  error_info=request.url)
 
+    # pylint: disable=too-many-locals
     def upload_from_file(self, file_obj, rewind=False, size=None,
-                         content_type=None, num_retries=6, client=None):
+                         encryption_key=None, content_type=None, num_retries=6,
+                         client=None):
         """Upload the contents of this blob from a file-like object.
 
         The content type of the upload will either be
@@ -378,6 +422,25 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
            `lifecycle <https://cloud.google.com/storage/docs/lifecycle>`_
            API documents for details.
 
+        Uploading a file with a `customer-supplied`_ encryption key::
+
+            >>> from gcloud import storage
+            >>> from gcloud.storage import Blob
+
+            >>> client = storage.Client(project='my-project')
+            >>> bucket = client.get_bucket('my-bucket')
+            >>> encryption_key = 'aa426195405adee2c8081bb9e7e74b19'
+            >>> blob = Blob('secure-data', bucket)
+            >>> with open('my-file', 'rb') as my_file:
+            >>>     blob.upload_from_file(my_file,
+            ...                           encryption_key=encryption_key)
+
+        The ``encryption_key`` should be a str or bytes with a length of at
+        least 32.
+
+        .. _customer-supplied: https://cloud.google.com/storage/docs/\
+                               encryption#customer-supplied
+
         :type file_obj: file
         :param file_obj: A file handle open for reading.
 
@@ -391,6 +454,10 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
                      :func:`os.fstat`. (If the file handle is not from the
                      filesystem this won't be possible.)
 
+        :type encryption_key: str or bytes
+        :param encryption_key: Optional 32 byte encryption key for
+                               customer-supplied encryption.
+
         :type content_type: string or ``NoneType``
         :param content_type: Optional type of content being uploaded.
 
@@ -434,6 +501,9 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
             'User-Agent': connection.USER_AGENT,
         }
 
+        if encryption_key:
+            _set_encryption_headers(encryption_key, headers)
+
         upload = Upload(file_obj, content_type, total_bytes,
                         auto_transfer=False)
 
@@ -473,9 +543,10 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
                           six.string_types):  # pragma: NO COVER  Python3
             response_content = response_content.decode('utf-8')
         self._set_properties(json.loads(response_content))
+    # pylint: enable=too-many-locals
 
     def upload_from_filename(self, filename, content_type=None,
-                             client=None):
+                             encryption_key=None, client=None):
         """Upload this blob's contents from the content of a named file.
 
         The content type of the upload will either be
@@ -500,6 +571,10 @@ def upload_from_filename(self, filename, content_type=None,
         :type content_type: string or ``NoneType``
         :param content_type: Optional type of content being uploaded.
 
+        :type encryption_key: str or bytes
+        :param encryption_key: Optional 32 byte encryption key for
+                               customer-supplied encryption.
+
         :type client: :class:`gcloud.storage.client.Client` or ``NoneType``
         :param client: Optional. The client to use.  If not passed, falls back
                        to the ``client`` stored on the blob's bucket.
@@ -510,10 +585,10 @@ def upload_from_filename(self, filename, content_type=None,
 
         with open(filename, 'rb') as file_obj:
             self.upload_from_file(file_obj, content_type=content_type,
-                                  client=client)
+                                  encryption_key=encryption_key, client=client)
 
     def upload_from_string(self, data, content_type='text/plain',
-                           client=None):
+                           encryption_key=None, client=None):
         """Upload contents of this blob from the provided string.
 
         .. note::
@@ -535,6 +610,10 @@ def upload_from_string(self, data, content_type='text/plain',
         :param content_type: Optional type of content being uploaded. Defaults
                              to ``'text/plain'``.
 
+        :type encryption_key: str or bytes
+        :param encryption_key: Optional 32 byte encryption key for
+                               customer-supplied encryption.
+
         :type client: :class:`gcloud.storage.client.Client` or ``NoneType``
         :param client: Optional. The client to use.  If not passed, falls back
                        to the ``client`` stored on the blob's bucket.
@@ -545,7 +624,7 @@ def upload_from_string(self, data, content_type='text/plain',
         string_buffer.write(data)
         self.upload_from_file(file_obj=string_buffer, rewind=True,
                               size=len(data), content_type=content_type,
-                              client=client)
+                              encryption_key=encryption_key, client=client)
 
     def make_public(self, client=None):
         """Make this blob public giving all users read access.
@@ -838,3 +917,21 @@ def __init__(self, bucket_name, object_name):
         self.query_params = {'name': object_name}
         self._bucket_name = bucket_name
         self._relative_path = ''
+
+
+def _set_encryption_headers(key, headers):
+    """Builds customer encyrption key headers
+
+    :type key: str or bytes
+    :param key: 32 byte key to build request key and hash.
+
+    :type headers: dict
+    :param headers: dict of HTTP headers being sent in request.
+    """
+    key = _to_bytes(key)
+    sha256_key = hashlib.sha256(key).digest()
+    key_hash = base64.b64encode(sha256_key).rstrip()
+    encoded_key = base64.b64encode(key).rstrip()
+    headers['X-Goog-Encryption-Algorithm'] = 'AES256'
+    headers['X-Goog-Encryption-Key'] = _bytes_to_unicode(encoded_key)
+    headers['X-Goog-Encryption-Key-Sha256'] = _bytes_to_unicode(key_hash)


Test patch:
diff --git a/gcloud/storage/test_blob.py b/gcloud/storage/test_blob.py
--- a/gcloud/storage/test_blob.py
+++ b/gcloud/storage/test_blob.py
@@ -383,6 +383,51 @@ def test_download_to_filename(self):
         self.assertEqual(wrote, b'abcdef')
         self.assertEqual(mtime, updatedTime)
 
+    def test_download_to_filename_w_key(self):
+        import os
+        import time
+        from six.moves.http_client import OK
+        from six.moves.http_client import PARTIAL_CONTENT
+        from gcloud._testing import _NamedTemporaryFile
+
+        BLOB_NAME = 'blob-name'
+        KEY = 'aa426195405adee2c8081bb9e7e74b19'
+        HEADER_KEY_VALUE = 'YWE0MjYxOTU0MDVhZGVlMmM4MDgxYmI5ZTdlNzRiMTk='
+        HEADER_KEY_HASH_VALUE = 'V3Kwe46nKc3xLv96+iJ707YfZfFvlObta8TQcx2gpm0='
+        chunk1_response = {'status': PARTIAL_CONTENT,
+                           'content-range': 'bytes 0-2/6'}
+        chunk2_response = {'status': OK,
+                           'content-range': 'bytes 3-5/6'}
+        connection = _Connection(
+            (chunk1_response, b'abc'),
+            (chunk2_response, b'def'),
+        )
+        client = _Client(connection)
+        bucket = _Bucket(client)
+        MEDIA_LINK = 'http://example.com/media/'
+        properties = {'mediaLink': MEDIA_LINK,
+                      'updated': '2014-12-06T13:13:50.690Z'}
+        blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties)
+        blob._CHUNK_SIZE_MULTIPLE = 1
+        blob.chunk_size = 3
+
+        with _NamedTemporaryFile() as temp:
+            blob.download_to_filename(temp.name, encryption_key=KEY)
+            with open(temp.name, 'rb') as file_obj:
+                wrote = file_obj.read()
+                mtime = os.path.getmtime(temp.name)
+                updatedTime = time.mktime(blob.updated.timetuple())
+
+        rq = connection.http._requested
+        headers = dict(
+            [(x.title(), str(y)) for x, y in rq[0]['headers'].items()])
+        self.assertEqual(headers['X-Goog-Encryption-Algorithm'], 'AES256')
+        self.assertEqual(headers['X-Goog-Encryption-Key'], HEADER_KEY_VALUE)
+        self.assertEqual(headers['X-Goog-Encryption-Key-Sha256'],
+                         HEADER_KEY_HASH_VALUE)
+        self.assertEqual(wrote, b'abcdef')
+        self.assertEqual(mtime, updatedTime)
+
     def test_download_as_string(self):
         from six.moves.http_client import OK
         from six.moves.http_client import PARTIAL_CONTENT
@@ -685,6 +730,63 @@ def test_upload_from_file_w_slash_in_name(self):
         self.assertEqual(headers['Content-Length'], '6')
         self.assertEqual(headers['Content-Type'], 'application/octet-stream')
 
+    def test_upload_from_filename_w_key(self):
+        from six.moves.http_client import OK
+        from six.moves.urllib.parse import parse_qsl
+        from six.moves.urllib.parse import urlsplit
+        from gcloud._testing import _NamedTemporaryFile
+        from gcloud.streaming import http_wrapper
+
+        BLOB_NAME = 'blob-name'
+        UPLOAD_URL = 'http://example.com/upload/name/key'
+        DATA = b'ABCDEF'
+        KEY = 'aa426195405adee2c8081bb9e7e74b19'
+        HEADER_KEY_VALUE = 'YWE0MjYxOTU0MDVhZGVlMmM4MDgxYmI5ZTdlNzRiMTk='
+        HEADER_KEY_HASH_VALUE = 'V3Kwe46nKc3xLv96+iJ707YfZfFvlObta8TQcx2gpm0='
+        EXPECTED_CONTENT_TYPE = 'foo/bar'
+        properties = {'contentType': EXPECTED_CONTENT_TYPE}
+        loc_response = {'status': OK, 'location': UPLOAD_URL}
+        chunk1_response = {'status': http_wrapper.RESUME_INCOMPLETE,
+                           'range': 'bytes 0-4'}
+        chunk2_response = {'status': OK}
+        connection = _Connection(
+            (loc_response, '{}'),
+            (chunk1_response, ''),
+            (chunk2_response, ''),
+        )
+        client = _Client(connection)
+        bucket = _Bucket(client)
+        blob = self._makeOne(BLOB_NAME, bucket=bucket,
+                             properties=properties)
+        blob._CHUNK_SIZE_MULTIPLE = 1
+        blob.chunk_size = 5
+
+        with _NamedTemporaryFile(suffix='.jpeg') as temp:
+            with open(temp.name, 'wb') as file_obj:
+                file_obj.write(DATA)
+            blob.upload_from_filename(temp.name,
+                                      content_type=EXPECTED_CONTENT_TYPE,
+                                      encryption_key=KEY)
+
+        rq = connection.http._requested
+        self.assertEqual(len(rq), 1)
+        self.assertEqual(rq[0]['method'], 'POST')
+        uri = rq[0]['uri']
+        scheme, netloc, path, qs, _ = urlsplit(uri)
+        self.assertEqual(scheme, 'http')
+        self.assertEqual(netloc, 'example.com')
+        self.assertEqual(path, '/b/name/o')
+        self.assertEqual(dict(parse_qsl(qs)),
+                         {'uploadType': 'media', 'name': BLOB_NAME})
+        headers = dict(
+            [(x.title(), str(y)) for x, y in rq[0]['headers'].items()])
+        self.assertEqual(headers['X-Goog-Encryption-Algorithm'], 'AES256')
+        self.assertEqual(headers['X-Goog-Encryption-Key'], HEADER_KEY_VALUE)
+        self.assertEqual(headers['X-Goog-Encryption-Key-Sha256'],
+                         HEADER_KEY_HASH_VALUE)
+        self.assertEqual(headers['Content-Length'], '6')
+        self.assertEqual(headers['Content-Type'], 'foo/bar')
+
     def _upload_from_filename_test_helper(self, properties=None,
                                           content_type_arg=None,
                                           expected_content_type=None):
@@ -837,6 +939,54 @@ def test_upload_from_string_w_text(self):
         self.assertEqual(headers['Content-Type'], 'text/plain')
         self.assertEqual(rq[0]['body'], ENCODED)
 
+    def test_upload_from_string_text_w_key(self):
+        from six.moves.http_client import OK
+        from six.moves.urllib.parse import parse_qsl
+        from six.moves.urllib.parse import urlsplit
+        from gcloud.streaming import http_wrapper
+        BLOB_NAME = 'blob-name'
+        KEY = 'aa426195405adee2c8081bb9e7e74b19'
+        HEADER_KEY_VALUE = 'YWE0MjYxOTU0MDVhZGVlMmM4MDgxYmI5ZTdlNzRiMTk='
+        HEADER_KEY_HASH_VALUE = 'V3Kwe46nKc3xLv96+iJ707YfZfFvlObta8TQcx2gpm0='
+        UPLOAD_URL = 'http://example.com/upload/name/key'
+        DATA = u'ABCDEF\u1234'
+        ENCODED = DATA.encode('utf-8')
+        loc_response = {'status': OK, 'location': UPLOAD_URL}
+        chunk1_response = {'status': http_wrapper.RESUME_INCOMPLETE,
+                           'range': 'bytes 0-4'}
+        chunk2_response = {'status': OK}
+        connection = _Connection(
+            (loc_response, '{}'),
+            (chunk1_response, ''),
+            (chunk2_response, ''),
+        )
+        client = _Client(connection)
+        bucket = _Bucket(client=client)
+        blob = self._makeOne(BLOB_NAME, bucket=bucket)
+        blob._CHUNK_SIZE_MULTIPLE = 1
+        blob.chunk_size = 5
+        blob.upload_from_string(DATA, encryption_key=KEY)
+        rq = connection.http._requested
+        self.assertEqual(len(rq), 1)
+        self.assertEqual(rq[0]['method'], 'POST')
+        uri = rq[0]['uri']
+        scheme, netloc, path, qs, _ = urlsplit(uri)
+        self.assertEqual(scheme, 'http')
+        self.assertEqual(netloc, 'example.com')
+        self.assertEqual(path, '/b/name/o')
+        self.assertEqual(dict(parse_qsl(qs)),
+                         {'uploadType': 'media', 'name': BLOB_NAME})
+        headers = dict(
+            [(x.title(), str(y)) for x, y in rq[0]['headers'].items()])
+
+        self.assertEqual(headers['X-Goog-Encryption-Algorithm'], 'AES256')
+        self.assertEqual(headers['X-Goog-Encryption-Key'], HEADER_KEY_VALUE)
+        self.assertEqual(headers['X-Goog-Encryption-Key-Sha256'],
+                         HEADER_KEY_HASH_VALUE)
+        self.assertEqual(headers['Content-Length'], str(len(ENCODED)))
+        self.assertEqual(headers['Content-Type'], 'text/plain')
+        self.assertEqual(rq[0]['body'], ENCODED)
+
     def test_make_public(self):
         from six.moves.http_client import OK
         from gcloud.storage.acl import _ACLEntity
diff --git a/gcloud/test__helpers.py b/gcloud/test__helpers.py
--- a/gcloud/test__helpers.py
+++ b/gcloud/test__helpers.py
@@ -586,6 +586,27 @@ def test_with_nonstring_type(self):
         self.assertRaises(TypeError, self._callFUT, value)
 
 
+class Test__bytes_to_unicode(unittest2.TestCase):
+
+    def _callFUT(self, *args, **kwargs):
+        from gcloud._helpers import _bytes_to_unicode
+        return _bytes_to_unicode(*args, **kwargs)
+
+    def test_with_bytes(self):
+        value = b'bytes-val'
+        encoded_value = 'bytes-val'
+        self.assertEqual(self._callFUT(value), encoded_value)
+
+    def test_with_unicode(self):
+        value = u'string-val'
+        encoded_value = 'string-val'
+        self.assertEqual(self._callFUT(value), encoded_value)
+
+    def test_with_nonstring_type(self):
+        value = object()
+        self.assertRaises(ValueError, self._callFUT, value)
+
+
 class Test__pb_timestamp_to_datetime(unittest2.TestCase):
 
     def _callFUT(self, timestamp):