Repository: googleapis/google-cloud-python

Issue:
resumable uploads should try to send the file in one chunk by default
Sending data in one chunk is the recommended approach, documented in a not very clear way [here](https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload#resumable) in the “Uploading the file in chunks” section.

We should allow the user to override the chunk size, making sure that the library ends up using a multiple of 256 KB, as expected from the API (except for the final chunk that completes the upload where the size can vary according to the total size of the file). [gsutil code](https://github.com/GoogleCloudPlatform/gsutil/blob/ccca18e64f3f7d87c989675f8673af7bdecae0d1/gslib/util.py#L464) dealing with this.



Discussion:

Resolving patch:
diff --git a/gcloud/storage/blob.py b/gcloud/storage/blob.py
--- a/gcloud/storage/blob.py
+++ b/gcloud/storage/blob.py
@@ -51,17 +51,21 @@ class Blob(_PropertyMixin):
     :param bucket: The bucket to which this blob belongs. Required, unless the
                    implicit default bucket has been set.
 
+    :type chunk_size: integer
+    :param chunk_size: The size of a chunk of data whenever iterating (1 MB).
+                       This must be a multiple of 256 KB per the API
+                       specification.
+
     :type properties: dict
     :param properties: All the other data provided by Cloud Storage.
     """
 
-    CHUNK_SIZE = 1024 * 1024  # 1 MB.
-    """The size of a chunk of data whenever iterating (1 MB).
+    _chunk_size = None  # Default value for each instance.
 
-    This must be a multiple of 256 KB per the API specification.
-    """
+    _CHUNK_SIZE_MULTIPLE = 256 * 1024
+    """Number (256 KB, in bytes) that must divide the chunk size."""
 
-    def __init__(self, name, bucket=None):
+    def __init__(self, name, bucket=None, chunk_size=None):
         if bucket is None:
             bucket = _implicit_environ.get_default_bucket()
 
@@ -70,9 +74,34 @@ def __init__(self, name, bucket=None):
 
         super(Blob, self).__init__(name=name)
 
+        self.chunk_size = chunk_size  # Check that setter accepts value.
         self.bucket = bucket
         self._acl = ObjectACL(self)
 
+    @property
+    def chunk_size(self):
+        """Get the blob's default chunk size.
+
+        :rtype: integer or ``NoneType``
+        :returns: The current blob's chunk size, if it is set.
+        """
+        return self._chunk_size
+
+    @chunk_size.setter
+    def chunk_size(self, value):
+        """Set the blob's default chunk size.
+
+        :type value: integer or ``NoneType``
+        :param value: The current blob's chunk size, if it is set.
+
+        :raises: :class:`ValueError` if ``value`` is not ``None`` and is not a
+                 multiple of 256 KB.
+        """
+        if value is not None and value % self._CHUNK_SIZE_MULTIPLE != 0:
+            raise ValueError('Chunk size must be a multiple of %d.' % (
+                self._CHUNK_SIZE_MULTIPLE,))
+        self._chunk_size = value
+
     @staticmethod
     def path_helper(bucket_path, blob_name):
         """Relative URL path for a blob.
@@ -226,8 +255,10 @@ def download_to_file(self, file_obj):
 
         # Use apitools 'Download' facility.
         download = transfer.Download.FromStream(file_obj, auto_transfer=False)
-        download.chunksize = self.CHUNK_SIZE
-        headers = {'Range': 'bytes=0-%d' % (self.CHUNK_SIZE - 1)}
+        headers = {}
+        if self.chunk_size is not None:
+            download.chunksize = self.chunk_size
+            headers['Range'] = 'bytes=0-%d' % (self.chunk_size - 1,)
         request = http_wrapper.Request(download_url, 'GET', headers)
 
         download.InitializeDownload(request, self.connection.http)
@@ -319,7 +350,7 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
 
         upload = transfer.Upload(file_obj, content_type, total_bytes,
                                  auto_transfer=False,
-                                 chunksize=self.CHUNK_SIZE)
+                                 chunksize=self.chunk_size)
 
         url_builder = _UrlBuilder(bucket_name=self.bucket.name,
                                   object_name=self.name)


Test patch:
diff --git a/gcloud/storage/test_blob.py b/gcloud/storage/test_blob.py
--- a/gcloud/storage/test_blob.py
+++ b/gcloud/storage/test_blob.py
@@ -69,6 +69,41 @@ def test_ctor_explicit(self):
         self.assertFalse(blob._acl.loaded)
         self.assertTrue(blob._acl.blob is blob)
 
+    def test_chunk_size_ctor(self):
+        from gcloud.storage.blob import Blob
+        BLOB_NAME = 'blob-name'
+        BUCKET = object()
+        chunk_size = 10 * Blob._CHUNK_SIZE_MULTIPLE
+        blob = self._makeOne(BLOB_NAME, bucket=BUCKET, chunk_size=chunk_size)
+        self.assertEqual(blob._chunk_size, chunk_size)
+
+    def test_chunk_size_getter(self):
+        BLOB_NAME = 'blob-name'
+        BUCKET = object()
+        blob = self._makeOne(BLOB_NAME, bucket=BUCKET)
+        self.assertEqual(blob.chunk_size, None)
+        VALUE = object()
+        blob._chunk_size = VALUE
+        self.assertTrue(blob.chunk_size is VALUE)
+
+    def test_chunk_size_setter(self):
+        BLOB_NAME = 'blob-name'
+        BUCKET = object()
+        blob = self._makeOne(BLOB_NAME, bucket=BUCKET)
+        self.assertEqual(blob._chunk_size, None)
+        blob._CHUNK_SIZE_MULTIPLE = 10
+        blob.chunk_size = 20
+        self.assertEqual(blob._chunk_size, 20)
+
+    def test_chunk_size_setter_bad_value(self):
+        BLOB_NAME = 'blob-name'
+        BUCKET = object()
+        blob = self._makeOne(BLOB_NAME, bucket=BUCKET)
+        self.assertEqual(blob._chunk_size, None)
+        blob._CHUNK_SIZE_MULTIPLE = 10
+        with self.assertRaises(ValueError):
+            blob.chunk_size = 11
+
     def test_acl_property(self):
         from gcloud.storage.acl import ObjectACL
         FAKE_BUCKET = _Bucket(None)
@@ -242,7 +277,7 @@ def test_delete(self):
         blob.delete()
         self.assertFalse(blob.exists())
 
-    def test_download_to_file(self):
+    def _download_to_file_helper(self, chunk_size=None):
         from six.moves.http_client import OK
         from six.moves.http_client import PARTIAL_CONTENT
         from io import BytesIO
@@ -259,11 +294,19 @@ def test_download_to_file(self):
         MEDIA_LINK = 'http://example.com/media/'
         properties = {'mediaLink': MEDIA_LINK}
         blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties)
-        blob.CHUNK_SIZE = 3
+        if chunk_size is not None:
+            blob._CHUNK_SIZE_MULTIPLE = 1
+            blob.chunk_size = chunk_size
         fh = BytesIO()
         blob.download_to_file(fh)
         self.assertEqual(fh.getvalue(), b'abcdef')
 
+    def test_download_to_file_default(self):
+        self._download_to_file_helper()
+
+    def test_download_to_file_with_chunk_size(self):
+        self._download_to_file_helper(chunk_size=3)
+
     def test_download_to_filename(self):
         import os
         import time
@@ -284,7 +327,8 @@ def test_download_to_filename(self):
         properties = {'mediaLink': MEDIA_LINK,
                       'updated': '2014-12-06T13:13:50.690Z'}
         blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties)
-        blob.CHUNK_SIZE = 3
+        blob._CHUNK_SIZE_MULTIPLE = 1
+        blob.chunk_size = 3
         with NamedTemporaryFile() as f:
             blob.download_to_filename(f.name)
             f.flush()
@@ -311,7 +355,8 @@ def test_download_as_string(self):
         MEDIA_LINK = 'http://example.com/media/'
         properties = {'mediaLink': MEDIA_LINK}
         blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties)
-        blob.CHUNK_SIZE = 3
+        blob._CHUNK_SIZE_MULTIPLE = 1
+        blob.chunk_size = 3
         fetched = blob.download_as_string()
         self.assertEqual(fetched, b'abcdef')
 
@@ -330,7 +375,8 @@ def _upload_from_file_simple_test_helper(self, properties=None,
         )
         bucket = _Bucket(connection)
         blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties)
-        blob.CHUNK_SIZE = 5
+        blob._CHUNK_SIZE_MULTIPLE = 1
+        blob.chunk_size = 5
         with NamedTemporaryFile() as fh:
             fh.write(DATA)
             fh.flush()
@@ -398,7 +444,8 @@ def test_upload_from_file_resumable(self):
         )
         bucket = _Bucket(connection)
         blob = self._makeOne(BLOB_NAME, bucket=bucket)
-        blob.CHUNK_SIZE = 5
+        blob._CHUNK_SIZE_MULTIPLE = 1
+        blob.chunk_size = 5
         # Set the threshhold low enough that we force a resumable uploada.
         with _Monkey(transfer, _RESUMABLE_UPLOAD_THRESHOLD=5):
             with NamedTemporaryFile() as fh:
@@ -455,7 +502,8 @@ def test_upload_from_file_w_slash_in_name(self):
         )
         bucket = _Bucket(connection)
         blob = self._makeOne(BLOB_NAME, bucket=bucket)
-        blob.CHUNK_SIZE = 5
+        blob._CHUNK_SIZE_MULTIPLE = 1
+        blob.chunk_size = 5
         with NamedTemporaryFile() as fh:
             fh.write(DATA)
             fh.flush()
@@ -502,7 +550,8 @@ def _upload_from_filename_test_helper(self, properties=None,
         bucket = _Bucket(connection)
         blob = self._makeOne(BLOB_NAME, bucket=bucket,
                              properties=properties)
-        blob.CHUNK_SIZE = 5
+        blob._CHUNK_SIZE_MULTIPLE = 1
+        blob.chunk_size = 5
         with NamedTemporaryFile(suffix='.jpeg') as fh:
             fh.write(DATA)
             fh.flush()
@@ -565,7 +614,8 @@ def test_upload_from_string_w_bytes(self):
         )
         bucket = _Bucket(connection)
         blob = self._makeOne(BLOB_NAME, bucket=bucket)
-        blob.CHUNK_SIZE = 5
+        blob._CHUNK_SIZE_MULTIPLE = 1
+        blob.chunk_size = 5
         blob.upload_from_string(DATA)
         rq = connection.http._requested
         self.assertEqual(len(rq), 1)
@@ -603,7 +653,8 @@ def test_upload_from_string_w_text(self):
         )
         bucket = _Bucket(connection)
         blob = self._makeOne(BLOB_NAME, bucket=bucket)
-        blob.CHUNK_SIZE = 5
+        blob._CHUNK_SIZE_MULTIPLE = 1
+        blob.chunk_size = 5
         blob.upload_from_string(DATA)
         rq = connection.http._requested
         self.assertEqual(len(rq), 1)