Repository: googleapis/google-cloud-python

Issue:
gcloud.storage.iterator.KeyDataIterator chunking is off-by-one
'get_headers' computes the "Range:" header as '_bytes_written' - '_bytes_written + CHUNK_SIZE':

https://github.com/GoogleCloudPlatform/gcloud-python/blob/1f2aaf3606802f4f02eeb30e22e1082ef7f4594e/gcloud/storage/iterator.py#L215

and the '_bytes_written' gets updated to bump by the size of the chunk the server returns.  Consider the initial read, assuming a CHUNK_SIZE of 100:

   Range: bytes=0-100

Note that, per RFC2616. this will return up to 101 bytes (both bytes-position values are inclusive):

http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35.1



Discussion:

Resolving patch:
diff --git a/gcloud/storage/iterator.py b/gcloud/storage/iterator.py
--- a/gcloud/storage/iterator.py
+++ b/gcloud/storage/iterator.py
@@ -211,7 +211,7 @@ def has_more_data(self):
 
   def get_headers(self):
     start = self._bytes_written
-    end = self._bytes_written + self.key.CHUNK_SIZE
+    end = self._bytes_written + self.key.CHUNK_SIZE - 1
 
     if self._total_bytes and end > self._total_bytes:
       end = ''


Test patch:
diff --git a/gcloud/storage/test_bucket.py b/gcloud/storage/test_bucket.py
--- a/gcloud/storage/test_bucket.py
+++ b/gcloud/storage/test_bucket.py
@@ -648,7 +648,7 @@ def test_save_default_object_acl_existing_set_new_passed(self):
         bucket = self._makeOne(connection, NAME, metadata)
         bucket.reload_default_object_acl()
         self.assertTrue(bucket.save_default_object_acl(new_acl) is bucket)
-        # See: https://github.com/GoogleCloudPlatform/gcloud-python/issues/138
+        # See: https://github.com/GoogleCloudPlatform/gcloud-python/issues/139
         #self.assertEqual(list(bucket.default_object_acl), new_acl)
         kw = connection._requested
         self.assertEqual(len(kw), 1)
@@ -666,14 +666,12 @@ def test_clear_default_object_acl(self):
         bucket = self._makeOne(connection, NAME, metadata)
         bucket.reload_default_object_acl()
         self.assertTrue(bucket.clear_default_object_acl() is bucket)
-        # See: https://github.com/GoogleCloudPlatform/gcloud-python/issues/138
+        # See: https://github.com/GoogleCloudPlatform/gcloud-python/issues/139
         #self.assertEqual(list(bucket.default_object_acl), [])
         kw = connection._requested
         self.assertEqual(len(kw), 1)
         self.assertEqual(kw[0]['method'], 'PATCH')
         self.assertEqual(kw[0]['path'], '/b/%s' % NAME)
-        # See: https://github.com/GoogleCloudPlatform/gcloud-python/issues/139
-        #self.assertEqual(list(bucket.default_object_acl), [])
         #self.assertEqual(kw[0]['data'], {'defaultObjectAcl': []})
         self.assertEqual(kw[0]['query_params'], {'projection': 'full'})
 
diff --git a/gcloud/storage/test_iterator.py b/gcloud/storage/test_iterator.py
--- a/gcloud/storage/test_iterator.py
+++ b/gcloud/storage/test_iterator.py
@@ -206,29 +206,29 @@ def test_ctor(self):
 
     def test__iter__(self):
         response1 = _Response(status=200)
-        response1['content-range'] = '0-10/15'
+        response1['content-range'] = '0-9/15'
         response2 = _Response(status=200)
-        response2['content-range'] = '11-14/15'
-        connection = _Connection((response1, '01234567890'),
-                                 (response2, '1234'),
+        response2['content-range'] = '10-14/15'
+        connection = _Connection((response1, '0123456789'),
+                                 (response2, '01234'),
                                 )
         key = _Key(connection)
         iterator = self._makeOne(key)
         chunks = list(iterator)
         self.assertEqual(len(chunks), 2)
-        self.assertEqual(chunks[0], '01234567890')
-        self.assertEqual(chunks[1], '1234')
+        self.assertEqual(chunks[0], '0123456789')
+        self.assertEqual(chunks[1], '01234')
         self.assertEqual(iterator._bytes_written, 15)
         self.assertEqual(iterator._total_bytes, 15)
         kws = connection._requested
         self.assertEqual(kws[0]['method'], 'GET')
-        self.assertEqual(kws[0]['url'], 
+        self.assertEqual(kws[0]['url'],
                          'http://example.com/b/name/o/key?alt=media')
-        self.assertEqual(kws[0]['headers'], {'Range': 'bytes=0-10'})
+        self.assertEqual(kws[0]['headers'], {'Range': 'bytes=0-9'})
         self.assertEqual(kws[1]['method'], 'GET')
-        self.assertEqual(kws[1]['url'], 
+        self.assertEqual(kws[1]['url'],
                          'http://example.com/b/name/o/key?alt=media')
-        self.assertEqual(kws[1]['headers'], {'Range': 'bytes=11-'})
+        self.assertEqual(kws[1]['headers'], {'Range': 'bytes=10-'})
 
     def test_reset(self):
         connection = _Connection()
@@ -275,7 +275,7 @@ def test_get_headers_new(self):
         iterator = self._makeOne(key)
         headers = iterator.get_headers()
         self.assertEqual(len(headers), 1)
-        self.assertEqual(headers['Range'], 'bytes=0-10')
+        self.assertEqual(headers['Range'], 'bytes=0-9')
 
     def test_get_headers_ok(self):
         connection = _Connection()
@@ -285,7 +285,7 @@ def test_get_headers_ok(self):
         iterator._total_bytes = 1000
         headers = iterator.get_headers()
         self.assertEqual(len(headers), 1)
-        self.assertEqual(headers['Range'], 'bytes=10-20')
+        self.assertEqual(headers['Range'], 'bytes=10-19')
 
     def test_get_headers_off_end(self):
         connection = _Connection()
@@ -313,7 +313,7 @@ def test_get_next_chunk_underflow(self):
 
     def test_get_next_chunk_200(self):
         response = _Response(status=200)
-        response['content-range'] = '0-10/100'
+        response['content-range'] = '0-9/100'
         connection = _Connection((response, 'CHUNK'))
         key = _Key(connection)
         iterator = self._makeOne(key)
@@ -323,9 +323,9 @@ def test_get_next_chunk_200(self):
         self.assertEqual(iterator._total_bytes, 100)
         kw, = connection._requested
         self.assertEqual(kw['method'], 'GET')
-        self.assertEqual(kw['url'], 
+        self.assertEqual(kw['url'],
                          'http://example.com/b/name/o/key?alt=media')
-        self.assertEqual(kw['headers'], {'Range': 'bytes=0-10'})
+        self.assertEqual(kw['headers'], {'Range': 'bytes=0-9'})
 
     def test_get_next_chunk_206(self):
         response = _Response(status=206)
@@ -338,9 +338,9 @@ def test_get_next_chunk_206(self):
         self.assertEqual(iterator._bytes_written, len(chunk))
         kw, = connection._requested
         self.assertEqual(kw['method'], 'GET')
-        self.assertEqual(kw['url'], 
+        self.assertEqual(kw['url'],
                          'http://example.com/b/name/o/key?alt=media')
-        self.assertEqual(kw['headers'], {'Range': 'bytes=0-10'})
+        self.assertEqual(kw['headers'], {'Range': 'bytes=0-9'})
 
     def test_get_next_chunk_416(self):
         response = _Response(status=416)