Repository: googleapis/google-resumable-media-python

Issue:
Blob.upload_from_file fails if 8MB < size-parameter < file-size 
#### Environment details

  - OS type and version: Linux Fedora 32 x86_64 
  - Python version: Python 3.8.2
  - pip version: pip 20.1
  - `google-cloud-storage` version: 1.28.1

#### Steps to reproduce

  1. create a file larger than 8MB, e.g. 16MB
  2. upload to GCS with Blob.upload_from_file() with size parameter set to 12 MB.

#### Code example

```python
with open("16MB-file.txt", "rb") as d:
    blob = Blob('upload-bug.txt', storage_client.get_bucket('some-bucket'))
    blob.upload_from_file(d, size=12_000_000)
```

#### Stack trace
```
  File "test-gcs-upload-bug.py", line 16, in <module>
    blob.upload_from_file(d, size=12_000_000)
  File "venv/lib/python3.8/site-packages/google/cloud/storage/blob.py", line 1557, in upload_from_file
    created_json = self._do_upload(
  File "venv/lib/python3.8/site-packages/google/cloud/storage/blob.py", line 1423, in _do_upload
    response = self._do_resumable_upload(
  File "venv/lib/python3.8/site-packages/google/cloud/storage/blob.py", line 1331, in _do_resumable_upload
    response = upload.transmit_next_chunk(transport)
  File "venv/lib/python3.8/site-packages/google/resumable_media/requests/upload.py", line 418, in transmit_next_chunk
    method, url, payload, headers = self._prepare_request()
  File "venv/lib/python3.8/site-packages/google/resumable_media/_upload.py", line 550, in _prepare_request
    start_byte, payload, content_range = get_next_chunk(
  File "venv/lib/python3.8/site-packages/google/resumable_media/_upload.py", line 866, in get_next_chunk
    raise ValueError(msg)
ValueError: 16000000 bytes have been read from the stream, which exceeds the expected total 12000000.
```


Discussion:

Patch:
diff --git a/google/resumable_media/_upload.py b/google/resumable_media/_upload.py
index bd4b197..dd5aaed 100644
--- a/google/resumable_media/_upload.py
+++ b/google/resumable_media/_upload.py
@@ -835,11 +835,12 @@ def get_next_chunk(stream, chunk_size, total_bytes):
         ValueError: If there is no data left to consume. This corresponds
             exactly to the case ``end_byte < start_byte``, which can only
             occur if ``end_byte == start_byte - 1``.
-        ValueError: If the stream has been read past ``total_bytes`` (this
-            is in the case that ``total_bytes`` is not :data:`None`).
     """
     start_byte = stream.tell()
-    payload = stream.read(chunk_size)
+    if total_bytes is not None and start_byte + chunk_size >= total_bytes > 0:
+        payload = stream.read(total_bytes - start_byte)
+    else:
+        payload = stream.read(chunk_size)
     end_byte = stream.tell() - 1
 
     num_bytes_read = len(payload)
@@ -861,10 +862,6 @@ def get_next_chunk(stream, chunk_size, total_bytes):
                 u"Stream is already exhausted. There is no content remaining."
             )
 
-        if end_byte >= total_bytes:
-            msg = _STREAM_READ_PAST_TEMPLATE.format(end_byte + 1, total_bytes)
-            raise ValueError(msg)
-
     content_range = get_content_range(start_byte, end_byte, total_bytes)
     return start_byte, payload, content_range
 


Test patch:
diff --git a/tests/unit/test__upload.py b/tests/unit/test__upload.py
index 6f87401..278eab8 100644
--- a/tests/unit/test__upload.py
+++ b/tests/unit/test__upload.py
@@ -859,20 +859,22 @@ class Test_get_next_chunk(object):
         stream = io.BytesIO(b"not empty WAT!")
         with pytest.raises(ValueError) as exc_info:
             _upload.get_next_chunk(stream, 1, 0)
-
         exc_info.match(u"Stream specified as empty, but produced non-empty content.")
 
-    def test_read_past_known_size(self):
-        data = b"more content than we expected"
+    def test_success_known_size_lt_stream_size(self):
+        data = b"0123456789"
         stream = io.BytesIO(data)
-        chunk_size = len(data)
-        total_bytes = chunk_size - 3
+        chunk_size = 3
+        total_bytes = len(data) - 2
 
-        with pytest.raises(ValueError) as exc_info:
-            _upload.get_next_chunk(stream, chunk_size, total_bytes)
+        # Splits into 3 chunks: 012, 345, 67
+        result0 = _upload.get_next_chunk(stream, chunk_size, total_bytes)
+        result1 = _upload.get_next_chunk(stream, chunk_size, total_bytes)
+        result2 = _upload.get_next_chunk(stream, chunk_size, total_bytes)
 
-        exc_info.match(u"bytes have been read from the stream")
-        exc_info.match(u"exceeds the expected total")
+        assert result0 == (0, b"012", u"bytes 0-2/8")
+        assert result1 == (3, b"345", u"bytes 3-5/8")
+        assert result2 == (6, b"67", u"bytes 6-7/8")
 
     def test_success_known_size(self):
         data = b"0123456789"