Repository: google/turbinia

Issue:
GCS writes fail for large files
Looks like it fails with files after 2GB.  Apparently large file sizes are the new norm after Plaso's on disk format change to sqllite.

[INFO] Task failed with exception: [string longer than 2147483647 bytes]
[INFO] Traceback (most recent call last):
  File "/home/turbinia/turbinia/turbinia/workers/__init__.py", line 378, in run_wrapper
    self.result = self.run(evidence, self.result)
  File "/home/turbinia/turbinia/turbinia/workers/plaso.py", line 58, in run
    new_evidence=[plaso_evidence], close=True)
  File "/home/turbinia/turbinia/turbinia/workers/__init__.py", line 280, in execute
    result.close(success=True)
  File "/home/turbinia/turbinia/turbinia/workers/__init__.py", line 115, in close
    self.save_local_file(evidence.local_path)
  File "/home/turbinia/turbinia/turbinia/workers/__init__.py", line 194, in save_local_file
    new_path = writer.write(file_)
  File "/home/turbinia/turbinia/turbinia/output_manager.py", line 177, in write
    blob.upload_from_filename(file_, client=self.client)
  File "/home/turbinia/turbinia2-env/local/lib/python2.7/site-packages/google/cloud/storage/blob.py", line 929, in upload_from_filename
    size=total_bytes)
  File "/home/turbinia/turbinia2-env/local/lib/python2.7/site-packages/google/cloud/storage/blob.py", line 886, in upload_from_file
    client, file_obj, content_type, size, num_retries)
  File "/home/turbinia/turbinia2-env/local/lib/python2.7/site-packages/google/cloud/storage/blob.py", line 810, in _do_upload
    client, stream, content_type, size, num_retries)
  File "/home/turbinia/turbinia2-env/local/lib/python2.7/site-packages/google/cloud/storage/blob.py", line 649, in _do_multipart_upload
    transport, data, object_metadata, content_type)
  File "/home/turbinia/turbinia2-env/local/lib/python2.7/site-packages/google/resumable_media/requests/upload.py", line 97, in transmit
    retry_strategy=self._retry_strategy)
  File "/home/turbinia/turbinia2-env/local/lib/python2.7/site-packages/google/resumable_media/requests/_helpers.py", line 101, in http_request
    func, RequestsMixin._get_status_code, retry_strategy)
  File "/home/turbinia/turbinia2-env/local/lib/python2.7/site-packages/google/resumable_media/_helpers.py", line 146, in wait_and_retry
    response = func()
  File "/home/turbinia/turbinia2-env/local/lib/python2.7/site-packages/google/auth/transport/requests.py", line 179, in request
    method, url, data=data, headers=request_headers, **kwargs)
  File "/home/turbinia/turbinia2-env/local/lib/python2.7/site-packages/requests/sessions.py", line 508, in request
    resp = self.send(prep, **send_kwargs)
  File "/home/turbinia/turbinia2-env/local/lib/python2.7/site-packages/requests/sessions.py", line 618, in send
    r = adapter.send(request, **kwargs)
  File "/home/turbinia/turbinia2-env/local/lib/python2.7/site-packages/requests/adapters.py", line 440, in send
    timeout=timeout
  File "/home/turbinia/turbinia2-env/local/lib/python2.7/site-packages/urllib3/connectionpool.py", line 601, in urlopen
    chunked=chunked)
  File "/home/turbinia/turbinia2-env/local/lib/python2.7/site-packages/urllib3/connectionpool.py", line 357, in _make_request
    conn.request(method, url, **httplib_request_kw)
  File "/usr/lib/python2.7/httplib.py", line 1042, in request
    self._send_request(method, url, body, headers)
  File "/usr/lib/python2.7/httplib.py", line 1082, in _send_request
    self.endheaders(body)
  File "/usr/lib/python2.7/httplib.py", line 1038, in endheaders
    self._send_output(message_body)
  File "/usr/lib/python2.7/httplib.py", line 882, in _send_output
    self.send(msg)
  File "/usr/lib/python2.7/httplib.py", line 858, in send
    self.sock.sendall(data)
  File "/usr/lib/python2.7/ssl.py", line 753, in sendall
    v = self.send(data[count:])
  File "/usr/lib/python2.7/ssl.py", line 719, in send
    v = self._sslobj.write(data)
OverflowError: string longer than 2147483647 bytes


Discussion:
Some relevant links.  It looks like this is a semi-known issue, and might require either upstream changes to support chunked encoding, or use streaming transfers (which might mean using the boto library? ugh):
https://stackoverflow.com/questions/47610283/cant-upload-2gb-to-google-cloud-storage
https://github.com/googledatalab/datalab/issues/784
https://cloud.google.com/storage/docs/streaming
Ok, so after some reading through the code, I found that setting the chunk_size forces it to use a different upload mechanism.  Here's the script I used to reproduce and verify the fix:
```
#!/usr/bin/env python

import os
import sys
import subprocess

from google.cloud import storage

filename = 'test-gcs-file.dd'
size = 3 * 1024
project = 'my-project-name'
bucket_name = 'my-bucket-name'
chunk_size = 10 * (2 ** 20)
print 'chunk_size is {0:d}'.format(chunk_size)

if not os.path.exists(filename):
    print 'Writing test file to {0:s}'.format(filename)
    subprocess.check_output(
        'dd if=/dev/zero of={0:s} bs=1M count={1:d}'.format(filename, size), shell=True)


client = storage.Client(project=project)
bucket = client.get_bucket(bucket_name)
full_path = os.path.join('output', '{0:d}-{1:s}'.format(1, filename))
blob = storage.Blob(full_path, bucket, chunk_size=chunk_size)
print 'Writing {0:s} to GCS path {1:s}'.format(filename, full_path)
blob.upload_from_filename(filename, client=client)
print 'Done'
```

Resolving patch:
diff --git a/turbinia/output_manager.py b/turbinia/output_manager.py
--- a/turbinia/output_manager.py
+++ b/turbinia/output_manager.py
@@ -266,6 +266,9 @@ class GCSOutputWriter(OutputWriter):
     client (google.cloud.storage.Client): GCS Client
   """
 
+  # 10MB by default
+  CHUNK_SIZE = 10 * (2 ** 20)
+
   def __init__(self, gcs_path, *args, **kwargs):
     """Initialization for GCSOutputWriter.
 
@@ -304,7 +307,7 @@ def copy_to(self, file_):
     full_path = os.path.join(
         self.base_output_dir, self.unique_dir, os.path.basename(file_))
     log.info('Writing {0:s} to GCS path {1:s}'.format(file_, full_path))
-    blob = storage.Blob(full_path, bucket)
+    blob = storage.Blob(full_path, bucket, chunk_size=self.CHUNK_SIZE)
     blob.upload_from_filename(file_, client=self.client)
     return os.path.join('gs://', self.bucket, full_path)
 
@@ -314,7 +317,7 @@ def copy_from(self, file_):
     full_path = os.path.join(self.local_output_dir, os.path.basename(file_))
     log.info('Writing GCS file {0:s} to local path {1:s}'.format(
         file_, full_path))
-    blob = storage.Blob(gcs_path, bucket)
+    blob = storage.Blob(gcs_path, bucket, chunk_size=self.CHUNK_SIZE)
     blob.download_to_filename(full_path, client=self.client)
     if not os.path.exists(full_path):
       raise TurbiniaException(


Test patch: