Repository: google/turbinia

Issue:
The value of property "data" is longer than 1048487 bytes. 
Looks like the traceback is happening in gcp libraries, are we adding a new datatype that we need to adjust for? 
```
Traceback (most recent call last):
  File "/usr/local/lib/python3.8/dist-packages/google/cloud/pubsub_v1/subscriber/_protocol/streaming_pull_manager.py", line 72, in _wrap_callback_errors
    callback(message)
  File "/usr/local/lib/python3.8/dist-packages/psq/queue.py", line 128, in _pubsub_message_callback
    task_callback(task)
  File "/usr/local/lib/python3.8/dist-packages/psq/worker.py", line 50, in run_task
    task.execute(self.queue)
  File "/usr/local/lib/python3.8/dist-packages/psq/task.py", line 101, in execute
    queue.storage.put_task(self)
  File "/usr/local/lib/python3.8/dist-packages/psq/datastore_storage.py", line 63, in put_task
    self.datastore.put(entity)
  File "/usr/local/lib/python3.8/dist-packages/google/cloud/datastore/client.py", line 570, in put
    self.put_multi(entities=[entity], retry=retry, timeout=timeout)
  File "/usr/local/lib/python3.8/dist-packages/google/cloud/datastore/client.py", line 610, in put_multi
    current.commit(retry=retry, timeout=timeout)
  File "/usr/local/lib/python3.8/dist-packages/google/cloud/datastore/batch.py", line 300, in commit
    self._commit(retry=retry, timeout=timeout)
  File "/usr/local/lib/python3.8/dist-packages/google/cloud/datastore/batch.py", line 257, in _commit
    commit_response_pb = self._client._datastore_api.commit(
  File "/usr/local/lib/python3.8/dist-packages/google/cloud/datastore_v1/services/datastore/client.py", line 627, in commit
    response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,)
  File "/usr/local/lib/python3.8/dist-packages/google/api_core/gapic_v1/method.py", line 145, in __call__
    return wrapped_func(*args, **kwargs)
  File "/usr/local/lib/python3.8/dist-packages/google/api_core/grpc_helpers.py", line 69, in error_remapped_callable
    six.raise_from(exceptions.from_grpc_error(exc), exc)
  File "<string>", line 3, in raise_from
google.api_core.exceptions.InvalidArgument: 400 The value of property "data" is longer than 1048487 bytes. 
```


Discussion:
I think it might have hit some kind of limit trying to write something >1MB.  It must have been a very large report.
Yeah, apparently an entity has a 1MB limit: https://cloud.google.com/datastore/docs/concepts/limits .  Maybe for now we can truncate the report and add a counter for that condition.  Long term we'll move away from datastore, but we can make those changes for now and monitor how often it happens to see if we need to do anything further.

Resolving patch:
diff --git a/turbinia/task_manager.py b/turbinia/task_manager.py
--- a/turbinia/task_manager.py
+++ b/turbinia/task_manager.py
@@ -684,8 +684,8 @@ def process_tasks(self):
         timeout = self.check_task_timeout(task)
         if timeout:
           log.warning(
-              'Task {0:s} timed on server out after {1:d} seconds. Auto-closing Task.'
-              .format(celery_task.id, timeout))
+              'Task {0:s} timed out on server after {1:d} seconds. '
+              'Auto-closing Task.'.format(celery_task.id, timeout))
           task = self.timeout_task(task, timeout)
           completed_tasks.append(task)
 
diff --git a/turbinia/workers/__init__.py b/turbinia/workers/__init__.py
--- a/turbinia/workers/__init__.py
+++ b/turbinia/workers/__init__.py
@@ -50,6 +50,12 @@
 from prometheus_client import Histogram
 
 METRICS = {}
+# Set the maximum size that the report can be before truncating it.  This is a
+# best effort estimate and not a guarantee and comes from the limit for
+# datastore entities[1] less some overhead for the rest of the attributes that
+# will be saved in the response.
+# [1]https://cloud.google.com/datastore/docs/concepts/limits
+REPORT_MAXSIZE = int(1048572 * 0.75)
 
 log = logging.getLogger('turbinia')
 
@@ -220,6 +226,18 @@ def close(self, task, success, status=None):
             'Evidence {0:s} has empty source_path so '
             'not saving.'.format(evidence.name))
 
+      # Truncate report text data if it is approaching the size of the max
+      # datastore entity size (See REPORT_MAXSIZE definition for details).
+      if (hasattr(evidence, 'text_data') and evidence.text_data and
+          len(evidence.text_data) > REPORT_MAXSIZE):
+        message = (
+            'The text_data attribute has a size {0:d} larger than the max '
+            'size {1:d} so truncating the response.'.format(
+                len(evidence.text_data), REPORT_MAXSIZE))
+        self.log(message, logging.WARN)
+        evidence.text_data = evidence.text_data[:REPORT_MAXSIZE] + '\n'
+        evidence.text_data += message
+
       if not evidence.request_id:
         evidence.request_id = self.request_id
 
@@ -659,7 +677,6 @@ def execute(
 
       ret = proc.returncode
 
-    result.error['stdout'] = str(stdout)
     result.error['stderr'] = str(stderr)
 
     if stderr_file and not stderr:


Test patch:
diff --git a/turbinia/workers/workers_test.py b/turbinia/workers/workers_test.py
--- a/turbinia/workers/workers_test.py
+++ b/turbinia/workers/workers_test.py
@@ -125,6 +125,18 @@ def unregisterMetrics(self):
 class TestTurbiniaTask(TestTurbiniaTaskBase):
   """Test TurbiniaTask class."""
 
+  def testTurbiniaTaskCloseTruncate(self):
+    """Tests that the close method will truncate large report output."""
+    evidence_ = evidence.ReportText(source_path='/no/path')
+    max_size = 2**20
+    evidence_.text_data = 'A' * max_size
+    self.result.add_evidence(evidence_, self.task._evidence_config)
+    self.result.close(self.task, success=True)
+    self.remove_files.append(
+        os.path.join(self.task.base_output_dir, 'worker-log.txt'))
+    self.assertIn('truncating', evidence_.text_data[-100:])
+    self.assertTrue(len(evidence_.text_data) <= (max_size * 0.8))
+
   def testTurbiniaTaskSerialize(self):
     """Test that we can properly serialize/deserialize tasks."""
     out_dict = self.plaso_task.serialize()
@@ -267,7 +279,6 @@ def testTurbiniaTaskExecute(self, popen_mock):
     # Command was executed, has the correct output saved and
     # TurbiniaTaskResult.close() was called with successful status.
     popen_mock.assert_called_with(cmd, stdout=-1, stderr=-1, cwd=None, env=None)
-    self.assertEqual(self.result.error['stdout'], str(output[0]))
     self.assertEqual(self.result.error['stderr'], str(output[1]))
     self.assertEqual(stdout_data, output[0])
     self.result.close.assert_called_with(self.task, success=True)