Repository: google/turbinia

Issue:
TurbiniaTaskResults not serializing properly
I suspect this is because we added the Datastore client to the TurbiniaTask recently.   We should a) make sure to unset the client in the serialization method, and b) make sure to check that `self.input_evidence` exists before calling `post_process()` (or other methods) on it during the Task cleanup.

```
[DEBUG] Checking TurbiniaTaskResult for serializability
[ERROR] Error pickling TurbiniaTaskResult object. Returning a new result with the pickling error, and all previous result data will be lost. Pickle Error: Pickling client objects is explicitly not supported.
Clients have non-trivial state that is local and unpickleable.
[INFO] Error pickling TurbiniaTaskResult object. Returning a new result with the pickling error, and all previous result data will be lost. Pickle Error: Pickling client objects is explicitly not supported.
Clients have non-trivial state that is local and unpickleable.
[ERROR] Task 2f0989499ce1464087f647b54e67ae93 failed.
Traceback (most recent call last):
  File "src/turbinia-docs/turbinia/workers/__init__.py", line 194, in close
    self.input_evidence.postprocess()
AttributeError: 'NoneType' object has no attribute 'postprocess'
```


Discussion:

Resolving patch:
diff --git a/turbinia/workers/__init__.py b/turbinia/workers/__init__.py
--- a/turbinia/workers/__init__.py
+++ b/turbinia/workers/__init__.py
@@ -190,16 +190,23 @@ def close(self, task, success, status=None):
       if not evidence.request_id:
         evidence.request_id = self.request_id
 
-    try:
-      self.input_evidence.postprocess()
-    # Adding a broad exception here because we want to try post-processing
-    # to clean things up even after other failures in the task, so this could
-    # also fail.
-    # pylint: disable=broad-except
-    except Exception as exception:
-      message = 'Evidence post-processing for {0!s} failed: {1!s}'.format(
-          self.input_evidence.name, exception)
-      self.log(message, level=logging.ERROR)
+    if self.input_evidence:
+      try:
+        self.input_evidence.postprocess()
+      # Adding a broad exception here because we want to try post-processing
+      # to clean things up even after other failures in the task, so this could
+      # also fail.
+      # pylint: disable=broad-except
+      except Exception as exception:
+        message = 'Evidence post-processing for {0!s} failed: {1!s}'.format(
+            self.input_evidence.name, exception)
+        self.log(message, level=logging.ERROR)
+    else:
+      self.log(
+          'No input evidence attached to the result object so post-processing '
+          'cannot be run. This usually means there were previous failures '
+          'during Task execution and this may result in resources (e.g. '
+          'mounted disks) accumulating on the Worker.', level=logging.WARNING)
 
     # Write result log info to file
     logfile = os.path.join(self.output_dir, 'worker-log.txt')
@@ -285,19 +292,23 @@ def set_error(self, error, traceback_):
     self.error['traceback'] = traceback_
 
   def serialize(self):
-    """Prepares result object for serialization.
+    """Creates serialized result object.
 
     Returns:
       dict: Object dictionary that is JSON serializable.
     """
     self.state_manager = None
-    self.run_time = self.run_time.total_seconds() if self.run_time else None
-    self.start_time = self.start_time.strftime(DATETIME_FORMAT)
+    result_copy = deepcopy(self.__dict__)
+    if self.run_time:
+      result_copy['run_time'] = self.run_time.total_seconds()
+    else:
+      result_copy['run_time'] = None
+    result_copy['start_time'] = self.start_time.strftime(DATETIME_FORMAT)
     if self.input_evidence:
-      self.input_evidence = self.input_evidence.serialize()
-    self.evidence = [x.serialize() for x in self.evidence]
+      result_copy['input_evidence'] = self.input_evidence.serialize()
+    result_copy['evidence'] = [x.serialize() for x in self.evidence]
 
-    return self.__dict__
+    return result_copy
 
   @classmethod
   def deserialize(cls, input_dict):
@@ -575,7 +586,7 @@ def validate_result(self, result):
     else:
       try:
         log.debug('Checking TurbiniaTaskResult for serializability')
-        pickle.dumps(result)
+        pickle.dumps(result.serialize())
       except (TypeError, pickle.PicklingError) as exception:
         bad_message = (
             'Error pickling TurbiniaTaskResult object. Returning a new result '


Test patch: