Repository: google/turbinia
Pull request #455 (2019-11-02 00:18:10)

Fix saving files on local instances
We still want to run the file copying when using a shared filesystem because we use it to copy from tmp files to the main shared filesystem.    We also want to save the result even if the file doesn't get copied.   This also updates some comments to be more clear.


Patch:
diff --git a/turbinia/output_manager.py b/turbinia/output_manager.py
index 768b258..d6b8d17 100644
--- a/turbinia/output_manager.py
+++ b/turbinia/output_manager.py
@@ -194,10 +194,15 @@ class OutputManager(object):
     local_path = None
     for writer in self._output_writers:
       new_path = writer.copy_to(file_)
-      if new_path and result:
-        result.saved_paths.append(new_path)
-        saved_path = new_path
-        saved_path_type = writer.name
+      if result:
+        if new_path:
+          result.saved_paths.append(new_path)
+          saved_path = new_path
+          saved_path_type = writer.name
+        elif os.path.exists(file_) and os.path.getsize(file_) > 0:
+          # We want to save the old path if the path is still valid.
+          result.saved_paths.append(file_)
+
       if writer.name == LocalOutputWriter.NAME:
         local_path = new_path
 
@@ -350,18 +355,20 @@ class LocalOutputWriter(OutputWriter):
     if self.local_output_dir in os.path.commonprefix([file_path,
                                                       destination_file]):
       log.debug(
-          'Not copying file {0:s} in output dir {1:s}'.format(
+          'Not copying source file {0:s} already in output dir {1:s}'.format(
               file_path, self.local_output_dir))
       return None
     if not os.path.exists(file_path):
-      log.warning('File [{0:s}] does not exist.'.format(file_path))
+      log.warning('Source file [{0:s}] does not exist.'.format(file_path))
       return None
     if os.path.exists(destination_file):
       log.warning(
-          'New file path [{0:s}] already exists.'.format(destination_file))
+          'Target output file path [{0:s}] already exists.'.format(
+              destination_file))
       return None
 
     shutil.copy(file_path, destination_file)
+    log.debug('Copied file {0:s} to {1:s}'.format(file_path, destination_file))
     return destination_file
 
   def copy_to(self, source_file):
diff --git a/turbinia/workers/__init__.py b/turbinia/workers/__init__.py
index 138e1fd..e9651aa 100644
--- a/turbinia/workers/__init__.py
+++ b/turbinia/workers/__init__.py
@@ -170,9 +170,8 @@ class TurbiniaTaskResult(object):
     for evidence in self.evidence:
       if evidence.local_path and os.path.exists(evidence.local_path):
         self.saved_paths.append(evidence.local_path)
-        if not task.run_local:
-          if evidence.copyable and not config.SHARED_FILESYSTEM:
-            task.output_manager.save_evidence(evidence, self)
+        if not task.run_local and evidence.copyable:
+          task.output_manager.save_evidence(evidence, self)
       else:
         self.log(
             'Evidence {0:s} has empty or missing file at local_path {1:s} so '
@@ -440,7 +439,7 @@ class TurbiniaTask(object):
             'Log file {0:s} is empty. Not saving'.format(file_),
             level=logging.DEBUG)
         continue
-      result.log('Output file at {0:s}'.format(file_))
+      result.log('Output log file found at {0:s}'.format(file_))
       if not self.run_local:
         self.output_manager.save_local_file(file_, result)
 
@@ -457,7 +456,7 @@ class TurbiniaTask(object):
               'Output file {0:s} is empty. Not saving'.format(file_),
               level=logging.DEBUG)
           continue
-        result.log('Output file at {0:s}'.format(file_))
+        result.log('Output save file at {0:s}'.format(file_))
         if not self.run_local:
           self.output_manager.save_local_file(file_, result)
 
diff --git a/turbinia/workers/bulk_extractor.py b/turbinia/workers/bulk_extractor.py
index 4817e46..6704559 100644
--- a/turbinia/workers/bulk_extractor.py
+++ b/turbinia/workers/bulk_extractor.py
@@ -72,6 +72,23 @@ class BulkExtractorTask(TurbiniaTask):
       return result
     return result
 
+  def check_xml_attrib(self, xml_key):
+    """Checks if a key exists within the xml report.
+
+    Args:
+      xml_key(str): the xml key to check for.
+
+    Returns:
+      xml_hit(str): the xml value else return N/A.
+    """
+    xml_hit = 'N/A'
+    xml_search = self.xml.find(xml_key)
+
+    # If exists, return the text value.
+    if xml_search is not None:
+      xml_hit = xml_search.text
+    return xml_hit
+
   def generate_summary_report(self, output_file_path):
     """Generate a summary report from the resulting bulk extractor run.
 
@@ -93,7 +110,7 @@ class BulkExtractorTask(TurbiniaTask):
       return (report, report)
 
     # Parse existing XML file.
-    xml = xml_tree.parse(report_path)
+    self.xml = xml_tree.parse(report_path)
 
     # Place in try/except statement to continue execution when
     # an attribute is not found and NoneType is returned.
@@ -104,23 +121,25 @@ class BulkExtractorTask(TurbiniaTask):
       findings.append(
           fmt.bullet(
               'Program: {0} - {1}'.format(
-                  xml.find('creator/program').text,
-                  xml.find('creator/version').text)))
+                  self.check_xml_attrib('creator/program'),
+                  self.check_xml_attrib('creator/version'))))
       findings.append(
           fmt.bullet(
               'Command Line: {0}'.format(
-                  xml.find('creator/execution_environment/command_line').text)))
+                  self.check_xml_attrib(
+                      'creator/execution_environment/command_line'))))
       findings.append(
           fmt.bullet(
               'Start Time: {0}'.format(
-                  xml.find('creator/execution_environment/start_time').text)))
+                  self.check_xml_attrib(
+                      'creator/execution_environment/start_time'))))
       findings.append(
           fmt.bullet(
               'Elapsed Time: {0}'.format(
-                  xml.find('report/elapsed_seconds').text)))
+                  self.check_xml_attrib('report/elapsed_seconds'))))
 
       # Retrieve results from each of the scanner runs
-      feature_files = xml.find('feature_files')
+      feature_files = self.xml.find('feature_files')
       if feature_files is not None:
         feature_iter = feature_files.iter()
         findings.append(fmt.heading5('Scanner Results'))


Test patch:
diff --git a/turbinia/workers/bulk_extractor_test.py b/turbinia/workers/bulk_extractor_test.py
index df81094..978e002 100644
--- a/turbinia/workers/bulk_extractor_test.py
+++ b/turbinia/workers/bulk_extractor_test.py
@@ -20,8 +20,8 @@ from io import StringIO
 
 import os
 import unittest
-import mock
 import textwrap
+import mock
 
 from turbinia.evidence import BulkExtractorOutput
 from turbinia.workers import bulk_extractor
@@ -66,7 +66,7 @@ class BulkExtractorTaskTest(TestTurbiniaTaskBase):
         * Program: BULK_EXTRACTOR - 1.6.0-dev
         * Command Line: bulk_extractor /tmp/test-small.img -o /output/test-small.img
         * Start Time: 2019-09-27T16:34:48Z
-        * Elapsed Time: 2.389155
+        * Elapsed Time: N/A
         ##### There are no findings to report.""")
 
     summary_sample = "0 artifacts have been extracted."
@@ -82,9 +82,6 @@ class BulkExtractorTaskTest(TestTurbiniaTaskBase):
             <start_time>2019-09-27T16:34:48Z</start_time>
           </execution_environment>
         </creator>
-        <report>
-          <elapsed_seconds>2.389155</elapsed_seconds>
-        </report>
         </dfxml>""")
 
     str_io = StringIO(xml_sample)