Repository: google/turbinia
Pull request #404 (2019-06-10 18:42:46)

Read volatility report and add tests
- Reads volatility report into evidence and Turbinia reporting
- Remove unused debug log
- Also don't need to re-add evidence since the execute() method will add it for us.

aarontp: @mwatkins-fb FYI

Patch:
diff --git a/README.md b/README.md
index 06de463..58178ed 100644
--- a/README.md
+++ b/README.md
@@ -108,7 +108,7 @@ optional arguments:
 * [Installation](docs/install.md)
 * [How it works](docs/how-it-works.md)
 * [Contributing to Turbinia](docs/contributing.md)
-* [Developing new Tasks](docs/how-it-works.md)
+* [Developing new Tasks](docs/developing-new-tasks.md)
 * [FAQ](docs/faq.md)
 * [Debugging and Common Errors](docs/debugging.md)
 
diff --git a/setup.py b/setup.py
index b63dca7..eb0181e 100644
--- a/setup.py
+++ b/setup.py
@@ -69,6 +69,6 @@ setup(
     extras_require={
         'dev': ['mock', 'nose', 'yapf'],
         'local': ['celery~=4.1', 'kombu~=4.1', 'redis~=3.0'],
-        'worker': ['plaso>=20171118']
+        'worker': ['plaso>=20171118', 'pyhindsight>=2.2.0']
     }
 )
diff --git a/turbinia/client.py b/turbinia/client.py
index b6d189c..9be1827 100644
--- a/turbinia/client.py
+++ b/turbinia/client.py
@@ -37,6 +37,7 @@ from turbinia.workers.analysis.wordpress import WordpressAccessLogAnalysisTask
 from turbinia.workers.analysis.jenkins import JenkinsAnalysisTask
 from turbinia.workers.grep import GrepTask
 from turbinia.workers.hadoop import HadoopAnalysisTask
+from turbinia.workers.hindsight import HindsightTask
 from turbinia.workers.plaso import PlasoTask
 from turbinia.workers.psort import PsortTask
 from turbinia.workers.sshd import SSHDAnalysisTask
@@ -54,6 +55,7 @@ TASK_MAP = {
     'jenkinsanalysistask': JenkinsAnalysisTask,
     'greptask': GrepTask,
     'hadoopanalysistask': HadoopAnalysisTask,
+    'hindsighttask': HindsightTask,
     'plasotask': PlasoTask,
     'psorttask': PsortTask,
     'sshdanalysistask': SSHDAnalysisTask,
diff --git a/turbinia/evidence.py b/turbinia/evidence.py
index 745353c..278036f 100644
--- a/turbinia/evidence.py
+++ b/turbinia/evidence.py
@@ -199,6 +199,23 @@ class Directory(Evidence):
   pass
 
 
+class ChromiumProfile(Evidence):
+  """Chromium based browser profile evidence.
+
+  Attributes:
+    browser_type: The type of browser.
+      Supported options are Chrome (default) and Brave.
+    format: Output format (default is sqlite, other options are xlsx and jsonl)
+  """
+
+  def __init__(self, browser_type=None, output_format=None, *args, **kwargs):
+    """Initialization for chromium profile evidence object."""
+    super(ChromiumProfile, self).__init__(*args, **kwargs)
+    self.browser_type = browser_type
+    self.output_format = output_format
+    self.copyable = True
+
+
 class RawDisk(Evidence):
   """Evidence object for Disk based evidence.
 
diff --git a/turbinia/jobs/__init__.py b/turbinia/jobs/__init__.py
index fb238ea..7df9221 100644
--- a/turbinia/jobs/__init__.py
+++ b/turbinia/jobs/__init__.py
@@ -17,6 +17,7 @@ from turbinia.jobs import grep
 from turbinia.jobs import hadoop
 from turbinia.jobs import http_access_logs
 from turbinia.jobs import jenkins
+from turbinia.jobs import hindsight
 from turbinia.jobs import plaso
 from turbinia.jobs import psort
 from turbinia.jobs import sshd
diff --git a/turbinia/jobs/hindsight.py b/turbinia/jobs/hindsight.py
new file mode 100644
index 0000000..dce33e7
--- /dev/null
+++ b/turbinia/jobs/hindsight.py
@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Job to execute Hindsight tasks."""
+
+from __future__ import unicode_literals
+
+from turbinia.evidence import ChromiumProfile
+from turbinia.evidence import TextFile
+from turbinia.jobs import interface
+from turbinia.jobs import manager
+from turbinia.workers.hindsight import HindsightTask
+
+
+class HindsightJob(interface.TurbiniaJob):
+  """Hindsight analysis job.
+    This will execute hindsight against a piece of evidence.
+    """
+
+  evidence_input = [ChromiumProfile]
+  evidence_output = [TextFile]
+
+  NAME = 'HindsightJob'
+
+  def create_tasks(self, evidence):
+    """Create task.
+        Args:
+            evidence: List of evidence objects to process
+        Returns:
+            A list of tasks to schedule.
+        """
+
+    tasks = [HindsightTask() for _ in evidence]
+    return tasks
+
+
+manager.JobsManager.RegisterJob(HindsightJob)
diff --git a/turbinia/lib/libcloudforensics.py b/turbinia/lib/libcloudforensics.py
index 612dcd9..532ae1e 100644
--- a/turbinia/lib/libcloudforensics.py
+++ b/turbinia/lib/libcloudforensics.py
@@ -330,11 +330,22 @@ class GoogleCloudProject(object):
     source_disk_image = ubuntu_image['selfLink']
 
     # Analysis software to install.
+    # yapf: disable
     packages_to_install = [
-        'python-plaso', 'xmount', 'sleuthkit', 'libfvde-tools', 'libbde-tools',
-        'plaso-tools', 'jq', 'ncdu', 'htop', 'binutils', 'upx-ucl',
+        'binutils',
         'docker-explorer-tools'
-    ]
+        'htop',
+        'jq',
+        'libbde-tools',
+        'libfsapfs-tools',
+        'libfvde-tools',
+        'ncdu',
+        'plaso-tools',
+        'sleuthkit',
+        'upx-ucl',
+        'xmount']
+
+    # yapf: enable
 
     startup_script = """
         #!/bin/bash
diff --git a/turbinia/turbiniactl.py b/turbinia/turbiniactl.py
index 0d22e3a..ec04a42 100644
--- a/turbinia/turbiniactl.py
+++ b/turbinia/turbiniactl.py
@@ -253,6 +253,20 @@ def main():
   parser_directory.add_argument(
       '-n', '--name', help='Descriptive name of the evidence', required=False)
 
+  # Parser options for ChromiumProfile evidence type
+  parser_hindsight = subparsers.add_parser(
+      'hindsight', help='Process ChromiumProfile as Evidence')
+  parser_hindsight.add_argument(
+      '-l', '--local_path', help='Local path to the evidence', required=True)
+  parser_hindsight.add_argument(
+      '-f', '--format', help='Output format (supported types are '
+      'xlsx, sqlite, jsonl)', default='sqlite')
+  parser_hindsight.add_argument(
+      '-b', '--browser_type', help='The type of browser the input files belong'
+      'to (supported types are Chrome, Brave)', default='Chrome')
+  parser_hindsight.add_argument(
+      '-n', '--name', help='Descriptive name of the evidence', required=False)
+
   # List Jobs
   subparsers.add_parser(
       'listjobs',
@@ -403,6 +417,18 @@ def main():
         name=args.name, disk_name=args.disk_name,
         embedded_path=args.embedded_path, mount_partition=args.mount_partition,
         project=args.project, zone=args.zone, source=args.source)
+  elif args.command == 'hindsight':
+    if args.format not in ['xlsx', 'sqlite', 'jsonl']:
+      log.error('Invalid output format.')
+      sys.exit(1)
+    if args.browser_type not in ['Chrome', 'Brave']:
+      log.error('Browser type not supported.')
+      sys.exit(1)
+    args.name = args.name if args.name else args.local_path
+    local_path = os.path.abspath(args.local_path)
+    evidence_ = evidence.ChromiumProfile(
+        name=args.name, local_path=local_path, output_format=args.format,
+        browser_type=args.browser_type)
   elif args.command == 'rawmemory':
     args.name = args.name if args.name else args.local_path
     local_path = os.path.abspath(args.local_path)
diff --git a/turbinia/workers/hindsight.py b/turbinia/workers/hindsight.py
new file mode 100644
index 0000000..a00da10
--- /dev/null
+++ b/turbinia/workers/hindsight.py
@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Task for executing hindsight."""
+
+from __future__ import unicode_literals
+
+import os
+
+from turbinia.evidence import TextFile
+from turbinia.workers import TurbiniaTask
+
+
+class HindsightTask(TurbiniaTask):
+  """Task to execute hindsight."""
+
+  def __init__(self):
+    super(HindsightTask, self).__init__()
+
+  def run(self, evidence, result):
+    """Task to execute hindsight."""
+
+    # Create the new Evidence object that will be generated by this Task.
+    output_evidence = TextFile()
+
+    # Create a path that we can write the new file to.
+    output_file_path = os.path.join(self.output_dir, 'hindsight_report')
+    output_evidence.local_path = output_file_path
+
+    # Create a path for the log file
+    hindsight_log = os.path.join(self.output_dir, 'hindsight.log')
+
+    cmd = [
+        'hindsight.py', '-i', evidence.local_path, '--format',
+        evidence.output_format, '--browser_type', evidence.browser_type,
+        '--output', output_file_path, '-l', hindsight_log
+    ]
+
+    result.log('Running hindsight as [{0:s}]'.format(' '.join(cmd)))
+    self.execute(
+        cmd, result, log_files=[hindsight_log], new_evidence=[output_evidence],
+        close=True)
+
+    return result
diff --git a/turbinia/workers/volatility.py b/turbinia/workers/volatility.py
index e54cc0c..6810d21 100644
--- a/turbinia/workers/volatility.py
+++ b/turbinia/workers/volatility.py
@@ -22,12 +22,18 @@ from turbinia import config
 from turbinia.evidence import VolatilityReport
 from turbinia.workers import TurbiniaTask
 
+MAX_REPORT_SIZE = 2**30  # 1 GiB
+
 
 class VolatilityTask(TurbiniaTask):
-  """Task to execute volatility."""
+  """Task to execute volatility.
+
+  Attributes:
+    module(str): The name of the volatility module to run.
+  """
 
-  def __init__(self, module='test'):
-    super(VolatilityTask, self).__init__()
+  def __init__(self, module='test', *args, **kwargs):
+    super(VolatilityTask, self).__init__(*args, **kwargs)
     self.module = module
 
   def run(self, evidence, result):
@@ -40,7 +46,6 @@ class VolatilityTask(TurbiniaTask):
     Returns:
         TurbiniaTaskResult object.
     """
-
     config.LoadConfig()
 
     # Create the new Evidence object that will be generated by this Task.
@@ -50,32 +55,53 @@ class VolatilityTask(TurbiniaTask):
     output_file_path = os.path.join(
         self.output_dir, '{0:s}.txt'.format(self.id))
     output_evidence.local_path = output_file_path
-    debug_log = os.path.join(self.output_dir, '{0:s}.log'.format(self.id))
 
     # TODO: Add in config options for Turbinia
     cmd = (
-        'vol2 -f {0:s} --profile={1:s} {2:s} --output=text '
+        'vol.py -f {0:s} --profile={1:s} {2:s} --output=text '
         '--output-file={3:s}').format(
             evidence.local_path, evidence.profile, self.module,
             output_file_path).split()
 
-    result.log('Running vol as [{0:s}]'.format(' '.join(cmd)))
-    res = self.execute(
-        cmd, result, log_files=[debug_log], new_evidence=[output_evidence],
-        close=True)
+    result.log('Running volatility as [{0:s}]'.format(' '.join(cmd)))
+    res = self.execute(cmd, result, new_evidence=[output_evidence], close=True)
 
     if res == 0:
-      # Write the report to the output file.
-      with open(output_file_path, 'w') as fh:
-        fh.write(output_evidence.text_data.encode('utf-8'))
-
-      # Add the resulting evidence to the result object.
-      result.add_evidence(output_evidence, evidence.config)
-      result.report_data = output_evidence
-      summary = 'Volatility module {0} successfully ran.'.format(self.module)
-      result.close(self, success=True, status=summary)
+      success = True
+      # Get report data from the output file.
+      try:
+        file_size = os.stat(output_file_path).st_size
+      except (IOError, OSError) as exception:
+        msg = 'Unable to determine size of output file {0:s}: {1!s}'.format(
+            output_file_path, exception)
+        summary = 'Volatility ran successfully, but no output file was created'
+        result.log(msg)
+        result.close(self, success=False, status=summary)
+        return result
+
+      if file_size > MAX_REPORT_SIZE:
+        result.log(
+            'Volatility report output size ({0:d}) is greater than max report '
+            'size ({1:d}). Truncating report to max size'.format(
+                file_size, MAX_REPORT_SIZE))
+        summary = (
+            'Volatility module {0:s} successfuly ran (report truncated)'.format(
+                self.module))
+      else:
+        summary = 'Volatility module {0:s} successfully ran'.format(self.module)
+
+      with open(output_file_path, 'rb') as fh:
+        report_data = fh.read(MAX_REPORT_SIZE)
+        try:
+          output_evidence.text_data = report_data.decode('utf-8')
+        except UnicodeDecodeError as e:
+          success = False
+          summary = 'Volatility report could not be read: {0!s}'.format(e)
+
+      result.report_data = output_evidence.text_data
+      result.close(self, success=success, status=summary)
     else:
-      summary = 'Volatility module {0} failed to run.'.format(self.module)
+      summary = 'Volatility module {0:s} failed to run'.format(self.module)
       result.close(self, success=False, status=summary)
 
     return result


Test patch:
diff --git a/turbinia/workers/volatility_test.py b/turbinia/workers/volatility_test.py
new file mode 100644
index 0000000..179cf0a
--- /dev/null
+++ b/turbinia/workers/volatility_test.py
@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for VolatilityTask."""
+
+from __future__ import unicode_literals
+
+import os
+import mock
+
+from turbinia.evidence import RawMemory
+from turbinia.workers.workers_test import TestTurbiniaTaskBase
+from turbinia.workers import volatility
+
+
+class VolatilityTaskTest(TestTurbiniaTaskBase):
+  """Tests for VolatilityTask."""
+
+  def setUp(self):
+    super(VolatilityTaskTest, self).setUp(
+        task_class=volatility.VolatilityTask, evidence_class=RawMemory)
+    self.original_max_report_size = volatility.MAX_REPORT_SIZE
+    self.evidence.profile = 'TestProfile'
+    self.evidence.module_list = ['TestModule']
+    self.task.output_dir = self.task.base_output_dir
+    self.output_file_path = os.path.join(
+        self.task.output_dir, '{0:s}.txt'.format(self.task.id))
+    self.remove_files.append(self.output_file_path)
+    self.setResults(mock_run=False)
+
+  def tearDown(self):
+    super(VolatilityTaskTest, self).tearDown()
+    volatility.MAX_REPORT_SIZE = self.original_max_report_size
+
+  def testVolatilityTaskRun(self):
+    """Test successful volatility task run."""
+    file_contents = 'Test file contents'
+    self.task.execute = mock.MagicMock(return_value=0)
+    with open(self.output_file_path, 'w') as fh:
+      fh.write(file_contents)
+    self.task.run(self.evidence, self.result)
+
+    self.assertEqual(file_contents, self.result.report_data)
+    self.result.close.assert_called_with(
+        self.task, success=True, status=mock.ANY)
+
+  def testVolatilityTaskRunLargeOutput(self):
+    """Test volatility run with large output."""
+    volatility.MAX_REPORT_SIZE = 9
+    file_contents = 'Test file contents'
+    truncated_file_contents = file_contents[:volatility.MAX_REPORT_SIZE]
+    self.task.execute = mock.MagicMock(return_value=0)
+    with open(self.output_file_path, 'w') as fh:
+      fh.write(file_contents)
+    self.task.run(self.evidence, self.result)
+
+    self.assertEqual(truncated_file_contents, self.result.report_data)
+    self.result.close.assert_called_with(
+        self.task, success=True, status=mock.ANY)
+
+  def testVolatilityTaskRunNoOutput(self):
+    """Test volatility task run with no output."""
+    self.task.execute = mock.MagicMock(return_value=0)
+    self.task.run(self.evidence, self.result)
+
+    self.assertEqual(None, self.result.report_data)
+    self.result.close.assert_called_with(
+        self.task, success=False, status=mock.ANY)
diff --git a/turbinia/workers/workers_test.py b/turbinia/workers/workers_test.py
index f90bb8d..65a1c8a 100644
--- a/turbinia/workers/workers_test.py
+++ b/turbinia/workers/workers_test.py
@@ -29,27 +29,39 @@ from turbinia.workers import TurbiniaTaskResult
 from turbinia.workers.plaso import PlasoTask
 
 
-class TestTurbiniaTask(unittest.TestCase):
-  """Test TurbiniaTask class."""
-
-  def setUp(self):
+class TestTurbiniaTaskBase(unittest.TestCase):
+  """Test TurbiniaTask class.
+
+  Attributes:
+    class_task(TurbiniaTask): The class the test should instantiated
+    remove_file(list(str)): Files that will be removed after the test run
+    remove_dirs(list(str)): Dirs that will be removed after the test run
+    base_output_dir(str): The base output directory used by the Task
+    task(TurbiniaTask): The instantiated Task under test
+    evidence(Evidence): The test evidence object used by the Task
+    result(TurbiniaResult): The result object used by the Task
+  """
+
+  def setUp(self, task_class=TurbiniaTask, evidence_class=evidence.RawDisk):
+    self.task_class = task_class
+    self.evidence_class = evidence_class
     self.remove_files = []
     self.remove_dirs = []
 
-    # Set up TurbiniaTask
+    # Set up Tasks under test
     self.base_output_dir = tempfile.mkdtemp()
     self.plaso_task = PlasoTask(base_output_dir=self.base_output_dir)
     self.plaso_task.output_manager = mock.MagicMock()
     self.plaso_task.output_manager.get_local_output_dirs.return_value = (
         None, None)
-    self.task = TurbiniaTask(base_output_dir=self.base_output_dir)
+    self.task = self.task_class(base_output_dir=self.base_output_dir)
     self.task.output_manager = mock.MagicMock()
     self.task.output_manager.get_local_output_dirs.return_value = (None, None)
 
-    # Set up RawDisk Evidence
-    test_disk_path = tempfile.mkstemp(dir=self.base_output_dir)[1]
-    self.remove_files.append(test_disk_path)
-    self.evidence = evidence.RawDisk(local_path=test_disk_path)
+    # Set up Evidence
+    test_artifact_path = tempfile.mkstemp(dir=self.base_output_dir)[1]
+    self.remove_files.append(test_artifact_path)
+    self.evidence = self.evidence_class(local_path=test_artifact_path)
 
     # Set up TurbiniaTaskResult
     self.result = TurbiniaTaskResult(base_output_dir=self.base_output_dir)
@@ -68,13 +80,15 @@ class TestTurbiniaTask(unittest.TestCase):
 
     os.rmdir(self.base_output_dir)
 
-  def setResults(self, setup=None, run=None, validate_result=None):
-    """Set up mock returns.
+  def setResults(
+      self, setup=None, run=None, validate_result=None, mock_run=True):
+    """Set up mock returns in TurbiniaTaskResult object.
 
     Args:
       setup: What value to return from setup()
       run: What value to return from run()
       validate_result: What value to return from validate_result()
+      mock_run(bool): Whether to mock out the run method
     """
     if setup is None:
       setup = self.result
@@ -86,9 +100,14 @@ class TestTurbiniaTask(unittest.TestCase):
     self.result.status = 'TestStatus'
     self.result.close = mock.MagicMock()
     self.task.setup = mock.MagicMock(return_value=setup)
-    self.task.run = mock.MagicMock(return_value=run)
+    if mock_run:
+      self.task.run = mock.MagicMock(return_value=run)
     self.task.validate_result = mock.MagicMock(return_value=validate_result)
 
+
+class TestTurbiniaTask(TestTurbiniaTaskBase):
+  """Test TurbiniaTask class."""
+
   def testTurbiniaTaskSerialize(self):
     """Test that we can properly serialize/deserialize tasks."""
     out_dict = self.plaso_task.serialize()