Repository: google/turbinia
Pull request #1018 (2022-03-10 21:11:39)

Properly handle case when a file entry cannot be read by dfVFS
dfVFS raises IOError exceptions if a file system entry cannot be read. This was causing the task to fail and return as soon as the exception was caught. 

This pull request proposes a fix to ensure that the exceptions are properly handled to allow the task to continue to the next file entry if the previous file entry could not be read.




Patch:
diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile
index 1a0a562..5438afa 100644
--- a/docker/worker/Dockerfile
+++ b/docker/worker/Dockerfile
@@ -71,10 +71,6 @@ RUN mkdir -p /opt/loki && chown -R turbinia:turbinia /opt/loki \
     && sudo chown -R turbinia:turbinia /opt/loki \
     && git clone https://github.com/Neo23x0/signature-base.git
 
-ADD . /tmp/
-# unshallow and fetch all tags so our build systems pickup the correct git tag if it's a shallow clone
-RUN if $(cd /tmp/ && git rev-parse --is-shallow-repository); then cd /tmp/ && git fetch --prune --unshallow && git fetch --depth=1 origin +refs/tags/*:refs/tags/*; fi
-RUN cd /tmp/ && python3 setup.py install
 
 RUN mkdir /etc/turbinia && mkdir -p /mnt/turbinia/ && mkdir -p /var/lib/turbinia/ \
     && mkdir -p /etc/turbinia/ && chown -R turbinia:turbinia /etc/turbinia/ \
@@ -91,6 +87,11 @@ RUN cp /home/turbinia/password.lst /root/
 COPY k8s/tools/check-lockfile.py /home/turbinia/check-lockfile.py
 RUN chown turbinia:turbinia /home/turbinia/check-lockfile.py
 
+ADD . /tmp/
+# unshallow and fetch all tags so our build systems pickup the correct git tag if it's a shallow clone
+RUN if $(cd /tmp/ && git rev-parse --is-shallow-repository); then cd /tmp/ && git fetch --prune --unshallow && git fetch --depth=1 origin +refs/tags/*:refs/tags/*; fi
+RUN cd /tmp/ && python3 setup.py install
+
 COPY docker/worker/start.sh /home/turbinia/start.sh
 RUN chmod +rwx /home/turbinia/start.sh
 USER turbinia
diff --git a/turbinia/client.py b/turbinia/client.py
index ba7c4ae..8b7a023 100644
--- a/turbinia/client.py
+++ b/turbinia/client.py
@@ -171,9 +171,9 @@ class BaseTurbiniaClient:
       jobs_allowlist=None, jobs_denylist=None, recipe_name=None, sketch_id=None,
       skip_recipe_validation=False, yara_rules=None):
     """Creates a Turbinia recipe.
-    
+
     If no recipe_name is specified, this  method returns a default recipe.
-    If a recipe_name is specified then this method will build the recipe 
+    If a recipe_name is specified then this method will build the recipe
     dictionary by reading the  contents of a recipe file. The path to
     the recipe file is inferre from the recipe_name and the RECIPE_FILE_DIR
     configuration parameter.
@@ -989,11 +989,22 @@ class BaseTurbiniaClient:
       report.append(fmt.heading1('{0:s} Tasks'.format(success_type)))
       if not task_map[success_type]:
         report.append(fmt.bullet('None'))
+      task_counter = defaultdict(int)
       for task in task_map[success_type]:
         if full_report and success_type == success_types[0]:
           report.extend(self.format_task_detail(task, show_files=all_fields))
-        else:
+        elif success_type == success_types[2]:
           report.extend(self.format_task(task, show_files=all_fields))
+        else:
+          task_counter['\n'.join(self.format_task(task,
+                                                  show_files=all_fields))] += 1
+
+      if len(task_counter):
+        for k, v in task_counter.items():
+          if v == 1:
+            report.append(k)
+          else:
+            report.append('{0:s} x {1:d}'.format(k, v))
 
     return '\n'.join(report)
 
diff --git a/turbinia/config/turbinia_config_tmpl.py b/turbinia/config/turbinia_config_tmpl.py
index e2dbf58..e77bc9c 100644
--- a/turbinia/config/turbinia_config_tmpl.py
+++ b/turbinia/config/turbinia_config_tmpl.py
@@ -153,6 +153,11 @@ DEPENDENCIES = [{
     'programs': ['fsstat'],
     'docker_image': None,
     'timeout': 1800
+}, {
+    'job': 'GitlabJob',
+    'programs': ['zgrep'],
+    'docker_image': None,
+    'timeout': 1800
 }, {
     'job': 'GrepJob',
     'programs': ['grep'],
diff --git a/turbinia/evidence.py b/turbinia/evidence.py
index 94f236a..f21ff4d 100644
--- a/turbinia/evidence.py
+++ b/turbinia/evidence.py
@@ -154,7 +154,7 @@ class Evidence:
         if that state is true.  This is used by the preprocessors to set the
         current state and Tasks can use this to determine if the Evidence is in
         the correct state for processing.
-    resource_tracked (bool): Evidence with this property set requires tracking 
+    resource_tracked (bool): Evidence with this property set requires tracking
         in a state file to allow for access amongst multiple workers.
     resource_id (str): The unique id used to track the state of a given Evidence
         type for stateful tracking.
diff --git a/turbinia/jobs/__init__.py b/turbinia/jobs/__init__.py
index e707b72..5a0d439 100644
--- a/turbinia/jobs/__init__.py
+++ b/turbinia/jobs/__init__.py
@@ -21,6 +21,7 @@ from turbinia.jobs import docker
 from turbinia.jobs import file_system_timeline
 from turbinia.jobs import finalize_request
 from turbinia.jobs import fsstat
+from turbinia.jobs import gitlab
 from turbinia.jobs import grep
 from turbinia.jobs import hadoop
 from turbinia.jobs import hindsight
diff --git a/turbinia/jobs/gitlab.py b/turbinia/jobs/gitlab.py
new file mode 100644
index 0000000..0932662
--- /dev/null
+++ b/turbinia/jobs/gitlab.py
@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+# Copyright 2022 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Job to execute the Gitlab task."""
+
+from turbinia.evidence import CompressedDirectory
+from turbinia.evidence import Directory
+from turbinia.evidence import DiskPartition
+from turbinia.evidence import DockerContainer
+from turbinia.evidence import ReportText
+from turbinia.jobs import interface
+from turbinia.jobs import manager
+from turbinia.workers import gitlab
+
+
+class GitlabJob(interface.TurbiniaJob):
+  """Gitlab job."""
+
+  evidence_input = [
+      Directory, DiskPartition, CompressedDirectory, DockerContainer
+  ]
+  evidence_output = [ReportText]
+
+  NAME = 'GitlabJob'
+
+  def create_tasks(self, evidence):
+    """Create task.
+
+    Args:
+      evidence: List of evidence objects to process
+    Returns:
+        A list of tasks to schedule.
+    """
+    tasks = [gitlab.GitlabTask() for _ in evidence]
+    return tasks
+
+
+manager.JobsManager.RegisterJob(GitlabJob)
diff --git a/turbinia/lib/utils.py b/turbinia/lib/utils.py
index fefb2a2..3e76ad8 100644
--- a/turbinia/lib/utils.py
+++ b/turbinia/lib/utils.py
@@ -112,6 +112,10 @@ def extract_files(file_name, disk_path, output_dir, credentials=[]):
   Raises:
     TurbiniaException: If an error occurs when running image_export.
   """
+  if not disk_path:
+    raise TurbiniaException(
+        'image_export.py failed: Attempted to run with no local_path')
+
   image_export_cmd = [
       'sudo', 'image_export.py', '--name', file_name, '--write', output_dir,
       '--partitions', 'all'
diff --git a/turbinia/task_utils.py b/turbinia/task_utils.py
index cf00004..268c575 100644
--- a/turbinia/task_utils.py
+++ b/turbinia/task_utils.py
@@ -49,6 +49,7 @@ class TaskLoader():
       'FileSystemTimelineTask',
       'FinalizeRequestTask',
       'FsstatTask',
+      'GitlabTask',
       'GrepTask',
       'HadoopAnalysisTask',
       'HindsightTask',
@@ -74,10 +75,10 @@ class TaskLoader():
 
   def check_task_name(self, task_name):
     """Checks whether a given task name is a valid task
-    
+
     Args:
       task_name(str): Name of the Task to check.
-      
+
     Returns:
       bool: True if task with the given name exists, else False
     """
@@ -88,10 +89,10 @@ class TaskLoader():
 
   def get_task(self, task_name):
     """Gets an instantiated Task object for the given name.
-    
+
     Args:
       task_name(str): Name of the Task to return.
-      
+
     Returns:
       TurbiniaTask: An instantiated Task object.
     """
@@ -116,6 +117,7 @@ class TaskLoader():
     from turbinia.workers.file_system_timeline import FileSystemTimelineTask
     from turbinia.workers.finalize_request import FinalizeRequestTask
     from turbinia.workers.fsstat import FsstatTask
+    from turbinia.workers.gitlab import GitlabTask
     from turbinia.workers.grep import GrepTask
     from turbinia.workers.hadoop import HadoopAnalysisTask
     from turbinia.workers.hindsight import HindsightTask
@@ -147,7 +149,7 @@ class TaskLoader():
 
   def get_task_names(self):
     """Returns a list of Task names.
-    
+
     Returns:
       (list) All Task names.
     """
diff --git a/turbinia/workers/artifact.py b/turbinia/workers/artifact.py
index 834d931..ea4e612 100644
--- a/turbinia/workers/artifact.py
+++ b/turbinia/workers/artifact.py
@@ -77,6 +77,13 @@ class FileArtifactExtractionTask(TurbiniaTask):
 
     # Path to the source image/directory.
     cmd.append(evidence.local_path)
+    if not evidence.local_path:
+      result.log('Tried to run image_export without local_path')
+      result.close(
+          self, False,
+          'image_export.py failed for artifact {0:s} - local_path not provided.'
+          .format(self.artifact_name))
+      return result
 
     result.log('Running image_export as [{0:s}]'.format(' '.join(cmd)))
 
diff --git a/turbinia/workers/file_system_timeline.py b/turbinia/workers/file_system_timeline.py
index f8af5ec..ee4fe5b 100644
--- a/turbinia/workers/file_system_timeline.py
+++ b/turbinia/workers/file_system_timeline.py
@@ -68,43 +68,45 @@ class FileSystemTimelineTask(TurbiniaTask):
     except dfvfs_errors.ScannerError as exception:
       status = 'Unable to open evidence: {0!s}'.format(exception)
       result.close(self, success=False, status=status)
+      return result
 
     # Iterate over all file entries and generate the output in bodyfile
     # format.
     try:
-      file_entries = None
-      with open(bodyfile_output, 'w', encoding='utf-8') as file_object:
-        file_entries = enumerate(entry_lister.ListFileEntries(base_path_specs))
-        while file_entries:
-          try:
-            _, (file_entry, path_segments) = next(file_entries)
-            bodyfile_entries = entry_lister.GetBodyfileEntries(
-                file_entry, path_segments)
-            for bodyfile_entry in bodyfile_entries:
-              file_object.write(bodyfile_entry)
-              file_object.write('\n')
-              number_of_entries += 1
-          except StopIteration:
-            break
-          except (dfvfs_errors.AccessError, dfvfs_errors.BackEndError,
-                  dfvfs_errors.MountPointError,
-                  dfvfs_errors.PathSpecError) as exception:
-            status = 'Unable to process file entry: {0!s}'.format(exception)
-            result.log(status)
-
-      if number_of_entries > 0:
-        output_evidence.number_of_entries = number_of_entries
-        result.add_evidence(output_evidence, evidence.config)
-        status = 'Generated file system timeline containing [{0:d}] entries'.format(
-            number_of_entries)
-        result.close(self, success=True, status=status)
-      else:
-        status = 'Unable to process any file entries.'
-        result.close(self, success=False, status=status)
-
+      bodyfile_fileobj = open(bodyfile_output, 'w', encoding='utf-8')
     except IOError as exception:
-      status = 'Unable to create bodyfile local output file: {0!s}'.format(
-          exception)
+      status = 'Failed to open local output file: {0!s}'.format(exception)
+      result.close(self, success=False, status=status)
+      return result
+
+    file_entries = enumerate(entry_lister.ListFileEntries(base_path_specs))
+    while file_entries:
+      try:
+        _, (file_entry, path_segments) = next(file_entries)
+        bodyfile_entries = entry_lister.GetBodyfileEntries(
+            file_entry, path_segments)
+        for bodyfile_entry in bodyfile_entries:
+          bodyfile_fileobj.write(bodyfile_entry)
+          bodyfile_fileobj.write('\n')
+          number_of_entries += 1
+      except StopIteration:
+        break
+      except (dfvfs_errors.AccessError, dfvfs_errors.BackEndError,
+              dfvfs_errors.MountPointError, dfvfs_errors.PathSpecError,
+              IOError) as exception:
+        status = 'Unable to process file entry: {0!s}'.format(exception)
+        result.log(status)
+
+    bodyfile_fileobj.close()
+
+    if number_of_entries > 0:
+      output_evidence.number_of_entries = number_of_entries
+      result.add_evidence(output_evidence, evidence.config)
+      status = 'Generated file system timeline containing [{0:d}] entries'.format(
+          number_of_entries)
+      result.close(self, success=True, status=status)
+    else:
+      status = 'Unable to process any file entries.'
       result.close(self, success=False, status=status)
 
     return result
diff --git a/turbinia/workers/gitlab.py b/turbinia/workers/gitlab.py
new file mode 100644
index 0000000..ab463ec
--- /dev/null
+++ b/turbinia/workers/gitlab.py
@@ -0,0 +1,163 @@
+# -*- coding: utf-8 -*-
+# Copyright 2022 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Task for analysing Gitlab instances."""
+
+from __future__ import unicode_literals
+
+import os
+import glob
+
+from turbinia.evidence import EvidenceState as state
+from turbinia.evidence import ReportText
+from turbinia.lib import text_formatter as fmt
+from turbinia.workers import TurbiniaTask
+from turbinia.workers import Priority
+
+
+class GitlabTask(TurbiniaTask):
+  """Task to analyze Gitlabs."""
+
+  REQUIRED_STATES = [
+      state.ATTACHED, state.CONTAINER_MOUNTED, state.DECOMPRESSED
+  ]
+
+  def run(self, evidence, result):
+    """Run the Gitlab worker.
+
+    Args:
+        evidence (Evidence object):  The evidence we will process.
+        result (TurbiniaTaskResult): The object to place task results into.
+
+    Returns:
+        TurbiniaTaskResult object.
+    """
+    # Where to store the resulting output file.
+    output_file_name = 'gitlab_analysis.txt'
+    output_file_path = os.path.join(self.output_dir, output_file_name)
+    # Set the output file as the data source for the output evidence.
+    output_evidence = ReportText(source_path=output_file_path)
+
+    reports = []
+    summaries = []
+
+    # Grep for exif in workhorse logs
+    (r, priority, s) = self._is_exif_in_logs(
+        result, evidence.local_path,
+        os.path.join('var', 'log', 'gitlab', 'workhorse.log'))
+    if r != '':
+      reports.append(r)
+    if s != '':
+      summaries.append(s)
+
+    (r, p2, s) = self._is_exif_in_logs(
+        result, evidence.local_path,
+        os.path.join('var', 'log', 'gitlab', 'gitlab-workhorse', '@*'))
+    if r != '':
+      reports.append(r)
+    if s != '':
+      summaries.append(s)
+    if p2 < priority:
+      priority = p2
+
+    # TODO: Check for Metasploit Module
+    # 'https://packetstormsecurity.com/files/160441/GitLab-File-Read-Remote-Code-Execution.html'
+
+    (r, p3, s) = self._is_traversal_in_logs(
+        result, evidence.local_path,
+        os.path.join('var', 'log', 'gitlab', 'nginx', '*access*'))
+    if r != '':
+      reports.append(r)
+    if s != '':
+      summaries.append(s)
+    if p3 < priority:
+      priority = p3
+
+    if priority == priority.LOW:
+      result.close(self, success=True, status='No Gitlab exploitation found')
+      return result
+
+    report = " ".join(reports)
+    summary = " ".join(summaries)
+
+    output_evidence.text_data = report
+    result.report_priority = priority
+    result.report_data = report
+
+    # Write the report to the output file.
+    with open(output_file_path, 'wb') as fh:
+      fh.write(output_evidence.text_data.encode('utf-8'))
+
+    # Add the resulting evidence to the result object.
+    result.add_evidence(output_evidence, evidence.config)
+    result.close(self, success=True, status=summary)
+    return result
+
+  def _is_exif_in_logs(self, result, basedir, logfiles):
+    """Checks to see if there is evidence of the exiftool exploit in the logs.
+
+    Args:
+      result (TurbiniaTaskResult): The object to place task results into.
+      basedir (str): the root of the evidence.
+      logfiles (str): The file(s) to check
+
+    Returns:
+      Tuple(
+        report_text(str): The report data
+        report_priority(int): The priority of the report (0 - 100)
+        summary(str): A summary of the report (used for task status)
+      )
+    """
+
+    check = " ".join(glob.glob(os.path.join(basedir, logfiles)))
+    if not len(check):
+      return ('', Priority.LOW, '')
+
+    cmd = ['zgrep', '"exiftool command failed"', check]
+    ret, result = self.execute(cmd, result, success_codes=[0, 1])
+    if ret == 0:
+      summary = 'exif exploit detected in {0:s}'.format(logfiles)
+      report = fmt.heading4(fmt.bold(summary))
+      return (report, Priority.HIGH, summary)
+
+    return ('', Priority.LOW, '')
+
+  def _is_traversal_in_logs(self, result, basedir, logfiles):
+    """Checks to see if there is evidence of directory traversal in the logs.
+
+    Args:
+      result (TurbiniaTaskResult): The object to place task results into.
+      basedir (str): the root of the evidence.
+      logfiles (str): The file(s) to check
+
+    Returns:
+      Tuple(
+        report_text(str): The report data
+        report_priority(int): The priority of the report (0 - 100)
+        summary(str): A summary of the report (used for task status)
+      )
+    """
+
+    check = " ".join(glob.glob(os.path.join(basedir, logfiles)))
+    if not len(check):
+      return ('', Priority.LOW, '')
+
+    cmd = ['zgrep', '"%2F..%2F..%2F..%2F"', check]
+    ret, result = self.execute(cmd, result, success_codes=[0, 1])
+    if ret == 0:
+      summary = 'directory traversal exploit detected in {}'.format(logfiles)
+      report = fmt.heading4(fmt.bold(summary))
+      return (report, Priority.HIGH, summary)
+
+    return ('', Priority.LOW, '')


Test patch:
diff --git a/test_data/var/log/gitlab/workhorse.log b/test_data/var/log/gitlab/workhorse.log
new file mode 100644
index 0000000..efcc4f9
--- /dev/null
+++ b/test_data/var/log/gitlab/workhorse.log
@@ -0,0 +1,1 @@
+time="2022-01-25T21:32:33Z" level=info msg="exiftool command failed" command="[exiftool -all= --IPTC:all --XMP-iptcExt:all -tagsFromFile @ -ResolutionUnit -XResolution -YResolution -YCbCrSubSampling -YCbCrPositioning -BitsPerSample -ImageHeight -ImageWidth -ImageSize -Copyright -CopyrightNotice -Orientation -]" error="exit status 1" stderr="  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n                                 Dload  Upload   Total   Spent    Left  Speed\n\r  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\r100   600  100   600    0     0   2933      0 --:--:-- --:--:-- --:--:--  2941\nsh: 1: wget: not found\nsh: 1: rsync: not found\nError: Writing of this type of file is not supported - -\n"
diff --git a/turbinia/workers/analysis/loki_test.py b/turbinia/workers/analysis/loki_test.py
index 0e12e4f..cff3782 100644
--- a/turbinia/workers/analysis/loki_test.py
+++ b/turbinia/workers/analysis/loki_test.py
@@ -25,7 +25,7 @@ from turbinia.workers.workers_test import TestTurbiniaTaskBase
 
 
 class LokiAnalysisTaskTest(TestTurbiniaTaskBase):
-  """Tests for LokisAnalysisTask Task."""
+  """Tests for LokiAnalysisTask Task."""
 
   LOKI_SUMMARY = 'Loki analysis found 1 alert(s)'
 
diff --git a/turbinia/workers/gitlab_test.py b/turbinia/workers/gitlab_test.py
new file mode 100644
index 0000000..9b863ca
--- /dev/null
+++ b/turbinia/workers/gitlab_test.py
@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+# Copyright 2022 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for the Gitlab task."""
+
+import os
+import shutil
+import unittest
+
+from turbinia import config
+from turbinia.workers import gitlab
+from turbinia.workers import TurbiniaTaskResult
+from turbinia.workers.workers_test import TestTurbiniaTaskBase
+
+
+class GitlabTaskTest(TestTurbiniaTaskBase):
+  """Tests for the Gitlab Task."""
+
+  SUMMARY = '#### **exif exploit detected in var/log/gitlab/workhorse.log**'
+  TEST_DATA = None
+
+  def setUp(self):
+    super(GitlabTaskTest, self).setUp(task_class=gitlab.GitlabTask)
+    self.setResults(mock_run=False)
+    filedir = os.path.dirname(os.path.realpath(__file__))
+    self.evidence.local_path = os.path.join(filedir, '..', '..', 'test_data')
+    self.task.output_dir = self.task.base_output_dir
+
+  def testGitlabRun(self):
+    """Test Gitlab task run."""
+    config.LoadConfig()
+    result = self.task.run(self.evidence, self.result)
+
+    self.assertIsInstance(result, TurbiniaTaskResult)
+
+    self.assertEqual(result.report_priority, 20)
+    self.assertEqual(result.report_data, self.SUMMARY)
+
+  def tearDown(self):
+    if os.path.exists(self.base_output_dir):
+      shutil.rmtree(self.base_output_dir)
+
+
+if __name__ == '__main__':
+  unittest.main()