Repository: google/turbinia
Pull request #400 (2019-06-05 14:19:50)

Unpin dependencies


Patch:
diff --git a/requirements.txt b/requirements.txt
index 51eef82..4441358 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,10 +1,10 @@
 filelock
-google-api-core>=1.2.1,<=1.4.0
-google-cloud-datastore==1.7.0
-google-api-python-client>=1.6.2
-google-cloud-pubsub==0.35.4
-google-cloud-storage==1.13.0
-google-cloud-core==0.28.1
+google-api-core
+google-cloud-datastore
+google-api-python-client
+google-cloud-pubsub
+google-cloud-storage
+google-cloud-core
 oauth2client
 psq
 six
diff --git a/turbinia/client.py b/turbinia/client.py
index 6458ebe..485a1d2 100644
--- a/turbinia/client.py
+++ b/turbinia/client.py
@@ -162,7 +162,8 @@ class TurbiniaClient(object):
       user (string): The user of the request we want tasks for.
       poll_interval (int): Interval of seconds between polling cycles.
     """
-    last_count = 0
+    last_completed_count = -1
+    last_uncompleted_count = -1
     while True:
       task_results = self.get_task_data(
           instance, project, region, request_id=request_id, user=user)
@@ -186,12 +187,14 @@ class TurbiniaClient(object):
           'Tasks completed ({0:d}/{1:d}): [{2:s}], waiting for [{3:s}].'.format(
               len(completed_tasks), total_count, completed_names,
               uncompleted_names))
-      if len(completed_tasks) > last_count:
+      if (len(completed_tasks) > last_completed_count or
+          len(uncompleted_tasks) > last_uncompleted_count):
         log.info(msg)
       else:
         log.debug(msg)
 
-      last_count = len(completed_tasks)
+      last_completed_count = len(completed_tasks)
+      last_uncompleted_count = len(uncompleted_tasks)
       time.sleep(poll_interval)
 
     log.info('All {0:d} Tasks completed'.format(len(task_results)))
diff --git a/turbinia/jobs/jenkins.py b/turbinia/jobs/jenkins.py
index 514a85f..c3a961b 100644
--- a/turbinia/jobs/jenkins.py
+++ b/turbinia/jobs/jenkins.py
@@ -22,6 +22,7 @@ from turbinia.evidence import GoogleCloudDisk
 from turbinia.evidence import GoogleCloudDiskRawEmbedded
 from turbinia.evidence import ReportText
 from turbinia.jobs import interface
+from turbinia.jobs import manager
 from turbinia.workers.analysis.jenkins import JenkinsAnalysisTask
 
 
@@ -46,3 +47,6 @@ class JenkinsAnalysisJob(interface.TurbiniaJob):
     """
     tasks = [JenkinsAnalysisTask() for _ in evidence]
     return tasks
+
+
+manager.JobsManager.RegisterJob(JenkinsAnalysisJob)
diff --git a/turbinia/lib/utils.py b/turbinia/lib/utils.py
index f943145..a1930cf 100644
--- a/turbinia/lib/utils.py
+++ b/turbinia/lib/utils.py
@@ -27,6 +27,37 @@ from turbinia import TurbiniaException
 log = logging.getLogger('turbinia')
 
 
+def _image_export(command, output_dir):
+  """Runs image_export command.
+
+  Args:
+    file_name: Name of file (without path) to be extracted.
+    output_dir: Path to directory to store the the extracted files.
+
+  Returns:
+    dict: file names and paths to extracted files.
+
+  Raises:
+    TurbiniaException: If an error occurs when running image_export.
+  """
+  # TODO: Consider using the exec helper to gather stdin/err.
+  log.debug('Running image_export as [{0:s}]'.format(' '.join(command)))
+  try:
+    subprocess.check_call(command)
+  except subprocess.CalledProcessError:
+    raise TurbiniaException('image_export.py failed.')
+
+  collected_file_paths = []
+  file_count = 0
+  for dirpath, _, filenames in os.walk(output_dir):
+    for filename in filenames:
+      collected_file_paths.append(os.path.join(dirpath, filename))
+      file_count += 1
+
+  log.debug('Collected {0:d} files with image_export'.format(file_count))
+  return collected_file_paths
+
+
 def extract_artifacts(artifact_names, disk_path, output_dir):
   """Extract artifacts using image_export from Plaso.
 
@@ -37,29 +68,39 @@ def extract_artifacts(artifact_names, disk_path, output_dir):
 
   Returns:
     dict: file names and paths to extracted files.
+
+  Raises:
+    TurbiniaException: If an error occurs when running image_export.
   """
   # Plaso image_export expects artifact names as a comma separated string.
   artifacts = ','.join(artifact_names)
-
   image_export_cmd = [
       'image_export.py', '--artifact_filters', artifacts, '--write', output_dir,
       '--partitions', 'all', disk_path
   ]
 
-  # TODO: Consider using the exec helper to gather stdin/err.
-  log.debug(
-      'Running image_export as [{0:s}]'.format(' '.join(image_export_cmd)))
-  try:
-    subprocess.check_call(image_export_cmd)
-  except subprocess.CalledProcessError:
-    raise TurbiniaException('image_export.py failed.')
+  return _image_export(image_export_cmd, output_dir)
 
-  collected_file_paths = []
-  for dirpath, _, filenames in os.walk(output_dir):
-    for filename in filenames:
-      collected_file_paths.append(os.path.join(dirpath, filename))
 
-  return collected_file_paths
+def extract_files(file_name, disk_path, output_dir):
+  """Extract files using image_export from Plaso.
+
+  Args:
+    file_name: Name of file (without path) to be extracted.
+    disk_path: Path to either a raw disk image or a block device.
+    output_dir: Path to directory to store the the extracted files.
+
+  Returns:
+    dict: file names and paths to extracted files.
+
+  Raises:
+    TurbiniaException: If an error occurs when running image_export.
+  """
+  image_export_cmd = [
+      'image_export.py', '--name', file_name, '--write', output_dir,
+      '--partitions', 'all', disk_path
+  ]
+  return _image_export(image_export_cmd, output_dir)
 
 
 def bruteforce_password_hashes(password_hashes, timeout=300):
diff --git a/turbinia/workers/analysis/jenkins.py b/turbinia/workers/analysis/jenkins.py
index e9a2be3..2eaad94 100644
--- a/turbinia/workers/analysis/jenkins.py
+++ b/turbinia/workers/analysis/jenkins.py
@@ -24,7 +24,7 @@ from turbinia.evidence import ReportText
 from turbinia.lib import text_formatter as fmt
 from turbinia.workers import TurbiniaTask
 from turbinia.workers import Priority
-from turbinia.lib.utils import extract_artifacts
+from turbinia.lib.utils import extract_files
 from turbinia.lib.utils import bruteforce_password_hashes
 
 
@@ -51,17 +51,26 @@ class JenkinsAnalysisTask(TurbiniaTask):
     # Set the output file as the data source for the output evidence.
     output_evidence.local_path = output_file_path
 
+    # TODO(aarontp): We should find a more optimal solution for this because
+    # this requires traversing the entire filesystem and extracting more files
+    # than we need.  Tracked in https://github.com/google/turbinia/issues/402
     try:
-      collected_artifacts = extract_artifacts(
-          artifact_names=['JenkinsConfigFile'], disk_path=evidence.local_path,
+      collected_artifacts = extract_files(
+          file_name='config.xml', disk_path=evidence.local_path,
           output_dir=os.path.join(self.output_dir, 'artifacts'))
     except TurbiniaException as e:
       result.close(self, success=False, status=str(e))
       return result
 
+    jenkins_artifacts = []
+    jenkins_re = re.compile(r'^.*jenkins[^\/]*(\/users\/[^\/]+)*\/config\.xml$')
+    for collected_artifact in collected_artifacts:
+      if re.match(jenkins_re, collected_artifact):
+        jenkins_artifacts.append(collected_artifact)
+
     version = None
     credentials = []
-    for filepath in collected_artifacts:
+    for filepath in jenkins_artifacts:
       with open(filepath, 'r') as input_file:
         config = input_file.read()
 
@@ -169,9 +178,14 @@ class JenkinsAnalysisTask(TurbiniaTask):
         line = 'User "{0:s}" with password "{1:s}"'.format(
             credentials_registry.get(password_hash), plaintext)
         report.append(fmt.bullet(line, level=2))
+    elif credentials_registry or version != 'Unknown':
+      summary = (
+          'Jenkins version {0:s} found with {1:d} credentials, but no issues '
+          'detected'.format(version, len(credentials_registry)))
+      report.insert(0, fmt.heading4(summary))
+      priority = Priority.MEDIUM
     else:
-      summary = 'Jenkins analysis found no issues'
-      priority = Priority.LOW
+      summary = 'No Jenkins instance found'
       report.insert(0, fmt.heading4(summary))
 
     report = '\n'.join(report)


Test patch:
diff --git a/turbinia/workers/analysis/jenkins_test.py b/turbinia/workers/analysis/jenkins_test.py
index 53850d9..cba7245 100644
--- a/turbinia/workers/analysis/jenkins_test.py
+++ b/turbinia/workers/analysis/jenkins_test.py
@@ -111,7 +111,7 @@ class JenkinsAnalysisTaskTest(unittest.TestCase):
     * User \"admin\" with password \"weakpassword\"
 """
 
-  JENKINS_ANALYSIS_EMPTY_REPORT = """#### Jenkins analysis found no issues
+  JENKINS_ANALYSIS_EMPTY_REPORT = """#### Jenkins version 2.121.2 found with 1 credentials, but no issues detected
 * Jenkins version: 2.121.2"""
 
   EXPECTED_VERSION = '2.121.2'
@@ -166,8 +166,11 @@ class JenkinsAnalysisTaskTest(unittest.TestCase):
         self.EXPECTED_VERSION, self.EXPECTED_CREDENTIALS)
 
     self.assertEqual(report, self.JENKINS_ANALYSIS_EMPTY_REPORT)
-    self.assertEqual(priority, 80)
-    self.assertEqual(summary, 'Jenkins analysis found no issues')
+    self.assertEqual(priority, 50)
+    self.assertEqual(
+        summary,
+        'Jenkins version 2.121.2 found with 1 credentials, but no issues '
+        'detected')
 
 
 if __name__ == '__main__':