Repository: google/turbinia
Pull request #226 (2018-08-10 14:26:01)

Jenkins analysis
Add analysis plugin for Jenkins. The plugin will:
* Extract Jenkins config files from artifact definition and image_export
* Extract version and credentials from Jenkins config files
* Try to find weak passwords by bruteforcing with John the Ripper

**Example output:**

Jenkins analysis found potential issues.

Jenkins version: 2.121.2
1 weak password(s) found:
 - User "admin2" with password "123456"

aarontp: Thanks!  I didn't get a chance to look today, but I'll review this tomorrow.
berggren: Codefactor don't let me add TODOs.
aarontp: Ping.  Can you take a look at the latest comments?  I think the only thing I was hoping to address was the status, and I had a question about whether we could see more than one user in these files.  Thanks!
aarontp: Actually, since I think you might be busy today, I'll ping you about just merging this, and then I can make a small change to update the status (and we can address the user issue later).

Patch:
diff --git a/turbinia/jobs/__init__.py b/turbinia/jobs/__init__.py
index e0884d2..2d973e7 100644
--- a/turbinia/jobs/__init__.py
+++ b/turbinia/jobs/__init__.py
@@ -35,12 +35,14 @@ def get_jobs():
   from turbinia.jobs.tomcat import TomcatAnalysisJob
   from turbinia.jobs.analysis.http_access_logs import HTTPAccessLogExtractionJob
   from turbinia.jobs.analysis.http_access_logs import HTTPAccessLogAnalysisJob
+  from turbinia.jobs.analysis.jenkins import JenkinsAnalysisJob
   # TODO(aarontp): Dynamically look up job objects and make enabling/disabling
   # configurable through config and/or recipes.
   return [
       StatJob(), PlasoJob(), PsortJob(), StringsJob(), GrepJob(),
       SSHDExtractionJob(), SSHDAnalysisJob(), HTTPAccessLogExtractionJob(),
-      HTTPAccessLogAnalysisJob(), TomcatExtractionJob(), TomcatAnalysisJob()]
+      HTTPAccessLogAnalysisJob(), TomcatExtractionJob(), TomcatAnalysisJob(),
+      JenkinsAnalysisJob()]
 
 
 class TurbiniaJob(object):
diff --git a/turbinia/jobs/analysis/jenkins.py b/turbinia/jobs/analysis/jenkins.py
new file mode 100644
index 0000000..e96ea38
--- /dev/null
+++ b/turbinia/jobs/analysis/jenkins.py
@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Job to execute Jenkins analysis task."""
+
+from __future__ import unicode_literals
+
+from turbinia.evidence import Directory
+from turbinia.evidence import RawDisk
+from turbinia.evidence import GoogleCloudDisk
+from turbinia.evidence import GoogleCloudDiskRawEmbedded
+from turbinia.evidence import ReportText
+from turbinia.jobs import TurbiniaJob
+from turbinia.workers.analysis.jenkins import JenkinsAnalysisTask
+
+
+class JenkinsAnalysisJob(TurbiniaJob):
+  """Jenkins analysis job."""
+
+  # Types of evidence that this Job will process.
+  evidence_input = [
+    Directory, RawDisk, GoogleCloudDisk, GoogleCloudDiskRawEmbedded]
+  evidence_output = [ReportText]
+
+  def __init__(self):
+    super(JenkinsAnalysisJob, self).__init__(name='JenkinsAnalysisJob')
+
+  def create_tasks(self, evidence):
+    """Create task for Jenkins analysis job.
+
+    Args:
+      evidence: List of evidence object to process
+
+    Returns:
+        A list of tasks to schedule.
+    """
+    tasks = [JenkinsAnalysisTask() for _ in evidence]
+    return tasks
diff --git a/turbinia/lib/utils.py b/turbinia/lib/utils.py
new file mode 100644
index 0000000..ff44fc4
--- /dev/null
+++ b/turbinia/lib/utils.py
@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Common utils."""
+
+from __future__ import unicode_literals
+
+import os
+import subprocess
+import tempfile
+import threading
+
+from turbinia import TurbiniaException
+
+
+def extract_artifacts(artifact_names, disk_path, output_dir):
+  """Extract artifacts using image_export from Plaso.
+
+  Args:
+    artifact_names: List of artifact definition names.
+    disk_path: Path to either a raw disk image or a block device.
+    output_dir: Path to directory to store the the extracted files.
+
+  Returns:
+    dict: file names and paths to extracted files.
+  """
+  # Plaso image_export expects artifact names as a comma separated string.
+  artifacts = ','.join(artifact_names)
+
+  image_export_cmd = [
+    'image_export.py',
+    '--artifact_filters', artifacts,
+    '--write', output_dir,
+    disk_path
+  ]
+
+  # TODO: Consider break the exec helper to gather stdin/err.
+  try:
+    subprocess.check_call(image_export_cmd)
+  except subprocess.CalledProcessError:
+    raise TurbiniaException('image_export.py failed.')
+
+  collected_file_paths = []
+  for dirpath, _, filenames in os.walk(output_dir):
+    for filename in filenames:
+      collected_file_paths.append(os.path.join(dirpath, filename))
+
+  return collected_file_paths
+
+
+def bruteforce_password_hashes(password_hashes, timeout=300):
+  """Bruteforce password hashes using John the Ripper.
+
+  Args:
+    password_hashes (list): Password hashes as strings.
+    timeout (int): Number of seconds to run for before terminating the process.
+
+  Returns:
+    list: of tuples with hashes and plain text passwords.
+
+  Raises:
+    TurbiniaException if execution failed.
+  """
+
+  with tempfile.NamedTemporaryFile(delete=False) as fh:
+    password_hashes_file_path = fh.name
+    fh.write('\n'.join(password_hashes))
+
+  cmd = ['john', password_hashes_file_path]
+
+  with open(os.devnull, 'w') as devnull:
+    try:
+      child = subprocess.Popen(cmd, stdout=devnull, stderr=devnull)
+      timer = threading.Timer(timeout, child.terminate)
+      timer.start()
+      child.communicate()
+      # Cancel the timer if the process is done before the timer.
+      if timer.is_alive():
+        timer.cancel()
+    except OSError:
+      raise TurbiniaException('john the ripper failed.')
+
+  result = []
+  # Default location of the result file, no way to change it.
+  pot_file = os.path.expanduser('~/.john/john.pot')
+
+  if os.path.isfile(pot_file):
+    with open(pot_file, 'r') as fh:
+      for line in fh.readlines():
+        password_hash, plaintext = line.rsplit(':', 1)
+        result.append((password_hash, plaintext.rstrip()))
+    os.remove(pot_file)
+
+  return result
diff --git a/turbinia/workers/analysis/jenkins.py b/turbinia/workers/analysis/jenkins.py
new file mode 100644
index 0000000..1684598
--- /dev/null
+++ b/turbinia/workers/analysis/jenkins.py
@@ -0,0 +1,162 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Task for analysing Jenkins."""
+
+from __future__ import unicode_literals
+
+import os
+import re
+
+from turbinia import TurbiniaException
+from turbinia.evidence import ReportText
+from turbinia.workers import TurbiniaTask
+from turbinia.lib.utils import extract_artifacts
+from turbinia.lib.utils import bruteforce_password_hashes
+
+
+class JenkinsAnalysisTask(TurbiniaTask):
+  """Task to analyze a Jenkins install."""
+
+  def run(self, evidence, result):
+    """Run the Jenkins worker.
+
+    Args:
+        evidence (Evidence object):  The evidence to process
+        result (TurbiniaTaskResult): The object to place task results into.
+
+    Returns:
+        TurbiniaTaskResult object.
+    """
+    # What type of evidence we should output.
+    output_evidence = ReportText()
+
+    # Where to store the resulting output file.
+    output_file_name = 'jenkins_analysis.txt'
+    output_file_path = os.path.join(self.output_dir, output_file_name)
+
+    # Set the output file as the data source for the output evidence.
+    output_evidence.local_path = output_file_path
+
+    try:
+      collected_artifacts = extract_artifacts(
+        artifact_names=['JenkinsConfigFile'],
+        disk_path=evidence.local_path,
+        output_dir=os.path.join(self.output_dir, 'artifacts')
+      )
+    except TurbiniaException as e:
+      result.close(self, success=False, status=str(e))
+      return result
+
+    version = None
+    credentials = []
+    for filepath in collected_artifacts:
+        with open(filepath, 'r') as input_file:
+          config = input_file.read()
+
+        extracted_version = self._extract_jenkins_version(config)
+        extracted_credentials = self._extract_jenkins_credentials(config)
+
+        if extracted_version:
+          version = extracted_version
+
+        credentials.extend(extracted_credentials)
+
+    analysis_report = self.analyze_jenkins(version, credentials)
+    output_evidence.text_data = analysis_report
+
+    # Write the report to the output file.
+    with open(output_file_path, 'w') as fh:
+      fh.write(output_evidence.text_data.encode('utf8'))
+      fh.write('\n'.encode('utf8'))
+
+    # Add the resulting evidence to the result object.
+    result.add_evidence(output_evidence, evidence.config)
+    result.close(self, success=True)
+
+    return result
+
+  @staticmethod
+  def _extract_jenkins_version(config):
+    """Extract version from Jenkins configuration files.
+
+    Args:
+      config (str): configuration file content.
+
+    Returns:
+      str: The version of Jenkins.
+    """
+    version = None
+    version_re = re.compile('<version>(.*)</version>')
+    version_match = re.search(version_re, config)
+
+    if version_match:
+      version = version_match.group(1)
+
+    return version
+
+  @staticmethod
+  def _extract_jenkins_credentials(config):
+    """Extract credentials from Jenkins configuration files.
+
+    Args:
+      config (str): configuration file content.
+
+    Returns:
+      list: of tuples with username and password hash.
+    """
+    credentials = []
+    password_hash_re = re.compile('<passwordHash>#jbcrypt:(.*)</passwordHash>')
+    username_re = re.compile('<fullName>(.*)</fullName>')
+
+    password_hash_match = re.search(password_hash_re, config)
+    username_match = re.search(username_re, config)
+
+    if username_match and password_hash_match:
+      username = username_match.group(1)
+      password_hash = password_hash_match.group(1)
+      credentials.append((username, password_hash))
+
+    return credentials
+
+  @staticmethod
+  def analyze_jenkins(version, credentials):
+    """Analyses a Jenkins configuration.
+
+    Args:
+      version (str): Version of Jenkins.
+      credentials (list): of tuples with username and password hash.
+
+    Returns:
+      str: of description of security of Jenkins configuration file.
+    """
+    findings = []
+    credentials_registry = {hash: username for username, hash in credentials}
+    # TODO: Add timeout parameter when dynamic configuration is ready.
+    # Ref: https://github.com/google/turbinia/issues/244
+    weak_passwords = bruteforce_password_hashes(credentials_registry.keys())
+
+    if not version:
+      version = 'Unknown'
+    findings.append('Jenkins version: {0:s}'.format(version))
+
+    if weak_passwords:
+      findings.insert(0, 'Jenkins analysis found potential issues.\n')
+      findings.append('{0:n} weak password(s) found:'.format(len(weak_passwords)))
+      for password_hash, plaintext in weak_passwords:
+        findings.append(
+          ' - User "{0:s}" with password "{1:s}"'.format(
+            credentials_registry.get(password_hash), plaintext))
+
+    return '\n'.join(findings)
diff --git a/turbinia/workers/analysis/wordpress.py b/turbinia/workers/analysis/wordpress.py
index 3e12945..25ff812 100644
--- a/turbinia/workers/analysis/wordpress.py
+++ b/turbinia/workers/analysis/wordpress.py
@@ -112,7 +112,7 @@ class WordpressAccessLogAnalysisTask(TurbiniaTask):
 
     if findings:
       findings.insert(0, 'Wordpress access logs found ({0:s})'.format(
-          ', '.join(list(findings_summary))))
+          ', '.join(sorted(list(findings_summary)))))
       return '\n'.join(findings)
 
     return 'No Wordpress install or theme editing found in access logs'


Test patch:
diff --git a/turbinia/workers/analysis/jenkins_test.py b/turbinia/workers/analysis/jenkins_test.py
new file mode 100644
index 0000000..c524f83
--- /dev/null
+++ b/turbinia/workers/analysis/jenkins_test.py
@@ -0,0 +1,131 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for the Jenkins analysis task."""
+
+from __future__ import unicode_literals
+
+import unittest
+
+from turbinia import config
+from turbinia.workers.analysis import jenkins
+
+
+class JenkinsAnalysisTaskTest(unittest.TestCase):
+
+  JENKINS_SYSTEM_CONFIG = """<?xml version='1.1' encoding='UTF-8'?>
+  <hudson>
+    <disabledAdministrativeMonitors/>
+    <version>2.121.2</version>
+    <installStateName>NEW</installStateName>
+    <numExecutors>2</numExecutors>
+    <mode>NORMAL</mode>
+    <useSecurity>true</useSecurity>
+    <authorizationStrategy class="hudson.security.FullControlOnceLoggedInAuthorizationStrategy">
+      <denyAnonymousReadAccess>true</denyAnonymousReadAccess>
+    </authorizationStrategy>
+    <securityRealm class="hudson.security.HudsonPrivateSecurityRealm">
+      <disableSignup>true</disableSignup>
+      <enableCaptcha>false</enableCaptcha>
+    </securityRealm>
+    <disableRememberMe>false</disableRememberMe>
+    <projectNamingStrategy class="jenkins.model.ProjectNamingStrategy$DefaultProjectNamingStrategy"/>
+    <workspaceDir>${JENKINS_HOME}/workspace/${ITEM_FULL_NAME}</workspaceDir>
+    <buildsDir>${ITEM_ROOTDIR}/builds</buildsDir>
+    <jdks/>
+    <viewsTabBar class="hudson.views.DefaultViewsTabBar"/>
+    <myViewsTabBar class="hudson.views.DefaultMyViewsTabBar"/>
+    <clouds/>
+    <scmCheckoutRetryCount>0</scmCheckoutRetryCount>
+    <views>
+      <hudson.model.AllView>
+        <owner class="hudson" reference="../../.."/>
+        <name>all</name>
+        <filterExecutors>false</filterExecutors>
+        <filterQueue>false</filterQueue>
+        <properties class="hudson.model.View$PropertyList"/>
+      </hudson.model.AllView>
+    </views>
+    <primaryView>all</primaryView>
+    <slaveAgentPort>-1</slaveAgentPort>
+    <disabledAgentProtocols>
+      <string>JNLP-connect</string>
+      <string>JNLP2-connect</string>
+    </disabledAgentProtocols>
+    <label></label>
+    <crumbIssuer class="hudson.security.csrf.DefaultCrumbIssuer">
+      <excludeClientIPFromCrumb>false</excludeClientIPFromCrumb>
+    </crumbIssuer>
+    <nodeProperties/>
+    <globalNodeProperties/>
+  </hudson>
+  """
+
+  JENKINS_USER_CONFIG = """<?xml version='1.1' encoding='UTF-8'?>
+  <user>
+    <fullName>admin</fullName>
+    <properties>
+      <jenkins.security.ApiTokenProperty>
+        <apiToken>{AQAAABAAAAAw2v+nvot+GLKWtq30DPlsk2lTEST==}</apiToken>
+      </jenkins.security.ApiTokenProperty>
+      <hudson.model.MyViewsProperty>
+        <views>
+          <hudson.model.AllView>
+            <owner class="hudson.model.MyViewsProperty" reference="../../.."/>
+            <name>all</name>
+            <filterExecutors>false</filterExecutors>
+            <filterQueue>false</filterQueue>
+            <properties class="hudson.model.View$PropertyList"/>
+          </hudson.model.AllView>
+        </views>
+      </hudson.model.MyViewsProperty>
+      <hudson.model.PaneStatusProperties>
+        <collapsed/>
+      </hudson.model.PaneStatusProperties>
+      <hudson.search.UserSearchProperty>
+        <insensitiveSearch>true</insensitiveSearch>
+      </hudson.search.UserSearchProperty>
+      <hudson.security.HudsonPrivateSecurityRealm_-Details>
+        <passwordHash>#jbcrypt:$2a$10$DSltvO4YXZuoLuUU77R871627TEST</passwordHash>
+      </hudson.security.HudsonPrivateSecurityRealm_-Details>
+    </properties>
+  </user>
+  """
+
+  EXPECTED_VERSION = '2.121.2'
+  EXPECTED_CREDENTIALS = [('admin', '$2a$10$DSltvO4YXZuoLuUU77R871627TEST')]
+
+  def test_extract_jenkins_version(self):
+    """Tests the extract_jenkins_version method."""
+    config.LoadConfig()
+    task = jenkins.JenkinsAnalysisTask()
+
+    version = task._extract_jenkins_version(
+      str(self.JENKINS_SYSTEM_CONFIG))
+
+    self.assertEqual(version, self.EXPECTED_VERSION)
+
+  def test_extract_jenkins_credentials(self):
+    """Tests the extract_jenkins_credentials method."""
+    config.LoadConfig()
+    task = jenkins.JenkinsAnalysisTask()
+
+    credentials = task._extract_jenkins_credentials(
+      str(self.JENKINS_USER_CONFIG))
+
+    self.assertEqual(credentials, self.EXPECTED_CREDENTIALS)
+
+
+if __name__ == '__main__':
+  unittest.main()
diff --git a/turbinia/workers/analysis/wordpress_test.py b/turbinia/workers/analysis/wordpress_test.py
index 1ef539e..f8f328c 100644
--- a/turbinia/workers/analysis/wordpress_test.py
+++ b/turbinia/workers/analysis/wordpress_test.py
@@ -27,7 +27,7 @@ class WordpressAccessLogAnalysisTaskTest(unittest.TestCase):
   """Tests for WordpressAccessLogAnalysis Task."""
 
   WORDPRESS_ACCESS_LOGS = None
-  WORDPRESS_PWNED_REPORT = """Wordpress access logs found (theme_edit, install)
+  WORDPRESS_PWNED_REPORT = """Wordpress access logs found (install, theme_edit)
 \t27/Jun/2018:19:29:54 +0000: Wordpress installation successful
 \t27/Jun/2018:19:31:15 +0000: Wordpress theme editor edited file (header.php)
 """ # pylint: disable=line-too-long