Repository: google/turbinia

Issue:
Support new PGSQL password method
SCRAM-SHA-256 as supported by hashcat v6.2.6

`-m 28600`


Discussion:

Resolving patch:
diff --git a/turbinia/workers/analysis/postgresql_acct.py b/turbinia/workers/analysis/postgresql_acct.py
--- a/turbinia/workers/analysis/postgresql_acct.py
+++ b/turbinia/workers/analysis/postgresql_acct.py
@@ -28,7 +28,8 @@
 from turbinia.workers import Priority
 from turbinia.workers import TurbiniaTask
 
-_CREDS_REGEXP = r'(?P<username>.*?)(?:\\xff)+I(?P<password>md5.{32})'
+_MD5_CREDS_REGEXP = r'(?P<username>.*?)(?:\\xff)+I(?P<password>md5.{32})'
+_SCRAM_CREDS_REGEXP = r'(?P<username>\w+).*?(?P<password>SCRAM-SHA-256\$\d+:\S{24}\$\S{44}:\S{44})'
 _PG_CONF_NAME = 'postgresql.conf'
 
 
@@ -80,7 +81,7 @@ def run(self, evidence, result):
       return result
     # 3) Extract creds
     try:
-      hashnames = self._extract_creds(data_dirs, evidence)
+      md5_hashnames, scram_hashnames = self._extract_creds(data_dirs, evidence)
     except TurbiniaException as exception:
       result.close(self, success=False, status=str(exception))
       return result
@@ -88,7 +89,7 @@ def run(self, evidence, result):
     # 4) Bruteforce
     timeout = self.task_config.get('bruteforce_timeout')
     (report, priority, summary) = self._analyse_postgres_creds(
-        hashnames, timeout=timeout)
+        md5_hashnames, scram_hashnames, timeout=timeout)
     output_evidence.text_data = report
     result.report_data = report
     result.report_priority = priority
@@ -136,8 +137,10 @@ def _extract_data_dir(self, location, result):
     Returns:
       data_dirs (list): List of locations of pgsql databases
     """
-    data_dirs = []
+    data_dirs = set()
     for dirs, _, _ in os.walk(location):
+      if os.path.isfile(os.path.join(dirs, _PG_CONF_NAME)):
+        data_dirs.add(dirs)
       try:
         grep = subprocess.run(
             ['grep', r'data_directory',
@@ -147,16 +150,19 @@ def _extract_data_dir(self, location, result):
           continue
 
         for directive in grep.stdout.strip().split('\n'):
+          if directive.startswith('#'):
+            continue
           parts = directive.split("'")
           if len(parts) == 3:
-            data_dirs.append(parts[1])
+            if os.path.sep in parts[1]:
+              data_dirs.add(parts[1])
           else:
             result.log(f'Unable to parse data_dir directive: {directive:s}')
       except subprocess.CalledProcessError as exception:
         raise TurbiniaException(
             f'Unable to grep Postgres config file: {str(exception):s}')
 
-    return data_dirs
+    return list(data_dirs)
 
   def _extract_creds(self, locations, evidence):
     """Attempts to extract raw encrypted credentials from the database
@@ -166,38 +172,57 @@ def _extract_creds(self, locations, evidence):
       evidence (Evidence object):  The evidence to process
 
     Returns:
-      hashnames (dict): Dict mapping hash back to username.
+      hashnames (Tuple[dict, dict]): Dicts mapping hash back to username.
     """
-    hashnames = {}
+    md5_hashnames = {}
+    scram_hashnames = {}
+
     for location in locations:
       dir = os.path.normpath(evidence.local_path + location)
       try:
         grep = subprocess.run(
             ['sudo', 'egrep', '-hari', r'md5[a-zA-Z0-9]{32}', dir], check=False,
             text=False, capture_output=True)
+        if grep.returncode == 0:
+          # Process the raw binary data
+          raw_lines = str(grep.stdout).split('\\n')
+          for line in raw_lines:
+            values = line.replace('\\x00', '').replace('\\x01', '')
+            m = re.match(_MD5_CREDS_REGEXP, values[2:])
+            if not m:
+              continue
+            (username, passwdhash) = (m.group('username'), m.group('password'))
+            if passwdhash[3:] not in md5_hashnames:
+              md5_hashnames[passwdhash[3:]] = str(username)
+
+        grep = subprocess.run(
+            ['sudo', 'grep', '-Phar', r'SCRAM-SHA-256\$\d+:', dir], check=False,
+            text=False, capture_output=True)
         if grep.returncode != 0:
           continue
+        raw_lines = grep.stdout.split(b'\n')
+        for line in raw_lines:
+          m = re.match(_SCRAM_CREDS_REGEXP, line.decode('utf-8', 'ignore'))
+          if not m:
+            continue
+          (username, passwdhash) = (m.group('username'), m.group('password'))
+          if passwdhash not in scram_hashnames:
+            scram_hashnames[passwdhash] = str(username)
       except subprocess.CalledProcessError as exception:
         raise TurbiniaException(
             f'Unable to grep raw database file: {str(exception):s}')
 
-      # Process the raw binary data
-      raw_lines = str(grep.stdout).split('\\n')
-      for line in raw_lines:
-        values = line.replace('\\x00', '').replace('\\x01', '')
-        m = re.match(_CREDS_REGEXP, values[2:])
-        if not m:
-          continue
-        (username, passwdhash) = (m.group('username'), m.group('password'))
-        if passwdhash[3:] not in hashnames:
-          hashnames[passwdhash[3:]] = str(username)
-    return hashnames
+    return (md5_hashnames, scram_hashnames)
 
-  def _analyse_postgres_creds(self, hashnames, timeout=300):
+  def _analyse_postgres_creds(
+      self, md5_hashnames, scram_hashnames, timeout=300):
     """Attempt to brute force extracted PostgreSQL credentials.
 
     Args:
-        hashnames (dict): Dict mapping hash back to username for convenience.
+        md5_hashnames (dict): Dict mapping hash back to username
+          for convenience.
+        scram_hashnames (dict): Dict mapping hash back to username
+          for convenience.
         timeout (int): How long to spend cracking.
 
     Returns:
@@ -213,19 +238,25 @@ def _analyse_postgres_creds(self, hashnames, timeout=300):
 
     # 0 is "md5"
     weak_passwords = bruteforce_password_hashes(
-        [v + ':' + k for (k, v) in hashnames.items()], tmp_dir=self.tmp_dir,
+        [v + ':' + k for (k, v) in md5_hashnames.items()], tmp_dir=self.tmp_dir,
         timeout=timeout, extra_args='--username -m 0')
 
+    # 28600 is PostgreSQL SCRAM-SHA-256
+    weak_passwords += bruteforce_password_hashes(
+        [v + ':' + k for (k, v) in scram_hashnames.items()],
+        tmp_dir=self.tmp_dir, timeout=timeout, extra_args='--username -m 28600')
+
     if weak_passwords:
       priority = Priority.CRITICAL
       summary = f'PostgreSQL analysis found {len(weak_passwords):d} weak password(s)'
       report.insert(0, fmt.heading4(fmt.bold(summary)))
       line = f'{len(weak_passwords):n} weak password(s) found:'
       report.append(fmt.bullet(fmt.bold(line)))
+      combined_hashnames = {**md5_hashnames, **scram_hashnames}
       for password_hash, plaintext in weak_passwords:
-        if password_hash in hashnames:
+        if password_hash in combined_hashnames:
           line = """User '{0:s}' with password '{1:s}'""".format(
-              hashnames[password_hash], plaintext)
+              combined_hashnames[password_hash], plaintext)
           report.append(fmt.bullet(line, level=2))
     report = '\n'.join(report)
     return (report, priority, summary)


Test patch:
diff --git a/test_data/postgresql.conf b/test_data/postgresql.conf
--- a/test_data/postgresql.conf
+++ b/test_data/postgresql.conf
@@ -40,3 +40,5 @@
 # option or PGDATA environment variable, represented here as ConfigDir.
 
 data_directory = 'test_data'		# use data in another directory
+#data_directory = 'ConfigDir'           # use data in another directory
+                                        # (change requires restart)
diff --git a/test_data/scram_database b/test_data/scram_database
new file mode 100644
Binary files /dev/null and b/test_data/scram_database differ
diff --git a/turbinia/workers/analysis/postgresql_acct_test.py b/turbinia/workers/analysis/postgresql_acct_test.py
--- a/turbinia/workers/analysis/postgresql_acct_test.py
+++ b/turbinia/workers/analysis/postgresql_acct_test.py
@@ -27,7 +27,11 @@ class PostgresAcctAnalysisTaskTest(TestTurbiniaTaskBase):
 
   TEST_DATA_DIR = None
 
-  EXPECTED_CREDENTIALS = {'5f4dcc3b5aa765d61d8327deb882cf99': 'postgres'}
+  EXPECTED_MD5_CREDENTIALS = {'5f4dcc3b5aa765d61d8327deb882cf99': 'postgres'}
+  EXPECTED_SCRAM_CREDENTIALS = {
+      'SCRAM-SHA-256$4096:APJq+0/Y/X3zrBg2AWyKkQ==$Qe9RKFYZJPhd14z1Iqs1agjzxGlBPexsTEHIhos6wrM=:Z9MGrSmyQvM4owINbGzK8HxhFzVWDcSYYD+s44sQvV8=':
+          'postgres'
+  }
 
   POSTGRES_REPORT = """#### **PostgreSQL analysis found 1 weak password(s)**
 * **1 weak password(s) found:**
@@ -48,24 +52,44 @@ def test_extract_data_dir(self):
     # pylint: disable=protected-access
     data_dirs = task._extract_data_dir(self.TEST_DATA_DIR, self.result)
     self.assertEqual(len(data_dirs), 1)
-    self.assertEqual(data_dirs, ['test_data'])
+    self.assertTrue(data_dirs[0].endswith('test_data'))
 
-  def test_extract_creds(self):
+  def test_extract_md5_creds(self):
     """Tests the _extract_creds method."""
     config.LoadConfig()
     task = postgresql_acct.PostgresAccountAnalysisTask()
 
     # pylint: disable=protected-access
-    hashes = task._extract_creds(['/database'], self.evidence)
-    self.assertDictEqual(hashes, self.EXPECTED_CREDENTIALS)
+    hashes, _ = task._extract_creds(['/database'], self.evidence)
+    self.assertDictEqual(hashes, self.EXPECTED_MD5_CREDENTIALS)
 
-  def test_analyse_postgres_creds(self):
+  def test_extract_scram_creds(self):
+    """Tests the _extract_creds method."""
+    config.LoadConfig()
+    task = postgresql_acct.PostgresAccountAnalysisTask()
+
+    # pylint: disable=protected-access
+    _, hashes = task._extract_creds(['/scram_database'], self.evidence)
+    self.assertDictEqual(hashes, self.EXPECTED_SCRAM_CREDENTIALS)
+
+  def test_analyse_md5_postgres_creds(self):
+    """Tests the _analyse_postgres_creds method."""
+    config.LoadConfig()
+    task = postgresql_acct.PostgresAccountAnalysisTask()
+
+    (report, priority, summary) = task._analyse_postgres_creds(
+        self.EXPECTED_MD5_CREDENTIALS, {})
+    self.assertEqual(report, self.POSTGRES_REPORT)
+    self.assertEqual(priority, 10)
+    self.assertEqual(summary, 'PostgreSQL analysis found 1 weak password(s)')
+
+  def test_analyse_scram_postgres_creds(self):
     """Tests the _analyse_postgres_creds method."""
     config.LoadConfig()
     task = postgresql_acct.PostgresAccountAnalysisTask()
 
     (report, priority, summary) = task._analyse_postgres_creds(
-        self.EXPECTED_CREDENTIALS)
+        {}, self.EXPECTED_SCRAM_CREDENTIALS)
     self.assertEqual(report, self.POSTGRES_REPORT)
     self.assertEqual(priority, 10)
-    self.assertEqual(summary, 'PostgreSQL analysis found 1 weak password(s)')
\ No newline at end of file
+    self.assertEqual(summary, 'PostgreSQL analysis found 1 weak password(s)')