Repository: google/turbinia
Pull request #387 (2019-05-02 23:37:08)

Print updates when uncompleted tasks increases


Patch:
diff --git a/CONTRIBUTING b/CONTRIBUTING
deleted file mode 100644
index 21e4c71..0000000
--- a/CONTRIBUTING
+++ /dev/null
@@ -1,25 +0,0 @@
-Want to contribute? Great! First, read this page (including the small print at the end).
-
-### Before you contribute
-Before we can use your code, you must sign the
-[Google Individual Contributor License Agreement](https://cla.developers.google.com/about/google-individual)
-(CLA), which you can do online. The CLA is necessary mainly because you own the
-copyright to your changes, even after your contribution becomes part of our
-codebase, so we need your permission to use and distribute your code. We also
-need to be sure of various other things—for instance that you'll tell us if you
-know that your code infringes on other people's patents. You don't have to sign
-the CLA until after you've submitted your code for review and a member has
-approved it, but you must do it before we can put your code into our codebase.
-Before you start working on a larger contribution, you should get in touch with
-us first through the issue tracker with your idea so that we can help out and
-possibly guide you. Coordinating up front makes it much easier to avoid
-frustration later on.
-
-### Code reviews
-All submissions, including submissions by project members, require review. We
-use Github pull requests for this purpose.
-
-### The small print
-Contributions made by corporations are covered by a different agreement than
-the one above, the
-[Software Grant and Corporate Contributor License Agreement](https://cla.developers.google.com/about/google-corporate).
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 0000000..bd3f271
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,2 @@
+# Want to contribute? 
+Great! First read this [page](https://github.com/google/turbinia/blob/master/docs/contributing.md).
diff --git a/tools/gcf_init/index.js b/tools/gcf_init/index.js
index 2919aca..2c07363 100644
--- a/tools/gcf_init/index.js
+++ b/tools/gcf_init/index.js
@@ -67,7 +67,7 @@ exports.gettasks = function gettasks(req, res) {
   }
   if (req.body.user) {
     console.log('Getting Turbinia Tasks by user: ' + req.body.user);
-    query = query.filter('user', '=', req.body.user);
+    query = query.filter('requester', '=', req.body.user);
   }
   if (req.body.start_time) {
     try {
@@ -150,8 +150,8 @@ exports.closetasks = function closetasks(req, res) {
         '__key__', '=', datastore.key([turbiniaKind, req.body.task_id]))
   }
   if (req.body.user) {
-    console.log('Adding filter - user: ' + req.body.user);
-    query = query.filter('user', '=', req.body.user)
+    console.log('Adding filter - requester: ' + req.body.user);
+    query = query.filter('requester', '=', req.body.user)
   }
 
   console.log(query);
diff --git a/tools/gcf_init/index.yaml b/tools/gcf_init/index.yaml
index c609e9e..6d617f8 100644
--- a/tools/gcf_init/index.yaml
+++ b/tools/gcf_init/index.yaml
@@ -45,7 +45,7 @@ indexes:
 - kind: TurbiniaTask
   ancestor: no
   properties:
-  - name: user
+  - name: requester
   - name: last_update
     direction: desc
 
@@ -54,7 +54,7 @@ indexes:
   properties:
   - name: instance
   - name: id
-  - name: user
+  - name: requester
   - name: last_update
     direction: desc
 
@@ -63,7 +63,7 @@ indexes:
   properties:
   - name: instance
   - name: request_id
-  - name: user
+  - name: requester
   - name: last_update
     direction: desc
 
diff --git a/turbinia/client.py b/turbinia/client.py
index b7c8aff..485a1d2 100644
--- a/turbinia/client.py
+++ b/turbinia/client.py
@@ -162,7 +162,8 @@ class TurbiniaClient(object):
       user (string): The user of the request we want tasks for.
       poll_interval (int): Interval of seconds between polling cycles.
     """
-    last_count = 0
+    last_completed_count = -1
+    last_uncompleted_count = -1
     while True:
       task_results = self.get_task_data(
           instance, project, region, request_id=request_id, user=user)
@@ -186,12 +187,14 @@ class TurbiniaClient(object):
           'Tasks completed ({0:d}/{1:d}): [{2:s}], waiting for [{3:s}].'.format(
               len(completed_tasks), total_count, completed_names,
               uncompleted_names))
-      if len(completed_tasks) > last_count:
+      if (len(completed_tasks) > last_completed_count or
+          len(uncompleted_tasks) > last_uncompleted_count):
         log.info(msg)
       else:
         log.debug(msg)
 
-      last_count = len(completed_tasks)
+      last_completed_count = len(completed_tasks)
+      last_uncompleted_count = len(uncompleted_tasks)
       time.sleep(poll_interval)
 
     log.info('All {0:d} Tasks completed'.format(len(task_results)))
@@ -347,7 +350,7 @@ class TurbiniaClient(object):
 
     # Build up data
     report = []
-    user = task_results[0].get('user')
+    requester = task_results[0].get('requester')
     request_id = task_results[0].get('request_id')
     success_types = ['Successful', 'Failed', 'Scheduled or Running']
     success_values = [True, False, None]
@@ -366,7 +369,8 @@ class TurbiniaClient(object):
     report.append(fmt.heading1('Turbinia report {0:s}'.format(request_id)))
     report.append(
         fmt.bullet(
-            'Processed {0:d} Tasks for user {1:s}'.format(num_results, user)))
+            'Processed {0:d} Tasks for user {1:s}'.format(
+                num_results, requester)))
 
     # Print report data for tasks
     for success_type in success_types:
diff --git a/turbinia/evidence.py b/turbinia/evidence.py
index e5326a4..815d33f 100644
--- a/turbinia/evidence.py
+++ b/turbinia/evidence.py
@@ -265,6 +265,24 @@ class BitlockerDisk(EncryptedDisk):
     self.encryption_type = self.__class__.__name__
 
 
+class APFSEncryptedDisk(EncryptedDisk):
+  """APFS encrypted disk file evidence.
+
+  Attributes:
+    recovery_key: A string of the recovery key for this disk
+    password: A string of the password used for this disk. If recovery key
+        is used, this argument is ignored
+    unencrypted_path: A string to the unencrypted local path
+  """
+
+  def __init__(self, recovery_key=None, password=None, *args, **kwargs):
+    """Initialization for Bitlocker disk evidence object"""
+    self.recovery_key = recovery_key
+    self.password = password
+    super(APFSEncryptedDisk, self).__init__(*args, **kwargs)
+    self.encryption_type = self.__class__.__name__
+
+
 class GoogleCloudDisk(RawDisk):
   """Evidence object for Google Cloud Disks.
 
diff --git a/turbinia/jobs/plaso.py b/turbinia/jobs/plaso.py
index c5aa4ca..7f1dd56 100644
--- a/turbinia/jobs/plaso.py
+++ b/turbinia/jobs/plaso.py
@@ -16,6 +16,7 @@
 
 from __future__ import unicode_literals
 
+from turbinia.evidence import APFSEncryptedDisk
 from turbinia.evidence import BitlockerDisk
 from turbinia.evidence import Directory
 from turbinia.evidence import GoogleCloudDisk
@@ -32,7 +33,7 @@ class PlasoJob(interface.TurbiniaJob):
   # The types of evidence that this Job will process
   evidence_input = [
       Directory, RawDisk, GoogleCloudDisk, GoogleCloudDiskRawEmbedded,
-      BitlockerDisk
+      BitlockerDisk, APFSEncryptedDisk
   ]
   evidence_output = [PlasoFile]
 
diff --git a/turbinia/lib/libcloudforensics.py b/turbinia/lib/libcloudforensics.py
index bd5a7b4..612dcd9 100644
--- a/turbinia/lib/libcloudforensics.py
+++ b/turbinia/lib/libcloudforensics.py
@@ -270,7 +270,8 @@ class GoogleCloudProject(object):
     # Max length of disk names in GCP is 63 characters
     project_id = snapshot.project.project_id
     disk_id = project_id + snapshot.disk.name
-    disk_id_crc32 = '{0:08x}'.format(binascii.crc32(disk_id) & 0xffffffff)
+    disk_id_crc32 = '{0:08x}'.format(
+        binascii.crc32(disk_id.encode()) & 0xffffffff)
     truncate_at = 62 - len(disk_id_crc32) - len('-copy') - len(project_id)
     if disk_name_prefix:
       disk_name_prefix += '-'
@@ -330,8 +331,9 @@ class GoogleCloudProject(object):
 
     # Analysis software to install.
     packages_to_install = [
-        'python-plaso', 'xmount', 'libfvde-tools', 'libbde-tools',
-        'plaso-tools', 'jq', 'ncdu', 'htop', 'binutils', 'upx-ucl'
+        'python-plaso', 'xmount', 'sleuthkit', 'libfvde-tools', 'libbde-tools',
+        'plaso-tools', 'jq', 'ncdu', 'htop', 'binutils', 'upx-ucl',
+        'docker-explorer-tools'
     ]
 
     startup_script = """
diff --git a/turbinia/message.py b/turbinia/message.py
index 3ee736e..f9b3708 100644
--- a/turbinia/message.py
+++ b/turbinia/message.py
@@ -35,16 +35,19 @@ class TurbiniaRequest(object):
   """An object to request evidence to be processed.
 
   Attributes:
-    request_id: A client specified ID for this request.
-    recipe: Recipe to use when processing this request.
-    context: A Dict of context data to be passed around with this request.
-    evidence: A list of Evidence objects.
+    request_id(str): A client specified ID for this request.
+    requestor(str): The username of who made the request.
+    recipe(dict): Recipe to use when processing this request.
+    context(dict): A Dict of context data to be passed around with this request.
+    evidence(list): A list of Evidence objects.
   """
 
   def __init__(
-      self, request_id=None, recipe=None, context=None, evidence_=None):
+      self, request_id=None, requester=None, recipe=None, context=None,
+      evidence_=None):
     """Initialization for TurbiniaRequest."""
     self.request_id = request_id if request_id else uuid.uuid4().hex
+    self.requester = requester if requester else 'user_unspecified'
     self.recipe = recipe if recipe else {}
     self.context = context if context else {}
     self.evidence = evidence_ if evidence_ else []
diff --git a/turbinia/task_manager.py b/turbinia/task_manager.py
index af6ecd1..df22a5c 100644
--- a/turbinia/task_manager.py
+++ b/turbinia/task_manager.py
@@ -168,6 +168,7 @@ class BaseTaskManager(object):
         job_count += 1
         for task in job.create_tasks([evidence_]):
           task.base_output_dir = config.OUTPUT_DIR
+          task.requester = evidence_.config.get('requester')
           self.add_task(task, evidence_)
 
     if not job_count:
@@ -352,6 +353,7 @@ class CeleryTaskManager(BaseTaskManager):
         if not evidence_.request_id:
           evidence_.request_id = request.request_id
         evidence_.config = request.recipe
+        evidence_.config['requester'] = request.requester
         log.info(
             'Received evidence [{0:s}] from Kombu message.'.format(
                 str(evidence_)))
@@ -436,6 +438,7 @@ class PSQTaskManager(BaseTaskManager):
         if not evidence_.request_id:
           evidence_.request_id = request.request_id
         evidence_.config = request.recipe
+        evidence_.config['requester'] = request.requester
         log.info(
             'Received evidence [{0:s}] from PubSub message.'.format(
                 str(evidence_)))
diff --git a/turbinia/turbiniactl.py b/turbinia/turbiniactl.py
index 65777d9..02acd3a 100644
--- a/turbinia/turbiniactl.py
+++ b/turbinia/turbiniactl.py
@@ -142,6 +142,25 @@ def main():
   parser_rawdisk.add_argument(
       '-n', '--name', help='Descriptive name of the evidence', required=False)
 
+  # Parser options for APFS Disk Evidence type
+  parser_apfs = subparsers.add_parser(
+      'apfs', help='Process APFSEncryptedDisk as Evidence')
+  parser_apfs.add_argument(
+      '-l', '--local_path', help='Local path to the encrypted APFS evidence',
+      required=True)
+  parser_apfs.add_argument(
+      '-r', '--recovery_key', help='Recovery key for the APFS evidence.  '
+      'Either recovery key or password must be specified.', required=False)
+  parser_apfs.add_argument(
+      '-p', '--password', help='Password for the APFS evidence.  '
+      'If a recovery key is specified concurrently, password will be ignored.',
+      required=False)
+  parser_apfs.add_argument(
+      '-s', '--source', help='Description of the source of the evidence',
+      required=False)
+  parser_apfs.add_argument(
+      '-n', '--name', help='Descriptive name of the evidence', required=False)
+
   # Parser options for Bitlocker Disk Evidence type
   parser_bitlocker = subparsers.add_parser(
       'bitlocker', help='Process Bitlocker Disk as Evidence')
@@ -334,6 +353,15 @@ def main():
     evidence_ = evidence.RawDisk(
         name=args.name, local_path=local_path,
         mount_partition=args.mount_partition, source=args.source)
+  elif args.command == 'apfs':
+    if not args.password and not args.recovery_key:
+      log.error('Neither recovery key nor password is specified.')
+      sys.exit(1)
+    args.name = args.name if args.name else args.local_path
+    local_path = os.path.abspath(args.local_path)
+    evidence_ = evidence.APFSEncryptedDisk(
+        name=args.name, local_path=local_path, recovery_key=args.recovery_key,
+        password=args.password, source=args.source)
   elif args.command == 'bitlocker':
     if not args.password and not args.recovery_key:
       log.error('Neither recovery key nor password is specified.')
@@ -452,7 +480,8 @@ def main():
     server.add_evidence(evidence_)
     server.start()
   elif evidence_:
-    request = TurbiniaRequest(request_id=args.request_id)
+    request = TurbiniaRequest(
+        request_id=args.request_id, requester=getpass.getuser())
     request.evidence.append(evidence_)
     if filter_patterns:
       request.recipe['filter_patterns'] = filter_patterns
@@ -462,6 +491,7 @@ def main():
       request.recipe['jobs_whitelist'] = args.jobs_whitelist
     if args.dump_json:
       print(request.to_json().encode('utf-8'))
+      sys.exit(0)
     else:
       log.info(
           'Creating request {0:s} with evidence {1:s}'.format(
diff --git a/turbinia/workers/__init__.py b/turbinia/workers/__init__.py
index b86a0e2..ce65023 100644
--- a/turbinia/workers/__init__.py
+++ b/turbinia/workers/__init__.py
@@ -75,7 +75,7 @@ class TurbiniaTaskResult(object):
       successful: Bool indicating success status.
       task_id: Task ID of the parent task.
       task_name: Name of parent task.
-      user: The user who requested the task.
+      requester: The user who requested the task.
       worker_name: Name of worker task executed on.
       _log: A list of log messages
   """
@@ -100,7 +100,7 @@ class TurbiniaTaskResult(object):
 
     self.task_id = None
     self.task_name = None
-    self.user = None
+    self.requester = None
     self.output_dir = None
 
     self.report_data = None
@@ -130,7 +130,7 @@ class TurbiniaTaskResult(object):
 
     self.task_id = task.id
     self.task_name = task.name
-    self.user = task.user
+    self.requester = task.requester
     if task.output_manager.is_setup:
       _, self.output_dir = task.output_manager.get_local_output_dirs()
     else:
@@ -315,16 +315,16 @@ class TurbiniaTask(object):
           this is a task result object, but other implementations have their
           own stub objects.
       tmp_dir: Temporary directory for Task to write to.
-      user: The user who requested the task.
+      requester: The user who requested the task.
       _evidence_config (dict): The config that we want to pass to all new
             evidence created from this task.
   """
 
   # The list of attributes that we will persist into storage
-  STORED_ATTRIBUTES = ['id', 'last_update', 'name', 'request_id', 'user']
+  STORED_ATTRIBUTES = ['id', 'last_update', 'name', 'request_id', 'requester']
 
   def __init__(
-      self, name=None, base_output_dir=None, request_id=None, user=None):
+      self, name=None, base_output_dir=None, request_id=None, requester=None):
     """Initialization for TurbiniaTask."""
     if base_output_dir:
       self.base_output_dir = base_output_dir
@@ -342,7 +342,7 @@ class TurbiniaTask(object):
     self.stub = None
     self.tmp_dir = None
     self.turbinia_version = turbinia.__version__
-    self.user = user if user else getpass.getuser()
+    self.requester = requester if requester else 'user_unspecified'
     self._evidence_config = {}
 
   def serialize(self):
diff --git a/turbinia/workers/plaso.py b/turbinia/workers/plaso.py
index 5e42f05..0215f09 100644
--- a/turbinia/workers/plaso.py
+++ b/turbinia/workers/plaso.py
@@ -19,6 +19,7 @@ from __future__ import unicode_literals
 import os
 
 from turbinia import config
+from turbinia.evidence import APFSEncryptedDisk
 from turbinia.evidence import BitlockerDisk
 from turbinia.evidence import PlasoFile
 from turbinia.workers import TurbiniaTask
@@ -53,7 +54,7 @@ class PlasoTask(TurbiniaTask):
     if config.DEBUG_TASKS:
       cmd.append('-d')
 
-    if isinstance(evidence, BitlockerDisk):
+    if isinstance(evidence, (APFSEncryptedDisk, BitlockerDisk)):
       if evidence.recovery_key:
         cmd.extend([
             '--credential', 'recovery_password:{0:s}'.format(


Test patch:
diff --git a/turbinia/client_test.py b/turbinia/client_test.py
index 3ed2666..e2bd8d4 100644
--- a/turbinia/client_test.py
+++ b/turbinia/client_test.py
@@ -127,7 +127,7 @@ class TestTurbiniaClient(unittest.TestCase):
             'saved_paths': ['/no/path/', '/fake/path'],
             'status': 'This fake task executed',
             'successful': True,
-            'user': 'myuser',
+            'requester': 'myuser',
             'worker_name': 'fake_worker'
         }, {
             'id': '0xfakeTaskId2',
@@ -140,7 +140,7 @@ class TestTurbiniaClient(unittest.TestCase):
             'saved_paths': ['/no/path/2', '/fake/path/2'],
             'status': 'This second fake task executed',
             'successful': True,
-            'user': 'myuser',
+            'requester': 'myuser',
             'worker_name': 'fake_worker'
         }, {
             'id': '0xfakeTaskId3',
@@ -153,7 +153,7 @@ class TestTurbiniaClient(unittest.TestCase):
             'saved_paths': ['/no/path/3', '/fake/path/3'],
             'status': 'Third Task Failed...',
             'successful': False,
-            'user': 'myuser',
+            'requester': 'myuser',
             'worker_name': 'fake_worker'
         }
     ] # yapf: disable