Repository: google/turbinia
Pull request #319 (2018-12-05 23:50:16)

Update Evidence local_path when it's saved
See #304 for details, but after #282, Tasks that held evidence in the tmp_dir saved the temp path instead of the locally saved path (thereby breaking when different Tasks processed the newly created Evidence).  This fixes that.

FYI @ericzinnikas , LMK what you think.


Patch:
diff --git a/tools/turbinia_job_graph.py b/tools/turbinia_job_graph.py
index d7a014f..c2dff8b 100755
--- a/tools/turbinia_job_graph.py
+++ b/tools/turbinia_job_graph.py
@@ -19,7 +19,7 @@ from __future__ import unicode_literals
 import argparse
 import graphviz
 
-from turbinia.jobs import get_jobs as turbinia_jobs
+from turbinia.jobs import manager as jobs_manager
 
 
 def create_graph():
@@ -29,15 +29,15 @@ def create_graph():
     Instance of graphviz.dot.Digraph
   """
   dot = graphviz.Digraph(comment='Turbinia Evidence graph', format='png')
-  for job in turbinia_jobs():
-    dot.node(job.name)
+  for _, job in jobs_manager.JobsManager.GetJobs():
+    dot.node(job.NAME)
     for evidence in job.evidence_input:
       dot.node(evidence.__name__, shape='box')
-      dot.edge(evidence.__name__, job.name)
+      dot.edge(evidence.__name__, job.NAME)
 
     for evidence in job.evidence_output:
       dot.node(evidence.__name__, shape='box')
-      dot.edge(job.name, evidence.__name__)
+      dot.edge(job.NAME, evidence.__name__)
   return dot
 
 
diff --git a/turbinia/client.py b/turbinia/client.py
index 77bd38f..95d87e6 100644
--- a/turbinia/client.py
+++ b/turbinia/client.py
@@ -398,11 +398,16 @@ class TurbiniaServer(object):
     task_manager (TaskManager): An object to manage turbinia tasks.
   """
 
-  def __init__(self):
-    """Initialize Turbinia Server."""
+  def __init__(self, jobs_blacklist=None, jobs_whitelist=None):
+    """Initializes Turbinia Server.
+
+    Args:
+      jobs_blacklist (Optional[list[str]]): Jobs we will exclude from running
+      jobs_whitelist (Optional[list[str]]): The only Jobs we will include to run
+    """
     config.LoadConfig()
     self.task_manager = task_manager.get_task_manager()
-    self.task_manager.setup()
+    self.task_manager.setup(jobs_blacklist, jobs_whitelist)
 
   def start(self):
     """Start Turbinia Server."""
diff --git a/turbinia/evidence.py b/turbinia/evidence.py
index 8da9018..30f6ccd 100644
--- a/turbinia/evidence.py
+++ b/turbinia/evidence.py
@@ -173,9 +173,9 @@ class Evidence(object):
   def preprocess(self):
     """Runs the possible parent's evidence preprocessing code, then ours.
 
-    This gets run in the context of the local task execution on the worker
-    nodes prior to the task itself running.  This can be used to prepare the
-    evidence to be processed (e.g. attach a cloud disk, mount a local disk etc).
+    This is a wrapper function that will call the chain of pre-processors
+    starting with the most distant ancestor.  After all of the ancestors have
+    been processed, then we run our pre-processor.
     """
     if self.parent_evidence:
       self.parent_evidence.preprocess()
@@ -184,9 +184,9 @@ class Evidence(object):
   def postprocess(self):
     """Runs our postprocessing code, then our possible parent's evidence.
 
-    This gets run in the context of the local task execution on the worker
-    nodes after the task has finished.  This can be used to clean-up after the
-    evidence is processed (e.g. detach a cloud disk, etc,).
+    This is is a wrapper function that will run our post-processor, and will
+    then recurse down the chain of parent Evidence and run those post-processors
+    in order.
     """
     self._postprocess()
     if self.parent_evidence:
diff --git a/turbinia/jobs/manager.py b/turbinia/jobs/manager.py
index 49b5cab..ba9030d 100644
--- a/turbinia/jobs/manager.py
+++ b/turbinia/jobs/manager.py
@@ -16,12 +16,65 @@
 
 from __future__ import unicode_literals
 
+from turbinia import TurbiniaException
+
 
 class JobsManager(object):
   """The jobs manager."""
 
   _job_classes = {}
 
+  @classmethod
+  def FilterJobNames(cls, job_names, jobs_blacklist=None, jobs_whitelist=None):
+    """Filters a list of job names against white/black lists.
+
+    jobs_whitelist and jobs_blacklist must not be specified at the same time.
+
+    Args:
+      job_names (list[str]): The names of the job_names to filter.
+      jobs_blacklist (Optional[list[str]]): Job names to exclude.
+      jobs_whitelist (Optional[list[str]]): Job names to include.
+
+    Returns:
+     list[str]: Job names
+
+    Raises:
+      TurbiniaException if both jobs_blacklist and jobs_whitelist are specified.
+    """
+    jobs_blacklist = jobs_blacklist if jobs_blacklist else []
+    jobs_blacklist = [job.lower() for job in jobs_blacklist]
+    jobs_whitelist = jobs_whitelist if jobs_whitelist else []
+    jobs_whitelist = [job.lower() for job in jobs_whitelist]
+
+    if jobs_whitelist and jobs_blacklist:
+      raise TurbiniaException(
+          'jobs_whitelist and jobs_blacklist cannot be specified at the same '
+          'time.')
+    elif jobs_blacklist:
+      return [job for job in job_names if job.lower() not in jobs_blacklist]
+    elif jobs_whitelist:
+      return [job for job in job_names if job.lower() in jobs_whitelist]
+    else:
+      return job_names
+
+  @classmethod
+  def FilterJobObjects(cls, jobs, jobs_blacklist=None, jobs_whitelist=None):
+    """Filters a list of job objects against white/black lists.
+
+    jobs_whitelist and jobs_blacklist must not be specified at the same time.
+
+    Args:
+      jobs (list[TurbiniaJob]): The jobs to filter.
+      jobs_blacklist (Optional[list[str]]): Job names to exclude.
+      jobs_whitelist (Optional[list[str]]): Job names to include.
+
+    Returns:
+     list[TurbiniaJob]: Job objects
+    """
+    job_names = [job.name.lower() for job in jobs]
+    job_names = cls.FilterJobNames(job_names, jobs_blacklist, jobs_whitelist)
+    return [job for job in jobs if job.name.lower() in job_names]
+
   @classmethod
   def DeregisterJob(cls, job_class):
     """Deregisters a job class.
diff --git a/turbinia/output_manager.py b/turbinia/output_manager.py
index bbf95f1..8ab388d 100644
--- a/turbinia/output_manager.py
+++ b/turbinia/output_manager.py
@@ -133,7 +133,13 @@ class OutputManager(object):
     Returns:
       An evidence object
     """
-    (path, path_type) = self.save_local_file(evidence_.local_path, result)
+    path, path_type, local_path = self.save_local_file(
+        evidence_.local_path, result)
+    # Set the evidence local_path from the saved path info so that in cases
+    # where tasks are saving evidence into the temp dir, we'll get the newly
+    # copied version from the saved output path.
+    if local_path:
+      evidence_.local_path = local_path
     evidence_.saved_path = path
     evidence_.saved_path_type = path_type
     log.info(
@@ -153,18 +159,22 @@ class OutputManager(object):
 
     Returns:
       Tuple of (String of last written file path,
-                String of last written file destination output type)
+                String of last written file destination output type,
+                Local path if saved locally, else None)
     """
     saved_path = None
     saved_path_type = None
+    local_path = None
     for writer in self._output_writers:
       new_path = writer.copy_to(file_)
       if new_path:
         result.saved_paths.append(new_path)
         saved_path = new_path
         saved_path_type = writer.name
+      if writer.name == LocalOutputWriter.NAME:
+        local_path = new_path
 
-    return saved_path, saved_path_type
+    return saved_path, saved_path_type, local_path
 
   def setup(self, task):
     """Setup OutputManager object."""
diff --git a/turbinia/task_manager.py b/turbinia/task_manager.py
index 8833fb5..f07fc85 100644
--- a/turbinia/task_manager.py
+++ b/turbinia/task_manager.py
@@ -101,12 +101,24 @@ class BaseTaskManager(object):
     """
     raise NotImplementedError
 
-  def setup(self, *args, **kwargs):
-    """Does setup of Task manager and its dependencies."""
+  def setup(self, jobs_blacklist=None, jobs_whitelist=None, *args, **kwargs):
+    """Does setup of Task manager and its dependencies.
+
+    Args:
+      jobs_blacklist (list): Jobs that will be excluded from running
+      jobs_whitelist (list): The only Jobs will be included to run
+    """
     self._backend_setup(*args, **kwargs)
     # TODO(aarontp): Consider instantiating a job per evidence object
     job_names = jobs_manager.JobsManager.GetJobNames()
+    if jobs_blacklist or jobs_whitelist:
+      log.info(
+          'Filtering Jobs with whitelist {0!s} and blacklist {1!s}'.format(
+              jobs_whitelist, jobs_blacklist))
+      job_names = jobs_manager.JobsManager.FilterJobNames(
+          job_names, jobs_blacklist, jobs_whitelist)
     self.jobs = jobs_manager.JobsManager.GetJobInstances(job_names)
+    log.debug('Registered job list: {0:s}'.format(str(job_names)))
 
   def add_evidence(self, evidence_):
     """Adds new evidence and creates tasks to process it.
@@ -125,11 +137,22 @@ class BaseTaskManager(object):
     log.info('Adding new evidence: {0:s}'.format(str(evidence_)))
     self.evidence.append(evidence_)
     job_count = 0
+    jobs_whitelist = evidence_.config.get('jobs_whitelist', [])
+    jobs_blacklist = evidence_.config.get('jobs_blacklist', [])
+    if jobs_blacklist or jobs_whitelist:
+      log.info(
+          'Filtering Jobs with whitelist {0!s} and blacklist {1!s}'.format(
+              jobs_whitelist, jobs_blacklist))
+      jobs_list = jobs_manager.JobsManager.FilterJobObjects(
+          self.jobs, jobs_blacklist, jobs_whitelist)
+    else:
+      jobs_list = self.jobs
+
     # TODO(aarontp): Add some kind of loop detection in here so that jobs can
     # register for Evidence(), or or other evidence types that may be a super
     # class of the output of the job itself.  Short term we could potentially
     # have a run time check for this upon Job instantiation to prevent it.
-    for job in self.jobs:
+    for job in jobs_list:
       # Doing a strict type check here for now until we can get the above
       # comment figured out.
       # pylint: disable=unidiomatic-typecheck
diff --git a/turbinia/turbiniactl.py b/turbinia/turbiniactl.py
old mode 100755
new mode 100644
index 84d3ca1..7f801f7
--- a/turbinia/turbiniactl.py
+++ b/turbinia/turbiniactl.py
@@ -40,6 +40,18 @@ log = logging.getLogger('turbinia')
 logger.setup()
 
 
+def csv_list(string):
+  """Helper method for having CSV argparse types.
+
+  Args:
+    string(str): Comma separated string to parse.
+
+  Returns:
+    list[str]: The parsed strings.
+  """
+  return string.split(',')
+
+
 def main():
   """Main function for turbiniactl"""
   # TODO(aarontp): Allow for single run mode when
@@ -89,12 +101,17 @@ def main():
       'text based evidence files with (in extended grep regex format). '
       'This filtered output will be in addition to the complete output')
   parser.add_argument(
-      '-j', '--jobs_whitelist',
-      help='A whitelist for Jobs that we will allow to run (note that it '
-      'will not force them to run).')
+      '-j', '--jobs_whitelist', default=[], type=csv_list,
+      help='A whitelist for Jobs that will be allowed to run (in CSV format, '
+      'no spaces). This will not force them to run if they are not configured '
+      'to. This is applied both at server start time and when the client makes '
+      'a processing request. When applied at server start time the change is '
+      'persistent while the server is running.  When applied by the client, it '
+      'will only affect that processing request.')
   parser.add_argument(
-      '-J', '--jobs_blacklist',
-      help='A blacklist for Jobs we will not allow to run')
+      '-J', '--jobs_blacklist', default=[], type=csv_list,
+      help='A blacklist for Jobs we will not allow to run.  See '
+      '--jobs_whitelist help for details on format and when it is applied.')
   parser.add_argument(
       '-p', '--poll_interval', default=60, type=int,
       help='Number of seconds to wait between polling for task state info')
@@ -189,7 +206,10 @@ def main():
       '-n', '--name', help='Descriptive name of the evidence', required=False)
 
   # List Jobs
-  subparsers.add_parser('listjobs', help='List all available jobs')
+  subparsers.add_parser(
+      'listjobs',
+      help='List all available Jobs. These Job names can be used by '
+      '--jobs_whitelist and --jobs_blacklist')
 
   # PSQ Worker
   parser_psqworker = subparsers.add_parser('psqworker', help='Run PSQ worker')
@@ -223,6 +243,9 @@ def main():
   # Server
   subparsers.add_parser('server', help='Run Turbinia Server')
 
+  #print version
+  log.info('Turbinia version: {0:s}'.format(__version__))
+
   args = parser.parse_args()
   if args.quiet:
     log.setLevel(logging.ERROR)
@@ -235,8 +258,8 @@ def main():
 
   if args.jobs_whitelist and args.jobs_blacklist:
     log.error(
-        'A Job filter whitelist and blacklist cannot be specified '
-        'at the same time')
+        'A Job filter whitelist and blacklist cannot be specified at the same '
+        'time')
     sys.exit(1)
 
   filter_patterns = None
@@ -317,7 +340,8 @@ def main():
     worker = TurbiniaCeleryWorker()
     worker.start()
   elif args.command == 'server':
-    server = TurbiniaServer()
+    server = TurbiniaServer(
+        jobs_blacklist=args.jobs_blacklist, jobs_whitelist=args.jobs_whitelist)
     server.start()
   elif args.command == 'status':
     region = config.TURBINIA_REGION
@@ -398,6 +422,10 @@ def main():
     request.evidence.append(evidence_)
     if filter_patterns:
       request.recipe['filter_patterns'] = filter_patterns
+    if args.jobs_blacklist:
+      request.recipe['jobs_blacklist'] = args.jobs_blacklist
+    if args.jobs_whitelist:
+      request.recipe['jobs_whitelist'] = args.jobs_whitelist
     if args.dump_json:
       print(request.to_json().encode('utf-8'))
     else:


Test patch:
diff --git a/turbinia/jobs/manager_test.py b/turbinia/jobs/manager_test.py
index 3c9896a..52544d6 100644
--- a/turbinia/jobs/manager_test.py
+++ b/turbinia/jobs/manager_test.py
@@ -6,6 +6,7 @@ from __future__ import unicode_literals
 
 import unittest
 
+from turbinia import TurbiniaException
 from turbinia.jobs import interface
 from turbinia.jobs import manager
 
@@ -94,6 +95,55 @@ class JobsManagerTest(unittest.TestCase):
     for job in jobs:
       self.assertIsInstance(job, interface.TurbiniaJob)
 
+  def testFilterJobNamesEmptyLists(self):
+    """Test FilterJobNames() with no filters."""
+    job_names = ['testjob1', 'testjob2']
+    return_job_names = manager.JobsManager.FilterJobNames(
+        job_names, jobs_blacklist=[], jobs_whitelist=[])
+    self.assertListEqual(job_names, return_job_names)
+
+  def testFilterJobNamesBlackList(self):
+    """Test FilterJobNames() with jobs_blacklist."""
+    job_names = ['testjob1', 'testjob2']
+    return_job_names = manager.JobsManager.FilterJobNames(
+        job_names, jobs_blacklist=[job_names[0]], jobs_whitelist=[])
+    self.assertListEqual(job_names[1:], return_job_names)
+
+  def testFilterJobObjectsBlackList(self):
+    """Test FilterJobObjects() with jobs_blacklist and objects."""
+    jobs = [TestJob1(), TestJob2()]
+    return_jobs = manager.JobsManager.FilterJobObjects(
+        jobs, jobs_blacklist=[jobs[0].name], jobs_whitelist=[])
+    self.assertListEqual(jobs[1:], return_jobs)
+
+  def testFilterJobNamesWhiteList(self):
+    """Test FilterJobNames() with jobs_whitelist."""
+    job_names = ['testjob1', 'testjob2']
+    return_job_names = manager.JobsManager.FilterJobNames(
+        job_names, jobs_blacklist=[], jobs_whitelist=[job_names[0]])
+    self.assertListEqual(job_names[:1], return_job_names)
+
+  def testFilterJobObjectsWhiteList(self):
+    """Test FilterJobObjects() with jobs_whitelist."""
+    jobs = [TestJob1(), TestJob2()]
+    return_jobs = manager.JobsManager.FilterJobObjects(
+        jobs, jobs_blacklist=[], jobs_whitelist=[jobs[1].name])
+    self.assertListEqual(jobs[1:], return_jobs)
+
+  def testFilterJobNamesException(self):
+    """Test FilterJobNames() with both jobs_blacklist and jobs_whitelist."""
+    job_names = ['testjob1', 'testjob2']
+    self.assertRaises(
+        TurbiniaException, manager.JobsManager.FilterJobNames, job_names,
+        jobs_blacklist=['a'], jobs_whitelist=['b'])
+
+  def testFilterJobNamesMixedCase(self):
+    """Test FilterJobNames() with mixed case inputs."""
+    job_names = ['testjob1', 'testjob2']
+    return_job_names = manager.JobsManager.FilterJobNames(
+        job_names, jobs_blacklist=[], jobs_whitelist=['TESTJOB1'])
+    self.assertListEqual(job_names[:1], return_job_names)
+
 
 if __name__ == '__main__':
   unittest.main()
diff --git a/turbinia/output_manager_test.py b/turbinia/output_manager_test.py
index 5cf8db3..65dfaac 100644
--- a/turbinia/output_manager_test.py
+++ b/turbinia/output_manager_test.py
@@ -92,9 +92,11 @@ class TestLocalOutputManager(unittest.TestCase):
       fh.write(test_contents)
 
     self.assertFalse(os.path.exists(dst_file))
-    self.task.output_manager.save_local_file(src_file, self.task.result)
+    _, __, local_file = self.task.output_manager.save_local_file(
+        src_file, self.task.result)
     self.assertTrue(os.path.exists(dst_file))
     self.assertIn(dst_file, self.task.result.saved_paths)
+    self.assertEqual(local_file, dst_file)
 
   def testSaveEvidence(self):
     """Test the save_evidence method."""