Repository: google/turbinia

Issue:
Missing logic to handle recipe name instead of recipe dictionary
Currently, the logic thad handles an input recipe name via cli arguments (-I <recipe_name>) is part of turbiniactl.py. 

This means that we cannot pass a recipe name directly as part of a TurbiniaRequest object's recipe attribute (for example, this would he useful for dfTimewolf to be able to just pass recipe names instead of an actual recipe dictionary) because it will be evaluated as a dictionary by recipe_hlpers.validate_recipe().

We should add some logic to accept recipe names (strings), and loading the recipe from a local file. The "load recipe from file" functionality already exist but some modifications are required if the recipe name comes from a different module than turbiniactl.


Discussion:

Resolving patch:
diff --git a/turbinia/client.py b/turbinia/client.py
--- a/turbinia/client.py
+++ b/turbinia/client.py
@@ -26,22 +26,17 @@
 from operator import itemgetter
 from operator import attrgetter
 import os
-import stat
 import time
-import subprocess
-import codecs
 
 from google import auth
-from prometheus_client import start_http_server
 from turbinia import config
 from turbinia.config import logger
 from turbinia.config import DATETIME_FORMAT
 from turbinia import task_manager
-from turbinia import task_utils
 from turbinia import TurbiniaException
+from turbinia.lib import recipe_helpers
 from turbinia.lib import text_formatter as fmt
-from turbinia.lib import docker_manager
-from turbinia.jobs import manager as job_manager
+from turbinia.message import TurbiniaRequest
 from turbinia.workers import Priority
 
 MAX_RETRIES = 10
@@ -166,6 +161,98 @@ def __init__(self):
     self.task_manager = task_manager.get_task_manager()
     self.task_manager.setup(server=False)
 
+  def _create_default_recipe(self):
+    """Creates a default Turbinia recipe."""
+    default_recipe = recipe_helpers.DEFAULT_RECIPE
+    return default_recipe
+
+  def create_recipe(
+      self, debug_tasks=False, filter_patterns=None, group_id='',
+      jobs_allowlist=None, jobs_denylist=None, recipe_name=None, sketch_id=None,
+      skip_recipe_validation=False, yara_rules=None):
+    """Creates a Turbinia recipe.
+    
+    If no recipe_name is specified, this  method returns a default recipe.
+    If a recipe_name is specified then this method will build the recipe 
+    dictionary by reading the  contents of a recipe file. The path to
+    the recipe file is inferre from the recipe_name and the RECIPE_FILE_DIR
+    configuration parameter.
+
+    Args:
+      debug_tasks (bool): flag to turn debug output on for supported tasks.
+      filter_patterns (list): a list of filter pattern strings.
+      group_id (str): a group identifier.
+      jobs_allowlist (list): a list of jobs allowed for execution.
+      jobs_denylist (list): a list of jobs that will not be executed.
+      recipe_name (str): Turbinia recipe name (e.g. triage-linux).
+      sketch_id (str): a Timesketch sketch identifier.
+      skip_recipe_validation (bool): flag indicates if the recipe will be
+          validated.
+      yara_rules (str): a string containing yara rules.
+
+    Returns:
+      dict: a Turbinia recipe dictionary.
+    """
+    recipe = None
+    if jobs_allowlist and jobs_denylist:
+      raise TurbiniaException(
+          'jobs_allowlist and jobs_denylist are mutually exclusive.')
+
+    if not recipe_name:
+      # if no recipe_name is given, create a default recipe.
+      recipe = self._create_default_recipe()
+      if filter_patterns:
+        recipe['globals']['filter_patterns'] = filter_patterns
+      if jobs_denylist:
+        recipe['globals']['jobs_denylist'] = jobs_denylist
+      if jobs_allowlist:
+        recipe['globals']['jobs_allowlist'] = jobs_allowlist
+      if yara_rules:
+        recipe['globals']['yara_rules'] = yara_rules
+    else:
+      # Load custom recipe from given path or name.
+      if (jobs_denylist or jobs_allowlist or filter_patterns or yara_rules):
+        msg = (
+            'Specifying a recipe name is incompatible with defining '
+            'jobs allow/deny lists, yara rules or a patterns file separately.')
+        raise TurbiniaException(msg)
+
+      if os.path.exists(recipe_name):
+        recipe_path = recipe_name
+      else:
+        recipe_path = recipe_helpers.get_recipe_path_from_name(recipe_name)
+
+      if not os.path.exists(recipe_path):
+        msg = 'Could not find recipe file at {0:s}'.format(recipe_path)
+        log.error(msg)
+        raise TurbiniaException(msg)
+
+      recipe = recipe_helpers.load_recipe_from_file(
+          recipe_path, skip_recipe_validation)
+      if not recipe:
+        msg = 'Could not load recipe from file at {0:s}.'.format(recipe_path)
+        raise TurbiniaException(msg)
+
+    # Set any additional recipe parameters, if specified.
+    if sketch_id:
+      recipe['globals']['sketch_id'] = sketch_id
+    if debug_tasks:
+      recipe['globals']['debug_tasks'] = debug_tasks
+    if group_id:
+      recipe['globals']['group_id'] = group_id
+
+    return recipe
+
+  def create_request(
+      self, request_id=None, group_id=None, requester=None, recipe=None,
+      context=None, evidence_=None):
+    """Wrapper method to create a Turbinia request."""
+    default_recipe = self.create_recipe()
+    request = TurbiniaRequest(
+        request_id=request_id, group_id=group_id, requester=requester,
+        recipe=default_recipe, context=context, evidence_=evidence_)
+    return request
+
   def list_jobs(self):
     """List the available jobs."""
     # TODO(aarontp): Refactor this out so that we don't need to depend on
diff --git a/turbinia/lib/recipe_helpers.py b/turbinia/lib/recipe_helpers.py
--- a/turbinia/lib/recipe_helpers.py
+++ b/turbinia/lib/recipe_helpers.py
@@ -17,8 +17,11 @@
 import copy
 import logging
 import yaml
+import os
+
 from yaml import Loader
 from yaml import load
+from turbinia import config
 from turbinia.lib.file_helpers import file_to_str
 from turbinia.lib.file_helpers import file_to_list
 from turbinia.task_utils import TaskLoader
@@ -166,4 +169,28 @@ def validate_recipe(recipe_dict):
         return (False, message)
       tasks_with_recipe.append(recipe_item)
 
-  return (True, '')
\ No newline at end of file
+  return (True, '')
+
+
+def get_recipe_path_from_name(recipe_name):
+  """Returns a recipe's path from a recipe name.
+
+  Args:
+    recipe_name (str): A recipe name.
+
+  Returns:
+    str: a recipe's file system path.
+  """
+  recipe_path = ''
+  if not recipe_name.endswith('.yaml'):
+    recipe_name = recipe_name + '.yaml'
+
+  if hasattr(config, 'RECIPE_FILE_DIR') and config.RECIPE_FILE_DIR:
+    recipe_path = os.path.join(config.RECIPE_FILE_DIR, recipe_name)
+  else:
+    recipe_path = os.path.realpath(__file__)
+    recipe_path = os.path.dirname(recipe_path)
+    recipe_path = os.path.join(recipe_path, 'config', 'recipes')
+    recipe_path = os.path.join(recipe_path, recipe_name)
+
+  return recipe_path
diff --git a/turbinia/turbiniactl.py b/turbinia/turbiniactl.py
--- a/turbinia/turbiniactl.py
+++ b/turbinia/turbiniactl.py
@@ -25,7 +25,6 @@
 import os
 import sys
 import uuid
-import copy
 
 from turbinia import config
 from turbinia import TurbiniaException
@@ -504,7 +503,6 @@ def process_args(args):
   # to point to config paths.
   from turbinia import notify
   from turbinia import client as TurbiniaClientProvider
-  from turbinia.client import TurbiniaCeleryClient
   from turbinia.worker import TurbiniaCeleryWorker
   from turbinia.worker import TurbiniaPsqWorker
   from turbinia.server import TurbiniaServer
@@ -533,11 +531,6 @@ def process_args(args):
 
   args.jobs_allowlist = [j.lower() for j in args.jobs_allowlist]
   args.jobs_denylist = [j.lower() for j in args.jobs_denylist]
-  if args.jobs_allowlist and args.jobs_denylist:
-    msg = (
-        'A Job filter allowlist and denylist cannot be specified at the same '
-        'time')
-    raise TurbiniaException(msg)
 
   # Read set set filter_patterns
   filter_patterns = []
@@ -860,7 +853,6 @@ def process_evidence(
     zone(str): Could zone used for cloud evidence. 
     """
   from turbinia import evidence
-  from turbinia.message import TurbiniaRequest
 
   # Set request id
   request_id = args.request_id if args.request_id else uuid.uuid4().hex
@@ -938,7 +930,7 @@ def process_evidence(
 
   request = None
   if evidence_:
-    request = TurbiniaRequest(
+    request = client.create_request(
         request_id=request_id, group_id=group_id, requester=getpass.getuser())
     request.evidence.append(evidence_)
 
@@ -953,54 +945,22 @@ def process_evidence(
                   credential, args.decryption_keys, exception))
           raise TurbiniaException(msg)
         evidence_.credentials.append((credential_type, credential_data))
-    if args.recipe or args.recipe_path:
-      if (args.jobs_denylist or args.jobs_allowlist or
-          args.filter_patterns_file or args.yara_rules_file):
-        msg = (
-            'Specifying a recipe is incompatible with defining '
-            'jobs allow/deny lists, yara rules or a patterns file separately.')
-        raise TurbiniaException(msg)
 
-      # Find the recipe file with the following precedence:
-      #   1. An absolute path to a recipe file provided with --recipe_path.
-      #   2. A combination of the RECIPE_FILE_DIR config option and the
-      #     --recipe flag.
-      #   3. A combination of the packaged recipe dir (turbinia/config/recipes)
-      #     and the --recipe flag.
-      if args.recipe_path:
-        recipe_file = args.recipe_path
-      elif hasattr(config, 'RECIPE_FILE_DIR') and config.RECIPE_FILE_DIR:
-        recipe_file = os.path.join(config.RECIPE_FILE_DIR, args.recipe)
-      else:
-        recipe_dir = os.path.realpath(__file__)
-        recipe_dir = os.path.dirname(recipe_dir)
-        recipe_dir = os.path.join(recipe_dir, 'config', 'recipes')
-        recipe_file = os.path.join(recipe_dir, args.recipe)
-
-      if not os.path.exists(recipe_file) and not recipe_file.endswith('.yaml'):
-        log.warning(
-            'Could not find recipe file at {0:s}, checking for file '
-            'with .yaml extension'.format(recipe_file))
-        recipe_file = recipe_file + '.yaml'
-      if not os.path.exists(recipe_file):
-        msg = 'Recipe file {0:s} could not be found. Exiting.'.format(
-            recipe_file)
-        raise TurbiniaException(msg)
+    # Recipe pre-condition checks.
+    if args.recipe and args.recipe_path:
+      msg = ('Expected a recipe name (-I) or path (-P) but found both.')
+      raise TurbiniaException(msg)
 
-      recipe_dict = recipe_helpers.load_recipe_from_file(
-          recipe_file, not args.skip_recipe_validation)
-      if not recipe_dict:
-        msg = 'Could not initiate a recipe_dict.'
-        raise TurbiniaException(msg)
-    else:
-      recipe_dict = copy.deepcopy(recipe_helpers.DEFAULT_RECIPE)
-      recipe_dict['globals']['debug_tasks'] = args.debug_tasks
-      recipe_dict['globals']['filter_patterns'] = filter_patterns
-      recipe_dict['globals']['jobs_denylist'] = args.jobs_denylist
-      recipe_dict['globals']['jobs_allowlist'] = args.jobs_allowlist
-      recipe_dict['globals']['yara_rules'] = yara_rules
-    recipe_dict['globals']['group_id'] = group_id
-    request.recipe = recipe_dict
+    if args.recipe or args.recipe_path:
+      # Load the specified recipe.
+      recipe_dict = client.create_recipe(
+          debug_tasks=args.debug_tasks, filter_patterns=filter_patterns,
+          group_id=group_id, jobs_allowlist=args.jobs_allowlist,
+          jobs_denylist=args.jobs_denylist,
+          recipe_name=args.recipe if args.recipe else args.recipe_path,
+          sketch_id=None, skip_recipe_validation=args.skip_recipe_validation,
+          yara_rules=yara_rules)
+      request.recipe = recipe_dict
 
     if args.dump_json:
       print(request.to_json().encode('utf-8'))


Test patch:
diff --git a/turbinia/turbiniactl_test.py b/turbinia/turbiniactl_test.py
--- a/turbinia/turbiniactl_test.py
+++ b/turbinia/turbiniactl_test.py
@@ -16,21 +16,22 @@
 
 from __future__ import unicode_literals
 
+import argparse
 import unittest
 import tempfile
+
 from unittest import mock
-import argparse
 from libcloudforensics.providers.gcp.internal import compute
 from turbinia import config
 from turbinia import TurbiniaException
 from turbinia import turbiniactl
-from turbinia.output_manager import OutputManager
+from turbinia.lib import recipe_helpers
 from turbinia.message import TurbiniaRequest
-from turbinia import evidence
 from turbinia.processors import archive
 
 
 class FakeEvidence():
+  """Class to represent a fake Evidence object. """
 
   def __init__(
       self, name='My Evidence', type=None, source_path=None, cloud_only=False,
@@ -47,6 +48,7 @@ def __init__(
     self.zone = zone
 
   def set_parent(self, _):
+    """Set evidence parent."""
     return
 
 
@@ -55,20 +57,19 @@ class TestTurbiniactl(unittest.TestCase):
 
   @mock.patch('turbinia.output_manager.OutputManager.setup')
   @mock.patch('turbinia.output_manager.OutputManager.save_evidence')
-  @mock.patch('turbinia.message.TurbiniaRequest')
-  def setUp(self, mockRequest, _, __):
+  # pylint: disable=arguments-differ
+  def setUp(self, _, __):
     super(TestTurbiniactl, self).setUp()
-    config.TASK_MANAGER = 'CELERY'
+    config.TASK_MANAGER = 'celery'
     self.output_manager = mock.MagicMock()
     self.base_dir = tempfile.mkdtemp()
     self.source_path = tempfile.mkstemp(dir=self.base_dir)[1]
-    mockRequest.request_id = 'fakeID'
-    mockRequest.group_id = 'FakeGroupID'
 
   @mock.patch('turbinia.client.get_turbinia_client')
   @mock.patch('turbinia.evidence.RawDisk')
   def testRawDiskEvidence(self, mockEvidence, mockClient):
     """Test RawDisk evidence."""
+    mockClient.create_request.return_value = TurbiniaRequest()
     args = argparse.Namespace(
         request_id=None, command='rawdisk', force_evidence=False,
         decryption_keys=None, recipe=None, recipe_path=None, dump_json=None,
@@ -89,6 +90,7 @@ def testRawDiskEvidence(self, mockEvidence, mockClient):
   def testDirectoryDiskEvidence(
       self, mockDirectory, mockCompressedEvidence, mockClient):
     """Test directory evidence."""
+    mockClient.create_request.return_value = TurbiniaRequest()
     args = argparse.Namespace(
         request_id=None, command='directory', force_evidence=False,
         decryption_keys=None, recipe=None, recipe_path=None, dump_json=None,
@@ -122,6 +124,7 @@ def testDirectoryDiskEvidence(
   @mock.patch('turbinia.evidence.CompressedDirectory')
   def testCompressedDirectory(self, mockEvidence, mockClient):
     """Test compressed directory evidence"""
+    mockClient.create_request.return_value = TurbiniaRequest()
     args = argparse.Namespace(
         request_id=None, command='compresseddirectory', force_evidence=False,
         decryption_keys=None, recipe=None, recipe_path=None, dump_json=None,
@@ -143,6 +146,7 @@ def testCompressedDirectory(self, mockEvidence, mockClient):
   @mock.patch('turbinia.evidence.GoogleCloudDisk')
   def testCloudDisk(self, mockEvidence, mockClient):
     """Test Google Cloud Disk evidence."""
+    mockClient.create_request.return_value = TurbiniaRequest()
     args = argparse.Namespace(
         request_id=None, command='googleclouddisk', force_evidence=False,
         decryption_keys=None, recipe=None, recipe_path=None, dump_json=None,
@@ -167,6 +171,7 @@ def testCloudDisk(self, mockEvidence, mockClient):
   def testCloudEmbedded(
       self, mockCloudEvidence, mockEmbeddedEvidence, mockClient, _, __):
     """Test Google Cloud Disk Embedded evidence."""
+    mockClient.create_request.return_value = TurbiniaRequest()
     args = argparse.Namespace(
         request_id=None, command='googleclouddiskembedded',
         force_evidence=False, decryption_keys=None, recipe=None,
@@ -196,6 +201,7 @@ def testCloudEmbedded(
   @mock.patch('turbinia.evidence.ChromiumProfile')
   def testHindsight(self, mockEvidence, mockClient):
     """Test hindsight evidence"""
+    mockClient.create_request.return_value = TurbiniaRequest()
     args = argparse.Namespace(
         request_id=None, command='hindsight', force_evidence=False,
         decryption_keys=None, recipe=None, recipe_path=None, dump_json=None,
@@ -226,6 +232,7 @@ def testHindsight(self, mockEvidence, mockClient):
   @mock.patch('turbinia.evidence.RawMemory')
   def testRawMemory(self, mockEvidence, mockClient):
     """Test raw memory evidence"""
+    mockClient.create_request.return_value = TurbiniaRequest()
     args = argparse.Namespace(
         request_id=None, command='rawmemory', force_evidence=False,
         decryption_keys=None, recipe=None, recipe_path=None, dump_json=None,
@@ -262,7 +269,7 @@ def testUnequalDirectoryArgs(self, _):
     self.assertTrue(turbiniactl.process_evidence.called)
 
   @mock.patch('turbinia.client.get_turbinia_client')
-  def testUnequalRawdiskArgs(self, _):
+  def testUnequalRawdiskArgs(self, mockClient):
     """Test unequal number of args for rawdisk evidence type."""
     self.assertRaises(
         TurbiniaException, turbiniactl.process_args, [
@@ -307,7 +314,7 @@ def testUnequalCompresseddirectoryArgs(self, _):
   def testUnequalCloudDiskArgs(self, mockParser, mock_copyDisk, _):
     """Test unequal number of args for cloud disk evidence type."""
     config.SHARED_FILESYSTEM = False
-
+    config.TASK_MANAGER = 'PSQ'
     mockArgs = argparse.Namespace(
         all_fields=False, command='googleclouddisk', config_file=None,
         copy_only=False, debug=False, debug_tasks=False, decryption_keys=[],
@@ -359,7 +366,6 @@ def testUnequalCloudDiskArgs(self, mockParser, mock_copyDisk, _):
         ])
 
     mockArgs.source = ['source1', 'source2', 'source3']
-    config.TASK_MANAGER = 'PSQ'
     turbiniactl.process_evidence = mock.MagicMock(return_value=None)
     mock_copyDisk.return_value = compute.GoogleComputeDisk(
         'fake-proj', 'fake-zone', 'fake-disk-copy')
@@ -375,6 +381,7 @@ def testUnequalCloudDiskEmbeddedArgs(self, mock_copyDisk, _):
     """Test unequal number of args for cloud embedded disk evidence type."""
     # Fail when zones don't match
     config.SHARED_FILESYSTEM = False
+    config.TASK_MANAGER = 'PSQ'
     self.assertRaises(
         TurbiniaException, turbiniactl.process_args, [
             'googleclouddiskembedded', '--disk_name', 'disk1,disk2,disk3',
@@ -416,7 +423,6 @@ def testUnequalCloudDiskEmbeddedArgs(self, mock_copyDisk, _):
         ])
 
     # Pass when all the args match
-    config.TASK_MANAGER = 'PSQ'
     turbiniactl.process_evidence = mock.MagicMock(return_value=None)
     mock_copyDisk.return_value = compute.GoogleComputeDisk(
         'fake-proj', 'fake-zone', 'fake-disk-copy')
@@ -481,3 +487,15 @@ def testUnequalHindsightArgs(self, _):
     turbiniactl.process_args(
         ['hindsight', '--source_path', 'disk1,disk2,disk3'])
     self.assertTrue(turbiniactl.process_evidence.called)
+
+  @mock.patch('turbinia.client.get_turbinia_client')
+  def testTurbiniaClientRequest(self, mockClient):
+    """Test Turbinia client request creation."""
+    config.TASK_MANAGER = 'celery'
+    mockClient.create_request = mock.MagicMock()
+    mockClient.create_request.return_value = TurbiniaRequest(
+        recipe=recipe_helpers.DEFAULT_RECIPE)
+    test_request = mockClient.create_request()
+    self.assertIsNotNone(test_request)
+    test_default_recipe = recipe_helpers.DEFAULT_RECIPE
+    self.assertEqual(test_request.recipe, test_default_recipe)
\ No newline at end of file