Repository: google/turbinia

Issue:
Jobs allow list flag not working
For some reason the `--jobs_allowlist` flag is not actually setting the allowlist in the request (as shown by dumping the request and seeing the `jobs_allowlist` parameter in the recipe part of the request being empty).  This is likely related to some of the recent recipe refactoring.


Discussion:
Looking into this. I think I identified the issue. The method that creates a default recipe does not include the jobs allow/deny lists passed from the command line via turbiniactl.

Resolving patch:
diff --git a/turbinia/turbiniactl.py b/turbinia/turbiniactl.py
--- a/turbinia/turbiniactl.py
+++ b/turbinia/turbiniactl.py
@@ -28,7 +28,6 @@
 
 from turbinia import config
 from turbinia import TurbiniaException
-from turbinia.lib import recipe_helpers
 from turbinia.config import logger
 from turbinia import __version__
 from turbinia.processors import archive
@@ -951,16 +950,19 @@ def process_evidence(
       msg = ('Expected a recipe name (-I) or path (-P) but found both.')
       raise TurbiniaException(msg)
 
-    if args.recipe or args.recipe_path:
-      # Load the specified recipe.
-      recipe_dict = client.create_recipe(
-          debug_tasks=args.debug_tasks, filter_patterns=filter_patterns,
-          group_id=group_id, jobs_allowlist=args.jobs_allowlist,
-          jobs_denylist=args.jobs_denylist,
-          recipe_name=args.recipe if args.recipe else args.recipe_path,
-          sketch_id=None, skip_recipe_validation=args.skip_recipe_validation,
-          yara_rules=yara_rules)
-      request.recipe = recipe_dict
+    # Set the recipe name/path or None if not set.
+    # If no recipe name or path is given, the create_recipe method will
+    # generate a default recipe but still honor any of other parameters
+    # such as jobs_allowlist/jobs_denylist.
+    recipe = args.recipe if args.recipe else args.recipe_path
+
+    recipe_dict = client.create_recipe(
+        debug_tasks=args.debug_tasks, filter_patterns=filter_patterns,
+        group_id=group_id, jobs_allowlist=args.jobs_allowlist,
+        jobs_denylist=args.jobs_denylist, recipe_name=recipe, sketch_id=None,
+        skip_recipe_validation=args.skip_recipe_validation,
+        yara_rules=yara_rules)
+    request.recipe = recipe_dict
 
     if args.dump_json:
       print(request.to_json().encode('utf-8'))


Test patch:
diff --git a/turbinia/turbiniactl_test.py b/turbinia/turbiniactl_test.py
--- a/turbinia/turbiniactl_test.py
+++ b/turbinia/turbiniactl_test.py
@@ -72,9 +72,9 @@ def testRawDiskEvidence(self, mockEvidence, mockClient):
     mockClient.create_request.return_value = TurbiniaRequest()
     args = argparse.Namespace(
         request_id=None, command='rawdisk', force_evidence=False,
-        decryption_keys=None, recipe=None, recipe_path=None, dump_json=None,
-        debug_tasks=None, jobs_denylist=None, jobs_allowlist=None,
-        run_local=False, wait=False)
+        decryption_keys=None, recipe=None, recipe_path=None,
+        skip_recipe_validation=False, dump_json=None, debug_tasks=None,
+        jobs_denylist=None, jobs_allowlist=None, run_local=False, wait=False)
     mockEvidence.return_value = FakeEvidence(
         type='rawdisk', source_path=self.source_path)
     config.SHARED_FILESYSTEM = True
@@ -93,9 +93,9 @@ def testDirectoryDiskEvidence(
     mockClient.create_request.return_value = TurbiniaRequest()
     args = argparse.Namespace(
         request_id=None, command='directory', force_evidence=False,
-        decryption_keys=None, recipe=None, recipe_path=None, dump_json=None,
-        debug_tasks=None, jobs_denylist=None, jobs_allowlist=None,
-        run_local=False, wait=False)
+        decryption_keys=None, recipe=None, recipe_path=None,
+        skip_recipe_validation=False, dump_json=None, debug_tasks=None,
+        jobs_denylist=None, jobs_allowlist=None, run_local=False, wait=False)
     # Test not shared filesystem
     archive.CompressDirectory = mock.MagicMock()
     config.SHARED_FILESYSTEM = False
@@ -127,9 +127,9 @@ def testCompressedDirectory(self, mockEvidence, mockClient):
     mockClient.create_request.return_value = TurbiniaRequest()
     args = argparse.Namespace(
         request_id=None, command='compresseddirectory', force_evidence=False,
-        decryption_keys=None, recipe=None, recipe_path=None, dump_json=None,
-        debug_tasks=None, jobs_denylist=None, jobs_allowlist=None,
-        run_local=False, wait=False)
+        decryption_keys=None, recipe=None, recipe_path=None,
+        skip_recipe_validation=False, dump_json=None, debug_tasks=None,
+        jobs_denylist=None, jobs_allowlist=None, run_local=False, wait=False)
     archive.ValidateTarFile = mock.MagicMock()
     mockEvidence.return_value = FakeEvidence(
         type='compresseddirectory', source_path=self.source_path,
@@ -149,9 +149,9 @@ def testCloudDisk(self, mockEvidence, mockClient):
     mockClient.create_request.return_value = TurbiniaRequest()
     args = argparse.Namespace(
         request_id=None, command='googleclouddisk', force_evidence=False,
-        decryption_keys=None, recipe=None, recipe_path=None, dump_json=None,
-        debug_tasks=None, jobs_denylist=None, jobs_allowlist=None,
-        run_local=False, wait=False)
+        decryption_keys=None, recipe=None, recipe_path=None,
+        skip_recipe_validation=False, dump_json=None, debug_tasks=None,
+        jobs_denylist=None, jobs_allowlist=None, run_local=False, wait=False)
     mockEvidence.return_value = FakeEvidence(
         type='googleclouddisk', project='testProject', disk_name='testDisk',
         cloud_only=True)
@@ -175,8 +175,9 @@ def testCloudEmbedded(
     args = argparse.Namespace(
         request_id=None, command='googleclouddiskembedded',
         force_evidence=False, decryption_keys=None, recipe=None,
-        recipe_path=None, dump_json=None, debug_tasks=None, jobs_denylist=None,
-        jobs_allowlist=None, run_local=False, wait=False)
+        recipe_path=None, dump_json=None, debug_tasks=None,
+        skip_recipe_validation=False, jobs_denylist=None, jobs_allowlist=None,
+        run_local=False, wait=False)
     mockCloudEvidence.return_value = FakeEvidence(
         type='googleclouddisk', project='testProject', disk_name='testDisk',
         cloud_only=True)
@@ -204,9 +205,9 @@ def testHindsight(self, mockEvidence, mockClient):
     mockClient.create_request.return_value = TurbiniaRequest()
     args = argparse.Namespace(
         request_id=None, command='hindsight', force_evidence=False,
-        decryption_keys=None, recipe=None, recipe_path=None, dump_json=None,
-        debug_tasks=None, jobs_denylist=None, jobs_allowlist=None,
-        run_local=False, wait=False)
+        decryption_keys=None, recipe=None, recipe_path=None,
+        skip_recipe_validation=False, dump_json=None, debug_tasks=None,
+        jobs_denylist=None, jobs_allowlist=None, run_local=False, wait=False)
     with self.assertRaisesRegex(TurbiniaException, 'Invalid output format.'):
       turbiniactl.process_evidence(
           name='My Evidence', source_path=self.source_path, args=args,
@@ -235,9 +236,10 @@ def testRawMemory(self, mockEvidence, mockClient):
     mockClient.create_request.return_value = TurbiniaRequest()
     args = argparse.Namespace(
         request_id=None, command='rawmemory', force_evidence=False,
-        decryption_keys=None, recipe=None, recipe_path=None, dump_json=None,
-        debug_tasks=None, jobs_denylist=None, jobs_allowlist=None,
-        run_local=False, wait=False, module_list=['mod1', 'mod2'])
+        decryption_keys=None, recipe=None, recipe_path=None,
+        skip_recipe_validation=False, dump_json=None, debug_tasks=None,
+        jobs_denylist=None, jobs_allowlist=None, run_local=False, wait=False,
+        module_list=['mod1', 'mod2'])
     mockEvidence.return_value = FakeEvidence(
         type='rawmemory', source_path=self.source_path)
     turbiniactl.process_evidence(