Repository: google/turbinia

Issue:
Results should be stored grouped per request
Currently Turbinia stores results in a flat folder structure with a folder per task. It would be a lot more convenient if output was stored in a structure with a folder per request.
Example: https://github.com/google/turbinia/commit/b9bf9dcaac2cbadb217278c77a5b90d4d286760b



Discussion:
Since we are also adding group_id to turbinia, perhaps it's better to make the group_id the parent folder -> request_ids ->  task_ids  respectively
As long as it's clear to the user what the group ID is when creating a
request (eg the client always returns a group ID).
It should be easy for a user to navigate the output structure on disk.

On Wed, Oct 20, 2021, 02:34 alimez ***@***.***> wrote:

> Since we are also adding group_id to turbinia, perhaps it's better to make
> the group_id the parent folder -> request_ids -> task_ids respectively
>
> —
> You are receiving this because you authored the thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/google/turbinia/issues/914#issuecomment-947209553>,
> or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/ABP5D4CTXZ3HC34PFUWIGPLUHYFATANCNFSM5FLBDQTQ>
> .
> Triage notifications on the go with GitHub Mobile for iOS
> <https://apps.apple.com/app/apple-store/id1477376905?ct=notification-email&mt=8&pt=524675>
> or Android
> <https://play.google.com/store/apps/details?id=com.github.android&referrer=utm_campaign%3Dnotification-email%26utm_medium%3Demail%26utm_source%3Dgithub>.
>
>


Resolving patch:
diff --git a/turbinia/output_manager.py b/turbinia/output_manager.py
--- a/turbinia/output_manager.py
+++ b/turbinia/output_manager.py
@@ -52,19 +52,21 @@ def __init__(self):
     self.is_setup = False
 
   @staticmethod
-  def get_output_writers(name, uid, remote_only):
+  def get_output_writers(name, uid, request_id, remote_only):
     """Get a list of output writers.
 
     Args:
       name (str): The name of the Request or Task.
       uid (str): The unique identifier of the Request or Task.
+      request_id(str): The ID of the Request.
 
     Returns:
       A list of OutputWriter objects.
     """
     config.LoadConfig()
     epoch = str(int(time.time()))
-    unique_dir = '{0:s}-{1:s}-{2:s}'.format(epoch, str(uid), name)
+    unique_dir = os.path.join(
+        str(request_id), '{0:s}-{1:s}-{2:s}'.format(epoch, str(uid), name))
     writers = []
     local_output_dir = None
 
@@ -216,9 +218,10 @@ def save_local_file(self, file_, result):
 
     return saved_path, saved_path_type, local_path
 
-  def setup(self, name, uid, remote_only=False):
+  def setup(self, name, uid, request_id, remote_only=False):
     """Setup OutputManager object."""
-    self._output_writers = self.get_output_writers(name, uid, remote_only)
+    self._output_writers = self.get_output_writers(
+        name, uid, request_id, remote_only)
     self.is_setup = True
 
 
diff --git a/turbinia/workers/__init__.py b/turbinia/workers/__init__.py
--- a/turbinia/workers/__init__.py
+++ b/turbinia/workers/__init__.py
@@ -733,7 +733,7 @@ def setup(self, evidence):
       TurbiniaException: If the evidence can not be found.
     """
     self.setup_metrics()
-    self.output_manager.setup(self.name, self.id)
+    self.output_manager.setup(self.name, self.id, self.request_id)
     self.tmp_dir, self.output_dir = self.output_manager.get_local_output_dirs()
     if not self.result:
       self.result = self.create_result(input_evidence=evidence)


Test patch:
diff --git a/turbinia/output_manager_test.py b/turbinia/output_manager_test.py
--- a/turbinia/output_manager_test.py
+++ b/turbinia/output_manager_test.py
@@ -63,7 +63,7 @@ def testGetOutputWriters(self):
     """Tests get_output_writers function for valid response."""
     output_manager.storage = mock.MagicMock()
     writers = output_manager.OutputManager.get_output_writers(
-        self.task.name, self.task.id, remote_only=False)
+        self.task.name, self.task.id, self.task.request_id, remote_only=False)
     self.assertEquals(len(writers), 2)
     for writer in writers:
       self.assertIsInstance(writer, output_manager.OutputWriter)
@@ -71,7 +71,8 @@ def testGetOutputWriters(self):
   def testGetLocalOutputDirs(self):
     """Tests get_local_output_dirs function for valid response."""
     output_manager.storage = mock.MagicMock()
-    self.task.output_manager.setup(self.task.name, self.task.id)
+    self.task.output_manager.setup(
+        self.task.name, self.task.id, self.task.request_id)
     tmp_dir, local_dir = self.task.output_manager.get_local_output_dirs()
 
     self.assertTrue(self.task.output_manager.is_setup)
@@ -84,7 +85,8 @@ def testSaveLocalFile(self):
     """Test the save_local_file method."""
     # Set path to None so we don't try to initialize GCS output writer.
     config.GCS_OUTPUT_PATH = None
-    self.task.output_manager.setup(self.task.name, self.task.id)
+    self.task.output_manager.setup(
+        self.task.name, self.task.id, self.task.request_id)
     tmp_dir, local_dir = self.task.output_manager.get_local_output_dirs()
     self.task.result = mock.MagicMock()
     self.task.result.saved_paths = []
@@ -107,7 +109,8 @@ def testSaveEvidence(self):
     """Test the save_evidence method."""
     # Set path to None so we don't try to initialize GCS output writer.
     config.GCS_OUTPUT_PATH = None
-    self.task.output_manager.setup(self.task.name, self.task.id)
+    self.task.output_manager.setup(
+        self.task.name, self.task.id, self.task.request_id)
     tmp_dir, local_dir = self.task.output_manager.get_local_output_dirs()
     self.task.result = mock.MagicMock()
     self.task.result.saved_paths = []
@@ -135,7 +138,8 @@ def testSaveEvidenceWithMetadata(self):
     """Test the save_evidence method with metadata file."""
     # Set path to None so we don't try to initialize GCS output writer.
     config.GCS_OUTPUT_PATH = None
-    self.task.output_manager.setup(self.task.name, self.task.id)
+    self.task.output_manager.setup(
+        self.task.name, self.task.id, self.task.request_id)
     tmp_dir, local_dir = self.task.output_manager.get_local_output_dirs()
     self.task.result = mock.MagicMock()
     self.task.result.saved_paths = []