Repository: log2timeline/dftimewolf

Issue:
FR - configure GRRFileFinder flows and hunts via state containers.
Currently, the GRRFileFinder flow and hunt modules accept file paths via a recipe argument.  

It would be useful if the file paths for these modules are also configurable via FSPath containers.  This would enable the GRRFileFinder modules to be used downstream in more complex recipes, where file paths are discovered or created by other modules.

Discussion:

Patch:
diff --git a/dftimewolf/lib/collectors/grr_hosts.py b/dftimewolf/lib/collectors/grr_hosts.py
index 58cf867..93a56d6 100644
--- a/dftimewolf/lib/collectors/grr_hosts.py
+++ b/dftimewolf/lib/collectors/grr_hosts.py
@@ -322,7 +322,7 @@ class GRRFlow(GRRBaseModule, module.ThreadAwareModule):
       flow_id (str): GRR identifier of the flow.
 
     Returns:
-      str: path of downloaded files.
+      str: path of the zipfile containing downloaded files.
     """
     output_file_path = os.path.join(
         self.output_path, '.'.join((flow_id, 'zip')))
@@ -633,6 +633,10 @@ class GRRFileCollector(GRRFlow):
 
   def PreProcess(self) -> None:
     """Check that we're actually doing something, and it's not a no-op."""
+    for file_container in self.state.GetContainers(
+        container_class=containers.FSPath):
+      self.files.append(file_container.path)
+
     if not self.files:
       message = 'Would fetch 0 files - bailing out instead.'
       self.logger.critical(message)
diff --git a/dftimewolf/lib/collectors/grr_hunt.py b/dftimewolf/lib/collectors/grr_hunt.py
index 6b2aec6..10c4fc1 100644
--- a/dftimewolf/lib/collectors/grr_hunt.py
+++ b/dftimewolf/lib/collectors/grr_hunt.py
@@ -293,13 +293,20 @@ class GRRHuntFileCollector(GRRHunt):
         approvers=approvers, verify=verify)
     self.file_path_list = [item.strip() for item
                            in file_path_list.strip().split(',')]
-    if not file_path_list:
-      self.ModuleError('Files must be specified for hunts', critical=True)
     if max_file_size:
       self.max_file_size = int(max_file_size)
 
     self.HuntSetup(match_mode, client_operating_systems, client_labels)
 
+  def PreProcess(self) -> None:
+    """Load File paths from containers and check there are files to download."""
+    for file_container in self.state.GetContainers(
+        container_class=containers.FSPath):
+      self.file_path_list.append(file_container.path)
+
+    if not self.file_path_list:
+      self.ModuleError('Files must be specified for hunts', critical=True)
+
   # TODO: this method does not raise itself, indicate what function call does.
   def Process(self) -> None:
     """Starts a new File Finder GRR hunt.


Test patch:
diff --git a/tests/lib/collectors/grr_hosts.py b/tests/lib/collectors/grr_hosts.py
index e8b9a21..f9ab593 100644
--- a/tests/lib/collectors/grr_hosts.py
+++ b/tests/lib/collectors/grr_hosts.py
@@ -357,6 +357,7 @@ class GRRFileCollectorTest(unittest.TestCase):
     self.mock_grr_api = mock.Mock()
     mock_InitHttp.return_value = self.mock_grr_api
     self.test_state = state.DFTimewolfState(config.Config)
+    self.test_state.StoreContainer(containers.FSPath(path='/etc/hosts'))
     self.grr_file_collector = grr_hosts.GRRFileCollector(self.test_state)
     self.grr_file_collector.SetUp(
         hostnames='C.0000000000000001',
@@ -382,7 +383,14 @@ class GRRFileCollectorTest(unittest.TestCase):
 
     self.assertIsNotNone(self.grr_file_collector)
     self.assertEqual(['C.0000000000000001'], actual_hosts)
-    self.assertEqual(self.grr_file_collector.files, ['/etc/passwd'])
+    self.assertEqual(
+        self.grr_file_collector.files, ['/etc/passwd'])
+
+  def testPreProcess(self):
+    """Tests the preprocess method."""
+    self.grr_file_collector.PreProcess()
+    self.assertEqual(
+        self.grr_file_collector.files, ['/etc/passwd', '/etc/hosts'])
 
   @mock.patch('dftimewolf.lib.collectors.grr_hosts.GRRFlow._AwaitFlow')
   @mock.patch('dftimewolf.lib.collectors.grr_hosts.GRRFlow._DownloadFiles')
@@ -404,7 +412,7 @@ class GRRFileCollectorTest(unittest.TestCase):
         mock_grr_hosts.MOCK_CLIENT_RECENT,
         'FileFinder',
         flows_pb2.FileFinderArgs(
-            paths=['/etc/passwd'],
+            paths=['/etc/passwd', '/etc/hosts'],
             action=flows_pb2.FileFinderAction(
                 action_type=flows_pb2.FileFinderAction.STAT,
                 download=flows_pb2.FileFinderDownloadActionOptions(
@@ -414,9 +422,8 @@ class GRRFileCollectorTest(unittest.TestCase):
     )
     results = self.test_state.GetContainers(containers.File)
     self.assertEqual(len(results), 1)
-    result = results[0]
-    self.assertEqual(result.name, 'tomchop')
-    self.assertEqual(result.path, '/tmp/something')
+    self.assertEqual(results[0].name, 'tomchop')
+    self.assertEqual(results[0].path, '/tmp/something')
 
   @mock.patch('grr_api_client.api.InitHttp')
   @mock.patch('dftimewolf.lib.collectors.grr_hosts.GRRFlow._AwaitFlow')
diff --git a/tests/lib/collectors/grr_hunt.py b/tests/lib/collectors/grr_hunt.py
index cf0b6e1..7e71d67 100644
--- a/tests/lib/collectors/grr_hunt.py
+++ b/tests/lib/collectors/grr_hunt.py
@@ -73,6 +73,7 @@ class GRRHuntFileCollectorTest(unittest.TestCase):
     self.mock_grr_api = mock.Mock()
     mock_InitHttp.return_value = self.mock_grr_api
     self.test_state = state.DFTimewolfState(config.Config)
+    self.test_state.StoreContainer(containers.FSPath(path='/etc/hosts'))
     self.grr_hunt_file_collector = grr_hunt.GRRHuntFileCollector(
         self.test_state)
     self.grr_hunt_file_collector.SetUp(
@@ -96,13 +97,21 @@ class GRRHuntFileCollectorTest(unittest.TestCase):
         ['/etc/passwd', '/etc/shadow']
     )
 
+  def testPreProcess(self):
+    """Tests the preprocess method."""
+    self.grr_hunt_file_collector.PreProcess()
+    self.assertEqual(
+        self.grr_hunt_file_collector.file_path_list,
+        ['/etc/passwd', '/etc/shadow', '/etc/hosts'])
+
   def testProcess(self):
     """Tests that the process method invokes the correct GRR API calls."""
+    self.grr_hunt_file_collector.PreProcess()
     self.grr_hunt_file_collector.Process()
     # extract call kwargs
     call_kwargs = self.mock_grr_api.CreateHunt.call_args[1]
     self.assertEqual(call_kwargs['flow_args'].paths,
-                     ['/etc/passwd', '/etc/shadow'])
+                     ['/etc/passwd', '/etc/shadow', '/etc/hosts'])
     self.assertEqual(call_kwargs['flow_args'].action.action_type,
                      flows_pb2.FileFinderAction.DOWNLOAD)
     self.assertEqual(call_kwargs['flow_name'], 'FileFinder')