Repository: google/turbinia

Issue:
Possible deadlock from excessive stdout in TurbiniaTask.execute()
When creating a task that uses `TurbiniaTask.execute()` to spawn a subprocess, and that subprocess generates a large amount of output on stdout (and probably stderr is affected also) the subprocess can finish what it's doing, but the subprocess will not actually terminate, and won't give control back to the task. I suspect it will after the configured timeout, but I didn't wait that long.

I suspect this is to do with the following note regarding `Popen.wait` from [0]:

```
Note: This will deadlock when using stdout=PIPE or stderr=PIPE and the child process generates enough output to a pipe such that it blocks waiting for the OS pipe buffer to accept more data. Use Popen.communicate() when using pipes to avoid that. 
```

I discovered this writing a task that uses log2timeline, and adding `--status_view none` to the command, thereby reducing the amount sent to stdout, rectified the issue.

[0] https://docs.python.org/3/library/subprocess.html


Discussion:

Resolving patch:
diff --git a/turbinia/workers/__init__.py b/turbinia/workers/__init__.py
--- a/turbinia/workers/__init__.py
+++ b/turbinia/workers/__init__.py
@@ -574,7 +574,7 @@ def get_metrics(self):
   def execute(
       self, cmd, result, save_files=None, log_files=None, new_evidence=None,
       close=False, shell=False, stderr_file=None, stdout_file=None,
-      success_codes=None, cwd=None, env=None):
+      success_codes=None, cwd=None, env=None, timeout=0):
     """Executes a given binary and saves output.
 
     Args:
@@ -592,6 +592,7 @@ def execute(
       stdout_file (str): Path to location to save stdout.
       cwd (str): Sets the current directory before the process is executed.
       env (list(str)): Process environment.
+      timeout (int): Override job timeout value.
 
     Returns:
       Tuple of the return code, and the TurbiniaTaskResult object
@@ -605,9 +606,13 @@ def execute(
     success_codes = success_codes if success_codes else [0]
     stdout = None
     stderr = None
+    fail_message = None
 
     # Get timeout value.
-    timeout_limit = job_manager.JobsManager.GetTimeoutValue(self.job_name)
+    if timeout:
+      timeout_limit = timeout
+    else:
+      timeout_limit = job_manager.JobsManager.GetTimeoutValue(self.job_name)
 
     # Execute the job via docker.
     docker_image = job_manager.JobsManager.GetDockerImage(self.job_name)
@@ -631,24 +636,24 @@ def execute(
           proc = subprocess.Popen(
               cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE,
               cwd=cwd, env=env)
-          proc.wait(timeout_limit)
+          stdout, stderr = proc.communicate(timeout=timeout_limit)
         else:
           proc = subprocess.Popen(
               cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, cwd=cwd,
               env=env)
-          proc.wait(timeout_limit)
+          stdout, stderr = proc.communicate(timeout=timeout_limit)
       except subprocess.TimeoutExpired as exception:
-        # Log error and close result.
-        message = (
+        proc.kill()
+        # Get any potential partial output so we can save it later.
+        stdout, stderr = proc.communicate()
+        fail_message = (
             'Execution of [{0!s}] failed due to job timeout of '
             '{1:d} seconds has been reached.'.format(cmd, timeout_limit))
-        result.log(message)
-        result.close(self, success=False, status=message)
-        # Increase timeout metric and raise exception
+        result.log(fail_message)
+        # Increase timeout metric. Not re-raising an exception so we can save
+        # any potential output.
         turbinia_worker_tasks_timeout_total.inc()
-        raise TurbiniaException(message)
 
-      stdout, stderr = proc.communicate()
       ret = proc.returncode
 
     result.error['stdout'] = str(stdout)
@@ -697,7 +702,9 @@ def execute(
       result.log('Output log file found at {0:s}'.format(file_))
       self.output_manager.save_local_file(file_, result)
 
-    if ret not in success_codes:
+    if fail_message:
+      result.close(self, success=False, status=fail_message)
+    elif ret not in success_codes:
       message = 'Execution of [{0!s}] failed with status {1:d}'.format(cmd, ret)
       result.log(message)
       if close:


Test patch:
diff --git a/turbinia/workers/workers_test.py b/turbinia/workers/workers_test.py
--- a/turbinia/workers/workers_test.py
+++ b/turbinia/workers/workers_test.py
@@ -284,6 +284,22 @@ def testTurbiniaTaskExecuteFailure(self, popen_mock):
     self.result.close.assert_called_with(
         self.task, success=False, status=mock.ANY)
 
+  def testTurbiniaTaskExecuteTimeout(self):
+    """Test execution with subprocess timeout case."""
+    cmd = 'sleep 10'
+    self.result.close = mock.MagicMock()
+    ret, result = self.task.execute(cmd, self.result, shell=True, timeout=1)
+
+    # Command was executed and TurbiniaTaskResult.close() was called with
+    # unsuccessful status.
+    self.result.close.assert_called_with(
+        self.task, success=False, status=mock.ANY)
+    result_call_args = self.result.close.call_args.kwargs
+    # 'timeout' string shows up in status message
+    self.assertIn('timeout', result_call_args['status'])
+    # Return value shows job was killed
+    self.assertEqual(ret, -9)
+
   @mock.patch('turbinia.workers.subprocess.Popen')
   def testTurbiniaTaskExecuteEvidenceExists(self, popen_mock):
     """Test execution with new evidence that has valid a source_path."""