Repository: google/turbinia

Issue:
Losetup device issues in post/pre processing.
As I was running some load tests, I discovered an issue with the Losetup devices not being deleted properly and may be either an issue with how partitions are being mounted or removed, either due to the PartitionEnumeration code or due to an unknown error. What makes it even more difficult is that so far, I've only been able to reproduce the error when the whole infrastructure is deployed in GKE, with running Worker pods that get scaled and descaled and unable to put a debugger on where the error is happening. 

Sample of errors observed
```
"Evidence post-processing for test-disk-25gb-1:/p15 failed: Could not delete losetup device /dev/loop1"
"Evidence post-processing for test-disk-25gb-4:/p1 failed: Could not delete losetup device /dev/loop1"
"Evidence post-processing for test-disk-25gb-3:/p15 failed: Could not delete losetup device /dev/loop1"
"Evidence post-processing for test-disk-25gb-1:/p1 failed: Could not delete losetup device /dev/loop1"
```




Discussion:
Getting some help from @aarontp, recommended that I try adding some Retry logic when the block device is being mounted to confirm that it is a block device. 

Added `IsBlockDevice` in `processors.mount_local.py`

```
def IsBlockDevice(path):
  """Checks path to determine whether it is a block device.

  Args:
      path: String of path to check.

  Returns:
      Bool indicating success.
  """
  if not os.path.exists(path):
    return False
  mode = os.stat(path).st_mode
  return stat.S_ISBLK(mode)
``` 

and retry logic in `processors.mount_local.PreprocessLosetup`
```
    # Make sure we have a proper block device
    for _ in range(RETRY_MAX):
      if IsBlockDevice(losetup_device):
        log.info(
            'Block device {0:s} successfully attached'.format(losetup_device))
        break
      if os.path.exists(losetup_device):
        log.info(
            'Block device {0:s} mode is {1}'.format(
                losetup_device,
                os.stat(losetup_device).st_mode))
      time.sleep(1)
    message = None
    if not os.path.exists(losetup_device):
      message = 'Device path {0:s} does not exist'.format(losetup_device)
    elif not IsBlockDevice(losetup_device):
      message = 'Device path {0:s} is not a block device'.format(losetup_device)
      if message:
        log.error(message)
        raise TurbiniaException(message)
```
 After running some tests, the above code doesn't seem to properly check if the device block is actually attached (due to seeing some errors trying to access the block devices manually)
@aarontp recommended to try calling `losetup -a` instead for checking if the Device has been properly attached.

In `processors.mount_local.py` replaced the `IsBlockDevice` function with `IsLosetup`
```
def IsLosetup(path):
  # Check that the device was attached
  losetup_cmd = ['sudo', 'losetup', '-a']
  log.info('Running: {0:s}'.format(' '.join(losetup_cmd)))
  try:
    output = subprocess.check_output(losetup_cmd)
  except subprocess.CalledProcessError as e:
    raise TurbiniaException(
        'Could not check losetup device status {0!s}'.format(e))
  if output.find(path.encode('utf-8')) != -1:
    return False
  log.info('losetup device [{0!s}] deleted.'.format(path))
  return True
```

Updated `processors.mount_local.PreprocessLosetup` to call `IsLosetup` instead
```
    # Make sure we have a proper block device
    for _ in range(RETRY_MAX):
      if IsLosetup(losetup_device):
        log.info(
            'Block device {0:s} successfully attached'.format(losetup_device))
        break
      if os.path.exists(losetup_device):
        log.info(
            'Block device {0:s} mode is {1}'.format(
                losetup_device,
                os.stat(losetup_device).st_mode))
      time.sleep(1)
    message = None
    if not os.path.exists(losetup_device):
      message = 'Device path {0:s} does not exist'.format(losetup_device)
    elif not IsLosetup(losetup_device):
      message = 'Device path {0:s} is not a block device'.format(losetup_device)
      if message:
        log.error(message)
        raise TurbiniaException(message)
``` 

After running some tests, there still seems to be an issue with detaching some of the loop devices by partition.
@aarontp next suggestion to also try adding the retry logic for when the loopback device is being detached to ensure that there is enough time for it to detach.

```
    losetup_cmd = ['sudo', 'losetup', '-d', device_path]
    for _ in range(RETRY_MAX):
      log.info('Running: {0:s}'.format(' '.join(losetup_cmd)))
      try:
        subprocess.check_call(losetup_cmd)
      except subprocess.CalledProcessError as e:
        turbinia_failed_loop_device_detach.inc()
        raise TurbiniaException(
            'Could not delete losetup device {0!s}'.format(e))
      # Check that the device was actually removed
      losetup_cmd = ['sudo', 'losetup', '-a']
      log.info('Running: {0:s}'.format(' '.join(losetup_cmd)))
      try:
        output = subprocess.check_output(losetup_cmd)
      except subprocess.CalledProcessError as e:
        raise TurbiniaException(
            'Could not check losetup device status {0!s}'.format(e))
      if output.find(device_path.encode('utf-8')) != -1:
        log.info(
            'Block device {0:s} mode is {1}'.format(
                device_path,
                os.stat(device_path).st_mode))
        else:
           break
      time.sleep(1)
    # Check if device failed.
    if output.find(device_path.encode('utf-8')) != -1:
      turbinia_failed_loop_device_detach.inc()
      raise TurbiniaException(
          'Could not delete losetup device {0!s}'.format(device_path))
    log.info('losetup device [{0!s}] deleted.'.format(device_path))
```

I ran some tests to see if the preprocess logic properly detected that the loop device is not a block device. 
```
Error running preprocessor for test-disk-25gb-15:/p1: Device path /dev/loop101 is not a block device
Device path /dev/loop101 is not a block device
Error running preprocessor for test-disk-25gb-15:/p1: Device path /dev/loop100 is not a block device
Device path /dev/loop100 is not a block device
```

I no longer see the `Could not delete losetup device` error message indicating that the preprocess is likely failing before the Task runs (given not equivalent device). However now seeing more failed Tasks as a result.
```
* YaraAnalysisTask (test-disk-25gb-6:/p1): Task Result was auto-closed from task executor on turbinia-worker-5899cf6d9c-48wkm likely due to previous failures.  Previous status: [YaraAnalysisTask Task failed with exception: [Evidence DiskPartition:test-disk-25gb-6:/p1:None being processed by Task YaraAnalysisTask requires Evidence to be in state ATTACHED, but earlier pre-processors may have failed.  Current state is [MOUNTED: False, ATTACHED: False, DECOMPRESSED: False, CONTAINER_MOUNTED: False]. See previous logs for more information.]]
--
* DockerContainersEnumerationTask (test-disk-25gb-7:/p1): Task Result was auto-closed from task executor on turbinia-worker-5899cf6d9c-gtwjd likely due to previous failures.  Previous status: [DockerContainersEnumerationTask Task failed with exception: [Evidence DiskPartition:test-disk-25gb-7:/p1:None being processed by Task DockerContainersEnumerationTask requires Evidence to be in state ATTACHED, but earlier pre-processors may have failed.  Current state is [MOUNTED: False, ATTACHED: False, DECOMPRESSED: False, CONTAINER_MOUNTED: False]. See previous logs for more information.]]
--
* StringsAsciiTask (test-disk-25gb-8:/p15): Task Result was auto-closed from task executor on turbinia-worker-5899cf6d9c-mcp5j likely due to previous failures.  Previous status: [StringsAsciiTask Task failed with exception: [Evidence DiskPartition:test-disk-25gb-8:/p15:None being processed by Task StringsAsciiTask requires Evidence to be in state ATTACHED, but earlier pre-processors may have failed.  Current state is [MOUNTED: False, ATTACHED: False, DECOMPRESSED: False, CONTAINER_MOUNTED: False]. See previous logs for more information.]]
--
FsstatTask (test-disk-25gb-9:/p1): Task Result was auto-closed from task executor on turbinia-worker-5899cf6d9c-tzflh likely due to previous failures.  Previous status: [FsstatTask Task failed with exception: [Evidence DiskPartition:test-disk-25gb-9:/p1:None being processed by Task FsstatTask requires Evidence to be in state ATTACHED, but earlier pre-processors may have failed.  Current state is [MOUNTED: False, ATTACHED: False, DECOMPRESSED: False, CONTAINER_MOUNTED: False]. See previous logs for more information.]]
```
@aarontp this is where I got stuck, if you have time and can PTAL per our chats around this.

Resolving patch:
diff --git a/turbinia/processors/mount_local.py b/turbinia/processors/mount_local.py
--- a/turbinia/processors/mount_local.py
+++ b/turbinia/processors/mount_local.py
@@ -21,6 +21,9 @@
 import subprocess
 import tempfile
 import time
+import filelock
+import re
+
 from prometheus_client import Gauge
 from turbinia import config
 from turbinia import TurbiniaException
@@ -209,11 +212,16 @@ def PreprocessLosetup(
     losetup_command.append(source_path)
     log.info('Running command {0:s}'.format(' '.join(losetup_command)))
     try:
-      losetup_device = subprocess.check_output(
-          losetup_command, universal_newlines=True).strip()
+      # File lock to prevent race condition with PostProcessLosetup.
+      with filelock.FileLock(config.RESOURCE_FILE_LOCK):
+        losetup_device = subprocess.check_output(
+            losetup_command, universal_newlines=True).strip()
     except subprocess.CalledProcessError as exception:
       raise TurbiniaException(
           'Could not set losetup devices {0!s}'.format(exception))
+    log.info(
+        'Loop device {0:s} created for evidence {1:s}'.format(
+            losetup_device, source_path))
 
   return losetup_device
 
@@ -477,25 +485,39 @@ def PostprocessDeleteLosetup(device_path, lv_uuid=None):
     # https://github.com/google/turbinia/issues/73
     losetup_cmd = ['sudo', 'losetup', '-d', device_path]
     log.info('Running: {0:s}'.format(' '.join(losetup_cmd)))
-    try:
-      subprocess.check_call(losetup_cmd)
-    except subprocess.CalledProcessError as exception:
+    # File lock to prevent race condition with PreProcessLosetup
+    with filelock.FileLock(config.RESOURCE_FILE_LOCK):
+      try:
+        subprocess.check_call(losetup_cmd)
+      except subprocess.CalledProcessError as exception:
+        turbinia_failed_loop_device_detach.inc()
+        raise TurbiniaException(
+            'Could not delete losetup device {0!s}'.format(exception))
+
+      # Check that the device was actually removed
+      losetup_cmd = ['sudo', 'losetup', '-a']
+      for _ in range(RETRY_MAX):
+        try:
+          output = subprocess.check_output(losetup_cmd, text=True)
+        except subprocess.CalledProcessError as exception:
+          raise TurbiniaException(
+              'Could not check losetup device status {0!s}'.format(exception))
+        reg_search = re.search(device_path + ':.*', output)
+        if reg_search:
+          # TODO(wyassine): Add lsof check for file handles on device path
+          # https://github.com/google/turbinia/issues/1148
+          log.debug('losetup retry check {0!s}/{1!s} for device {2!s}').format(
+              _, RETRY_MAX, device_path)
+          time.sleep(1)
+        else:
+          break
+    # Raise if losetup device still exists
+    if reg_search:
       turbinia_failed_loop_device_detach.inc()
       raise TurbiniaException(
-          'Could not delete losetup device {0!s}'.format(exception))
+          'losetup device still present, unable to delete the device {0!s}'
+          .format(device_path))
 
-    # Check that the device was actually removed
-    losetup_cmd = ['sudo', 'losetup', '-a']
-    log.info('Running: {0:s}'.format(' '.join(losetup_cmd)))
-    try:
-      output = subprocess.check_output(losetup_cmd)
-    except subprocess.CalledProcessError as exception:
-      raise TurbiniaException(
-          'Could not check losetup device status {0!s}'.format(exception))
-    if output.find(device_path.encode('utf-8')) != -1:
-      turbinia_failed_loop_device_detach.inc()
-      raise TurbiniaException(
-          'Could not delete losetup device {0!s}'.format(device_path))
     log.info('losetup device [{0!s}] deleted.'.format(device_path))
 
 


Test patch:
diff --git a/turbinia/processors/mount_local_test.py b/turbinia/processors/mount_local_test.py
--- a/turbinia/processors/mount_local_test.py
+++ b/turbinia/processors/mount_local_test.py
@@ -145,11 +145,13 @@ def testPreprocessBitLocker(
         source_path, partition_offset=65536, credentials=credentials)
     mock_subprocess.assert_not_called()
 
+  @mock.patch('turbinia.processors.mount_local.config')
   @mock.patch('subprocess.check_output')
-  def testPreprocessLosetup(self, mock_subprocess):
+  def testPreprocessLosetup(self, mock_subprocess, mock_config):
     """Test PreprocessLosetup method."""
     current_path = os.path.abspath(os.path.dirname(__file__))
     source_path = os.path.join(current_path, '..', '..', 'test_data', 'mbr.raw')
+    mock_config.RESOURCE_FILE_LOCK = '/tmp/turbinia_resource.lock'
     mock_subprocess.return_value = '/dev/loop0'
     device = mount_local.PreprocessLosetup(source_path)
     expected_args = ['sudo', 'losetup', '--show', '--find', '-r', source_path]
@@ -330,11 +332,14 @@ def testGetFilesystem(self, mock_subprocess):
     self.assertEqual(fstype, 'ext4')
     self.assertEqual(mock_subprocess.call_count, 2)
 
+  @mock.patch('turbinia.processors.mount_local.config')
   @mock.patch('subprocess.check_output')
   @mock.patch('subprocess.check_call')
-  def testPostprocessDeleteLosetup(self, mock_subprocess, mock_output):
+  def testPostprocessDeleteLosetup(
+      self, mock_subprocess, mock_output, mock_config):
     """Test PostprocessDeleteLosetup method."""
-    mock_output.return_value = b''
+    mock_config.RESOURCE_FILE_LOCK = '/tmp/turbinia_resource.lock'
+    mock_output.return_value = ''
     mount_local.PostprocessDeleteLosetup('/dev/loop0')
     mock_subprocess.assert_called_once_with(
         ['sudo', 'losetup', '-d', '/dev/loop0'])