Repository: DataBiosphere/toil

Issue:
Fix brittle and obscure assertion in jobCacheTest
See my comments on PR #898.

https://github.com/BD2KGenomics/toil/pull/898#discussion-diff-65220948



Discussion:

Resolving patch:
diff --git a/src/toil/job.py b/src/toil/job.py
--- a/src/toil/job.py
+++ b/src/toil/job.py
@@ -1288,9 +1288,7 @@ def deleteLocalFile(self, fileStoreID):
                                     logger.debug(fileToDelete,  'was read mutably and deleted by '
                                                  'the user')
                                 else:
-                                    raise CacheError('Illegal operation detected. Cache tracked'
-                                                     'file deleted explicitly by user. Use'
-                                                     'deleteLocalFile to delete such files.')
+                                    raise IllegalDeletionCacheError(fileToDelete)
                         allOwnedFiles[fileToDelete].remove(fileStoreID)
                         filesToDelete.pop(fileToDelete)
                         cacheInfo.jobState[self.hashedJobCommand] = jobState.__dict__
@@ -1300,9 +1298,7 @@ def deleteLocalFile(self, fileStoreID):
                     # Get the size of the file to be deleted, and the number of jobs using the file
                     # at the moment.
                     if not os.path.exists(fileToDelete):
-                        raise CacheError('Illegal operation detected. Cache tracked file deleted '
-                                         'explicitly by user. Use deleteLocalFile to delete such '
-                                         'files.')
+                        raise IllegalDeletionCacheError(fileToDelete)
                     fileStats = os.stat(fileToDelete)
                     if fileSize != fileStats.st_size:
                         logger.warn("the size on record differed from the real size by " +
@@ -1511,11 +1507,11 @@ def addToCache(self, localFilePath, jobStoreFileID, callingFunc, mutable=None):
                 # The file to be cached MUST originate in the environment of the TOIL temp directory
                 if (os.stat(self.localCacheDir).st_dev !=
                         os.stat(os.path.dirname(localFilePath)).st_dev):
-                    raise CacheInvalidSrcError('Attempting to cache a file across file systems '
+                    raise InvalidSourceCacheError('Attempting to cache a file across file systems '
                                                'cachedir = %s, file = %s.' % (self.localCacheDir,
                                                                               localFilePath))
                 if not localFilePath.startswith(self.localTempDir):
-                    raise CacheInvalidSrcError('Attempting a cache operation on a non-local file '
+                    raise InvalidSourceCacheError('Attempting a cache operation on a non-local file '
                                                '%s.' % localFilePath)
                 if callingFunc == 'read' and mutable:
                     shutil.copyfile(cachedFile, localFilePath)
@@ -2454,13 +2450,24 @@ def __init__(self, message):
         super(CacheError, self).__init__(message)
 
 
-class CacheInvalidSrcError(Exception):
+class IllegalDeletionCacheError(CacheError):
+    '''
+    Error Raised if the Toil detects the user deletes a cached file
+    '''
+
+    def __init__(self, deletedFile):
+        message = 'Cache tracked file (%s) deleted explicitly by user. Use deleteLocalFile to ' \
+                  'delete such files.' % deletedFile
+        super(IllegalDeletionCacheError, self).__init__(message)
+
+
+class InvalidSourceCacheError(CacheError):
     '''
     Error Raised if the user attempts to add a non-local file to cache
     '''
 
     def __init__(self, message):
-        super(CacheInvalidSrcError, self).__init__(message)
+        super(InvalidSourceCacheError, self).__init__(message)
 
 
 class FunctionWrappingJob(Job):


Test patch:
diff --git a/src/toil/test/src/jobCacheTest.py b/src/toil/test/src/jobCacheTest.py
--- a/src/toil/test/src/jobCacheTest.py
+++ b/src/toil/test/src/jobCacheTest.py
@@ -29,7 +29,7 @@
 from struct import pack, unpack
 from uuid import uuid4
 
-from toil.job import Job, CacheError
+from toil.job import Job, IllegalDeletionCacheError
 from toil.test import ToilTest, needs_aws, needs_azure, needs_google
 from toil.leader import FailedJobsException
 from toil.jobStores.abstractJobStore import NoSuchFileException
@@ -244,11 +244,12 @@ def _testCacheEviction(self, file1MB, file2MB, diskRequestMB):
                 Job.Runner.startToil(A, self.options)
             except FailedJobsException as err:
                 self.assertEqual(err.numberOfFailedJobs, 1)
-                errMsg = self._parseAssertionError(self.options.logFile)
-                if errMsg == 'Unable to free up enough space for caching.':
+                errType, errMsg = self._parseAssertionError(self.options.logFile)
+                if (errType == 'AssertionError' and
+                        errMsg == 'Unable to free up enough space for caching.'):
                     self.assertEqual(expectedResult, 'Fail')
                 else:
-                    self.fail('Shouldn\'t see this')
+                    self.fail('An AssertionError was not thrown by Toil')
 
         @staticmethod
         def _writeFileToJobStore(job, isLocalFile, nonLocalDir=None, fileMB=1):
@@ -334,7 +335,7 @@ def _parseAssertionError(logFile):
             Parse the assertion error message from a failed toil logfile
 
             :param logFile: path to the logfile
-            :return: Str of the error message
+            :return: tuple of (error type, string) of the error
             """
             workerLogName = None
             with open(logFile, 'r') as logFileHandle:
@@ -348,9 +349,9 @@ def _parseAssertionError(logFile):
                         continue
                     else:
                         fields = line.split()
-                        if fields[1] == 'AssertionError:' or fields[1] == 'CacheError:':
-                            return ' '.join(fields[2:])
-            raise RuntimeError('This shouldn\'t Happen')
+                        if fields[1].endswith('Error:'):
+                            return fields[1][:-1], ' '.join(fields[2:])
+            raise RuntimeError('An error was not found in the error log.')
 
         # writeGlobalFile tests
         def testWriteNonLocalFileToJobStore(self):
@@ -704,15 +705,15 @@ def testRemoveLocalImmutablyReadFile(self):
         def _deleteLocallyReadFilesFn(self, readAsMutable):
             self.options.retryCount = 0
             A = Job.wrapJobFn(self._writeFileToJobStore, isLocalFile=True, memory='10M')
-            B = Job.wrapJobFn(self._removeReadFileFn, A.rv(), readAsMutable=readAsMutable, memory='20M')
+            B = Job.wrapJobFn(self._removeReadFileFn, A.rv(), readAsMutable=readAsMutable,
+                              memory='20M')
             A.addChild(B)
             try:
                 Job.Runner.startToil(A, self.options)
             except FailedJobsException as err:
                 self.assertEqual(err.numberOfFailedJobs, 2)
-                errMsg = self._parseAssertionError(self.options.logFile)
-                if 'explicitly' not in errMsg:
-                    self.fail('Shouldn\'t see this')
+                errType, errMsg = self._parseAssertionError(self.options.logFile)
+                self.assertEqual(errType, 'IllegalDeletionCacheError')
 
         @staticmethod
         def _removeReadFileFn(job, fileToDelete, readAsMutable):
@@ -738,9 +739,8 @@ def _removeReadFileFn(job, fileToDelete, readAsMutable):
                 os.remove(outfile)
                 try:
                     job.fileStore.deleteLocalFile(fileToDelete)
-                except CacheError as err:
-                    if 'explicitly' not in err.message:
-                        raise
+                except IllegalDeletionCacheError:
+                    pass
                 else:
                     # If we are processing the write test, or if we are testing the immutably read
                     # file, we should not reach here.