Repository: DataBiosphere/toil

Issue:
Add back ability to disable caching
Create a second FileStore implementation that can fill in for the current caching one. Add `--disableCaching` and mention it in the error message that is raised when a BS is used that doesn't support the caching FileStore.



Discussion:
Arjun volunteered! Due to his time constraints, we'll leave testing out of it for now if @chapmanb is willing to test it manually. Ultimately, it would be nice if we could run some tests agains both FileStore implementations.

Arjun and Hannes -- this is awesome, thanks so much for helping to put this in place. I'm happy to help with testing and anything else needed once you have an opportunity to look it.

@cket , this is the ticket I've been talking about in today's scrum. @arkal and I looked at the code and determined that the most difficult aspect of this is how worker and FileStore interact for the asynchronous aka background writing of local files to the job store. The new non-caching FileStore implementation should not do background writing so you only need to study that interaction up to the point of being able to stub it out with something that "does nothing" or "immediately returns" or "does it immediately instead of in the background" if YKWIM. The less code you change in worker, the better.

Part two of this is testing. Think about a way of how the new FileStore can be covered by  the existing tests, e.g. by running a test with and without `--disableCaching`.


Resolving patch:
diff --git a/src/toil/common.py b/src/toil/common.py
--- a/src/toil/common.py
+++ b/src/toil/common.py
@@ -95,6 +95,7 @@ def __init__(self):
         self.rescueJobsFrequency = 3600
 
         #Misc
+        self.disableCaching = False
         self.maxLogFileSize=50120
         self.sseKey = None
         self.cseKey = None
@@ -202,6 +203,7 @@ def fC(minValue, maxValue=None):
         setOption("rescueJobsFrequency", int, iC(1))
 
         #Misc
+        setOption("disableCaching")
         setOption("maxLogFileSize", h2b, iC(1))
         def checkSse(sseKey):
             with open(sseKey) as f:
@@ -407,6 +409,10 @@ def _addOptionFn(*name, **kwargs):
     #Misc options
     #
     addOptionFn = addGroupFn("toil miscellaneous options", "Miscellaneous options")
+    addOptionFn('--disableCaching', dest='disableCaching', action='store_true', default=False,
+                help='Disables caching in the file store. This flag must be set to use '
+                     'a batch system that does not support caching such as Grid Engine, Parasol, '
+                     'LSF, or Slurm')
     addOptionFn("--maxLogFileSize", dest="maxLogFileSize", default=None,
                       help=("The maximum size of a job log file to keep (in bytes), log files larger "
                             "than this will be truncated to the last X bytes. Default is 50 "
@@ -676,8 +682,8 @@ def createBatchSystem(config, jobStore=None, userScript=None):
             raise RuntimeError('Unrecognised batch system: %s' % config.batchSystem)
 
         if not batchSystemClass.supportsWorkerCleanup():
-            raise RuntimeError('%s currently does not support shared caching.  Use Toil version '
-                               '3.1.6 along with the --disableSharedCache option if you want to '
+            raise RuntimeError('%s currently does not support shared caching.  Set the '
+                               '--disableCaching flag if you want to '
                                'use this batch system.' % config.batchSystem)
         logger.info('Using the %s' %
                     re.sub("([a-z])([A-Z])", "\g<1> \g<2>", batchSystemClass.__name__).lower())
diff --git a/src/toil/fileStore.py b/src/toil/fileStore.py
--- a/src/toil/fileStore.py
+++ b/src/toil/fileStore.py
@@ -92,6 +92,11 @@ def __init__(self, jobStore, jobWrapper, localTempDir, inputBlockFn):
         # job if required.
         self._setupCache()
 
+    @staticmethod
+    def createFileStore(jobStore, jobWrapper, localTempDir, inputBlockFn, caching):
+        fileStoreCls = FileStore if caching else NonCachingFileStore
+        return fileStoreCls(jobStore, jobWrapper, localTempDir, inputBlockFn)
+
     @contextmanager
     def open(self, job):
         '''
@@ -442,8 +447,9 @@ def deleteLocalFile(self, fileStoreID):
         # False.
         with self._CacheState.open(self) as cacheInfo:
             jobState = self._JobState(cacheInfo.jobState[self.hashedJobCommand])
-            assert fileStoreID in jobState.jobSpecificFiles.keys(), 'Attempting to delete ' + \
-                                                                    'a non-local file'
+            if fileStoreID not in jobState.jobSpecificFiles.keys():
+                # EOENT indicates that the file did not exist
+                raise OSError(errno.ENOENT, "Attempting to delete a non-local file")
             # filesToDelete is a dictionary of file: fileSize
             filesToDelete = jobState.jobSpecificFiles[fileStoreID]
             allOwnedFiles = jobState.filesToFSIDs
@@ -1306,7 +1312,9 @@ def asyncUpdate():
 
     def _blockFn(self):
         """
-        Blocks while _updateJobWhenDone is running.
+        Blocks while _updateJobWhenDone is running. This function is called by this job's
+        successor to insure that it does not begin modifying the job store until after this job has
+        finished doing so.
         """
         self.updateSemaphore.acquire()
         self.updateSemaphore.release()  # Release so that the block function can be recalled
@@ -1326,6 +1334,96 @@ def __del__(self):
         self.updateSemaphore.release()
 
 
+class NonCachingFileStore(FileStore):
+
+    def __init__(self, jobStore, jobWrapper, localTempDir, inputBlockFn):
+        self.jobStore = jobStore
+        self.jobWrapper = jobWrapper
+        self.localTempDir = os.path.abspath(localTempDir)
+        self.inputBlockFn = inputBlockFn
+        self.jobsToDelete = set()
+        self.loggingMessages = []
+        self.localFileMap = {}
+        self.filesToDelete = set()
+
+    @contextmanager
+    def open(self, job):
+        startingDir = os.getcwd()
+        self.localTempDir = makePublicDir(os.path.join(self.localTempDir, str(uuid.uuid4())))
+        try:
+            os.chdir(self.localTempDir)
+            yield
+        finally:
+            os.chdir(startingDir)
+
+    def writeGlobalFile(self, localFileName, cleanup=False):
+        absLocalFileName = self._abspath(localFileName)
+        cleanupID = None if not cleanup else self.jobWrapper.jobStoreID
+        return self.jobStore.writeFile(absLocalFileName, cleanupID)
+
+    def readGlobalFile(self, fileStoreID, userPath=None, cache=True, mutable=None):
+        if userPath is not None:
+            localFilePath = self._abspath(userPath)
+            if os.path.exists(localFilePath):
+                raise RuntimeError(' File %s ' % localFilePath + ' exists. Cannot Overwrite.')
+        else:
+            localFilePath = self.getLocalTempFileName()
+
+        self.jobStore.readFile(fileStoreID, localFilePath)
+        self.localFileMap[fileStoreID] = localFilePath
+        return localFilePath
+
+    @contextmanager
+    def readGlobalFileStream(self, fileStoreID):
+        with self.jobStore.readFileStream(fileStoreID) as f:
+            yield f
+
+    def deleteLocalFile(self, fileStoreID):
+        try:
+            localFilePath = self.localFileMap.pop(fileStoreID)
+        except KeyError:
+            raise OSError(errno.ENOENT, "Attempting to delete a non-local file")
+        else:
+            os.remove(localFilePath)
+
+    def deleteGlobalFile(self, fileStoreID):
+        try:
+            self.deleteLocalFile(fileStoreID)
+        except OSError as e:
+            if e.errno == errno.ENOENT:
+                # the file does not exist locally, so no local deletion necessary
+                pass
+            else:
+                raise
+        self.filesToDelete.add(fileStoreID)
+
+    def _blockFn(self):
+        # there is no asynchronicity in this file store so no need to block at all
+        return True
+
+    def _updateJobWhenDone(self):
+        try:
+            # Indicate any files that should be deleted once the update of
+            # the job wrapper is completed.
+            self.jobWrapper.filesToDelete = list(self.filesToDelete)
+            # Complete the job
+            self.jobStore.update(self.jobWrapper)
+            # Delete any remnant jobs
+            map(self.jobStore.delete, self.jobsToDelete)
+            # Delete any remnant files
+            map(self.jobStore.deleteFile, self.filesToDelete)
+            # Remove the files to delete list, having successfully removed the files
+            if len(self.filesToDelete) > 0:
+                self.jobWrapper.filesToDelete = []
+                # Update, removing emptying files to delete
+                self.jobStore.update(self.jobWrapper)
+        except:
+            self._terminateEvent.set()
+            raise
+
+    def __del__(self):
+        pass
+
 class FileID(str):
     """
     A class to wrap the job store file id returned by writeGlobalFile and any attributes we may want
diff --git a/src/toil/worker.py b/src/toil/worker.py
--- a/src/toil/worker.py
+++ b/src/toil/worker.py
@@ -319,7 +319,8 @@ def recursiveDelete(jobWrapper2):
                     jobWrapper.checkpoint = jobWrapper.command
 
                 # Create a fileStore object for the job
-                fileStore = FileStore(jobStore, jobWrapper, localWorkerTempDir, blockFn)
+                fileStore = FileStore.createFileStore(jobStore, jobWrapper, localWorkerTempDir, blockFn,
+                                                      caching=not config.disableCaching)
                 with job._executor(jobWrapper=jobWrapper,
                                    stats=statsDict if config.stats else None,
                                    fileStore=fileStore):
@@ -421,8 +422,9 @@ def recursiveDelete(jobWrapper2):
             assert jobWrapper.cores >= successorJobWrapper.cores
             
             #Build a fileStore to update the job
-            fileStore = FileStore(jobStore, jobWrapper, localWorkerTempDir, blockFn)
-            
+            fileStore = FileStore.createFileStore(jobStore, jobWrapper, localWorkerTempDir, blockFn,
+                                                  caching=not config.disableCaching)
+
             #Update blockFn
             blockFn = fileStore._blockFn
             


Test patch:
diff --git a/src/toil/test/sort/sortTest.py b/src/toil/test/sort/sortTest.py
--- a/src/toil/test/sort/sortTest.py
+++ b/src/toil/test/sort/sortTest.py
@@ -57,7 +57,7 @@ def setUp(self):
 
     def _toilSort(self, jobStoreLocator, batchSystem,
                   lines=defaultLines, N=defaultN, testNo=1, lineLen=defaultLineLen,
-                  retryCount=2, badWorker=0.5, downCheckpoints=False):
+                  retryCount=2, badWorker=0.5, downCheckpoints=False, disableCaching=False):
         """
         Generate a file consisting of the given number of random lines, each line of the given
         length. Sort the file with Toil by splitting the file recursively until each part is less
@@ -86,6 +86,7 @@ def _toilSort(self, jobStoreLocator, batchSystem,
                 options.clean = "never"
                 options.badWorker = badWorker
                 options.badWorkerFailInterval = 0.05
+                options.disableCaching = disableCaching
 
                 # Make the file to sort
                 tempSortFile = os.path.join(self.tempDir, "fileToSort.txt")
@@ -213,6 +214,10 @@ def testGoogleMesos(self):
     def testFileSingle(self):
         self._toilSort(jobStoreLocator=self._getTestJobStorePath(), batchSystem='singleMachine')
 
+    def testFileSingleNonCaching(self):
+        self._toilSort(jobStoreLocator=self._getTestJobStorePath(), batchSystem='singleMachine',
+                       disableCaching=True)
+
     def testFileSingleCheckpoints(self):
         self._toilSort(jobStoreLocator=self._getTestJobStorePath(), batchSystem='singleMachine', retryCount=2, downCheckpoints=True)
 
diff --git a/src/toil/test/src/jobFileStoreTest.py b/src/toil/test/src/jobFileStoreTest.py
--- a/src/toil/test/src/jobFileStoreTest.py
+++ b/src/toil/test/src/jobFileStoreTest.py
@@ -14,6 +14,9 @@
 from __future__ import absolute_import
 import random
 import os
+import errno
+
+from toil.common import Toil
 from toil.job import Job
 from toil.test import ToilTest
 
@@ -23,7 +26,18 @@ class JobFileStoreTest(ToilTest):
     """
     Tests testing the Job.FileStore class
     """
-    def testJobFileStoreWithSmallCache(self, retryCount=0, badWorker=0.0, 
+    def testCachingFileStore(self):
+        options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
+        with Toil(options) as workflow:
+            workflow.start(Job.wrapJobFn(simpleFileStoreJob))
+
+    def testNonCachingFileStore(self):
+        options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
+        options.disableCaching = True
+        with Toil(options) as workflow:
+            workflow.start(Job.wrapJobFn(simpleFileStoreJob))
+
+    def testJobFileStoreWithSmallCache(self, retryCount=0, badWorker=0.0,
                          stringNo=1, stringLength=1000000, cacheSize=10000, testNo=2):
         """
         Creates a chain of jobs, each reading and writing files using the 
@@ -130,3 +144,45 @@ def fileTestJob(job, inputFileStoreIDs, testStrings, chainLength):
     if chainLength > 0:
         #Make a child that will read these files and check it gets the same results
         job.addChildJobFn(fileTestJob, outputFileStoreIds, testStrings, chainLength-1)
+
+
+fileStoreString = "Testing writeGlobalFile"
+streamingFileStoreString = "Testing writeGlobalFileStream"
+
+
+def simpleFileStoreJob(job):
+    localFilePath = os.path.join(job.fileStore.getLocalTempDir(), "parentTemp.txt")
+    with open(localFilePath, 'w') as f:
+        f.write(fileStoreString)
+    testID1 = job.fileStore.writeGlobalFile(localFilePath)
+
+    testID2 = None
+    with job.fileStore.writeGlobalFileStream() as (f, fileID):
+        f.write(streamingFileStoreString)
+        testID2 = fileID
+
+    job.addChildJobFn(fileStoreChild, testID1, testID2)
+
+
+def fileStoreChild(job, testID1, testID2):
+    with job.fileStore.readGlobalFileStream(testID1) as f:
+        assert(f.read() == fileStoreString)
+
+    localFilePath = os.path.join(job.fileStore.getLocalTempDir(), "childTemp.txt")
+    job.fileStore.readGlobalFile(testID2, localFilePath)
+    with open(localFilePath, 'r') as f:
+        assert(f.read() == streamingFileStoreString)
+
+    job.fileStore.deleteLocalFile(testID2)
+    try:
+        job.fileStore.deleteLocalFile(testID1)
+    except OSError as e:
+        if e.errno == errno.ENOENT:  # indicates that the file was not found
+            pass
+        else:
+            raise
+    else:
+        raise RuntimeError("Deleting a non-existant file did not throw an exception")
+
+    for fileID in (testID1, testID2):
+        job.fileStore.deleteGlobalFile(fileID)