Repository: DataBiosphere/toil

Issue:
Add defer to inferior filestore
Depended on by #987

The uncached filestore does not support defer and it should. Proposing to add a jobstate file to each job's working directory such that it can be used to store defer information

Add defer to inferior filestore
Depended on by #987

The uncached filestore does not support defer and it should. Proposing to add a jobstate file to each job's working directory such that it can be used to store defer information



Discussion:

Resolving patch:
diff --git a/src/toil/batchSystems/abstractBatchSystem.py b/src/toil/batchSystems/abstractBatchSystem.py
--- a/src/toil/batchSystems/abstractBatchSystem.py
+++ b/src/toil/batchSystems/abstractBatchSystem.py
@@ -23,7 +23,7 @@
 from bd2k.util.objects import abstractclassmethod
 
 from toil.common import Toil, cacheDirName
-from toil.fileStore import shutdownCache
+from toil.fileStore import shutdownFileStore
 
 # A class containing the information required for worker cleanup on shutdown of the batch system.
 WorkerCleanupInfo = namedtuple('WorkerCleanupInfo', (
@@ -302,7 +302,7 @@ def workerCleanup(info):
         assert isinstance(info, WorkerCleanupInfo)
         workflowDir = Toil.getWorkflowDir(info.workflowID, info.workDir)
         workflowDirContents = os.listdir(workflowDir)
-        shutdownCache(os.path.join(workflowDir, cacheDirName(info.workflowID)))
+        shutdownFileStore(workflowDir, info.workflowID)
         if (info.cleanWorkDir == 'always'
             or info.cleanWorkDir in ('onSuccess', 'onError')
             and workflowDirContents in ([], [cacheDirName(info.workflowID)])):
diff --git a/src/toil/fileStore.py b/src/toil/fileStore.py
--- a/src/toil/fileStore.py
+++ b/src/toil/fileStore.py
@@ -13,10 +13,23 @@
 # limitations under the License.
 
 from __future__ import absolute_import, print_function
+from abc import abstractmethod, ABCMeta
+from contextlib import contextmanager
+from fcntl import flock, LOCK_EX, LOCK_UN
+from functools import partial
+from hashlib import sha1
+from threading import Thread, Semaphore, Event
+from Queue import Queue, Empty
+
+
+from bd2k.util.humanize import bytes2human
+from bd2k.util.objects import abstractclassmethod
+from toil.common import cacheDirName, getDirSizeRecursively
+from toil.lib.bioio import makePublicDir
 
 import base64
-import dill
 import collections
+import dill
 import errno
 import logging
 import os
@@ -26,17 +39,6 @@
 import time
 import uuid
 
-from contextlib import contextmanager
-from fcntl import flock, LOCK_EX, LOCK_UN
-from functools import partial
-from hashlib import sha1
-from Queue import Queue, Empty
-from threading import Thread, Semaphore, Event
-
-from bd2k.util.humanize import bytes2human
-from toil.common import cacheDirName, getDirSizeRecursively
-from toil.lib.bioio import makePublicDir
-
 logger = logging.getLogger( __name__ )
 
 DeferredFunction = collections.namedtuple('DeferredFunction', (
@@ -49,26 +51,352 @@
 
 
 class FileStore(object):
-    '''
-    Class used to manage temporary files, read and write files from the job store\
-    and log messages, passed as argument to the :func:`toil.job.Job.run` method.
-    '''
+    """
+    An abstract base class to represent the interface between a worker and the job store.  Concrete
+    subclasses will be used to manage temporary files, read and write files from the job store and
+    log messages, passed as argument to the :meth:`toil.job.Job.run` method.
+    """
     # Variables used for syncing reads/writes
     _pendingFileWritesLock = Semaphore()
     _pendingFileWrites = set()
-    # For files in jobStore that are on the local disk,
-    # map of jobStoreFileIDs to locations in localTempDir.
-    _jobStoreFileIDToCacheLocation = {}
     _terminateEvent = Event()  # Used to signify crashes in threads
 
+    __metaclass__ = ABCMeta
+
     def __init__(self, jobStore, jobGraph, localTempDir, inputBlockFn):
         self.jobStore = jobStore
         self.jobGraph = jobGraph
         self.localTempDir = os.path.abspath(localTempDir)
+        self.workFlowDir = os.path.dirname(self.localTempDir)
+        self.jobName = self.jobGraph.command.split()[1]
         self.inputBlockFn = inputBlockFn
         self.loggingMessages = []
         self.filesToDelete = set()
         self.jobsToDelete = set()
+
+    @staticmethod
+    def createFileStore(jobStore, jobGraph, localTempDir, inputBlockFn, caching):
+        fileStoreCls = CachingFileStore if caching else NonCachingFileStore
+        return fileStoreCls(jobStore, jobGraph, localTempDir, inputBlockFn)
+
+    @abstractmethod
+    @contextmanager
+    def open(self, job):
+        """
+        The context manager used to conduct tasks prior-to, and after a job has been run.
+
+        :param toil.job.Job job: The job instance of the toil job to run.
+        """
+        raise NotImplementedError()
+
+    # Functions related to temp files and directories
+    def getLocalTempDir(self):
+        """
+        Get a new local temporary directory in which to write files that persist for the duration of
+        the job.
+
+        :return: The absolute path to a new local temporary directory. This directory will exist
+        for the duration of the job only, and is guaranteed to be deleted once the job terminates,
+        removing all files it contains recursively.
+        :rtype: str
+        """
+        return os.path.abspath(tempfile.mkdtemp(prefix="t", dir=self.localTempDir))
+
+    def getLocalTempFile(self):
+        """
+        Get a new local temporary file that will persist for the duration of the job.
+
+        :return: The absolute path to a local temporary file. This file will exist for the duration
+        of the job only, and is guaranteed to be deleted once the job terminates.
+        :rtype: str
+        """
+        handle, tmpFile = tempfile.mkstemp(prefix="tmp", suffix=".tmp", dir=self.localTempDir)
+        os.close(handle)
+        return os.path.abspath(tmpFile)
+
+    def getLocalTempFileName(self):
+        """
+        Get a valid name for a new local file. Don't actually create a file at the path.
+
+        :return: Path to valid file
+        :rtype: str
+        """
+        # Create, and then delete a temp file. Creating will guarantee you a unique, unused
+        # file name. There is a very, very, very low chance that another job will create the
+        # same file name in the span of this one being deleted and then being used by the user.
+        tempFile = self.getLocalTempFile()
+        os.remove(tempFile)
+        return tempFile
+
+    # Functions related to reading, writing and removing files to/from the job store
+    @abstractmethod
+    def writeGlobalFile(self, localFileName, cleanup=False):
+        """
+        Takes a file (as a path) and uploads it to the job store.
+
+        :param string localFileName: The path to the local file to upload.
+        :param Boolean cleanup: if True then the copy of the global file will be deleted once the
+               job and all its successors have completed running.  If not the global file must be
+               deleted manually.
+        :return: an ID that can be used to retrieve the file.
+        :rtype: FileID
+        """
+        raise NotImplementedError()
+
+    def writeGlobalFileStream(self, cleanup=False):
+        """
+        Similar to writeGlobalFile, but allows the writing of a stream to the job store.
+        The yielded file handle does not need to and should not be closed explicitly.
+
+        :param Boolean cleanup: is as in :func:`toil.fileStore.FileStore.writeGlobalFile`.
+        :return: A context manager yielding a tuple of
+                  1) a file handle which can be written to and
+                  2) the ID of the resulting file in the job store.
+        """
+        # TODO: Make this work with FileID
+        return self.jobStore.writeFileStream(None if not cleanup else self.jobGraph.jobStoreID)
+
+    @abstractmethod
+    def readGlobalFile(self, fileStoreID, userPath=None, cache=True, mutable=None):
+        """
+        Downloads a file described by fileStoreID from the file store to the local directory.
+
+        If a user path is specified, it is used as the destination. If a user path isn't
+        specified, the file is stored in the local temp directory with an encoded name.
+
+        :param FileID fileStoreID: job store id for the file
+        :param string userPath: a path to the name of file to which the global file will be copied
+               or hard-linked (see below).
+        :param boolean cache: Described in :func:`~filestore.Filestore.readGlobalFile`
+        :param boolean mutable: Described in :func:`~filestore.Filestore.readGlobalFile`
+        :return: An absolute path to a local, temporary copy of the file keyed by fileStoreID.
+        :rtype: str
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def readGlobalFileStream(self, fileStoreID):
+        """
+        Similar to readGlobalFile, but allows a stream to be read from the job store. The yielded
+        file handle does not need to and should not be closed explicitly.
+
+        :return: a context manager yielding a file handle which can be read from.
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def deleteLocalFile(self, fileStoreID):
+        """
+        Deletes Local copies of files associated with the provided job store ID.
+
+        :param str fileStoreID: File Store ID of the file to be deleted.
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def deleteGlobalFile(self, fileStoreID):
+        """
+        Deletes local files with the provided job store ID and then permanently deletes them from
+        the job store. To ensure that the job can be restarted if necessary, the delete will not
+        happen until after the job's run method has completed.
+
+        :param fileStoreID: the job store ID of the file to be deleted.
+        """
+        raise NotImplementedError()
+
+    # Functions used to read and write files directly between a source url and the job store.
+    def importFile(self, srcUrl, sharedFileName=None):
+        return self.jobStore.importFile(srcUrl, sharedFileName=sharedFileName)
+
+    def exportFile(self, jobStoreFileID, dstUrl):
+        self.jobStore.exportFile(jobStoreFileID, dstUrl)
+
+    # A utility method for accessing filenames
+    def _resolveAbsoluteLocalPath(self, filePath):
+        """
+        Return the absolute path to filePath.  This is a wrapper for os.path.abspath because mac OS
+        symlinks /tmp and /var (the most common places for a default tempdir) to /private/tmp and
+        /private/var respectively.
+
+        :param str filePath: The absolute or relative path to the file. If relative, it must be
+               relative to the local temp working dir
+        :return: Absolute path to key
+        :rtype: str
+        """
+        if os.path.isabs(filePath):
+            return os.path.abspath(filePath)
+        else:
+            return os.path.join(self.localTempDir, filePath)
+
+    class _StateFile(object):
+        """
+        Utility class to read and write dill-ed state dictionaries from/to a file into a namespace.
+        """
+        def __init__(self, stateDict):
+            assert isinstance(stateDict, dict)
+            self.__dict__.update(stateDict)
+
+        @abstractclassmethod
+        @contextmanager
+        def open(cls, outer=None):
+            """
+            This is a context manager that state file and reads it into an object that is returned
+            to the user in the yield.
+
+            :param outer: Instance of the calling class (to use outer methods).
+            """
+            raise NotImplementedError()
+
+        @classmethod
+        def _load(cls, fileName):
+            """
+            Load the state of the cache from the state file
+
+            :param str fileName: Path to the cache state file.
+            :return: An instance of the state as a namespace.
+            :rtype: _StateFile
+            """
+            # Read the value from the cache state file then initialize and instance of
+            # _CacheState with it.
+            with open(fileName, 'r') as fH:
+                infoDict = dill.load(fH)
+            return cls(infoDict)
+
+        def write(self, fileName):
+            """
+            Write the current state into a temporary file then atomically rename it to the main
+            state file.
+
+            :param str fileName: Path to the state file.
+            """
+            with open(fileName + '.tmp', 'w') as fH:
+                # Based on answer by user "Mark" at:
+                # http://stackoverflow.com/questions/2709800/how-to-pickle-yourself
+                # We can't pickle nested classes. So we have to pickle the variables of the class
+                # If we ever change this, we need to ensure it doesn't break FileID
+                dill.dump(self.__dict__, fH)
+            os.rename(fileName + '.tmp', fileName)
+
+    # Methods related to the deferred function logic
+    @abstractclassmethod
+    def findAndHandleDeadJobs(cls, nodeInfo, batchSystemShutdown=False):
+        """
+        This function looks at the state of all jobs registered on the node and will handle them
+        (clean up their presence ont he node, and run any registered defer functions)
+
+        :param nodeInfo: Information regarding the node required for identifying dead jobs.
+        :param bool batchSystemShutdown: Is the batch system in the process of shutting down?
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def _registerDeferredFunction(self, callable_, *args, **kwargs):
+        """
+        This is the backend function to register a function that will be run after the completion of
+        the current job on the same node
+
+        :param callable_: The function or lambda to be run after this job -- Must return True for
+               callable(callable_).
+        :param list args: The arguments to the function.
+        :param dict kwargs: The keyword arguments to the function.
+        """
+        raise NotImplementedError()
+
+    @staticmethod
+    def _runDeferredFunctions(deferredFunctions):
+        """
+        This function will run all the registered deferred functions for the job.
+
+        :param list deferredFunctions: A list of DeferredFunctions to run the cache
+        :return: list of failed functions
+        :rtype: list
+        """
+        failures = []
+        for deferredFunction in deferredFunctions:
+            # TODO: better representation of the callable in the output string
+            try:
+                logger.debug('Running deferred function "%s".', deferredFunction.callable.__name__)
+                deferredFunction.callable(*deferredFunction.args, **deferredFunction.kwargs)
+            except:
+                # This has to be generic because we don't know what is in the function
+                failures.append(deferredFunction.callable.__name__)
+                logger.exception('Deferred function "%s" failed.',
+                                 deferredFunction.callable.__name__)
+        return failures
+
+    # Functions related to logging
+    def logToMaster(self, text, level=logging.INFO):
+        """
+        Send a logging message to the leader. The message will also be \
+        logged by the worker at the same level.
+
+        :param text: The string to log.
+        :param int level: The logging level.
+        """
+        logger.log(level=level, msg=("LOG-TO-MASTER: " + text))
+        self.loggingMessages.append(dict(text=text, level=level))
+
+    # Functions run after the completion of the job.
+    @abstractmethod
+    def _updateJobWhenDone(self):
+        """
+        Update the status of the job on the disk.
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def _blockFn(self):
+        """
+        Blocks while _updateJobWhenDone is running. This function is called by this job's
+        successor to ensure that it does not begin modifying the job store until after this job has
+        finished doing so.
+        """
+        raise NotImplementedError()
+
+    # Utility function used to identify if a pid is still running on the node.
+    @staticmethod
+    def _pidExists(pid):
+        """
+        This will return True if the process associated with pid is still running on the machine.
+        This is based on stackoverflow question 568271.
+
+        :param int pid: ID of the process to check for
+        :return: True/False
+        :rtype: bool
+        """
+        assert pid > 0
+        try:
+            os.kill(pid, 0)
+        except OSError as err:
+            if err.errno == errno.ESRCH:
+                # ESRCH == No such process
+                return False
+            else:
+                raise
+        else:
+            return True
+
+    @abstractclassmethod
+    def shutdown(cls, dir_):
+        """
+        Shutdown the filestore on this node.
+
+        This is intended to be called on batch system shutdown.
+
+        :param dir_: The jeystone directory containing the required information for fixing the state
+               of failed workers on the node before cleaning up.
+        """
+        raise NotImplementedError()
+
+
+class CachingFileStore(FileStore):
+    """
+    A cache-enabled file store that attempts to use hard-links and asynchronous job store writes to
+    reduce I/O between, and during jobs.
+    """
+
+    def __init__(self, jobStore, jobGraph, localTempDir, inputBlockFn):
+        super(CachingFileStore, self).__init__(jobStore, jobGraph, localTempDir, inputBlockFn)
         # Variables related to asynchronous writes.
         self.workerNumber = 2
         self.queue = Queue()
@@ -82,16 +410,14 @@ def __init__(self, jobStore, jobGraph, localTempDir, inputBlockFn):
         # cacheDir has to be 1 levels above local worker tempdir, at the same level as the
         # worker dirs. At this point, localTempDir is the worker directory, not the job
         # directory.
-        self.localTempDir = localTempDir
         self.localCacheDir = os.path.join(os.path.dirname(localTempDir),
                                           cacheDirName(self.jobStore.config.workflowID))
         self.cacheLockFile = os.path.join(self.localCacheDir, '.cacheLock')
         self.cacheStateFile = os.path.join(self.localCacheDir, '_cacheState')
-        # Since each worker has it's own unique FileStore instance, and only one Job can run at
-        # a time on a worker, we can bookkeep the job's file store operated files in a
+        # Since each worker has it's own unique CachingFileStore instance, and only one Job can run
+        # at a time on a worker, we can bookkeep the job's file store operated files in a
         # dictionary.
         self.jobSpecificFiles = {}
-        self.jobName = self.jobGraph.command.split()[1]
         self.jobID = sha1(self.jobName).hexdigest()
         logger.info('Starting job (%s) with ID (%s).', self.jobName, self.jobID)
         # A variable to describe how many hard links an unused file in the cache will have.
@@ -103,19 +429,12 @@ def __init__(self, jobStore, jobGraph, localTempDir, inputBlockFn):
         # job if required.
         self._setupCache()
 
-    @staticmethod
-    def createFileStore(jobStore, jobGraph, localTempDir, inputBlockFn, caching):
-        fileStoreCls = FileStore if caching else NonCachingFileStore
-        return fileStoreCls(jobStore, jobGraph, localTempDir, inputBlockFn)
-
     @contextmanager
     def open(self, job):
-        '''
+        """
         This context manager decorated method allows cache-specific operations to be conducted
         before and after the execution of a job in worker.py
-        :param job:
-        :return:
-        '''
+        """
         # Create a working directory for the job
         startingDir = os.getcwd()
         self.localTempDir = makePublicDir(os.path.join(self.localTempDir, str(uuid.uuid4())))
@@ -162,63 +481,13 @@ def open(self, job):
                 # Finally delete the job from the cache state file
                 cacheInfo.jobState.pop(self.jobID)
 
-    # Functions related to temp file and directories
-    def getLocalTempDir(self):
-        """
-        Get a new local temporary directory in which to write files that persist \
-        for the duration of the job.
-
-        :return: The absolute path to a new local temporary directory. \
-        This directory will exist for the duration of the job only, and is \
-        guaranteed to be deleted once the job terminates, removing all files \
-        it contains recursively.
-        :rtype: string
-        """
-        return os.path.abspath(tempfile.mkdtemp(prefix="t", dir=self.localTempDir))
-
-    def getLocalTempFile(self):
-        """
-        Get a new local temporary file that will persist for the duration of the job.
-
-        :return: The absolute path to a local temporary file. \
-        This file will exist for the duration of the job only, and \
-        is guaranteed to be deleted once the job terminates.
-        :rtype: string
-        """
-        handle, tmpFile = tempfile.mkstemp(prefix="tmp",
-                                           suffix=".tmp", dir=self.localTempDir)
-        os.close(handle)
-        return os.path.abspath(tmpFile)
-
-    def getLocalTempFileName(self):
-        '''
-        Get a valid name for a new local file. Don't actually create a file at the path.
-        :return: Path to valid file
-        :rtype: str
-        '''
-        # Create, and then delete a temp file. Creating will guarantee you a unique, unused
-        # file name. There is a very, very, very low chance that another job will create the
-        # same file name in the span of this one being deleted and then being used by the user.
-        tempFile = self.getLocalTempFile()
-        os.remove(tempFile)
-        return tempFile
-
     # Functions related to reading, writing and removing files to/from the job store
     def writeGlobalFile(self, localFileName, cleanup=False):
         """
         Takes a file (as a path) and uploads it to the job store.  Depending on the jobstore
         used, carry out the appropriate cache functions.
-
-        :param string localFileName: The path to the local file to upload.
-
-        :param Boolean cleanup: if True then the copy of the global file will \
-        be deleted once the job and all its successors have completed running. \
-        If not the global file must be deleted manually.
-
-        :returns: an ID that can be used to retrieve the file.
-        :rtype: FileID
         """
-        absLocalFileName = self._abspath(localFileName)
+        absLocalFileName = self._resolveAbsoluteLocalPath(localFileName)
         # What does this do?
         cleanupID = None if not cleanup else self.jobGraph.jobStoreID
         # If the file is from the scope of local temp dir
@@ -279,18 +548,8 @@ def writeGlobalFile(self, localFileName, cleanup=False):
         return FileID.forPath(jobStoreFileID, absLocalFileName)
 
     def writeGlobalFileStream(self, cleanup=False):
-        """
-        Similar to writeGlobalFile, but allows the writing of a stream to the job store.
-
-        :param Boolean cleanup: is as in :func:`toil.job.Job.FileStore.writeGlobalFile`.
-
-        :returns: a context manager yielding a tuple of 1) a file handle which \
-        can be written to and 2) the ID of the resulting file in the job store. \
-        The yielded file handle does not need to and should not be closed explicitly.
-        """
-        # TODO: Make this work with the caching??
-        # TODO: Make this work with FileID
-        return self.jobStore.writeFileStream(None if not cleanup else self.jobGraph.jobStoreID)
+        # TODO: Make this work with caching
+        return super(CachingFileStore, self).writeGlobalFileStream(cleanup)
 
     def readGlobalFile(self, fileStoreID, userPath=None, cache=True, mutable=None):
         """
@@ -298,29 +557,17 @@ def readGlobalFile(self, fileStoreID, userPath=None, cache=True, mutable=None):
         The function first looks for the file in the cache and if found, it hardlinks to the
         cached copy instead of downloading.
 
-        If a user path is specified, it is used as the destination. If a user path isn't
-        specified, the file is stored in the local temp directory with an encoded name.
-
         The cache parameter will be used only if the file isn't already in the cache, and
         provided user path (if specified) is in the scope of local temp dir.
 
-        :param FileID fileStoreID: job store id for the file
-
-        :param string userPath: a path to the name of file to which the global
-            file will be copied or hard-linked (see below).
-
         :param boolean cache: If True, a copy of the file will be saved into a cache that can be
-            used by other workers. caching supports multiple concurrent workers requesting the same
-            file by allowing only one to download the file while the others wait for it to complete.
-
+               used by other workers. caching supports multiple concurrent workers requesting the
+               same file by allowing only one to download the file while the others wait for it to
+               complete.
         :param boolean mutable: If True, the file path returned points to a file that is
-            modifiable by the user. Using False is recommended as it saves disk by making multiple
-            workers share a file via hard links. The value defaults to False unless backwards
-            compatibility was requested.
-
-        :return: an absolute path to a local, temporary copy of the file keyed
-            by fileStoreID.
-        :rtype: string
+               modifiable by the user. Using False is recommended as it saves disk by making
+               multiple workers share a file via hard links. The default is False unless backwards
+               compatibility was requested.
         """
         # Check that the file hasn't been deleted by the user
         if fileStoreID in self.filesToDelete:
@@ -340,7 +587,7 @@ def readGlobalFile(self, fileStoreID, userPath=None, cache=True, mutable=None):
         # Local File. If a name isn't provided, use the base64 encoded name such that we can
         # easily identify the files later on.
         if userPath is not None:
-            localFilePath = self._abspath(userPath)
+            localFilePath = self._resolveAbsoluteLocalPath(userPath)
             if os.path.exists(localFilePath):
                 # yes, this is illegal now.
                 raise RuntimeError(' File %s ' % localFilePath + ' exists. Cannot Overwrite.')
@@ -437,13 +684,6 @@ def readGlobalFile(self, fileStoreID, userPath=None, cache=True, mutable=None):
         return localFilePath
 
     def readGlobalFileStream(self, fileStoreID):
-        """
-        Similar to readGlobalFile, but allows a stream to be read from the job \
-        store.
-
-        :returns: a context manager yielding a file handle which can be read from. \
-        The yielded file handle does not need to and should not be closed explicitly.
-        """
         if fileStoreID in self.filesToDelete:
             raise RuntimeError(
                 "Trying to access a file in the jobStore you've deleted: %s" % fileStoreID)
@@ -456,18 +696,7 @@ def readGlobalFileStream(self, fileStoreID):
             logger.info('CACHE: Cache miss on file with ID \'%s\'.' % fileStoreID)
             return self.jobStore.readFileStream(fileStoreID)
 
-    def importFile(self, srcUrl, sharedFileName=None):
-        return self.jobStore.importFile(srcUrl, sharedFileName=sharedFileName)
-
-    def exportFile(self, jobStoreFileID, dstUrl):
-        self.jobStore.exportFile(jobStoreFileID, dstUrl)
-
     def deleteLocalFile(self, fileStoreID):
-        '''
-        Deletes a local file with the given job store ID.
-        :param str fileStoreID: File Store ID of the file to be deleted.
-        :return: None
-        '''
         # The local file may or may not have been cached. If it was, we need to do some
         # bookkeeping. If it wasn't, we just delete the file and continue with no might need
         # some bookkeeping if the file store and cache live on the same filesystem. We can know
@@ -544,14 +773,6 @@ def deleteLocalFile(self, fileStoreID):
                              '\'%s\'.' % fileStoreID)
 
     def deleteGlobalFile(self, fileStoreID):
-        """
-        Deletes a global file with the given job store ID.
-
-        To ensure that the job can be restarted if necessary, the delete will not happen until
-        after the job's run method has completed.
-
-        :param fileStoreID: the job store ID of the file to be deleted.
-        """
         jobStateIsPopulated = False
         with self._CacheState.open(self) as cacheInfo:
             if self.jobID in cacheInfo.jobState:
@@ -578,11 +799,11 @@ def deleteGlobalFile(self, fileStoreID):
     # Cache related methods
     @contextmanager
     def cacheLock(self):
-        '''
+        """
         This is a context manager to acquire a lock on the Lock file that will be used to
         prevent synchronous cache operations between workers.
-        :yield: File descriptor for cache lock file in r+ mode
-        '''
+        :yields: File descriptor for cache lock file in r+ mode
+        """
         cacheLockFile = open(self.cacheLockFile, 'w')
         try:
             flock(cacheLockFile, LOCK_EX)
@@ -596,10 +817,9 @@ def cacheLock(self):
             logger.debug("CACHE: Released lock")
 
     def _setupCache(self):
-        '''
+        """
         Setup the cache based on the provided values for localCacheDir.
-        :return: None
-        '''
+        """
         # we first check whether the cache directory exists. If it doesn't, create it.
         if not os.path.exists(self.localCacheDir):
             # Create a temporary directory as this worker's private cache. If all goes well, it
@@ -639,13 +859,12 @@ def _setupCache(self):
             self.nlinkThreshold = cacheInfo.nlink
 
     def _createCacheLockFile(self, tempCacheDir):
-        '''
+        """
         Create the cache lock file file to contain the state of the cache on the node.
 
         :param str tempCacheDir: Temporary directory to use for setting up a cache lock file the
-        first time.
-        :return: None
-        '''
+               first time.
+        """
         # The nlink threshold is setup along with the first instance of the cache class on the
         # node.
         self.setNlinkThreshold()
@@ -669,38 +888,39 @@ def _createCacheLockFile(self, tempCacheDir):
         cacheInfo.write(personalCacheStateFile)
 
     def encodedFileID(self, jobStoreFileID):
-        '''
+        """
         Uses a url safe base64 encoding to encode the jobStoreFileID into a unique identifier to
         use as filename within the cache folder.  jobstore IDs are essentially urls/paths to
         files and thus cannot be used as is. Base64 encoding is used since it is reversible.
 
         :param jobStoreFileID: string representing a job store file ID
-        :return: str outCachedFile: A path to the hashed file in localCacheDir
-        '''
+        :return: outCachedFile: A path to the hashed file in localCacheDir
+        :rtype: str
+        """
         outCachedFile = os.path.join(self.localCacheDir,
                                      base64.urlsafe_b64encode(jobStoreFileID))
         return outCachedFile
 
     def _fileIsCached(self, jobStoreFileID):
-        '''
+        """
         Is the file identified by jobStoreFileID in cache or not.
-        '''
+        """
         return os.path.exists(self.encodedFileID(jobStoreFileID))
 
     def decodedFileID(self, cachedFilePath):
-        '''
+        """
         Decode a cached fileName back to a job store file ID.
 
         :param str cachedFilePath: Path to the cached file
         :return: The jobstore file ID associated with the file
         :rtype: str
-        '''
+        """
         fileDir, fileName = os.path.split(cachedFilePath)
         assert fileDir == self.localCacheDir, 'Can\'t decode uncached file names'
         return base64.urlsafe_b64decode(fileName)
 
     def addToCache(self, localFilePath, jobStoreFileID, callingFunc, mutable=None):
-        '''
+        """
         Used to process the caching of a file. This depends on whether a file is being written
         to file store, or read from it.
         WRITING
@@ -714,8 +934,7 @@ def addToCache(self, localFilePath, jobStoreFileID, callingFunc, mutable=None):
         :param jobStoreFileID: jobStoreID for the file
         :param str callingFunc: Who called this function, 'write' or 'read'
         :param boolean mutable: See modifiable in readGlobalFile
-        :return: None
-        '''
+        """
         assert callingFunc in ('read', 'write')
         # Set up the modifiable variable if it wasn't provided by the user in the function call.
         if mutable is None:
@@ -799,8 +1018,7 @@ def returnFileSize(self, fileStoreID, cachedFileSource, lockFileHandle,
         :param str cachedFileSource: File being added to cache
         :param file lockFileHandle: Open file handle to the cache lock file
         :param bool fileAlreadyCached: A flag to indicate whether the file was already cached or
-            not. If it was, then it means that you don't need to add the filesize to cache again.
-        :return: None
+               not. If it was, then it means that you don't need to add the filesize to cache again.
         """
         fileSize = os.stat(cachedFileSource).st_size
         cacheInfo = self._CacheState._load(self.cacheStateFile)
@@ -821,26 +1039,23 @@ def returnFileSize(self, fileStoreID, cachedFileSource, lockFileHandle,
 
     @staticmethod
     def _isHidden(filePath):
-        '''
+        """
         This is a function that checks whether filePath is hidden
+
         :param str filePath: Path to the file under consideration
-        :return:
-        '''
-        # >>> timeit.timeit('x = a[0] in {".", "_"}', setup='a=".abc"', number=1000000)
-        # 0.14043903350830078
-        # >>> timeit.timeit('x = (a.startswith(".") or a.startswith("_"))', ...)
-        # 0.216400146484375
-        # >>> timeit.timeit('x = a.startswith((".", "_"))', ...)
-        # 0.21130609512329102
+        :return: A boolean indicating whether the file is hidden or not.
+        :rtype: bool
+        """
         assert isinstance(filePath, str)
         # I can safely assume i will never see an empty string because this is always called on
         # the results of an os.listdir()
-        return filePath[0] in {'.', '_'}
+        return filePath[0] in ('.', '_')
 
     def cleanCache(self, newJobReqs):
         """
         Cleanup all files in the cache directory to ensure that at lead newJobReqs are available
         for use.
+
         :param float newJobReqs: the total number of bytes of files allowed in the cache.
         """
         with self._CacheState.open(self) as cacheInfo:
@@ -898,10 +1113,9 @@ def cleanCache(self, newJobReqs):
                          (cacheInfo.cached + cacheInfo.sigmaJob))
 
     def removeSingleCachedFile(self, fileStoreID):
-        '''
+        """
         Removes a single file described by the fileStoreID from the cache forcibly.
-        :return:
-        '''
+        """
         with self._CacheState.open(self) as cacheInfo:
             cachedFile = self.encodedFileID(fileStoreID)
             cachedFileStats = os.stat(cachedFile)
@@ -930,14 +1144,28 @@ def setNlinkThreshold(self):
         else:
             self.nlinkThreshold = 1
 
+    def _accountForNlinkEquals2(self, localFilePath):
+        """
+        This is a utility function that accounts for the fact that if nlinkThreshold == 2, the
+        size of the file is accounted for by the file store copy of the file and thus the file
+        size shouldn't be added to the cached file sizes.
+
+        :param str localFilePath: Path to the local file that was linked to the file store copy.
+        """
+        fileStats = os.stat(localFilePath)
+        assert fileStats.st_nlink >= self.nlinkThreshold
+        with self._CacheState.open(self) as cacheInfo:
+            cacheInfo.sigmaJob -= fileStats.st_size
+            jobState = self._JobState(cacheInfo.jobState[self.jobID])
+            jobState.updateJobReqs(fileStats.st_size, 'remove')
+
     def returnJobReqs(self, jobReqs):
-        '''
+        """
         This function returns the effective job requirements back to the pool after the job
         completes. It also deletes the local copies of files with the cache lock held.
 
         :param float jobReqs: Original size requirement of the job
-        :return: None
-        '''
+        """
         # Since we are only reading this job's specific values from the state file, we don't
         # need a lock
         jobState = self._JobState(self._CacheState._load(self.cacheStateFile
@@ -948,109 +1176,63 @@ def returnJobReqs(self, jobReqs):
             cacheInfo.sigmaJob -= jobReqs
             # assert cacheInfo.isBalanced() # commenting this out for now. God speed
 
-    def _abspath(self, key):
+    class _CacheState(FileStore._StateFile):
         """
-        Return the absolute path to key.  This is a wrapepr for os.path.abspath because mac OS
-        symlinks /tmp and /var (the most common places for a default tempdir) to
-        private/<tmp or var>.
-        :param str key: The absolute or relative path to the file. If relative, it must be
-        relative to the local temp working dir
-        :return: Absolute path to key
-        """
-        if key.startswith('/'):
-            return os.path.abspath(key)
-        else:
-            return os.path.join(self.localTempDir, key)
-
-    class _CacheState(object):
-        '''
         Utility class to read and write the cache lock file. Also for checking whether the
-        caching equation is balanced or not.
-        '''
-
-        def __init__(self, stateDict):
-            # Should i assert the dictionary contains the required elements? This class is
-            # instantiated often {nlink, attemptNumber, total, cached, sigmaJob}
-            assert isinstance(stateDict, dict)
-            self.__dict__.update(stateDict)
-
+        caching equation is balanced or not.  It extends the _StateFile class to add other cache
+        related functions.
+        """
         @classmethod
         @contextmanager
-        def open(cls, outer):
-            '''
-            This is a context manager that basically opens the cache state file and reads it
-            into an object that is returned to the user in the yield
-            :param outer: instance of the CachedFileStore class (to use the cachelock method)
-            '''
+        def open(cls, outer=None):
+            """
+            This is a context manager that opens the cache state file and reads it into an object
+            that is returned to the user in the yield
+            """
+            assert outer is not None
             with outer.cacheLock():
                 cacheInfo = cls._load(outer.cacheStateFile)
                 yield cacheInfo
                 cacheInfo.write(outer.cacheStateFile)
 
-        @classmethod
-        def _load(cls, fileName):
-            '''
-            Load the state of the cache from the cache state file (via the file handle provided)
-            :param str fileName: Path to the cache state file
-            '''
-            # Read the value from the cache state file then initialize and instance of
-            # _CacheState with it.
-            with open(fileName, 'r') as fH:
-                cacheInfoDict = dill.load(fH)
-            return cls(cacheInfoDict)
-
-        def write(self, fileName):
-            '''
-            Write the current state of the cache into a temporary file then atomically rename it
-            to the main cache state file.
-            :param str fileName: Path to the cache state file
-            :return:
-            '''
-            with open(fileName + '.tmp', 'w') as fH:
-                # Based on answer by user "Mark" at:
-                # http://stackoverflow.com/questions/2709800/how-to-pickle-yourself
-                # We can't pickle nested classes. So we have to pickle the variables of the class
-                # If we ever change this, we need to ensure it doesn't break FileID
-                dill.dump(self.__dict__, fH)
-            os.rename(fileName + '.tmp', fileName)
-
         def isBalanced(self):
-            '''
+            """
             Checks for the inequality of the caching equation, i.e.
                             cachedSpace + sigmaJobDisk <= totalFreeSpace
             Essentially, the sum of all cached file + disk requirements of all running jobs
             should always be less than the available space on the system
+
             :return: Boolean for equation is balanced (T) or not (F)
-            '''
+            :rtype: bool
+            """
             return self.cached + self.sigmaJob <= self.total
 
         def purgeRequired(self, jobReqs):
-            '''
+            """
             Similar to isBalanced, however it looks at the actual state of the system and
             decides whether an eviction is required.
-            :return bool: Is a purge required(T) or no(F)
-            '''
+
+            :return: Is a purge required(T) or no(F)
+            :rtype: bool
+            """
             return not self.isBalanced()
             # totalStats = os.statvfs(self.cacheDir)
             # totalFree = totalStats.f_bavail * totalStats.f_frsize
             # return totalFree < jobReqs
 
-    # Functions related to the deferred function logic
+    # Methods related to the deferred function logic
     @classmethod
-    def findAndHandleDeadJobs(cls, cacheInfo, batchSystemShutdown=False):
+    def findAndHandleDeadJobs(cls, nodeInfo, batchSystemShutdown=False):
         """
-        This function looks at the state of all jobs registered in the cache state file and will
-        handle them (clean up the cache, and run any registered defer functions)
 
-        :param FileStore._CacheState cacheInfo: The state of the cache a _CacheState object
-        :param bool batchSystemShutdown: Is the batch system in the process of shutting down?
-        :return:
+        :param toil.fileStore.CachingFileStore._CacheState nodeInfo: The state of the node cache as
+               a _CacheState object
         """
         # A list of tuples of (hashed job id, pid or process running job)
-        registeredJobs = [(jid, state['pid']) for jid, state in cacheInfo.jobState.items()]
+        registeredJobs = [(jid, state['pid']) for jid, state in nodeInfo.jobState.items()]
         for jobID, jobPID in registeredJobs:
-            if not cls.HarbingerFile._pidExists(jobPID):
-                jobState = FileStore._JobState(cacheInfo.jobState[jobID])
+            if not cls._pidExists(jobPID):
+                jobState = CachingFileStore._JobState(nodeInfo.jobState[jobID])
                 logger.warning('Detected that job (%s) prematurely terminated.  Fixing the state '
                                'of the cache.', jobState.jobName)
                 if not batchSystemShutdown:
@@ -1059,75 +1241,30 @@ def findAndHandleDeadJobs(cls, cacheInfo, batchSystemShutdown=False):
                     # Do this only during the life of the program and dont' do it during the
                     # batch system cleanup.  Leave that to the batch system cleanup code.
                     if os.path.exists(jobState.jobDir):
-                        shutil.rmtree(jobState.jobDir)  # Ignore_errors?
-                    cacheInfo.sigmaJob -= jobState.jobReqs
+                        shutil.rmtree(jobState.jobDir)
+                    nodeInfo.sigmaJob -= jobState.jobReqs
                 # Run any deferred functions associated with the job
                 logger.debug('Running user-defined deferred functions.')
                 deferredFunctions = jobState.deferredFunctions
                 cls._runDeferredFunctions(deferredFunctions)
                 # Remove it from the cache state file
-                cacheInfo.jobState.pop(jobID)
+                nodeInfo.jobState.pop(jobID)
 
-    def _registerDeferredFunction(self, callable, *args, **kwargs):
-        """
-        This is the backend function to register a function that will be run after the completion of
-        the current job on the same node
-        :param function callable: The function to be run after this job.
-        :param list args: The arguments to the function
-        :param dict kwargs: The keyword arguments to the function
-        """
+    def _registerDeferredFunction(self, callable_, *args, **kwargs):
+        assert callable(callable_)
         with self._CacheState.open(self) as cacheInfo:
-            deferredFunction = DeferredFunction(callable=callable,
+            deferredFunction = DeferredFunction(callable=callable_,
                                                 args=args,
                                                 kwargs=kwargs)
             cacheInfo.jobState[self.jobID]['deferredFunctions'].append(deferredFunction)
-            logger.info('Registered the deferred function "%s" to job "%s".', callable.__name__,
+            logger.info('Registered the deferred function "%s" to job "%s".', callable_.__name__,
                         self.jobName)
 
-    @staticmethod
-    def _runDeferredFunctions(deferredFunctions):
-        """
-        This function will run all the registered deferred functions for the job.
-
-        :param list deferredFunctions: A list of DeferredFunctions to run
-        the cache
-        :returns: list of failed functions
-        :rtype: list
-        """
-        failures = []
-        for deferredFunction in deferredFunctions:
-            #TODO: better representation of the callable in the output string
-            try:
-                logger.debug('Running deferred function "%s".', deferredFunction.callable.__name__)
-                deferredFunction.callable(*deferredFunction.args, **deferredFunction.kwargs)
-            except:
-                # This has to be generic because we don't know what is in the function
-                failures.append(deferredFunction.callable.__name__)
-                logger.exception('Deferred function "%s" failed.',
-                                 deferredFunction.callable.__name__)
-        return failures
-
-
-    def _accountForNlinkEquals2(self, localFilePath):
-        '''
-        This is a utility function that accounts for the fact that if nlinkThreshold == 2, the
-        size of the file is accounted for by the file store copy of the file and thus the file
-        size shouldn't be added to the cached file sizes.
-        :param str localFilePath: Path to the local file that was linked to the file store copy.
-        :return: None
-        '''
-        fileStats = os.stat(localFilePath)
-        assert fileStats.st_nlink >= self.nlinkThreshold
-        with self._CacheState.open(self) as cacheInfo:
-            cacheInfo.sigmaJob -= fileStats.st_size
-            jobState = self._JobState(cacheInfo.jobState[self.jobID])
-            jobState.updateJobReqs(fileStats.st_size, 'remove')
-
     class _JobState(object):
-        '''
+        """
         This is a utility class to handle the state of a job in terms of it's current disk
         requirements, working directory, and job specific files.
-        '''
+        """
 
         def __init__(self, dictObj):
             assert isinstance(dictObj, dict)
@@ -1135,30 +1272,30 @@ def __init__(self, dictObj):
 
         @classmethod
         def updateJobSpecificFiles(cls, outer, jobStoreFileID, filePath, fileSize, cached):
-            '''
+            """
             This method will update the job specifc files in the job state object. It deals with
             opening a cache lock file, etc.
-            :param FileStore outer: An instance of FileStore
+
+            :param toil.fileStore.CachingFileStore outer: An instance of CachingFileStore
             :param str jobStoreFileID: job store Identifier for the file
             :param str filePath: The path to the file
             :param float fileSize: The size of the file (may be deprecated soon)
             :param bool cached: T : F : None :: cached : not cached : mutably read
-            :return: None
-            '''
+            """
             with outer._CacheState.open(outer) as cacheInfo:
                 jobState = cls(cacheInfo.jobState[outer.jobID])
                 jobState.addToJobSpecFiles(jobStoreFileID, filePath, fileSize, cached)
                 cacheInfo.jobState[outer.jobID] = jobState.__dict__
 
         def addToJobSpecFiles(self, jobStoreFileID, filePath, fileSize, cached):
-            '''
+            """
             This is the real method that actually does the updations.
+
             :param jobStoreFileID: job store Identifier for the file
             :param filePath: The path to the file
             :param fileSize: The size of the file (may be deprecated soon)
             :param cached: T : F : None :: cached : not cached : mutably read
-            :return: None
-            '''
+            """
             # If there is no entry for the jsfID, make one. self.jobSpecificFiles is a default
             # dict of default dicts and the absence of a key will return an empty dict
             # (equivalent to a None for the if)
@@ -1178,13 +1315,13 @@ def addToJobSpecFiles(self, jobStoreFileID, filePath, fileSize, cached):
                 self.updateJobReqs(fileSize, 'add')
 
         def updateJobReqs(self, fileSize, actions):
-            '''
+            """
             This method will update the current state of the disk required by the job after the
             most recent cache operation.
+
             :param fileSize: Size of the last file added/removed from the cache
             :param actions: 'add' or 'remove'
-            :return:
-            '''
+            """
             assert actions in ('add', 'remove')
             multiplier = 1 if actions == 'add' else -1
             # If the file was added to the cache, the value is subtracted from the requirements,
@@ -1233,7 +1370,6 @@ def waitOnDownload(self, lockFileHandle):
             write a file to the cache.
 
             :param lockFileHandle: The open handle to the cache lock file
-            :return: None
             """
             while self.exists():
                 logger.info('CACHE: Waiting for another worker to download file with ID %s.'
@@ -1241,7 +1377,7 @@ def waitOnDownload(self, lockFileHandle):
                 # Ensure that the process downloading the file is still alive.  The PID will
                 # be in the harbinger file.
                 pid = self.read()
-                if self._pidExists(pid):
+                if FileStore._pidExists(pid):
                     # Release the file lock and then wait for a bit before repeating.
                     flock(lockFileHandle, LOCK_UN)
                     time.sleep(20)
@@ -1274,47 +1410,11 @@ def _delete(self):
                                        self.fileStoreID, logging.DEBUG)
             os.remove(self.harbingerFileName)
 
-        @staticmethod
-        def _pidExists(pid):
-            """
-            This will return True if the process associated with pid is still running on the
-            machine.
-            This is based on stackoverflow question 568271.
-
-            :param int pid: ID of the process to check for
-            :return: True/False
-            :rtype: bool
-            """
-            assert pid > 0
-            try:
-                os.kill(pid, 0)
-            except OSError as err:
-                if err.errno == errno.ESRCH:
-                    # ESRCH == No such process
-                    return False
-                else:
-                    raise
-            else:
-                return True
-
-    # Logging
-    def logToMaster(self, text, level=logging.INFO):
-        """
-        Send a logging message to the leader. The message will also be \
-        logged by the worker at the same level.
-
-        :param text: The string to log.
-        :param int level: The logging level.
-        """
-        logger.log(level=level, msg=("LOG-TO-MASTER: " + text))
-        self.loggingMessages.append(dict(text=text, level=level))
-
     # Functions related to async updates
     def asyncWrite(self):
         """
         A function to write files asynchronously to the job store such that subsequent jobs are
         not delayed by a long write operation.
-        :return: None
         """
         try:
             while True:
@@ -1418,20 +1518,25 @@ def asyncUpdate():
             raise
 
     def _blockFn(self):
-        """
-        Blocks while _updateJobWhenDone is running. This function is called by this job's
-        successor to insure that it does not begin modifying the job store until after this job has
-        finished doing so.
-        """
         self.updateSemaphore.acquire()
         self.updateSemaphore.release()  # Release so that the block function can be recalled
         # This works, because once acquired the semaphore will not be acquired
         # by _updateJobWhenDone again.
         return
 
+    @classmethod
+    def shutdown(cls, dir_):
+        """
+        :param dir_: The directory that will contain the cache state file.
+        """
+        cacheInfo = cls._CacheState._load(os.path.join(dir_, '_cacheState'))
+        cls.findAndHandleDeadJobs(cacheInfo, batchSystemShutdown=True)
+        shutil.rmtree(dir_)
+
     def __del__(self):
-        """Cleanup function that is run when destroying the class instance \
-        that ensures that all the file writing threads exit.
+        """
+        Cleanup function that is run when destroying the class instance that ensures that all the
+        file writing threads exit.
         """
         self.updateSemaphore.acquire()
         for i in xrange(len(self.workers)):
@@ -1442,55 +1547,64 @@ def __del__(self):
 
 
 class NonCachingFileStore(FileStore):
-
     def __init__(self, jobStore, jobGraph, localTempDir, inputBlockFn):
-        self.jobStore = jobStore
-        self.jobGraph = jobGraph
-        self.jobName = self.jobGraph.command.split()[1]
-        self.localTempDir = os.path.abspath(localTempDir)
-        self.inputBlockFn = inputBlockFn
-        self.jobsToDelete = set()
-        self.loggingMessages = []
-        self.localFileMap = {}
-        self.filesToDelete = set()
+        super(NonCachingFileStore, self).__init__(jobStore, jobGraph, localTempDir, inputBlockFn)
+        # This will be defined in the `open` method.
+        self.jobStateFile = None
+        self.localFileMap = collections.defaultdict(list)
 
     @contextmanager
     def open(self, job):
         jobReqs = job.disk
         startingDir = os.getcwd()
         self.localTempDir = makePublicDir(os.path.join(self.localTempDir, str(uuid.uuid4())))
+        self.findAndHandleDeadJobs(self.workFlowDir)
+        self.jobStateFile = self._createJobStateFile()
         try:
             os.chdir(self.localTempDir)
             yield
         finally:
             diskUsed = getDirSizeRecursively(self.localTempDir)
-            logString = "Job %s used %.2f%% (%sB [%sB] used, %sB [%sB] requested) at the end of " \
-                        "its run." % (self.jobName,
-                                      float(diskUsed) / jobReqs * 100 if jobReqs > 0 else 0.0,
-                                      bytes2human(diskUsed), diskUsed, bytes2human(jobReqs),
-                                      jobReqs)
+            logString = ("Job {jobName} used {percent:.2f}% ({humanDisk}B [{disk}B] used, "
+                         "{humanRequestedDisk}B [{requestedDisk}B] requested) at the end of "
+                         "its run.".format(jobName=self.jobName,
+                                           percent=(float(diskUsed) / jobReqs * 100 if
+                                                    jobReqs > 0 else 0.0),
+                                           humanDisk=bytes2human(diskUsed),
+                                           disk=diskUsed,
+                                           humanRequestedDisk=bytes2human(jobReqs),
+                                           requestedDisk=jobReqs))
             self.logToMaster(logString, level=logging.INFO)
             if diskUsed > jobReqs:
                 self.logToMaster("Job used more disk than requested. Cconsider modifying the user "
                                  "script to avoid the chance of failure due to incorrectly "
                                  "requested resources. " + logString, level=logging.WARNING)
             os.chdir(startingDir)
+            jobState = self._readJobState(self.jobStateFile)
+            deferredFunctions = jobState['deferredFunctions']
+            failures = self._runDeferredFunctions(deferredFunctions)
+            for failure in failures:
+                self.logToMaster('Deferred function "%s" failed.' % failure, logging.WARN)
+            # Finally delete the job from the worker
+            os.remove(self.jobStateFile)
 
     def writeGlobalFile(self, localFileName, cleanup=False):
-        absLocalFileName = self._abspath(localFileName)
+        absLocalFileName = self._resolveAbsoluteLocalPath(localFileName)
         cleanupID = None if not cleanup else self.jobGraph.jobStoreID
-        return FileID.forPath(self.jobStore.writeFile(absLocalFileName, cleanupID), absLocalFileName)
+        fileStoreID = self.jobStore.writeFile(absLocalFileName, cleanupID)
+        self.localFileMap[fileStoreID].append(absLocalFileName)
+        return FileID.forPath(fileStoreID, absLocalFileName)
 
     def readGlobalFile(self, fileStoreID, userPath=None, cache=True, mutable=None):
         if userPath is not None:
-            localFilePath = self._abspath(userPath)
+            localFilePath = self._resolveAbsoluteLocalPath(userPath)
             if os.path.exists(localFilePath):
                 raise RuntimeError(' File %s ' % localFilePath + ' exists. Cannot Overwrite.')
         else:
             localFilePath = self.getLocalTempFileName()
 
         self.jobStore.readFile(fileStoreID, localFilePath)
-        self.localFileMap[fileStoreID] = localFilePath
+        self.localFileMap[fileStoreID].append(localFilePath)
         return localFilePath
 
     @contextmanager
@@ -1500,11 +1614,12 @@ def readGlobalFileStream(self, fileStoreID):
 
     def deleteLocalFile(self, fileStoreID):
         try:
-            localFilePath = self.localFileMap.pop(fileStoreID)
+            localFilePaths = self.localFileMap.pop(fileStoreID)
         except KeyError:
             raise OSError(errno.ENOENT, "Attempting to delete a non-local file")
         else:
-            os.remove(localFilePath)
+            for localFilePath in localFilePaths:
+                os.remove(localFilePath)
 
     def deleteGlobalFile(self, fileStoreID):
         try:
@@ -1542,8 +1657,125 @@ def _updateJobWhenDone(self):
             raise
 
     def __del__(self):
+        """
+        Cleanup function that is run when destroying the class instance.  Nothing to do since there
+        are no async write events.
+        """
         pass
 
+    # Functions related to the deferred function logic
+    @classmethod
+    def findAndHandleDeadJobs(cls, nodeInfo, batchSystemShutdown=False):
+        """
+        Look at the state of all jobs registered in the individual job state files, and handle them
+        (clean up the disk, and run any registered defer functions)
+
+        :param str nodeInfo: The location of the workflow directory on the node.
+        :param bool batchSystemShutdown: Is the batch system in the process of shutting down?
+        :return:
+        """
+        # A list of tuples of (job name, pid or process running job, registered defer functions)
+        for jobState in cls._getAllJobStates(nodeInfo):
+            if not cls._pidExists(jobState['jobPID']):
+                # using same logic to prevent races as CachingFileStore._setupCache
+                myPID = str(os.getpid())
+                cleanupFile = os.path.join(jobState['jobDir'], '.cleanup')
+                with open(os.path.join(jobState['jobDir'], '.' + myPID), 'w') as f:
+                    f.write(myPID)
+                while True:
+                    try:
+                        os.rename(f.name, cleanupFile)
+                    except OSError as err:
+                        if err.errno == errno.ENOTEMPTY:
+                            with open(cleanupFile, 'r') as f:
+                                cleanupPID = f.read()
+                            if cls._pidExists(int(cleanupPID)):
+                                # Cleanup your own mess.  It's only polite.
+                                os.remove(f.name)
+                                break
+                            else:
+                                os.remove(cleanupFile)
+                                continue
+                        else:
+                            raise
+                    else:
+                        logger.warning('Detected that job (%s) prematurely terminated.  Fixing the '
+                                       'state of the job on disk.', jobState['jobName'])
+                        if not batchSystemShutdown:
+                            logger.debug("Deleting the stale working directory.")
+                            # Delete the old work directory if it still exists.  Do this only during
+                            # the life of the program and dont' do it during the batch system
+                            # cleanup.  Leave that to the batch system cleanup code.
+                            shutil.rmtree(jobState['jobDir'])
+                        # Run any deferred functions associated with the job
+                        logger.debug('Running user-defined deferred functions.')
+                        cls._runDeferredFunctions(jobState['deferredFunctions'])
+                        break
+
+    @staticmethod
+    def _getAllJobStates(workflowDir):
+        """
+        Generator function that deserializes and yields the job state for every job on the node,
+        one at a time.
+
+        :param str workflowDir: The location of the workflow directory on the node.
+        :return: dict with keys (jobName,  jobPID, jobDir, deferredFunctions)
+        :rtype: dict
+        """
+        jobStateFiles = []
+        for root, dirs, files in os.walk(workflowDir):
+            for filename in files:
+                if filename == '.jobState':
+                    jobStateFiles.append(os.path.join(root, filename))
+        for filename in jobStateFiles:
+            yield NonCachingFileStore._readJobState(filename)
+
+    @staticmethod
+    def _readJobState(jobStateFileName):
+        with open(jobStateFileName) as fH:
+            state = dill.load(fH)
+        return state
+
+    def _registerDeferredFunction(self, callable_, *args, **kwargs):
+        assert callable(callable_)
+        with open(self.jobStateFile) as fH:
+            jobState = dill.load(fH)
+            deferredFunction = DeferredFunction(callable=callable_,
+                                                args=args,
+                                                kwargs=kwargs)
+            jobState['deferredFunctions'].append(deferredFunction)
+        with open(self.jobStateFile + '.tmp', 'w') as fH:
+            dill.dump(jobState, fH)
+        os.rename(self.jobStateFile + '.tmp', self.jobStateFile)
+        logger.info('Registered the deferred function "%s" to job "%s".', callable_.__name__,
+                    self.jobName)
+
+    def _createJobStateFile(self):
+        """
+        Create the job state file for the current job and fill in the required
+        values.
+
+        :return: Path to the job state file
+        :rtype: str
+        """
+        jobStateFile = os.path.join(self.localTempDir, '.jobState')
+        jobState = {'jobPID': os.getpid(),
+                    'jobName': self.jobName,
+                    'jobDir': self.localTempDir,
+                    'deferredFunctions': []}
+        with open(jobStateFile + '.tmp', 'w') as fH:
+            dill.dump(jobState, fH)
+        os.rename(jobStateFile + '.tmp', jobStateFile)
+        return jobStateFile
+
+    @classmethod
+    def shutdown(cls, dir_):
+        """
+        :param dir_: The workflow directory that will contain all the individual worker directories.
+        """
+        cls.findAndHandleDeadJobs(dir_, batchSystemShutdown=True)
+
+
 class FileID(str):
     """
     A class to wrap the job store file id returned by writeGlobalFile and any attributes we may want
@@ -1561,33 +1793,36 @@ def forPath(cls, fileStoreID, filePath):
         return cls(fileStoreID, os.stat(filePath).st_size)
 
 
-def shutdownCache(cacheDir):
+def shutdownFileStore(workflowDir, workflowID):
     """
     Run the deferred functions from any prematurely terminated jobs still lingering on the system
-    and delete the cache directory.
+    and carry out any necessary filestore-specific cleanup.
 
     This is a destructive operation and it is important to ensure that there are no other running
-    processes on the system that are modifying or using the cache state file.
+    processes on the system that are modifying or using the file store for this workflow.
+
 
     This is the intended to be the last call to the file store in a Toil run, called by the
     batch system cleanup function upon batch system shutdown.
 
-    :param cacheDir: The path to the cache directory
-    :return: None
+    :param str workflowDir: The path to the cache directory
+    :param str workflowID: The workflow ID for this invocation of the workflow
     """
+    cacheDir = os.path.join(workflowDir, cacheDirName(workflowID))
     if os.path.exists(cacheDir):
         # The presence of the cacheDir suggests this was a cached run. We don't need the cache lock
         # for any of this since this is the final cleanup of a job and there should be  no other
         # conflicting processes using the cache.
-        cacheInfo = FileStore._CacheState._load(os.path.join(cacheDir, '_cacheState'))
-        FileStore.findAndHandleDeadJobs(cacheInfo, batchSystemShutdown=True)
-        shutil.rmtree(cacheDir)
+        CachingFileStore.shutdown(cacheDir)
+    else:
+        # This absence of cacheDir suggests otherwise.
+        NonCachingFileStore.shutdown(workflowDir)
 
 
 class CacheError(Exception):
-    '''
+    """
     Error Raised if the user attempts to add a non-local file to cache
-    '''
+    """
 
     def __init__(self, message):
         super(CacheError, self).__init__(message)
@@ -1603,9 +1838,9 @@ def __init__(self):
 
 
 class IllegalDeletionCacheError(CacheError):
-    '''
+    """
     Error Raised if the Toil detects the user deletes a cached file
-    '''
+    """
 
     def __init__(self, deletedFile):
         message = 'Cache tracked file (%s) deleted explicitly by user. Use deleteLocalFile to ' \
@@ -1614,9 +1849,9 @@ def __init__(self, deletedFile):
 
 
 class InvalidSourceCacheError(CacheError):
-    '''
+    """
     Error Raised if the user attempts to add a non-local file to cache
-    '''
+    """
 
     def __init__(self, message):
         super(InvalidSourceCacheError, self).__init__(message)
diff --git a/src/toil/job.py b/src/toil/job.py
--- a/src/toil/job.py
+++ b/src/toil/job.py
@@ -294,11 +294,11 @@ def run(self, fileStore):
         """
         Override this function to perform work and dynamically create successor jobs.
 
-        :param toil.job.Job.FileStore fileStore: Used to create local and globally \
-        sharable temporary files and to send log messages to the leader process.
+        :param toil.fileStore.FileStore fileStore: Used to create local and globally
+               sharable temporary files and to send log messages to the leader process.
 
-        :return: The return value of the function can be passed to other jobs \
-        by means of :func:`toil.job.Job.rv`.
+        :return: The return value of the function can be passed to other jobs by means of
+                 :func:`toil.job.Job.rv`.
         """
         pass
 
@@ -676,7 +676,6 @@ def defer(self, callable, *args, **kwargs):
     ####################################################
     #The following nested classes are used for
     #creating jobtrees (Job.Runner),
-    #managing temporary files (Job.FileStore),
     #and defining a service (Job.Service)
     ####################################################
 
@@ -1267,7 +1266,8 @@ def _runner(self, jobGraph, jobStore, fileStore):
 
         :param class jobGraph: Instance of a jobGraph object
         :param class jobStore: Instance of the job store
-        :param class fileStore: Instance of Job.FileStore or Job.CachedFileStore
+        :param toil.fileStore.FileStore fileStore: Instance of a Cached on uncached
+               filestore
         :return:
         """
         # Run the job
@@ -1377,7 +1377,7 @@ class JobFunctionWrappingJob(FunctionWrappingJob):
     add successor jobs for the function and perform all the functions the \
     :class:`job.Job` class provides.
 
-    To enable the job function to get access to the :class:`toil.job.Job.FileStore` \
+    To enable the job function to get access to the :class:`toil.fileStore.FileStore` \
     instance (see :func:`toil.job.Job.Run`), it is made a variable of the wrapping job \
     called fileStore.
     """


Test patch:
diff --git a/run_tests.py b/run_tests.py
--- a/run_tests.py
+++ b/run_tests.py
@@ -22,10 +22,14 @@
         'AzureJobStoreTest',
         'FileJobStoreTest',
         'GoogleJobStoreTest',
-        'AwsJobStoreCacheTest',
-        'AzureJobStoreCacheTest',
-        'FileJobStoreCacheTest',
-        'GoogleJobStoreCacheTest',
+        'NonCachingFileStoreTestWithFileJobStore',
+        'CachingFileStoreTestWithFileJobStore',
+        'NonCachingFileStoreTestWithAwsJobStore',
+        'CachingFileStoreTestWithAwsJobStore',
+        'NonCachingFileStoreTestWithAzureJobStore',
+        'CachingFileStoreTestWithAzureJobStore',
+        'NonCachingFileStoreTestWithGoogleJobStore',
+        'CachingFileStoreTestWithGoogleJobStore',
         None],
     'integration-test': [
         'CGCloudRNASeqTest',
diff --git a/src/toil/test/src/jobCacheTest.py b/src/toil/test/src/fileStoreTest.py
similarity index 83%
rename from src/toil/test/src/jobCacheTest.py
rename to src/toil/test/src/fileStoreTest.py
--- a/src/toil/test/src/jobCacheTest.py
+++ b/src/toil/test/src/fileStoreTest.py
@@ -14,26 +14,26 @@
 
 from __future__ import absolute_import
 from __future__ import print_function
-
-import collections
-import inspect
-import os
-import random
-import signal
-import time
-import unittest
 from abc import abstractmethod, ABCMeta
-
 from struct import pack, unpack
 from uuid import uuid4
 
 from toil.job import Job
-from toil.fileStore import IllegalDeletionCacheError, FileStore
+from toil.fileStore import IllegalDeletionCacheError, CachingFileStore
 from toil.test import ToilTest, needs_aws, needs_azure, needs_google, experimental
 from toil.leader import FailedJobsException
 from toil.jobStores.abstractJobStore import NoSuchFileException
 from toil.fileStore import CacheUnbalancedError
 
+import collections
+import inspect
+import os
+import random
+import signal
+import time
+import unittest
+
+
 # Some tests take too long on the AWS and Azure Job stores and are unquitable for CI.  They can be
 # be run during manual tests by setting this to False.
 testingIsAutomatic = True
@@ -41,18 +41,33 @@
 
 class hidden:
     """
-    Hiding the abstract test class from the Unittest loader so it can be inherited in different test
-    suites for the different job stores.
+    Hiding the abstract test classes from the Unittest loader so it can be inherited in different
+    test suites for the different job stores.
     """
-
-    class AbstractCacheTest(ToilTest):
+    class AbstractFileStoreTest(ToilTest):
         """
-        Abstract tests for the the various cache functions in Job.CachedFileStore
+        An abstract base class for testing the various general functions described in
+        :class:toil.fileStore.FileStore
         """
+        # This is overwritten in the inheriting classs
+        jobStoreType = None
         __metaclass__ = ABCMeta
 
+        def _getTestJobStore(self):
+            if self.jobStoreType == 'file':
+                return self._getTestJobStorePath()
+            elif self.jobStoreType == 'aws':
+                return 'aws:%s:cache-tests-%s' % (self.awsRegion(), uuid4())
+            elif self.jobStoreType == 'azure':
+                return 'azure:toiltest:cache-tests-' + str(uuid4())
+            elif self.jobStoreType == 'google':
+                projectID = 'cgc-05-0006'
+                return 'google:' + projectID + ':cache-tests-' + str(uuid4())
+            else:
+                raise RuntimeError('Illegal job store type.')
+
         def setUp(self):
-            super(hidden.AbstractCacheTest, self).setUp()
+            super(hidden.AbstractFileStoreTest, self).setUp()
             testDir = self._createTempDir()
             self.options = Job.Runner.getDefaultOptions(self._getTestJobStore())
             self.options.logLevel = 'INFO'
@@ -60,11 +75,14 @@ def setUp(self):
             self.options.clean = 'always'
             self.options.logFile = os.path.join(testDir, 'logFile')
 
-        @abstractmethod
-        def _getTestJobStore(self):
-            raise NotImplementedError()
+        @staticmethod
+        def _uselessFunc(job):
+            """
+            I do nothing.  Don't judge me.
+            """
+            return None
 
-        # Sanity
+        # Sanity test
         def testToilIsNotBroken(self):
             """
             Runs a simple DAG to test if if any features other that caching were broken.
@@ -79,133 +97,486 @@ def testToilIsNotBroken(self):
             C.addChild(D)
             Job.Runner.startToil(A, self.options)
 
-        # noinspection PyUnusedLocal
+        # Test filestore operations.  This is a slightly less intense version of the cache specific
+        # test `testReturnFileSizes`
+        def testFileStoreOperations(self):
+            """
+            Write a couple of files to the jobstore.  Delete a couple of them.  Read back written
+            and locally deleted files.
+            """
+            workdir = self._createTempDir(purpose='nonLocalDir')
+            F = Job.wrapJobFn(self._testFileStoreOperations,
+                              nonLocalDir=workdir,
+                              numIters=30, disk='2G')
+            Job.Runner.startToil(F, self.options)
+
         @staticmethod
-        def _uselessFunc(job):
+        def _testFileStoreOperations(job, nonLocalDir, numIters=100):
             """
-            I do nothing.  Don't judge me.
+            Aux function for testFileStoreOperations Conduct numIters operations.
             """
-            return None
+            work_dir = job.fileStore.getLocalTempDir()
+            writtenFiles = {}  # fsID: (size, isLocal)
+            localFileIDs = set()
+            # Add one file for the sake of having something in the job store
+            writeFileSize = random.randint(0, 30)
+            cls = hidden.AbstractNonCachingFileStoreTest
+            fsId, _ = cls._writeFileToJobStore(job, isLocalFile=True, nonLocalDir=nonLocalDir,
+                                               fileMB=writeFileSize)
+            writtenFiles[fsId] = writeFileSize
+            localFileIDs.add(writtenFiles.keys()[0])
+            i = 0
+            while i <= numIters:
+                randVal = random.random()
+                if randVal < 0.33:  # Write
+                    writeFileSize = random.randint(0, 30)
+                    isLocalFile = True if random.random() <= 0.5 else False
+                    fsID, _ = cls._writeFileToJobStore(job, isLocalFile=isLocalFile,
+                                                       nonLocalDir=nonLocalDir,
+                                                       fileMB=writeFileSize)
+                    writtenFiles[fsID] = writeFileSize
+                    localFileIDs.add(fsID)
+                else:
+                    if len(writtenFiles) == 0:
+                        continue
+                    else:
+                        fsID, rdelFileSize = random.choice(writtenFiles.items())
+                        rdelRandVal = random.random()
+                    if randVal < 0.66:  # Read
+                        mutable = True if random.random() <= 0.5 else False
+                        cache = True if random.random() <= 0.5 else False
+                        job.fileStore.readGlobalFile(fsID, '/'.join([work_dir, str(uuid4())]),
+                                                     cache=cache, mutable=mutable)
+                        localFileIDs.add(fsID)
+                    else:  # Delete
+                        if rdelRandVal <= 0.5:  # Local Delete
+                            if fsID not in localFileIDs:
+                                continue
+                            job.fileStore.deleteLocalFile(fsID)
+                        else:  # Global Delete
+                            job.fileStore.deleteGlobalFile(fsID)
+                            writtenFiles.pop(fsID)
+                        if fsID in localFileIDs:
+                            localFileIDs.remove(fsID)
+                i += 1
 
-        def testExtremeCacheSetup(self):
+        # Tests for the various defer possibilities
+        def testDeferredFunctionRunsWithMethod(self):
             """
-            Try to create the cache with bad worker active and then have 10 child jobs try to run in
-            the chain.  This tests whether the cache is created properly even when the job crashes
-            randomly.
+            Refer docstring in _testDeferredFunctionRuns.
+            Test with Method
             """
-            self.options.retryCount = 20
-            self.options.badWorker = 0.5
-            self.options.badWorkerFailInterval = 0.1
-            for test in xrange(0, 20):
-                E = Job.wrapJobFn(self._uselessFunc)
-                F = Job.wrapJobFn(self._uselessFunc)
-                jobs = {}
-                for i in xrange(0, 10):
-                    jobs[i] = Job.wrapJobFn(self._uselessFunc)
-                    E.addChild(jobs[i])
-                    jobs[i].addChild(F)
-                Job.Runner.startToil(E, self.options)
+            self._testDeferredFunctionRuns(self._writeNonLocalFilesMethod)
 
-        def testCacheLockRace(self):
+        def testDeferredFunctionRunsWithClassMethod(self):
             """
-            Make 3 jobs compete for the same cache lock file.  If they have the lock at the same
-            time, the test will fail.  This test abuses the _CacheState class and modifies values in
-            the lock file.  DON'T TRY THIS AT HOME.
+            Refer docstring in _testDeferredFunctionRuns.
+            Test with Class Method
             """
-            A = Job.wrapJobFn(self._setUpLockFile)
-            B = Job.wrapJobFn(self._selfishLocker, cores=1)
-            C = Job.wrapJobFn(self._selfishLocker, cores=1)
-            D = Job.wrapJobFn(self._selfishLocker, cores=1)
-            E = Job.wrapJobFn(self._raceTestSuccess)
-            A.addChild(B)
-            A.addChild(C)
-            A.addChild(D)
-            B.addChild(E)
-            C.addChild(E)
-            D.addChild(E)
-            Job.Runner.startToil(A, self.options)
+            self._testDeferredFunctionRuns(self._writeNonLocalFilesClassMethod)
 
-        @staticmethod
-        def _setUpLockFile(job):
+        def testDeferredFunctionRunsWithLambda(self):
             """
-            Set nlink=0 for the cache test
+            Refer docstring in _testDeferredFunctionRuns.
+            Test with Lambda
             """
-            with job.fileStore.cacheLock():
-                cacheInfo = job.fileStore._CacheState._load(job.fileStore.cacheStateFile)
-                cacheInfo.nlink = 0
-                cacheInfo.write(job.fileStore.cacheStateFile)
+            self._testDeferredFunctionRuns(self._writeNonLocalFilesLambda)
 
-        @staticmethod
-        def _selfishLocker(job):
+        def _testDeferredFunctionRuns(self, callableFn):
             """
-            Try to acquire a lock on the lock file.  If 2 threads have the lock concurrently, then
-            abort.
+            Create 2 files. Make a job that writes data to them. Register a deferred function that
+            deletes the two files (one passed as an arg, adn one as a kwarg) and later assert that
+            the files have been deleted.
+
+            :param function callableFn: The function to use in the test.
+            :return: None
             """
-            for i in xrange(0, 1000):
-                with job.fileStore.cacheLock():
-                    cacheInfo = job.fileStore._CacheState._load(job.fileStore.cacheStateFile)
-                    cacheInfo.nlink += 1
-                    cacheInfo.cached = max(cacheInfo.nlink, cacheInfo.cached)
-                    cacheInfo.write(job.fileStore.cacheStateFile)
-                time.sleep(0.001)
-                with job.fileStore.cacheLock():
-                    cacheInfo = job.fileStore._CacheState._load(job.fileStore.cacheStateFile)
-                    cacheInfo.nlink -= 1
-                    cacheInfo.write(job.fileStore.cacheStateFile)
+            workdir = self._createTempDir(purpose='nonLocalDir')
+            nonLocalFile1 = os.path.join(workdir, str(uuid4()))
+            nonLocalFile2 = os.path.join(workdir, str(uuid4()))
+            open(nonLocalFile1, 'w').close()
+            open(nonLocalFile2, 'w').close()
+            assert os.path.exists(nonLocalFile1)
+            assert os.path.exists(nonLocalFile2)
+            A = Job.wrapJobFn(callableFn, files=(nonLocalFile1, nonLocalFile2))
+            Job.Runner.startToil(A, self.options)
+            assert not os.path.exists(nonLocalFile1)
+            assert not os.path.exists(nonLocalFile2)
 
         @staticmethod
-        def _raceTestSuccess(job):
-            """
-            Assert that the cache test passed successfully.
+        def _writeNonLocalFilesMethod(job, files):
             """
-            with job.fileStore.cacheLock():
-                cacheInfo = job.fileStore._CacheState._load(job.fileStore.cacheStateFile)
-                # Value of the nlink has to be zero for successful run
-                assert cacheInfo.nlink == 0
-                assert cacheInfo.cached > 1
+            Write some data to 2 files.  Pass them to a registered deferred method.
 
-        def testCacheEvictionPartialEvict(self):
+            :param tuple files: the tuple of the two files to work with
+            :return: None
             """
-            Ensure the cache eviction happens as expected.  Two files (20MB and 30MB) are written
-            sequentially into the job store in separate jobs.  The cache max is force set to 50MB.
-            A Third Job requests 10MB of disk requiring eviction of the 1st file.  Ensure that the
-            behavior is as expected.
+            for nlf in files:
+                with open(nlf, 'w') as nonLocalFileHandle:
+                    nonLocalFileHandle.write(os.urandom(1 * 1024 * 1024))
+            job.defer(_deleteMethods._deleteFileMethod, files[0], nlf=files[1])
+            return None
+
+        @staticmethod
+        def _writeNonLocalFilesClassMethod(job, files):
             """
-            self._testValidityOfCacheEvictTest()
+            Write some data to 2 files.  Pass them to a registered deferred class method.
 
-            # Explicitly set clean to always so even the failed cases get cleaned (This will
-            # overwrite the value set in setUp if it is ever changed in the future)
-            self.options.clean = 'always'
+            :param tuple files: the tuple of the two files to work with
+            :return: None
+            """
+            for nlf in files:
+                with open(nlf, 'w') as nonLocalFileHandle:
+                    nonLocalFileHandle.write(os.urandom(1 * 1024 * 1024))
+            job.defer(_deleteMethods._deleteFileClassMethod, files[0], nlf=files[1])
+            return None
 
-            self._testCacheEviction(file1MB=20, file2MB=30, diskRequestMB=10)
+        @staticmethod
+        def _writeNonLocalFilesLambda(job, files):
+            """
+            Write some data to 2 files.  Pass them to a registered deferred Lambda.
 
-        def testCacheEvictionTotalEvict(self):
+            :param tuple files: the tuple of the two files to work with
+            :return: None
             """
-            Ensure the cache eviction happens as expected.  Two files (20MB and 30MB) are written
-            sequentially into the job store in separate jobs.  The cache max is force set to 50MB.
-            A Third Job requests 10MB of disk requiring eviction of the 1st file.  Ensure that the
-            behavior is as expected.
+            lmd = lambda x, nlf: [os.remove(x), os.remove(nlf)]
+            for nlf in files:
+                with open(nlf, 'w') as nonLocalFileHandle:
+                    nonLocalFileHandle.write(os.urandom(1 * 1024 * 1024))
+            job.defer(lmd, files[0], nlf=files[1])
+            return None
+
+        def testDeferredFunctionRunsWithFailures(self):
             """
-            self._testValidityOfCacheEvictTest()
+            Create 2 non local filesto use as flags.  Create a job that registers a function that
+            deletes one non local file.  If that file exists, the job SIGKILLs itself. If it doesn't
+            exist, the job registers a second deferred function to delete the second non local file
+            and exits normally.
 
-            # Explicitly set clean to always so even the failed cases get cleaned (This will
-            # overwrite the value set in setUp if it is ever changed in the future)
-            self.options.clean = 'always'
+            Initially the first file exists, so the job should SIGKILL itself and neither deferred
+            function will run (in fact, the second should not even be registered). On the restart,
+            the first deferred function should run and the first file should not exist, but the
+            second one should.  We assert the presence of the second, then register the second
+            deferred function and exit normally.  At the end of the test, neither file should exist.
 
-            self._testCacheEviction(file1MB=20, file2MB=30, diskRequestMB=30)
+            Incidentally, this also tests for multiple registered deferred functions, and the case
+            where a deferred function fails (since the first file doesn't exist on the retry).
+            """
+            self.options.retryCount = 1
+            workdir = self._createTempDir(purpose='nonLocalDir')
+            nonLocalFile1 = os.path.join(workdir, str(uuid4()))
+            nonLocalFile2 = os.path.join(workdir, str(uuid4()))
+            open(nonLocalFile1, 'w').close()
+            open(nonLocalFile2, 'w').close()
+            assert os.path.exists(nonLocalFile1)
+            assert os.path.exists(nonLocalFile2)
+            A = Job.wrapJobFn(self._deferredFunctionRunsWithFailuresFn,
+                              files=(nonLocalFile1, nonLocalFile2))
+            Job.Runner.startToil(A, self.options)
+            assert not os.path.exists(nonLocalFile1)
+            assert not os.path.exists(nonLocalFile2)
 
-        def testCacheEvictionFailCase(self):
+        @staticmethod
+        def _deferredFunctionRunsWithFailuresFn(job, files):
             """
-            Ensure the cache eviction happens as expected.  Two files (20MB and 30MB) are written
-            sequentially into the job store in separate jobs.  The cache max is force set to 50MB.
-            A Third Job requests 10MB of disk requiring eviction of the 1st file.  Ensure that the
-            behavior is as expected.
+            Refer testDeferredFunctionRunsWithFailures
+
+            :param tuple files: the tuple of the two files to work with
+            :return: None
             """
-            self._testValidityOfCacheEvictTest()
+            cls = hidden.AbstractNonCachingFileStoreTest
+            job.defer(cls._deleteFile, files[0])
+            if os.path.exists(files[0]):
+                os.kill(os.getpid(), signal.SIGKILL)
+            else:
+                assert os.path.exists(files[1])
+                job.defer(cls._deleteFile, files[1])
 
-            # Explicitly set clean to always so even the failed cases get cleaned (This will
-            # overwrite the value set in setUp if it is ever changed in the future)
-            self.options.clean = 'always'
+        @staticmethod
+        def _deleteFile(nonLocalFile, nlf=None):
+            """
+            Delete nonLocalFile and nlf
+            :param str nonLocalFile:
+            :param str nlf:
+            :return: None
+            """
+            os.remove(nonLocalFile)
+            if nlf is not None:
+                os.remove(nlf)
+
+        def testNewJobsCanHandleOtherJobDeaths(self):
+            """
+            Create 2 non-local files and then create 2 jobs. The first job registers a deferred job
+            to delete the second non-local file, deletes the first non-local file and then kills
+            itself.  The second job waits for the first file to be deleted, then sleeps for a few
+            seconds and then spawns a child. the child of the second does nothing. However starting
+            it should handle the untimely demise of the first job and run the registered deferred
+            function that deletes the first file.  We assert the absence of the two files at the
+            end of the run.
+            """
+            # There can be no retries
+            self.options.retryCount = 0
+            workdir = self._createTempDir(purpose='nonLocalDir')
+            nonLocalFile1 = os.path.join(workdir, str(uuid4()))
+            nonLocalFile2 = os.path.join(workdir, str(uuid4()))
+            open(nonLocalFile1, 'w').close()
+            open(nonLocalFile2, 'w').close()
+            assert os.path.exists(nonLocalFile1)
+            assert os.path.exists(nonLocalFile2)
+            files = [nonLocalFile1, nonLocalFile2]
+            root = Job()
+            A = Job.wrapJobFn(self._testNewJobsCanHandleOtherJobDeaths_A, files=files)
+            B = Job.wrapJobFn(self._testNewJobsCanHandleOtherJobDeaths_B, files=files)
+            C = Job.wrapJobFn(self._testNewJobsCanHandleOtherJobDeaths_C, files=files,
+                              expectedResult=False)
+            root.addChild(A)
+            root.addChild(B)
+            B.addChild(C)
+            try:
+                Job.Runner.startToil(root, self.options)
+            except FailedJobsException as e:
+                pass
+
+        @staticmethod
+        def _testNewJobsCanHandleOtherJobDeaths_A(job, files):
+            """
+            Defer deletion of files[1], then wait for _testNewJobsCanHandleOtherJobDeaths_B to
+            start up, and finally delete files[0] before sigkilling self.
+
+            :param tuple files: the tuple of the two files to work with
+            :return: None
+            """
+            # Write the pid to files[1] such that we can be sure that this process has died before
+            # we spawn the next job that will do the cleanup.
+            with open(files[1], 'w') as fileHandle:
+                fileHandle.write(str(os.getpid()))
+            job.defer(hidden.AbstractNonCachingFileStoreTest._deleteFile, files[1])
+            while os.stat(files[0]).st_size == 0:
+                time.sleep(0.5)
+            os.remove(files[0])
+            os.kill(os.getpid(), signal.SIGKILL)
+
+        @staticmethod
+        def _testNewJobsCanHandleOtherJobDeaths_B(job, files):
+            # Write something to files[0] such that we can be sure that this process has started
+            # before _testNewJobsCanHandleOtherJobDeaths_A kills itself.
+            with open(files[0], 'w') as fileHandle:
+                fileHandle.write(str(os.getpid()))
+            while os.path.exists(files[0]):
+                time.sleep(0.5)
+            # Get the pid of _testNewJobsCanHandleOtherJobDeaths_A and wait for it to truly be dead.
+            with open(files[1], 'r') as fileHandle:
+                meeseeksPID = int(fileHandle.read())
+            while CachingFileStore._pidExists(meeseeksPID):
+                time.sleep(0.5)
+            # Now that we are convinced that_testNewJobsCanHandleOtherJobDeaths_A has died, we can
+            # spawn the next job
+            return None
+
+        @staticmethod
+        def _testNewJobsCanHandleOtherJobDeaths_C(job, files, expectedResult):
+            """
+            Asserts whether the files exist or not.
+
+            :param Job job: Job
+            :param list files: list of files to test
+            :param bool expectedResult: Are we expecting the files to exist or not?
+            """
+            for testFile in files:
+                assert os.path.exists(testFile) is expectedResult
+
+        def testBatchSystemCleanupCanHandleWorkerDeaths(self):
+            """
+            Create a non-local files. Create a job that registers a deferred job to delete the file
+            and then kills itself.
+
+            Assert that the file is missing after the pipeline fails.
+            """
+            # There can be no retries
+            self.options.retryCount = 0
+            workdir = self._createTempDir(purpose='nonLocalDir')
+            nonLocalFile1 = os.path.join(workdir, str(uuid4()))
+            nonLocalFile2 = os.path.join(workdir, str(uuid4()))
+            # The first file has to be non zero or meseeks will go into an infinite sleep
+            file1 = open(nonLocalFile1, 'w')
+            file1.write('test')
+            file1.close()
+            open(nonLocalFile2, 'w').close()
+            assert os.path.exists(nonLocalFile1)
+            assert os.path.exists(nonLocalFile2)
+            A = Job.wrapJobFn(self._testNewJobsCanHandleOtherJobDeaths_A,
+                              files=(nonLocalFile1, nonLocalFile2))
+            try:
+                Job.Runner.startToil(A, self.options)
+            except FailedJobsException:
+                pass
+            assert not os.path.exists(nonLocalFile1)
+            assert not os.path.exists(nonLocalFile2)
+
+        @staticmethod
+        def _writeFileToJobStore(job, isLocalFile, nonLocalDir=None, fileMB=1):
+            """
+            This function creates a file and writes it to the jobstore.
+
+            :param bool isLocalFile: Is the file local(T) or Non-Local(F)?
+            :param str nonLocalDir: A dir to write the file to.  If unspecified, a local directory
+                                    is created.
+            :param int fileMB: Size of the created file in MB
+            """
+            if isLocalFile:
+                work_dir = job.fileStore.getLocalTempDir()
+            else:
+                assert nonLocalDir is not None
+                work_dir = nonLocalDir
+            with open(os.path.join(work_dir, str(uuid4())), 'w') as testFile:
+                testFile.write(os.urandom(fileMB * 1024 * 1024))
+
+            return job.fileStore.writeGlobalFile(testFile.name), testFile
+
+    class AbstractNonCachingFileStoreTest(AbstractFileStoreTest):
+        """
+        Abstract tests for the the various functions in :class:toil.fileStore.NonCachingFileStore.
+        These tests are general enough that they can also be used for
+        :class:toil.fileStore.CachingFileStore.
+        """
+        __metaclass__ = ABCMeta
+
+        def setUp(self):
+            super(hidden.AbstractNonCachingFileStoreTest, self).setUp()
+            self.options.disableCaching = True
+
+    class AbstractCachingFileStoreTest(AbstractFileStoreTest):
+        """
+        Abstract tests for the the various cache-related functions in
+        :class:toil.fileStore.CachingFileStore.
+        """
+        __metaclass__ = ABCMeta
+
+        def setUp(self):
+            super(hidden.AbstractCachingFileStoreTest, self).setUp()
+            self.options.disableCaching = False
+
+        def testExtremeCacheSetup(self):
+            """
+            Try to create the cache with bad worker active and then have 10 child jobs try to run in
+            the chain.  This tests whether the cache is created properly even when the job crashes
+            randomly.
+            """
+            if testingIsAutomatic and self.jobStoreType != 'file':
+                self.skipTest("To save time")
+            self.options.retryCount = 20
+            self.options.badWorker = 0.5
+            self.options.badWorkerFailInterval = 0.1
+            for test in xrange(0, 20):
+                E = Job.wrapJobFn(self._uselessFunc)
+                F = Job.wrapJobFn(self._uselessFunc)
+                jobs = {}
+                for i in xrange(0, 10):
+                    jobs[i] = Job.wrapJobFn(self._uselessFunc)
+                    E.addChild(jobs[i])
+                    jobs[i].addChild(F)
+                Job.Runner.startToil(E, self.options)
+
+        def testCacheLockRace(self):
+            """
+            Make 3 jobs compete for the same cache lock file.  If they have the lock at the same
+            time, the test will fail.  This test abuses the _CacheState class and modifies values in
+            the lock file.  DON'T TRY THIS AT HOME.
+            """
+            A = Job.wrapJobFn(self._setUpLockFile)
+            B = Job.wrapJobFn(self._selfishLocker, cores=1)
+            C = Job.wrapJobFn(self._selfishLocker, cores=1)
+            D = Job.wrapJobFn(self._selfishLocker, cores=1)
+            E = Job.wrapJobFn(self._raceTestSuccess)
+            A.addChild(B)
+            A.addChild(C)
+            A.addChild(D)
+            B.addChild(E)
+            C.addChild(E)
+            D.addChild(E)
+            Job.Runner.startToil(A, self.options)
+
+        @staticmethod
+        def _setUpLockFile(job):
+            """
+            Set nlink=0 for the cache test
+            """
+            with job.fileStore.cacheLock():
+                cacheInfo = job.fileStore._CacheState._load(job.fileStore.cacheStateFile)
+                cacheInfo.nlink = 0
+                cacheInfo.write(job.fileStore.cacheStateFile)
+
+        @staticmethod
+        def _selfishLocker(job):
+            """
+            Try to acquire a lock on the lock file.  If 2 threads have the lock concurrently, then
+            abort.
+            """
+            for i in xrange(0, 1000):
+                with job.fileStore.cacheLock():
+                    cacheInfo = job.fileStore._CacheState._load(job.fileStore.cacheStateFile)
+                    cacheInfo.nlink += 1
+                    cacheInfo.cached = max(cacheInfo.nlink, cacheInfo.cached)
+                    cacheInfo.write(job.fileStore.cacheStateFile)
+                time.sleep(0.001)
+                with job.fileStore.cacheLock():
+                    cacheInfo = job.fileStore._CacheState._load(job.fileStore.cacheStateFile)
+                    cacheInfo.nlink -= 1
+                    cacheInfo.write(job.fileStore.cacheStateFile)
+
+        @staticmethod
+        def _raceTestSuccess(job):
+            """
+            Assert that the cache test passed successfully.
+            """
+            with job.fileStore.cacheLock():
+                cacheInfo = job.fileStore._CacheState._load(job.fileStore.cacheStateFile)
+                # Value of the nlink has to be zero for successful run
+                assert cacheInfo.nlink == 0
+                assert cacheInfo.cached > 1
+
+        def testCacheEvictionPartialEvict(self):
+            """
+            Ensure the cache eviction happens as expected.  Two files (20MB and 30MB) are written
+            sequentially into the job store in separate jobs.  The cache max is force set to 50MB.
+            A Third Job requests 10MB of disk requiring eviction of the 1st file.  Ensure that the
+            behavior is as expected.
+            """
+            self._testValidityOfCacheEvictTest()
+
+            # Explicitly set clean to always so even the failed cases get cleaned (This will
+            # overwrite the value set in setUp if it is ever changed in the future)
+            self.options.clean = 'always'
+
+            self._testCacheEviction(file1MB=20, file2MB=30, diskRequestMB=10)
+
+        def testCacheEvictionTotalEvict(self):
+            """
+            Ensure the cache eviction happens as expected.  Two files (20MB and 30MB) are written
+            sequentially into the job store in separate jobs.  The cache max is force set to 50MB.
+            A Third Job requests 10MB of disk requiring eviction of the 1st file.  Ensure that the
+            behavior is as expected.
+            """
+            self._testValidityOfCacheEvictTest()
+
+            # Explicitly set clean to always so even the failed cases get cleaned (This will
+            # overwrite the value set in setUp if it is ever changed in the future)
+            self.options.clean = 'always'
+
+            self._testCacheEviction(file1MB=20, file2MB=30, diskRequestMB=30)
+
+        def testCacheEvictionFailCase(self):
+            """
+            Ensure the cache eviction happens as expected.  Two files (20MB and 30MB) are written
+            sequentially into the job store in separate jobs.  The cache max is force set to 50MB.
+            A Third Job requests 10MB of disk requiring eviction of the 1st file.  Ensure that the
+            behavior is as expected.
+            """
+            self._testValidityOfCacheEvictTest()
+
+            # Explicitly set clean to always so even the failed cases get cleaned (This will
+            # overwrite the value set in setUp if it is ever changed in the future)
+            self.options.clean = 'always'
 
             self._testCacheEviction(file1MB=20, file2MB=30, diskRequestMB=60)
 
@@ -233,11 +604,13 @@ def _testCacheEviction(self, file1MB, file2MB, diskRequestMB):
             else:
                 expectedResult = 50 - file1MB if diskRequestMB <= file1MB else 0
             try:
-                A = Job.wrapJobFn(self._writeFileToJobStore, isLocalFile=True, fileMB=file1MB)
+                A = Job.wrapJobFn(self._writeFileToJobStoreWithAsserts, isLocalFile=True,
+                                  fileMB=file1MB)
                 # Sleep for 1 second after writing the first file so that their ctimes are
                 # guaranteed to be distinct for the purpose of this test.
                 B = Job.wrapJobFn(self._sleepy, timeToSleep=1)
-                C = Job.wrapJobFn(self._writeFileToJobStore, isLocalFile=True, fileMB=file2MB)
+                C = Job.wrapJobFn(self._writeFileToJobStoreWithAsserts, isLocalFile=True,
+                                  fileMB=file2MB)
                 D = Job.wrapJobFn(self._forceModifyCacheLockFile, newTotalMB=50, disk='0M')
                 E = Job.wrapJobFn(self._uselessFunc, disk=''.join([str(diskRequestMB), 'M']))
                 # Set it to > 2GB such that the cleanup jobs don't die in the non-fail cases
@@ -261,7 +634,7 @@ def _testCacheEviction(self, file1MB, file2MB, diskRequestMB):
                     self.fail('Toil did not raise the expected AssertionError')
 
         @staticmethod
-        def _writeFileToJobStore(job, isLocalFile, nonLocalDir=None, fileMB=1):
+        def _writeFileToJobStoreWithAsserts(job, isLocalFile, nonLocalDir=None, fileMB=1):
             """
             This function creates a file and writes it to the jobstore.
 
@@ -270,16 +643,8 @@ def _writeFileToJobStore(job, isLocalFile, nonLocalDir=None, fileMB=1):
                                     is created.
             :param int fileMB: Size of the created file in MB
             """
-            if isLocalFile:
-                work_dir = job.fileStore.getLocalTempDir()
-            else:
-                assert nonLocalDir is not None
-                work_dir = nonLocalDir
-            with open(os.path.join(work_dir, str(uuid4())), 'w') as testFile:
-                testFile.write(os.urandom(fileMB * 1024 * 1024))
-
-            fsID = job.fileStore.writeGlobalFile(testFile.name)
-
+            cls = hidden.AbstractNonCachingFileStoreTest
+            fsID, testFile = cls._writeFileToJobStore(job, isLocalFile, nonLocalDir, fileMB)
             actual = os.stat(testFile.name).st_nlink
             if isLocalFile:
                 # Since the file has been hard linked it should have nlink_count = threshold + 1
@@ -291,7 +656,6 @@ def _writeFileToJobStore(job, isLocalFile, nonLocalDir=None, fileMB=1):
                 assert actual == 1, 'Should have one nlink. Got %i.' % actual
             return fsID
 
-        # noinspection PyUnusedLocal
         @staticmethod
         def _sleepy(job, timeToSleep):
             """
@@ -342,21 +706,20 @@ def testAsyncWriteWithCaching(self):
             """
             Ensure the Async Writing of files happens as expected.  The first Job forcefully
             modifies the cache lock file to 1GB. The second asks for 1GB of disk and  writes a 900MB
-            file into cache and then rewrites it to the job store. The third asks for 1GB of disk
-            and then requests the file.
-
-            The second writeGlobalFile in the writing job should trigger an async write to the job
-            store and the since the first was written to cache, the second won't have space to be.
-            The third job should wait for the second to finish writing to the job store (through the
-            harbinger mechanism) and there should be no read-only error thrown by the async write
-            threads.
+            file into cache then rewrites it to the job store triggering an async write since the
+            two unique jobstore IDs point to the same local file.  Also, the second write is not
+            cached since the first was written to cache, and there "isn't enough space" to cache the
+            second.  Imediately assert that the second write isn't cached, and is being
+            asynchronously written to the job store (through the presence of a harbinger file).
+             Attempting to get the file from the jobstore should not fail.
             """
             self.options.retryCount = 0
+            self.options.logLevel = 'DEBUG'
             A = Job.wrapJobFn(self._forceModifyCacheLockFile, newTotalMB=1024, disk='1G')
-            B = Job.wrapJobFn(self._doubleWriteFileToJobStore, fileMB=900, disk='1G')
-            C = Job.wrapJobFn(self._readFromJobStoreWithoutAsssertions, fsID=B.rv(), disk='1G')
+            B = Job.wrapJobFn(self._doubleWriteFileToJobStore, fileMB=850, disk='900M')
             # Set it to > 2GB such that the cleanup jobs don't die.
-            D = Job.wrapJobFn(self._forceModifyCacheLockFile, newTotalMB=5000, disk='10M')
+            C = Job.wrapJobFn(self._readFromJobStoreWithoutAsssertions, fsID=B.rv(), disk='1G')
+            D = Job.wrapJobFn(self._forceModifyCacheLockFile, newTotalMB=5000, disk='1G')
             A.addChild(B)
             B.addChild(C)
             C.addChild(D)
@@ -372,12 +735,27 @@ def _doubleWriteFileToJobStore(job, fileMB):
             :param fileMB: File Size
             :return: Job store file ID for second written file
             """
+            jobStore = job.fileStore.jobStore
+            oldUpdateFileStream = jobStore.updateFileStream
+            import types
+            from contextlib import contextmanager
+
+            @contextmanager
+            def newUpdateFileStream(self, jobStoreFileID):
+                with oldUpdateFileStream(jobStoreFileID) as fileHandle:
+                    time.sleep(20)
+                    yield fileHandle
+
             job.fileStore.logToMaster('Double writing a file into job store')
             work_dir = job.fileStore.getLocalTempDir()
             with open(os.path.join(work_dir, str(uuid4())), 'w') as testFile:
                 testFile.write(os.urandom(fileMB * 1024 * 1024))
 
             job.fileStore.writeGlobalFile(testFile.name)
+            fsID = job.fileStore.writeGlobalFile(testFile.name)
+            hidden.AbstractCachingFileStoreTest._readFromJobStoreWithoutAsssertions(job, fsID)
+            # Make this take longer so we can test asynchronous writes across jobs/workers.
+            jobStore.updateFileStream = types.MethodType(newUpdateFileStream, jobStore)
             return job.fileStore.writeGlobalFile(testFile.name)
 
         @staticmethod
@@ -401,7 +779,8 @@ def testWriteNonLocalFileToJobStore(self):
             Ensure the file is not cached.
             """
             workdir = self._createTempDir(purpose='nonLocalDir')
-            A = Job.wrapJobFn(self._writeFileToJobStore, isLocalFile=False, nonLocalDir=workdir)
+            A = Job.wrapJobFn(self._writeFileToJobStoreWithAsserts, isLocalFile=False,
+                              nonLocalDir=workdir)
             Job.Runner.startToil(A, self.options)
 
         def testWriteLocalFileToJobStore(self):
@@ -409,7 +788,7 @@ def testWriteLocalFileToJobStore(self):
             Write a file from the localTempDir to the job store.  Such a file will be cached by
             default.  Ensure the file is cached.
             """
-            A = Job.wrapJobFn(self._writeFileToJobStore, isLocalFile=True)
+            A = Job.wrapJobFn(self._writeFileToJobStoreWithAsserts, isLocalFile=True)
             Job.Runner.startToil(A, self.options)
 
         # readGlobalFile tests
@@ -434,7 +813,8 @@ def _testCacheMissFunction(self, cacheReadFile):
             :param cacheReadFile: Does the read file need to be cached(T) or not(F)
             """
             workdir = self._createTempDir(purpose='nonLocalDir')
-            A = Job.wrapJobFn(self._writeFileToJobStore, isLocalFile=False, nonLocalDir=workdir)
+            A = Job.wrapJobFn(self._writeFileToJobStoreWithAsserts, isLocalFile=False,
+                              nonLocalDir=workdir)
             B = Job.wrapJobFn(self._readFromJobStore, isCachedFile=False,
                               cacheReadFile=cacheReadFile, fsID=A.rv())
             A.addChild(B)
@@ -480,7 +860,7 @@ def testReadCachHitFileFromJobStore(self):
             Read a file from the file store that has a corresponding cached copy.  Ensure the number
             of links on the file are appropriate.
             """
-            A = Job.wrapJobFn(self._writeFileToJobStore, isLocalFile=True)
+            A = Job.wrapJobFn(self._writeFileToJobStoreWithAsserts, isLocalFile=True)
             B = Job.wrapJobFn(self._readFromJobStore, isCachedFile=True, cacheReadFile=None,
                               fsID=A.rv())
             A.addChild(B)
@@ -516,7 +896,8 @@ def _testMultipleJobsReadGlobalFileFunction(self, cacheHit):
             workdir = self._createTempDir(purpose=dirPurpose)
             with open(os.path.join(workdir, 'test'), 'w') as x:
                 x.write(str(0))
-            A = Job.wrapJobFn(self._writeFileToJobStore, isLocalFile=cacheHit, nonLocalDir=workdir,
+            A = Job.wrapJobFn(self._writeFileToJobStoreWithAsserts, isLocalFile=cacheHit,
+                              nonLocalDir=workdir,
                               fileMB=256)
             B = Job.wrapJobFn(self._probeJobReqs, sigmaJob=100, disk='100M')
             jobs = {}
@@ -617,8 +998,8 @@ def _returnFileTestFn(job, jobDisk, initialCachedSize, nonLocalDir, numIters=100
             # Add one file for the sake of having something in the job store
             writeFileSize = random.randint(0, 30)
             jobDisk -= writeFileSize * 1024 * 1024
-            cls = hidden.AbstractCacheTest
-            fsId = cls._writeFileToJobStore(job, isLocalFile=True, fileMB=writeFileSize)
+            cls = hidden.AbstractCachingFileStoreTest
+            fsId = cls._writeFileToJobStoreWithAsserts(job, isLocalFile=True, fileMB=writeFileSize)
             writtenFiles[fsId] = writeFileSize
             if job.fileStore._fileIsCached(writtenFiles.keys()[0]):
                 cached += writeFileSize * 1024 * 1024
@@ -630,16 +1011,17 @@ def _returnFileTestFn(job, jobDisk, initialCachedSize, nonLocalDir, numIters=100
                 if randVal < 0.33:  # Write
                     writeFileSize = random.randint(0, 30)
                     if random.random() <= 0.5:  # Write a local file
-                        fsID = cls._writeFileToJobStore(job, isLocalFile=True, fileMB=writeFileSize)
+                        fsID = cls._writeFileToJobStoreWithAsserts(job, isLocalFile=True,
+                                                                   fileMB=writeFileSize)
                         writtenFiles[fsID] = writeFileSize
                         localFileIDs[fsID].append('local')
                         jobDisk -= writeFileSize * 1024 * 1024
                         if job.fileStore._fileIsCached(fsID):
                             cached += writeFileSize * 1024 * 1024
                     else:  # Write a non-local file
-                        fsID = cls._writeFileToJobStore(job, isLocalFile=False,
-                                                        nonLocalDir=nonLocalDir,
-                                                        fileMB=writeFileSize)
+                        fsID = cls._writeFileToJobStoreWithAsserts(job, isLocalFile=False,
+                                                                   nonLocalDir=nonLocalDir,
+                                                                   fileMB=writeFileSize)
                         writtenFiles[fsID] = writeFileSize
                         localFileIDs[fsID].append('non-local')
                         # No change to the job since there was no caching
@@ -701,405 +1083,152 @@ def _requirementsConcur(job, jobDisk, cached):
                     assert cacheInfo.cached == cached
                 else:
                     assert cacheInfo.cached == 0
-            assert jobState['jobReqs'] == jobDisk
-
-        # Testing the resumability of a failed worker
-        def testControlledFailedWorkerRetry(self):
-            """
-            Conduct a couple of job store operations.  Then die.  Ensure that the restarted job is
-            tracking values in the cache state file appropriately.
-            """
-            workdir = self._createTempDir(purpose='nonLocalDir')
-            self.options.retryCount = 1
-            F = Job.wrapJobFn(self._controlledFailTestFn, jobDisk=2 * 1024 * 1024 * 1024,
-                              testDir=workdir,
-                              disk='2G')
-            G = Job.wrapJobFn(self._probeJobReqs, sigmaJob=100, disk='100M')
-            F.addChild(G)
-            Job.Runner.startToil(F, self.options)
-
-        @staticmethod
-        def _controlledFailTestFn(job, jobDisk, testDir):
-            """
-            This is the aux function for the controlled failed worker test.  It does a couple of
-            cache operations, fails, then checks whether the new worker starts with the expected
-            value, and whether it exits with zero for sigmaJob.
-
-            :param float jobDisk: Disk space supplied for this job
-            :param str testDir: T3sting directory
-            """
-            cls = hidden.AbstractCacheTest
-            if os.path.exists(os.path.join(testDir, 'testfile.test')):
-                with open(os.path.join(testDir, 'testfile.test'), 'r') as fH:
-                    cached = unpack('d', fH.read())[0]
-                cls._requirementsConcur(job, jobDisk, cached)
-                cls._returnFileTestFn(job, jobDisk, cached, testDir, 20)
-            else:
-                modifiedJobReqs, cached = cls._returnFileTestFn(job, jobDisk, 0, testDir, 20)
-                with open(os.path.join(testDir, 'testfile.test'), 'w') as fH:
-                    fH.write(pack('d', cached))
-                os.kill(os.getpid(), signal.SIGKILL)
-
-        def testRemoveLocalMutablyReadFile(self):
-            """
-            If a mutably read file is deleted by the user, it is ok.
-            """
-            self._deleteLocallyReadFilesFn(readAsMutable=True)
-
-        def testRemoveLocalImmutablyReadFile(self):
-            """
-            If an immutably read file is deleted by the user, it is not ok.
-            """
-            self._deleteLocallyReadFilesFn(readAsMutable=False)
-
-        def _deleteLocallyReadFilesFn(self, readAsMutable):
-            self.options.retryCount = 0
-            A = Job.wrapJobFn(self._writeFileToJobStore, isLocalFile=True, memory='10M')
-            B = Job.wrapJobFn(self._removeReadFileFn, A.rv(), readAsMutable=readAsMutable,
-                              memory='20M')
-            A.addChild(B)
-            Job.Runner.startToil(A, self.options)
-
-        @staticmethod
-        def _removeReadFileFn(job, fileToDelete, readAsMutable):
-            """
-            Accept a file. Run os.remove on it. Then attempt to delete it locally. This will raise
-            an error for files read immutably.
-            Then write a new file to the jobstore and try to do the same. This should always raise
-            an error
-
-            :param fileToDelete: File written to the job store that is tracked by the cache
-            """
-            work_dir = job.fileStore.getLocalTempDir()
-            # Are we processing the read file or the written file?
-            processsingReadFile = True
-            # Read in the file
-            outfile = job.fileStore.readGlobalFile(fileToDelete, os.path.join(work_dir, 'temp'),
-                                                   mutable=readAsMutable)
-            tempfile = os.path.join(work_dir, 'tmp.tmp')
-            # The first time we run this loop, processsingReadFile is True and fileToDelete is the
-            # file read from the job store.  The second time, processsingReadFile is False and
-            # fileToDelete is one that was just written in to the job store. Ensure the correct
-            # behaviour is seen in both conditions.
-            while True:
-                os.rename(outfile, tempfile)
-                try:
-                    job.fileStore.deleteLocalFile(fileToDelete)
-                except IllegalDeletionCacheError:
-                    job.fileStore.logToMaster('Detected a deleted file %s.' % fileToDelete)
-                    os.rename(tempfile, outfile)
-                else:
-                    # If we are processing the write test, or if we are testing the immutably read
-                    # file, we should not reach here.
-                    assert processsingReadFile and readAsMutable
-                if processsingReadFile:
-                    processsingReadFile = False
-                    # Write a file
-                    with open(os.path.join(work_dir, str(uuid4())), 'w') as testFile:
-                        testFile.write(os.urandom(1 * 1024 * 1024))
-                    fileToDelete = job.fileStore.writeGlobalFile(testFile.name)
-                    outfile = testFile.name
-                else:
-                    break
-
-        def testDeleteLocalFile(self):
-            """
-            Test the deletion capabilities of deleteLocalFile
-            """
-            self.options.retryCount = 0
-            workdir = self._createTempDir(purpose='nonLocalDir')
-            A = Job.wrapJobFn(self._deleteLocalFileFn, nonLocalDir=workdir)
-            Job.Runner.startToil(A, self.options)
-
-        @staticmethod
-        def _deleteLocalFileFn(job, nonLocalDir):
-            """
-            Test deleteLocalFile on a local write, non-local write, read, mutable read, and bogus
-            jobstore IDs.
-            """
-            work_dir = job.fileStore.getLocalTempDir()
-            # Write local file
-            with open(os.path.join(work_dir, str(uuid4())), 'w') as localFile:
-                localFile.write(os.urandom(1 * 1024 * 1024))
-            localFsID = job.fileStore.writeGlobalFile(localFile.name)
-            # write Non-Local File
-            with open(os.path.join(nonLocalDir, str(uuid4())), 'w') as nonLocalFile:
-                nonLocalFile.write(os.urandom(1 * 1024 * 1024))
-            nonLocalFsID = job.fileStore.writeGlobalFile(nonLocalFile.name)
-            # Delete fsid of local file. The file should be deleted
-            job.fileStore.deleteLocalFile(localFsID)
-            assert not os.path.exists(localFile.name)
-            # Delete fsid of non-local file. The file should persist
-            job.fileStore.deleteLocalFile(nonLocalFsID)
-            assert os.path.exists(nonLocalFile.name)
-            # Read back one file and then delete it
-            readBackFile1 = job.fileStore.readGlobalFile(localFsID)
-            job.fileStore.deleteLocalFile(localFsID)
-            assert not os.path.exists(readBackFile1)
-            # Read back one file with 2 different names and then delete it. Assert both get deleted
-            readBackFile1 = job.fileStore.readGlobalFile(localFsID)
-            readBackFile2 = job.fileStore.readGlobalFile(localFsID)
-            job.fileStore.deleteLocalFile(localFsID)
-            assert not os.path.exists(readBackFile1)
-            assert not os.path.exists(readBackFile2)
-            # Try to get a bogus FSID
-            try:
-                job.fileStore.readGlobalFile('bogus')
-            except NoSuchFileException:
-                pass
-
-        # These tests don't test the cache functions but they test code that heavily inherits cache
-        # code
-        def testDeferredFunctionRunsWithMethod(self):
-            """
-            Refer docstring in _testDeferredFunctionRuns.
-            Test with Method
-            """
-            self._testDeferredFunctionRuns(self._writeNonLocalFilesMethod)
-
-        def testDeferredFunctionRunsWithClassMethod(self):
-            """
-            Refer docstring in _testDeferredFunctionRuns.
-            Test with Class Method
-            """
-            self._testDeferredFunctionRuns(self._writeNonLocalFilesClassMethod)
-
-        def testDeferredFunctionRunsWithLambda(self):
-            """
-            Refer docstring in _testDeferredFunctionRuns.
-            Test with Lambda
-            """
-            self._testDeferredFunctionRuns(self._writeNonLocalFilesLambda)
-
-        def _testDeferredFunctionRuns(self, callableFn):
-            """
-            Create 2 files. Make a job that writes data to them. Register a deferred function that
-            deletes the two files (one passed as an arg, adn one as a kwarg) and later assert that
-            the files have been deleted.
-
-            :param function callableFn: The function to use in the test.
-            :return: None
-            """
-            workdir = self._createTempDir(purpose='nonLocalDir')
-            nonLocalFile1 = os.path.join(workdir, str(uuid4()))
-            nonLocalFile2 = os.path.join(workdir, str(uuid4()))
-            open(nonLocalFile1, 'w').close()
-            open(nonLocalFile2, 'w').close()
-            assert os.path.exists(nonLocalFile1)
-            assert os.path.exists(nonLocalFile2)
-            A = Job.wrapJobFn(callableFn, files=(nonLocalFile1, nonLocalFile2))
-            Job.Runner.startToil(A, self.options)
-            assert not os.path.exists(nonLocalFile1)
-            assert not os.path.exists(nonLocalFile2)
-
-        @staticmethod
-        def _writeNonLocalFilesMethod(job, files):
-            """
-            Write some data to 2 files.  Pass them to a registered deferred method.
-
-            :param tuple files: the tuple of the two files to work with
-            :return: None
-            """
-            for nlf in files:
-                with open(nlf, 'w') as nonLocalFileHandle:
-                    nonLocalFileHandle.write(os.urandom(1 * 1024 * 1024))
-            job.defer(_deleteMethods._deleteFileMethod, files[0], nlf=files[1])
-            return None
-
-        @staticmethod
-        def _writeNonLocalFilesClassMethod(job, files):
-            """
-            Write some data to 2 files.  Pass them to a registered deferred class method.
-
-            :param tuple files: the tuple of the two files to work with
-            :return: None
-            """
-            for nlf in files:
-                with open(nlf, 'w') as nonLocalFileHandle:
-                    nonLocalFileHandle.write(os.urandom(1 * 1024 * 1024))
-            job.defer(_deleteMethods._deleteFileClassMethod, files[0], nlf=files[1])
-            return None
-
-        @staticmethod
-        def _writeNonLocalFilesLambda(job, files):
-            """
-            Write some data to 2 files.  Pass them to a registered deferred Lambda.
-
-            :param tuple files: the tuple of the two files to work with
-            :return: None
-            """
-            lmd = lambda x, nlf: [os.remove(x), os.remove(nlf)]
-            for nlf in files:
-                with open(nlf, 'w') as nonLocalFileHandle:
-                    nonLocalFileHandle.write(os.urandom(1 * 1024 * 1024))
-            job.defer(lmd, files[0], nlf=files[1])
-            return None
+            assert jobState['jobReqs'] == jobDisk
 
-        def testDeferredFunctionRunsWithFailures(self):
+        # Testing the resumability of a failed worker
+        def testControlledFailedWorkerRetry(self):
             """
-            Create 2 non local filesto use as flags.  Create a job that registers a function that
-            deletes one non local file.  If that file exists, the job SIGKILLs itself. If it doesn't
-            exist, the job registers a second deferred function to delete the second non local file
-            and exits normally.
-
-            Initially the first file exists, so the job should SIGKILL itself and neither deferred
-            function will run (in fact, the second should not even be registered). On the restart,
-            the first deferred function should run and the first file should not exist, but the
-            second one should.  We assert the presence of the second, then register the second
-            deferred function and exit normally.  At the end of the test, neither file should exist.
-
-            Incidentally, this also tests for multiple registered deferred functions, and the case
-            where a deferred function fails (since the first file doesn't exist on the retry).
+            Conduct a couple of job store operations.  Then die.  Ensure that the restarted job is
+            tracking values in the cache state file appropriately.
             """
-            self.options.retryCount = 1
             workdir = self._createTempDir(purpose='nonLocalDir')
-            nonLocalFile1 = os.path.join(workdir, str(uuid4()))
-            nonLocalFile2 = os.path.join(workdir, str(uuid4()))
-            open(nonLocalFile1, 'w').close()
-            open(nonLocalFile2, 'w').close()
-            assert os.path.exists(nonLocalFile1)
-            assert os.path.exists(nonLocalFile2)
-            A = Job.wrapJobFn(self._deferredFunctionRunsWithFailuresFn,
-                              files=(nonLocalFile1, nonLocalFile2))
-            Job.Runner.startToil(A, self.options)
-            assert not os.path.exists(nonLocalFile1)
-            assert not os.path.exists(nonLocalFile2)
+            self.options.retryCount = 1
+            F = Job.wrapJobFn(self._controlledFailTestFn, jobDisk=2 * 1024 * 1024 * 1024,
+                              testDir=workdir,
+                              disk='2G')
+            G = Job.wrapJobFn(self._probeJobReqs, sigmaJob=100, disk='100M')
+            F.addChild(G)
+            Job.Runner.startToil(F, self.options)
 
         @staticmethod
-        def _deferredFunctionRunsWithFailuresFn(job, files):
+        def _controlledFailTestFn(job, jobDisk, testDir):
             """
-            Refer testDeferredFunctionRunsWithFailures
+            This is the aux function for the controlled failed worker test.  It does a couple of
+            cache operations, fails, then checks whether the new worker starts with the expected
+            value, and whether it exits with zero for sigmaJob.
 
-            :param tuple files: the tuple of the two files to work with
-            :return: None
+            :param float jobDisk: Disk space supplied for this job
+            :param str testDir: T3sting directory
             """
-            job.defer(hidden.AbstractCacheTest._deleteFile, files[0])
-            if os.path.exists(files[0]):
-                os.kill(os.getpid(), signal.SIGKILL)
+            cls = hidden.AbstractCachingFileStoreTest
+            if os.path.exists(os.path.join(testDir, 'testfile.test')):
+                with open(os.path.join(testDir, 'testfile.test'), 'r') as fH:
+                    cached = unpack('d', fH.read())[0]
+                cls._requirementsConcur(job, jobDisk, cached)
+                cls._returnFileTestFn(job, jobDisk, cached, testDir, 20)
             else:
-                assert os.path.exists(files[1])
-                job.defer(hidden.AbstractCacheTest._deleteFile, files[1])
+                modifiedJobReqs, cached = cls._returnFileTestFn(job, jobDisk, 0, testDir, 20)
+                with open(os.path.join(testDir, 'testfile.test'), 'w') as fH:
+                    fH.write(pack('d', cached))
+                os.kill(os.getpid(), signal.SIGKILL)
 
-        @staticmethod
-        def _deleteFile(nonLocalFile, nlf=None):
+        def testRemoveLocalMutablyReadFile(self):
             """
-            Delete nonLocalFile and nlf
-            :param str nonLocalFile:
-            :param str nlf:
-            :return: None
+            If a mutably read file is deleted by the user, it is ok.
             """
-            os.remove(nonLocalFile)
-            if nlf is not None:
-                os.remove(nlf)
+            self._deleteLocallyReadFilesFn(readAsMutable=True)
 
-        def testNewJobsCanHandleOtherJobDeaths(self):
+        def testRemoveLocalImmutablyReadFile(self):
             """
-            Create 2 non-local files and then create 2 jobs. The first job registers a deferred job
-            to delete the second non-local file, deletes the first non-local file and then kills
-            itself.  The second job waits for the first file to be deleted, then sleeps for a few
-            seconds and then spawns a child. the child of the second does nothing. However starting
-            it should handle the untimely demise of the first job and run the registered deferred
-            function that deletes the first file.  We assert the absence of the two files at the
-            end of the run.
+            If an immutably read file is deleted by the user, it is not ok.
             """
-            # There can be no retries
+            self._deleteLocallyReadFilesFn(readAsMutable=False)
+
+        def _deleteLocallyReadFilesFn(self, readAsMutable):
             self.options.retryCount = 0
-            workdir = self._createTempDir(purpose='nonLocalDir')
-            nonLocalFile1 = os.path.join(workdir, str(uuid4()))
-            nonLocalFile2 = os.path.join(workdir, str(uuid4()))
-            open(nonLocalFile1, 'w').close()
-            open(nonLocalFile2, 'w').close()
-            assert os.path.exists(nonLocalFile1)
-            assert os.path.exists(nonLocalFile2)
-            files = [nonLocalFile1, nonLocalFile2]
-            root = Job()
-            A = Job.wrapJobFn(self._testNewJobsCanHandleOtherJobDeaths_A, files=files)
-            B = Job.wrapJobFn(self._testNewJobsCanHandleOtherJobDeaths_B, files=files)
-            C = Job.wrapJobFn(self._testNewJobsCanHandleOtherJobDeaths_C, files=files,
-                              expectedResult=False)
-            root.addChild(A)
-            root.addChild(B)
-            B.addChild(C)
-            try:
-                Job.Runner.startToil(root, self.options)
-            except FailedJobsException as e:
-                pass
+            A = Job.wrapJobFn(self._writeFileToJobStoreWithAsserts, isLocalFile=True, memory='10M')
+            B = Job.wrapJobFn(self._removeReadFileFn, A.rv(), readAsMutable=readAsMutable,
+                              memory='20M')
+            A.addChild(B)
+            Job.Runner.startToil(A, self.options)
 
         @staticmethod
-        def _testNewJobsCanHandleOtherJobDeaths_A(job, files):
+        def _removeReadFileFn(job, fileToDelete, readAsMutable):
             """
-            Defer deletion of files[1], then wait for _testNewJobsCanHandleOtherJobDeaths_B to
-            start up, and finally delete files[0] before sigkilling self.
+            Accept a file. Run os.remove on it. Then attempt to delete it locally. This will raise
+            an error for files read immutably.
+            Then write a new file to the jobstore and try to do the same. This should always raise
+            an error
 
-            :param tuple files: the tuple of the two files to work with
-            :return: None
+            :param fileToDelete: File written to the job store that is tracked by the cache
             """
-            # Write the pid to files[1] such that we can be sure that this process has died before
-            # we spawn the next job that will do the cleanup.
-            with open(files[1], 'w') as fileHandle:
-                fileHandle.write(str(os.getpid()))
-            job.defer(hidden.AbstractCacheTest._deleteFile, files[1])
-            while os.stat(files[0]).st_size == 0:
-                time.sleep(0.5)
-            os.remove(files[0])
-            os.kill(os.getpid(), signal.SIGKILL)
-
-        @staticmethod
-        def _testNewJobsCanHandleOtherJobDeaths_B(job, files):
-            # Write something to files[0] such that we can be sure that this process has started
-            # before _testNewJobsCanHandleOtherJobDeaths_A kills itself.
-            with open(files[0], 'w') as fileHandle:
-                fileHandle.write(str(os.getpid()))
-            while os.path.exists(files[0]):
-                time.sleep(0.5)
-            # Get the pid of _testNewJobsCanHandleOtherJobDeaths_A and wait for it to truly be dead.
-            with open(files[1], 'r') as fileHandle:
-                meeseeksPID = int(fileHandle.read())
-            while FileStore.HarbingerFile._pidExists(meeseeksPID):
-                time.sleep(0.5)
-            # Now that we are convinced that_testNewJobsCanHandleOtherJobDeaths_A has died, we can
-            # spawn the next job
-            return None
+            work_dir = job.fileStore.getLocalTempDir()
+            # Are we processing the read file or the written file?
+            processsingReadFile = True
+            # Read in the file
+            outfile = job.fileStore.readGlobalFile(fileToDelete, os.path.join(work_dir, 'temp'),
+                                                   mutable=readAsMutable)
+            tempfile = os.path.join(work_dir, 'tmp.tmp')
+            # The first time we run this loop, processsingReadFile is True and fileToDelete is the
+            # file read from the job store.  The second time, processsingReadFile is False and
+            # fileToDelete is one that was just written in to the job store. Ensure the correct
+            # behaviour is seen in both conditions.
+            while True:
+                os.rename(outfile, tempfile)
+                try:
+                    job.fileStore.deleteLocalFile(fileToDelete)
+                except IllegalDeletionCacheError:
+                    job.fileStore.logToMaster('Detected a deleted file %s.' % fileToDelete)
+                    os.rename(tempfile, outfile)
+                else:
+                    # If we are processing the write test, or if we are testing the immutably read
+                    # file, we should not reach here.
+                    assert processsingReadFile and readAsMutable
+                if processsingReadFile:
+                    processsingReadFile = False
+                    # Write a file
+                    with open(os.path.join(work_dir, str(uuid4())), 'w') as testFile:
+                        testFile.write(os.urandom(1 * 1024 * 1024))
+                    fileToDelete = job.fileStore.writeGlobalFile(testFile.name)
+                    outfile = testFile.name
+                else:
+                    break
 
-        @staticmethod
-        def _testNewJobsCanHandleOtherJobDeaths_C(job, files, expectedResult):
+        def testDeleteLocalFile(self):
             """
-            Asserts whether the files exist or not.
-
-            :param Job job: Job
-            :param list files: list of files to test
-            :param bool expectedResult: Are we expecting the files to exist or not?
+            Test the deletion capabilities of deleteLocalFile
             """
-            for testFile in files:
-                assert os.path.exists(testFile) is expectedResult
+            self.options.retryCount = 0
+            workdir = self._createTempDir(purpose='nonLocalDir')
+            A = Job.wrapJobFn(self._deleteLocalFileFn, nonLocalDir=workdir)
+            Job.Runner.startToil(A, self.options)
 
-        def testBatchSystemCleanupCanHandleWorkerDeaths(self):
+        @staticmethod
+        def _deleteLocalFileFn(job, nonLocalDir):
             """
-            Create a non-local files. Create a job that registers a deferred job to delete the file
-            and then kills itself.
-
-            Assert that the file is missing after the pipeline fails.
+            Test deleteLocalFile on a local write, non-local write, read, mutable read, and bogus
+            jobstore IDs.
             """
-            # There can be no retries
-            self.options.retryCount = 0
-            workdir = self._createTempDir(purpose='nonLocalDir')
-            nonLocalFile1 = os.path.join(workdir, str(uuid4()))
-            nonLocalFile2 = os.path.join(workdir, str(uuid4()))
-            # The first file has to be non zero or meseeks will go into an infinite sleep
-            file1 = open(nonLocalFile1, 'w')
-            file1.write('test')
-            file1.close()
-            open(nonLocalFile2, 'w').close()
-            assert os.path.exists(nonLocalFile1)
-            assert os.path.exists(nonLocalFile2)
-            A = Job.wrapJobFn(self._testNewJobsCanHandleOtherJobDeaths_A,
-                              files=(nonLocalFile1, nonLocalFile2))
+            work_dir = job.fileStore.getLocalTempDir()
+            # Write local file
+            with open(os.path.join(work_dir, str(uuid4())), 'w') as localFile:
+                localFile.write(os.urandom(1 * 1024 * 1024))
+            localFsID = job.fileStore.writeGlobalFile(localFile.name)
+            # write Non-Local File
+            with open(os.path.join(nonLocalDir, str(uuid4())), 'w') as nonLocalFile:
+                nonLocalFile.write(os.urandom(1 * 1024 * 1024))
+            nonLocalFsID = job.fileStore.writeGlobalFile(nonLocalFile.name)
+            # Delete fsid of local file. The file should be deleted
+            job.fileStore.deleteLocalFile(localFsID)
+            assert not os.path.exists(localFile.name)
+            # Delete fsid of non-local file. The file should persist
+            job.fileStore.deleteLocalFile(nonLocalFsID)
+            assert os.path.exists(nonLocalFile.name)
+            # Read back one file and then delete it
+            readBackFile1 = job.fileStore.readGlobalFile(localFsID)
+            job.fileStore.deleteLocalFile(localFsID)
+            assert not os.path.exists(readBackFile1)
+            # Read back one file with 2 different names and then delete it. Assert both get deleted
+            readBackFile1 = job.fileStore.readGlobalFile(localFsID)
+            readBackFile2 = job.fileStore.readGlobalFile(localFsID)
+            job.fileStore.deleteLocalFile(localFsID)
+            assert not os.path.exists(readBackFile1)
+            assert not os.path.exists(readBackFile2)
+            # Try to get a bogus FSID
             try:
-                Job.Runner.startToil(A, self.options)
-            except FailedJobsException:
+                job.fileStore.readGlobalFile('bogus')
+            except NoSuchFileException:
                 pass
-            assert not os.path.exists(nonLocalFile1)
-            assert not os.path.exists(nonLocalFile2)
 
 
 class _deleteMethods(object):
@@ -1126,42 +1255,46 @@ def _deleteFileClassMethod(cls, nonLocalFile, nlf=None):
             os.remove(nlf)
 
 
-class FileJobStoreCacheTest(hidden.AbstractCacheTest):
-    def _getTestJobStore(self):
-        return self._getTestJobStorePath()
+class NonCachingFileStoreTestWithFileJobStore(hidden.AbstractNonCachingFileStoreTest):
+    jobStoreType = 'file'
+
+
+class CachingFileStoreTestWithFileJobStore(hidden.AbstractCachingFileStoreTest):
+    jobStoreType = 'file'
 
 
 @needs_aws
-class AwsJobStoreCacheTest(hidden.AbstractCacheTest):
-    def _getTestJobStore(self):
-        return 'aws:%s:cache-tests-%s' % (self.awsRegion(), uuid4())
+class NonCachingFileStoreTestWithAwsJobStore(hidden.AbstractNonCachingFileStoreTest):
+    jobStoreType = 'aws'
 
-    @unittest.skipIf(testingIsAutomatic, "To save time")
-    def testExtremeCacheSetup(self):
-        super(AwsJobStoreCacheTest, self).testExtremeCacheSetup()
+
+@needs_aws
+class CachingFileStoreTestWithAwsJobStore(hidden.AbstractCachingFileStoreTest):
+    jobStoreType = 'aws'
 
 
 @needs_azure
 @experimental
-class AzureJobStoreCacheTest(hidden.AbstractCacheTest):
-    def _getTestJobStore(self):
-        return 'azure:toiltest:cache-tests-' + str(uuid4())
+class NonCachingFileStoreTestWithAzureJobStore(hidden.AbstractNonCachingFileStoreTest):
+    jobStoreType = 'azure'
 
-    @unittest.skipIf(testingIsAutomatic, "To save time")
-    def testExtremeCacheSetup(self):
-        super(AzureJobStoreCacheTest, self).testExtremeCacheSetup()
+
+@needs_azure
+@experimental
+class CachingFileStoreTestWithAzureJobStore(hidden.AbstractCachingFileStoreTest):
+    jobStoreType = 'azure'
 
 
 @experimental
 @needs_google
-class GoogleJobStoreCacheTest(hidden.AbstractCacheTest):
-    def _getTestJobStore(self):
-        projectID = 'cgc-05-0006'
-        return 'google:' + projectID + ':cache-tests-' + str(uuid4())
+class NonCachingFileStoreTestWithGoogleJobStore(hidden.AbstractNonCachingFileStoreTest):
+    jobStoreType = 'google'
+
 
-    @unittest.skipIf(testingIsAutomatic, "To save time")
-    def testExtremeCacheSetup(self):
-        super(GoogleJobStoreCacheTest, self).testExtremeCacheSetup()
+@experimental
+@needs_google
+class CachingFileStoreTestWithGoogleJobStore(hidden.AbstractCachingFileStoreTest):
+    jobStoreType = 'google'
 
 
 def _exportStaticMethodAsGlobalFunctions(cls):
@@ -1177,4 +1310,6 @@ def _exportStaticMethodAsGlobalFunctions(cls):
                 globals()[name] = method
 
 
-_exportStaticMethodAsGlobalFunctions(hidden.AbstractCacheTest)
+_exportStaticMethodAsGlobalFunctions(hidden.AbstractFileStoreTest)
+_exportStaticMethodAsGlobalFunctions(hidden.AbstractCachingFileStoreTest)
+_exportStaticMethodAsGlobalFunctions(hidden.AbstractNonCachingFileStoreTest)
diff --git a/src/toil/test/src/jobCacheEjectionTest.py b/src/toil/test/src/jobCacheEjectionTest.py
deleted file mode 100644
--- a/src/toil/test/src/jobCacheEjectionTest.py
+++ /dev/null
@@ -1,92 +0,0 @@
-# Copyright (C) 2015-2016 Regents of the University of California
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-from __future__ import absolute_import, print_function
-import random
-import os
-import tempfile
-
-from toil.job import Job
-from toil.test import ToilTest
-
-class JobCacheEjectionTest(ToilTest):
-    """
-    Tests testing the Job.FileStore class
-    """
-    def testCacheEjection(self):
-        """
-        Test cache always always ejects least recently created file
-        """
-        # Makes three jobs that create an output file each which they write to filestore.  The combined size of any two
-        # files is always less that cacheSize but the combined size of all 3 is always more so 1 file always has to be
-        # ejected. Test to ensure that A is always ejected regardless of size.
-        #  Make a temp directory for the test
-        test_dir = self._createTempDir()
-        for test in xrange(10):
-            options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
-            options.logLevel = "DEBUG"
-            options.cacheSize = 100000
-            options.retryCount=100
-            options.badWorker=0.5
-            options.badWorkerFailInterval = 1.0
-            # Create a temp file to write the test results
-            handle, logfile = tempfile.mkstemp(dir=test_dir)
-            os.close(handle)
-            file_sizes = [50000, 40000, 30000]
-            # Randomize to (potentially) test all combinations
-            random.shuffle(file_sizes)
-            # Run the workflow. A, B and C do the cache operations, and D prints test status to tempFile
-            A = Job.wrapJobFn(fileTestJob, file_sizes[0])
-            B = Job.wrapJobFn(fileTestJob, file_sizes[0])
-            C = Job.wrapJobFn(fileTestJob, file_sizes[0])
-            D = Job.wrapJobFn(fileTestCache, A.rv(), B.rv(), C.rv(), logfile)
-            A.addChild(B)
-            B.addChild(C)
-            C.addChild(D)
-            Job.Runner.startToil(A, options)
-            #  Assert jobs passed by reading test results from tempFile
-            with open(logfile, 'r') as outfile:
-                for test_status in outfile:
-                    assert test_status.strip() == 'True'
-
-
-
-def fileTestCache(job, file_a, file_b, file_c, logfile):
-    """
-    Test job exercises Job.FileStore functions
-    """
-    with open(logfile, 'w') as lf:
-        print(file_a not in job.fileStore._jobStoreFileIDToCacheLocation.items(), file=lf)
-        print(file_b not in job.fileStore._jobStoreFileIDToCacheLocation.items(), file=lf)
-        print(file_c not in job.fileStore._jobStoreFileIDToCacheLocation.items(), file=lf)
-
-def fileTestJob(job, file_size):
-    """
-    Test job exercises Job.FileStore functions
-    """
-    def randomString(str_len):
-        chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-        s = "".join(map(lambda i : random.choice(chars), xrange(str_len)))
-        return s
-    if random.random() > 0.5:
-        #Make a local copy of the file
-        tempFile = job.fileStore.getLocalTempFile()
-    else:
-        tempDir = job.fileStore.getLocalTempDir()
-        handle, tempFile = tempfile.mkstemp(dir=tempDir)
-        os.close(handle)
-    with open(tempFile, 'w') as fH:
-        fH.write(os.urandom(file_size))
-    job.fileStore.writeGlobalFile(tempFile)
-    return fH.name
-
diff --git a/src/toil/test/src/jobFileStoreTest.py b/src/toil/test/src/jobFileStoreTest.py
--- a/src/toil/test/src/jobFileStoreTest.py
+++ b/src/toil/test/src/jobFileStoreTest.py
@@ -22,9 +22,12 @@
 
 PREFIX_LENGTH=200
 
+
+# TODO: This test is ancient and while similar tests exist in `fileStoreTest.py`, none of them look
+# at the contents of read files and thus we will let this test remain as-is.
 class JobFileStoreTest(ToilTest):
     """
-    Tests testing the Job.FileStore class
+    Tests testing the methods defined in :class:toil.fileStore.FileStore.
     """
     def testCachingFileStore(self):
         options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
@@ -37,12 +40,12 @@ def testNonCachingFileStore(self):
         with Toil(options) as workflow:
             workflow.start(Job.wrapJobFn(simpleFileStoreJob))
 
-    def testJobFileStoreWithSmallCache(self, retryCount=0, badWorker=0.0,
-                         stringNo=1, stringLength=1000000, cacheSize=10000, testNo=2):
+    def _testJobFileStore(self, retryCount=0, badWorker=0.0, stringNo=1, stringLength=1000000,
+                          testNo=2):
         """
         Creates a chain of jobs, each reading and writing files using the 
-        Job.FileStore interface. Verifies the files written are always what we expect.
-        The chain tests the caching behavior. 
+        toil.fileStore.FileStore interface. Verifies the files written are always what we
+        expect.
         """
         for test in xrange(testNo):
             #Make a list of random strings, each of 100k chars and hash the first 200 
@@ -55,7 +58,6 @@ def randomString():
             testStrings = dict(map(lambda i : randomString(), xrange(stringNo)))
             options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
             options.logLevel = "INFO"
-            options.cacheSize = cacheSize
             options.retryCount=retryCount
             options.badWorker=badWorker
             options.badWorkerFailInterval = 1.0
@@ -64,35 +66,24 @@ def randomString():
             Job.Runner.startToil(Job.wrapJobFn(fileTestJob, [], 
                                                testStrings, chainLength), 
                                  options)
-            
-    def testJobFileStoreWithOverLargeCache(self):
-        """
-        Tests case that all files are cached.
-        """
-        self.testJobFileStoreWithSmallCache(retryCount=0, badWorker=0.0, 
-                         stringNo=5, stringLength=1000000, 
-                         cacheSize=10000000)
         
-    def testJobFileStoreWithMediumCache(self):
+    def testJobFileStore(self):
         """
         Tests case that about half the files are cached
         """
-        self.testJobFileStoreWithSmallCache(retryCount=0, badWorker=0.0, 
-                         stringNo=5, stringLength=1000000, 
-                         cacheSize=3000000)
+        self._testJobFileStore(retryCount=0, badWorker=0.0,  stringNo=5, stringLength=1000000)
     
-    def testJobFileStoreWithMediumCacheAndBadWorker(self):
+    def testJobFileStoreWithBadWorker(self):
         """
         Tests case that about half the files are cached and the worker is randomly
         failing.
         """
-        self.testJobFileStoreWithSmallCache(retryCount=100, badWorker=0.5, 
-                         stringNo=5, stringLength=1000000, 
-                         cacheSize=3000000)
-        
+        self._testJobFileStore(retryCount=100, badWorker=0.5,  stringNo=5, stringLength=1000000)
+
+
 def fileTestJob(job, inputFileStoreIDs, testStrings, chainLength):
     """
-    Test job exercises Job.FileStore functions
+    Test job exercises toil.fileStore.FileStore functions
     """
     outputFileStoreIds = [] #Strings passed to the next job in the chain
     
diff --git a/src/toil/test/src/retainTempDirTest.py b/src/toil/test/src/retainTempDirTest.py
--- a/src/toil/test/src/retainTempDirTest.py
+++ b/src/toil/test/src/retainTempDirTest.py
@@ -20,7 +20,7 @@
 
 class CleanWorkDirTest(ToilTest):
     """
-    Tests testing the Job.FileStore class
+    Tests testing :class:toil.fileStore.FileStore
     """
     def setUp(self):
         super(CleanWorkDirTest, self).setUp()