# Buggy code
def _cleanLocalTempDir(self, cacheSize):
    """
    At the end of the job, remove all localTempDir files except those whose value is in
    jobStoreFileIDToCacheLocation.

    The param cacheSize is the total number of bytes of files allowed in the cache.
    """
    # Remove files so that the total cached files are smaller than a cacheSize

    # List of pairs of (fileSize, fileStoreID) for cached files
    cachedFileSizes = map(
        lambda x: (os.stat(self.jobStoreFileIDToCacheLocation[x]).st_size, x),
        self.jobStoreFileIDToCacheLocation.keys(),
    )
    # Total number of bytes stored in cached files
    totalCachedFileSizes = sum(map(lambda x: x[0], cachedFileSizes))
    # Remove smallest files first - this is not obviously best, could do it a different
    # way
    cachedFileSizes.sort()
    cachedFileSizes.reverse()
    # Now do the actual file removal
    while totalCachedFileSizes > cacheSize:
        fileSize, fileStoreID = cachedFileSizes.pop()
        filePath = self.jobStoreFileIDToCacheLocation[fileStoreID]
        self.jobStoreFileIDToCacheLocation.pop(fileStoreID)
        os.remove(filePath)
        totalCachedFileSizes -= fileSize
        assert totalCachedFileSizes >= 0

    # Iterate from the base of localTempDir and remove all
    # files/empty directories, recursively
    cachedFiles = set(self.jobStoreFileIDToCacheLocation.values())

    def clean(dirOrFile):
        canRemove = True
        if os.path.isdir(dirOrFile):
            for f in os.listdir(dirOrFile):
                canRemove = canRemove and clean(os.path.join(dirOrFile, f))
            if canRemove:
                os.rmdir(dirOrFile)  # Dir should be empty if canRemove is true
            return canRemove
        if dirOrFile in cachedFiles:
            return False
        os.remove(dirOrFile)
        return True

    clean(self.localTempDir)


# Traceback (RuntimeError)
[centos@arjun-fake-podk toil_tests]$ python module_tests.py
No handlers could be found for logger "toil.resource"
INFO:toil.lib.bioio:Logging set at level: DEBUG
INFO:toil.lib.bioio:Logging to file: /data1/star_log
INFO:toil.common:Using the single machine batch system
INFO:toil.jobStores.fileJobStore:Jobstore directory is: /data1/toil_tempdir
WARNING:toil.batchSystems.singleMachine:Limiting maxCores to CPU count of system (8).
INFO:toil.batchSystems.singleMachine:Setting up the thread pool with 80 workers, given a minimum CPU fraction of 0.100000 and a maximum CPU value of 8.
INFO:toil.common:Written the environment for the jobs to the environment file
WARNING:toil.resource:Can't globalize module ModuleDescriptor(dirPath='/usr/local/lib/python2.7/site-packages', name='toil.job', extension='.pyc').
WARNING:toil.resource:Can't globalize module ModuleDescriptor(dirPath='/usr/local/lib/python2.7/site-packages', name='toil.job', extension='.pyc').
WARNING:toil.resource:Can't globalize module ModuleDescriptor(dirPath='/usr/local/lib/python2.7/site-packages', name='toil.job', extension='.pyc').
INFO:toil.leader:Checked batch system has no running jobs and no updated jobs
INFO:toil.leader:Found 1 jobs to start and 0 jobs with successors to run
INFO:toil.leader:Starting the main loop
DEBUG:toil.leader:Built the jobs list, currently have 1 jobs to update and 0 jobs issued
DEBUG:toil.batchSystems.singleMachine:Issuing the command: /usr/local/bin/python -E /usr/local/lib/python2.7/site-packages/toil/worker.py /data1/toil_tempdir e/j/jobaSjV4A with memory: 2147483648, cores: 1, disk: 2147483648
DEBUG:toil.leader:Issued job with job store ID: e/j/jobaSjV4A and job batch system ID: 0 and cores: 1, disk: 2147483648, and memory: 2147483648
DEBUG:toil.batchSystems.singleMachine:Acquiring 2147483648 bytes of memory from a pool of 9223372036854775807.
DEBUG:toil.batchSystems.singleMachine:Acquiring 10 fractional cores from a pool of 80 to satisfy a request of 1.000000 cores
INFO:toil.batchSystems.singleMachine:Executing command: '/usr/local/bin/python -E /usr/local/lib/python2.7/site-packages/toil/worker.py /data1/toil_tempdir e/j/jobaSjV4A'.
Exception in thread Thread-13:
Traceback (most recent call last):
File "/usr/local/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/local/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 398, in asyncWrite
raise RuntimeError("The termination flag is set, exiting")
RuntimeError: The termination flag is set, exiting

Exception in thread Thread-14:
Traceback (most recent call last):
File "/usr/local/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/local/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 398, in asyncWrite
raise RuntimeError("The termination flag is set, exiting")
RuntimeError: The termination flag is set, exiting

Exception RuntimeError: RuntimeError('cannot join current thread',) in <bound method FileStore.__del__ of <toil.job.FileStore object at 0x7f5142899f90>> ignored

# Fixed code
def _cleanLocalTempDir(self, cacheSize):
    """
    At the end of the job, remove all localTempDir files except those whose value is in
    jobStoreFileIDToCacheLocation.

    :param cacheSize: the total number of bytes of files allowed in the cache.
    """
    # Remove files so that the total cached files are smaller than a cacheSize

    # List of pairs of (fileSize, fileStoreID) for cached files
    cachedFileSizes = map(
        lambda x: (os.stat(self.jobStoreFileIDToCacheLocation[x]).st_size, x),
        self.jobStoreFileIDToCacheLocation.keys(),
    )
    # Total number of bytes stored in cached files
    totalCachedFileSizes = sum(map(lambda x: x[0], cachedFileSizes))
    # Remove smallest files first - this is not obviously best, could do it a different
    # way
    cachedFileSizes.sort()
    cachedFileSizes.reverse()
    # Now do the actual file removal
    while totalCachedFileSizes > cacheSize:
        fileSize, fileStoreID = cachedFileSizes.pop()
        filePath = self.jobStoreFileIDToCacheLocation[fileStoreID]
        self.jobStoreFileIDToCacheLocation.pop(fileStoreID)
        os.remove(filePath)
        totalCachedFileSizes -= fileSize
        assert totalCachedFileSizes >= 0

    # Iterate from the base of localTempDir and remove all
    # files/empty directories, recursively
    cachedFiles = set(self.jobStoreFileIDToCacheLocation.values())

    def clean(dirOrFile):
        canRemove = True
        if os.path.isdir(dirOrFile):
            for f in os.listdir(dirOrFile):
                canRemove = canRemove and clean(os.path.join(dirOrFile, f))
            if canRemove:
                os.rmdir(dirOrFile)  # Dir should be empty if canRemove is true
            return canRemove
        if dirOrFile in cachedFiles:
            return False
        os.remove(dirOrFile)
        return True

    clean(self.localTempDir)