# Buggy code
def clean(self, rootJobWrapper):
    """
    Function to cleanup the state of a jobStore after a restart.
    Fixes jobs that might have been partially updated.
    Resets the try counts.
    Removes jobs that are not successors of the rootJobWrapper.
    """
    # Iterate from the root jobWrapper and collate all jobs that are reachable from it
    # All other jobs returned by self.jobs() are orphaned and can be removed
    reachableFromRoot = set()

    def getConnectedJobs(jobWrapper):
        if jobWrapper.jobStoreID in reachableFromRoot:
            return
        reachableFromRoot.add(jobWrapper.jobStoreID)
        for jobs in jobWrapper.stack:
            for successorJobStoreID in map(lambda x: x[0], jobs):
                if successorJobStoreID not in reachableFromRoot and self.exists(
                    successorJobStoreID
                ):
                    getConnectedJobs(self.load(successorJobStoreID))

    getConnectedJobs(rootJobWrapper)

    # Cleanup the state of each jobWrapper
    for jobWrapper in self.jobs():
        changed = False  # Flag to indicate if we need to update the jobWrapper
        # on disk

        if len(jobWrapper.filesToDelete) != 0:
            # Delete any files that should already be deleted
            for fileID in jobWrapper.filesToDelete:
                logger.critical(
                    "Removing file in job store: %s that was marked for deletion but not previously removed"
                    % fileID
                )
                self.deleteFile(fileID)
            jobWrapper.filesToDelete = set()
            changed = True

        # Delete a jobWrapper if it is not reachable from the rootJobWrapper
        if jobWrapper.jobStoreID not in reachableFromRoot:
            logger.critical(
                "Removing job: %s that is not a successor of the root job in cleanup"
                % jobWrapper.jobStoreID
            )
            self.delete(jobWrapper.jobStoreID)
            continue

        # While jobs at the end of the stack are already deleted remove
        # those jobs from the stack (this cleans up the case that the jobWrapper
        # had successors to run, but had not been updated to reflect this)
        while len(jobWrapper.stack) > 0:
            jobs = [
                command for command in jobWrapper.stack[-1] if self.exists(command[0])
            ]
            if len(jobs) < len(jobWrapper.stack[-1]):
                changed = True
                if len(jobs) > 0:
                    jobWrapper.stack[-1] = jobs
                    break
                else:
                    jobWrapper.stack.pop()
            else:
                break

        # Reset the retry count of the jobWrapper
        if jobWrapper.remainingRetryCount != self._defaultTryCount():
            jobWrapper.remainingRetryCount = self._defaultTryCount()
            changed = True

        # This cleans the old log file which may
        # have been left if the jobWrapper is being retried after a jobWrapper failure.
        if jobWrapper.logJobStoreFileID != None:
            self.delete(jobWrapper.logJobStoreFileID)
            jobWrapper.logJobStoreFileID = None
            changed = True

        if changed:  # Update, but only if a change has occurred
            self.update(jobWrapper)

    # Remove any crufty stats/logging files from the previous run
    self.readStatsAndLogging(lambda x: None)


# Traceback (RuntimeError)
[centos@arjun-fake-podk toil_tests]$ python module_tests.py
No handlers could be found for logger "toil.resource"
INFO:toil.lib.bioio:Logging set at level: DEBUG
INFO:toil.lib.bioio:Logging to file: /data1/star_log
INFO:toil.common:Using the single machine batch system
INFO:toil.jobStores.fileJobStore:Jobstore directory is: /data1/toil_tempdir
WARNING:toil.batchSystems.singleMachine:Limiting maxCores to CPU count of system (8).
INFO:toil.batchSystems.singleMachine:Setting up the thread pool with 80 workers, given a minimum CPU fraction of 0.100000 and a maximum CPU value of 8.
INFO:toil.common:Written the environment for the jobs to the environment file
WARNING:toil.resource:Can't globalize module ModuleDescriptor(dirPath='/usr/local/lib/python2.7/site-packages', name='toil.job', extension='.pyc').
WARNING:toil.resource:Can't globalize module ModuleDescriptor(dirPath='/usr/local/lib/python2.7/site-packages', name='toil.job', extension='.pyc').
WARNING:toil.resource:Can't globalize module ModuleDescriptor(dirPath='/usr/local/lib/python2.7/site-packages', name='toil.job', extension='.pyc').
INFO:toil.leader:Checked batch system has no running jobs and no updated jobs
INFO:toil.leader:Found 1 jobs to start and 0 jobs with successors to run
INFO:toil.leader:Starting the main loop
DEBUG:toil.leader:Built the jobs list, currently have 1 jobs to update and 0 jobs issued
DEBUG:toil.batchSystems.singleMachine:Issuing the command: /usr/local/bin/python -E /usr/local/lib/python2.7/site-packages/toil/worker.py /data1/toil_tempdir e/j/jobaSjV4A with memory: 2147483648, cores: 1, disk: 2147483648
DEBUG:toil.leader:Issued job with job store ID: e/j/jobaSjV4A and job batch system ID: 0 and cores: 1, disk: 2147483648, and memory: 2147483648
DEBUG:toil.batchSystems.singleMachine:Acquiring 2147483648 bytes of memory from a pool of 9223372036854775807.
DEBUG:toil.batchSystems.singleMachine:Acquiring 10 fractional cores from a pool of 80 to satisfy a request of 1.000000 cores
INFO:toil.batchSystems.singleMachine:Executing command: '/usr/local/bin/python -E /usr/local/lib/python2.7/site-packages/toil/worker.py /data1/toil_tempdir e/j/jobaSjV4A'.
Exception in thread Thread-13:
Traceback (most recent call last):
File "/usr/local/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/local/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 398, in asyncWrite
raise RuntimeError("The termination flag is set, exiting")
RuntimeError: The termination flag is set, exiting

Exception in thread Thread-14:
Traceback (most recent call last):
File "/usr/local/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/local/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 398, in asyncWrite
raise RuntimeError("The termination flag is set, exiting")
RuntimeError: The termination flag is set, exiting

Exception RuntimeError: RuntimeError('cannot join current thread',) in <bound method FileStore.__del__ of <toil.job.FileStore object at 0x7f5142899f90>> ignored

# Fixed code
def clean(self, rootJobWrapper):
    """
    Function to cleanup the state of a jobStore after a restart.
    Fixes jobs that might have been partially updated.
    Resets the try counts.
    Removes jobs that are not successors of the rootJobWrapper.
    """
    # Iterate from the root jobWrapper and collate all jobs that are reachable from it
    # All other jobs returned by self.jobs() are orphaned and can be removed
    reachableFromRoot = set()

    def getConnectedJobs(jobWrapper):
        if jobWrapper.jobStoreID in reachableFromRoot:
            return
        reachableFromRoot.add(jobWrapper.jobStoreID)
        for jobs in jobWrapper.stack:
            for successorJobStoreID in map(lambda x: x[0], jobs):
                if successorJobStoreID not in reachableFromRoot and self.exists(
                    successorJobStoreID
                ):
                    getConnectedJobs(self.load(successorJobStoreID))

    getConnectedJobs(rootJobWrapper)

    # Cleanup the state of each jobWrapper
    for jobWrapper in self.jobs():
        changed = False  # Flag to indicate if we need to update the jobWrapper
        # on disk

        if len(jobWrapper.filesToDelete) != 0:
            # Delete any files that should already be deleted
            for fileID in jobWrapper.filesToDelete:
                logger.critical(
                    "Removing file in job store: %s that was marked for deletion but not previously removed"
                    % fileID
                )
                self.deleteFile(fileID)
            jobWrapper.filesToDelete = []
            changed = True

        # Delete a jobWrapper if it is not reachable from the rootJobWrapper
        if jobWrapper.jobStoreID not in reachableFromRoot:
            logger.critical(
                "Removing job: %s that is not a successor of the root job in cleanup"
                % jobWrapper.jobStoreID
            )
            self.delete(jobWrapper.jobStoreID)
            continue

        # While jobs at the end of the stack are already deleted remove
        # those jobs from the stack (this cleans up the case that the jobWrapper
        # had successors to run, but had not been updated to reflect this)
        while len(jobWrapper.stack) > 0:
            jobs = [
                command for command in jobWrapper.stack[-1] if self.exists(command[0])
            ]
            if len(jobs) < len(jobWrapper.stack[-1]):
                changed = True
                if len(jobs) > 0:
                    jobWrapper.stack[-1] = jobs
                    break
                else:
                    jobWrapper.stack.pop()
            else:
                break

        # Reset the retry count of the jobWrapper
        if jobWrapper.remainingRetryCount != self._defaultTryCount():
            jobWrapper.remainingRetryCount = self._defaultTryCount()
            changed = True

        # This cleans the old log file which may
        # have been left if the jobWrapper is being retried after a jobWrapper failure.
        if jobWrapper.logJobStoreFileID != None:
            self.delete(jobWrapper.logJobStoreFileID)
            jobWrapper.logJobStoreFileID = None
            changed = True

        if changed:  # Update, but only if a change has occurred
            self.update(jobWrapper)

    # Remove any crufty stats/logging files from the previous run
    self.readStatsAndLogging(lambda x: None)