# Buggy code
def asyncUpdate():
    try:
        # Wait till all file writes have completed
        for i in xrange(len(self.workers)):
            self.queue.put(None)

        for thread in self.workers:
            thread.join()

        # Wait till input block-fn returns - in the event of an exception
        # this will eventually terminate
        self.inputBlockFn()

        # Check the terminate event, if set we can not guarantee
        # that the workers ended correctly, therefore we exit without
        # completing the update
        if self.terminateEvent.isSet():
            raise RuntimeError("The termination flag is set, exiting before update")

        # Indicate any files that should be deleted once the update of
        # the job wrapper is completed.
        self.jobWrapper.filesToDelete = len(self.filesToDelete)

        # Complete the job
        self.jobStore.update(self.jobWrapper)

        # Delete any remnant jobs
        map(self.jobStore.delete, self.jobsToDelete)

        # Delete any remnant files
        map(self.jobStore.deleteFile, self.filesToDelete)

        # Remove the files to delete list, having successfully removed the files
        if len(self.filesToDelete) > 0:
            self.jobWrapper.filesToDelete = []
            # Update, removing emptying files to delete
            self.jobStore.update(self.jobWrapper)
    except:
        self.terminateEvent.set()
        raise
    finally:
        # Indicate that _blockFn can return
        # This code will always run
        self.updateSemaphore.release()


# Traceback (RuntimeError)
[centos@arjun-fake-podk toil_tests]$ python module_tests.py
No handlers could be found for logger "toil.resource"
INFO:toil.lib.bioio:Logging set at level: DEBUG
INFO:toil.lib.bioio:Logging to file: /data1/star_log
INFO:toil.common:Using the single machine batch system
INFO:toil.jobStores.fileJobStore:Jobstore directory is: /data1/toil_tempdir
WARNING:toil.batchSystems.singleMachine:Limiting maxCores to CPU count of system (8).
INFO:toil.batchSystems.singleMachine:Setting up the thread pool with 80 workers, given a minimum CPU fraction of 0.100000 and a maximum CPU value of 8.
INFO:toil.common:Written the environment for the jobs to the environment file
WARNING:toil.resource:Can't globalize module ModuleDescriptor(dirPath='/usr/local/lib/python2.7/site-packages', name='toil.job', extension='.pyc').
WARNING:toil.resource:Can't globalize module ModuleDescriptor(dirPath='/usr/local/lib/python2.7/site-packages', name='toil.job', extension='.pyc').
WARNING:toil.resource:Can't globalize module ModuleDescriptor(dirPath='/usr/local/lib/python2.7/site-packages', name='toil.job', extension='.pyc').
INFO:toil.leader:Checked batch system has no running jobs and no updated jobs
INFO:toil.leader:Found 1 jobs to start and 0 jobs with successors to run
INFO:toil.leader:Starting the main loop
DEBUG:toil.leader:Built the jobs list, currently have 1 jobs to update and 0 jobs issued
DEBUG:toil.batchSystems.singleMachine:Issuing the command: /usr/local/bin/python -E /usr/local/lib/python2.7/site-packages/toil/worker.py /data1/toil_tempdir e/j/jobaSjV4A with memory: 2147483648, cores: 1, disk: 2147483648
DEBUG:toil.leader:Issued job with job store ID: e/j/jobaSjV4A and job batch system ID: 0 and cores: 1, disk: 2147483648, and memory: 2147483648
DEBUG:toil.batchSystems.singleMachine:Acquiring 2147483648 bytes of memory from a pool of 9223372036854775807.
DEBUG:toil.batchSystems.singleMachine:Acquiring 10 fractional cores from a pool of 80 to satisfy a request of 1.000000 cores
INFO:toil.batchSystems.singleMachine:Executing command: '/usr/local/bin/python -E /usr/local/lib/python2.7/site-packages/toil/worker.py /data1/toil_tempdir e/j/jobaSjV4A'.
Exception in thread Thread-13:
Traceback (most recent call last):
File "/usr/local/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/local/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 398, in asyncWrite
raise RuntimeError("The termination flag is set, exiting")
RuntimeError: The termination flag is set, exiting

Exception in thread Thread-14:
Traceback (most recent call last):
File "/usr/local/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/local/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 398, in asyncWrite
raise RuntimeError("The termination flag is set, exiting")
RuntimeError: The termination flag is set, exiting

Exception RuntimeError: RuntimeError('cannot join current thread',) in <bound method FileStore.__del__ of <toil.job.FileStore object at 0x7f5142899f90>> ignored

# Fixed code
def asyncUpdate():
    try:
        # Wait till all file writes have completed
        for i in xrange(len(self.workers)):
            self.queue.put(None)

        for thread in self.workers:
            thread.join()

        # Wait till input block-fn returns - in the event of an exception
        # this will eventually terminate
        self.inputBlockFn()

        # Check the terminate event, if set we can not guarantee
        # that the workers ended correctly, therefore we exit without
        # completing the update
        if self.terminateEvent.isSet():
            raise RuntimeError("The termination flag is set, exiting before update")

        # Indicate any files that should be deleted once the update of
        # the job wrapper is completed.
        self.jobWrapper.filesToDelete = list(self.filesToDelete)

        # Complete the job
        self.jobStore.update(self.jobWrapper)

        # Delete any remnant jobs
        map(self.jobStore.delete, self.jobsToDelete)

        # Delete any remnant files
        map(self.jobStore.deleteFile, self.filesToDelete)

        # Remove the files to delete list, having successfully removed the files
        if len(self.filesToDelete) > 0:
            self.jobWrapper.filesToDelete = []
            # Update, removing emptying files to delete
            self.jobStore.update(self.jobWrapper)
    except:
        self.terminateEvent.set()
        raise
    finally:
        # Indicate that _blockFn can return
        # This code will always run
        self.updateSemaphore.release()