# Buggy code
def _updateJobWhenDone(self):
    """
    Asynchronously update the status of the job on the disk, first waiting
    until the writing threads have finished and the inputBlockFn has stopped
    blocking.
    """

    def asyncUpdate():
        try:
            # Wait till all file writes have completed
            for i in xrange(len(self.workers)):
                self.queue.put(None)

            for thread in self.workers:
                thread.join()

            # Wait till input block-fn returns - in the event of an exception
            # this will eventually terminate
            self.inputBlockFn()

            # Check the terminate event, if set we can not guarantee
            # that the workers ended correctly, therefore we exit without
            # completing the update
            if self.terminateEvent.isSet():
                raise RuntimeError("The termination flag is set, exiting before update")

            # Indicate any files that should be deleted once the update of
            # the job wrapper is completed.
            self.jobWrapper.filesToDelete = len(self.filesToDelete)

            # Complete the job
            self.jobStore.update(self.jobWrapper)

            # Delete any remnant jobs
            map(self.jobStore.delete, self.jobsToDelete)

            # Delete any remnant files
            map(self.jobStore.deleteFile, self.filesToDelete)

            # Remove the files to delete list, having successfully removed the files
            if len(self.filesToDelete) > 0:
                self.jobWrapper.filesToDelete = []
                # Update, removing emptying files to delete
                self.jobStore.update(self.jobWrapper)
        except:
            self.terminateEvent.set()
            raise
        finally:
            # Indicate that _blockFn can return
            # This code will always run
            self.updateSemaphore.release()

    # The update semaphore is held while the jobWrapper is written to disk
    try:
        self.updateSemaphore.acquire()
        t = Thread(target=asyncUpdate)
        t.start()
    except:  # This is to ensure that the semaphore is released in a crash to stop a deadlock scenario
        self.updateSemaphore.release()
        raise


# Traceback (RuntimeError)
[centos@arjun-fake-podk toil_tests]$ python module_tests.py
No handlers could be found for logger "toil.resource"
INFO:toil.lib.bioio:Logging set at level: DEBUG
INFO:toil.lib.bioio:Logging to file: /data1/star_log
INFO:toil.common:Using the single machine batch system
INFO:toil.jobStores.fileJobStore:Jobstore directory is: /data1/toil_tempdir
WARNING:toil.batchSystems.singleMachine:Limiting maxCores to CPU count of system (8).
INFO:toil.batchSystems.singleMachine:Setting up the thread pool with 80 workers, given a minimum CPU fraction of 0.100000 and a maximum CPU value of 8.
INFO:toil.common:Written the environment for the jobs to the environment file
WARNING:toil.resource:Can't globalize module ModuleDescriptor(dirPath='/usr/local/lib/python2.7/site-packages', name='toil.job', extension='.pyc').
WARNING:toil.resource:Can't globalize module ModuleDescriptor(dirPath='/usr/local/lib/python2.7/site-packages', name='toil.job', extension='.pyc').
WARNING:toil.resource:Can't globalize module ModuleDescriptor(dirPath='/usr/local/lib/python2.7/site-packages', name='toil.job', extension='.pyc').
INFO:toil.leader:Checked batch system has no running jobs and no updated jobs
INFO:toil.leader:Found 1 jobs to start and 0 jobs with successors to run
INFO:toil.leader:Starting the main loop
DEBUG:toil.leader:Built the jobs list, currently have 1 jobs to update and 0 jobs issued
DEBUG:toil.batchSystems.singleMachine:Issuing the command: /usr/local/bin/python -E /usr/local/lib/python2.7/site-packages/toil/worker.py /data1/toil_tempdir e/j/jobaSjV4A with memory: 2147483648, cores: 1, disk: 2147483648
DEBUG:toil.leader:Issued job with job store ID: e/j/jobaSjV4A and job batch system ID: 0 and cores: 1, disk: 2147483648, and memory: 2147483648
DEBUG:toil.batchSystems.singleMachine:Acquiring 2147483648 bytes of memory from a pool of 9223372036854775807.
DEBUG:toil.batchSystems.singleMachine:Acquiring 10 fractional cores from a pool of 80 to satisfy a request of 1.000000 cores
INFO:toil.batchSystems.singleMachine:Executing command: '/usr/local/bin/python -E /usr/local/lib/python2.7/site-packages/toil/worker.py /data1/toil_tempdir e/j/jobaSjV4A'.
Exception in thread Thread-13:
Traceback (most recent call last):
File "/usr/local/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/local/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 398, in asyncWrite
raise RuntimeError("The termination flag is set, exiting")
RuntimeError: The termination flag is set, exiting

Exception in thread Thread-14:
Traceback (most recent call last):
File "/usr/local/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/local/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 398, in asyncWrite
raise RuntimeError("The termination flag is set, exiting")
RuntimeError: The termination flag is set, exiting

Exception RuntimeError: RuntimeError('cannot join current thread',) in <bound method FileStore.__del__ of <toil.job.FileStore object at 0x7f5142899f90>> ignored

# Fixed code
def _updateJobWhenDone(self):
    """
    Asynchronously update the status of the job on the disk, first waiting
    until the writing threads have finished and the inputBlockFn has stopped
    blocking.
    """

    def asyncUpdate():
        try:
            # Wait till all file writes have completed
            for i in xrange(len(self.workers)):
                self.queue.put(None)

            for thread in self.workers:
                thread.join()

            # Wait till input block-fn returns - in the event of an exception
            # this will eventually terminate
            self.inputBlockFn()

            # Check the terminate event, if set we can not guarantee
            # that the workers ended correctly, therefore we exit without
            # completing the update
            if self.terminateEvent.isSet():
                raise RuntimeError("The termination flag is set, exiting before update")

            # Indicate any files that should be deleted once the update of
            # the job wrapper is completed.
            self.jobWrapper.filesToDelete = list(self.filesToDelete)

            # Complete the job
            self.jobStore.update(self.jobWrapper)

            # Delete any remnant jobs
            map(self.jobStore.delete, self.jobsToDelete)

            # Delete any remnant files
            map(self.jobStore.deleteFile, self.filesToDelete)

            # Remove the files to delete list, having successfully removed the files
            if len(self.filesToDelete) > 0:
                self.jobWrapper.filesToDelete = []
                # Update, removing emptying files to delete
                self.jobStore.update(self.jobWrapper)
        except:
            self.terminateEvent.set()
            raise
        finally:
            # Indicate that _blockFn can return
            # This code will always run
            self.updateSemaphore.release()

    # The update semaphore is held while the jobWrapper is written to disk
    try:
        self.updateSemaphore.acquire()
        t = Thread(target=asyncUpdate)
        t.start()
    except:  # This is to ensure that the semaphore is released in a crash to stop a deadlock scenario
        self.updateSemaphore.release()
        raise