Repository: DataBiosphere/toil

Issue:
Make caching optional

writeGlobalFile probably needs a block if the input is a not-local file and won't be put into cache
I have a test pipeline as follows

```
F1 = Job.wrapJobFn(aux.write_to_filestore, self.fastqs)
F2 = Job.wrapJobFn(aux.write_to_filestore, self.star_options)
G = Job.wrapJobFn(PI.get_files_from_filestore, F2.rv(), self.test_dir)
F1.addChild(F2)
F2.addChild(G)
Job.Runner.startToil(F1, params)
```

F1 writes two 4.5G fastqs to the jobstore
F2 writes a 24G tar.gz file to the jobstore
G in this example will attempt to retrieve the index.tar.gz from jobstore and untar it.

aux.write_to_filestore is

```
def write_to_filestore(job, files):
    '''
    This will write a dict of files to the write_to_filestore.
    The input FILES object is expected to be a dict of
    key = filename or sample_type
    value = filepath if key == filename
            list of filepaths if key == sample_type
    Eg. {'SRR123.fastq': '/data/SRR123.fastq',
         'sample1': ['/data/s_1.fastq', '/data/s_2.fastq']}

    FILES will be updated to
    key = filename or sample_type
    value = jobStoreFileID if key == filename
            list of jobStoreFileIDs if key == sample_type
    '''
    for name in files.keys():
        #  if list, replace the items with the filestoreID for each item 
        #  obtained from writeGlobalFile
        if isinstance(files[name], list):
            files[name] = [job.fileStore.writeGlobalFile(x) for x in files[name]]
        else:
            files[name] = job.fileStore.writeGlobalFile(files[name])
    return files
```

PI.get_files_from_filestore is

```
def get_files_from_filestore(job, files, work_dir, docker=False):
    """
    This is adapted from John Vivian's return_input_paths from the RNA-Seq
    pipeline.

    Returns the paths of files from the FileStore if they are not present.
    if docker=True, return the docker path for the file
    if the file extension is tar.gz, then tar -zxvf it
    files is a dict with
        keys = the name of the file to be returned in toil space
        value = the input value for the file (can be toil temp file)
    """
    for name in files.keys():
        files[name] = job.fileStore.readGlobalFile(files[name],
                                                   '/'.join([work_dir, name]))
        #  If the file pointed to a tarball, extract it to WORK_DIR
        if tarfile.is_tarfile(files[name]):
            files[name] = untargz(files[name], work_dir)
        #  If the files will be sent to docker, we will mount work_dir to the 
        #  container as /data and we want the /data prefixed path to the file
        if docker == True:
            files[name] = docker_path(files[name])
    return files

def untargz(input_targz_file, untar_to_dir):
    '''
    This module accepts a tar.gz archive and untars it.

    RETURN VALUE: path to the untar-ed directory/file

    NOTE: this module expects the multiple files to be in a directory before
          being tar-ed.
    '''
    assert tarfile.is_tarfile(input_targz_file), 'Not a tar file.'
    tarball = tarfile.open(input_targz_file)
    return_value = os.path.join(untar_to_dir, tarball.getmembers()[0].name)
    tarball.extractall(path=untar_to_dir)
    tarball.close()
    return return_value
```

If i comment out the `F2.addChild(G)` line then the data is successfully written to the filestore and the program exits normally.
If i don't, I get this Error almost immediately

```
[centos@arjun-fake-podk toil_tests]$ python module_tests.py
No handlers could be found for logger "toil.resource"
INFO:toil.lib.bioio:Logging set at level: DEBUG
INFO:toil.lib.bioio:Logging to file: /data1/star_log
INFO:toil.common:Using the single machine batch system
INFO:toil.jobStores.fileJobStore:Jobstore directory is: /data1/toil_tempdir
WARNING:toil.batchSystems.singleMachine:Limiting maxCores to CPU count of system (8).
INFO:toil.batchSystems.singleMachine:Setting up the thread pool with 80 workers, given a minimum CPU fraction of 0.100000 and a maximum CPU value of 8.
INFO:toil.common:Written the environment for the jobs to the environment file
WARNING:toil.resource:Can't globalize module ModuleDescriptor(dirPath='/usr/local/lib/python2.7/site-packages', name='toil.job', extension='.pyc').
WARNING:toil.resource:Can't globalize module ModuleDescriptor(dirPath='/usr/local/lib/python2.7/site-packages', name='toil.job', extension='.pyc').
WARNING:toil.resource:Can't globalize module ModuleDescriptor(dirPath='/usr/local/lib/python2.7/site-packages', name='toil.job', extension='.pyc').
INFO:toil.leader:Checked batch system has no running jobs and no updated jobs
INFO:toil.leader:Found 1 jobs to start and 0 jobs with successors to run
INFO:toil.leader:Starting the main loop
DEBUG:toil.leader:Built the jobs list, currently have 1 jobs to update and 0 jobs issued
DEBUG:toil.batchSystems.singleMachine:Issuing the command: /usr/local/bin/python -E /usr/local/lib/python2.7/site-packages/toil/worker.py /data1/toil_tempdir e/j/jobaSjV4A with memory: 2147483648, cores: 1, disk: 2147483648
DEBUG:toil.leader:Issued job with job store ID: e/j/jobaSjV4A and job batch system ID: 0 and cores: 1, disk: 2147483648, and memory: 2147483648
DEBUG:toil.batchSystems.singleMachine:Acquiring 2147483648 bytes of memory from a pool of 9223372036854775807.
DEBUG:toil.batchSystems.singleMachine:Acquiring 10 fractional cores from a pool of 80 to satisfy a request of 1.000000 cores
INFO:toil.batchSystems.singleMachine:Executing command: '/usr/local/bin/python -E /usr/local/lib/python2.7/site-packages/toil/worker.py /data1/toil_tempdir e/j/jobaSjV4A'.
Exception in thread Thread-13:
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/threading.py", line 810, in __bootstrap_inner
    self.run()
  File "/usr/local/lib/python2.7/threading.py", line 763, in run
    self.__target(*self.__args, **self.__kwargs)
  File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 398, in asyncWrite
    raise RuntimeError("The termination flag is set, exiting")
RuntimeError: The termination flag is set, exiting

Exception in thread Thread-14:
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/threading.py", line 810, in __bootstrap_inner
    self.run()
  File "/usr/local/lib/python2.7/threading.py", line 763, in run
    self.__target(*self.__args, **self.__kwargs)
  File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 398, in asyncWrite
    raise RuntimeError("The termination flag is set, exiting")
RuntimeError: The termination flag is set, exiting

Exception RuntimeError: RuntimeError('cannot join current thread',) in <bound method FileStore.__del__ of <toil.job.FileStore object at 0x7f5142899f90>> ignored
```

I assume that a block on the write will prevent this from happening. I'm going to place a 10 minute sleep in F2 to allow the write to complete and rerun this. I'm putting this up for now anyway.



Discussion:
What commit resolves this?

None, but I don't think it's worth putting in. It adds complexity for really no point.

Oops, yes. Bugger - this is what happens if you try to do ten things at
once. I agree, the simple thing is to just block for files in the temporary
store.

On Fri, Oct 9, 2015 at 3:46 PM, Arjun Arkal Rao notifications@github.com
wrote:

> I have a test pipeline as follows
> 
> F1 = Job.wrapJobFn(aux.write_to_filestore, self.fastqs)
> F2 = Job.wrapJobFn(aux.write_to_filestore, self.star_options)
> G = Job.wrapJobFn(PI.get_files_from_filestore, F2.rv(), self.test_dir)
> F1.addChild(F2)
> F2.addChild(G)
> Job.Runner.startToil(F1, params)
> 
> aux.write_to_filestore is
> 
> def write_to_filestore(job, files):
>     '''
>     This will write a dict of files to the write_to_filestore.
>     The input FILES object is expected to be a dict of
>     key = filename or sample_type
>     value = filepath if key == filename
>             list of filepaths if key == sample_type
>     Eg. {'SRR123.fastq': '/data/SRR123.fastq',
>          'sample1': ['/data/s_1.fastq', '/data/s_2.fastq']}
> 
> ```
> FILES will be updated to
> key = filename or sample_type
> value = jobStoreFileID if key == filename
>         list of jobStoreFileIDs if key == sample_type
> '''
> for name in files.keys():
>     #  if list, replace the items with the filestoreID for each item
>     #  obtained from writeGlobalFile
>     if isinstance(files[name], list):
>         files[name] = [job.fileStore.writeGlobalFile(x) for x in files[name]]
>     else:
>         files[name] = job.fileStore.writeGlobalFile(files[name])
> return files
> ```
> 
> PI.get_files_from_filestore is
> 
> def get_files_from_filestore(job, files, work_dir, docker=False):
>     """
>     This is adapted from John Vivian's return_input_paths from the RNA-Seq
>     pipeline.
> 
> ```
> Returns the paths of files from the FileStore if they are not present.
> if docker=True, return the docker path for the file
> if the file extension is tar.gz, then tar -zxvf it
> files is a dict with
>     keys = the name of the file to be returned in toil space
>     value = the input value for the file (can be toil temp file)
> """
> for name in files.keys():
>     files[name] = job.fileStore.readGlobalFile(files[name],
>                                                '/'.join([work_dir, name]))
>     #  If the file pointed to a tarball, extract it to WORK_DIR
>     if tarfile.is_tarfile(files[name]):
>         files[name] = untargz(files[name], work_dir)
>     #  If the files will be sent to docker, we will mount work_dir to the
>     #  container as /data and we want the /data prefixed path to the file
>     if docker == True:
>         files[name] = docker_path(files[name])
> return files
> ```
> 
> —
> Reply to this email directly or view it on GitHub
> https://github.com/BD2KGenomics/toil/issues/484.

Doesn't look like it's a blocking problem on the write. I ran it on a 1.5G tarball with a 15s sleep function

```
def beddybye(job, sleeptime):
    '''
    This will sleep for SLEEPTIME seconds. It also creates a temp working
    directory that it returns
    '''
    work_dir = job.fileStore.getLocalTempDir()
    with open('/data/star_logfile', 'w') as logfile:
        print('My workdir is ', work_dir, file=logfile)
    sleep(sleeptime)
    return work_dir
```

that returns a temp working directory. The main script is modified to

```
F2 = Job.wrapJobFn(aux.write_to_filestore, self.star_options)
S = Job.wrapJobFn(aux.beddybye, 15)
H = Job.wrapJobFn(PI.get_files_from_filestore, F2.rv(), S.rv())

F2.addChild(S)
S.addChild(H)

Job.Runner.startToil(F2, params)
```

So TL;DR - write a 1.5G bam into jobstore, wait 15s (ample time to write the entire file to jobstore), then attempt to read it back from filestore. This is the trace ( --logDebug)

```
[centos@arjun-fake-podk toil_tests]$ python module_tests.py
No handlers could be found for logger "toil.resource"
INFO:toil.lib.bioio:Logging set at level: DEBUG
INFO:toil.lib.bioio:Logging to file: /data1/star_log
INFO:toil.common:Using the single machine batch system
INFO:toil.jobStores.fileJobStore:Jobstore directory is: /data1/toil_tempdir
WARNING:toil.batchSystems.singleMachine:Limiting maxCores to CPU count of system (8).
INFO:toil.batchSystems.singleMachine:Setting up the thread pool with 80 workers, given a minimum CPU fraction of 0.100000 and a maximum CPU value of 8.
INFO:toil.common:Written the environment for the jobs to the environment file
WARNING:toil.resource:Can't globalize module ModuleDescriptor(dirPath='/usr/local/lib/python2.7/site-packages', name='toil.job', extension='.pyc').
WARNING:toil.resource:Can't globalize module ModuleDescriptor(dirPath='/usr/local/lib/python2.7/site-packages', name='toil.job', extension='.pyc').
WARNING:toil.resource:Can't globalize module ModuleDescriptor(dirPath='/usr/local/lib/python2.7/site-packages', name='toil.job', extension='.pyc').
INFO:toil.leader:Checked batch system has no running jobs and no updated jobs
INFO:toil.leader:Found 1 jobs to start and 0 jobs with successors to run
INFO:toil.leader:Starting the main loop
DEBUG:toil.leader:Built the jobs list, currently have 1 jobs to update and 0 jobs issued
DEBUG:toil.batchSystems.singleMachine:Issuing the command: /usr/local/bin/python -E /usr/local/lib/python2.7/site-packages/toil/worker.py /data1/toil_tempdir 0/c/job7pbt0o with memory: 2147483648, cores: 1, disk: 2147483648
DEBUG:toil.leader:Issued job with job store ID: 0/c/job7pbt0o and job batch system ID: 0 and cores: 1, disk: 2147483648, and memory: 2147483648
DEBUG:toil.batchSystems.singleMachine:Acquiring 2147483648 bytes of memory from a pool of 9223372036854775807.
DEBUG:toil.batchSystems.singleMachine:Acquiring 10 fractional cores from a pool of 80 to satisfy a request of 1.000000 cores
INFO:toil.batchSystems.singleMachine:Executing command: '/usr/local/bin/python -E /usr/local/lib/python2.7/site-packages/toil/worker.py /data1/toil_tempdir 0/c/job7pbt0o'.
DEBUG:toil.batchSystems.singleMachine:Finished job. self.coreFractions ~ 80 and self.memory ~ 9223372036854775807
DEBUG:toil.batchSystems.singleMachine:Ran jobID: 0 with exit value: 0
DEBUG:toil.leader:Batch system is reporting that the jobWrapper with batch system ID: 0 and jobWrapper store ID: 0/c/job7pbt0o ended successfully
WARNING:toil.leader:The jobWrapper seems to have left a log file, indicating failure: 0/c/job7pbt0o
WARNING:toil.leader:Reporting file: 0/c/job7pbt0o
WARNING:toil.leader:0/c/job7pbt0o:  ---TOIL WORKER OUTPUT LOG---
WARNING:toil.leader:0/c/job7pbt0o:  Next available file descriptor: 8
WARNING:toil.leader:0/c/job7pbt0o:  DEBUG:__main__:Next available file descriptor: 8
WARNING:toil.leader:0/c/job7pbt0o:  Parsed jobWrapper
WARNING:toil.leader:0/c/job7pbt0o:  DEBUG:__main__:Parsed jobWrapper
WARNING:toil.leader:0/c/job7pbt0o:  WARNING:toil.resource:Can't find resource for leader path '/data/test_docker/toil_tests/aux_scripts.pyc'
WARNING:toil.leader:0/c/job7pbt0o:  WARNING:toil.resource:Can't localize module ModuleDescriptor(dirPath='/data/test_docker/toil_tests', name='aux_scripts', extension='.pyc')
WARNING:toil.leader:0/c/job7pbt0o:  Starting the next jobWrapper
WARNING:toil.leader:0/c/job7pbt0o:  DEBUG:__main__:Starting the next jobWrapper
WARNING:toil.leader:0/c/job7pbt0o:  WARNING:toil.resource:Can't find resource for leader path '/data/test_docker/toil_tests/aux_scripts.pyc'
WARNING:toil.leader:0/c/job7pbt0o:  WARNING:toil.resource:Can't localize module ModuleDescriptor(dirPath='/data/test_docker/toil_tests', name='aux_scripts', extension='.pyc')
WARNING:toil.leader:0/c/job7pbt0o:  Starting the next jobWrapper
WARNING:toil.leader:0/c/job7pbt0o:  DEBUG:__main__:Starting the next jobWrapper
WARNING:toil.leader:0/c/job7pbt0o:  INFO:toil.jobStores.fileJobStore:Jobstore directory is: /data1/toil_tempdir
WARNING:toil.leader:0/c/job7pbt0o:  WARNING:toil.resource:Can't find resource for leader path '/data/test_docker/docker_scripts/precision_immuno.pyc'
WARNING:toil.leader:0/c/job7pbt0o:  WARNING:toil.resource:Can't localize module ModuleDescriptor(dirPath='/data/test_docker/docker_scripts', name='precision_immuno', extension='.pyc')
WARNING:toil.leader:0/c/job7pbt0o:  Traceback (most recent call last):
WARNING:toil.leader:0/c/job7pbt0o:    File "/usr/local/lib/python2.7/site-packages/toil/worker.py", line 284, in main
WARNING:toil.leader:0/c/job7pbt0o:      fileStore=fileStore)
WARNING:toil.leader:0/c/job7pbt0o:    File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 1104, in _execute
WARNING:toil.leader:0/c/job7pbt0o:      returnValues = self.run(fileStore)
WARNING:toil.leader:0/c/job7pbt0o:    File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 1218, in run
WARNING:toil.leader:0/c/job7pbt0o:      rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs)
WARNING:toil.leader:0/c/job7pbt0o:    File "/data/test_docker/docker_scripts/precision_immuno.py", line 1100, in get_files_from_filestore
WARNING:toil.leader:0/c/job7pbt0o:      '/'.join([work_dir, name]))
WARNING:toil.leader:0/c/job7pbt0o:    File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 528, in readGlobalFile
WARNING:toil.leader:0/c/job7pbt0o:      self.jobStore.readFile(fileStoreID, localFilePath)
WARNING:toil.leader:0/c/job7pbt0o:    File "/usr/local/lib/python2.7/site-packages/toil/jobStores/fileJobStore.py", line 167, in readFile
WARNING:toil.leader:0/c/job7pbt0o:      shutil.copyfile(self._getAbsPath(jobStoreFileID), localFilePath)
WARNING:toil.leader:0/c/job7pbt0o:    File "/usr/local/lib/python2.7/shutil.py", line 83, in copyfile
WARNING:toil.leader:0/c/job7pbt0o:      with open(dst, 'wb') as fdst:
WARNING:toil.leader:0/c/job7pbt0o:  IOError: [Errno 2] No such file or directory: '/data1/temp_dir/tmp_FPCRNmnBcY/localTempDir/t8wTgwR/index_tar'
WARNING:toil.leader:0/c/job7pbt0o:  Exiting the worker because of a failed jobWrapper on host arjun-fake-podk.novalocal
WARNING:toil.leader:0/c/job7pbt0o:  ERROR:__main__:Exiting the worker because of a failed jobWrapper on host arjun-fake-podk.novalocal
WARNING:toil.leader:0/c/job7pbt0o:  WARNING:toil.jobWrapper:Due to failure we are reducing the remaining retry count of job 0/c/job7pbt0o to 0
WARNING:toil.leader:0/c/job7pbt0o:  Exception in thread Thread-13:
WARNING:toil.leader:0/c/job7pbt0o:  Traceback (most recent call last):
WARNING:toil.leader:0/c/job7pbt0o:    File "/usr/local/lib/python2.7/threading.py", line 810, in __bootstrap_inner
WARNING:toil.leader:0/c/job7pbt0o:      self.run()
WARNING:toil.leader:0/c/job7pbt0o:    File "/usr/local/lib/python2.7/threading.py", line 763, in run
WARNING:toil.leader:0/c/job7pbt0o:      self.__target(*self.__args, **self.__kwargs)
WARNING:toil.leader:0/c/job7pbt0o:    File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 398, in asyncWrite
WARNING:toil.leader:0/c/job7pbt0o:      raise RuntimeError("The termination flag is set, exiting")
WARNING:toil.leader:0/c/job7pbt0o:  RuntimeError: The termination flag is set, exiting
WARNING:toil.leader:0/c/job7pbt0o:  
WARNING:toil.leader:0/c/job7pbt0o:  Exception in thread Thread-14:
WARNING:toil.leader:0/c/job7pbt0o:  Traceback (most recent call last):
WARNING:toil.leader:0/c/job7pbt0o:    File "/usr/local/lib/python2.7/threading.py", line 810, in __bootstrap_inner
WARNING:toil.leader:0/c/job7pbt0o:      self.run()
WARNING:toil.leader:0/c/job7pbt0o:    File "/usr/local/lib/python2.7/threading.py", line 763, in run
WARNING:toil.leader:0/c/job7pbt0o:      self.__target(*self.__args, **self.__kwargs)
WARNING:toil.leader:0/c/job7pbt0o:    File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 398, in asyncWrite
WARNING:toil.leader:0/c/job7pbt0o:      raise RuntimeError("The termination flag is set, exiting")
WARNING:toil.leader:0/c/job7pbt0o:  RuntimeError: The termination flag is set, exiting
WARNING:toil.leader:0/c/job7pbt0o:  
DEBUG:toil.leader:Added jobWrapper: 0/c/job7pbt0o to active jobs
DEBUG:toil.leader:Built the jobs list, currently have 1 jobs to update and 0 jobs issued
WARNING:toil.leader:Job: 0/c/job7pbt0o is completely failed
INFO:toil.leader:Only failed jobs and their dependents (1 total) are remaining, so exiting.
INFO:toil.leader:Finished the main loop
INFO:toil.leader:Waiting for stats and logging collator process to finish
INFO:toil.leader:Stats/logging finished collating in 0.0275518894196 seconds
DEBUG:toil.common:Shutting down batch system
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
Traceback (most recent call last):
  File "module_tests.py", line 210, in <module>
    A.test_star()
  File "module_tests.py", line 107, in test_star
    Job.Runner.startToil(F2, params)
  File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 362, in startToil
    return mainLoop(config, batchSystem, jobStore, rootJob)
  File "/usr/local/lib/python2.7/site-packages/toil/leader.py", line 505, in mainLoop
    raise FailedJobsException( config.jobStore, totalFailedJobs )
toil.leader.FailedJobsException: The job store '/data1/toil_tempdir' contains 1 failed jobs
```

I honestly don't understand the `IOError: [Errno 2] No such file or directory: '/data1/temp_dir/tmp_FPCRNmnBcY/localTempDir/t8wTgwR/index_tar'` error since the temp directory exists and star_logifle confirms it. 

I also don't understand why the thread errors are coming after this OSError since I can see from `watch tree-fh /data1/toil_tempdir` that the entire file is already in the filestore. In my previous trace, the thread errors were before the OSError.

```
[centos@arjun-fake-podk toil_tests]$ python module_tests.py
No handlers could be found for logger "toil.resource"
INFO:toil.lib.bioio:Logging set at level: DEBUG
INFO:toil.lib.bioio:Logging to file: /data1/star_log
INFO:toil.common:Using the single machine batch system
INFO:toil.jobStores.fileJobStore:Jobstore directory is: /data1/toil_tempdir
WARNING:toil.batchSystems.singleMachine:Limiting maxCores to CPU count of system (8).
INFO:toil.batchSystems.singleMachine:Setting up the thread pool with 80 workers, given a minimum CPU fraction of 0.100000 and a maximum CPU value of 8.
INFO:toil.common:Written the environment for the jobs to the environment file
WARNING:toil.resource:Can't globalize module ModuleDescriptor(dirPath='/usr/local/lib/python2.7/site-packages', name='toil.job', extension='.pyc').
WARNING:toil.resource:Can't globalize module ModuleDescriptor(dirPath='/usr/local/lib/python2.7/site-packages', name='toil.job', extension='.pyc').
WARNING:toil.resource:Can't globalize module ModuleDescriptor(dirPath='/usr/local/lib/python2.7/site-packages', name='toil.job', extension='.pyc').
INFO:toil.leader:Checked batch system has no running jobs and no updated jobs
INFO:toil.leader:Found 1 jobs to start and 0 jobs with successors to run
INFO:toil.leader:Starting the main loop
DEBUG:toil.leader:Built the jobs list, currently have 1 jobs to update and 0 jobs issued
DEBUG:toil.batchSystems.singleMachine:Issuing the command: /usr/local/bin/python -E /usr/local/lib/python2.7/site-packages/toil/worker.py /data1/toil_tempdir r/4/jobP1dGoT with memory: 2147483648, cores: 1, disk: 2147483648
DEBUG:toil.leader:Issued job with job store ID: r/4/jobP1dGoT and job batch system ID: 0 and cores: 1, disk: 2147483648, and memory: 2147483648
DEBUG:toil.batchSystems.singleMachine:Acquiring 2147483648 bytes of memory from a pool of 9223372036854775807.
DEBUG:toil.batchSystems.singleMachine:Acquiring 10 fractional cores from a pool of 80 to satisfy a request of 1.000000 cores
INFO:toil.batchSystems.singleMachine:Executing command: '/usr/local/bin/python -E /usr/local/lib/python2.7/site-packages/toil/worker.py /data1/toil_tempdir r/4/jobP1dGoT'.
Exception in thread Thread-9:
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/threading.py", line 810, in __bootstrap_inner
    self.run()
  File "/usr/local/lib/python2.7/threading.py", line 763, in run
    self.__target(*self.__args, **self.__kwargs)
  File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 600, in asyncUpdate
    raise RuntimeError("The termination flag is set, exiting before update")
RuntimeError: The termination flag is set, exiting before update

Exception in thread Thread-12:
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/threading.py", line 810, in __bootstrap_inner
    self.run()
  File "/usr/local/lib/python2.7/threading.py", line 763, in run
    self.__target(*self.__args, **self.__kwargs)
  File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 600, in asyncUpdate
    raise RuntimeError("The termination flag is set, exiting before update")
RuntimeError: The termination flag is set, exiting before update

Exception in thread Thread-13:
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/threading.py", line 810, in __bootstrap_inner
    self.run()
  File "/usr/local/lib/python2.7/threading.py", line 763, in run
    self.__target(*self.__args, **self.__kwargs)
  File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 398, in asyncWrite
    raise RuntimeError("The termination flag is set, exiting")
RuntimeError: The termination flag is set, exiting

Exception in thread Thread-14:
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/threading.py", line 810, in __bootstrap_inner
    self.run()
  File "/usr/local/lib/python2.7/threading.py", line 763, in run
    self.__target(*self.__args, **self.__kwargs)
  File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 398, in asyncWrite
    raise RuntimeError("The termination flag is set, exiting")
RuntimeError: The termination flag is set, exiting

Exception RuntimeError: RuntimeError('cannot join current thread',) in <bound method FileStore.__del__ of <toil.job.FileStore object at 0x7f9de4d2cc10>> ignored
DEBUG:toil.batchSystems.singleMachine:Finished job. self.coreFractions ~ 80 and self.memory ~ 9223372036854775807
DEBUG:toil.batchSystems.singleMachine:Ran jobID: 0 with exit value: 0
DEBUG:toil.leader:Batch system is reporting that the jobWrapper with batch system ID: 0 and jobWrapper store ID: r/4/jobP1dGoT ended successfully
WARNING:toil.leader:The jobWrapper seems to have left a log file, indicating failure: r/4/jobP1dGoT
WARNING:toil.leader:Reporting file: r/4/jobP1dGoT
WARNING:toil.leader:r/4/jobP1dGoT:  ---TOIL WORKER OUTPUT LOG---
WARNING:toil.leader:r/4/jobP1dGoT:  Next available file descriptor: 8
WARNING:toil.leader:r/4/jobP1dGoT:  DEBUG:__main__:Next available file descriptor: 8
WARNING:toil.leader:r/4/jobP1dGoT:  Parsed jobWrapper
WARNING:toil.leader:r/4/jobP1dGoT:  DEBUG:__main__:Parsed jobWrapper
WARNING:toil.leader:r/4/jobP1dGoT:  WARNING:toil.resource:Can't find resource for leader path '/data/test_docker/toil_tests/aux_scripts.pyc'
WARNING:toil.leader:r/4/jobP1dGoT:  WARNING:toil.resource:Can't localize module ModuleDescriptor(dirPath='/data/test_docker/toil_tests', name='aux_scripts', extension='.pyc')
WARNING:toil.leader:r/4/jobP1dGoT:  Starting the next jobWrapper
WARNING:toil.leader:r/4/jobP1dGoT:  DEBUG:__main__:Starting the next jobWrapper
WARNING:toil.leader:r/4/jobP1dGoT:  WARNING:toil.resource:Can't find resource for leader path '/data/test_docker/toil_tests/aux_scripts.pyc'
WARNING:toil.leader:r/4/jobP1dGoT:  WARNING:toil.resource:Can't localize module ModuleDescriptor(dirPath='/data/test_docker/toil_tests', name='aux_scripts', extension='.pyc')
WARNING:toil.leader:r/4/jobP1dGoT:  Starting the next jobWrapper
WARNING:toil.leader:r/4/jobP1dGoT:  DEBUG:__main__:Starting the next jobWrapper
WARNING:toil.leader:r/4/jobP1dGoT:  INFO:toil.jobStores.fileJobStore:Jobstore directory is: /data1/toil_tempdir
WARNING:toil.leader:r/4/jobP1dGoT:  WARNING:toil.resource:Can't find resource for leader path '/data/test_docker/docker_scripts/precision_immuno.pyc'
WARNING:toil.leader:r/4/jobP1dGoT:  WARNING:toil.resource:Can't localize module ModuleDescriptor(dirPath='/data/test_docker/docker_scripts', name='precision_immuno', extension='.pyc')
WARNING:toil.leader:r/4/jobP1dGoT:  Traceback (most recent call last):
WARNING:toil.leader:r/4/jobP1dGoT:    File "/usr/local/lib/python2.7/site-packages/toil/worker.py", line 284, in main
WARNING:toil.leader:r/4/jobP1dGoT:      fileStore=fileStore)
WARNING:toil.leader:r/4/jobP1dGoT:    File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 1104, in _execute
WARNING:toil.leader:r/4/jobP1dGoT:      returnValues = self.run(fileStore)
WARNING:toil.leader:r/4/jobP1dGoT:    File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 1218, in run
WARNING:toil.leader:r/4/jobP1dGoT:      rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs)
WARNING:toil.leader:r/4/jobP1dGoT:    File "/data/test_docker/docker_scripts/precision_immuno.py", line 1100, in get_files_from_filestore
WARNING:toil.leader:r/4/jobP1dGoT:      '/'.join([work_dir, name]))
WARNING:toil.leader:r/4/jobP1dGoT:    File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 528, in readGlobalFile
WARNING:toil.leader:r/4/jobP1dGoT:      self.jobStore.readFile(fileStoreID, localFilePath)
WARNING:toil.leader:r/4/jobP1dGoT:    File "/usr/local/lib/python2.7/site-packages/toil/jobStores/fileJobStore.py", line 167, in readFile
WARNING:toil.leader:r/4/jobP1dGoT:      shutil.copyfile(self._getAbsPath(jobStoreFileID), localFilePath)
WARNING:toil.leader:r/4/jobP1dGoT:    File "/usr/local/lib/python2.7/shutil.py", line 83, in copyfile
WARNING:toil.leader:r/4/jobP1dGoT:      with open(dst, 'wb') as fdst:
WARNING:toil.leader:r/4/jobP1dGoT:  IOError: [Errno 2] No such file or directory: '/data1/temp_dir/tmp_RMz7RGIgJx/localTempDir/tJSFLJM/index_tar'
WARNING:toil.leader:r/4/jobP1dGoT:  Exiting the worker because of a failed jobWrapper on host arjun-fake-podk.novalocal
WARNING:toil.leader:r/4/jobP1dGoT:  ERROR:__main__:Exiting the worker because of a failed jobWrapper on host arjun-fake-podk.novalocal
WARNING:toil.leader:r/4/jobP1dGoT:  WARNING:toil.jobWrapper:Due to failure we are reducing the remaining retry count of job r/4/jobP1dGoT to 0
DEBUG:toil.leader:Added jobWrapper: r/4/jobP1dGoT to active jobs
DEBUG:toil.leader:Built the jobs list, currently have 1 jobs to update and 0 jobs issued
WARNING:toil.leader:Job: r/4/jobP1dGoT is completely failed
INFO:toil.leader:Only failed jobs and their dependents (1 total) are remaining, so exiting.
INFO:toil.leader:Finished the main loop
INFO:toil.leader:Waiting for stats and logging collator process to finish
INFO:toil.leader:Stats/logging finished collating in 0.399203062057 seconds
DEBUG:toil.common:Shutting down batch system
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
Traceback (most recent call last):
  File "module_tests.py", line 210, in <module>
    A.test_star()
  File "module_tests.py", line 107, in test_star
    Job.Runner.startToil(F2, params)
  File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 362, in startToil
    return mainLoop(config, batchSystem, jobStore, rootJob)
  File "/usr/local/lib/python2.7/site-packages/toil/leader.py", line 505, in mainLoop
    raise FailedJobsException( config.jobStore, totalFailedJobs )
toil.leader.FailedJobsException: The job store '/data1/toil_tempdir' contains 1 failed jobs
```

My guess is that the OSError is being thrown because the failed threads cause the program to crash and localTempDir to get deleted before the untargz is called hence the folder 'doesn't exist'. What is weird is that the threads crash but the tarfile in the jobstore extracts without error (when i manually attempt it) so there's no data corruption. So this begs teh question, why are those threads erroring out? Increasing the wait time to 10 minutes (way wayyyyy more than required to write the file) changes nothing. There are no errors thrown till the next job starts and the error is the same.

```
[centos@arjun-fake-podk toil_tests]$ python module_tests.py
No handlers could be found for logger "toil.resource"
INFO:toil.lib.bioio:Logging set at level: DEBUG
INFO:toil.lib.bioio:Logging to file: /data1/star_log
INFO:toil.common:Using the single machine batch system
INFO:toil.jobStores.fileJobStore:Jobstore directory is: /data1/toil_tempdir
WARNING:toil.batchSystems.singleMachine:Limiting maxCores to CPU count of system (8).
INFO:toil.batchSystems.singleMachine:Setting up the thread pool with 80 workers, given a minimum CPU fraction of 0.100000 and a maximum CPU value of 8.
INFO:toil.common:Written the environment for the jobs to the environment file
WARNING:toil.resource:Can't globalize module ModuleDescriptor(dirPath='/usr/local/lib/python2.7/site-packages', name='toil.job', extension='.pyc').
WARNING:toil.resource:Can't globalize module ModuleDescriptor(dirPath='/usr/local/lib/python2.7/site-packages', name='toil.job', extension='.pyc').
WARNING:toil.resource:Can't globalize module ModuleDescriptor(dirPath='/usr/local/lib/python2.7/site-packages', name='toil.job', extension='.pyc').
INFO:toil.leader:Checked batch system has no running jobs and no updated jobs
INFO:toil.leader:Found 1 jobs to start and 0 jobs with successors to run
INFO:toil.leader:Starting the main loop
DEBUG:toil.leader:Built the jobs list, currently have 1 jobs to update and 0 jobs issued
DEBUG:toil.batchSystems.singleMachine:Issuing the command: /usr/local/bin/python -E /usr/local/lib/python2.7/site-packages/toil/worker.py /data1/toil_tempdir l/J/jobcMf1Cx with memory: 2147483648, cores: 1, disk: 2147483648
DEBUG:toil.leader:Issued job with job store ID: l/J/jobcMf1Cx and job batch system ID: 0 and cores: 1, disk: 2147483648, and memory: 2147483648
DEBUG:toil.batchSystems.singleMachine:Acquiring 2147483648 bytes of memory from a pool of 9223372036854775807.
DEBUG:toil.batchSystems.singleMachine:Acquiring 10 fractional cores from a pool of 80 to satisfy a request of 1.000000 cores
INFO:toil.batchSystems.singleMachine:Executing command: '/usr/local/bin/python -E /usr/local/lib/python2.7/site-packages/toil/worker.py /data1/toil_tempdir l/J/jobcMf1Cx'.
Exception in thread Thread-13:
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/threading.py", line 810, in __bootstrap_inner
    self.run()
  File "/usr/local/lib/python2.7/threading.py", line 763, in run
    self.__target(*self.__args, **self.__kwargs)
  File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 398, in asyncWrite
    raise RuntimeError("The termination flag is set, exiting")
RuntimeError: The termination flag is set, exiting

Exception in thread Thread-14:
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/threading.py", line 810, in __bootstrap_inner
    self.run()
  File "/usr/local/lib/python2.7/threading.py", line 763, in run
    self.__target(*self.__args, **self.__kwargs)
  File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 398, in asyncWrite
    raise RuntimeError("The termination flag is set, exiting")
RuntimeError: The termination flag is set, exiting

Exception RuntimeError: RuntimeError('cannot join current thread',) in <bound method FileStore.__del__ of <toil.job.FileStore object at 0x7fbe7835cc10>> ignored
DEBUG:toil.batchSystems.singleMachine:Finished job. self.coreFractions ~ 80 and self.memory ~ 9223372036854775807
DEBUG:toil.batchSystems.singleMachine:Ran jobID: 0 with exit value: 0
DEBUG:toil.leader:Batch system is reporting that the jobWrapper with batch system ID: 0 and jobWrapper store ID: l/J/jobcMf1Cx ended successfully
WARNING:toil.leader:The jobWrapper seems to have left a log file, indicating failure: l/J/jobcMf1Cx
WARNING:toil.leader:Reporting file: l/J/jobcMf1Cx
WARNING:toil.leader:l/J/jobcMf1Cx:  ---TOIL WORKER OUTPUT LOG---
WARNING:toil.leader:l/J/jobcMf1Cx:  Next available file descriptor: 8
WARNING:toil.leader:l/J/jobcMf1Cx:  DEBUG:__main__:Next available file descriptor: 8
WARNING:toil.leader:l/J/jobcMf1Cx:  Parsed jobWrapper
WARNING:toil.leader:l/J/jobcMf1Cx:  DEBUG:__main__:Parsed jobWrapper
WARNING:toil.leader:l/J/jobcMf1Cx:  WARNING:toil.resource:Can't find resource for leader path '/data/test_docker/toil_tests/aux_scripts.pyc'
WARNING:toil.leader:l/J/jobcMf1Cx:  WARNING:toil.resource:Can't localize module ModuleDescriptor(dirPath='/data/test_docker/toil_tests', name='aux_scripts', extension='.pyc')
WARNING:toil.leader:l/J/jobcMf1Cx:  Starting the next jobWrapper
WARNING:toil.leader:l/J/jobcMf1Cx:  DEBUG:__main__:Starting the next jobWrapper
WARNING:toil.leader:l/J/jobcMf1Cx:  WARNING:toil.resource:Can't find resource for leader path '/data/test_docker/toil_tests/aux_scripts.pyc'
WARNING:toil.leader:l/J/jobcMf1Cx:  WARNING:toil.resource:Can't localize module ModuleDescriptor(dirPath='/data/test_docker/toil_tests', name='aux_scripts', extension='.pyc')
WARNING:toil.leader:l/J/jobcMf1Cx:  Starting the next jobWrapper
WARNING:toil.leader:l/J/jobcMf1Cx:  DEBUG:__main__:Starting the next jobWrapper
WARNING:toil.leader:l/J/jobcMf1Cx:  INFO:toil.jobStores.fileJobStore:Jobstore directory is: /data1/toil_tempdir
WARNING:toil.leader:l/J/jobcMf1Cx:  WARNING:toil.resource:Can't find resource for leader path '/data/test_docker/docker_scripts/precision_immuno.pyc'
WARNING:toil.leader:l/J/jobcMf1Cx:  WARNING:toil.resource:Can't localize module ModuleDescriptor(dirPath='/data/test_docker/docker_scripts', name='precision_immuno', extension='.pyc')
WARNING:toil.leader:l/J/jobcMf1Cx:  Traceback (most recent call last):
WARNING:toil.leader:l/J/jobcMf1Cx:    File "/usr/local/lib/python2.7/site-packages/toil/worker.py", line 284, in main
WARNING:toil.leader:l/J/jobcMf1Cx:      fileStore=fileStore)
WARNING:toil.leader:l/J/jobcMf1Cx:    File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 1104, in _execute
WARNING:toil.leader:l/J/jobcMf1Cx:      returnValues = self.run(fileStore)
WARNING:toil.leader:l/J/jobcMf1Cx:    File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 1218, in run
WARNING:toil.leader:l/J/jobcMf1Cx:      rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs)
WARNING:toil.leader:l/J/jobcMf1Cx:    File "/data/test_docker/docker_scripts/precision_immuno.py", line 1105, in get_files_from_filestore
WARNING:toil.leader:l/J/jobcMf1Cx:      '/'.join([work_dir, name]))
WARNING:toil.leader:l/J/jobcMf1Cx:    File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 528, in readGlobalFile
WARNING:toil.leader:l/J/jobcMf1Cx:      self.jobStore.readFile(fileStoreID, localFilePath)
WARNING:toil.leader:l/J/jobcMf1Cx:    File "/usr/local/lib/python2.7/site-packages/toil/jobStores/fileJobStore.py", line 167, in readFile
WARNING:toil.leader:l/J/jobcMf1Cx:      shutil.copyfile(self._getAbsPath(jobStoreFileID), localFilePath)
WARNING:toil.leader:l/J/jobcMf1Cx:    File "/usr/local/lib/python2.7/shutil.py", line 83, in copyfile
WARNING:toil.leader:l/J/jobcMf1Cx:      with open(dst, 'wb') as fdst:
WARNING:toil.leader:l/J/jobcMf1Cx:  IOError: [Errno 2] No such file or directory: '/data1/temp_dir/tmp_PW74NwRRLa/localTempDir/t0OuPSi/index_tar'
WARNING:toil.leader:l/J/jobcMf1Cx:  Exiting the worker because of a failed jobWrapper on host arjun-fake-podk.novalocal
WARNING:toil.leader:l/J/jobcMf1Cx:  ERROR:__main__:Exiting the worker because of a failed jobWrapper on host arjun-fake-podk.novalocal
WARNING:toil.leader:l/J/jobcMf1Cx:  WARNING:toil.jobWrapper:Due to failure we are reducing the remaining retry count of job l/J/jobcMf1Cx to 0
WARNING:toil.leader:l/J/jobcMf1Cx:  Exception in thread Thread-9:
WARNING:toil.leader:l/J/jobcMf1Cx:  Traceback (most recent call last):
WARNING:toil.leader:l/J/jobcMf1Cx:    File "/usr/local/lib/python2.7/threading.py", line 810, in __bootstrap_inner
WARNING:toil.leader:l/J/jobcMf1Cx:      self.run()
WARNING:toil.leader:l/J/jobcMf1Cx:    File "/usr/local/lib/python2.7/threading.py", line 763, in run
WARNING:toil.leader:l/J/jobcMf1Cx:      self.__target(*self.__args, **self.__kwargs)
WARNING:toil.leader:l/J/jobcMf1Cx:    File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 600, in asyncUpdate
WARNING:toil.leader:l/J/jobcMf1Cx:      raise RuntimeError("The termination flag is set, exiting before update")
WARNING:toil.leader:l/J/jobcMf1Cx:  RuntimeError: The termination flag is set, exiting before update
WARNING:toil.leader:l/J/jobcMf1Cx:  
WARNING:toil.leader:l/J/jobcMf1Cx:  Exception in thread Thread-12:
WARNING:toil.leader:l/J/jobcMf1Cx:  Traceback (most recent call last):
WARNING:toil.leader:l/J/jobcMf1Cx:    File "/usr/local/lib/python2.7/threading.py", line 810, in __bootstrap_inner
WARNING:toil.leader:l/J/jobcMf1Cx:      self.run()
WARNING:toil.leader:l/J/jobcMf1Cx:    File "/usr/local/lib/python2.7/threading.py", line 763, in run
WARNING:toil.leader:l/J/jobcMf1Cx:      self.__target(*self.__args, **self.__kwargs)
WARNING:toil.leader:l/J/jobcMf1Cx:    File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 600, in asyncUpdate
WARNING:toil.leader:l/J/jobcMf1Cx:      raise RuntimeError("The termination flag is set, exiting before update")
WARNING:toil.leader:l/J/jobcMf1Cx:  RuntimeError: The termination flag is set, exiting before update
WARNING:toil.leader:l/J/jobcMf1Cx:  
DEBUG:toil.leader:Added jobWrapper: l/J/jobcMf1Cx to active jobs
DEBUG:toil.leader:Built the jobs list, currently have 1 jobs to update and 0 jobs issued
WARNING:toil.leader:Job: l/J/jobcMf1Cx is completely failed
INFO:toil.leader:Only failed jobs and their dependents (1 total) are remaining, so exiting.
INFO:toil.leader:Finished the main loop
INFO:toil.leader:Waiting for stats and logging collator process to finish
INFO:toil.leader:Stats/logging finished collating in 0.489140987396 seconds
DEBUG:toil.common:Shutting down batch system
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Received queue sentinel.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
DEBUG:toil.batchSystems.singleMachine:Exiting worker thread normally.
Traceback (most recent call last):
  File "module_tests.py", line 210, in <module>
    A.test_star()
  File "module_tests.py", line 107, in test_star
    Job.Runner.startToil(F2, params)
  File "/usr/local/lib/python2.7/site-packages/toil/job.py", line 362, in startToil
    return mainLoop(config, batchSystem, jobStore, rootJob)
  File "/usr/local/lib/python2.7/site-packages/toil/leader.py", line 505, in mainLoop
    raise FailedJobsException( config.jobStore, totalFailedJobs )
toil.leader.FailedJobsException: The job store '/data1/toil_tempdir' contains 1 failed jobs
```


Resolving patch:
diff --git a/src/toil/job.py b/src/toil/job.py
--- a/src/toil/job.py
+++ b/src/toil/job.py
@@ -375,7 +375,7 @@ def __init__(self, jobStore, jobWrapper, localTempDir,
             """
             self.jobStore = jobStore
             self.jobWrapper = jobWrapper
-            self.localTempDir = localTempDir
+            self.localTempDir = os.path.abspath(localTempDir)
             self.loggingMessages = []
             self.filesToDelete = set()
             self.jobsToDelete = set()
@@ -445,7 +445,7 @@ def getLocalTempFile(self):
             handle, tmpFile = tempfile.mkstemp(prefix="tmp", 
                                                suffix=".tmp", dir=self.localTempDir)
             os.close(handle)
-            return os.path.abspath(tmpFile)
+            return os.path.abspath(tmpFile) 
 
         def writeGlobalFile(self, localFileName, cleanup=False):
             """
@@ -456,18 +456,25 @@ def writeGlobalFile(self, localFileName, cleanup=False):
             and all its successors have completed running. If not the global file 
             must be deleted manually.
             
-            The write is asynchronous, so further modifications during execution 
-            to the file pointed by localFileName will result in undetermined behavior. 
+            If the local file is a file returned by Job.FileStore.getLocalTempFile
+            or is in a directory, or, recursively, a subdirectory, returned by 
+            Job.FileStore.getLocalTempDir then the write is asynchronous, 
+            so further modifications during execution to the file pointed by 
+            localFileName will result in undetermined behavior. Otherwise, the 
+            method will block until the file is written to the file store. 
             """
-            jobStoreFileID = self.jobStore.getEmptyFileStoreID(None 
-                            if not cleanup else self.jobWrapper.jobStoreID)
-            self.queue.put((open(localFileName, 'r'), jobStoreFileID))
-            #Now put the file into the cache if it is a path within localTempDir
+            #Put the file into the cache if it is a path within localTempDir
             absLocalFileName = os.path.abspath(localFileName)
+            cleanupID = None if not cleanup else self.jobWrapper.jobStoreID
             if absLocalFileName.startswith(self.localTempDir):
-                #Chmod to make file read only
+                jobStoreFileID = self.jobStore.getEmptyFileStoreID(cleanupID)
+                self.queue.put((open(absLocalFileName, 'r'), jobStoreFileID))
+                #Chmod to make file read only to try to prevent accidental user modification
                 os.chmod(absLocalFileName, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
                 self.jobStoreFileIDToCacheLocation[jobStoreFileID] = absLocalFileName
+            else:
+                #Write the file directly to the file store
+                jobStoreFileID = self.jobStore.writeFile(localFileName, cleanupID)
             return jobStoreFileID
         
         def writeGlobalFileStream(self, cleanup=False):
@@ -482,53 +489,71 @@ def writeGlobalFileStream(self, cleanup=False):
             #TODO: Make this work with the caching??
             return self.jobStore.writeFileStream(None if not cleanup else self.jobWrapper.jobStoreID)
         
-        def readGlobalFile(self, fileStoreID, userPath=None):
+        def readGlobalFile(self, fileStoreID, userPath=None, cache=True):
             """
             Returns an absolute path to a local, temporary copy of the file 
             keyed by fileStoreID. 
             
-            *The returned file will be read only (have permissions 444).* 
-            
             :param userPath: a path to the name of file to which the global file will be 
-            copied or hard-linked (see below). userPath must either be: (1) a 
-            file path contained within a directory or, recursively, a subdirectory 
-            of a temporary directory returned by Job.FileStore.getLocalTempDir(), 
-            or (2) a file path returned by Job.FileStore.getLocalTempFile(). 
-            If userPath is specified and this is not true a RuntimeError exception 
-            will be raised. If userPath is specified and the file is already cached, 
+            copied or hard-linked (see below). 
+            
+            :param cache: a boolean to switch on caching (see below). Caching will 
+            attempt to keep copies of files between sequences of jobs run on the same 
+            worker. 
+            
+            If cache=True and userPath is either: 
+            (1) a file path contained within a directory or, 
+            recursively, a subdirectory of a temporary directory returned by 
+            Job.FileStore.getLocalTempDir(), or (2) a file path returned by 
+            Job.FileStore.getLocalTempFile() then the file will be cached and
+            returned file will be read only (have permissions 444).
+            If userPath is specified and the file is already cached, 
             the userPath file will be a hard link to the actual location, else it 
-            will be an actual copy of the file.  
+            will be an actual copy of the file. 
+            
+            If the cache=False or userPath is not either of the above the file 
+            will not be cached and will have default permissions. Note, if the file
+            is already cached this will result in two copies of the file on the system.
+            
+            :rtype : the absolute path to the read file
             """
             if fileStoreID in self.filesToDelete:
                 raise RuntimeError("Trying to access a file in the jobStore you've deleted: %s" % fileStoreID)
             if userPath != None:
                 userPath = os.path.abspath(userPath) #Make an absolute path
-                #Check it is a valid location
-                if not userPath.startswith(self.localTempDir):
-                    raise RuntimeError("The user path is not contained within the"
-                                       " temporary file hierarchy created by the job."
-                                       " User path: %s, temporary file root path: %s" % 
-                                       (userPath, self.localTempDir))
+                #Turn off caching if user file is not in localTempDir
+                if cache and not userPath.startswith(self.localTempDir):
+                    cache = False  
             #When requesting a new file from the jobStore first check if fileStoreID
             #is a key in jobStoreFileIDToCacheLocation.
             if fileStoreID in self.jobStoreFileIDToCacheLocation:
                 cachedAbsFilePath = self.jobStoreFileIDToCacheLocation[fileStoreID]   
-                #If the user specifies a location and it is not the current location
-                # return a hardlink to the location, else return the original location
-                if userPath == None or userPath == cachedAbsFilePath:
-                    return cachedAbsFilePath
-                if os.path.exists(userPath):
-                    os.remove(userPath)
-                os.link(cachedAbsFilePath, userPath)
-                return userPath
+                if cache:
+                    #If the user specifies a location and it is not the current location
+                    #return a hardlink to the location, else return the original location
+                    if userPath == None or userPath == cachedAbsFilePath:
+                        return cachedAbsFilePath
+                    #Chmod to make file read only
+                    if os.path.exists(userPath):
+                        os.remove(userPath)
+                    os.link(cachedAbsFilePath, userPath)
+                    return userPath
+                else:
+                    #If caching is not true then make a copy of the file
+                    localFilePath = userPath if userPath != None else self.getLocalTempFile()
+                    shutil.copyfile(cachedAbsFilePath, localFilePath)
+                    return localFilePath
             else:
                 #If it is not in the cache read it from the jobStore to the 
                 #desired location
                 localFilePath = userPath if userPath != None else self.getLocalTempFile()
                 self.jobStore.readFile(fileStoreID, localFilePath)
-                self.jobStoreFileIDToCacheLocation[fileStoreID] = localFilePath
-                #Chmod to make file read only
-                os.chmod(localFilePath, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
+                #If caching is enabled and the file is in local temp dir then
+                #add to cache and make read only
+                if cache:
+                    assert localFilePath.startswith(self.localTempDir)
+                    self.jobStoreFileIDToCacheLocation[fileStoreID] = localFilePath
+                    os.chmod(localFilePath, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
                 return localFilePath
 
         def readGlobalFileStream(self, fileStoreID):
@@ -601,7 +626,7 @@ def asyncUpdate():
                     
                     #Indicate any files that should be deleted once the update of 
                     #the job wrapper is completed.
-                    self.jobWrapper.filesToDelete = len(self.filesToDelete)
+                    self.jobWrapper.filesToDelete = list(self.filesToDelete)
                     
                     #Complete the job
                     self.jobStore.update(self.jobWrapper)
@@ -638,7 +663,7 @@ def _cleanLocalTempDir(self, cacheSize):
             At the end of the job, remove all localTempDir files except those whose value is in
             jobStoreFileIDToCacheLocation.
             
-            The param cacheSize is the total number of bytes of files allowed in the cache.
+            :param cacheSize: the total number of bytes of files allowed in the cache.
             """
             #Remove files so that the total cached files are smaller than a cacheSize
             
diff --git a/src/toil/jobStores/abstractJobStore.py b/src/toil/jobStores/abstractJobStore.py
--- a/src/toil/jobStores/abstractJobStore.py
+++ b/src/toil/jobStores/abstractJobStore.py
@@ -134,7 +134,7 @@ def getConnectedJobs(jobWrapper):
                 for fileID in jobWrapper.filesToDelete:
                     logger.critical("Removing file in job store: %s that was marked for deletion but not previously removed" % fileID)
                     self.deleteFile(fileID)
-                jobWrapper.filesToDelete = set()
+                jobWrapper.filesToDelete = []
                 changed = True
             
             #Delete a jobWrapper if it is not reachable from the rootJobWrapper


Test patch:
diff --git a/src/toil/test/sort/sort.py b/src/toil/test/sort/sort.py
--- a/src/toil/test/sort/sort.py
+++ b/src/toil/test/sort/sort.py
@@ -32,34 +32,44 @@
 def setup(job, inputFile, N):
     """Sets up the sort.
     """
+    #Write the input file to the file store
+    inputFileStoreID = job.fileStore.writeGlobalFile(inputFile, True)
     job.addFollowOnJobFn(cleanup, job.addChildJobFn(down, 
-        inputFile, 0, os.path.getsize(inputFile), N).rv(), inputFile, memory=sortMemory)
+                        inputFileStoreID, N).rv(), inputFile, memory=sortMemory)
 
-def down(job, inputFile, fileStart, fileEnd, N):
+def down(job, inputFileStoreID, N):
     """Input is a file and a range into that file to sort and an output location in which
     to write the sorted file.
     If the range is larger than a threshold N the range is divided recursively and
     a follow on job is then created which merges back the results else
     the file is sorted and placed in the output.
     """
-    length = fileEnd - fileStart
+    #Read the file
+    inputFile = job.fileStore.readGlobalFile(inputFileStoreID, cache=False)
+    length = os.path.getsize(inputFile)
     if length > N:
         #We will subdivide the file
-        job.fileStore.logToMaster( "Splitting range (%i..%i) of file: %s"
-                                      % (fileStart, fileEnd, inputFile), level=logging.CRITICAL )
-        midPoint = getMidPoint(inputFile, fileStart, fileEnd)
+        job.fileStore.logToMaster( "Splitting file: %s of size: %s"
+                                      % (inputFileStoreID, length), level=logging.CRITICAL )
+        #Split the file into two copies
+        midPoint = getMidPoint(inputFile, 0, length)
+        t1 = job.fileStore.getLocalTempFile()
+        with open(t1, 'w') as fH:
+            copySubRangeOfFile(inputFile, 0, midPoint+1, fH)
+        t2 = job.fileStore.getLocalTempFile()
+        with open(t2, 'w') as fH:
+            copySubRangeOfFile(inputFile, midPoint+1, length, fH)
+        #Call down recursively
         return job.addFollowOnJobFn(up,
-            job.addChildJobFn(down, inputFile, fileStart, midPoint+1, N, memory=sortMemory).rv(),
-            job.addChildJobFn(down, inputFile, midPoint+1, fileEnd, N, memory=sortMemory).rv()).rv()          
+            job.addChildJobFn(down, job.fileStore.writeGlobalFile(t1), N, memory=sortMemory).rv(),
+            job.addChildJobFn(down, job.fileStore.writeGlobalFile(t2), N, memory=sortMemory).rv()).rv()          
     else:
         #We can sort this bit of the file
-        job.fileStore.logToMaster( "Sorting range (%i..%i) of file: %s"
-                                      % (fileStart, fileEnd, inputFile), level=logging.CRITICAL )
-        t = job.fileStore.getLocalTempFile()
-        with open(t, 'w') as fH:
-            copySubRangeOfFile(inputFile, fileStart, fileEnd, fH)
-        sort(t)
-        return job.fileStore.writeGlobalFile(t)
+        job.fileStore.logToMaster( "Sorting file: %s of size: %s"
+                                      % (inputFileStoreID, length), level=logging.CRITICAL )
+        #Sort the copy and write back to the fileStore
+        sort(inputFile)
+        return job.fileStore.writeGlobalFile(inputFile)
 
 def up(job, inputFileID1, inputFileID2):
     """Merges the two files and places them in the output.
@@ -78,9 +88,7 @@ def up(job, inputFileID1, inputFileID2):
 def cleanup(job, tempOutputFileStoreID, outputFile):
     """Copies back the temporary file to input once we've successfully sorted the temporary file.
     """
-    localTempFile = job.fileStore.readGlobalFile(tempOutputFileStoreID)
-    shutil.copyfile(localTempFile, outputFile)
-    #sort(outputFile)
+    job.fileStore.readGlobalFile(tempOutputFileStoreID, userPath=outputFile)
 
 def main():
     parser = ArgumentParser()
diff --git a/src/toil/test/src/jobFileStoreTest.py b/src/toil/test/src/jobFileStoreTest.py
--- a/src/toil/test/src/jobFileStoreTest.py
+++ b/src/toil/test/src/jobFileStoreTest.py
@@ -58,22 +58,21 @@ def fileTestJob(job, inputFileStoreIDs, testStrings, chainLength):
         #Load the input jobStoreFileIDs and check that they map to the 
         #same set of random input strings
         if random.random() > 0.666:
-            #Check we get an exception if we try a file out side of the file path
-            try:
-                job.fileStore.readGlobalFile(fileStoreID, userPath="THIS/IS/NOT/A/REAL/PATH")
-            except RuntimeError:
-                pass
+            #Read the file for the fileStoreID, randomly picking a way to invoke readGlobalFile
             if random.random() > 0.5:
-                tempFile = job.fileStore.readGlobalFile(fileStoreID)
+                tempFile = job.fileStore.readGlobalFile(fileStoreID, 
+                                                        cache=random.random() > 0.5)
             else:
                 userPath = job.fileStore.getLocalTempFile()
-                tempFile = job.fileStore.readGlobalFile(fileStoreID, userPath)
+                tempFile = job.fileStore.readGlobalFile(fileStoreID, userPath, 
+                                                        cache=random.random() > 0.5)
                 assert userPath == tempFile
+            #Check the local file is as we expect
             with open(tempFile, 'r') as fH:
                 string = fH.readline()
                 assert testStrings[string[:PREFIX_LENGTH]] == string
         elif random.random() > 0.666:
-            #with job.fileStore.jobStore.readFileStream(fileStoreID) as fH:
+            #Try reading the file using the stream method.
             with job.fileStore.readGlobalFileStream(fileStoreID) as fH:
                 string = fH.readline()
                 assert testStrings[string[:PREFIX_LENGTH]] == string