# Buggy code
def __init__(self, jobStore, rootJob, jobCache=None):
    # This is a hash of jobs, referenced by jobStoreID, to their predecessor jobs.
    self.successorJobStoreIDToPredecessorJobs = {}

    # Hash of jobStoreIDs to counts of numbers of successors issued.
    # There are no entries for jobs
    # without successors in this map.
    self.successorCounts = {}

    # This is a hash of service jobs, referenced by jobStoreID, to their predecessor job
    self.serviceJobStoreIDToPredecessorJob = {}

    # Hash of jobStoreIDs to maps of services issued for the job
    # Each for job, the map is a dictionary of service jobStoreIDs
    # to the flags used to communicate the with service
    self.servicesIssued = {}

    # Jobs (as jobStoreIDs) with successors that have totally failed
    self.hasFailedSuccessors = set()

    # Jobs that are ready to be processed
    self.updatedJobs = set()

    # The set of totally failed jobs - this needs to be filtered at the
    # end to remove jobs that were removed by checkpoints
    self.totalFailedJobs = set()

    ##Algorithm to build this information
    logger.info("(Re)building internal scheduler state")
    self._buildToilState(rootJob, jobStore, jobCache)


# Traceback (AssertionError)
kunkel 2016-08-26 11:18:31,313 MainThread WARNING toil.leader: The jobWrapper seems to have left a log file, indicating failure: B/U/jobWBMw6I
kunkel 2016-08-26 11:18:31,313 MainThread WARNING toil.leader: Reporting file: B/U/jobWBMw6I
kunkel 2016-08-26 11:18:31,313 MainThread WARNING toil.leader: B/U/jobWBMw6I:   ---TOIL WORKER OUTPUT LOG---
kunkel 2016-08-26 11:18:31,313 MainThread WARNING toil.leader: B/U/jobWBMw6I:   INFO:rdflib:RDFLib Version: 4.2.1
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   INFO:toil.fileStore:CACHE: Read file with ID 'a/2/tmp2RYy7z.tmp' from the cache.
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   [job revtool.cwl] /tmp/user/1001/tmpZ7hLZ6$ rev-fail \
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:       /tmp/user/1001/tmppbMf7F/stg5c0b758e-c348-411b-b260-877b9491c6da/whale.txt > /tmp/user/1001/tmpZ7hLZ6/output.txt
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   INFO:cwltool:[job revtool.cwl] /tmp/user/1001/tmpZ7hLZ6$ rev-fail \
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:       /tmp/user/1001/tmppbMf7F/stg5c0b758e-c348-411b-b260-877b9491c6da/whale.txt > /tmp/user/1001/tmpZ7hLZ6/output.txt
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   'rev-fail' not found
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   ERROR:cwltool:'rev-fail' not found
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   [job revtool.cwl] completed permanentFail
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   WARNING:cwltool:[job revtool.cwl] completed permanentFail
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   Final process status is permanentFail
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   WARNING:cwltool:Final process status is permanentFail
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:   INFO:toil.fileStore:LOG-TO-MASTER: Successfully deleted local copies of file with ID 'a/2/tmp2RYy7z.tmp'.
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:   Traceback (most recent call last):
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:     File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/worker.py", line 331, in main
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:       job._runner(jobWrapper=jobWrapper, jobStore=jobStore, fileStore=fileStore)
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:     File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/job.py", line 1085, in _runner
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:       returnValues = self._run(jobWrapper, fileStore)
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:     File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/job.py", line 1033, in _run
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:       return self.run(fileStore)
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:     File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/cwl/cwltoil.py", line 256, in run
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:       **opts)
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:     File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/cwltool/main.py", line 218, in single_job_executor
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:       raise WorkflowException(u"Process status is %s" % (final_status))
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:   WorkflowException: Process status is ['permanentFail']
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:   Exiting the worker because of a failed jobWrapper on host kunkel
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:   ERROR:toil.worker:Exiting the worker because of a failed jobWrapper on host kunkel
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:   WARNING:toil.jobWrapper:Due to failure we are reducing the remaining retry count of job B/U/jobWBMw6I to 0
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:   WARNING:toil.jobWrapper:We have increased the default memory of the failed job to 2147483648 bytes
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: Job: B/U/jobWBMw6I is completely failed
kunkel 2016-08-26 11:18:31,316 MainThread INFO toil.leader: No jobs left to run so exiting.
kunkel 2016-08-26 11:18:31,316 MainThread INFO toil.leader: Finished the main loop
kunkel 2016-08-26 11:18:31,316 MainThread INFO toil.leader: Waiting for stats and logging collator thread to finish ...
kunkel 2016-08-26 11:18:31,638 MainThread INFO toil.leader: ... finished collating stats and logs. Took 0.321724891663 seconds
kunkel 2016-08-26 11:18:31,639 MainThread INFO toil.leader: Waiting for service manager thread to finish ...
kunkel 2016-08-26 11:18:32,129 MainThread INFO toil.leader: ... finished shutting down the service manager. Took 0.490026950836 seconds
Traceback (most recent call last):
File "/usr/local/bin/cwltoil", line 9, in <module>
load_entry_point('toil==3.5.0a1', 'console_scripts', 'cwltoil')()
File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/cwl/cwltoil.py", line 665, in main
outobj = toil.start(wf1)
File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/common.py", line 580, in start
return self._runMainLoop(job)
File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/common.py", line 794, in _runMainLoop
jobCache=self._jobCache)
File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/leader.py", line 692, in mainLoop
innerLoop(jobStore, config, batchSystem, toilState, jobBatcher, serviceManager, statsAndLogging)
File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/leader.py", line 970, in innerLoop
assert toilState.successorCounts == {}
AssertionError

# Fixed code
def __init__(self, jobStore, rootJob, jobCache=None):
    # This is a hash of jobs, referenced by jobStoreID, to their predecessor jobs.
    self.successorJobStoreIDToPredecessorJobs = {}

    # Hash of jobStoreIDs to counts of numbers of successors issued.
    # There are no entries for jobs
    # without successors in this map.
    self.successorCounts = {}

    # This is a hash of service jobs, referenced by jobStoreID, to their predecessor job
    self.serviceJobStoreIDToPredecessorJob = {}

    # Hash of jobStoreIDs to maps of services issued for the job
    # Each for job, the map is a dictionary of service jobStoreIDs
    # to the flags used to communicate the with service
    self.servicesIssued = {}

    # Jobs that are ready to be processed
    self.updatedJobs = set()

    # The set of totally failed jobs - this needs to be filtered at the
    # end to remove jobs that were removed by checkpoints
    self.totalFailedJobs = set()

    # Jobs (as jobStoreIDs) with successors that have totally failed
    self.hasFailedSuccessors = set()

    # The set of successors of failed jobs as a set of jobStoreIds
    self.failedSuccessors = set()

    # Set of jobs that have multiple predecessors that have one or more predecessors
    # finished, but not all of them. This acts as a cache for these jobs.
    # Stored as hash from jobStoreIDs to jobWrappers
    self.jobsToBeScheduledWithMultiplePredecessors = {}

    ##Algorithm to build this information
    logger.info("(Re)building internal scheduler state")
    self._buildToilState(rootJob, jobStore, jobCache)