# Buggy code
def _buildToilState(self, jobWrapper, jobStore, jobCache=None):
    """
    Traverses tree of jobs from the root jobWrapper (rootJob) building the
    ToilState class.

    If jobCache is passed, it must be a dict from job ID to JobWrapper
    object. Jobs will be loaded from the cache (which can be downloaded from
    the jobStore in a batch) instead of piecemeal when recursed into.
    """

    def getJob(jobId):
        if jobCache is not None:
            try:
                return jobCache[jobId]
            except ValueError:
                return jobStore.load(jobId)
        else:
            return jobStore.load(jobId)

    # If the jobWrapper has a command, is a checkpoint, has services or is ready to be
    # deleted it is ready to be processed
    if (
        jobWrapper.command is not None
        or jobWrapper.checkpoint is not None
        or len(jobWrapper.services) > 0
        or len(jobWrapper.stack) == 0
    ):
        logger.debug(
            "Found job to run: %s, with command: %s, with checkpoint: %s, "
            "with  services: %s, with stack: %s",
            jobWrapper.jobStoreID,
            jobWrapper.command is not None,
            jobWrapper.checkpoint is not None,
            len(jobWrapper.services) > 0,
            len(jobWrapper.stack) == 0,
        )
        self.updatedJobs.add((jobWrapper, 0))

        if jobWrapper.checkpoint is not None:
            jobWrapper.command = jobWrapper.checkpoint

    else:  # There exist successors
        self.successorCounts[jobWrapper.jobStoreID] = len(jobWrapper.stack[-1])
        for successorJobStoreTuple in jobWrapper.stack[-1]:
            successorJobStoreID = successorJobStoreTuple[0]
            if successorJobStoreID not in self.successorJobStoreIDToPredecessorJobs:
                # Given that the successor jobWrapper does not yet point back at a
                # predecessor we have not yet considered it, so we call the function
                # on the successor
                self.successorJobStoreIDToPredecessorJobs[successorJobStoreID] = [
                    jobWrapper
                ]
                self._buildToilState(
                    getJob(successorJobStoreID), jobStore, jobCache=jobCache
                )
            else:
                # We have already looked at the successor, so we don't recurse,
                # but we add back a predecessor link
                self.successorJobStoreIDToPredecessorJobs[successorJobStoreID].append(
                    jobWrapper
                )


# Traceback (AssertionError)
kunkel 2016-08-26 11:18:31,313 MainThread WARNING toil.leader: The jobWrapper seems to have left a log file, indicating failure: B/U/jobWBMw6I
kunkel 2016-08-26 11:18:31,313 MainThread WARNING toil.leader: Reporting file: B/U/jobWBMw6I
kunkel 2016-08-26 11:18:31,313 MainThread WARNING toil.leader: B/U/jobWBMw6I:   ---TOIL WORKER OUTPUT LOG---
kunkel 2016-08-26 11:18:31,313 MainThread WARNING toil.leader: B/U/jobWBMw6I:   INFO:rdflib:RDFLib Version: 4.2.1
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   INFO:toil.fileStore:CACHE: Read file with ID 'a/2/tmp2RYy7z.tmp' from the cache.
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   [job revtool.cwl] /tmp/user/1001/tmpZ7hLZ6$ rev-fail \
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:       /tmp/user/1001/tmppbMf7F/stg5c0b758e-c348-411b-b260-877b9491c6da/whale.txt > /tmp/user/1001/tmpZ7hLZ6/output.txt
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   INFO:cwltool:[job revtool.cwl] /tmp/user/1001/tmpZ7hLZ6$ rev-fail \
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:       /tmp/user/1001/tmppbMf7F/stg5c0b758e-c348-411b-b260-877b9491c6da/whale.txt > /tmp/user/1001/tmpZ7hLZ6/output.txt
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   'rev-fail' not found
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   ERROR:cwltool:'rev-fail' not found
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   [job revtool.cwl] completed permanentFail
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   WARNING:cwltool:[job revtool.cwl] completed permanentFail
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   Final process status is permanentFail
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   WARNING:cwltool:Final process status is permanentFail
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:   INFO:toil.fileStore:LOG-TO-MASTER: Successfully deleted local copies of file with ID 'a/2/tmp2RYy7z.tmp'.
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:   Traceback (most recent call last):
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:     File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/worker.py", line 331, in main
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:       job._runner(jobWrapper=jobWrapper, jobStore=jobStore, fileStore=fileStore)
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:     File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/job.py", line 1085, in _runner
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:       returnValues = self._run(jobWrapper, fileStore)
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:     File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/job.py", line 1033, in _run
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:       return self.run(fileStore)
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:     File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/cwl/cwltoil.py", line 256, in run
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:       **opts)
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:     File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/cwltool/main.py", line 218, in single_job_executor
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:       raise WorkflowException(u"Process status is %s" % (final_status))
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:   WorkflowException: Process status is ['permanentFail']
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:   Exiting the worker because of a failed jobWrapper on host kunkel
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:   ERROR:toil.worker:Exiting the worker because of a failed jobWrapper on host kunkel
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:   WARNING:toil.jobWrapper:Due to failure we are reducing the remaining retry count of job B/U/jobWBMw6I to 0
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:   WARNING:toil.jobWrapper:We have increased the default memory of the failed job to 2147483648 bytes
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: Job: B/U/jobWBMw6I is completely failed
kunkel 2016-08-26 11:18:31,316 MainThread INFO toil.leader: No jobs left to run so exiting.
kunkel 2016-08-26 11:18:31,316 MainThread INFO toil.leader: Finished the main loop
kunkel 2016-08-26 11:18:31,316 MainThread INFO toil.leader: Waiting for stats and logging collator thread to finish ...
kunkel 2016-08-26 11:18:31,638 MainThread INFO toil.leader: ... finished collating stats and logs. Took 0.321724891663 seconds
kunkel 2016-08-26 11:18:31,639 MainThread INFO toil.leader: Waiting for service manager thread to finish ...
kunkel 2016-08-26 11:18:32,129 MainThread INFO toil.leader: ... finished shutting down the service manager. Took 0.490026950836 seconds
Traceback (most recent call last):
File "/usr/local/bin/cwltoil", line 9, in <module>
load_entry_point('toil==3.5.0a1', 'console_scripts', 'cwltoil')()
File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/cwl/cwltoil.py", line 665, in main
outobj = toil.start(wf1)
File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/common.py", line 580, in start
return self._runMainLoop(job)
File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/common.py", line 794, in _runMainLoop
jobCache=self._jobCache)
File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/leader.py", line 692, in mainLoop
innerLoop(jobStore, config, batchSystem, toilState, jobBatcher, serviceManager, statsAndLogging)
File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/leader.py", line 970, in innerLoop
assert toilState.successorCounts == {}
AssertionError

# Fixed code
def _buildToilState(self, jobWrapper, jobStore, jobCache=None):
    """
    Traverses tree of jobs from the root jobWrapper (rootJob) building the
    ToilState class.

    If jobCache is passed, it must be a dict from job ID to JobWrapper
    object. Jobs will be loaded from the cache (which can be downloaded from
    the jobStore in a batch) instead of piecemeal when recursed into.
    """

    def getJob(jobId):
        if jobCache is not None:
            try:
                return jobCache[jobId]
            except ValueError:
                return jobStore.load(jobId)
        else:
            return jobStore.load(jobId)

    # If the jobWrapper has a command, is a checkpoint, has services or is ready to be
    # deleted it is ready to be processed
    if (
        jobWrapper.command is not None
        or jobWrapper.checkpoint is not None
        or len(jobWrapper.services) > 0
        or len(jobWrapper.stack) == 0
    ):
        logger.debug(
            "Found job to run: %s, with command: %s, with checkpoint: %s, "
            "with  services: %s, with stack: %s",
            jobWrapper.jobStoreID,
            jobWrapper.command is not None,
            jobWrapper.checkpoint is not None,
            len(jobWrapper.services) > 0,
            len(jobWrapper.stack) == 0,
        )
        self.updatedJobs.add((jobWrapper, 0))

        if jobWrapper.checkpoint is not None:
            jobWrapper.command = jobWrapper.checkpoint

    else:  # There exist successors
        logger.debug(
            "Adding job: %s to the state with %s successors"
            % (jobWrapper.jobStoreID, len(jobWrapper.stack[-1]))
        )

        # Record the number of successors
        self.successorCounts[jobWrapper.jobStoreID] = len(jobWrapper.stack[-1])

        def processSuccessorWithMultiplePredecessors(successorJobWrapper):
            # If jobWrapper job is not reported as complete by the successor
            if jobWrapper.jobStoreID not in successorJobWrapper.predecessorsFinished:
                # Update the sucessor's status to mark the predecessor complete
                successorJobWrapper.predecessorsFinished.add(jobWrapper.jobStoreID)

            # If the successor has no predecessors to finish
            assert (
                len(successorJobWrapper.predecessorsFinished)
                <= successorJobWrapper.predecessorNumber
            )
            if (
                len(successorJobWrapper.predecessorsFinished)
                == successorJobWrapper.predecessorNumber
            ):
                # It is ready to be run, so remove it from the cache
                self.jobsToBeScheduledWithMultiplePredecessors.pop(successorJobStoreID)

                # Recursively consider the successor
                self._buildToilState(successorJobWrapper, jobStore, jobCache=jobCache)

        # For each successor
        for successorJobStoreTuple in jobWrapper.stack[-1]:
            successorJobStoreID = successorJobStoreTuple[0]

            # If the successor jobWrapper does not yet point back at a
            # predecessor we have not yet considered it
            if successorJobStoreID not in self.successorJobStoreIDToPredecessorJobs:
                # Add the job as a predecessor
                self.successorJobStoreIDToPredecessorJobs[successorJobStoreID] = [
                    jobWrapper
                ]

                # If predecessorJobStoreID is not None then the successor has multiple predecessors
                predecessorJobStoreID = successorJobStoreTuple[-1]
                if predecessorJobStoreID != None:
                    # We load the successor job
                    successorJobWrapper = getJob(successorJobStoreID)

                    # We put the successor job in the cache of successor jobs with multiple predecessors
                    assert (
                        successorJobStoreID
                        not in self.jobsToBeScheduledWithMultiplePredecessors
                    )
                    self.jobsToBeScheduledWithMultiplePredecessors[
                        successorJobStoreID
                    ] = successorJobWrapper

                    # Process successor
                    processSuccessorWithMultiplePredecessors(successorJobWrapper)

                else:
                    # The successor has only the jobWrapper job as a predecessor so
                    # recursively consider the successor
                    self._buildToilState(
                        getJob(successorJobStoreID), jobStore, jobCache=jobCache
                    )

            else:
                # We've already seen the successor

                # Add the job as a predecessor
                assert (
                    jobWrapper
                    not in self.successorJobStoreIDToPredecessorJobs[
                        successorJobStoreID
                    ]
                )
                self.successorJobStoreIDToPredecessorJobs[successorJobStoreID].append(
                    jobWrapper
                )

                # If the successor has multiple predecessors
                if (
                    successorJobStoreID
                    in self.jobsToBeScheduledWithMultiplePredecessors
                ):
                    # Get the successor from cache
                    successorJobWrapper = (
                        self.jobsToBeScheduledWithMultiplePredecessors[
                            successorJobStoreID
                        ]
                    )

                    # Process successor
                    processSuccessorWithMultiplePredecessors(successorJobWrapper)