# Buggy code
def _updatePredecessorStatus(self, jobStoreID):
    """
    Update status of a predecessor for finished successor job.
    """
    if jobStoreID in self.toilState.serviceJobStoreIDToPredecessorJob:
        # Is a service job
        predecessorJob = self.toilState.serviceJobStoreIDToPredecessorJob.pop(
            jobStoreID
        )
        self.toilState.servicesIssued[predecessorJob.jobStoreID].pop(jobStoreID)
        if (
            len(self.toilState.servicesIssued[predecessorJob.jobStoreID]) == 0
        ):  # Predecessor job has
            # all its services terminated
            self.toilState.servicesIssued.pop(
                predecessorJob.jobStoreID
            )  # The job has no running services
            self.toilState.updatedJobs.add((predecessorJob, 0))  # Now we know
            # the job is done we can add it to the list of updated job files
            logger.debug(
                "Job %s services have completed or totally failed, adding to updated jobs"
                % predecessorJob.jobStoreID
            )

    elif jobStoreID not in self.toilState.successorJobStoreIDToPredecessorJobs:
        # We have reach the root job
        assert len(self.toilState.updatedJobs) == 0
        assert len(self.toilState.successorJobStoreIDToPredecessorJobs) == 0
        assert len(self.toilState.successorCounts) == 0

    else:
        for predecessorJob in self.toilState.successorJobStoreIDToPredecessorJobs.pop(
            jobStoreID
        ):
            self.toilState.successorCounts[predecessorJob.jobStoreID] -= 1
            assert self.toilState.successorCounts[predecessorJob.jobStoreID] >= 0
            if (
                self.toilState.successorCounts[predecessorJob.jobStoreID] == 0
            ):  # Job is done
                self.toilState.successorCounts.pop(predecessorJob.jobStoreID)
                predecessorJob.stack.pop()
                logger.debug(
                    "Job %s has all its non-service successors completed or totally "
                    "failed",
                    predecessorJob.jobStoreID,
                )
                assert predecessorJob not in self.toilState.updatedJobs
                self.toilState.updatedJobs.add((predecessorJob, 0))  # Now we know


# Traceback (AssertionError)
kunkel 2016-08-26 11:18:31,313 MainThread WARNING toil.leader: The jobWrapper seems to have left a log file, indicating failure: B/U/jobWBMw6I
kunkel 2016-08-26 11:18:31,313 MainThread WARNING toil.leader: Reporting file: B/U/jobWBMw6I
kunkel 2016-08-26 11:18:31,313 MainThread WARNING toil.leader: B/U/jobWBMw6I:   ---TOIL WORKER OUTPUT LOG---
kunkel 2016-08-26 11:18:31,313 MainThread WARNING toil.leader: B/U/jobWBMw6I:   INFO:rdflib:RDFLib Version: 4.2.1
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   INFO:toil.fileStore:CACHE: Read file with ID 'a/2/tmp2RYy7z.tmp' from the cache.
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   [job revtool.cwl] /tmp/user/1001/tmpZ7hLZ6$ rev-fail \
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:       /tmp/user/1001/tmppbMf7F/stg5c0b758e-c348-411b-b260-877b9491c6da/whale.txt > /tmp/user/1001/tmpZ7hLZ6/output.txt
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   INFO:cwltool:[job revtool.cwl] /tmp/user/1001/tmpZ7hLZ6$ rev-fail \
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:       /tmp/user/1001/tmppbMf7F/stg5c0b758e-c348-411b-b260-877b9491c6da/whale.txt > /tmp/user/1001/tmpZ7hLZ6/output.txt
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   'rev-fail' not found
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   ERROR:cwltool:'rev-fail' not found
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   [job revtool.cwl] completed permanentFail
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   WARNING:cwltool:[job revtool.cwl] completed permanentFail
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   Final process status is permanentFail
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   WARNING:cwltool:Final process status is permanentFail
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:   INFO:toil.fileStore:LOG-TO-MASTER: Successfully deleted local copies of file with ID 'a/2/tmp2RYy7z.tmp'.
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:   Traceback (most recent call last):
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:     File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/worker.py", line 331, in main
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:       job._runner(jobWrapper=jobWrapper, jobStore=jobStore, fileStore=fileStore)
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:     File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/job.py", line 1085, in _runner
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:       returnValues = self._run(jobWrapper, fileStore)
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:     File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/job.py", line 1033, in _run
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:       return self.run(fileStore)
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:     File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/cwl/cwltoil.py", line 256, in run
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:       **opts)
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:     File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/cwltool/main.py", line 218, in single_job_executor
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:       raise WorkflowException(u"Process status is %s" % (final_status))
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:   WorkflowException: Process status is ['permanentFail']
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:   Exiting the worker because of a failed jobWrapper on host kunkel
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:   ERROR:toil.worker:Exiting the worker because of a failed jobWrapper on host kunkel
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:   WARNING:toil.jobWrapper:Due to failure we are reducing the remaining retry count of job B/U/jobWBMw6I to 0
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:   WARNING:toil.jobWrapper:We have increased the default memory of the failed job to 2147483648 bytes
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: Job: B/U/jobWBMw6I is completely failed
kunkel 2016-08-26 11:18:31,316 MainThread INFO toil.leader: No jobs left to run so exiting.
kunkel 2016-08-26 11:18:31,316 MainThread INFO toil.leader: Finished the main loop
kunkel 2016-08-26 11:18:31,316 MainThread INFO toil.leader: Waiting for stats and logging collator thread to finish ...
kunkel 2016-08-26 11:18:31,638 MainThread INFO toil.leader: ... finished collating stats and logs. Took 0.321724891663 seconds
kunkel 2016-08-26 11:18:31,639 MainThread INFO toil.leader: Waiting for service manager thread to finish ...
kunkel 2016-08-26 11:18:32,129 MainThread INFO toil.leader: ... finished shutting down the service manager. Took 0.490026950836 seconds
Traceback (most recent call last):
File "/usr/local/bin/cwltoil", line 9, in <module>
load_entry_point('toil==3.5.0a1', 'console_scripts', 'cwltoil')()
File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/cwl/cwltoil.py", line 665, in main
outobj = toil.start(wf1)
File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/common.py", line 580, in start
return self._runMainLoop(job)
File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/common.py", line 794, in _runMainLoop
jobCache=self._jobCache)
File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/leader.py", line 692, in mainLoop
innerLoop(jobStore, config, batchSystem, toilState, jobBatcher, serviceManager, statsAndLogging)
File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/leader.py", line 970, in innerLoop
assert toilState.successorCounts == {}
AssertionError

# Fixed code
def _updatePredecessorStatus(self, jobStoreID):
    """
    Update status of predecessors for finished successor job.
    """
    if jobStoreID in self.toilState.serviceJobStoreIDToPredecessorJob:
        # Is a service job
        predecessorJob = self.toilState.serviceJobStoreIDToPredecessorJob.pop(
            jobStoreID
        )
        self.toilState.servicesIssued[predecessorJob.jobStoreID].pop(jobStoreID)
        if (
            len(self.toilState.servicesIssued[predecessorJob.jobStoreID]) == 0
        ):  # Predecessor job has
            # all its services terminated
            self.toilState.servicesIssued.pop(
                predecessorJob.jobStoreID
            )  # The job has no running services
            self.toilState.updatedJobs.add((predecessorJob, 0))  # Now we know
            # the job is done we can add it to the list of updated job files
            logger.debug(
                "Job %s services have completed or totally failed, adding to updated jobs"
                % predecessorJob.jobStoreID
            )

    elif jobStoreID not in self.toilState.successorJobStoreIDToPredecessorJobs:
        # We have reach the root job
        assert len(self.toilState.updatedJobs) == 0
        assert len(self.toilState.successorJobStoreIDToPredecessorJobs) == 0
        assert len(self.toilState.successorCounts) == 0
        logger.debug("Reached root job %s so no predecessors to clean up" % jobStoreID)

    else:
        # Is a non-root, non-service job
        logger.debug("Cleaning the predecessors of %s" % jobStoreID)

        # For each predecessor
        for predecessorJob in self.toilState.successorJobStoreIDToPredecessorJobs.pop(
            jobStoreID
        ):
            # Reduce the predecessor's number of successors by one to indicate the
            # completion of the jobStoreID job
            self.toilState.successorCounts[predecessorJob.jobStoreID] -= 1

            # If the predecessor job is done and all the successors are complete
            if self.toilState.successorCounts[predecessorJob.jobStoreID] == 0:
                # Remove it from the set of jobs with active successors
                self.toilState.successorCounts.pop(predecessorJob.jobStoreID)

                # Pop stack at this point, as we can get rid of its successors
                predecessorJob.stack.pop()

                # Now we know the job is done we can add it to the list of updated job files
                assert predecessorJob not in self.toilState.updatedJobs
                self.toilState.updatedJobs.add((predecessorJob, 0))

                logger.debug(
                    "Job %s has all its non-service successors completed or totally "
                    "failed",
                    predecessorJob.jobStoreID,
                )