# Buggy code
def processTotallyFailedJob(self, jobWrapper):
    """
    Processes a totally failed job.
    """
    # Mark job as a totally failed job
    self.toilState.totalFailedJobs.add(jobWrapper.jobStoreID)

    if jobWrapper.jobStoreID in self.toilState.serviceJobStoreIDToPredecessorJob:  # Is
        # a service job
        logger.debug(
            "Service job is being processed as a totally failed job: %s"
            % jobWrapper.jobStoreID
        )

        predecessorJobWrapper = self.toilState.serviceJobStoreIDToPredecessorJob[
            jobWrapper.jobStoreID
        ]

        # This removes the service job as a service of the predecessor
        # and potentially makes the predecessor active
        self._updatePredecessorStatus(jobWrapper.jobStoreID)

        # Remove the start flag, if it still exists. This indicates
        # to the service manager that the job has "started", this prevents
        # the service manager from deadlocking while waiting
        self.jobStore.deleteFile(jobWrapper.startJobStoreID)

        # Signal to any other services in the group that they should
        # terminate. We do this to prevent other services in the set
        # of services from deadlocking waiting for this service to start properly
        if predecessorJobWrapper.jobStoreID in self.toilState.servicesIssued:
            self.serviceManager.killServices(
                self.toilState.servicesIssued[predecessorJobWrapper.jobStoreID],
                error=True,
            )
            logger.debug(
                "Job: %s is instructing all the services of its parent job to quit",
                jobWrapper.jobStoreID,
            )

        self.toilState.hasFailedSuccessors.add(
            predecessorJobWrapper.jobStoreID
        )  # This ensures that the
        # job will not attempt to run any of it's successors on the stack
    else:
        assert jobWrapper.jobStoreID not in self.toilState.servicesIssued

        # Is a non-service job, walk up the tree killing services of any jobs with nothing left to do.
        if jobWrapper.jobStoreID in self.toilState.successorJobStoreIDToPredecessorJobs:
            for (
                predecessorJobWrapper
            ) in self.toilState.successorJobStoreIDToPredecessorJobs[
                jobWrapper.jobStoreID
            ]:
                self.toilState.hasFailedSuccessors.add(predecessorJobWrapper.jobStoreID)
                logger.debug(
                    "Totally failed job: %s is marking direct predecessors %s as having failed jobs",
                    jobWrapper.jobStoreID,
                    predecessorJobWrapper.jobStoreID,
                )
            self._updatePredecessorStatus(jobWrapper.jobStoreID)


# Traceback (AssertionError)
kunkel 2016-08-26 11:18:31,313 MainThread WARNING toil.leader: The jobWrapper seems to have left a log file, indicating failure: B/U/jobWBMw6I
kunkel 2016-08-26 11:18:31,313 MainThread WARNING toil.leader: Reporting file: B/U/jobWBMw6I
kunkel 2016-08-26 11:18:31,313 MainThread WARNING toil.leader: B/U/jobWBMw6I:   ---TOIL WORKER OUTPUT LOG---
kunkel 2016-08-26 11:18:31,313 MainThread WARNING toil.leader: B/U/jobWBMw6I:   INFO:rdflib:RDFLib Version: 4.2.1
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   INFO:toil.fileStore:CACHE: Read file with ID 'a/2/tmp2RYy7z.tmp' from the cache.
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   [job revtool.cwl] /tmp/user/1001/tmpZ7hLZ6$ rev-fail \
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:       /tmp/user/1001/tmppbMf7F/stg5c0b758e-c348-411b-b260-877b9491c6da/whale.txt > /tmp/user/1001/tmpZ7hLZ6/output.txt
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   INFO:cwltool:[job revtool.cwl] /tmp/user/1001/tmpZ7hLZ6$ rev-fail \
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:       /tmp/user/1001/tmppbMf7F/stg5c0b758e-c348-411b-b260-877b9491c6da/whale.txt > /tmp/user/1001/tmpZ7hLZ6/output.txt
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   'rev-fail' not found
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   ERROR:cwltool:'rev-fail' not found
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   [job revtool.cwl] completed permanentFail
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   WARNING:cwltool:[job revtool.cwl] completed permanentFail
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   Final process status is permanentFail
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   WARNING:cwltool:Final process status is permanentFail
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:   INFO:toil.fileStore:LOG-TO-MASTER: Successfully deleted local copies of file with ID 'a/2/tmp2RYy7z.tmp'.
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:   Traceback (most recent call last):
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:     File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/worker.py", line 331, in main
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:       job._runner(jobWrapper=jobWrapper, jobStore=jobStore, fileStore=fileStore)
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:     File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/job.py", line 1085, in _runner
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:       returnValues = self._run(jobWrapper, fileStore)
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:     File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/job.py", line 1033, in _run
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:       return self.run(fileStore)
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:     File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/cwl/cwltoil.py", line 256, in run
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:       **opts)
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:     File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/cwltool/main.py", line 218, in single_job_executor
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:       raise WorkflowException(u"Process status is %s" % (final_status))
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:   WorkflowException: Process status is ['permanentFail']
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:   Exiting the worker because of a failed jobWrapper on host kunkel
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:   ERROR:toil.worker:Exiting the worker because of a failed jobWrapper on host kunkel
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:   WARNING:toil.jobWrapper:Due to failure we are reducing the remaining retry count of job B/U/jobWBMw6I to 0
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:   WARNING:toil.jobWrapper:We have increased the default memory of the failed job to 2147483648 bytes
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: Job: B/U/jobWBMw6I is completely failed
kunkel 2016-08-26 11:18:31,316 MainThread INFO toil.leader: No jobs left to run so exiting.
kunkel 2016-08-26 11:18:31,316 MainThread INFO toil.leader: Finished the main loop
kunkel 2016-08-26 11:18:31,316 MainThread INFO toil.leader: Waiting for stats and logging collator thread to finish ...
kunkel 2016-08-26 11:18:31,638 MainThread INFO toil.leader: ... finished collating stats and logs. Took 0.321724891663 seconds
kunkel 2016-08-26 11:18:31,639 MainThread INFO toil.leader: Waiting for service manager thread to finish ...
kunkel 2016-08-26 11:18:32,129 MainThread INFO toil.leader: ... finished shutting down the service manager. Took 0.490026950836 seconds
Traceback (most recent call last):
File "/usr/local/bin/cwltoil", line 9, in <module>
load_entry_point('toil==3.5.0a1', 'console_scripts', 'cwltoil')()
File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/cwl/cwltoil.py", line 665, in main
outobj = toil.start(wf1)
File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/common.py", line 580, in start
return self._runMainLoop(job)
File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/common.py", line 794, in _runMainLoop
jobCache=self._jobCache)
File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/leader.py", line 692, in mainLoop
innerLoop(jobStore, config, batchSystem, toilState, jobBatcher, serviceManager, statsAndLogging)
File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/leader.py", line 970, in innerLoop
assert toilState.successorCounts == {}
AssertionError

# Fixed code
def processTotallyFailedJob(self, jobWrapper):
    """
    Processes a totally failed job.
    """
    # Mark job as a totally failed job
    self.toilState.totalFailedJobs.add(jobWrapper.jobStoreID)

    if jobWrapper.jobStoreID in self.toilState.serviceJobStoreIDToPredecessorJob:  # Is
        # a service job
        logger.debug(
            "Service job is being processed as a totally failed job: %s"
            % jobWrapper.jobStoreID
        )

        predecessorJobWrapper = self.toilState.serviceJobStoreIDToPredecessorJob[
            jobWrapper.jobStoreID
        ]

        # This removes the service job as a service of the predecessor
        # and potentially makes the predecessor active
        self._updatePredecessorStatus(jobWrapper.jobStoreID)

        # Remove the start flag, if it still exists. This indicates
        # to the service manager that the job has "started", this prevents
        # the service manager from deadlocking while waiting
        self.jobStore.deleteFile(jobWrapper.startJobStoreID)

        # Signal to any other services in the group that they should
        # terminate. We do this to prevent other services in the set
        # of services from deadlocking waiting for this service to start properly
        if predecessorJobWrapper.jobStoreID in self.toilState.servicesIssued:
            self.serviceManager.killServices(
                self.toilState.servicesIssued[predecessorJobWrapper.jobStoreID],
                error=True,
            )
            logger.debug(
                "Job: %s is instructing all the services of its parent job to quit",
                jobWrapper.jobStoreID,
            )

        self.toilState.hasFailedSuccessors.add(
            predecessorJobWrapper.jobStoreID
        )  # This ensures that the
        # job will not attempt to run any of it's successors on the stack
    else:
        # Is a non-service job
        assert jobWrapper.jobStoreID not in self.toilState.servicesIssued

        # Traverse failed job's successor graph and get the jobStoreID of new successors.
        # Any successor already in toilState.failedSuccessors will not be traversed
        # All successors traversed will be added to toilState.failedSuccessors and returned
        # as a set (unseenSuccessors).
        unseenSuccessors = self.getSuccessors(
            jobWrapper, self.toilState.failedSuccessors, self.jobStore
        )
        logger.debug(
            "Found new failed successors: %s of job: %s"
            % (" ".join(unseenSuccessors), jobWrapper.jobStoreID)
        )

        # For each newly found successor
        for successorJobStoreID in unseenSuccessors:
            # If the successor is a successor of other jobs that have already tried to schedule it
            if (
                successorJobStoreID
                in self.toilState.successorJobStoreIDToPredecessorJobs
            ):
                # For each such predecessor job
                # (we remove the successor from toilState.successorJobStoreIDToPredecessorJobs to avoid doing
                # this multiple times for each failed predecessor)
                for (
                    predecessorJob
                ) in self.toilState.successorJobStoreIDToPredecessorJobs.pop(
                    successorJobStoreID
                ):
                    # Reduce the predecessor job's successor count.
                    self.toilState.successorCounts[predecessorJob.jobStoreID] -= 1

                    # Indicate that it has failed jobs.
                    self.toilState.hasFailedSuccessors.add(predecessorJob.jobStoreID)
                    logger.debug(
                        "Marking job: %s as having failed successors (found by reading successors failed job)"
                        % predecessorJob.jobStoreID
                    )

                    # If the predecessor has no remaining successors, add to list of active jobs
                    assert (
                        self.toilState.successorCounts[predecessorJob.jobStoreID] >= 0
                    )
                    if self.toilState.successorCounts[predecessorJob.jobStoreID] == 0:
                        self.toilState.updatedJobs.add((predecessorJob, 0))

                        # Remove the predecessor job from the set of jobs with successors.
                        self.toilState.successorCounts.pop(predecessorJob.jobStoreID)

        # If the job has predecessor(s)
        if jobWrapper.jobStoreID in self.toilState.successorJobStoreIDToPredecessorJobs:
            # For each predecessor of the job
            for (
                predecessorJobWrapper
            ) in self.toilState.successorJobStoreIDToPredecessorJobs[
                jobWrapper.jobStoreID
            ]:
                # Mark the predecessor as failed
                self.toilState.hasFailedSuccessors.add(predecessorJobWrapper.jobStoreID)
                logger.debug(
                    "Totally failed job: %s is marking direct predecessor: %s as having failed jobs",
                    jobWrapper.jobStoreID,
                    predecessorJobWrapper.jobStoreID,
                )

            self._updatePredecessorStatus(jobWrapper.jobStoreID)