# Buggy code
def innerLoop(
    jobStore,
    config,
    batchSystem,
    toilState,
    jobBatcher,
    serviceManager,
    statsAndLogging,
):
    """
    :param toil.jobStores.abstractJobStore.AbstractJobStore jobStore:
    :param toil.common.Config config:
    :param toil.batchSystems.abstractBatchSystem.AbstractBatchSystem batchSystem:
    :param ToilState toilState:
    :param JobBatcher jobBatcher:
    :param ServiceManager serviceManager:
    :param StatsAndLogging statsAndLogging:
    """
    # Putting this in separate function for easier reading

    # Sets up the timing of the jobWrapper rescuing method
    timeSinceJobsLastRescued = time.time()

    logger.info("Starting the main loop")
    while True:
        # Process jobs that are ready to be scheduled/have successors to schedule
        if len(toilState.updatedJobs) > 0:
            logger.debug(
                "Built the jobs list, currently have %i jobs to update and %i jobs issued",
                len(toilState.updatedJobs),
                jobBatcher.getNumberOfJobsIssued(),
            )

            updatedJobs = toilState.updatedJobs  # The updated jobs to consider below
            toilState.updatedJobs = set()  # Resetting the list for the next set

            for jobWrapper, resultStatus in updatedJobs:
                logger.debug(
                    "Updating status of job: %s with result status: %s",
                    jobWrapper.jobStoreID,
                    resultStatus,
                )

                # This stops a job with services being issued by the serviceManager from
                # being considered further in this loop. This catch is necessary because
                # the job's service's can fail while being issued, causing the job to be
                # added to updated jobs.
                if jobWrapper in serviceManager.jobWrappersWithServicesBeingStarted:
                    logger.debug(
                        "Got a job to update which is still owned by the service "
                        "manager: %s",
                        jobWrapper.jobStoreID,
                    )
                    continue

                # If some of the jobs successors failed then either fail the job
                # or restart it if it has retries left and is a checkpoint job
                if jobWrapper.jobStoreID in toilState.hasFailedSuccessors:
                    # If the job has services running, signal for them to be killed
                    # once they are killed then the jobWrapper will be re-added to the
                    # updatedJobs set and then scheduled to be removed
                    if jobWrapper.jobStoreID in toilState.servicesIssued:
                        logger.debug(
                            "Telling job: %s to terminate its services due to successor failure",
                            jobWrapper.jobStoreID,
                        )
                        serviceManager.killServices(
                            toilState.servicesIssued[jobWrapper.jobStoreID], error=True
                        )

                    # If the job has non-service jobs running wait for them to finish
                    # the job will be re-added to the updated jobs when these jobs are done
                    elif jobWrapper.jobStoreID in toilState.successorCounts:
                        logger.debug(
                            "Job: %s with failed successors still has successor jobs running",
                            jobWrapper.jobStoreID,
                        )
                        continue

                    # If the job is a checkpoint and has remaining retries then reissue it.
                    elif (
                        jobWrapper.checkpoint is not None
                        and jobWrapper.remainingRetryCount > 0
                    ):
                        logger.warn(
                            "Job: %s is being restarted as a checkpoint after the total "
                            "failure of jobs in its subtree.",
                            jobWrapper.jobStoreID,
                        )
                        jobBatcher.issueJob(
                            jobWrapper.jobStoreID,
                            memory=jobWrapper.memory,
                            cores=jobWrapper.cores,
                            disk=jobWrapper.disk,
                            preemptable=jobWrapper.preemptable,
                        )
                    else:  # Mark it totally failed
                        logger.debug(
                            "Job %s is being processed as completely failed",
                            jobWrapper.jobStoreID,
                        )
                        jobBatcher.processTotallyFailedJob(jobWrapper)

                # If the jobWrapper has a command it must be run before any successors.
                # Similarly, if the job previously failed we rerun it, even if it doesn't have a
                # command to run, to eliminate any parts of the stack now completed.
                elif jobWrapper.command is not None or resultStatus != 0:
                    isServiceJob = (
                        jobWrapper.jobStoreID
                        in toilState.serviceJobStoreIDToPredecessorJob
                    )

                    # If the job has run out of retries or is a service job whose error flag has
                    # been indicated, fail the job.
                    if (
                        jobWrapper.remainingRetryCount == 0
                        or isServiceJob
                        and not jobStore.fileExists(jobWrapper.errorJobStoreID)
                    ):
                        jobBatcher.processTotallyFailedJob(jobWrapper)
                        logger.warn(
                            "Job: %s is completely failed", jobWrapper.jobStoreID
                        )
                    else:
                        # Otherwise try the job again
                        jobBatcher.issueJob(
                            jobWrapper.jobStoreID,
                            jobWrapper.memory,
                            jobWrapper.cores,
                            jobWrapper.disk,
                            jobWrapper.preemptable,
                        )

                # If the job has services to run, which have not been started, start them
                elif len(jobWrapper.services) > 0:
                    # Build a map from the service jobs to the job and a map
                    # of the services created for the job
                    assert jobWrapper.jobStoreID not in toilState.servicesIssued
                    toilState.servicesIssued[jobWrapper.jobStoreID] = {}
                    for serviceJobList in jobWrapper.services:
                        for serviceTuple in serviceJobList:
                            serviceID = serviceTuple[0]
                            assert (
                                serviceID
                                not in toilState.serviceJobStoreIDToPredecessorJob
                            )
                            toilState.serviceJobStoreIDToPredecessorJob[serviceID] = (
                                jobWrapper
                            )
                            toilState.servicesIssued[jobWrapper.jobStoreID][
                                serviceID
                            ] = serviceTuple[4:7]

                    # Use the service manager to start the services
                    serviceManager.scheduleServices(jobWrapper)

                    logger.debug(
                        "Giving job: %s to service manager to schedule its jobs",
                        jobWrapper.jobStoreID,
                    )

                # There exist successors to run
                elif len(jobWrapper.stack) > 0:
                    assert len(jobWrapper.stack[-1]) > 0
                    logger.debug(
                        "Job: %s has %i successors to schedule",
                        jobWrapper.jobStoreID,
                        len(jobWrapper.stack[-1]),
                    )
                    # Record the number of successors that must be completed before
                    # the jobWrapper can be considered again
                    assert jobWrapper.jobStoreID not in toilState.successorCounts
                    toilState.successorCounts[jobWrapper.jobStoreID] = len(
                        jobWrapper.stack[-1]
                    )
                    # List of successors to schedule
                    successors = []
                    # For each successor schedule if all predecessors have been completed
                    for (
                        successorJobStoreID,
                        memory,
                        cores,
                        disk,
                        preemptable,
                        predecessorID,
                    ) in jobWrapper.stack[-1]:
                        # Build map from successor to predecessors.
                        if (
                            successorJobStoreID
                            not in toilState.successorJobStoreIDToPredecessorJobs
                        ):
                            toilState.successorJobStoreIDToPredecessorJobs[
                                successorJobStoreID
                            ] = []
                        toilState.successorJobStoreIDToPredecessorJobs[
                            successorJobStoreID
                        ].append(jobWrapper)
                        # Case that the jobWrapper has multiple predecessors
                        if predecessorID is not None:
                            # Load the wrapped jobWrapper
                            job2 = jobStore.load(successorJobStoreID)
                            # Remove the predecessor from the list of predecessors
                            job2.predecessorsFinished.add(predecessorID)
                            # Checkpoint
                            jobStore.update(job2)
                            # If the jobs predecessors have all not all completed then
                            # ignore the jobWrapper
                            assert len(job2.predecessorsFinished) >= 1
                            assert (
                                len(job2.predecessorsFinished) <= job2.predecessorNumber
                            )
                            if len(job2.predecessorsFinished) < job2.predecessorNumber:
                                continue
                        successors.append(
                            (successorJobStoreID, memory, cores, disk, preemptable)
                        )
                    jobBatcher.issueJobs(successors)

                elif jobWrapper.jobStoreID in toilState.servicesIssued:
                    logger.debug(
                        "Telling job: %s to terminate its due to the successful completion of its successor jobs",
                        jobWrapper.jobStoreID,
                    )
                    serviceManager.killServices(
                        toilState.servicesIssued[jobWrapper.jobStoreID], error=False
                    )

                # There are no remaining tasks to schedule within the jobWrapper, but
                # we schedule it anyway to allow it to be deleted.

                # TODO: An alternative would be simple delete it here and add it to the
                # list of jobs to process, or (better) to create an asynchronous
                # process that deletes jobs and then feeds them back into the set
                # of jobs to be processed
                else:
                    # Remove the job
                    if jobWrapper.remainingRetryCount > 0:
                        jobBatcher.issueJob(
                            jobWrapper.jobStoreID,
                            memory=config.defaultMemory,
                            cores=config.defaultCores,
                            disk=config.defaultDisk,
                            # We allow this cleanup to potentially occur on a
                            # preemptable instance.
                            preemptable=True,
                        )
                        logger.debug(
                            "Job: %s is empty, we are scheduling to clean it up",
                            jobWrapper.jobStoreID,
                        )
                    else:
                        jobBatcher.processTotallyFailedJob(jobWrapper)
                        logger.warn(
                            "Job: %s is empty but completely failed - something is very wrong",
                            jobWrapper.jobStoreID,
                        )

        # The exit criterion
        if (
            len(toilState.updatedJobs) == 0
            and jobBatcher.getNumberOfJobsIssued() == 0
            and serviceManager.serviceJobsIssuedToServiceManager == 0
        ):
            logger.info("No jobs left to run so exiting.")
            break

        # Start any service jobs available from the service manager
        while True:
            serviceJobTuple = serviceManager.getServiceJobsToStart(0)
            # Stop trying to get jobs when function returns None
            if serviceJobTuple is None:
                break
            serviceJobStoreID, memory, cores, disk = serviceJobTuple
            logger.debug("Launching service job: %s", serviceJobStoreID)
            # This loop issues the jobs to the batch system because the batch system is not
            # thread-safe. FIXME: don't understand this comment
            jobBatcher.issueJob(serviceJobStoreID, memory, cores, disk, False)

        # Get jobs whose services have started
        while True:
            jobWrapper = serviceManager.getJobWrapperWhoseServicesAreRunning(0)
            if jobWrapper is None:  # Stop trying to get jobs when function returns None
                break
            logger.debug("Job: %s has established its services.", jobWrapper.jobStoreID)
            jobWrapper.services = []
            toilState.updatedJobs.add((jobWrapper, 0))

        # Gather any new, updated jobWrapper from the batch system
        updatedJob = batchSystem.getUpdatedBatchJob(2)
        if updatedJob is not None:
            jobBatchSystemID, result, wallTime = updatedJob
            if jobBatcher.hasJob(jobBatchSystemID):
                if result == 0:
                    logger.debug(
                        "Batch system is reporting that the jobWrapper with "
                        "batch system ID: %s and jobWrapper store ID: %s ended successfully",
                        jobBatchSystemID,
                        jobBatcher.getJob(jobBatchSystemID),
                    )
                else:
                    logger.warn(
                        "Batch system is reporting that the jobWrapper with "
                        "batch system ID: %s and jobWrapper store ID: %s failed with exit value %i",
                        jobBatchSystemID,
                        jobBatcher.getJob(jobBatchSystemID),
                        result,
                    )
                jobBatcher.processFinishedJob(
                    jobBatchSystemID, result, wallTime=wallTime
                )
            else:
                logger.warn(
                    "A result seems to already have been processed "
                    "for jobWrapper with batch system ID: %i",
                    jobBatchSystemID,
                )
        else:
            # Process jobs that have gone awry

            # In the case that there is nothing happening
            # (no updated jobWrapper to gather for 10 seconds)
            # check if their are any jobs that have run too long
            # (see JobBatcher.reissueOverLongJobs) or which
            # have gone missing from the batch system (see JobBatcher.reissueMissingJobs)
            if (
                time.time() - timeSinceJobsLastRescued >= config.rescueJobsFrequency
            ):  # We only
                # rescue jobs every N seconds, and when we have
                # apparently exhausted the current jobWrapper supply
                jobBatcher.reissueOverLongJobs()
                logger.info("Reissued any over long jobs")

                hasNoMissingJobs = jobBatcher.reissueMissingJobs()
                if hasNoMissingJobs:
                    timeSinceJobsLastRescued = time.time()
                else:
                    timeSinceJobsLastRescued += 60  # This means we'll try again
                    # in a minute, providing things are quiet
                logger.info("Rescued any (long) missing jobs")

        # Check on the associated threads and exit if a failure is detected
        statsAndLogging.check()
        serviceManager.check()
        # the cluster scaler object will only be instantiated if autoscaling is enabled
        if jobBatcher.clusterScaler is not None:
            jobBatcher.clusterScaler.check()

    logger.info("Finished the main loop")

    # Consistency check the toil state
    assert toilState.updatedJobs == set()
    assert toilState.successorCounts == {}
    assert toilState.successorJobStoreIDToPredecessorJobs == {}
    assert toilState.serviceJobStoreIDToPredecessorJob == {}
    assert toilState.servicesIssued == {}


# Traceback (AssertionError)
kunkel 2016-08-26 11:18:31,313 MainThread WARNING toil.leader: The jobWrapper seems to have left a log file, indicating failure: B/U/jobWBMw6I
kunkel 2016-08-26 11:18:31,313 MainThread WARNING toil.leader: Reporting file: B/U/jobWBMw6I
kunkel 2016-08-26 11:18:31,313 MainThread WARNING toil.leader: B/U/jobWBMw6I:   ---TOIL WORKER OUTPUT LOG---
kunkel 2016-08-26 11:18:31,313 MainThread WARNING toil.leader: B/U/jobWBMw6I:   INFO:rdflib:RDFLib Version: 4.2.1
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   INFO:toil.fileStore:CACHE: Read file with ID 'a/2/tmp2RYy7z.tmp' from the cache.
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   [job revtool.cwl] /tmp/user/1001/tmpZ7hLZ6$ rev-fail \
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:       /tmp/user/1001/tmppbMf7F/stg5c0b758e-c348-411b-b260-877b9491c6da/whale.txt > /tmp/user/1001/tmpZ7hLZ6/output.txt
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   INFO:cwltool:[job revtool.cwl] /tmp/user/1001/tmpZ7hLZ6$ rev-fail \
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:       /tmp/user/1001/tmppbMf7F/stg5c0b758e-c348-411b-b260-877b9491c6da/whale.txt > /tmp/user/1001/tmpZ7hLZ6/output.txt
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   'rev-fail' not found
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   ERROR:cwltool:'rev-fail' not found
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   [job revtool.cwl] completed permanentFail
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   WARNING:cwltool:[job revtool.cwl] completed permanentFail
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   Final process status is permanentFail
kunkel 2016-08-26 11:18:31,314 MainThread WARNING toil.leader: B/U/jobWBMw6I:   WARNING:cwltool:Final process status is permanentFail
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:   INFO:toil.fileStore:LOG-TO-MASTER: Successfully deleted local copies of file with ID 'a/2/tmp2RYy7z.tmp'.
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:   Traceback (most recent call last):
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:     File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/worker.py", line 331, in main
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:       job._runner(jobWrapper=jobWrapper, jobStore=jobStore, fileStore=fileStore)
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:     File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/job.py", line 1085, in _runner
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:       returnValues = self._run(jobWrapper, fileStore)
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:     File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/job.py", line 1033, in _run
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:       return self.run(fileStore)
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:     File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/cwl/cwltoil.py", line 256, in run
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:       **opts)
kunkel 2016-08-26 11:18:31,315 MainThread WARNING toil.leader: B/U/jobWBMw6I:     File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/cwltool/main.py", line 218, in single_job_executor
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:       raise WorkflowException(u"Process status is %s" % (final_status))
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:   WorkflowException: Process status is ['permanentFail']
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:   Exiting the worker because of a failed jobWrapper on host kunkel
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:   ERROR:toil.worker:Exiting the worker because of a failed jobWrapper on host kunkel
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:   WARNING:toil.jobWrapper:Due to failure we are reducing the remaining retry count of job B/U/jobWBMw6I to 0
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: B/U/jobWBMw6I:   WARNING:toil.jobWrapper:We have increased the default memory of the failed job to 2147483648 bytes
kunkel 2016-08-26 11:18:31,316 MainThread WARNING toil.leader: Job: B/U/jobWBMw6I is completely failed
kunkel 2016-08-26 11:18:31,316 MainThread INFO toil.leader: No jobs left to run so exiting.
kunkel 2016-08-26 11:18:31,316 MainThread INFO toil.leader: Finished the main loop
kunkel 2016-08-26 11:18:31,316 MainThread INFO toil.leader: Waiting for stats and logging collator thread to finish ...
kunkel 2016-08-26 11:18:31,638 MainThread INFO toil.leader: ... finished collating stats and logs. Took 0.321724891663 seconds
kunkel 2016-08-26 11:18:31,639 MainThread INFO toil.leader: Waiting for service manager thread to finish ...
kunkel 2016-08-26 11:18:32,129 MainThread INFO toil.leader: ... finished shutting down the service manager. Took 0.490026950836 seconds
Traceback (most recent call last):
File "/usr/local/bin/cwltoil", line 9, in <module>
load_entry_point('toil==3.5.0a1', 'console_scripts', 'cwltoil')()
File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/cwl/cwltoil.py", line 665, in main
outobj = toil.start(wf1)
File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/common.py", line 580, in start
return self._runMainLoop(job)
File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/common.py", line 794, in _runMainLoop
jobCache=self._jobCache)
File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/leader.py", line 692, in mainLoop
innerLoop(jobStore, config, batchSystem, toilState, jobBatcher, serviceManager, statsAndLogging)
File "/home/chapmanb/install/bcbio-vm/anaconda/lib/python2.7/site-packages/toil/leader.py", line 970, in innerLoop
assert toilState.successorCounts == {}
AssertionError

# Fixed code
def innerLoop(
    jobStore,
    config,
    batchSystem,
    toilState,
    jobBatcher,
    serviceManager,
    statsAndLogging,
):
    """
    :param toil.jobStores.abstractJobStore.AbstractJobStore jobStore:
    :param toil.common.Config config:
    :param toil.batchSystems.abstractBatchSystem.AbstractBatchSystem batchSystem:
    :param ToilState toilState:
    :param JobBatcher jobBatcher:
    :param ServiceManager serviceManager:
    :param StatsAndLogging statsAndLogging:
    """
    # Putting this in separate function for easier reading

    # Sets up the timing of the jobWrapper rescuing method
    timeSinceJobsLastRescued = time.time()

    logger.info("Starting the main loop")
    while True:
        # Process jobs that are ready to be scheduled/have successors to schedule
        if len(toilState.updatedJobs) > 0:
            logger.debug(
                "Built the jobs list, currently have %i jobs to update and %i jobs issued",
                len(toilState.updatedJobs),
                jobBatcher.getNumberOfJobsIssued(),
            )

            updatedJobs = toilState.updatedJobs  # The updated jobs to consider below
            toilState.updatedJobs = set()  # Resetting the list for the next set

            for jobWrapper, resultStatus in updatedJobs:
                logger.debug(
                    "Updating status of job: %s with result status: %s",
                    jobWrapper.jobStoreID,
                    resultStatus,
                )

                # This stops a job with services being issued by the serviceManager from
                # being considered further in this loop. This catch is necessary because
                # the job's service's can fail while being issued, causing the job to be
                # added to updated jobs.
                if jobWrapper in serviceManager.jobWrappersWithServicesBeingStarted:
                    logger.debug(
                        "Got a job to update which is still owned by the service "
                        "manager: %s",
                        jobWrapper.jobStoreID,
                    )
                    continue

                # If some of the jobs successors failed then either fail the job
                # or restart it if it has retries left and is a checkpoint job
                if jobWrapper.jobStoreID in toilState.hasFailedSuccessors:
                    # If the job has services running, signal for them to be killed
                    # once they are killed then the jobWrapper will be re-added to the
                    # updatedJobs set and then scheduled to be removed
                    if jobWrapper.jobStoreID in toilState.servicesIssued:
                        logger.debug(
                            "Telling job: %s to terminate its services due to successor failure",
                            jobWrapper.jobStoreID,
                        )
                        serviceManager.killServices(
                            toilState.servicesIssued[jobWrapper.jobStoreID], error=True
                        )

                    # If the job has non-service jobs running wait for them to finish
                    # the job will be re-added to the updated jobs when these jobs are done
                    elif jobWrapper.jobStoreID in toilState.successorCounts:
                        logger.debug(
                            "Job: %s with failed successors still has successor jobs running",
                            jobWrapper.jobStoreID,
                        )
                        continue

                    # If the job is a checkpoint and has remaining retries then reissue it.
                    elif (
                        jobWrapper.checkpoint is not None
                        and jobWrapper.remainingRetryCount > 0
                    ):
                        logger.warn(
                            "Job: %s is being restarted as a checkpoint after the total "
                            "failure of jobs in its subtree.",
                            jobWrapper.jobStoreID,
                        )
                        jobBatcher.issueJob(
                            jobWrapper.jobStoreID,
                            memory=jobWrapper.memory,
                            cores=jobWrapper.cores,
                            disk=jobWrapper.disk,
                            preemptable=jobWrapper.preemptable,
                        )

                    else:  # Mark it totally failed
                        logger.debug(
                            "Job %s is being processed as completely failed",
                            jobWrapper.jobStoreID,
                        )
                        jobBatcher.processTotallyFailedJob(jobWrapper)

                # If the jobWrapper has a command it must be run before any successors.
                # Similarly, if the job previously failed we rerun it, even if it doesn't have a
                # command to run, to eliminate any parts of the stack now completed.
                elif jobWrapper.command is not None or resultStatus != 0:
                    isServiceJob = (
                        jobWrapper.jobStoreID
                        in toilState.serviceJobStoreIDToPredecessorJob
                    )

                    # If the job has run out of retries or is a service job whose error flag has
                    # been indicated, fail the job.
                    if (
                        jobWrapper.remainingRetryCount == 0
                        or isServiceJob
                        and not jobStore.fileExists(jobWrapper.errorJobStoreID)
                    ):
                        jobBatcher.processTotallyFailedJob(jobWrapper)
                        logger.warn(
                            "Job: %s is completely failed", jobWrapper.jobStoreID
                        )
                    else:
                        # Otherwise try the job again
                        jobBatcher.issueJob(
                            jobWrapper.jobStoreID,
                            jobWrapper.memory,
                            jobWrapper.cores,
                            jobWrapper.disk,
                            jobWrapper.preemptable,
                        )

                # If the job has services to run, which have not been started, start them
                elif len(jobWrapper.services) > 0:
                    # Build a map from the service jobs to the job and a map
                    # of the services created for the job
                    assert jobWrapper.jobStoreID not in toilState.servicesIssued
                    toilState.servicesIssued[jobWrapper.jobStoreID] = {}
                    for serviceJobList in jobWrapper.services:
                        for serviceTuple in serviceJobList:
                            serviceID = serviceTuple[0]
                            assert (
                                serviceID
                                not in toilState.serviceJobStoreIDToPredecessorJob
                            )
                            toilState.serviceJobStoreIDToPredecessorJob[serviceID] = (
                                jobWrapper
                            )
                            toilState.servicesIssued[jobWrapper.jobStoreID][
                                serviceID
                            ] = serviceTuple[4:7]

                    # Use the service manager to start the services
                    serviceManager.scheduleServices(jobWrapper)

                    logger.debug(
                        "Giving job: %s to service manager to schedule its jobs",
                        jobWrapper.jobStoreID,
                    )

                # There exist successors to run
                elif len(jobWrapper.stack) > 0:
                    assert len(jobWrapper.stack[-1]) > 0
                    logger.debug(
                        "Job: %s has %i successors to schedule",
                        jobWrapper.jobStoreID,
                        len(jobWrapper.stack[-1]),
                    )

                    # Record the number of successors that must be completed before
                    # the jobWrapper can be considered again
                    assert jobWrapper.jobStoreID not in toilState.successorCounts
                    toilState.successorCounts[jobWrapper.jobStoreID] = len(
                        jobWrapper.stack[-1]
                    )

                    # List of successors to schedule
                    successors = []

                    # For each successor schedule if all predecessors have been completed
                    for (
                        successorJobStoreID,
                        memory,
                        cores,
                        disk,
                        preemptable,
                        predecessorID,
                    ) in jobWrapper.stack[-1]:
                        # Build map from successor to predecessors.
                        if (
                            successorJobStoreID
                            not in toilState.successorJobStoreIDToPredecessorJobs
                        ):
                            toilState.successorJobStoreIDToPredecessorJobs[
                                successorJobStoreID
                            ] = []
                        toilState.successorJobStoreIDToPredecessorJobs[
                            successorJobStoreID
                        ].append(jobWrapper)
                        # Case that the jobWrapper has multiple predecessors
                        if predecessorID is not None:
                            logger.debug(
                                "Successor job: %s of job: %s has multiple predecessors"
                                % (successorJobStoreID, jobWrapper.jobStoreID)
                            )

                            # Get the successor job, using a cache
                            # (if the successor job has already been seen it will be in this cache,
                            # but otherwise put it in the cache)
                            if (
                                successorJobStoreID
                                not in toilState.jobsToBeScheduledWithMultiplePredecessors
                            ):
                                toilState.jobsToBeScheduledWithMultiplePredecessors[
                                    successorJobStoreID
                                ] = jobStore.load(successorJobStoreID)
                            successorJobWrapper = (
                                toilState.jobsToBeScheduledWithMultiplePredecessors[
                                    successorJobStoreID
                                ]
                            )

                            # Add the jobWrapper job as a finished predecessor to the successor
                            successorJobWrapper.predecessorsFinished.add(predecessorID)

                            # If the successor is in the set of successors of failed jobs
                            if successorJobStoreID in toilState.failedSuccessors:
                                logger.debug(
                                    "Successor job: %s of job: %s has failed predecessors"
                                    % (successorJobStoreID, jobWrapper.jobStoreID)
                                )

                                # Add the job to the set having failed successors
                                toilState.hasFailedSuccessors.add(jobWrapper.jobStoreID)

                                # Reduce active successor count and remove the successor as an active successor of the job
                                toilState.successorCounts[jobWrapper.jobStoreID] -= 1
                                assert (
                                    toilState.successorCounts[jobWrapper.jobStoreID]
                                    >= 0
                                )
                                toilState.successorJobStoreIDToPredecessorJobs[
                                    successorJobStoreID
                                ].remove(jobWrapper)
                                if (
                                    len(
                                        toilState.successorJobStoreIDToPredecessorJobs[
                                            successorJobStoreID
                                        ]
                                    )
                                    == 0
                                ):
                                    toilState.successorJobStoreIDToPredecessorJobs.pop(
                                        successorJobStoreID
                                    )

                                # If the job now has no active successors add to active jobs
                                # so it can be processed as a job with failed successors
                                if (
                                    toilState.successorCounts[jobWrapper.jobStoreID]
                                    == 0
                                ):
                                    logger.debug(
                                        "Job: %s has no successors to run and some are failed, adding to list of jobs with failed successors"
                                        % jobWrapper.jobStoreID
                                    )
                                    toilState.successorCounts.pop(jobWrapper.jobStoreID)
                                    toilState.updatedJobs.add((jobWrapper, 0))
                                    continue

                            # If the successor job's predecessors have all not all completed then
                            # ignore the jobWrapper as is not yet ready to run
                            assert (
                                len(successorJobWrapper.predecessorsFinished)
                                <= successorJobWrapper.predecessorNumber
                            )
                            if (
                                len(successorJobWrapper.predecessorsFinished)
                                < successorJobWrapper.predecessorNumber
                            ):
                                continue
                            else:
                                # Remove the successor job from the cache
                                toilState.jobsToBeScheduledWithMultiplePredecessors.pop(
                                    successorJobStoreID
                                )

                        # Add successor to list of successors to schedule
                        successors.append(
                            (successorJobStoreID, memory, cores, disk, preemptable)
                        )
                    jobBatcher.issueJobs(successors)

                elif jobWrapper.jobStoreID in toilState.servicesIssued:
                    logger.debug(
                        "Telling job: %s to terminate its services due to the successful completion of its successor jobs",
                        jobWrapper.jobStoreID,
                    )
                    serviceManager.killServices(
                        toilState.servicesIssued[jobWrapper.jobStoreID], error=False
                    )

                # There are no remaining tasks to schedule within the jobWrapper, but
                # we schedule it anyway to allow it to be deleted.

                # TODO: An alternative would be simple delete it here and add it to the
                # list of jobs to process, or (better) to create an asynchronous
                # process that deletes jobs and then feeds them back into the set
                # of jobs to be processed
                else:
                    # Remove the job
                    if jobWrapper.remainingRetryCount > 0:
                        jobBatcher.issueJob(
                            jobWrapper.jobStoreID,
                            memory=config.defaultMemory,
                            cores=config.defaultCores,
                            disk=config.defaultDisk,
                            # We allow this cleanup to potentially occur on a
                            # preemptable instance.
                            preemptable=True,
                        )
                        logger.debug(
                            "Job: %s is empty, we are scheduling to clean it up",
                            jobWrapper.jobStoreID,
                        )
                    else:
                        jobBatcher.processTotallyFailedJob(jobWrapper)
                        logger.warn(
                            "Job: %s is empty but completely failed - something is very wrong",
                            jobWrapper.jobStoreID,
                        )

        # The exit criterion
        if (
            len(toilState.updatedJobs) == 0
            and jobBatcher.getNumberOfJobsIssued() == 0
            and serviceManager.serviceJobsIssuedToServiceManager == 0
        ):
            logger.info("No jobs left to run so exiting.")
            break

        # Start any service jobs available from the service manager
        while True:
            serviceJobTuple = serviceManager.getServiceJobsToStart(0)
            # Stop trying to get jobs when function returns None
            if serviceJobTuple is None:
                break
            serviceJobStoreID, memory, cores, disk = serviceJobTuple
            logger.debug("Launching service job: %s", serviceJobStoreID)
            # This loop issues the jobs to the batch system because the batch system is not
            # thread-safe. FIXME: don't understand this comment
            jobBatcher.issueJob(serviceJobStoreID, memory, cores, disk, False)

        # Get jobs whose services have started
        while True:
            jobWrapper = serviceManager.getJobWrapperWhoseServicesAreRunning(0)
            if jobWrapper is None:  # Stop trying to get jobs when function returns None
                break
            logger.debug("Job: %s has established its services.", jobWrapper.jobStoreID)
            jobWrapper.services = []
            toilState.updatedJobs.add((jobWrapper, 0))

        # Gather any new, updated jobWrapper from the batch system
        updatedJob = batchSystem.getUpdatedBatchJob(2)
        if updatedJob is not None:
            jobBatchSystemID, result, wallTime = updatedJob
            if jobBatcher.hasJob(jobBatchSystemID):
                if result == 0:
                    logger.debug(
                        "Batch system is reporting that the jobWrapper with "
                        "batch system ID: %s and jobWrapper store ID: %s ended successfully",
                        jobBatchSystemID,
                        jobBatcher.getJob(jobBatchSystemID),
                    )
                else:
                    logger.warn(
                        "Batch system is reporting that the jobWrapper with "
                        "batch system ID: %s and jobWrapper store ID: %s failed with exit value %i",
                        jobBatchSystemID,
                        jobBatcher.getJob(jobBatchSystemID),
                        result,
                    )
                jobBatcher.processFinishedJob(
                    jobBatchSystemID, result, wallTime=wallTime
                )
            else:
                logger.warn(
                    "A result seems to already have been processed "
                    "for jobWrapper with batch system ID: %i",
                    jobBatchSystemID,
                )
        else:
            # Process jobs that have gone awry

            # In the case that there is nothing happening
            # (no updated jobWrapper to gather for 10 seconds)
            # check if their are any jobs that have run too long
            # (see JobBatcher.reissueOverLongJobs) or which
            # have gone missing from the batch system (see JobBatcher.reissueMissingJobs)
            if (
                time.time() - timeSinceJobsLastRescued >= config.rescueJobsFrequency
            ):  # We only
                # rescue jobs every N seconds, and when we have
                # apparently exhausted the current jobWrapper supply
                jobBatcher.reissueOverLongJobs()
                logger.info("Reissued any over long jobs")

                hasNoMissingJobs = jobBatcher.reissueMissingJobs()
                if hasNoMissingJobs:
                    timeSinceJobsLastRescued = time.time()
                else:
                    timeSinceJobsLastRescued += 60  # This means we'll try again
                    # in a minute, providing things are quiet
                logger.info("Rescued any (long) missing jobs")

        # Check on the associated threads and exit if a failure is detected
        statsAndLogging.check()
        serviceManager.check()
        # the cluster scaler object will only be instantiated if autoscaling is enabled
        if jobBatcher.clusterScaler is not None:
            jobBatcher.clusterScaler.check()

    logger.info("Finished the main loop")

    # Consistency check the toil state
    assert toilState.updatedJobs == set()
    assert toilState.successorCounts == {}
    assert toilState.successorJobStoreIDToPredecessorJobs == {}
    assert toilState.serviceJobStoreIDToPredecessorJob == {}
    assert toilState.servicesIssued == {}