Repository: DataBiosphere/toil

Issue:
Leader.innerloop is very complex
It requires huge amounts of work to understand the monolithic and heavily nested innerloop function, with difficult to follow flow, implicit gotos (ie. continues), and cases were the rationale is not clear.

Very 1970s


Discussion:

Resolving patch:
diff --git a/src/toil/leader.py b/src/toil/leader.py
--- a/src/toil/leader.py
+++ b/src/toil/leader.py
@@ -53,6 +53,23 @@
 
 logger = logging.getLogger( __name__ )
 
+###############################################################################
+# Implementation Notes
+#
+# Multiple predecessors:
+#   There is special-case handling for jobs with multiple predecessors as a
+#   performance optimization. This minimize number of expensive loads of
+#   jobGraphs from jobStores.  However, this special case could be unnecessary.
+#   The jobGraph is loaded to update predecessorsFinished, in
+#   _checkSuccssorReadyToRunMultiplePredecessors, however it doesn't appear to
+#   write the jobGraph back the jobStore.  Thus predecessorsFinished may really
+#   be leader state and could moved out of the jobGraph.  This would make this
+#   special-cases handling unnecessary and simplify the leader.
+#   Issue #2136
+###############################################################################
+
+
+
 ####################################################
 # Exception thrown by the Leader class when one or more jobs fails
 ####################################################
@@ -136,6 +153,9 @@ def __init__(self, config, batchSystem, provisioner, jobStore, rootJob, jobCache
         self.preemptableServiceJobsIssued = 0
         self.preemptableServiceJobsToBeIssued = []
 
+        # Timing of the jobGraph rescuing method
+        self.timeSinceJobsLastRescued = None
+
         # Hash to store number of times a job is lost by the batch system,
         # used to decide if to reissue an apparently missing job
         self.reissueMissingJobs_missingHash = {}
@@ -143,7 +163,7 @@ def __init__(self, config, batchSystem, provisioner, jobStore, rootJob, jobCache
         # Class used to create/destroy nodes in the cluster, may be None if
         # using a statically defined cluster
         self.provisioner = provisioner
- 
+
         # Create cluster scaling thread if the provisioner is not None
         self.clusterScaler = None
         if self.provisioner is not None and len(self.provisioner.nodeTypes) > 0:
@@ -228,262 +248,290 @@ def run(self):
 
         return self.jobStore.getRootJobReturnValue()
 
+    def _handledFailedSuccessor(self, jobNode, jobGraph, successorJobStoreID):
+        """Deal with the successor having failed. Return True if there are
+        still active successors. Return False if all successors have failed
+        and the job is queued to run to handle the failed successors."""
+        logger.debug("Successor job: %s of job: %s has failed """
+                     "predecessors", jobNode, jobGraph)
+
+        # Add the job to the set having failed successors
+        self.toilState.hasFailedSuccessors.add(jobGraph.jobStoreID)
+
+        # Reduce active successor count and remove the successor as an active successor of the job
+        self.toilState.successorCounts[jobGraph.jobStoreID] -= 1
+        assert self.toilState.successorCounts[jobGraph.jobStoreID] >= 0
+        self.toilState.successorJobStoreIDToPredecessorJobs[successorJobStoreID].remove(jobGraph)
+        if len(self.toilState.successorJobStoreIDToPredecessorJobs[successorJobStoreID]) == 0:
+            self.toilState.successorJobStoreIDToPredecessorJobs.pop(successorJobStoreID)
+
+        # If the job now has no active successors add to active jobs
+        # so it can be processed as a job with failed successors
+        if self.toilState.successorCounts[jobGraph.jobStoreID] == 0:
+            logger.debug("Job: %s has no successors to run "
+                         "and some are failed, adding to list of jobs "
+                         "with failed successors", jobGraph)
+            self.toilState.successorCounts.pop(jobGraph.jobStoreID)
+            self.toilState.updatedJobs.add((jobGraph, 0))
+            return False
+
+
+    def _checkSuccssorReadyToRunMultiplePredecessors(self, jobGraph, jobNode, successorJobStoreID):
+        """Handle the special cases of checking if a successor job is
+        ready to run when there are multiple predecessors"""
+        # See implementation note at the top of this file for discussion of multiple predecessors
+        logger.debug("Successor job: %s of job: %s has multiple "
+                     "predecessors", jobNode, jobGraph)
+
+        # Get the successor job graph, which is caches
+        if successorJobStoreID not in self.toilState.jobsToBeScheduledWithMultiplePredecessors:
+            self.toilState.jobsToBeScheduledWithMultiplePredecessors[successorJobStoreID] = self.jobStore.load(successorJobStoreID)
+        successorJobGraph = self.toilState.jobsToBeScheduledWithMultiplePredecessors[successorJobStoreID]
+
+        # Add the jobGraph as a finished predecessor to the successor
+        successorJobGraph.predecessorsFinished.add(jobGraph.jobStoreID)
+
+        # If the successor is in the set of successors of failed jobs
+        if successorJobStoreID in self.toilState.failedSuccessors:
+            if not self._handledFailedSuccessor(jobNode, jobGraph, successorJobStoreID):
+                return False
+
+        # If the successor job's predecessors have all not all completed then
+        # ignore the jobGraph as is not yet ready to run
+        assert len(successorJobGraph.predecessorsFinished) <= successorJobGraph.predecessorNumber
+        if len(successorJobGraph.predecessorsFinished) < successorJobGraph.predecessorNumber:
+            return False
+        else:
+            # Remove the successor job from the cache
+            self.toilState.jobsToBeScheduledWithMultiplePredecessors.pop(successorJobStoreID)
+            return True
+
+    def _makeJobSuccssorReadyToRun(self, jobGraph, jobNode):
+        """make a successor job ready to run, returning False if they should
+        not yet be run"""
+        successorJobStoreID = jobNode.jobStoreID
+        #Build map from successor to predecessors.
+        if successorJobStoreID not in self.toilState.successorJobStoreIDToPredecessorJobs:
+            self.toilState.successorJobStoreIDToPredecessorJobs[successorJobStoreID] = []
+        self.toilState.successorJobStoreIDToPredecessorJobs[successorJobStoreID].append(jobGraph)
+
+        if jobNode.predecessorNumber > 1:
+            return self._checkSuccssorReadyToRunMultiplePredecessors(jobGraph, jobNode, successorJobStoreID)
+        else:
+            return True
+
+    def _runJobSuccessors(self, jobGraph):
+        assert len(jobGraph.stack[-1]) > 0
+        logger.debug("Job: %s has %i successors to schedule",
+                     jobGraph.jobStoreID, len(jobGraph.stack[-1]))
+        #Record the number of successors that must be completed before
+        #the jobGraph can be considered again
+        assert jobGraph.jobStoreID not in self.toilState.successorCounts
+        self.toilState.successorCounts[jobGraph.jobStoreID] = len(jobGraph.stack[-1])
+
+        # For each successor schedule if all predecessors have been completed
+        successors = []
+        for jobNode in jobGraph.stack[-1]:
+            if self._makeJobSuccssorReadyToRun(jobGraph, jobNode):
+                successors.append(jobNode)
+        self.issueJobs(successors)
+
+    def _processFailedSuccessors(self, jobGraph):
+        """Some of the jobs successors failed then either fail the job
+        or restart it if it has retries left and is a checkpoint job"""
+
+        if jobGraph.jobStoreID in self.toilState.servicesIssued:
+            # The job has services running, signal for them to be killed
+            # once they are killed then the jobGraph will be re-added to
+            # the updatedJobs set and then scheduled to be removed
+            logger.debug("Telling job: %s to terminate its services due to successor failure",
+                         jobGraph.jobStoreID)
+            self.serviceManager.killServices(self.toilState.servicesIssued[jobGraph.jobStoreID],
+                                             error=True)
+        elif jobGraph.jobStoreID in self.toilState.successorCounts:
+            # The job has non-service jobs running wait for them to finish
+            # the job will be re-added to the updated jobs when these jobs
+            # are done
+            logger.debug("Job %s with ID: %s with failed successors still has successor jobs running",
+                         jobGraph, jobGraph.jobStoreID)
+        elif jobGraph.checkpoint is not None and jobGraph.remainingRetryCount > 1:
+            # If the job is a checkpoint and has remaining retries then reissue it.
+            # The logic behind using > 1 rather than > 0 here: Since this job has
+            # been tried once (without decreasing its retry count as the job
+            # itself was successful), and its subtree failed, it shouldn't be retried
+            # unless it has more than 1 try.
+            logger.warn('Job: %s is being restarted as a checkpoint after the total '
+                        'failure of jobs in its subtree.', jobGraph.jobStoreID)
+            self.issueJob(JobNode.fromJobGraph(jobGraph))
+        else:
+            # Mark it totally failed
+            logger.debug("Job %s is being processed as completely failed", jobGraph.jobStoreID)
+            self.processTotallyFailedJob(jobGraph)
+
+    def _processReadyJob(self, jobGraph, resultStatus):
+        logger.debug('Updating status of job %s with ID %s: with result status: %s',
+                     jobGraph, jobGraph.jobStoreID, resultStatus)
+
+        if jobGraph in self.serviceManager.jobGraphsWithServicesBeingStarted:
+            # This stops a job with services being issued by the serviceManager from
+            # being considered further in this loop. This catch is necessary because
+            # the job's service's can fail while being issued, causing the job to be
+            # added to updated jobs.
+            logger.debug("Got a job to update which is still owned by the service "
+                         "manager: %s", jobGraph.jobStoreID)
+        elif jobGraph.jobStoreID in self.toilState.hasFailedSuccessors:
+            self._processFailedSuccessors(jobGraph)
+        elif jobGraph.command is not None or resultStatus != 0:
+            # The jobGraph has a command it must be run before any successors.
+            # Similarly, if the job previously failed we rerun it, even if it doesn't have a
+            # command to run, to eliminate any parts of the stack now completed.
+            isServiceJob = jobGraph.jobStoreID in self.toilState.serviceJobStoreIDToPredecessorJob
+
+            # If the job has run out of retries or is a service job whose error flag has
+            # been indicated, fail the job.
+            if (jobGraph.remainingRetryCount == 0
+                or isServiceJob and not self.jobStore.fileExists(jobGraph.errorJobStoreID)):
+                self.processTotallyFailedJob(jobGraph)
+                logger.warn("Job %s with ID %s is completely failed",
+                            jobGraph, jobGraph.jobStoreID)
+            else:
+                # Otherwise try the job again
+                self.issueJob(JobNode.fromJobGraph(jobGraph))
+        elif len(jobGraph.services) > 0:
+            # the job has services to run, which have not been started, start them
+            # Build a map from the service jobs to the job and a map
+            # of the services created for the job
+            assert jobGraph.jobStoreID not in self.toilState.servicesIssued
+            self.toilState.servicesIssued[jobGraph.jobStoreID] = {}
+            for serviceJobList in jobGraph.services:
+                for serviceTuple in serviceJobList:
+                    serviceID = serviceTuple.jobStoreID
+                    assert serviceID not in self.toilState.serviceJobStoreIDToPredecessorJob
+                    self.toilState.serviceJobStoreIDToPredecessorJob[serviceID] = jobGraph
+                    self.toilState.servicesIssued[jobGraph.jobStoreID][serviceID] = serviceTuple
+
+            # Use the service manager to start the services
+            self.serviceManager.scheduleServices(jobGraph)
+
+            logger.debug("Giving job: %s to service manager to schedule its jobs", jobGraph.jobStoreID)
+        elif len(jobGraph.stack) > 0:
+            # There are exist successors to run
+            self._runJobSuccessors(jobGraph)
+        elif jobGraph.jobStoreID in self.toilState.servicesIssued:
+            logger.debug("Telling job: %s to terminate its services due to the "
+                         "successful completion of its successor jobs",
+                         jobGraph)
+            self.serviceManager.killServices(self.toilState.servicesIssued[jobGraph.jobStoreID], error=False)
+        else:
+            #There are no remaining tasks to schedule within the jobGraph, but
+            #we schedule it anyway to allow it to be deleted. Remove the job
+
+            #TODO: An alternative would be simple delete it here and add it to the
+            #list of jobs to process, or (better) to create an asynchronous
+            #process that deletes jobs and then feeds them back into the set
+            #of jobs to be processed
+            if jobGraph.remainingRetryCount > 0:
+                self.issueJob(JobNode.fromJobGraph(jobGraph))
+                logger.debug("Job: %s is empty, we are scheduling to clean it up", jobGraph.jobStoreID)
+            else:
+                self.processTotallyFailedJob(jobGraph)
+                logger.warn("Job: %s is empty but completely failed - something is very wrong", jobGraph.jobStoreID)
+
+    def _processReadyJobs(self):
+        """Process jobs that are ready to be scheduled/have successors to schedule"""
+        logger.debug('Built the jobs list, currently have %i jobs to update and %i jobs issued',
+                     len(self.toilState.updatedJobs), self.getNumberOfJobsIssued())
+
+        updatedJobs = self.toilState.updatedJobs # The updated jobs to consider below
+        self.toilState.updatedJobs = set() # Resetting the list for the next set
+
+        for jobGraph, resultStatus in updatedJobs:
+            self._processReadyJob(jobGraph, resultStatus)
+
+    def _startServiceJobs(self):
+        """Start any service jobs available from the service manager"""
+        self.issueQueingServiceJobs()
+        while True:
+            serviceJob = self.serviceManager.getServiceJobsToStart(0)
+            # Stop trying to get jobs when function returns None
+            if serviceJob is None:
+                break
+            logger.debug('Launching service job: %s', serviceJob)
+            self.issueServiceJob(serviceJob)
+
+    def _processJobsWithRunningServices(self):
+        """Get jobs whose services have started"""
+        while True:
+            jobGraph = self.serviceManager.getJobGraphWhoseServicesAreRunning(0)
+            if jobGraph is None: # Stop trying to get jobs when function returns None
+                break
+            logger.debug('Job: %s has established its services.', jobGraph.jobStoreID)
+            jobGraph.services = []
+            self.toilState.updatedJobs.add((jobGraph, 0))
+
+    def _gatherUpdatedJobs(self, updatedJobTuple):
+        """Gather any new, updated jobGraph from the batch system"""
+        jobID, result, wallTime = updatedJobTuple
+        # easy, track different state
+        try:
+            updatedJob = self.jobBatchSystemIDToIssuedJob[jobID]
+        except KeyError:
+            logger.warn("A result seems to already have been processed "
+                        "for job %s", jobID)
+        else:
+            if result == 0:
+                cur_logger = (logger.debug if str(updatedJob.jobName).startswith(CWL_INTERNAL_JOBS)
+                              else logger.info)
+                cur_logger('Job ended successfully: %s', updatedJob)
+                if self.toilMetrics:
+                    self.toilMetrics.logCompletedJob(updatedJob)
+            else:
+                logger.warn('Job failed with exit value %i: %s',
+                            result, updatedJob)
+            self.processFinishedJob(jobID, result, wallTime=wallTime)
+
+    def _processLostJobs(self):
+        """Process jobs that have gone awry"""
+        # In the case that there is nothing happening (no updated jobs to
+        # gather for rescueJobsFrequency seconds) check if there are any jobs
+        # that have run too long (see self.reissueOverLongJobs) or which have
+        # gone missing from the batch system (see self.reissueMissingJobs)
+        if ((time.time() - self.timeSinceJobsLastRescued) >= self.config.rescueJobsFrequency):
+            # We only rescue jobs every N seconds, and when we have apparently
+            # exhausted the current jobGraph supply
+            self.reissueOverLongJobs()
+            logger.info("Reissued any over long jobs")
+
+            hasNoMissingJobs = self.reissueMissingJobs()
+            if hasNoMissingJobs:
+                self.timeSinceJobsLastRescued = time.time()
+            else:
+                # This means we'll try again in a minute, providing things are quiet
+                self.timeSinceJobsLastRescued += 60
+            logger.info("Rescued any (long) missing jobs")
+
+
     def innerLoop(self):
         """
         The main loop for processing jobs by the leader.
         """
-        # Sets up the timing of the jobGraph rescuing method
-        timeSinceJobsLastRescued = time.time()
-
         logger.info("Starting the main loop")
-        while True:
-            # Process jobs that are ready to be scheduled/have successors to schedule
+        self.timeSinceJobsLastRescued = time.time()
+
+        while self._anythingLeftToDo():
             if len(self.toilState.updatedJobs) > 0:
-                logger.debug('Built the jobs list, currently have %i jobs to update and %i jobs issued',
-                             len(self.toilState.updatedJobs), self.getNumberOfJobsIssued())
-
-                updatedJobs = self.toilState.updatedJobs # The updated jobs to consider below
-                self.toilState.updatedJobs = set() # Resetting the list for the next set
-
-                for jobGraph, resultStatus in updatedJobs:
-
-                    logger.debug('Updating status of job %s with ID %s: with result status: %s',
-                                 jobGraph, jobGraph.jobStoreID, resultStatus)
-
-                    # This stops a job with services being issued by the serviceManager from
-                    # being considered further in this loop. This catch is necessary because
-                    # the job's service's can fail while being issued, causing the job to be
-                    # added to updated jobs.
-                    if jobGraph in self.serviceManager.jobGraphsWithServicesBeingStarted:
-                        logger.debug("Got a job to update which is still owned by the service "
-                                     "manager: %s", jobGraph.jobStoreID)
-                        continue
-
-                    # If some of the jobs successors failed then either fail the job
-                    # or restart it if it has retries left and is a checkpoint job
-                    if jobGraph.jobStoreID in self.toilState.hasFailedSuccessors:
-
-                        # If the job has services running, signal for them to be killed
-                        # once they are killed then the jobGraph will be re-added to the
-                        # updatedJobs set and then scheduled to be removed
-                        if jobGraph.jobStoreID in self.toilState.servicesIssued:
-                            logger.debug("Telling job: %s to terminate its services due to successor failure",
-                                         jobGraph.jobStoreID)
-                            self.serviceManager.killServices(self.toilState.servicesIssued[jobGraph.jobStoreID],
-                                                        error=True)
-
-                        # If the job has non-service jobs running wait for them to finish
-                        # the job will be re-added to the updated jobs when these jobs are done
-                        elif jobGraph.jobStoreID in self.toilState.successorCounts:
-                            logger.debug("Job %s with ID: %s with failed successors still has successor jobs running",
-                                         jobGraph, jobGraph.jobStoreID)
-                            continue
-
-                        # If the job is a checkpoint and has remaining retries then reissue it.
-                        # The logic behind using > 1 rather than > 0 here: Since this job has
-                        # been tried once (without decreasing its retry count as the job
-                        # itself was successful), and its subtree failed, it shouldn't be retried
-                        # unless it has more than 1 try.
-                        elif jobGraph.checkpoint is not None and jobGraph.remainingRetryCount > 1:
-                            logger.warn('Job: %s is being restarted as a checkpoint after the total '
-                                        'failure of jobs in its subtree.', jobGraph.jobStoreID)
-                            self.issueJob(JobNode.fromJobGraph(jobGraph))
-                        else: # Mark it totally failed
-                            logger.debug("Job %s is being processed as completely failed", jobGraph.jobStoreID)
-                            self.processTotallyFailedJob(jobGraph)
-
-                    # If the jobGraph has a command it must be run before any successors.
-                    # Similarly, if the job previously failed we rerun it, even if it doesn't have a
-                    # command to run, to eliminate any parts of the stack now completed.
-                    elif jobGraph.command is not None or resultStatus != 0:
-                        isServiceJob = jobGraph.jobStoreID in self.toilState.serviceJobStoreIDToPredecessorJob
-
-                        # If the job has run out of retries or is a service job whose error flag has
-                        # been indicated, fail the job.
-                        if (jobGraph.remainingRetryCount == 0
-                            or isServiceJob and not self.jobStore.fileExists(jobGraph.errorJobStoreID)):
-                            self.processTotallyFailedJob(jobGraph)
-                            logger.warn("Job %s with ID %s is completely failed",
-                                        jobGraph, jobGraph.jobStoreID)
-                        else:
-                            # Otherwise try the job again
-                            self.issueJob(JobNode.fromJobGraph(jobGraph))
-
-                    # If the job has services to run, which have not been started, start them
-                    elif len(jobGraph.services) > 0:
-                        # Build a map from the service jobs to the job and a map
-                        # of the services created for the job
-                        assert jobGraph.jobStoreID not in self.toilState.servicesIssued
-                        self.toilState.servicesIssued[jobGraph.jobStoreID] = {}
-                        for serviceJobList in jobGraph.services:
-                            for serviceTuple in serviceJobList:
-                                serviceID = serviceTuple.jobStoreID
-                                assert serviceID not in self.toilState.serviceJobStoreIDToPredecessorJob
-                                self.toilState.serviceJobStoreIDToPredecessorJob[serviceID] = jobGraph
-                                self.toilState.servicesIssued[jobGraph.jobStoreID][serviceID] = serviceTuple
-
-                        # Use the service manager to start the services
-                        self.serviceManager.scheduleServices(jobGraph)
-
-                        logger.debug("Giving job: %s to service manager to schedule its jobs", jobGraph.jobStoreID)
-
-                    # There exist successors to run
-                    elif len(jobGraph.stack) > 0:
-                        assert len(jobGraph.stack[-1]) > 0
-                        logger.debug("Job: %s has %i successors to schedule",
-                                     jobGraph.jobStoreID, len(jobGraph.stack[-1]))
-                        #Record the number of successors that must be completed before
-                        #the jobGraph can be considered again
-                        assert jobGraph.jobStoreID not in self.toilState.successorCounts
-                        self.toilState.successorCounts[jobGraph.jobStoreID] = len(jobGraph.stack[-1])
-                        #List of successors to schedule
-                        successors = []
-
-                        #For each successor schedule if all predecessors have been completed
-                        for jobNode in jobGraph.stack[-1]:
-                            successorJobStoreID = jobNode.jobStoreID
-                            #Build map from successor to predecessors.
-                            if successorJobStoreID not in self.toilState.successorJobStoreIDToPredecessorJobs:
-                                self.toilState.successorJobStoreIDToPredecessorJobs[successorJobStoreID] = []
-                            self.toilState.successorJobStoreIDToPredecessorJobs[successorJobStoreID].append(jobGraph)
-                            #Case that the jobGraph has multiple predecessors
-                            if jobNode.predecessorNumber > 1:
-                                logger.debug("Successor job: %s of job: %s has multiple "
-                                             "predecessors", jobNode, jobGraph)
-
-                                # Get the successor job, using a cache
-                                # (if the successor job has already been seen it will be in this cache,
-                                # but otherwise put it in the cache)
-                                if successorJobStoreID not in self.toilState.jobsToBeScheduledWithMultiplePredecessors:
-                                    self.toilState.jobsToBeScheduledWithMultiplePredecessors[successorJobStoreID] = self.jobStore.load(successorJobStoreID)
-                                successorJobGraph = self.toilState.jobsToBeScheduledWithMultiplePredecessors[successorJobStoreID]
-
-                                #Add the jobGraph as a finished predecessor to the successor
-                                successorJobGraph.predecessorsFinished.add(jobGraph.jobStoreID)
-
-                                # If the successor is in the set of successors of failed jobs
-                                if successorJobStoreID in self.toilState.failedSuccessors:
-                                    logger.debug("Successor job: %s of job: %s has failed "
-                                                 "predecessors", jobNode, jobGraph)
-
-                                    # Add the job to the set having failed successors
-                                    self.toilState.hasFailedSuccessors.add(jobGraph.jobStoreID)
-
-                                    # Reduce active successor count and remove the successor as an active successor of the job
-                                    self.toilState.successorCounts[jobGraph.jobStoreID] -= 1
-                                    assert self.toilState.successorCounts[jobGraph.jobStoreID] >= 0
-                                    self.toilState.successorJobStoreIDToPredecessorJobs[successorJobStoreID].remove(jobGraph)
-                                    if len(self.toilState.successorJobStoreIDToPredecessorJobs[successorJobStoreID]) == 0:
-                                        self.toilState.successorJobStoreIDToPredecessorJobs.pop(successorJobStoreID)
-
-                                    # If the job now has no active successors add to active jobs
-                                    # so it can be processed as a job with failed successors
-                                    if self.toilState.successorCounts[jobGraph.jobStoreID] == 0:
-                                        logger.debug("Job: %s has no successors to run "
-                                                     "and some are failed, adding to list of jobs "
-                                                     "with failed successors", jobGraph)
-                                        self.toilState.successorCounts.pop(jobGraph.jobStoreID)
-                                        self.toilState.updatedJobs.add((jobGraph, 0))
-                                        continue
-
-                                # If the successor job's predecessors have all not all completed then
-                                # ignore the jobGraph as is not yet ready to run
-                                assert len(successorJobGraph.predecessorsFinished) <= successorJobGraph.predecessorNumber
-                                if len(successorJobGraph.predecessorsFinished) < successorJobGraph.predecessorNumber:
-                                    continue
-                                else:
-                                    # Remove the successor job from the cache
-                                    self.toilState.jobsToBeScheduledWithMultiplePredecessors.pop(successorJobStoreID)
-
-                            # Add successor to list of successors to schedule
-                            successors.append(jobNode)
-                        self.issueJobs(successors)
-
-                    elif jobGraph.jobStoreID in self.toilState.servicesIssued:
-                        logger.debug("Telling job: %s to terminate its services due to the "
-                                     "successful completion of its successor jobs",
-                                     jobGraph)
-                        self.serviceManager.killServices(self.toilState.servicesIssued[jobGraph.jobStoreID], error=False)
-
-                    #There are no remaining tasks to schedule within the jobGraph, but
-                    #we schedule it anyway to allow it to be deleted.
-
-                    #TODO: An alternative would be simple delete it here and add it to the
-                    #list of jobs to process, or (better) to create an asynchronous
-                    #process that deletes jobs and then feeds them back into the set
-                    #of jobs to be processed
-                    else:
-                        # Remove the job
-                        if jobGraph.remainingRetryCount > 0:
-                            self.issueJob(JobNode.fromJobGraph(jobGraph))
-                            logger.debug("Job: %s is empty, we are scheduling to clean it up", jobGraph.jobStoreID)
-                        else:
-                            self.processTotallyFailedJob(jobGraph)
-                            logger.warn("Job: %s is empty but completely failed - something is very wrong", jobGraph.jobStoreID)
-
-            # Start any service jobs available from the service manager
-            self.issueQueingServiceJobs()
-            while True:
-                serviceJob = self.serviceManager.getServiceJobsToStart(0)
-                # Stop trying to get jobs when function returns None
-                if serviceJob is None:
-                    break
-                logger.debug('Launching service job: %s', serviceJob)
-                self.issueServiceJob(serviceJob)
-
-            # Get jobs whose services have started
-            while True:
-                jobGraph = self.serviceManager.getJobGraphWhoseServicesAreRunning(0)
-                if jobGraph is None: # Stop trying to get jobs when function returns None
-                    break
-                logger.debug('Job: %s has established its services.', jobGraph.jobStoreID)
-                jobGraph.services = []
-                self.toilState.updatedJobs.add((jobGraph, 0))
-
-            # Gather any new, updated jobGraph from the batch system
+                self._processReadyJobs()
+
+            # deal with service-related jobs
+            self._startServiceJobs()
+            self._processJobsWithRunningServices()
+
+            # check in with the batch system
             updatedJobTuple = self.batchSystem.getUpdatedBatchJob(2)
             if updatedJobTuple is not None:
-                jobID, result, wallTime = updatedJobTuple
-                # easy, track different state
-                try:
-                    updatedJob = self.jobBatchSystemIDToIssuedJob[jobID]
-                except KeyError:
-                    logger.warn("A result seems to already have been processed "
-                                "for job %s", jobID)
-                else:
-                    if result == 0:
-                        cur_logger = (logger.debug if str(updatedJob.jobName).startswith(CWL_INTERNAL_JOBS)
-                                      else logger.info)
-                        cur_logger('Job ended successfully: %s', updatedJob)
-                        if self.toilMetrics:
-                            self.toilMetrics.logCompletedJob(updatedJob)
-                    else:
-                        logger.warn('Job failed with exit value %i: %s',
-                                    result, updatedJob)
-                    self.processFinishedJob(jobID, result, wallTime=wallTime)
-
+                self._gatherUpdatedJobs(updatedJobTuple)
             else:
-                # Process jobs that have gone awry
-
-                #In the case that there is nothing happening
-                #(no updated jobs to gather for 10 seconds)
-                #check if there are any jobs that have run too long
-                #(see self.reissueOverLongJobs) or which
-                #have gone missing from the batch system (see self.reissueMissingJobs)
-                if (time.time() - timeSinceJobsLastRescued >=
-                    self.config.rescueJobsFrequency): #We only
-                    #rescue jobs every N seconds, and when we have
-                    #apparently exhausted the current jobGraph supply
-                    self.reissueOverLongJobs()
-                    logger.info("Reissued any over long jobs")
-
-                    hasNoMissingJobs = self.reissueMissingJobs()
-                    if hasNoMissingJobs:
-                        timeSinceJobsLastRescued = time.time()
-                    else:
-                        timeSinceJobsLastRescued += 60 #This means we'll try again
-                        #in a minute, providing things are quiet
-                    logger.info("Rescued any (long) missing jobs")
+                self._processLostJobs()
 
             # Check on the associated threads and exit if a failure is detected
             self.statsAndLogging.check()
@@ -492,15 +540,10 @@ def innerLoop(self):
             if self.clusterScaler is not None:
                 self.clusterScaler.check()
 
-            # The exit criterion
-            if len(self.toilState.updatedJobs) == 0 and self.getNumberOfJobsIssued() == 0 and self.serviceManager.jobsIssuedToServiceManager == 0:
-                logger.info("No jobs left to run so exiting.")
-                break
-
             # Check for deadlocks
             self.checkForDeadlocks()
 
-        logger.info("Finished the main loop")
+        logger.info("Finished the main loop: no jobs left to run")
 
         # Consistency check the toil state
         assert self.toilState.updatedJobs == set()
@@ -540,6 +583,8 @@ def checkForDeadlocks(self):
             self.potentialDeadlockedJobs = set()
             self.potentialDeadlockTime = 0
 
+    def _anythingLeftToDo(self):
+        return (len(self.toilState.updatedJobs) > 0) or (self.getNumberOfJobsIssued() > 0) or (self.serviceManager.jobsIssuedToServiceManager > 0)
 
     def issueJob(self, jobNode):
         """
@@ -637,6 +682,7 @@ def removeJob(self, jobBatchSystemID):
                 self.serviceJobsIssued -= 1
 
         return jobNode
+
     def getJobs(self, preemptable=None):
         jobs = self.jobBatchSystemIDToIssuedJob.values()
         if preemptable is not None:


Test patch: