Repository: DataBiosphere/toil

Issue:
Fix job scheduling on restart after partially failed DAGs
If a job in a DAG fails and there is another path to its child, the child is run in the restart irrespective of the status of the previously failed parent in the new run.

@arkal will write a test to prove this behaviour and wither @cket or @benedictpaten will probably write the fix.
Retry 409s configuring versioning of job store buckets (resolves #1093)
resolves #1093



Discussion:
Likely related to #1100 

Jenkins, test this please.


Resolving patch:
diff --git a/src/toil/leader.py b/src/toil/leader.py
--- a/src/toil/leader.py
+++ b/src/toil/leader.py
@@ -348,6 +348,42 @@ def processRemovedJob(jobStoreID):
             logger.debug("Added jobWrapper: %s to active jobs", jobStoreID)
         else:  #The jobWrapper is done
             processRemovedJob(jobStoreID)
+    
+    @staticmethod    
+    def getSuccessors(jobWrapper, alreadySeenSuccessors, jobStore):
+        """
+        Gets successors of the given job (jobWrapper) by walking the job graph from the job
+        through all its successors recursively. 
+        Any successor in alreadySeenSuccessors is ignored and not traversed.
+        Returns the set of found successors. This set is added to alreadySeenSuccessors.
+        """
+        successors = set()
+        
+        def successorRecursion(jobWrapper):
+            # For lists of successors
+            for successorList in jobWrapper.stack:
+                
+                # For each successor in list of successors
+                for successorJobStoreTuple in successorList:
+                    
+                    # Id of the successor
+                    successorJobStoreID = successorJobStoreTuple[0]
+                    
+                    # If successor not already visited
+                    if successorJobStoreID not in alreadySeenSuccessors:
+                        
+                        # Add to set of successors
+                        successors.add(successorJobStoreID)
+                        alreadySeenSuccessors.add(successorJobStoreID)
+                        
+                        # Recurse if job exists
+                        # (job may not exist if already completed)
+                        if jobStore.exists(successorJobStoreID):
+                            successorRecursion(jobStore.load(successorJobStoreID))
+    
+        successorRecursion(jobWrapper) # Recurse from jobWrapper
+        
+        return successors   
 
     def processTotallyFailedJob(self, jobWrapper):
         """
@@ -381,18 +417,57 @@ def processTotallyFailedJob(self, jobWrapper):
             self.toilState.hasFailedSuccessors.add(predecessorJobWrapper.jobStoreID) # This ensures that the
             # job will not attempt to run any of it's successors on the stack
         else:
+            # Is a non-service job
             assert jobWrapper.jobStoreID not in self.toilState.servicesIssued
-
-            # Is a non-service job, walk up the tree killing services of any jobs with nothing left to do.
+            
+            # Traverse failed job's successor graph and get the jobStoreID of new successors.
+            # Any successor already in toilState.failedSuccessors will not be traversed
+            # All successors traversed will be added to toilState.failedSuccessors and returned
+            # as a set (unseenSuccessors).
+            unseenSuccessors = self.getSuccessors(jobWrapper, self.toilState.failedSuccessors, self.jobStore)
+            logger.debug("Found new failed successors: %s of job: %s" % (" ".join(unseenSuccessors), jobWrapper.jobStoreID))
+            
+            # For each newly found successor
+            for successorJobStoreID in unseenSuccessors:
+                
+                # If the successor is a successor of other jobs that have already tried to schedule it
+                if successorJobStoreID in self.toilState.successorJobStoreIDToPredecessorJobs:
+                    
+                    # For each such predecessor job
+                    # (we remove the successor from toilState.successorJobStoreIDToPredecessorJobs to avoid doing 
+                    # this multiple times for each failed predecessor)
+                    for predecessorJob in self.toilState.successorJobStoreIDToPredecessorJobs.pop(successorJobStoreID):
+                        
+                        # Reduce the predecessor job's successor count.
+                        self.toilState.successorCounts[predecessorJob.jobStoreID] -= 1
+                        
+                        # Indicate that it has failed jobs.  
+                        self.toilState.hasFailedSuccessors.add(predecessorJob.jobStoreID)
+                        logger.debug("Marking job: %s as having failed successors (found by reading successors failed job)" % predecessorJob.jobStoreID)
+                        
+                        # If the predecessor has no remaining successors, add to list of active jobs
+                        assert self.toilState.successorCounts[predecessorJob.jobStoreID] >= 0
+                        if self.toilState.successorCounts[predecessorJob.jobStoreID] == 0:
+                            self.toilState.updatedJobs.add((predecessorJob, 0))
+                            
+                            # Remove the predecessor job from the set of jobs with successors. 
+                            self.toilState.successorCounts.pop(predecessorJob.jobStoreID) 
+
+            # If the job has predecessor(s)
             if jobWrapper.jobStoreID in self.toilState.successorJobStoreIDToPredecessorJobs:
+                
+                # For each predecessor of the job
                 for predecessorJobWrapper in self.toilState.successorJobStoreIDToPredecessorJobs[jobWrapper.jobStoreID]:
+                    
+                    # Mark the predecessor as failed
                     self.toilState.hasFailedSuccessors.add(predecessorJobWrapper.jobStoreID)
-                    logger.debug("Totally failed job: %s is marking direct predecessors %s as having failed jobs", jobWrapper.jobStoreID, predecessorJobWrapper.jobStoreID)
+                    logger.debug("Totally failed job: %s is marking direct predecessor: %s as having failed jobs", jobWrapper.jobStoreID, predecessorJobWrapper.jobStoreID)
+
                 self._updatePredecessorStatus(jobWrapper.jobStoreID)
 
     def _updatePredecessorStatus(self, jobStoreID):
         """
-        Update status of a predecessor for finished successor job.
+        Update status of predecessors for finished successor job.
         """
         if jobStoreID in self.toilState.serviceJobStoreIDToPredecessorJob:
             # Is a service job
@@ -410,19 +485,34 @@ def _updatePredecessorStatus(self, jobStoreID):
             assert len(self.toilState.updatedJobs) == 0
             assert len(self.toilState.successorJobStoreIDToPredecessorJobs) == 0
             assert len(self.toilState.successorCounts) == 0
+            logger.debug("Reached root job %s so no predecessors to clean up" % jobStoreID)
 
         else:
+            # Is a non-root, non-service job
+            logger.debug("Cleaning the predecessors of %s" % jobStoreID)
+            
+            # For each predecessor
             for predecessorJob in self.toilState.successorJobStoreIDToPredecessorJobs.pop(jobStoreID):
+                
+                # Reduce the predecessor's number of successors by one to indicate the 
+                # completion of the jobStoreID job
                 self.toilState.successorCounts[predecessorJob.jobStoreID] -= 1
-                assert self.toilState.successorCounts[predecessorJob.jobStoreID] >= 0
-                if self.toilState.successorCounts[predecessorJob.jobStoreID] == 0: #Job is done
+
+                # If the predecessor job is done and all the successors are complete 
+                if self.toilState.successorCounts[predecessorJob.jobStoreID] == 0:
+                    
+                    # Remove it from the set of jobs with active successors
                     self.toilState.successorCounts.pop(predecessorJob.jobStoreID)
+
+                    # Pop stack at this point, as we can get rid of its successors
                     predecessorJob.stack.pop()
+                    
+                    # Now we know the job is done we can add it to the list of updated job files
+                    assert predecessorJob not in self.toilState.updatedJobs
+                    self.toilState.updatedJobs.add((predecessorJob, 0))
+                    
                     logger.debug('Job %s has all its non-service successors completed or totally '
                                  'failed', predecessorJob.jobStoreID)
-                    assert predecessorJob not in self.toilState.updatedJobs
-                    self.toilState.updatedJobs.add((predecessorJob, 0)) #Now we know
-                    #the job is done we can add it to the list of updated job files
 
 ##########################################
 #Class to represent the state of the toil in memory. Loads this
@@ -450,9 +540,6 @@ def __init__( self, jobStore, rootJob, jobCache=None):
         # Each for job, the map is a dictionary of service jobStoreIDs
         # to the flags used to communicate the with service
         self.servicesIssued = { }
-
-        # Jobs (as jobStoreIDs) with successors that have totally failed
-        self.hasFailedSuccessors = set()
         
         # Jobs that are ready to be processed
         self.updatedJobs = set( )
@@ -461,6 +548,17 @@ def __init__( self, jobStore, rootJob, jobCache=None):
         # end to remove jobs that were removed by checkpoints
         self.totalFailedJobs = set()
         
+        # Jobs (as jobStoreIDs) with successors that have totally failed
+        self.hasFailedSuccessors = set()
+        
+        # The set of successors of failed jobs as a set of jobStoreIds
+        self.failedSuccessors = set()
+        
+        # Set of jobs that have multiple predecessors that have one or more predecessors
+        # finished, but not all of them. This acts as a cache for these jobs.
+        # Stored as hash from jobStoreIDs to jobWrappers
+        self.jobsToBeScheduledWithMultiplePredecessors = {}
+        
         ##Algorithm to build this information
         logger.info("(Re)building internal scheduler state")
         self._buildToilState(rootJob, jobStore, jobCache)
@@ -500,19 +598,73 @@ def getJob(jobId):
                 jobWrapper.command = jobWrapper.checkpoint
 
         else: # There exist successors
+            logger.debug("Adding job: %s to the state with %s successors" % (jobWrapper.jobStoreID, len(jobWrapper.stack[-1])))
+            
+            # Record the number of successors
             self.successorCounts[jobWrapper.jobStoreID] = len(jobWrapper.stack[-1])
+            
+            def processSuccessorWithMultiplePredecessors(successorJobWrapper):
+                # If jobWrapper job is not reported as complete by the successor
+                if jobWrapper.jobStoreID not in successorJobWrapper.predecessorsFinished:
+                    
+                    # Update the sucessor's status to mark the predecessor complete
+                    successorJobWrapper.predecessorsFinished.add(jobWrapper.jobStoreID)
+            
+                # If the successor has no predecessors to finish
+                assert len(successorJobWrapper.predecessorsFinished) <= successorJobWrapper.predecessorNumber
+                if len(successorJobWrapper.predecessorsFinished) == successorJobWrapper.predecessorNumber:
+                    
+                    # It is ready to be run, so remove it from the cache
+                    self.jobsToBeScheduledWithMultiplePredecessors.pop(successorJobStoreID)
+                    
+                    # Recursively consider the successor
+                    self._buildToilState(successorJobWrapper, jobStore, jobCache=jobCache)
+            
+            # For each successor
             for successorJobStoreTuple in jobWrapper.stack[-1]:
                 successorJobStoreID = successorJobStoreTuple[0]
+                
+                # If the successor jobWrapper does not yet point back at a
+                # predecessor we have not yet considered it
                 if successorJobStoreID not in self.successorJobStoreIDToPredecessorJobs:
-                    #Given that the successor jobWrapper does not yet point back at a
-                    #predecessor we have not yet considered it, so we call the function
-                    #on the successor
+
+                    # Add the job as a predecessor
                     self.successorJobStoreIDToPredecessorJobs[successorJobStoreID] = [jobWrapper]
-                    self._buildToilState(getJob(successorJobStoreID), jobStore, jobCache=jobCache)
+                    
+                    # If predecessorJobStoreID is not None then the successor has multiple predecessors
+                    predecessorJobStoreID = successorJobStoreTuple[-1]
+                    if predecessorJobStoreID != None: 
+                        
+                        # We load the successor job
+                        successorJobWrapper =  getJob(successorJobStoreID)
+                        
+                        # We put the successor job in the cache of successor jobs with multiple predecessors
+                        assert successorJobStoreID not in self.jobsToBeScheduledWithMultiplePredecessors
+                        self.jobsToBeScheduledWithMultiplePredecessors[successorJobStoreID] = successorJobWrapper
+                        
+                        # Process successor
+                        processSuccessorWithMultiplePredecessors(successorJobWrapper)
+                            
+                    else:
+                        # The successor has only the jobWrapper job as a predecessor so
+                        # recursively consider the successor
+                        self._buildToilState(getJob(successorJobStoreID), jobStore, jobCache=jobCache)
+                
                 else:
-                    #We have already looked at the successor, so we don't recurse,
-                    #but we add back a predecessor link
-                    self.successorJobStoreIDToPredecessorJobs[successorJobStoreID].append(jobWrapper)
+                    # We've already seen the successor
+                    
+                    # Add the job as a predecessor
+                    assert jobWrapper not in self.successorJobStoreIDToPredecessorJobs[successorJobStoreID]
+                    self.successorJobStoreIDToPredecessorJobs[successorJobStoreID].append(jobWrapper) 
+                    
+                    # If the successor has multiple predecessors
+                    if successorJobStoreID in self.jobsToBeScheduledWithMultiplePredecessors:
+                        
+                        # Get the successor from cache
+                        successorJobWrapper = self.jobsToBeScheduledWithMultiplePredecessors[successorJobStoreID]
+                        
+                        # Process successor
+                        processSuccessorWithMultiplePredecessors(successorJobWrapper)
 
 class FailedJobsException( Exception ):
     def __init__(self, jobStoreLocator, failedJobs, jobStore):
@@ -760,8 +912,6 @@ def mainLoop(config, batchSystem, provisioner, jobStore, rootJobWrapper, jobCach
             logger.exception('Failed to unpickle root job return value')
             raise FailedJobsException(config.jobStore, toilState.totalFailedJobs, jobStore)
 
-
-
 def innerLoop(jobStore, config, batchSystem, toilState, jobBatcher, serviceManager, statsAndLogging):
     """
     :param toil.jobStores.abstractJobStore.AbstractJobStore jobStore:
@@ -829,6 +979,7 @@ def innerLoop(jobStore, config, batchSystem, toilState, jobBatcher, serviceManag
                                             cores=jobWrapper.cores,
                                             disk=jobWrapper.disk,
                                             preemptable=jobWrapper.preemptable)
+                        
                     else: # Mark it totally failed
                         logger.debug("Job %s is being processed as completely failed", jobWrapper.jobStoreID)
                         jobBatcher.processTotallyFailedJob(jobWrapper)
@@ -873,12 +1024,15 @@ def innerLoop(jobStore, config, batchSystem, toilState, jobBatcher, serviceManag
                     assert len(jobWrapper.stack[-1]) > 0
                     logger.debug("Job: %s has %i successors to schedule",
                                  jobWrapper.jobStoreID, len(jobWrapper.stack[-1]))
+                    
                     #Record the number of successors that must be completed before
                     #the jobWrapper can be considered again
                     assert jobWrapper.jobStoreID not in toilState.successorCounts
                     toilState.successorCounts[jobWrapper.jobStoreID] = len(jobWrapper.stack[-1])
+                    
                     #List of successors to schedule
                     successors = []
+                    
                     #For each successor schedule if all predecessors have been completed
                     for successorJobStoreID, memory, cores, disk, preemptable, predecessorID in jobWrapper.stack[-1]:
                         #Build map from successor to predecessors.
@@ -887,26 +1041,57 @@ def innerLoop(jobStore, config, batchSystem, toilState, jobBatcher, serviceManag
                         toilState.successorJobStoreIDToPredecessorJobs[successorJobStoreID].append(jobWrapper)
                         #Case that the jobWrapper has multiple predecessors
                         if predecessorID is not None:
-                            #Load the wrapped jobWrapper
-                            job2 = jobStore.load(successorJobStoreID)
-                            #Remove the predecessor from the list of predecessors
-                            job2.predecessorsFinished.add(predecessorID)
-                            #Checkpoint
-                            jobStore.update(job2)
-                            #If the jobs predecessors have all not all completed then
-                            #ignore the jobWrapper
-                            assert len(job2.predecessorsFinished) >= 1
-                            assert len(job2.predecessorsFinished) <= job2.predecessorNumber
-                            if len(job2.predecessorsFinished) < job2.predecessorNumber:
+                            logger.debug("Successor job: %s of job: %s has multiple predecessors" % (successorJobStoreID, jobWrapper.jobStoreID))
+                            
+                            # Get the successor job, using a cache 
+                            # (if the successor job has already been seen it will be in this cache, 
+                            # but otherwise put it in the cache)
+                            if successorJobStoreID not in toilState.jobsToBeScheduledWithMultiplePredecessors:
+                                toilState.jobsToBeScheduledWithMultiplePredecessors[successorJobStoreID] = jobStore.load(successorJobStoreID)      
+                            successorJobWrapper = toilState.jobsToBeScheduledWithMultiplePredecessors[successorJobStoreID]
+                            
+                            #Add the jobWrapper job as a finished predecessor to the successor 
+                            successorJobWrapper.predecessorsFinished.add(predecessorID)
+
+                            # If the successor is in the set of successors of failed jobs 
+                            if successorJobStoreID in toilState.failedSuccessors:
+                                logger.debug("Successor job: %s of job: %s has failed predecessors" % (successorJobStoreID, jobWrapper.jobStoreID))
+                                
+                                # Add the job to the set having failed successors
+                                toilState.hasFailedSuccessors.add(jobWrapper.jobStoreID)
+                                
+                                # Reduce active successor count and remove the successor as an active successor of the job
+                                toilState.successorCounts[jobWrapper.jobStoreID] -= 1
+                                assert toilState.successorCounts[jobWrapper.jobStoreID] >= 0
+                                toilState.successorJobStoreIDToPredecessorJobs[successorJobStoreID].remove(jobWrapper)
+                                if len(toilState.successorJobStoreIDToPredecessorJobs[successorJobStoreID]) == 0:
+                                    toilState.successorJobStoreIDToPredecessorJobs.pop(successorJobStoreID)
+                                
+                                # If the job now has no active successors add to active jobs
+                                # so it can be processed as a job with failed successors
+                                if toilState.successorCounts[jobWrapper.jobStoreID] == 0:
+                                    logger.debug("Job: %s has no successors to run and some are failed, adding to list of jobs with failed successors" % jobWrapper.jobStoreID)
+                                    toilState.successorCounts.pop(jobWrapper.jobStoreID)
+                                    toilState.updatedJobs.add((jobWrapper, 0))
+                                    continue
+
+                            # If the successor job's predecessors have all not all completed then
+                            # ignore the jobWrapper as is not yet ready to run
+                            assert len(successorJobWrapper.predecessorsFinished) <= successorJobWrapper.predecessorNumber
+                            if len(successorJobWrapper.predecessorsFinished) < successorJobWrapper.predecessorNumber:
                                 continue
+                            else:
+                                # Remove the successor job from the cache
+                                toilState.jobsToBeScheduledWithMultiplePredecessors.pop(successorJobStoreID)
+                        
+                        # Add successor to list of successors to schedule   
                         successors.append((successorJobStoreID, memory, cores, disk, preemptable))
                     jobBatcher.issueJobs(successors)
 
                 elif jobWrapper.jobStoreID in toilState.servicesIssued:
-                    logger.debug("Telling job: %s to terminate its due to the successful completion of its successor jobs",
-                                 jobWrapper.jobStoreID)
-                    serviceManager.killServices(toilState.servicesIssued[jobWrapper.jobStoreID],
-                                                error=False)
+                    logger.debug("Telling job: %s to terminate its services due to the successful completion of its successor jobs",
+                                jobWrapper.jobStoreID)
+                    serviceManager.killServices(toilState.servicesIssued[jobWrapper.jobStoreID], error=False)
 
                 #There are no remaining tasks to schedule within the jobWrapper, but
                 #we schedule it anyway to allow it to be deleted.
@@ -1011,4 +1196,5 @@ def innerLoop(jobStore, config, batchSystem, toilState, jobBatcher, serviceManag
     assert toilState.successorJobStoreIDToPredecessorJobs == {}
     assert toilState.serviceJobStoreIDToPredecessorJob == {}
     assert toilState.servicesIssued == {}
+    # assert toilState.jobsToBeScheduledWithMultiplePredecessors # These are not properly emptied yet
     # assert toilState.hasFailedSuccessors == set() # These are not properly emptied yet


Test patch:
diff --git a/src/toil/test/src/jobTest.py b/src/toil/test/src/jobTest.py
--- a/src/toil/test/src/jobTest.py
+++ b/src/toil/test/src/jobTest.py
@@ -16,8 +16,10 @@
 import os
 import random
 
+from toil.common import Toil
+from toil.leader import FailedJobsException
 from toil.lib.bioio import getTempFile
-from toil.job import Job, JobGraphDeadlockException
+from toil.job import Job, JobGraphDeadlockException, JobFunctionWrappingJob
 from toil.test import ToilTest
 
 
@@ -109,6 +111,51 @@ def testStatic2(self):
             self.assertEquals(open(outFile, 'r').readline(), "ABCDE")
         finally:
             os.remove(outFile)
+            
+    def testTrivialDAGConsistency(self):
+        options = Job.Runner.getDefaultOptions(self._createTempDir() + '/jobStore')
+        options.clean = 'always'
+        options.logLevel = 'debug'
+        i = Job.wrapJobFn(trivialParent)
+        with Toil(options) as toil:
+            try:
+                toil.start(i)
+            except FailedJobsException:
+                # we expect this exception to be raised
+                pass
+            else:
+                self.fail()
+
+    def testDAGConsistency(self):
+        options = Job.Runner.getDefaultOptions(self._createTempDir() + '/jobStore')
+        options.clean = 'always'
+        i = Job.wrapJobFn(parent)
+        with Toil(options) as toil:
+            try:
+                toil.start(i)
+            except FailedJobsException:
+                # we expect this exception to be raised
+                pass
+            else:
+                self.fail()
+
+    def testSiblingDAGConsistency(self):
+        """
+        Slightly more complex case. The stranded job's predecessors are siblings instead of
+        parent/child.
+        """
+        options = Job.Runner.getDefaultOptions(self._createTempDir() + '/jobStore')
+        options.clean = 'always'
+        options.logLevel = 'debug'
+        i = Job.wrapJobFn(diamond)
+        with Toil(options) as toil:
+            try:
+                toil.start(i)
+            except FailedJobsException:
+                # we expect this exception to be raised
+                pass
+            else:
+                self.fail()
 
     def testDeadlockDetection(self):
         """
@@ -116,14 +163,15 @@ def testDeadlockDetection(self):
         check they cause an exception properly. Also check that multiple roots 
         causes a deadlock exception.
         """
-        for test in xrange(100):
+        for test in xrange(10):
             # Make a random DAG for the set of child edges
             nodeNumber = random.choice(xrange(2, 20))
             childEdges = self.makeRandomDAG(nodeNumber)
             # Get an adjacency list representation and check is acyclic
             adjacencyList = self.getAdjacencyList(nodeNumber, childEdges)
             self.assertTrue(self.isAcyclic(adjacencyList))
-            # Add in follow on edges - these are returned as a list, and as a set of augmented
+            
+            # Add in follow-on edges - these are returned as a list, and as a set of augmented
             # edges in the adjacency list
             # edges in the adjacency list
             followOnEdges = self.addRandomFollowOnEdges(adjacencyList)
@@ -145,7 +193,7 @@ def testDeadlockDetection(self):
             except JobGraphDeadlockException:
                 pass  # This is the expected behaviour
 
-            def checkChildEdgeCycleDetection(fNode, tNode):
+            def checkChildEdgeCycleDetection(fNode, tNode):                
                 childEdges.add((fNode, tNode))  # Create a cycle
                 adjacencyList[fNode].add(tNode)
                 self.assertTrue(not self.isAcyclic(adjacencyList))
@@ -161,19 +209,7 @@ def checkChildEdgeCycleDetection(fNode, tNode):
                 # Check is now acyclic again
                 self.makeJobGraph(nodeNumber, childEdges,
                                   followOnEdges, None).checkJobGraphAcylic()
-
-            # Now try adding edges that create a cycle
-
-            # Try adding a child edge from a descendant to an ancestor
-            fNode, tNode = self.getRandomEdge(nodeNumber)
-            while fNode not in self.reachable(tNode, adjacencyList):
-                fNode, tNode = self.getRandomEdge(nodeNumber)
-            checkChildEdgeCycleDetection(fNode, tNode)
-
-            # Try adding a self child edge
-            node = random.choice(xrange(nodeNumber))
-            checkChildEdgeCycleDetection(node, node)
-
+                                  
             def checkFollowOnEdgeCycleDetection(fNode, tNode):
                 followOnEdges.add((fNode, tNode))  # Create a cycle
                 try:
@@ -188,34 +224,45 @@ def checkFollowOnEdgeCycleDetection(fNode, tNode):
                 self.makeJobGraph(nodeNumber, childEdges,
                                   followOnEdges, None).checkJobGraphAcylic()
 
-            # Try adding a follow on edge from a descendant to an ancestor
+            # Now try adding edges that create a cycle
+
+            # Pick a random existing order relationship
             fNode, tNode = self.getRandomEdge(nodeNumber)
-            while fNode not in self.reachable(tNode, adjacencyList):
-                fNode, tNode = self.getRandomEdge(nodeNumber)
-            checkFollowOnEdgeCycleDetection(fNode, tNode)
+            while tNode not in self.reachable(fNode, adjacencyList):
+                fNode, tNode = self.getRandomEdge(nodeNumber)  
+            
+            # Try creating a cycle of child edges
+            checkChildEdgeCycleDetection(tNode, fNode)   
 
-            # Try adding a self follow on edge
+            # Try adding a self child edge
             node = random.choice(xrange(nodeNumber))
+            checkChildEdgeCycleDetection(node, node)
+
+            # Try adding a follow on edge from a descendant to an ancestor
+            checkFollowOnEdgeCycleDetection(tNode, fNode)
+
+            # Try adding a self follow on edge
             checkFollowOnEdgeCycleDetection(node, node)
 
             # Try adding a follow on edge between two nodes with shared descendants
             fNode, tNode = self.getRandomEdge(nodeNumber)
             if (len(self.reachable(tNode, adjacencyList)
                         .intersection(self.reachable(fNode, adjacencyList))) > 0
-                and (fNode, tNode) not in childEdges):
+                and (fNode, tNode) not in childEdges and (fNode, tNode) not in followOnEdges):
                 checkFollowOnEdgeCycleDetection(fNode, tNode)
 
     def testEvaluatingRandomDAG(self):
         """
-        Randomly generate test input then check that the ordering of the running respected the
-        constraints.
+        Randomly generate test input then check that the job graph can be 
+        run successfully, using the existence of promises
+        to validate the run.
         """
         jobStore = self._getTestJobStorePath()
-        for test in xrange(30):
+        for test in xrange(10):
             # Temporary file
             tempDir = self._createTempDir(purpose='tempDir')
             # Make a random DAG for the set of child edges
-            nodeNumber = random.choice(xrange(2, 20))
+            nodeNumber = random.choice(xrange(2, 8))
             childEdges = self.makeRandomDAG(nodeNumber)
             # Get an adjacency list representation and check is acyclic
             adjacencyList = self.getAdjacencyList(nodeNumber, childEdges)
@@ -228,10 +275,33 @@ def testEvaluatingRandomDAG(self):
             rootJob = self.makeJobGraph(nodeNumber, childEdges, followOnEdges, tempDir)
             # Run the job  graph
             options = Job.Runner.getDefaultOptions("%s.%i" % (jobStore, test))
-            options.retryCount = 100
-            options.badWorker = 0.5
+            options.logLevel = "DEBUG"
+            options.retryCount = 1
+            options.badWorker = 0.25
             options.badWorkerFailInterval = 0.01
-            Job.Runner.startToil(rootJob, options)
+
+            # Now actually run the workflow
+            try:
+                with Toil(options) as toil:
+                    toil.start(rootJob)
+                numberOfFailedJobs = 0
+            except FailedJobsException as e:
+                numberOfFailedJobs = e.numberOfFailedJobs
+            
+            # Restart until successful or failed
+            totalTrys = 1
+            options.restart = True
+            while numberOfFailedJobs != 0:
+                try:
+                    with Toil(options) as toil:
+                        toil.restart()
+                    numberOfFailedJobs = 0
+                except FailedJobsException as e:
+                    numberOfFailedJobs = e.numberOfFailedJobs
+                    if totalTrys > 32: #p(fail after this many restarts) = 0.5**32
+                        self.fail() #Exceeded a reasonable number of restarts    
+                    totalTrys += 1
+            
             # For each job check it created a valid output file and add the ordering
             # relationships contained within the output file to the ordering relationship,
             # so we can check they are compatible with the relationships defined by the job DAG.
@@ -241,7 +311,7 @@ def testEvaluatingRandomDAG(self):
                     ordering = map(int, fH.readline().split())
                     self.assertEquals(int(ordering[-1]), i)
                     for j in ordering[:-1]:
-                        adjacencyList[int(j)].add(int(ordering[-1]))
+                        adjacencyList[int(j)].add(i)
             # Check the ordering retains an acyclic graph
             if not self.isAcyclic(adjacencyList):
                 print "ORDERING", ordering
@@ -254,7 +324,7 @@ def testEvaluatingRandomDAG(self):
     def getRandomEdge(nodeNumber):
         assert nodeNumber > 1
         fNode = random.choice(xrange(nodeNumber - 1))
-        return fNode, random.choice(xrange(fNode, nodeNumber))
+        return fNode, random.choice(xrange(fNode+1, nodeNumber))
 
     @staticmethod
     def makeRandomDAG(nodeNumber):
@@ -268,7 +338,7 @@ def makeRandomDAG(nodeNumber):
         # Make a spanning tree of edges so that nodes are connected
         edges = set(map(lambda i: (random.choice(xrange(i)), i), xrange(1, nodeNumber)))
         # Add extra random edges until there are edgeNumber edges
-        while edgeNumber < len(edges):
+        while len(edges) < edgeNumber:
             edges.add(JobTest.getRandomEdge(nodeNumber))
         return edges
 
@@ -292,11 +362,9 @@ def reachable(self, node, adjacencyList, followOnAdjacencyList=None):
         def dfs(fNode):
             if fNode not in visited:
                 visited.add(fNode)
-                for tNode in adjacencyList[fNode]:
-                    dfs(tNode)
+                map(dfs, adjacencyList[fNode])
                 if followOnAdjacencyList is not None:
-                    for tNode in followOnAdjacencyList[fNode]:
-                        dfs(tNode)
+                    map(dfs, followOnAdjacencyList[fNode])
 
         dfs(node)
         return visited
@@ -312,15 +380,19 @@ def makeAugmentedAdjacencyList():
                                       for i in range(len(childAdjacencyList))]
 
             def addImpliedEdges(node, followOnEdges):
+                # Let node2 be a child of node or a successor of a child of node.
+                # For all node2 the following adds an edge to the augmented 
+                # adjacency list from node2 to each followOn of node
+                
                 visited = set()
 
-                def f(node):
-                    if node not in visited:
-                        visited.add(node)
+                def f(node2):
+                    if node2 not in visited:
+                        visited.add(node2)
                         for i in followOnEdges:
-                            augmentedAdjacencyList[node].add(i)
-                        map(f, childAdjacencyList[node])
-                        map(f, followOnAdjacencyList[node])
+                            augmentedAdjacencyList[node2].add(i)
+                        map(f, childAdjacencyList[node2])
+                        map(f, followOnAdjacencyList[node2])
 
                 map(f, childAdjacencyList[node])
 
@@ -333,12 +405,6 @@ def f(node):
         # Loop to create the follow on edges (try 1000 times)
         while random.random() > 0.001:
             fNode, tNode = JobTest.getRandomEdge(len(childAdjacencyList))
-            # Get the  descendants of fNode not on a path of edges starting with a follow-on edge
-            # from fNode
-            fDescendants = reduce(set.union,
-                                  (self.reachable(c, childAdjacencyList, followOnAdjacencyList)
-                                   for c in childAdjacencyList[fNode]), set())
-            fDescendants.add(fNode)
 
             # Make an adjacency list including augmented edges and proposed
             # follow on edge
@@ -423,7 +489,6 @@ def cyclic(fNode, visited, stack):
                 return False
         return True
 
-
 def fn1Test(string, outputFile):
     """
     Function appends string to output file, then returns the next ascii character of the first
@@ -446,5 +511,43 @@ def fn2Test(pStrings, s, outputFile):
     return s
 
 
+def trivialParent(job):
+    strandedJob = JobFunctionWrappingJob(child)
+    failingJob = JobFunctionWrappingJob(errorChild)
+
+    job.addChild(failingJob)
+    job.addChild(strandedJob)
+    failingJob.addChild(strandedJob)
+
+
+def parent(job):
+    childJob = JobFunctionWrappingJob(child)
+    strandedJob = JobFunctionWrappingJob(child)
+    failingJob = JobFunctionWrappingJob(errorChild)
+
+    job.addChild(childJob)
+    job.addChild(strandedJob)
+    childJob.addChild(failingJob)
+    failingJob.addChild(strandedJob)
+
+
+def diamond(job):
+    childJob = JobFunctionWrappingJob(child)
+    strandedJob = JobFunctionWrappingJob(child)
+    failingJob = JobFunctionWrappingJob(errorChild)
+
+    job.addChild(childJob)
+    job.addChild(failingJob)
+    childJob.addChild(strandedJob)
+    failingJob.addChild(strandedJob)
+
+def child(job):
+    pass
+
+
+def errorChild(job):
+    raise RuntimeError('Child failure')
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/src/toil/test/src/restartDAGTest.py b/src/toil/test/src/restartDAGTest.py
new file mode 100644
--- /dev/null
+++ b/src/toil/test/src/restartDAGTest.py
@@ -0,0 +1,157 @@
+# Copyright (C) 2015-2016 Regents of the University of California
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import logging
+
+from toil.common import Toil
+from toil.job import Job
+from toil.leader import FailedJobsException
+from toil.test import ToilTest
+
+import inspect
+import os
+import shutil
+import signal
+
+
+logger = logging.getLogger(__name__)
+
+
+class RestartDAGTest(ToilTest):
+    """
+    Tests that restarted job DAGs don't run children of jobs that failed in the first run till the
+    parent completes successfully in the restart.
+    """
+    def setUp(self):
+        super(RestartDAGTest, self).setUp()
+        self.tempDir = self._createTempDir(purpose='tempDir')
+        self.testJobStore = self._getTestJobStorePath()
+
+    def tearDown(self):
+        super(RestartDAGTest, self).tearDown()
+        shutil.rmtree(self.testJobStore)
+
+    def testRestartedWorkflowSchedulesCorrectJobsOnFailedParent(self):
+        self._testRestartedWorkflowSchedulesCorrectJobs('raise')
+
+    def testRestartedWorkflowSchedulesCorrectJobsOnKilledParent(self):
+        self._testRestartedWorkflowSchedulesCorrectJobs('kill')
+
+    def _testRestartedWorkflowSchedulesCorrectJobs(self, failType):
+        """
+        Creates a diamond DAG
+            /->passingParent-\
+        root                 |-->child
+           \->failingParent--/
+
+        where root and passingParent are guaranteed to pass, while failingParent will fail.
+        child should not run on start or restart and we assert that by ensuring that a file create
+        iff child is run is not present on the system.
+
+        :param str failType: Does failingParent fail on an assertionError, or is it killed.
+        """
+        # Specify options
+        options = Job.Runner.getDefaultOptions(self.testJobStore)
+        options.logLevel = 'DEBUG'
+        options.retryCount = 0
+        options.clean = "never"
+
+        parentFile = os.path.join(self.tempDir, 'parent')
+        childFile = os.path.join(self.tempDir, 'child')
+
+        # Make the first job
+        root = Job.wrapJobFn(passingFn)
+        passingParent = Job.wrapJobFn(passingFn)
+        failingParent = Job.wrapJobFn(failingFn, failType=failType, fileName=parentFile)
+        child = Job.wrapJobFn(passingFn, fileName=childFile)
+
+        # define the DAG
+        root.addChild(passingParent)
+        root.addChild(failingParent)
+        passingParent.addChild(child)
+        failingParent.addChild(child)
+
+        failReasons = []
+
+        assert not os.path.exists(childFile)
+
+        # Run the test
+        for runMode in 'start', 'restart':
+            self.errorRaised = None
+            try:
+                with Toil(options) as toil:
+                    if runMode == 'start':
+                        toil.start(root)
+                    else:
+                        toil.restart()
+            except Exception as e:
+                logger.exception(e)
+                self.errorRaised = e
+            finally:
+                # The processing of an AssertionError and FailedJobsException is the same so we do
+                # it together in this finally clause.
+                if self.errorRaised is not None:
+                    if not os.path.exists(parentFile):
+                        failReasons.append('The failing parent file did not exist on toil "%s".'
+                                           % runMode)
+                    if os.path.exists(childFile):
+                        failReasons.append('The child file existed.  i.e. the child was run on '
+                                           'toil "%s".' % runMode)
+                    if isinstance(self.errorRaised, FailedJobsException):
+                        if self.errorRaised.numberOfFailedJobs != 3:
+                            failReasons.append('FailedJobsException was raised on toil "%s" but '
+                                               'the number of failed jobs (%s) was not 3.'
+                                               % (runMode, self.errorRaised.numberOfFailedJobs))
+                    elif isinstance(self.errorRaised, AssertionError):
+                        failReasons.append('Toil raised an AssertionError instead of a '
+                                           'FailedJobsException on toil "%s".' % runMode)
+                    else:
+                        failReasons.append("Toil raised error: %s" % self.errorRaised)
+                    self.errorRaised = None
+                    options.restart = True
+                else:
+                    self.fail('No errors were raised on toil "%s".' % runMode)
+        if failReasons:
+            self.fail('Test failed for (%s) reasons:\n\t%s' % (len(failReasons),
+                                                               '\n\t'.join(failReasons)))
+
+def passingFn(job, fileName=None):
+    """
+    This function is guaranteed to pass as it does nothing out of the ordinary.  If fileName is
+    provided, it will be created.
+
+    :param str fileName: The name of a file that must be created if provided.
+    """
+    if fileName is not None:
+        # Emulates system touch.
+        open(fileName, 'w').close()
+
+def failingFn(job, failType, fileName):
+    """
+    This function is guaranteed to fail via a raised assertion, or an os.kill
+
+    :param job: Job
+    :param str failType: 'raise' or 'kill
+    :param str fileName: The name of a file that must be created.
+    """
+    assert failType in ('raise', 'kill')
+    # Use that function to avoid code redundancy
+    passingFn(job, fileName)
+
+    if failType == 'raise':
+        assert False
+    else:
+        os.kill(os.getpid(), signal.SIGKILL)
diff --git a/src/toil/test/src/resumabilityTest.py b/src/toil/test/src/resumabilityTest.py
new file mode 100644
--- /dev/null
+++ b/src/toil/test/src/resumabilityTest.py
@@ -0,0 +1,79 @@
+# Copyright (C) 2016 UCSC Computational Genomics Lab
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+import os
+from StringIO import StringIO
+from toil.job import Job
+from toil.test import ToilTest
+from toil.jobStores.abstractJobStore import NoSuchFileException
+from toil.leader import FailedJobsException
+
+class ResumabilityTest(ToilTest):
+    """
+    https://github.com/BD2KGenomics/toil/issues/808
+    """
+    def test(self):
+        """
+        Tests that a toil workflow that fails once can be resumed without a NoSuchJobException.
+        """
+        options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
+        options.logLevel = "INFO"
+        root = Job.wrapJobFn(parent)
+        with self.assertRaises(FailedJobsException):
+            # This one is intended to fail.
+            Job.Runner.startToil(root, options)
+
+        # Resume the workflow. Unfortunately, we have to check for
+        # this bug using the logging output, since although the
+        # NoSuchJobException causes the worker to fail, the batch
+        # system code notices that the job has been deleted despite
+        # the failure and avoids the failure.
+        options.restart = True
+        tempDir = self._createTempDir()
+        options.logFile = os.path.join(tempDir, "log.txt")
+        Job.Runner.startToil(root, options)
+        with open(options.logFile) as f:
+            logString = f.read()
+            # We are looking for e.g. "Batch system is reporting that
+            # the jobWrapper with batch system ID: 1 and jobWrapper
+            # store ID: n/t/jobwbijqL failed with exit value 1"
+            self.assertTrue("failed with exit value" not in logString)
+
+def parent(job):
+    """
+    Set up a bunch of dummy child jobs, and a bad job that needs to be
+    restarted as the follow on.
+    """
+    for _ in xrange(5):
+        job.addChildJobFn(goodChild)
+    job.addFollowOnJobFn(badChild)
+
+def goodChild(job):
+    """
+    Does nothing.
+    """
+    return
+
+def badChild(job):
+    """
+    Fails the first time it's run, succeeds the second time.
+    """
+    try:
+        with job.fileStore.jobStore.readSharedFileStream("alreadyRun") as fileHandle:
+            fileHandle.read()
+    except NoSuchFileException:
+        with job.fileStore.jobStore.writeSharedFileStream("alreadyRun", isProtected=False) as fileHandle:
+            fileHandle.write("failed once\n")
+            raise RuntimeError("this is an expected error")