Repository: DataBiosphere/toil

Issue:
change return value behavior of Job.Runner.startToil
Currently Job.Runner.startToil returns the number of failed jobs. I think this is, on reflection, not very pythonic and easy to miss. I think it would be better if it threw some kind unfinished exception if there were failed jobs and instead returned None. As this is an interface changing I'm labelling it for the current release

change return value behavior of Job.Runner.startToil
Currently Job.Runner.startToil returns the number of failed jobs. I think this is, on reflection, not very pythonic and easy to miss. I think it would be better if it threw some kind unfinished exception if there were failed jobs and instead returned None. As this is an interface changing I'm labelling it for the current release



Discussion:

Resolving patch:
diff --git a/src/toil/job.py b/src/toil/job.py
--- a/src/toil/job.py
+++ b/src/toil/job.py
@@ -345,7 +345,10 @@ def startToil(job, options):
             """
             Runs the toil workflow using the given options 
             (see Job.Runner.getDefaultOptions and Job.Runner.addToilOptions) 
-            starting with this job.
+            starting with this job. 
+            
+            :raises: toil.leader.FailedJobsException if at the end of function their remain
+    failed jobs
             """
             setLoggingFromOptions(options)
             with setupToil(options, userScript=job.getUserScript()) as (config, batchSystem, jobStore):
@@ -356,7 +359,7 @@ def startToil(job, options):
                     #Setup the first wrapper.
                     rootJob = job._serialiseFirstJob(jobStore)
                 return mainLoop(config, batchSystem, jobStore, rootJob)
-        
+
     class FileStore:
         """
         Class used to manage temporary files and log messages, 
diff --git a/src/toil/leader.py b/src/toil/leader.py
--- a/src/toil/leader.py
+++ b/src/toil/leader.py
@@ -304,9 +304,18 @@ def _buildToilState(self, job, jobStore):
                     #but we add back a predecessor link
                     self.successorJobStoreIDToPredecessorJobs[successorJobStoreID].append(job)
 
+class FailedJobsException( Exception ):
+    def __init__( self, jobStoreString, numberOfFailedJobs ):
+        super( FailedJobsException, self ).__init__( "The job store '%s' contains %i failed jobs" % (jobStoreString, numberOfFailedJobs))
+        self.jobStoreString = jobStoreString
+        self.numberOfFailedJobs = numberOfFailedJobs
+        
 def mainLoop(config, batchSystem, jobStore, rootJob):
     """
     This is the main loop from which jobs are issued and processed.
+    
+    :raises: toil.leader.FailedJobsException if at the end of function their remain
+    failed jobs
     """
 
     ##########################################
@@ -490,5 +499,6 @@ def mainLoop(config, batchSystem, jobStore, rootJob):
         clean = config.clean
         if clean=="always" or clean == "onError" and totalFailedJobs>0 or clean == "onSuccess" and totalFailedJobs==0:
             jobStore.deleteJobStore()
-
-    return totalFailedJobs #Returns number of failed jobs
+    
+    if totalFailedJobs > 0:
+        raise FailedJobsException( config.jobStore, totalFailedJobs )
diff --git a/src/toil/utils/toilRestart.py b/src/toil/utils/toilRestart.py
--- a/src/toil/utils/toilRestart.py
+++ b/src/toil/utils/toilRestart.py
@@ -58,7 +58,7 @@ def main():
     options.restart = True
     with setupToil(options) as (config, batchSystem, jobStore):
         jobStore.clean()
-        return mainLoop(config, batchSystem, jobStore, Job._loadRootJob(jobStore))
+        mainLoop(config, batchSystem, jobStore, Job._loadRootJob(jobStore))
     
 def _test():
     import doctest      


Test patch:
diff --git a/src/toil/test/dependencies/dependenciesTest.py b/src/toil/test/dependencies/dependenciesTest.py
--- a/src/toil/test/dependencies/dependenciesTest.py
+++ b/src/toil/test/dependencies/dependenciesTest.py
@@ -96,12 +96,9 @@ def fn(tree, maxCores, maxThreads, size, coresPerJob, sleepTime):
             options.maxCores = maxCores
 
             baseJob = FirstJob(tree, "Anc00", sleepTime, startTime, int(coresPerJob))
-            i = Job.Runner.startToil(baseJob, options)
-
+            Job.Runner.startToil(baseJob, options)
             checkLog(logName, maxCores, maxThreads, coresPerJob, sleepTime)
 
-            self.assertEquals(i, 0)
-
             os.close(logFd)
 
         fn("comb", 10, 100, 100, 1, 10)
diff --git a/src/toil/test/mesos/stress.py b/src/toil/test/mesos/stress.py
--- a/src/toil/test/mesos/stress.py
+++ b/src/toil/test/mesos/stress.py
@@ -71,8 +71,7 @@ def main(numJobs):
 
     # Launch first toil Job
     i = LongTestJob( numJobs )
-    j = Job.Runner.startToil(i,  options )
-    assert(j==0) # confirm that no jobs failed
+    Job.Runner.startToil(i,  options )
 
 if __name__=="__main__":
     main(numJobs=5)
diff --git a/src/toil/test/sort/sortTest.py b/src/toil/test/sort/sortTest.py
old mode 100644
new mode 100755
--- a/src/toil/test/sort/sortTest.py
+++ b/src/toil/test/sort/sortTest.py
@@ -33,6 +33,7 @@
 from toil.test.sort.sort import setup
 from toil.test import ToilTest, needs_aws, needs_mesos
 from toil.jobStores.abstractJobStore import JobStoreCreationException
+from toil.leader import FailedJobsException
 
 log = logging.getLogger(__name__)
 
@@ -93,7 +94,11 @@ def toilSortTest(self, testNo=1, batchSystem="singleMachine", jobStore='file',
             options.restart = False
 
             # Now actually run the workflow
-            i = Job.Runner.startToil(firstJob, options)
+            try:
+                Job.Runner.startToil(firstJob, options)
+                i = 0
+            except FailedJobsException as e:
+                i = e.numberOfFailedJobs
 
             # Check we get an exception if we try to run without restart on an existing job store
             try:
@@ -107,7 +112,11 @@ def toilSortTest(self, testNo=1, batchSystem="singleMachine", jobStore='file',
             # This loop tests the restart behavior
             while i != 0:
                 options.useExistingOptions = random.random() > 0.5
-                i = Job.Runner.startToil(firstJob, options)
+                try:
+                    Job.Runner.startToil(firstJob, options)
+                    i = 0
+                except FailedJobsException as e:
+                    i = e.numberOfFailedJobs
 
             # Now check that if you try to restart from here it will raise an exception
             # indicating that there are no jobs remaining in the workflow.
diff --git a/src/toil/test/src/jobEncapsulationTest.py b/src/toil/test/src/jobEncapsulationTest.py
--- a/src/toil/test/src/jobEncapsulationTest.py
+++ b/src/toil/test/src/jobEncapsulationTest.py
@@ -49,7 +49,7 @@ def testEncapsulation(self):
             options.jobStore = self._getTestJobStorePath()
             options.logLevel = "INFO"
             # Run the workflow, the return value being the number of failed jobs
-            self.assertEquals(T.Runner.startToil(a, options), 0)
+            T.Runner.startToil(a, options)
             # Check output
             self.assertEquals(open(outFile, 'r').readline(), "ABCDE")
         finally:
diff --git a/src/toil/test/src/jobServiceTest.py b/src/toil/test/src/jobServiceTest.py
--- a/src/toil/test/src/jobServiceTest.py
+++ b/src/toil/test/src/jobServiceTest.py
@@ -65,7 +65,7 @@ def testService(self):
             options.logLevel = "INFO"
             options.jobStore = self._getTestJobStorePath()
             # Run the workflow, the return value being the number of failed jobs
-            self.assertEquals(Job.Runner.startToil(t, options), 0)
+            Job.Runner.startToil(t, options)
             # Check output
             self.assertEquals(open(outFile, 'r').readline(), "123")
         finally:
diff --git a/src/toil/test/src/jobTest.py b/src/toil/test/src/jobTest.py
--- a/src/toil/test/src/jobTest.py
+++ b/src/toil/test/src/jobTest.py
@@ -62,7 +62,7 @@ def testStatic(self):
             options.jobStore = self._getTestJobStorePath()
             options.logLevel = "INFO"
             # Run the workflow, the return value being the number of failed jobs
-            self.assertEquals(Job.Runner.startToil(A, options), 0)
+            Job.Runner.startToil(A, options)
 
             # Check output
             self.assertEquals(open(outFile, 'r').readline(), "ABCDEF")
@@ -187,8 +187,7 @@ def testEvaluatingRandomDAG(self):
             # Run the job  graph
             options = Job.Runner.getDefaultOptions()
             options.jobStore = "%s.%i" % (jobStore, test)
-            failedJobs = Job.Runner.startToil(rootJob, options)
-            self.assertEquals(failedJobs, 0)
+            Job.Runner.startToil(rootJob, options)
             # Get the ordering add the implied ordering to the graph
             with open(outFile, 'r') as fH:
                 ordering = map(int, fH.readline().split())