Repository: DataBiosphere/toil

Issue:
Another wonky try: finally: in the services code
https://github.com/BD2KGenomics/toil/commit/fb8eba9b87f4698bcd026c3f64f17338dbf0c661#diff-4eeea14b1dc6e72c2dcb9ad1471d1cf3R642

is broken, please see

https://github.com/BD2KGenomics/toil/wiki/Python-Dos-and-Don'ts#dont-piggy-back-resource-deallocation

Another wonky try: finally: in the services code
https://github.com/BD2KGenomics/toil/commit/fb8eba9b87f4698bcd026c3f64f17338dbf0c661#diff-4eeea14b1dc6e72c2dcb9ad1471d1cf3R642

is broken, please see

https://github.com/BD2KGenomics/toil/wiki/Python-Dos-and-Don'ts#dont-piggy-back-resource-deallocation



Discussion:
Cleared milestone prior to tomorrow's planning.

Sorry, @hannes-ucsc , which one specifically? Line number/file would would help

@benedictpaten, the line I link to in the description is the piggy-backed cleanup. It should get its own `try: except:` block or a comment explaining why the piggy-backing is ok and intended. For an explanation of what I mean by "piggy-backing" see the wiki page I link to in the description.

Cleared milestone prior to tomorrow's planning.

Sorry, @hannes-ucsc , which one specifically? Line number/file would would help

@benedictpaten, the line I link to in the description is the piggy-backed cleanup. It should get its own `try: except:` block or a comment explaining why the piggy-backing is ok and intended. For an explanation of what I mean by "piggy-backing" see the wiki page I link to in the description.


Resolving patch:
diff --git a/src/toil/leader.py b/src/toil/leader.py
--- a/src/toil/leader.py
+++ b/src/toil/leader.py
@@ -637,44 +637,47 @@ def mainLoop(config, batchSystem, provisioner, jobStore, rootJobWrapper, jobCach
     toilState = ToilState(jobStore, rootJobWrapper, jobCache=jobCache)
 
     # Create a service manager to start and terminate services
-    serviceManager = ServiceManager(jobStore)
-
-    assert len(batchSystem.getIssuedBatchJobIDs()) == 0 #Batch system must start with no active jobs!
-    logger.info("Checked batch system has no running jobs and no updated jobs")
-
-    # Load the jobBatcher class - used to track jobs submitted to the batch-system
-    jobBatcher = JobBatcher(config, batchSystem, jobStore, toilState, serviceManager)
-    logger.info("Found %s jobs to start and %i jobs with successors to run",
-                len(toilState.updatedJobs), len(toilState.successorCounts))
-
-    # Start the stats/logging aggregation process
-    statsAndLogging = StatsAndLogging(jobStore)
     try:
-        # Create cluster scaling processes if the provisioner is not None
-        if provisioner is None:
-            clusterScaler = None
-        else:
-            clusterScaler = ClusterScaler(provisioner, jobBatcher, config)
-            jobBatcher.clusterScaler = clusterScaler
+        serviceManager = ServiceManager(jobStore)
+    
+        assert len(batchSystem.getIssuedBatchJobIDs()) == 0 #Batch system must start with no active jobs!
+        logger.info("Checked batch system has no running jobs and no updated jobs")
+    
+        # Load the jobBatcher class - used to track jobs submitted to the batch-system
+        jobBatcher = JobBatcher(config, batchSystem, jobStore, toilState, serviceManager)
+        logger.info("Found %s jobs to start and %i jobs with successors to run",
+                    len(toilState.updatedJobs), len(toilState.successorCounts))
+    
         try:
-            innerLoop(jobStore, config, batchSystem, toilState, jobBatcher, serviceManager, statsAndLogging)
+            # Start the stats/logging aggregation process
+            statsAndLogging = StatsAndLogging(jobStore)
+            
+            try:
+                # Create cluster scaling processes if the provisioner is not None
+                if provisioner is None:
+                    clusterScaler = None
+                else:
+                    clusterScaler = ClusterScaler(provisioner, jobBatcher, config)
+                    jobBatcher.clusterScaler = clusterScaler
+                innerLoop(jobStore, config, batchSystem, toilState, jobBatcher, serviceManager, statsAndLogging)
+            finally:
+                if provisioner is not None:
+                    logger.info('Waiting for workers to shutdown')
+                    startTime = time.time()
+                    clusterScaler.shutdown()
+                    logger.info('Worker shutdown complete in %s seconds', time.time() - startTime)
         finally:
-            if provisioner is not None:
-                logger.info('Waiting for workers to shutdown')
-                startTime = time.time()
-                clusterScaler.shutdown()
-                logger.info('Worker shutdown complete in %s seconds', time.time() - startTime)
+            # Shutdown the stats and logging process
+            statsAndLogging.shutdown()
     finally:
-        # Shutdown the stats and logging process
-        statsAndLogging.shutdown()
         serviceManager.shutdown()
 
 
     # Filter the failed jobs
     toilState.totalFailedJobs = set(filter(jobStore.exists, toilState.totalFailedJobs))
 
-    logger.info("Finished toil run with %s failed jobs" %
-                 ("no" if len(toilState.totalFailedJobs) == 0 else len(toilState.totalFailedJobs)))
+    logger.info("Finished toil run %s" %
+                 ("successfully" if len(toilState.totalFailedJobs) == 0 else ("with %s failed jobs" % len(toilState.totalFailedJobs))))
     if len(toilState.totalFailedJobs):
         logger.info("Failed jobs at end of the run: %s", toilState.totalFailedJobs)
 


Test patch: