Repository: DataBiosphere/toil

Issue:
Control-C doesn't stop a Toil script run completely
When using control-c to stop a toil script, it successfully shuts down the batch system and stops execution of commands, but doesn't exit the python script and relinquish control of the command prompt to the user. The python toil script child processes are still running.

However, I can clean the jobstore using the `toil clean <jobstore_name>` on an Azure storage account. And after manually killing the python toil child processes, I can rerun another instance of toil using the same jobstore name.



Discussion:
My hunch is that this is caused by the use of multiprocessing.Process for the stats-and-logging aggregator in leader.py. multiprocessing is notorious for interfering with Ctrl-C. This is how s3am works around it:

https://github.com/BD2KGenomics/s3am/blob/9f7d35e18ce3a139a601d0ded94a0bfddc2a4858/src/s3am/operations.py#L1067

Anyhow, we might just eliminate Process and use Thread for this. Process is only needed on CPU-bound  tasks that are likely to contend on the GIL. 

But first, we need to replicate the described behavior and see if the simple fix employed by s3am resolves the issue.


Resolving patch:
diff --git a/src/toil/leader.py b/src/toil/leader.py
--- a/src/toil/leader.py
+++ b/src/toil/leader.py
@@ -23,8 +23,6 @@
 import time
 from Queue import Queue, Empty
 from collections import namedtuple
-from multiprocessing import Event as ProcessEvent
-from multiprocessing import Process
 from threading import Thread, Event
 
 from bd2k.util.expando import Expando
@@ -41,21 +39,21 @@
 
 class StatsAndLogging( object ):
     """
-    Class manages process to aggregate statistics and logging information on a toil run.
+    Class manages a thread that aggregates statistics and logging information on a toil run.
     """
 
     def __init__(self, jobStore):
-        # Start the stats/logging aggregation process
-        self._stop = ProcessEvent()
-        self._worker = Process(target=self.statsAndLoggingAggregatorProcess,
-                         args=(jobStore, self._stop))
+        # Start the stats/logging aggregation thread
+        self._stop = Event()
+        self._worker = Thread(target=self.statsAndLoggingAggregator,
+                              args=(jobStore, self._stop))
         self._worker.start()
 
     @staticmethod
-    def statsAndLoggingAggregatorProcess(jobStore, stop):
+    def statsAndLoggingAggregator(jobStore, stop):
         """
         The following function is used for collating stats/reporting log messages from the workers.
-        Works inside of a separate process, collates as long as the stop flag is not True.
+        Works inside of a thread, collates as long as the stop flag is not True.
         """
         #  Overall timing
         startTime = time.time()
@@ -82,7 +80,7 @@ def callback(fileHandle):
                     logger.info("%s:    %s", log.jobStoreID, log.text)
 
         while True:
-            # This is a indirect way of getting a message to the process to exit
+            # This is a indirect way of getting a message to the thread to exit
             if stop.is_set():
                 jobStore.readStatsAndLogging(callback)
                 break
@@ -96,22 +94,20 @@ def callback(fileHandle):
 
     def check(self):
         """
-        Check on the stats and logging process.
-        :raise RuntimeError: If the underlying process has quit.
+        Check on the stats and logging aggregator.
+        :raise RuntimeError: If the underlying thread has quit.
         """
         if not self._worker.is_alive():
-            raise RuntimeError("Stats and logging process has quit with exit code: %d." % self._worker.exitcode)
+            raise RuntimeError("Stats and logging thread has quit")
 
     def shutdown(self):
         """
-        Finish up the stats/logging aggregation process
+        Finish up the stats/logging aggregation thread
         """
-        logger.info('Waiting for stats and logging collator process to finish ...')
+        logger.info('Waiting for stats and logging collator thread to finish ...')
         startTime = time.time()
         self._stop.set()
         self._worker.join()
-        if self._worker.exitcode != 0:
-            raise RuntimeError('Stats/logging collator failed with exit code %d.' % self._worker.exitcode)
         logger.info('... finished collating stats and logs. Took %s seconds', time.time() - startTime)
         # in addition to cleaning on exceptions, onError should clean if there are any failed jobs
 
@@ -665,7 +661,7 @@ def mainLoop(config, batchSystem, provisioner, jobStore, rootJobWrapper, jobCach
         jobBatcher = JobBatcher(config, batchSystem, jobStore, toilState, serviceManager)
         logger.info("Found %s jobs to start and %i jobs with successors to run",
                     len(toilState.updatedJobs), len(toilState.successorCounts))
-        # Start the stats/logging aggregation process
+        # Start the stats/logging aggregation thread
         statsAndLogging = StatsAndLogging(jobStore)
         try:
             # Create cluster scaling processes if the provisioner is not None
@@ -683,7 +679,7 @@ def mainLoop(config, batchSystem, provisioner, jobStore, rootJobWrapper, jobCach
                     clusterScaler.shutdown()
                     logger.info('Worker shutdown complete in %s seconds', time.time() - startTime)
         finally:
-            # Shutdown the stats and logging process
+            # Shutdown the stats and logging thread
             statsAndLogging.shutdown()
     finally:
         serviceManager.shutdown()


Test patch: