Repository: DataBiosphere/toil

Issue:
Fork-free mode for debugging
So that we can use `ipdb` to debug Toil


Discussion:
Is this mostly for the system calls in src/toil/test/utils/utilsTest.py?
Forking also happens during regular execution of Toil outside of a testing
I've finally taken some time to look into this and it looks like the singleMachine batchSystem uses threads. I haven't yet found where else forking might be happening. @mr-c can you point me to where you were running into problems?
Nevermind, I see that jobs are launched with Popen.
After looking into this more, completing this will require more in-depth knowledge of how jobs are serialized, stored, and prepared to be run than I currently have... 

Perhaps I can revisit this in the indefinite future. Meanwhile I'll see what I can learn.

Resolving patch:
diff --git a/src/toil/batchSystems/singleMachine.py b/src/toil/batchSystems/singleMachine.py
--- a/src/toil/batchSystems/singleMachine.py
+++ b/src/toil/batchSystems/singleMachine.py
@@ -32,10 +32,10 @@
 
 # Python 3 compatibility imports
 from six.moves.queue import Empty, Queue
-from six.moves import xrange
 
 import toil
-from toil.batchSystems.abstractBatchSystem import BatchSystemSupport, InsufficientSystemResources
+from toil.batchSystems.abstractBatchSystem import BatchSystemSupport
+from toil import worker as toil_worker
 
 log = logging.getLogger(__name__)
 
@@ -85,6 +85,7 @@ def __init__(self, config, maxCores, maxMemory, maxDisk):
         # (scale > 1).
         self.scale = config.scale
         # Number of worker threads that will be started
+        self.debugWorker = config.debugWorker
         self.numWorkers = int(old_div(self.maxCores, self.minCores))
         # A counter to generate job IDs and a lock to guard it
         self.jobIndex = 0
@@ -122,19 +123,62 @@ def __init__(self, config, maxCores, maxMemory, maxDisk):
         # A pool representing the available space in bytes
         self.disk = ResourcePool(self.maxDisk, 'disk', self.acquisitionTimeout)
 
-        log.debug('Setting up the thread pool with %i workers, '
-                 'given a minimum CPU fraction of %f '
-                 'and a maximum CPU value of %i.', self.numWorkers, self.minCores, maxCores)
-        for i in range(self.numWorkers):
-            worker = Thread(target=self.worker, args=(self.inputQueue,))
-            self.workerThreads.append(worker)
-            worker.start()
+        if not self.debugWorker:
+            log.debug('Setting up the thread pool with %i workers, '
+                     'given a minimum CPU fraction of %f '
+                     'and a maximum CPU value of %i.', self.numWorkers, self.minCores, maxCores)
+            for i in range(self.numWorkers):
+                worker = Thread(target=self.worker, args=(self.inputQueue,))
+                self.workerThreads.append(worker)
+                worker.start()
+        else:
+            log.debug('Started in worker debug mode.')
+
+    def _runWorker(self, jobCommand, jobID, environment):
+        """
+        Run the jobCommand using the worker and wait for it to finish.
+        The worker is forked unless it is a '_toil_worker' job and
+        debugWorker is True.
+        """
+        startTime = time.time()  # Time job is started
+        popen = None
+        statusCode = None
+        forkWorker = not (self.debugWorker and "_toil_worker" in jobCommand)
+        if forkWorker:
+            with self.popenLock:
+                popen = subprocess.Popen(jobCommand,
+                                         shell=True,
+                                         env=dict(os.environ, **environment))
+        else:
+            statusCode = toil_worker.main(jobCommand.split())
+            if statusCode is None:
+                statusCode = 0
+
+        info = Info(time.time(), popen, killIntended=False)
+        try:
+            self.runningJobs[jobID] = info
+            try:
+                if forkWorker:
+                    statusCode = popen.wait()
+                if 0 != statusCode:
+                    if statusCode != -9 or not info.killIntended:
+                        log.error("Got exit code %i (indicating failure) "
+                                  "from job %s.", statusCode,
+                                  self.jobs[jobID])
+            finally:
+                self.runningJobs.pop(jobID)
+        finally:
+            if statusCode is not None and not info.killIntended:
+                self.outputQueue.put((jobID, statusCode,
+                                      time.time() - startTime))
 
     # Note: The input queue is passed as an argument because the corresponding attribute is reset
     # to None in shutdown()
 
     def worker(self, inputQueue):
         while True:
+            if self.debugWorker and inputQueue.empty():
+                return
             args = inputQueue.get()
             if args is None:
                 log.debug('Received queue sentinel.')
@@ -151,28 +195,8 @@ def worker(self, inputQueue):
                                   jobCores)
                         with self.coreFractions.acquisitionOf(coreFractions):
                             with self.disk.acquisitionOf(jobDisk):
-                                startTime = time.time() #Time job is started
-                                with self.popenLock:
-                                    popen = subprocess.Popen(jobCommand,
-                                                             shell=True,
-                                                             env=dict(os.environ, **environment))
-                                statusCode = None
-                                info = Info(time.time(), popen, killIntended=False)
-                                try:
-                                    self.runningJobs[jobID] = info
-                                    try:
-                                        statusCode = popen.wait()
-                                        if 0 != statusCode:
-                                            if statusCode != -9 or not info.killIntended:
-                                                log.error("Got exit code %i (indicating failure) "
-                                                          "from job %s.", statusCode,
-                                                          self.jobs[jobID])
-                                    finally:
-                                        self.runningJobs.pop(jobID)
-                                finally:
-                                    if statusCode is not None and not info.killIntended:
-                                        self.outputQueue.put((jobID, statusCode,
-                                                              time.time() - startTime))
+                                self._runWorker(jobCommand, jobID, environment)
+
                 except ResourcePool.AcquisitionTimeoutException as e:
                     log.debug('Could not acquire enough (%s) to run job (%s). Requested: (%s), '
                               'Avaliable: %s. Sleeping for 10s.', e.resource, jobID, e.requested,
@@ -214,6 +238,8 @@ def issueBatchJob(self, jobNode):
         self.jobs[jobID] = jobNode.command
         self.inputQueue.put((jobNode.command, jobID, cores, jobNode.memory,
                              jobNode.disk, self.environment.copy()))
+        if self.debugWorker:  # then run immediately, blocking for return
+            self.worker(self.inputQueue)
         return jobID
 
     def killBatchJobs(self, jobIDs):
diff --git a/src/toil/common.py b/src/toil/common.py
--- a/src/toil/common.py
+++ b/src/toil/common.py
@@ -27,7 +27,6 @@
 import time
 import uuid
 from argparse import ArgumentParser
-from threading import Thread
 
 try:
     import cPickle as pickle
@@ -128,6 +127,7 @@ def __init__(self):
         self.useAsync = True
 
         #Debug options
+        self.debugWorker = False
         self.badWorker = 0.0
         self.badWorkerFailInterval = 0.01
 
@@ -263,6 +263,7 @@ def checkSse(sseKey):
         setOption("servicePollingInterval", float, fC(0.0))
 
         #Debug options
+        setOption("debugWorker")
         setOption("badWorker", float, fC(0.0, 1.0))
         setOption("badWorkerFailInterval", float, fC(0.0))
 
@@ -536,6 +537,10 @@ def _addOptions(addGroupFn, config):
     #Debug options
     #
     addOptionFn = addGroupFn("toil debug options", "Debug options")
+    addOptionFn("--debug-worker", default=False, action="store_true",
+            help="Experimental no forking mode for local debugging."
+                 " Specifically, workers are not forked and"
+                 " stderr/stdout are not redirected to the log.")
     addOptionFn("--badWorker", dest="badWorker", default=None,
                       help=("For testing purposes randomly kill 'badWorker' proportion of jobs using SIGKILL, default=%s" % config.badWorker))
     addOptionFn("--badWorkerFailInterval", dest="badWorkerFailInterval", default=None,
diff --git a/src/toil/worker.py b/src/toil/worker.py
--- a/src/toil/worker.py
+++ b/src/toil/worker.py
@@ -38,15 +38,13 @@
 except ImportError:
     import pickle
 
-from bd2k.util.expando import Expando, MagicExpando
+from bd2k.util.expando import MagicExpando
 from toil.common import Toil
 from toil.fileStore import FileStore
 from toil import logProcessContext
 import signal
 
-logger = logging.getLogger( __name__ )
-
-
+logger = logging.getLogger(__name__)
 
 
 def nextOpenDescriptor():
@@ -56,6 +54,7 @@ def nextOpenDescriptor():
     os.close(descriptor)
     return descriptor
 
+
 class AsyncJobStoreWrite(object):
     def __init__(self, jobStore):
         pass
@@ -69,8 +68,11 @@ def writeFileStream(self):
     def blockUntilSync(self):
         pass
 
-def main():
+
+def main(argv=None):
     logging.basicConfig()
+    if argv is None:
+        argv = sys.argv
 
     ##########################################
     #Import necessary modules 
@@ -97,11 +99,11 @@ def main():
     ##########################################
     #Input args
     ##########################################
-    
-    listOfJobs = [sys.argv[1]]
-    jobStoreLocator = sys.argv[2]
-    jobStoreID = sys.argv[3]
-    
+
+    listOfJobs = [argv[1]]
+    jobStoreLocator = argv[2]
+    jobStoreID = argv[3]
+
     ##########################################
     #Load the jobStore/config file
     ##########################################
@@ -175,29 +177,30 @@ def badWorker():
 
     #What file do we want to point FDs 1 and 2 to?
     tempWorkerLogPath = os.path.join(localWorkerTempDir, "worker_log.txt")
-    
-    #Save the original stdout and stderr (by opening new file descriptors to the
-    #same files)
-    origStdOut = os.dup(1)
-    origStdErr = os.dup(2)
 
-    #Open the file to send stdout/stderr to.
-    logFh = os.open(tempWorkerLogPath, os.O_WRONLY | os.O_CREAT | os.O_APPEND)
+    if not config.debugWorker:
+        # Save the original stdout and stderr (by opening new file descriptors
+        # to the same files)
+        origStdOut = os.dup(1)
+        origStdErr = os.dup(2)
 
-    #Replace standard output with a descriptor for the log file
-    os.dup2(logFh, 1)
-    
-    #Replace standard error with a descriptor for the log file
-    os.dup2(logFh, 2)
-    
-    #Since we only opened the file once, all the descriptors duped from the
-    #original will share offset information, and won't clobber each others'
-    #writes. See <http://stackoverflow.com/a/5284108/402891>. This shouldn't
-    #matter, since O_APPEND seeks to the end of the file before every write, but
-    #maybe there's something odd going on...
-    
-    #Close the descriptor we used to open the file
-    os.close(logFh)
+        # Open the file to send stdout/stderr to.
+        logFh = os.open(tempWorkerLogPath, os.O_WRONLY | os.O_CREAT | os.O_APPEND)
+
+        # Replace standard output with a descriptor for the log file
+        os.dup2(logFh, 1)
+
+        # Replace standard error with a descriptor for the log file
+        os.dup2(logFh, 2)
+
+        # Since we only opened the file once, all the descriptors duped from
+        # the original will share offset information, and won't clobber each
+        # others' writes. See <http://stackoverflow.com/a/5284108/402891>. This
+        # shouldn't matter, since O_APPEND seeks to the end of the file before
+        # every write, but maybe there's something odd going on...
+
+        # Close the descriptor we used to open the file
+        os.close(logFh)
 
     debugging = logging.getLogger().isEnabledFor(logging.DEBUG)
     ##########################################
@@ -472,30 +475,32 @@ def badWorker():
     ##########################################
     #Cleanup
     ##########################################
-    
-    #Close the worker logging
-    #Flush at the Python level
+
+    # Close the worker logging
+    # Flush at the Python level
     sys.stdout.flush()
     sys.stderr.flush()
-    #Flush at the OS level
-    os.fsync(1)
-    os.fsync(2)
+    if not config.debugWorker:
+        # Flush at the OS level
+        os.fsync(1)
+        os.fsync(2)
 
-    #Close redirected stdout and replace with the original standard output.
-    os.dup2(origStdOut, 1)
+        # Close redirected stdout and replace with the original standard output.
+        os.dup2(origStdOut, 1)
 
-    #Close redirected stderr and replace with the original standard error.
-    os.dup2(origStdErr, 2)
+        # Close redirected stderr and replace with the original standard error.
+        os.dup2(origStdErr, 2)
 
-    #sys.stdout and sys.stderr don't need to be modified at all. We don't need
-    #to call redirectLoggerStreamHandlers since they still log to sys.stderr
+        # sys.stdout and sys.stderr don't need to be modified at all. We don't
+        # need to call redirectLoggerStreamHandlers since they still log to
+        # sys.stderr
 
-    #Close our extra handles to the original standard output and standard error
-    #streams, so we don't leak file handles.
-    os.close(origStdOut)
-    os.close(origStdErr)
+        # Close our extra handles to the original standard output and standard
+        # error streams, so we don't leak file handles.
+        os.close(origStdOut)
+        os.close(origStdErr)
 
-    #Now our file handles are in exactly the state they were in before.
+    # Now our file handles are in exactly the state they were in before.
 
     #Copy back the log file to the global dir, if needed
     if workerFailed:
@@ -511,7 +516,7 @@ def badWorker():
                 w.write(f.read())
         jobStore.update(jobGraph)
 
-    elif debugging:  # write log messages
+    elif debugging and not config.debugWorker:  # write log messages
         with open(tempWorkerLogPath, 'r') as logFile:
             if os.path.getsize(tempWorkerLogPath) > logFileByteReportLimit != 0:
                 if logFileByteReportLimit > 0:


Test patch:
diff --git a/src/toil/test/cwl/cwlTest.py b/src/toil/test/cwl/cwlTest.py
--- a/src/toil/test/cwl/cwlTest.py
+++ b/src/toil/test/cwl/cwlTest.py
@@ -13,15 +13,12 @@
 # limitations under the License.
 from __future__ import absolute_import
 from __future__ import print_function
-from future import standard_library
 import json
 import os
 import subprocess
 import re
 import shutil
-from future.moves.urllib.parse import urlparse, urlencode
-from future.moves.urllib.request import urlopen, Request, urlretrieve
-from future.moves.urllib.error import HTTPError
+from future.moves.urllib.request import urlretrieve
 import zipfile
 
 # Python 3 compatibility imports
@@ -30,6 +27,7 @@
 
 from toil.test import ToilTest, needs_cwl, slow
 
+
 @needs_cwl
 class CWLTest(ToilTest):
 
@@ -47,6 +45,19 @@ def _tester(self, cwlfile, jobfile, outDir, expect):
         out["output"].pop("nameroot", None)
         self.assertEquals(out, expect)
 
+    def _debug_worker_tester(self, cwlfile, jobfile, outDir, expect):
+        from toil.cwl import cwltoil
+        rootDir = self._projectRootPath()
+        st = StringIO()
+        cwltoil.main(['--debug-worker', '--outdir', outDir,
+                     os.path.join(rootDir, cwlfile),
+                     os.path.join(rootDir, jobfile)], stdout=st)
+        out = json.loads(st.getvalue())
+        out["output"].pop("http://commonwl.org/cwltool#generation", None)
+        out["output"].pop("nameext", None)
+        out["output"].pop("nameroot", None)
+        self.assertEquals(out, expect)
+
     def test_run_revsort(self):
         outDir = self._createTempDir()
         self._tester('src/toil/test/cwl/revsort.cwl',
@@ -61,6 +72,21 @@ def test_run_revsort(self):
                 u'class': u'File',
                 u'checksum': u'sha1$b9214658cc453331b62c2282b772a5c063dbd284'}})
 
+    def test_run_revsort_debug_worker(self):
+        outDir = self._createTempDir()
+        # Having unicode string literals isn't necessary for the assertion
+        # but makes for a less noisy diff in case the assertion fails.
+        self._debug_worker_tester(
+                'src/toil/test/cwl/revsort.cwl',
+                'src/toil/test/cwl/revsort-job.json', outDir, {u'output': {
+                    u'location': "file://" + str(os.path.join(
+                        outDir, 'output.txt')),
+                    u'basename': str("output.txt"),
+                    u'size': 1111,
+                    u'class': u'File',
+                    u'checksum':
+                        u'sha1$b9214658cc453331b62c2282b772a5c063dbd284'}})
+
     @slow
     def test_restart(self):
         """Enable restarts with CWLtoil -- run failing test, re-run correct test.
@@ -72,6 +98,7 @@ def test_restart(self):
         cwlDir = os.path.join(self._projectRootPath(), "src", "toil", "test", "cwl")
         cmd = ['--outdir', outDir, '--jobStore', os.path.join(outDir, 'jobStore'), "--no-container",
                os.path.join(cwlDir, "revsort.cwl"), os.path.join(cwlDir, "revsort-job.json")]
+
         def path_without_rev():
             return ":".join([d for d in os.environ["PATH"].split(":")
                              if not os.path.exists(os.path.join(d, "rev"))])