Repository: DataBiosphere/toil

Issue:
If log level is DEBUG, send all worker output back to master
... regardless of whether a failure occurred.



Discussion:
Do you mind if CJ takes this on? He needs more work.

Please, be my guest! Sorry to not have got it done earlier.

On Thu, Aug 27, 2015 at 12:41 PM, Hannes Schmidt notifications@github.com
wrote:

> Do you mind if CJ takes this on? He needs more work.
> 
> —
> Reply to this email directly or view it on GitHub
> https://github.com/BD2KGenomics/toil/issues/301#issuecomment-135532625.
> 
> ## 
> 
> You received this message because you are subscribed to the Google Groups
> "cgloud-jenkins" group.
> To unsubscribe from this group and stop receiving emails from it, send an
> email to cgloud-jenkins+unsubscribe@googlegroups.com.
> To post to this group, send email to cgloud-jenkins@googlegroups.com.
> To view this discussion on the web visit
> https://groups.google.com/d/msgid/cgloud-jenkins/BD2KGenomics/toil/issues/301/135532625%40github.com
> https://groups.google.com/d/msgid/cgloud-jenkins/BD2KGenomics/toil/issues/301/135532625%40github.com?utm_medium=email&utm_source=footer
> .
> 
> For more options, visit https://groups.google.com/d/optout.


Resolving patch:
diff --git a/src/toil/batchSystems/gridengine.py b/src/toil/batchSystems/gridengine.py
--- a/src/toil/batchSystems/gridengine.py
+++ b/src/toil/batchSystems/gridengine.py
@@ -25,8 +25,8 @@
 from toil.batchSystems.abstractBatchSystem import AbstractBatchSystem
 from toil.batchSystems.parasol import getParasolResultsFileName
 
+logger = logging.getLogger(__name__)
 
-logger = logging.getLogger( __name__ )
 
 class MemoryString:
     def __init__(self, string):
@@ -57,24 +57,31 @@ def byteVal(self):
     def __cmp__(self, other):
         return cmp(self.bytes, other.bytes)
 
-def prepareQsub(cpu, mem):
-    qsubline = ["qsub","-b","y","-terse","-j" ,"y", "-cwd", "-o", "/dev/null", "-e", "/dev/null", "-v",
-                     "LD_LIBRARY_PATH=%s" % os.environ["LD_LIBRARY_PATH"]]
+
+def prepareQsub(cpu, mem, jobID):
+    qsubline = ["qsub", "-b", "y", "-terse", "-j", "y", "-cwd", "-o", "/dev/null",
+                "-e", "/dev/null", "-N", "Toil-Job_" + str(jobID)]
+    try:
+        path = os.environ["LD_LIBRARY_PATH"]
+    except KeyError:
+        pass
+    else:
+        qsubline.append("LD_LIBRARY_PATH=%s" % path)
+
     reqline = list()
     if mem is not None:
-        reqline.append("vf="+str(mem/ 1024)+"K")
-        reqline.append("h_vmem="+str(mem/ 1024)+"K")
+        reqline.append("vf=" + str(mem / 1024) + "K")
+        reqline.append("h_vmem=" + str(mem / 1024) + "K")
     if len(reqline) > 0:
         qsubline.extend(["-hard","-l", ",".join(reqline)])
-    if cpu is not None:
+    if cpu is not None and math.ceil(cpu) > 1:
         qsubline.extend(["-pe", "smp", str(int(math.ceil(cpu)))])
     return qsubline
 
 def qsub(qsubline):
-    logger.debug("**"+" ".join(qsubline))
+    logger.debug("**" + " ".join(qsubline))
     process = subprocess.Popen(qsubline, stdout=subprocess.PIPE)
     result = int(process.stdout.readline().strip().split('.')[0])
-    logger.debug("Got the job id: %s" % (str(result)))
     return result
 
 def getjobexitcode(sgeJobID):
@@ -83,11 +90,12 @@ def getjobexitcode(sgeJobID):
         if task is not None:
              args.extend(["-t", str(task)])
 
-        process = subprocess.Popen(args, stdout = subprocess.PIPE,stderr = subprocess.STDOUT)
+        process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
         for line in process.stdout:
             if line.startswith("failed") and int(line.split()[1]) == 1:
                 return 1
             elif line.startswith("exit_status"):
+                logger.debug('Exit Status: {}'.format(line.split()[1]))
                 return int(line.split()[1])
         return None
 
@@ -106,16 +114,17 @@ def __init__(self, newJobsQueue, updatedJobsQueue, killQueue, killedJobsQueue, b
 
     def getRunningJobIDs(self):
         times = {}
-        currentjobs = dict((self.sgeJobIDs[x], x) for x in self.runningJobs)
-        process = subprocess.Popen(["qstat"], stdout = subprocess.PIPE)
+        currentjobs = dict((str(self.sgeJobIDs[x][0]), x) for x in self.runningJobs)
+        process = subprocess.Popen(["qstat"], stdout=subprocess.PIPE)
         stdout, stderr = process.communicate()
         
         for currline in stdout.split('\n'):
             items = currline.strip().split()
-            if ((len(items) > 9 and (items[0],items[9]) in currentjobs) or (items[0], None) in currentjobs) and items[4] == 'r':
-                jobstart = " ".join(items[5:7])
-                jobstart = time.mktime(time.strptime(jobstart,"%m/%d/%Y %H:%M:%S"))
-                times[currentjobs[(items[0],items[9])]] = time.time() - jobstart 
+            if items:
+                if items[0] in currentjobs and items[4] == 'r':
+                    jobstart = " ".join(items[5:7])
+                    jobstart = time.mktime(time.strptime(jobstart,"%m/%d/%Y %H:%M:%S"))
+                    times[currentjobs[items[0]]] = time.time() - jobstart
 
         return times
 
@@ -143,6 +152,7 @@ def killJobs(self):
         # Do the dirty job
         for jobID in list(killList):
             if jobID in self.runningJobs:
+                logger.debug('Killing job: {}'.format(jobID))
                 process = subprocess.Popen(["qdel", self.getSgeID(jobID)])
             else:
                 if jobID in self.waitingJobs:
@@ -154,6 +164,7 @@ def killJobs(self):
         while len(killList) > 0:
             for jobID in list(killList):
                 if getjobexitcode(self.sgeJobIDs[jobID]) is not None:
+                    logger.debug('Adding {} jobID to killedJobsQueue'.format(jobID))
                     self.killedJobsQueue.put(jobID)
                     killList.remove(jobID)
                     self.forgetJob(jobID)
@@ -171,10 +182,8 @@ def createJobs(self, new_job):
         # Launch jobs as necessary:
         while len(self.waitingJobs) > 0 and sum(self.allocatedCpus.values()) < int(self.boss.maxCores):
             jobID, cpu, memory, command = self.waitingJobs.pop(0)
-            qsubline = prepareQsub(cpu, memory) + [command]
-            logger.debug('qsubline: {}'.format(qsubline))
+            qsubline = prepareQsub(cpu, memory, jobID) + [command]
             sgeJobID = qsub(qsubline)
-            logger.debug('sgeJobID: {}'.format(sgeJobID))
             self.sgeJobIDs[jobID] = (sgeJobID, None)
             self.runningJobs.add(jobID)
             self.allocatedCpus[jobID] = cpu
@@ -183,7 +192,6 @@ def checkOnJobs(self):
         logger.debug('List of runningJobs: {}'.format(self.runningJobs))
         for jobID in list(self.runningJobs):
             exit = getjobexitcode(self.sgeJobIDs[jobID])
-            logger.debug('Exit status: {}'.format(exit))
             if exit is not None:
                 self.updatedJobsQueue.put((jobID, exit))
                 self.forgetJob(jobID)
@@ -214,7 +222,6 @@ def __init__(self, config, maxCores, maxMemory, maxDisk):
         self.currentjobs = set()
         self.obtainSystemConstants()
         self.nextJobID = 0
-
         self.newJobsQueue = Queue()
         self.updatedJobsQueue = Queue()
         self.killQueue = Queue()
@@ -230,7 +237,6 @@ def issueBatchJob(self, command, memory, cores, disk):
         self.checkResourceRequest(memory, cores, disk)
         jobID = self.nextJobID
         self.nextJobID += 1
-
         self.currentjobs.add(jobID)
         self.newJobsQueue.put((jobID, cores, memory, command))
         logger.debug("Issued the job command: %s with job id: %s " % (command, str(jobID)))
@@ -243,19 +249,20 @@ def killBatchJobs(self, jobIDs):
         for jobID in jobIDs:
             self.killQueue.put(jobID)
 
+        logger.debug('Jobs to be killed: {}'.format(jobIDs))
         killList = set(jobIDs)
         while len(killList) > 0:
-            while True:
-                i = self.killedJobsQueue.get()
-                if i is not None:
-                    killList.remove(jobID)
-                    self.currentjobs.remove(jobID)
-                else:
-                    break
+            i = self.killedJobsQueue.get()
+            if i is not None:
+                killList.remove(i)
+                if i in self.currentjobs:
+                    self.currentjobs.remove(i)
+            else:
+                break
+
+            if len(killList) > 0:
+                time.sleep(5)
 
-        if len(killList) > 0:
-            time.sleep(5)
-    
     def getIssuedBatchJobIDs(self):
         """Gets the list of jobs issued to SGE.
         """
@@ -266,6 +273,7 @@ def getRunningBatchJobIDs(self):
     
     def getUpdatedBatchJob(self, maxWait):
         i = self.updatedJobsQueue.get()
+        logger.debug('UpdatedJobsQueue Item: {}'.format(i))
         if i == None:
             return None
         jobID, retcode = i
@@ -277,9 +285,10 @@ def shutdown(self):
         """
         Signals worker to shutdown (via sentinel) then cleanly joins the thread
         """
-        self.newJobsQueue.put(None)
-        # Remove reference to newJobsQueue (raises exception if inputQueue is used after method call)
+        newJobsQueue = self.newJobsQueue
         self.newJobsQueue = None
+
+        newJobsQueue.put(None)
         self.worker.join()
 
     def getWaitDuration(self):
diff --git a/src/toil/batchSystems/singleMachine.py b/src/toil/batchSystems/singleMachine.py
--- a/src/toil/batchSystems/singleMachine.py
+++ b/src/toil/batchSystems/singleMachine.py
@@ -222,10 +222,12 @@ def shutdown(self):
         """
         Cleanly terminate worker threads. Add sentinels to inputQueue equal to maxThreads. Join all worker threads.
         """
-        for i in xrange(self.numWorkers):
-            self.inputQueue.put(None)
         # Remove reference to inputQueue (raises exception if inputQueue is used after method call)
+        inputQueue = self.inputQueue
         self.inputQueue = None
+        for i in xrange(self.numWorkers):
+            inputQueue.put(None)
+
         for thread in self.workerThreads:
             thread.join()
 


Test patch:
diff --git a/src/toil/test/__init__.py b/src/toil/test/__init__.py
--- a/src/toil/test/__init__.py
+++ b/src/toil/test/__init__.py
@@ -15,6 +15,7 @@
 from __future__ import absolute_import
 import logging
 import os
+import subprocess
 import tempfile
 import unittest
 import shutil
@@ -155,6 +156,19 @@ def needs_azure(test_item):
                                  "'toiltest' storage account." % credential_file_path)(test_item)
         return test_item
 
+def needs_gridengine(test_item):
+    """
+    Use as a decorator before test classes or methods to only run them if GridEngine is installed.
+    """
+    try:
+        subprocess.Popen('qsub')
+    except OSError:
+        return unittest.skip("Skipping test. Install GridEngine to include this test.")(test_item)
+    except:
+        raise
+    else:
+        return test_item
+
 
 def needs_mesos(test_item):
     """
diff --git a/src/toil/test/batchSystems/batchSystemTest.py b/src/toil/test/batchSystems/batchSystemTest.py
--- a/src/toil/test/batchSystems/batchSystemTest.py
+++ b/src/toil/test/batchSystems/batchSystemTest.py
@@ -15,8 +15,6 @@
 from abc import ABCMeta, abstractmethod
 import logging
 import os
-import shutil
-import tempfile
 import time
 import multiprocessing
 
@@ -26,7 +24,7 @@
 from toil.batchSystems.parasol import ParasolBatchSystem
 from toil.batchSystems.singleMachine import SingleMachineBatchSystem
 from toil.batchSystems.abstractBatchSystem import InsufficientSystemResources
-from toil.test import ToilTest, needs_mesos, needs_parasol
+from toil.test import ToilTest, needs_mesos, needs_parasol, needs_gridengine
 
 log = logging.getLogger(__name__)
 
@@ -64,27 +62,14 @@ def createBatchSystem(self):
             """
             raise NotImplementedError
 
-        config = None
-        tempDir = None
-
-        @classmethod
-        def setUpClass(cls):
-            super(hidden.AbstractBatchSystemTest, cls).setUpClass()
-            cls.config = cls._createDummyConfig()
-            cls.tempDir = tempfile.mkdtemp()
-            if not os.path.exists(cls.config.jobStore):
-                os.mkdir(cls.config.jobStore)
-
-        @classmethod
-        def tearDownClass(cls):
-            shutil.rmtree(cls.tempDir)
-            shutil.rmtree(cls.config.jobStore)
-            super(hidden.AbstractBatchSystemTest, cls).tearDownClass()
+        def _createDummyConfig(self):
+            return Config()
 
         def setUp(self):
             super(hidden.AbstractBatchSystemTest, self).setUp()
+            self.config = self._createDummyConfig()
             self.batchSystem = self.createBatchSystem()
-            self.tempDir = tempfile.mkdtemp()
+            self.tempDir = self._createTempDir('testFiles')
 
         def tearDown(self):
             self.batchSystem.shutdown()
@@ -97,19 +82,19 @@ def testIssueJob(self):
             test_path = os.path.join(self.tempDir, 'test.txt')
             # sleep 1 coupled to command as 'touch' was too fast for wait_for_jobs to catch
             jobCommand = 'touch {}; sleep 1'.format(test_path)
-            self.batchSystem.issueBatchJob(jobCommand, memory=10, cores=.1, disk=1000)
+            self.batchSystem.issueBatchJob(jobCommand, memory=10, cores=1, disk=1000)
             self.wait_for_jobs(wait_for_completion=True)
             self.assertTrue(os.path.exists(test_path))
 
         def testCheckResourceRequest(self):
             self.assertRaises(InsufficientSystemResources, self.batchSystem.checkResourceRequest,
-                              memory=1000, cores=200, disk=1000)
+                              memory=1000, cores=200, disk=1e9)
             self.assertRaises(InsufficientSystemResources, self.batchSystem.checkResourceRequest,
-                              memory=5, cores=200, disk=1000)
+                              memory=5, cores=200, disk=1e9)
             self.assertRaises(InsufficientSystemResources, self.batchSystem.checkResourceRequest,
-                              memory=2e9, cores=1, disk=1000)
+                              memory=1001e9, cores=1, disk=1e9)
             self.assertRaises(InsufficientSystemResources, self.batchSystem.checkResourceRequest,
-                              memory=5, cores=1, disk=1002)
+                              memory=5, cores=1, disk=2e9)
             self.assertRaises(AssertionError, self.batchSystem.checkResourceRequest, memory=None,
                               cores=1, disk=1000)
             self.assertRaises(AssertionError, self.batchSystem.checkResourceRequest, memory=10,
@@ -152,7 +137,7 @@ def testKillJobs(self):
             self.batchSystem.killBatchJobs([0])
 
         def testGetUpdatedJob(self):
-            delay = 1
+            delay = 20
             jobCommand = 'sleep %i' % delay
             issuedIDs = []
             for i in range(numJobs):
@@ -168,26 +153,6 @@ def testGetUpdatedJob(self):
         def testGetRescueJobFrequency(self):
             self.assertTrue(self.batchSystem.getRescueBatchJobFrequency() > 0)
 
-        @staticmethod
-        def _createDummyConfig():
-            config = Config()
-            """
-            config = ElementTree.Element("config")
-            config.attrib["log_level"] = 'DEBUG'
-            config.attrib["job_store"] = '.'
-            config.attrib["parasol_command"] = 'parasol'
-            config.attrib["try_count"] = str(2)
-            config.attrib["max_job_duration"] = str(1)
-            config.attrib["batch_system"] = None
-            config.attrib["max_log_file_size"] = str(1)
-            config.attrib["default_memory"] = str(1)
-            config.attrib["default_cores"] = str(1)
-            config.attrib["max_cores"] = str(1)
-            config.attrib["max_memory"] = str(1)
-            config.attrib["scale"] = str(1)
-            """
-            return config
-
         def wait_for_jobs(self, numJobs=1, wait_for_completion=False):
             while not self.batchSystem.getIssuedBatchJobIDs():
                 pass
@@ -227,6 +192,21 @@ class ParasolBatchSystemTest(hidden.AbstractBatchSystemTest, ParasolTestSupport)
     """
     Tests the Parasol batch system
     """
+    def _createDummyConfig(self):
+        config = super(ParasolBatchSystemTest, self)._createDummyConfig()
+        # can't use _getTestJobStorePath since that method removes the directory
+        config.jobStore = self._createTempDir('jobStore')
+        return config
+
+    def createBatchSystem(self):
+        self._startParasol(numCores)
+        return ParasolBatchSystem(config=self.config, maxCores=numCores, maxMemory=1e9,
+                                  maxDisk=1001)
+
+    def tearDown(self):
+        self._stopParasol()
+        super(ParasolBatchSystemTest, self).tearDown()
+
 
     def testIssueJob(self):
         # TODO
@@ -239,12 +219,25 @@ def testIssueJob(self):
         self.wait_for_jobs(wait_for_completion=True)
         self.assertTrue(os.path.exists(test_path))
 
+
+
+@needs_gridengine
+class GridEngineTest(hidden.AbstractBatchSystemTest):
+    """
+    Tests against the GridEngine batch system
+    """
+
+    def _createDummyConfig(self):
+        config = super(GridEngineTest, self)._createDummyConfig()
+        # can't use _getTestJobStorePath since that method removes the directory
+        config.jobStore = self._createTempDir('jobStore')
+        return config
+
     def createBatchSystem(self):
-        self._startParasol(numCores)
-        return ParasolBatchSystem(config=self.config, maxCores=numCores, maxMemory=1e9,
-                                  maxDisk=1001)
+        from toil.batchSystems.gridengine import GridengineBatchSystem
+        return GridengineBatchSystem(config=self.config, maxCores=numCores, maxMemory=1000e9, maxDisk=1e9)
 
-    def tearDown(self):
-        self._stopParasol()
-        super(ParasolBatchSystemTest, self).tearDown()
-        self.batchSystem.shutdown()
+    @classmethod
+    def setUpClass(cls):
+        super(GridEngineTest, cls).setUpClass()
+        logging.basicConfig(level=logging.DEBUG)