Repository: DataBiosphere/toil

Issue:
Fix GridEngine batch system



Discussion:
- [x] Install Toil on PODK
- [x] Test "hello world" user script with:  `--batchSystem=gridEngine`


Resolving patch:
diff --git a/src/toil/batchSystems/gridengine.py b/src/toil/batchSystems/gridengine.py
--- a/src/toil/batchSystems/gridengine.py
+++ b/src/toil/batchSystems/gridengine.py
@@ -18,6 +18,7 @@
 import os
 import subprocess
 import time
+import math
 from Queue import Queue
 from threading import Thread
 
@@ -39,9 +40,9 @@ def __init__(self, string):
 
     def __str__(self):
         if self.unit != 'B':
-            return str(val) + unit
+            return str(self.val) + self.unit
         else:
-            return str(val)
+            return str(self.val)
 
     def byteVal(self):
         if self.unit == 'B':
@@ -60,13 +61,13 @@ def prepareQsub(cpu, mem):
     qsubline = ["qsub","-b","y","-terse","-j" ,"y", "-cwd", "-o", "/dev/null", "-e", "/dev/null", "-v",
                      "LD_LIBRARY_PATH=%s" % os.environ["LD_LIBRARY_PATH"]]
     reqline = list()
-    if cpu is not None:
-        reqline.append("p="+str(cpu))
     if mem is not None:
         reqline.append("vf="+str(mem/ 1024)+"K")
         reqline.append("h_vmem="+str(mem/ 1024)+"K")
     if len(reqline) > 0:
         qsubline.extend(["-hard","-l", ",".join(reqline)])
+    if cpu is not None:
+        qsubline.extend(["-pe", "smp", str(int(math.ceil(cpu)))])
     return qsubline
 
 def qsub(qsubline):
@@ -110,7 +111,7 @@ def getRunningJobIDs(self):
         stdout, stderr = process.communicate()
         
         for currline in stdout.split('\n'):
-            items = curline.strip().split()
+            items = currline.strip().split()
             if ((len(items) > 9 and (items[0],items[9]) in currentjobs) or (items[0], None) in currentjobs) and items[4] == 'r':
                 jobstart = " ".join(items[5:7])
                 jobstart = time.mktime(time.strptime(jobstart,"%m/%d/%Y %H:%M:%S"))
@@ -162,31 +163,41 @@ def killJobs(self):
                             "so I'll try again")
                 time.sleep(5)
 
-    def createJobs(self):
-        # Load new job ids:
-        while not self.newJobsQueue.empty():
-            self.waitingJobs.append(self.newJobsQueue.get())
+    def createJobs(self, new_job):
+        # Load new job id if present:
+        if new_job is not None:
+            self.waitingJobs.append(new_job)
 
         # Launch jobs as necessary:
-        while len(self.waitingJobs) > 0 and sum(self.allocatedCpus.values()) < int(self.boss.maxCpus):
+        while len(self.waitingJobs) > 0 and sum(self.allocatedCpus.values()) < int(self.boss.maxCores):
             jobID, cpu, memory, command = self.waitingJobs.pop(0)
             qsubline = prepareQsub(cpu, memory) + [command]
+            logger.debug('qsubline: {}'.format(qsubline))
             sgeJobID = qsub(qsubline)
+            logger.debug('sgeJobID: {}'.format(sgeJobID))
             self.sgeJobIDs[jobID] = (sgeJobID, None)
             self.runningJobs.add(jobID)
             self.allocatedCpus[jobID] = cpu
 
     def checkOnJobs(self):
+        logger.debug('List of runningJobs: {}'.format(self.runningJobs))
         for jobID in list(self.runningJobs):
             exit = getjobexitcode(self.sgeJobIDs[jobID])
+            logger.debug('Exit status: {}'.format(exit))
             if exit is not None:
                 self.updatedJobsQueue.put((jobID, exit))
                 self.forgetJob(jobID)
     
     def run(self):
         while True:
+            new_job = None
+            if not self.newJobsQueue.empty():
+                new_job = self.newJobsQueue.get()
+                if new_job is None:
+                    logger.debug('Received queue sentinel.')
+                    break
             self.killJobs()
-            self.createJobs()
+            self.createJobs(new_job)
             self.checkOnJobs()
             time.sleep(10)
 
@@ -194,8 +205,8 @@ class GridengineBatchSystem(AbstractBatchSystem):
     """The interface for gridengine.
     """
     
-    def __init__(self, config, maxCores, maxMemory):
-        AbstractBatchSystem.__init__(self, config, maxCores, maxMemory) #Call the parent constructor
+    def __init__(self, config, maxCores, maxMemory, maxDisk):
+        AbstractBatchSystem.__init__(self, config, maxCores, maxMemory, maxDisk)
         self.gridengineResultsFile = getParasolResultsFileName(config.jobStore)
         #Reset the job queue and results (initially, we do this again once we've killed the jobs)
         self.gridengineResultsFileHandle = open(self.gridengineResultsFile, 'w')
@@ -209,15 +220,14 @@ def __init__(self, config, maxCores, maxMemory):
         self.killQueue = Queue()
         self.killedJobsQueue = Queue()
         self.worker = Worker(self.newJobsQueue, self.updatedJobsQueue, self.killQueue, self.killedJobsQueue, self)
-        self.worker.setDaemon(True)
         self.worker.start()
         
     def __des__(self):
         #Closes the file handle associated with the results file.
         self.gridengineResultsFileHandle.close() #Close the results file, cos were done.
 
-    def issueBatchJob(self, command, memory, cores):
-        self.checkResourceRequest(memory, cores)
+    def issueBatchJob(self, command, memory, cores, disk):
+        self.checkResourceRequest(memory, cores, disk)
         jobID = self.nextJobID
         self.nextJobID += 1
 
@@ -236,7 +246,7 @@ def killBatchJobs(self, jobIDs):
         killList = set(jobIDs)
         while len(killList) > 0:
             while True:
-                i = self.getFromQueueSafely(self.killedJobsQueue, maxWait)
+                i = self.killedJobsQueue.get()
                 if i is not None:
                     killList.remove(jobID)
                     self.currentjobs.remove(jobID)
@@ -255,14 +265,23 @@ def getRunningBatchJobIDs(self):
         return self.worker.getRunningJobIDs()
     
     def getUpdatedBatchJob(self, maxWait):
-        i = self.getFromQueueSafely(self.updatedJobsQueue, maxWait)
+        i = self.updatedJobsQueue.get()
         if i == None:
             return None
         jobID, retcode = i
         self.updatedJobsQueue.task_done()
         self.currentjobs.remove(jobID)
         return i
-    
+
+    def shutdown(self):
+        """
+        Signals worker to shutdown (via sentinel) then cleanly joins the thread
+        """
+        self.newJobsQueue.put(None)
+        # Remove reference to newJobsQueue (raises exception if inputQueue is used after method call)
+        self.newJobsQueue = None
+        self.worker.join()
+
     def getWaitDuration(self):
         """We give parasol a second to catch its breath (in seconds)
         """
diff --git a/src/toil/batchSystems/singleMachine.py b/src/toil/batchSystems/singleMachine.py
--- a/src/toil/batchSystems/singleMachine.py
+++ b/src/toil/batchSystems/singleMachine.py
@@ -191,7 +191,7 @@ def issueBatchJob(self, command, memory, cores, disk):
 
     def killBatchJobs(self, jobIDs):
         """
-        As jobs are already run, this method has no effect.
+        Kills jobs by ID
         """
         logger.debug('Killing jobs: {}'.format(jobIDs))
         for id in jobIDs:
diff --git a/src/toil/common.py b/src/toil/common.py
--- a/src/toil/common.py
+++ b/src/toil/common.py
@@ -410,4 +410,5 @@ def setupToil(options, userScript=None):
         serialiseEnvironment(jobStore)
         yield (config, batchSystem, jobStore)
     finally:
+        logger.debug('Shutting down batch system')
         batchSystem.shutdown()


Test patch: