Repository: DataBiosphere/toil

Issue:
Fix parasol job IDs
Parasol doesn't start the job ID's at zero

> self = <toil.test.batchSystems.batchSystemTest.ParasolBatchSystemTest >testMethod=testGetIssuedJobIDs>
> 
> def testGetIssuedJobIDs(self):
>        self.batchSystem.issueBatchJob('sleep 1', memory=10, cpu=numCoresPerJob, disk=1000)
>       self.batchSystem.issueBatchJob('sleep 1', memory=10, cpu=numCoresPerJob, disk=1000)
>       self.assertEqual({0, 1}, set(self.batchSystem.getIssuedBatchJobIDs()))
> E       AssertionError: Items in the first set but not the second:
> E       0
> E       1
> E       Items in the second set but not the first:
> E       3800017
> E       3800018



Discussion:

Resolving patch:
diff --git a/src/toil/batchSystems/parasol.py b/src/toil/batchSystems/parasol.py
--- a/src/toil/batchSystems/parasol.py
+++ b/src/toil/batchSystems/parasol.py
@@ -93,8 +93,8 @@ def getUpdatedJob(parasolResultsFile, outputQueue1, outputQueue2):
 class ParasolBatchSystem(AbstractBatchSystem):
     """The interface for Parasol.
     """
-    def __init__(self, config, maxCores, maxMemory):
-        AbstractBatchSystem.__init__(self, config, maxCores, maxMemory) #Call the parent constructor
+    def __init__(self, config, maxCores, maxMemory, maxDisk):
+        AbstractBatchSystem.__init__(self, config, maxCores, maxMemory, maxDisk) #Call the parent constructor
         if maxMemory != sys.maxint:
             logger.warn("A max memory has been specified for the parasol batch system class of %i, but currently "
                         "this batchsystem interface does not support such limiting" % maxMemory)
@@ -128,12 +128,13 @@ def __init__(self, config, maxCores, maxMemory):
         self.usedCpus = 0
         self.jobIDsToCpu = {}
          
-    def issueBatchJob(self, command, memory, cores):
+    def issueBatchJob(self, command, memory, cores, disk):
         """Issues parasol with job commands.
         """
-        self.checkResourceRequest(memory, cores)
+        self.checkResourceRequest(memory, cores, disk)
         pattern = re.compile("your job ([0-9]+).*")
-        parasolCommand = "%s -verbose -ram=%i -cores=%i -results=%s add job '%s'" % (self.parasolCommand, memory, cores, self.parasolResultsFile, command)
+        parasolCommand = "%s -verbose -ram=%i -cpu=%i -results=%s add job '%s'" % (self.parasolCommand, memory, cores, self.parasolResultsFile, command)
+
         #Deal with the cpus
         self.usedCpus += cores
         while True: #Process finished results with no wait
@@ -222,6 +223,8 @@ def getRescueBatchJobFrequency(cls):
         making it expensive. 
         """
         return 5400 #Once every 90 minutes
+    def shutdown(self):
+        pass
         
 def main():
     pass
diff --git a/src/toil/batchSystems/parasolTestSupport.py b/src/toil/batchSystems/parasolTestSupport.py
new file mode 100644
--- /dev/null
+++ b/src/toil/batchSystems/parasolTestSupport.py
@@ -0,0 +1,78 @@
+from __future__ import absolute_import
+from abc import ABCMeta, abstractmethod
+import logging
+import shutil
+import threading
+import time
+import subprocess
+import multiprocessing
+import os
+from toil.lib.bioio import getTempFile
+
+log = logging.getLogger(__name__)
+
+
+class ParasolTestSupport(object):
+    """
+    For test cases that need a running Parasol leader and worker on the local host
+    """
+    def _startParasol(self, numCores=None):
+        if numCores is None:
+            numCores = multiprocessing.cpu_count()
+        self.machineList = os.path.join(os.getcwd(), "machineList.txt")
+        with open(self.machineList, "w") as out:
+            out.write("localhost 4 3000 /tmp /scratch 36000 r1")
+
+        self.leader = self.ParasolLeaderThread(numCores)
+        self.leader.start()
+        self.worker = self.ParasolWorkerThread(numCores)
+        self.worker.start()
+        while self.leader.popen is None or self.worker.popen is None:
+            log.info("Waiting for leader and worker processes")
+            time.sleep(.1)
+
+    def _stopParasol(self):
+        self.worker.popen.kill()
+        self.worker.join()
+        self.leader.popen.kill()
+        self.leader.join()
+        os.remove(self.machineList)
+        if os.path.exists("para.results"):
+            os.remove("para.results")
+        if os.path.exists("parasol.jid"):
+            os.remove("parasol.jid")
+
+    class ParasolThread(threading.Thread):
+        __metaclass__ = ABCMeta
+
+        # Lock is used because subprocess is NOT thread safe: http://tinyurl.com/pkp5pgq
+        lock = threading.Lock()
+
+        def __init__(self, numCores):
+            threading.Thread.__init__(self)
+            self.numCores = numCores
+            self.popen = None
+            
+
+        @abstractmethod
+        def parasolCommand(self):
+            raise NotImplementedError
+
+        def run(self):
+            with self.lock:
+                self.popen = subprocess.Popen(self.parasolCommand())
+            self.popen.wait()
+            log.info('Exiting %s', self.__class__.__name__)
+
+    class ParasolLeaderThread(ParasolThread):
+        def parasolCommand(self):
+            return ['paraHub',
+                    '-debug',
+                    os.path.join(os.getcwd(), "machineList.txt")]
+
+    class ParasolWorkerThread(ParasolThread):
+        def parasolCommand(self):
+            return ['paraNode',
+                    '-cpu=4',
+                    '-debug',
+                    'start']


Test patch:
diff --git a/src/toil/test/__init__.py b/src/toil/test/__init__.py
--- a/src/toil/test/__init__.py
+++ b/src/toil/test/__init__.py
@@ -19,6 +19,7 @@
 import unittest
 import shutil
 import re
+import subprocess
 
 from bd2k.util.files import mkdir_p
 
@@ -169,3 +170,17 @@ def needs_mesos(test_item):
         raise
     else:
         return test_item
+
+
+def needs_parasol(test_item):
+    """
+    Use as decorator so tests are only run if Parasol is installed.
+    """
+    try:
+        subprocess.Popen("parasol")
+    except OSError:
+        return unittest.skip("Skipping test. Install Parasol to include this test.")(test_item)
+    except:
+        raise
+    else:
+        return test_item
diff --git a/src/toil/test/batchSystems/batchSystemTest.py b/src/toil/test/batchSystems/batchSystemTest.py
--- a/src/toil/test/batchSystems/batchSystemTest.py
+++ b/src/toil/test/batchSystems/batchSystemTest.py
@@ -19,11 +19,14 @@
 import tempfile
 import time
 import multiprocessing
+
 from toil.common import Config
 from toil.batchSystems.mesos.test import MesosTestSupport
+from toil.batchSystems.parasolTestSupport import ParasolTestSupport
+from toil.batchSystems.parasol import ParasolBatchSystem
 from toil.batchSystems.singleMachine import SingleMachineBatchSystem
 from toil.batchSystems.abstractBatchSystem import InsufficientSystemResources
-from toil.test import ToilTest, needs_mesos
+from toil.test import ToilTest, needs_mesos, needs_parasol
 
 log = logging.getLogger(__name__)
 
@@ -69,20 +72,23 @@ def setUpClass(cls):
             super(hidden.AbstractBatchSystemTest, cls).setUpClass()
             cls.config = cls._createDummyConfig()
             cls.tempDir = tempfile.mkdtemp()
+            if not os.path.exists(cls.config.jobStore):
+                os.mkdir(cls.config.jobStore)
 
         @classmethod
-        def tearDownClass( cls ):
+        def tearDownClass(cls):
             shutil.rmtree(cls.tempDir)
-            super( hidden.AbstractBatchSystemTest, cls ).tearDownClass( )
+            shutil.rmtree(cls.config.jobStore)
+            super(hidden.AbstractBatchSystemTest, cls).tearDownClass()
 
         def setUp(self):
             super(hidden.AbstractBatchSystemTest, self).setUp()
             self.batchSystem = self.createBatchSystem()
             self.tempDir = tempfile.mkdtemp()
 
-        def tearDown( self ):
+        def tearDown(self):
             self.batchSystem.shutdown()
-            super( hidden.AbstractBatchSystemTest, self ).tearDown( )
+            super(hidden.AbstractBatchSystemTest, self).tearDown()
 
         def testAvailableCores(self):
             self.assertTrue(multiprocessing.cpu_count() >= numCores)
@@ -96,32 +102,48 @@ def testIssueJob(self):
             self.assertTrue(os.path.exists(test_path))
 
         def testCheckResourceRequest(self):
-            self.assertRaises(InsufficientSystemResources, self.batchSystem.checkResourceRequest, memory=1000, cores=200, disk=1000)
-            self.assertRaises(InsufficientSystemResources, self.batchSystem.checkResourceRequest, memory=5, cores=200,disk=1000)
-            self.assertRaises(InsufficientSystemResources, self.batchSystem.checkResourceRequest, memory=100, cores=1,disk=1000)
-            self.assertRaises(InsufficientSystemResources, self.batchSystem.checkResourceRequest, memory=5, cores=1,disk=1002)
-            self.assertRaises(AssertionError, self.batchSystem.checkResourceRequest, memory=None, cores=1,disk=1000)
-            self.assertRaises(AssertionError, self.batchSystem.checkResourceRequest, memory=10, cores=None,disk=1000)
+            self.assertRaises(InsufficientSystemResources, self.batchSystem.checkResourceRequest,
+                              memory=1000, cores=200, disk=1000)
+            self.assertRaises(InsufficientSystemResources, self.batchSystem.checkResourceRequest,
+                              memory=5, cores=200, disk=1000)
+            self.assertRaises(InsufficientSystemResources, self.batchSystem.checkResourceRequest,
+                              memory=2e9, cores=1, disk=1000)
+            self.assertRaises(InsufficientSystemResources, self.batchSystem.checkResourceRequest,
+                              memory=5, cores=1, disk=1002)
+            self.assertRaises(AssertionError, self.batchSystem.checkResourceRequest, memory=None,
+                              cores=1, disk=1000)
+            self.assertRaises(AssertionError, self.batchSystem.checkResourceRequest, memory=10,
+                              cores=None, disk=1000)
             self.batchSystem.checkResourceRequest(memory=10, cores=1, disk=100)
 
         def testGetIssuedJobIDs(self):
-            self.batchSystem.issueBatchJob('sleep 1', memory=10, cores=numCoresPerJob, disk=1000)
-            self.batchSystem.issueBatchJob('sleep 1', memory=10, cores=numCoresPerJob, disk=1000)
-            self.assertEqual({0, 1}, set(self.batchSystem.getIssuedBatchJobIDs()))
+            issuedIDs = []
+            issuedIDs.append(
+                self.batchSystem.issueBatchJob('sleep 1', memory=10, cores=numCoresPerJob,
+                                               disk=1000))
+            issuedIDs.append(
+                self.batchSystem.issueBatchJob('sleep 1', memory=10, cores=numCoresPerJob,
+                                               disk=1000))
+            self.assertEqual(set(issuedIDs), set(self.batchSystem.getIssuedBatchJobIDs()))
 
         def testGetRunningJobIDs(self):
-            self.batchSystem.issueBatchJob('sleep 100', memory=10, cores=.1, disk=1000)
-            self.batchSystem.issueBatchJob('sleep 100', memory=10, cores=.1, disk=1000)
+            issuedIDs = []
+            issuedIDs.append(
+                self.batchSystem.issueBatchJob('sleep 100', memory=100e6, cores=1, disk=1000))
+            issuedIDs.append(
+                self.batchSystem.issueBatchJob('sleep 100', memory=100e6, cores=1, disk=1000))
             self.wait_for_jobs(numJobs=2)
-            # Assert that jobs were correctly labeled by JobID
-            self.assertEqual({0, 1}, set(self.batchSystem.getRunningBatchJobIDs().keys()))
+            # Assert that the issued jobs are running
+            self.assertEqual(set(issuedIDs), set(self.batchSystem.getRunningBatchJobIDs().keys()))
+            log.info("running jobs: %s" % self.batchSystem.getRunningBatchJobIDs().keys())
             # Assert that the length of the job was recorded
-            self.assertTrue(len([t for t in self.batchSystem.getRunningBatchJobIDs().values() if t > 0]) == 2)
-            self.batchSystem.killBatchJobs([0, 1])
+            self.assertTrue(
+                len([t for t in self.batchSystem.getRunningBatchJobIDs().values() if t > 0]) == 2)
+            self.batchSystem.killBatchJobs(issuedIDs)
 
         def testKillJobs(self):
             jobCommand = 'sleep 100'
-            jobID = self.batchSystem.issueBatchJob(jobCommand, memory=10, cores=.1, disk=1000)
+            jobID = self.batchSystem.issueBatchJob(jobCommand, memory=100e6, cores=1, disk=1000)
             self.wait_for_jobs()
             # self.assertEqual([0], self.batchSystem.getRunningJobIDs().keys())
             self.batchSystem.killBatchJobs([jobID])
@@ -132,9 +154,12 @@ def testKillJobs(self):
         def testGetUpdatedJob(self):
             delay = 1
             jobCommand = 'sleep %i' % delay
+            issuedIDs = []
             for i in range(numJobs):
-                self.batchSystem.issueBatchJob(jobCommand, memory=10, cores=numCoresPerJob, disk=1000)
-            jobs = set((i, 0) for i in range(numJobs))
+                issuedIDs.append(
+                    self.batchSystem.issueBatchJob(jobCommand, memory=100e6, cores=numCoresPerJob,
+                                                   disk=1000))
+            jobs = set((issuedIDs[i], 0) for i in range(numJobs))
             self.wait_for_jobs(numJobs=numJobs, wait_for_completion=True)
             for i in range(numJobs):
                 jobs.remove(self.batchSystem.getUpdatedBatchJob(delay * 2))
@@ -150,7 +175,7 @@ def _createDummyConfig():
             config = ElementTree.Element("config")
             config.attrib["log_level"] = 'DEBUG'
             config.attrib["job_store"] = '.'
-            config.attrib["parasol_command"] = None
+            config.attrib["parasol_command"] = 'parasol'
             config.attrib["try_count"] = str(2)
             config.attrib["max_job_duration"] = str(1)
             config.attrib["batch_system"] = None
@@ -173,6 +198,7 @@ def wait_for_jobs(self, numJobs=1, wait_for_completion=False):
                     time.sleep(0.1)
                     # pass updates too quickly (~24e6 iter/sec), which is why I'm using time.sleep(0.1):
 
+
 @needs_mesos
 class MesosBatchSystemTest(hidden.AbstractBatchSystemTest, MesosTestSupport):
     """
@@ -182,7 +208,8 @@ class MesosBatchSystemTest(hidden.AbstractBatchSystemTest, MesosTestSupport):
     def createBatchSystem(self):
         from toil.batchSystems.mesos.batchSystem import MesosBatchSystem
         self._startMesos(numCores)
-        return MesosBatchSystem(config=self.config, maxCores=numCores, maxMemory=20, maxDisk=1001,masterIP='127.0.0.1:5050')
+        return MesosBatchSystem(config=self.config, maxCores=numCores, maxMemory=1e9, maxDisk=1001,
+                                masterIP='127.0.0.1:5050')
 
     def tearDown(self):
         self._stopMesos()
@@ -191,4 +218,33 @@ def tearDown(self):
 
 class SingleMachineBatchSystemTest(hidden.AbstractBatchSystemTest):
     def createBatchSystem(self):
-        return SingleMachineBatchSystem(config=self.config, maxCores=numCores, maxMemory=50, maxDisk=1001)
+        return SingleMachineBatchSystem(config=self.config, maxCores=numCores, maxMemory=1e9,
+                                        maxDisk=1001)
+
+
+@needs_parasol
+class ParasolBatchSystemTest(hidden.AbstractBatchSystemTest, ParasolTestSupport):
+    """
+    Tests the Parasol batch system
+    """
+
+    def testIssueJob(self):
+        # TODO
+        # parasol treats 'touch test.txt; sleep 1' as one
+        # command with the arguments 'test.txt;', 'sleep', and '1'
+        # For now, override the test
+        test_path = os.path.join(self.tempDir, 'test.txt')
+        jobCommand = 'touch {}'.format(test_path)
+        self.batchSystem.issueBatchJob(jobCommand, memory=100e6, cores=1, disk=1000)
+        self.wait_for_jobs(wait_for_completion=True)
+        self.assertTrue(os.path.exists(test_path))
+
+    def createBatchSystem(self):
+        self._startParasol(numCores)
+        return ParasolBatchSystem(config=self.config, maxCores=numCores, maxMemory=1e9,
+                                  maxDisk=1001)
+
+    def tearDown(self):
+        self._stopParasol()
+        super(ParasolBatchSystemTest, self).tearDown()
+        self.batchSystem.shutdown()