Repository: DataBiosphere/toil

Issue:
--masterIP should be --mesosMaster and accept host names

--masterIP should be --mesosMaster and accept host names



Discussion:

Resolving patch:
diff --git a/src/toil/batchSystems/mesos/batchSystem.py b/src/toil/batchSystems/mesos/batchSystem.py
--- a/src/toil/batchSystems/mesos/batchSystem.py
+++ b/src/toil/batchSystems/mesos/batchSystem.py
@@ -16,6 +16,7 @@
 
 from collections import defaultdict
 import os
+import socket
 import time
 import pickle
 from Queue import Queue, Empty
@@ -47,7 +48,7 @@ class MesosBatchSystem(AbstractBatchSystem, mesos.interface.Scheduler):
     def supportsHotDeployment():
         return True
 
-    def __init__(self, config, maxCores, maxMemory, maxDisk, masterIP,
+    def __init__(self, config, maxCores, maxMemory, maxDisk, masterAddress,
                  userScript=None, toilDistribution=None):
         AbstractBatchSystem.__init__(self, config, maxCores, maxMemory, maxDisk)
         # The hot-deployed resources representing the user script and the toil distribution
@@ -63,8 +64,8 @@ def __init__(self, config, maxCores, maxMemory, maxDisk, masterIP,
         # defined by resource usage
         self.jobQueueList = defaultdict(list)
 
-        # IP of mesos master. specified in MesosBatchSystem, currently loopback
-        self.masterIP = masterIP
+        # Address of Mesos master in the form host:port where host can be an IP or a hostname
+        self.masterAddress = masterAddress
 
         # queue of jobs to kill, by jobID.
         self.killSet = set()
@@ -240,7 +241,6 @@ def _startDriver(self):
         framework.user = ""  # Have Mesos fill in the current user.
         framework.name = "toil"
 
-
         if os.getenv("MESOS_CHECKPOINT"):
             log.debug("Enabling checkpoint for the framework")
             framework.checkpoint = True
@@ -249,10 +249,32 @@ def _startDriver(self):
             raise NotImplementedError("Authentication is currently not supported")
         else:
             framework.principal = framework.name
-            self.driver = mesos.native.MesosSchedulerDriver(self, framework, self.masterIP,
+            self.driver = mesos.native.MesosSchedulerDriver(self, framework,
+                                                            self.resolveAddress(self.masterAddress),
                                                             self.implicitAcknowledgements)
         assert self.driver.start() == mesos_pb2.DRIVER_RUNNING
 
+    @staticmethod
+    def resolveAddress(address):
+        """
+        Resolves the host in the given string. The input is of the form host[:port]. This method
+        is idempotent, i.e. the host may already be a dotted IP address.
+
+        >>> f=MesosBatchSystem.resolveAddress
+        >>> f('localhost')
+        '127.0.0.1'
+        >>> f('127.0.0.1')
+        '127.0.0.1'
+        >>> f('localhost:123')
+        '127.0.0.1:123'
+        >>> f('127.0.0.1:123')
+        '127.0.0.1:123'
+        """
+        address = address.split(':')
+        assert len(address) in (1,2)
+        address[0] = socket.gethostbyname(address[0])
+        return ':'.join(address)
+
     def shutdown(self):
         log.info("Stopping Mesos driver")
         self.driver.stop()
diff --git a/src/toil/batchSystems/parasol.py b/src/toil/batchSystems/parasol.py
--- a/src/toil/batchSystems/parasol.py
+++ b/src/toil/batchSystems/parasol.py
@@ -80,7 +80,7 @@ def __init__(self, config, maxCores, maxMemory, maxDisk):
         #is created whenever a job has a new unique combination of cpu and memory
         #requirements.
         self.resultsFiles = dict()
-        self.maxBatches = config.maxParasolBatches
+        self.maxBatches = config.parasolMaxBatches
 
         #Allows the worker process to send back the IDs of
         #jobs that have finished, so the batch system can
diff --git a/src/toil/common.py b/src/toil/common.py
--- a/src/toil/common.py
+++ b/src/toil/common.py
@@ -50,9 +50,9 @@ def __init__(self):
         #Batch system options
         self.batchSystem = "singleMachine"
         self.scale = 1
-        self.masterIP = '127.0.0.1:5050'
+        self.mesosMasterAddress = 'localhost:5050'
         self.parasolCommand = "parasol"
-        self.maxParasolBatches = 10000
+        self.parasolMaxBatches = 10000
         
         #Resource requirements
         self.defaultMemory = 2147483648
@@ -129,9 +129,9 @@ def f(x):
         #Batch system options
         setOption("batchSystem")
         setOption("scale", float) 
-        setOption("masterIP") 
+        setOption("mesosMasterAddress")
         setOption("parasolCommand")
-        setOption("maxParasolBatches", int, iC(1))
+        setOption("parasolMaxBatches", int, iC(1))
         
         #Resource requirements
         setOption("defaultMemory", h2b, iC(1))
@@ -202,17 +202,17 @@ def _addOptions(addGroupFn, config):
     addOptionFn("--batchSystem", dest="batchSystem", default=None,
                       help=("The type of batch system to run the job(s) with, currently can be one "
                             "of singleMachine, parasol, gridEngine, lsf or mesos'. default=%s" % config.batchSystem))
-    #TODO - what is this?
     addOptionFn("--scale", dest="scale", default=None,
                 help=("A scaling factor to change the value of all submitted tasks's submitted cores. "
                       "Used in singleMachine batch system. default=%s" % config.scale))
-    addOptionFn("--masterIP", dest="masterIP", default=None,
-                help=("The master node's ip and port number. Used in mesos batch system. default=%s" % config.masterIP))
+    addOptionFn("--mesosMaster", dest="mesosMasterAddress", default=None,
+                help=("The host and port of the Mesos master separated by colon. default=%s" % config.mesosMasterAddress))
     addOptionFn("--parasolCommand", dest="parasolCommand", default=None,
-                      help="The command to run the parasol program default=%s" % config.parasolCommand)
-    addOptionFn("--maxParasolBatches", dest="maxParasolBatches", default=None,
-                help="Maximum number of batches Parasol is allowed to create - a batch \
-                is created for each job that has a unique set of resource requirements. Default=%i" % config.maxParasolBatches)
+                      help="The path to the parasol program. default=%s" % config.parasolCommand)
+    addOptionFn("--parasolMaxBatches", dest="parasolMaxBatches", default=None,
+                help="Maximum number of job batches the Parasol batch is allowed to create. One "
+                     "batch is created for jobs with a a unique set of resource requirements. "
+                     "default=%i" % config.parasolMaxBatches)
 
     #
     #Resource requirements
@@ -333,7 +333,7 @@ def loadBatchSystemClass(config):
     elif batchSystemName == 'mesos' or batchSystemName == 'Mesos':
         from toil.batchSystems.mesos.batchSystem import MesosBatchSystem
         batchSystemClass = MesosBatchSystem
-        kwargs["masterIP"] = config.masterIP
+        kwargs['masterAddress'] = config.mesosMasterAddress
         logger.info('Using the mesos batch system')
     else:
         raise RuntimeError('Unrecognised batch system: %s' % batchSystemName)


Test patch:
diff --git a/src/toil/batchSystems/mesos/test/__init__.py b/src/toil/batchSystems/mesos/test/__init__.py
--- a/src/toil/batchSystems/mesos/test/__init__.py
+++ b/src/toil/batchSystems/mesos/test/__init__.py
@@ -58,12 +58,13 @@ def mesosCommand(self):
             return ['mesos-master',
                     '--registry=in_memory',
                     '--ip=127.0.0.1',
+                    '--port=5050',
                     '--allocation_interval=500ms']
 
     class MesosSlaveThread(MesosThread):
         def mesosCommand(self):
-            # NB: The --resources parameter forces this test to use a predictable number of cores, independent of how
-            # many cores the system running the test actually has.
+            # NB: The --resources parameter forces this test to use a predictable number of
+            # cores, independent of how many cores the system running the test actually has.
             return ['mesos-slave',
                     '--ip=127.0.0.1',
                     '--master=127.0.0.1:5050',
diff --git a/src/toil/test/batchSystems/batchSystemTest.py b/src/toil/test/batchSystems/batchSystemTest.py
--- a/src/toil/test/batchSystems/batchSystemTest.py
+++ b/src/toil/test/batchSystems/batchSystemTest.py
@@ -164,7 +164,7 @@ def createBatchSystem(self):
         from toil.batchSystems.mesos.batchSystem import MesosBatchSystem
         self._startMesos(numCores)
         return MesosBatchSystem(config=self.config, maxCores=numCores, maxMemory=1e9, maxDisk=1001,
-                                masterIP='127.0.0.1:5050')
+                                masterAddress='127.0.0.1:5050')
 
     def tearDown(self):
         self._stopMesos()