Repository: DataBiosphere/toil

Issue:
mesosMasterAddress should default to leader's private IP, not localhost
When starting a workflow on the leader node of a cluster for autoscaling one needs to specify the option `mesosMasterAddress=<my_local_IP_address>:5050` because the default (localhost:5050) is not useful at all and causes the workflow to fail more or less ambiguously. 

To get the local IP address without any external dependencies the following code hunk from [here](http://stackoverflow.com/a/24196955) seems to be the most robust and straight forward.

```
import socket
import fcntl
import struct

def get_ip_address(ifname):
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    return socket.inet_ntoa(fcntl.ioctl(
        s.fileno(),
        0x8915,  # SIOCGIFADDR
        struct.pack('256s', ifname[:15])
    )[20:24])

get_ip_address('eth0')
```


Discussion:

Resolving patch:
diff --git a/src/toil/batchSystems/options.py b/src/toil/batchSystems/options.py
--- a/src/toil/batchSystems/options.py
+++ b/src/toil/batchSystems/options.py
@@ -15,6 +15,12 @@
 
 from .registry import batchSystemFactoryFor, defaultBatchSystem, uniqueNames
 
+import socket
+
+
+def getLocalIP():
+    # may return localhost on some systems (not osx and coreos) https://stackoverflow.com/a/166520
+    return socket.gethostbyname(socket.gethostname())
 
 def _parasolOptions(addOptionFn):
     addOptionFn("--parasolCommand", dest="parasolCommand", default=None,
@@ -90,7 +96,7 @@ def setDefaultOptions(config):
     config.linkImports = False
 
     # mesos
-    config.mesosMasterAddress = 'localhost:5050'
+    config.mesosMasterAddress = '%s:5050' % getLocalIP()
 
     # parasol
     config.parasolCommand = 'parasol'
diff --git a/src/toil/common.py b/src/toil/common.py
--- a/src/toil/common.py
+++ b/src/toil/common.py
@@ -20,7 +20,6 @@
 import sys
 import tempfile
 import time
-import socket
 import uuid
 from argparse import ArgumentParser
 from threading import Thread
@@ -35,7 +34,6 @@
 from toil import logProcessContext
 from toil.lib.bioio import addLoggingOptions, getLogLevelString, setLoggingFromOptions
 from toil.realtimeLogger import RealtimeLogger
-from toil.batchSystems.options import setOptions as setBatchOptions
 from toil.batchSystems.options import addOptions as addBatchOptions
 from toil.batchSystems.options import setDefaultOptions as setDefaultBatchOptions
 


Test patch:
diff --git a/src/toil/test/batchSystems/batchSystemTest.py b/src/toil/test/batchSystems/batchSystemTest.py
--- a/src/toil/test/batchSystems/batchSystemTest.py
+++ b/src/toil/test/batchSystems/batchSystemTest.py
@@ -256,6 +256,19 @@ def getBatchSystemName(self):
             """
             raise NotImplementedError
 
+        def getOptions(self, tempDir):
+            """
+            Configures options for Toil workflow and makes job store.
+            :param str tempDir: path to test directory
+            :return: Toil options object
+            """
+            options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
+            options.logLevel = "DEBUG"
+            options.batchSystem = self.batchSystemName
+            options.workDir = tempDir
+            options.maxCores = self.cpuCount
+            return options
+
         def setUp(self):
             self.batchSystemName = self.getBatchSystemName()
             super(hidden.AbstractBatchSystemJobTest, self).setUp()
@@ -269,11 +282,7 @@ def testJobConcurrency(self):
             """
             for coresPerJob in self.allocatedCores:
                 tempDir = self._createTempDir('testFiles')
-
-                options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
-                options.workDir = tempDir
-                options.maxCores = self.cpuCount
-                options.batchSystem = self.batchSystemName
+                options = self.getOptions(tempDir)
 
                 counterPath = os.path.join(tempDir, 'counter')
                 resetCounters(counterPath)
@@ -318,6 +327,15 @@ class MesosBatchSystemTest(hidden.AbstractBatchSystemTest, MesosTestSupport):
     Tests against the Mesos batch system
     """
 
+    def createConfig(cls):
+        """
+        needs to set mesosMasterAddress to localhost for testing since the default is now the
+        private IP address
+        """
+        config = super(MesosBatchSystemTest, cls).createConfig()
+        config.mesosMasterAddress = 'localhost:5050'
+        return config
+
     def supportsWallTime(self):
         return True
 
@@ -782,6 +800,11 @@ class MesosBatchSystemJobTest(hidden.AbstractBatchSystemJobTest, MesosTestSuppor
     Tests Toil workflow against the Mesos batch system
     """
 
+    def getOptions(self, tempDir):
+        options = super(MesosBatchSystemJobTest, self).getOptions(tempDir)
+        options.mesosMasterAddress = 'localhost:5050'
+        return options
+
     def getBatchSystemName(self):
         self._startMesos(self.cpuCount)
         return "mesos"
diff --git a/src/toil/test/mesos/mesosTest.py b/src/toil/test/mesos/mesosTest.py
--- a/src/toil/test/mesos/mesosTest.py
+++ b/src/toil/test/mesos/mesosTest.py
@@ -61,7 +61,8 @@ def test_hello_world(self):
         system([sys.executable,
                 '-m', helloWorld.__name__,
                 'file:./toilTest',
-                '--batchSystem=mesos',
+                '--batchSystem', 'mesos',
+                '--mesosMaster', 'localhost:5050',
                 '--logLevel', getLogLevelString()])
 
     def test_stress_good(self):
diff --git a/src/toil/test/mesos/stress.py b/src/toil/test/mesos/stress.py
--- a/src/toil/test/mesos/stress.py
+++ b/src/toil/test/mesos/stress.py
@@ -66,6 +66,7 @@ def main(numJobs):
     Job.Runner.addToilOptions(parser)
     options = parser.parse_args( args=['./toilTest'] )
     options.batchSystem="mesos"
+    options.mesosMasterAddress="localhost:5050"
     # Launch first toil Job
     i = LongTestJob( numJobs )
     Job.Runner.startToil(i,  options )
diff --git a/src/toil/test/provisioners/aws/awsProvisionerTest.py b/src/toil/test/provisioners/aws/awsProvisionerTest.py
--- a/src/toil/test/provisioners/aws/awsProvisionerTest.py
+++ b/src/toil/test/provisioners/aws/awsProvisionerTest.py
@@ -149,7 +149,6 @@ def _test(self, spotInstances=False, fulfillableBid=True):
         toilOptions = [self.jobStore,
                        '--batchSystem=mesos',
                        '--workDir=/var/lib/toil',
-                       '--mesosMaster=%s:5050' % self.leader.private_ip_address,
                        '--clean=always',
                        '--retryCount=2',
                        '--clusterStats=/home/',
diff --git a/src/toil/test/sort/sortTest.py b/src/toil/test/sort/sortTest.py
--- a/src/toil/test/sort/sortTest.py
+++ b/src/toil/test/sort/sortTest.py
@@ -89,7 +89,11 @@ def _toilSort(self, jobStoreLocator, batchSystem,
                 options.clean = "never"
                 options.badWorker = badWorker
                 options.badWorkerFailInterval = 0.05
-                options.disableCaching = disableCaching  # FIXME maybe this line should be deleted
+                options.disableCaching = disableCaching
+                # This is required because mesosMasterAddress now defaults to the IP of the machine
+                # that is starting the workflow while the mesos *tests* run locally.
+                if batchSystem == 'mesos':
+                    options.mesosMasterAddress = 'localhost:5050'
                 options.downCheckpoints = downCheckpoints
                 options.N = N
 
diff --git a/src/toil/test/src/promisedRequirementTest.py b/src/toil/test/src/promisedRequirementTest.py
--- a/src/toil/test/src/promisedRequirementTest.py
+++ b/src/toil/test/src/promisedRequirementTest.py
@@ -75,16 +75,7 @@ def testConcurrencyStatic(self):
                 self.assertEqual(maxValue, self.cpuCount / coresPerJob)
 
         def getOptions(self, tempDir, caching=True):
-            """
-            Configures options for Toil workflow and makes job store.
-            :param str tempDir: path to test directory
-            :return: Toil options object
-            """
-            options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
-            options.logLevel = "DEBUG"
-            options.batchSystem = self.batchSystemName
-            options.workDir = tempDir
-            options.maxCores = self.cpuCount
+            options = super(hidden.AbstractPromisedRequirementsTest, self).getOptions(tempDir)
             # defaultCores defaults to 1 - this is coincidentally the core requirement relied upon by this
             # test, so we change defaultCores to 2 to make the test more strict
             options.defaultCores = 2
@@ -228,6 +219,11 @@ class MesosPromisedRequirementsTest(hidden.AbstractPromisedRequirementsTest, Mes
     Tests against the Mesos batch system
     """
 
+    def getOptions(self, tempDir, caching=True):
+        options = super(MesosPromisedRequirementsTest, self).getOptions(tempDir, caching=caching)
+        options.mesosMasterAddress = 'localhost:5050'
+        return options
+
     def getBatchSystemName(self):
         self._startMesos(self.cpuCount)
         return "mesos"