Repository: DataBiosphere/toil

Issue:
Private IP support for Toil leader
Toil currently assumes that the leader node has a public IP ([example](https://github.com/DataBiosphere/toil/blob/863e2f0e0a0e7bca028ba7b66a9e6a4201ec9f58/src/toil/provisioners/node.py#L240)). However, there are cluster setups where public IPs are not available.

It would be ideal to have an option to use private IPs.

Note: once the cluster is launched, Mesos works fine with private IPs, so this is just a blocker for launching the cluster and using tools like `toil ssh-cluster`.


Discussion:

Resolving patch:
diff --git a/src/toil/provisioners/aws/awsProvisioner.py b/src/toil/provisioners/aws/awsProvisioner.py
--- a/src/toil/provisioners/aws/awsProvisioner.py
+++ b/src/toil/provisioners/aws/awsProvisioner.py
@@ -401,7 +401,7 @@ def _waitForIP(cls, instance):
         while True:
             time.sleep(a_short_time)
             instance.update()
-            if instance.ip_address or instance.public_dns_name:
+            if instance.ip_address or instance.public_dns_name or instance.private_ip_address:
                 logger.debug('...got ip')
                 break
 
diff --git a/src/toil/provisioners/node.py b/src/toil/provisioners/node.py
--- a/src/toil/provisioners/node.py
+++ b/src/toil/provisioners/node.py
@@ -33,6 +33,7 @@ class Node(object):
     def __init__(self, publicIP, privateIP, name, launchTime, nodeType, preemptable, tags=None):
         self.publicIP = publicIP
         self.privateIP = privateIP
+        self.effectiveIP = self.publicIP or self.privateIP
         self.name = name
         self.launchTime = launchTime
         self.nodeType = nodeType
@@ -40,13 +41,13 @@ def __init__(self, publicIP, privateIP, name, launchTime, nodeType, preemptable,
         self.tags = tags
 
     def __str__(self):
-        return "%s at %s" % (self.name, self.publicIP)
+        return "%s at %s" % (self.name, self.effectiveIP)
 
     def __repr__(self):
         return str(self)
 
     def __hash__(self):
-        return hash(self.publicIP)
+        return hash(self.effectiveIP)
 
     def remainingBillingInterval(self):
         """
@@ -102,7 +103,7 @@ def injectFile(self, fromFile, toFile, role):
             except Exception as e:
                 logger.debug("Rsync to new node failed, trying again. Error message: %s" % e)
                 time.sleep(10*retry)
-        raise RuntimeError("Failed to inject file %s to %s with ip %s" % (fromFile, role, self.publicIP) )
+        raise RuntimeError("Failed to inject file %s to %s with ip %s" % (fromFile, role, self.effectiveIP) )
 
     def _waitForSSHKeys(self, keyName='core'):
         # the propagation of public ssh keys vs. opening the SSH port is racey, so this method blocks until
@@ -110,7 +111,7 @@ def _waitForSSHKeys(self, keyName='core'):
         startTime = time.time()
         while True:
             if time.time() - startTime > self.maxWaitTime:
-                raise RuntimeError("Key propagation failed on machine with ip %s" % self.publicIP)
+                raise RuntimeError("Key propagation failed on machine with ip %s" % self.effectiveIP)
             try:
                 logger.info('Attempting to establish SSH connection...')
                 self.sshInstance('ps', sshOptions=['-oBatchMode=yes'], user=keyName)
@@ -123,12 +124,12 @@ def _waitForSSHKeys(self, keyName='core'):
                 return
 
     def _waitForDockerDaemon(self, keyName='core'):
-        logger.info('Waiting for docker on %s to start...', self.publicIP)
+        logger.info('Waiting for docker on %s to start...', self.effectiveIP)
         sleepTime = 10
         startTime = time.time()
         while True:
             if time.time() - startTime > self.maxWaitTime:
-                raise RuntimeError("Docker daemon failed to start on machine with ip %s" % self.publicIP)
+                raise RuntimeError("Docker daemon failed to start on machine with ip %s" % self.effectiveIP)
             try:
                 output = self.sshInstance('/usr/bin/ps', 'auxww', sshOptions=['-oBatchMode=yes'], user=keyName)
                 if b'dockerd' in output:
@@ -150,7 +151,7 @@ def _waitForAppliance(self, role, keyName='core'):
             if time.time() - startTime > self.maxWaitTime:
                 raise RuntimeError("Appliance failed to start on machine with ip %s"
                                    " Check if the appliance is valid, e.g. check if the environment variable"
-                                    " TOIL_APPLIANCE_SELF is set correctly and the container exists." % self.publicIP)
+                                    " TOIL_APPLIANCE_SELF is set correctly and the container exists." % self.effectiveIP)
             try:
                 output = self.sshInstance('/usr/bin/docker', 'ps',
                                           sshOptions=['-oBatchMode=yes'], user=keyName)
@@ -182,7 +183,7 @@ def _waitForSSHPort(self):
             s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
             try:
                 s.settimeout(a_short_time)
-                s.connect((self.publicIP, 22))
+                s.connect((self.effectiveIP, 22))
                 logger.debug('...ssh port open')
                 return i
             except socket.error:
@@ -237,7 +238,7 @@ def coreSSH(self, *args, **kwargs):
             commandTokens.extend(sshOptions)
         # specify host
         user = kwargs.pop('user', 'core')   # CHANGED: Is this needed?
-        commandTokens.append('%s@%s' % (user,str(self.publicIP)))
+        commandTokens.append('%s@%s' % (user,str(self.effectiveIP)))
         appliance = kwargs.pop('appliance', None)
         if appliance:
             # run the args in the appliance
@@ -253,7 +254,7 @@ def coreSSH(self, *args, **kwargs):
             kwargs['stdout'] = subprocess.PIPE
         kwargs['stderr'] = subprocess.PIPE
 
-        logger.debug('Node %s: %s', self.publicIP, ' '.join(args))
+        logger.debug('Node %s: %s', self.effectiveIP, ' '.join(args))
         args = list(map(pipes.quote, args))
         commandTokens += args
         logger.debug('Full command %s', ' '.join(commandTokens))
@@ -281,7 +282,7 @@ def coreRsync(self, args, applianceName='toil_leader', **kwargs):
         for i in args:
             if i.startswith(":") and not hostInserted:
                 user = kwargs.pop('user', 'core')   # CHANGED: Is this needed?
-                i = ("%s@%s" % (user, self.publicIP)) + i
+                i = ("%s@%s" % (user, self.effectiveIP)) + i
                 hostInserted = True
             elif i.startswith(":") and hostInserted:
                 raise ValueError("Cannot rsync between two remote hosts")


Test patch: