Repository: ray-project/ray

Issue:
Autoscaler requires public IPs
<!--
General questions should be asked on the mailing list ray-dev@googlegroups.com.

Before submitting an issue, please fill out the following form.
-->

### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**:  Ubuntu 17.04
- **Ray installed from (source or binary)**: binary
- **Ray version**:  0.5.3
- **Python version**:  3.6.6
- **Exact command to reproduce**:
1.  Configure a cluster with a subnet which has `"MapPublicIpOnLaunch": false`
2.  Run `ray up <yaml file>`


### Describe the problem
We see the following stack  trace:
```Traceback (most recent call last):                                                                                                                                                                                                                                                                                           
  File "/home/matthew/conda/envs/venv/bin/ray", line 11, in <module>                                                                                                                            
    load_entry_point('ray==0.5.3', 'console_scripts', 'ray')()                                                                                                                                                                                                                                                               
  File "/home/matthew/conda/envs/venv/lib/python3.6/site-packages/ray-0.5.3-py3.6-linux-x86_64.egg/ray/scripts/scripts.py", line 569, in main                                                                                                                                                                                
    return cli()                                                                                                                                                                                
  File "/home/matthew/conda/envs/venv/lib/python3.6/site-packages/Click-7.0-py3.6.egg/click/core.py", line 764, in __call__                                                   
    return self.main(*args, **kwargs)                                                                                                                         
  File "/home/matthew/conda/envs/venv/lib/python3.6/site-packages/Click-7.0-py3.6.egg/click/core.py", line 717, in main                                             
    rv = self.invoke(ctx)                                                                                                                                                                                                                                                                                                    
  File "/home/matthew/conda/envs/venv/lib/python3.6/site-packages/Click-7.0-py3.6.egg/click/core.py", line 1137, in invoke                                                                                 
    return _process_result(sub_ctx.command.invoke(sub_ctx))                                                                                                                                                           
  File "/home/matthew/conda/envs/venv/lib/python3.6/site-packages/Click-7.0-py3.6.egg/click/core.py", line 956, in invoke                        
    return ctx.invoke(self.callback, **ctx.params)                                                                                                       
  File "/home/matthew/conda/envs/venv/lib/python3.6/site-packages/Click-7.0-py3.6.egg/click/core.py", line 555, in invoke                                                                                                                                                                                                    
    return callback(*args, **kwargs)                                                                                                    
  File "/home/matthew/conda/envs/venv/lib/python3.6/site-packages/ray-0.5.3-py3.6-linux-x86_64.egg/ray/scripts/scripts.py", line 440, in create_or_update
    no_restart, restart_only, yes, cluster_name)                                                                                                                      
  File "/home/matthew/conda/envs/venv/lib/python3.6/site-packages/ray-0.5.3-py3.6-linux-x86_64.egg/ray/autoscaler/commands.py", line 42, in create_or_update_cluster               
    config = _bootstrap_config(config)                                                                                                                                                                                                                                                                                       
  File "/home/matthew/conda/envs/venv/lib/python3.6/site-packages/ray-0.5.3-py3.6-linux-x86_64.egg/ray/autoscaler/commands.py", line 64, in _bootstrap_config                                                                                                                                                                
    resolved_config = bootstrap_config(config)                                                                                                                         
  File "/home/matthew/conda/envs/venv/lib/python3.6/site-packages/ray-0.5.3-py3.6-linux-x86_64.egg/ray/autoscaler/aws/config.py", line 48, in bootstrap_aws
    config = _configure_subnet(config)                                                                                                                 
  File "/home/matthew/conda/envs/venv/lib/python3.6/site-packages/ray-0.5.3-py3.6-linux-x86_64.egg/ray/autoscaler/aws/config.py", line 156, in _configure_subnet            
    "No usable subnets found, try manually creating an instance in "                                                                                       
Exception: No usable subnets found, try manually creating an instance in your specified region to populate the list of subnets and trying this again. Note that the subnet must map public IPs on instance launch.
```

Is it possible to support private IPs?  We operate inside a VPN and public IPs aren't an option for us.  Thanks in advance for your help


Discussion:
I believe the only reason to map public ips is for the command to be able to SSH to the node it just launched. You can instead set up a jump node in the VPC, SSH to that, and run ray up there.

You'll also have to remove the check for public ips https://github.com/ray-project/ray/blob/master/python/ray/autoscaler/aws/config.py#L157 

Let me know if this works. We can merge this as an option if it does.
Thanks for the quick response.  I tried changing this line already and things seemed to work (at least better than before), although I also had to change the `use_internal_ip` default arg to `True` in in the `NodeUpdater` class.  I also didn't need to create a jump node as my machine is already on the VPC VPN and so can automatically see the private IPs. Is it possible to get this change committed?

Resolving patch:
diff --git a/python/ray/autoscaler/autoscaler.py b/python/ray/autoscaler/autoscaler.py
--- a/python/ray/autoscaler/autoscaler.py
+++ b/python/ray/autoscaler/autoscaler.py
@@ -69,6 +69,7 @@
             "project_id": (None, OPTIONAL),  # gcp project id, if using gcp
             "head_ip": (str, OPTIONAL),  # local cluster head node
             "worker_ips": (list, OPTIONAL),  # local cluster worker nodes
+            "use_internal_ips": (bool, OPTIONAL),  # don't require public ips
         },
         REQUIRED),
 
diff --git a/python/ray/autoscaler/aws/config.py b/python/ray/autoscaler/aws/config.py
--- a/python/ray/autoscaler/aws/config.py
+++ b/python/ray/autoscaler/aws/config.py
@@ -152,9 +152,10 @@ def _configure_key_pair(config):
 
 def _configure_subnet(config):
     ec2 = _resource("ec2", config)
+    use_internal_ips = config["provider"].get("use_internal_ips", False)
     subnets = sorted(
-        (s for s in ec2.subnets.all()
-         if s.state == "available" and s.map_public_ip_on_launch),
+        (s for s in ec2.subnets.all() if s.state == "available" and (
+            use_internal_ips or s.map_public_ip_on_launch)),
         reverse=True,  # sort from Z-A
         key=lambda subnet: subnet.availability_zone)
     if not subnets:
@@ -162,7 +163,8 @@ def _configure_subnet(config):
             "No usable subnets found, try manually creating an instance in "
             "your specified region to populate the list of subnets "
             "and trying this again. Note that the subnet must map public IPs "
-            "on instance launch.")
+            "on instance launch unless you set 'use_internal_ips': True in "
+            "the 'provider' config.")
     if "availability_zone" in config["provider"]:
         azs = config["provider"]["availability_zone"].split(',')
         subnets = [s for s in subnets if s.availability_zone in azs]
diff --git a/python/ray/autoscaler/updater.py b/python/ray/autoscaler/updater.py
--- a/python/ray/autoscaler/updater.py
+++ b/python/ray/autoscaler/updater.py
@@ -47,7 +47,8 @@ def __init__(self,
         self.daemon = True
         self.process_runner = process_runner
         self.node_id = node_id
-        self.use_internal_ip = use_internal_ip
+        self.use_internal_ip = (use_internal_ip or provider_config.get(
+            "use_internal_ips", False))
         self.provider = get_node_provider(provider_config, cluster_name)
         self.ssh_private_key = auth_config["ssh_private_key"]
         self.ssh_user = auth_config["ssh_user"]


Test patch: