Repository: ray-project/ray

Issue:
[Autoscaler] Only recognizes workers it launches
<!--Please include [tune], [rllib], [autoscaler] etc. in the issue title if relevant-->

### What is the problem?

*Ray version and other system information (Python version, TensorFlow version, OS):*
0.8.6+

### Reproduction (REQUIRED)
Please provide a script that can be run to reproduce the issue. The script should have **no external library dependencies** (i.e., use fake or mock data / environments):

* Start a ray cluster with gpu workers (non-resource demand autoscaler), head should not have gpu's.
* Manually add a node (not using autoscaler), with a custom resource {"custom": 1}

```
@ray.remote(num_gpus=1)
def gpu_task():
    pass

@ray.remote(resources={"custom":1})
def launch_gpu():
    # This blocks and never starts a gpu worker.
    ray.get(gpu_task.remote())

ray.get(launch_gpu.remote())
```

If we cannot run your script, we cannot fix your issue.

- [ ] I have verified my script runs in a clean environment and reproduces the issue.
- [ ] I have verified the issue also occurs with the [latest wheels](https://docs.ray.io/en/latest/installation.html).



Discussion:
The root cause of this is that `StandardAutoscaler::workers()` explicitely filters
```
    def workers(self):
        return self.provider.non_terminated_nodes(
            tag_filters={TAG_RAY_NODE_KIND: NODE_KIND_WORKER})

```
@wuisawesome Can you set the priority of this task? 
> The root cause of this is that `StandardAutoscaler::workers()` explicitely filters
> 
> ```
>     def workers(self):
>         return self.provider.non_terminated_nodes(
>             tag_filters={TAG_RAY_NODE_KIND: NODE_KIND_WORKER})
> ```

This is indeed the root cause. When this would be fixed, the autoscaler will however still try to update the manual node because the `ray-launch-config` and `ray-runtime-config` labels are not set on the manual node either.

Additionally, the script you posted looks very similar to the thing I'm trying to do, but is still a little different.
In my case the `gpu_task` task is not started from within an actor but simply from within python code that is running on the manually started worker. I don't know if that makes a difference or not...

Resolving patch:
diff --git a/python/ray/autoscaler/autoscaler.py b/python/ray/autoscaler/autoscaler.py
--- a/python/ray/autoscaler/autoscaler.py
+++ b/python/ray/autoscaler/autoscaler.py
@@ -13,10 +13,11 @@
 from ray.experimental.internal_kv import _internal_kv_put, \
     _internal_kv_initialized
 from ray.autoscaler.node_provider import get_node_provider
-from ray.autoscaler.tags import (
-    TAG_RAY_LAUNCH_CONFIG, TAG_RAY_RUNTIME_CONFIG,
-    TAG_RAY_FILE_MOUNTS_CONTENTS, TAG_RAY_NODE_STATUS, TAG_RAY_NODE_KIND,
-    TAG_RAY_USER_NODE_TYPE, STATUS_UP_TO_DATE, NODE_KIND_WORKER)
+from ray.autoscaler.tags import (TAG_RAY_LAUNCH_CONFIG, TAG_RAY_RUNTIME_CONFIG,
+                                 TAG_RAY_FILE_MOUNTS_CONTENTS,
+                                 TAG_RAY_NODE_STATUS, TAG_RAY_NODE_KIND,
+                                 TAG_RAY_USER_NODE_TYPE, STATUS_UP_TO_DATE,
+                                 NODE_KIND_WORKER, NODE_KIND_UNMANAGED)
 from ray.autoscaler.updater import NodeUpdaterThread
 from ray.autoscaler.node_launcher import NodeLauncher
 from ray.autoscaler.resource_demand_scheduler import ResourceDemandScheduler
@@ -146,10 +147,12 @@ def _update(self):
         nodes = self.workers()
         # Check pending nodes immediately after fetching the number of running
         # nodes to minimize chance number of pending nodes changing after
-        # additional nodes are launched.
+        # additional nodes (managed and unmanaged) are launched.
         num_pending = self.pending_launches.value
-        self.load_metrics.prune_active_ips(
-            [self.provider.internal_ip(node_id) for node_id in nodes])
+        self.load_metrics.prune_active_ips([
+            self.provider.internal_ip(node_id)
+            for node_id in self.all_workers()
+        ])
         target_workers = self.target_num_workers()
 
         if len(nodes) >= target_workers:
@@ -165,8 +168,9 @@ def _update(self):
         nodes_to_terminate = []
         for node_id in nodes:
             node_ip = self.provider.internal_ip(node_id)
-            if node_ip in last_used and last_used[node_ip] < horizon and \
-                    len(nodes) - len(nodes_to_terminate) > target_workers:
+            if (node_ip in last_used and last_used[node_ip] < horizon) and \
+                    (len(nodes) - len(nodes_to_terminate)
+                     > target_workers):
                 logger.info("StandardAutoscaler: "
                             "{}: Terminating idle node".format(node_id))
                 nodes_to_terminate.append(node_id)
@@ -182,11 +186,12 @@ def _update(self):
 
         # Terminate nodes if there are too many
         nodes_to_terminate = []
-        while len(nodes) > self.config["max_workers"]:
+        while (len(nodes) -
+               len(nodes_to_terminate)) > self.config["max_workers"] and nodes:
+            to_terminate = nodes.pop()
             logger.info("StandardAutoscaler: "
-                        "{}: Terminating unneeded node".format(nodes[-1]))
-            nodes_to_terminate.append(nodes[-1])
-            nodes = nodes[:-1]
+                        "{}: Terminating unneeded node".format(to_terminate))
+            nodes_to_terminate.append(to_terminate)
 
         if nodes_to_terminate:
             self.provider.terminate_nodes(nodes_to_terminate)
@@ -511,10 +516,17 @@ def launch_new_node(self, count: int, node_type: Optional[str]) -> None:
         config = copy.deepcopy(self.config)
         self.launch_queue.put((config, count, node_type))
 
+    def all_workers(self):
+        return self.workers() + self.unmanaged_workers()
+
     def workers(self):
         return self.provider.non_terminated_nodes(
             tag_filters={TAG_RAY_NODE_KIND: NODE_KIND_WORKER})
 
+    def unmanaged_workers(self):
+        return self.provider.non_terminated_nodes(
+            tag_filters={TAG_RAY_NODE_KIND: NODE_KIND_UNMANAGED})
+
     def log_info_string(self, nodes, target):
         tmp = "Cluster status: "
         tmp += self.info_string(nodes, target)
diff --git a/python/ray/autoscaler/resource_demand_scheduler.py b/python/ray/autoscaler/resource_demand_scheduler.py
--- a/python/ray/autoscaler/resource_demand_scheduler.py
+++ b/python/ray/autoscaler/resource_demand_scheduler.py
@@ -14,7 +14,7 @@
 from typing import List, Dict, Tuple
 
 from ray.autoscaler.node_provider import NodeProvider
-from ray.autoscaler.tags import TAG_RAY_USER_NODE_TYPE
+from ray.autoscaler.tags import TAG_RAY_USER_NODE_TYPE, NODE_KIND_UNMANAGED
 
 logger = logging.getLogger(__name__)
 
@@ -90,9 +90,14 @@ def calculate_node_resources(
 
         def add_node(node_type, existing_resource_usages=None):
             if node_type not in self.node_types:
-                raise RuntimeError("Missing entry for node_type {} in "
-                                   "available_node_types config: {}".format(
-                                       node_type, self.node_types))
+                logger.warn(
+                    f"Missing entry for node_type {node_type} in "
+                    f"cluster config: {self.node_types} under entry "
+                    f"available_node_types. This node's resources will be "
+                    f"ignored. If you are using an unmanaged node, manually "
+                    f"set the user_node_type tag to \"{NODE_KIND_UNMANAGED}\""
+                    f"in your cloud provider's management console.")
+                return None
             # Careful not to include the same dict object multiple times.
             available = copy.deepcopy(self.node_types[node_type]["resources"])
             if existing_resource_usages:
diff --git a/python/ray/autoscaler/tags.py b/python/ray/autoscaler/tags.py
--- a/python/ray/autoscaler/tags.py
+++ b/python/ray/autoscaler/tags.py
@@ -8,6 +8,7 @@
 TAG_RAY_NODE_KIND = "ray-node-type"
 NODE_KIND_HEAD = "head"
 NODE_KIND_WORKER = "worker"
+NODE_KIND_UNMANAGED = "unmanaged"
 
 # Tag for user defined node types (e.g., m4xl_spot). This is used for multi
 # node type clusters.


Test patch:
diff --git a/python/ray/tests/test_autoscaler.py b/python/ray/tests/test_autoscaler.py
--- a/python/ray/tests/test_autoscaler.py
+++ b/python/ray/tests/test_autoscaler.py
@@ -597,6 +597,50 @@ def testAggressiveAutoscaling(self):
         autoscaler.update()
         self.waitForNodes(11)
 
+    def testUnmanagedNodes(self):
+        config = SMALL_CLUSTER.copy()
+        config["min_workers"] = 0
+        config["max_workers"] = 20
+        config["initial_workers"] = 0
+        config["idle_timeout_minutes"] = 0
+        config["autoscaling_mode"] = "aggressive"
+        config["target_utilization_fraction"] = 0.8
+        config_path = self.write_config(config)
+
+        self.provider = MockProvider()
+        self.provider.create_node({}, {TAG_RAY_NODE_KIND: "head"}, 1)
+        head_ip = self.provider.non_terminated_node_ips(
+            tag_filters={TAG_RAY_NODE_KIND: "head"}, )[0]
+
+        self.provider.create_node({}, {TAG_RAY_NODE_KIND: "unmanaged"}, 1)
+        unmanaged_ip = self.provider.non_terminated_node_ips(
+            tag_filters={TAG_RAY_NODE_KIND: "unmanaged"}, )[0]
+
+        runner = MockProcessRunner()
+
+        lm = LoadMetrics()
+        lm.local_ip = head_ip
+
+        autoscaler = StandardAutoscaler(
+            config_path,
+            lm,
+            max_launch_batch=5,
+            max_concurrent_launches=5,
+            max_failures=0,
+            process_runner=runner,
+            update_interval_s=0)
+
+        autoscaler.update()
+        self.waitForNodes(2)
+        # This node has num_cpus=0
+        lm.update(unmanaged_ip, {"CPU": 0}, {"CPU": 0}, {})
+        autoscaler.update()
+        self.waitForNodes(2)
+        # 1 CPU task cannot be scheduled.
+        lm.update(unmanaged_ip, {"CPU": 0}, {"CPU": 0}, {"CPU": 1})
+        autoscaler.update()
+        self.waitForNodes(3)
+
     def testDelayedLaunch(self):
         config_path = self.write_config(SMALL_CLUSTER)
         self.provider = MockProvider()