# Buggy code
def terminate_node(self, node_id):
    node = self._get_cached_node(node_id)
    if self.cache_stopped_nodes:
        if node.spot_instance_request_id:
            cli_logger.print(
                "Terminating instance {} "
                + cf.dimmed("(cannot stop spot instances, only terminate)"),
                node_id,
            )  # todo: show node name?
            node.terminate()
        else:
            cli_logger.print(
                "Stopping instance {} "
                + cf.dimmed(
                    "(to terminate instead, "
                    "set `cache_stopped_nodes: False` "
                    "under `provider` in the cluster configuration)"
                ),
                node_id,
            )  # todo: show node name?
            node.stop()
    else:
        node.terminate()

    self.tag_cache.pop(node_id, None)
    self.tag_cache_pending.pop(node_id, None)


# Traceback (KeyError)
2021-02-17 14:55:34,817 INFO monitor.py:207 – :event_summary:Removing 1 nodes of type cpu_48_spot (idle).
2021-02-17 14:55:34,817 INFO monitor.py:207 – :event_summary:Adding 1 nodes of type cpu_48_spot.
2021-02-17 14:55:40,430 INFO load_metrics.py:102 – LoadMetrics: Removed mapping: 172.31.23.116 - 1613573430.7000167
2021-02-17 14:55:40,430 INFO load_metrics.py:109 – LoadMetrics: Removed 1 stale ip mappings: {‘172.31.23.116’} not in {‘172.31.16.240’, ‘172.31.27.173’, ‘172.31.26.163’, ‘172.31.20.177’, ‘172.31.25.79’, ‘172.31.28.159’, ‘172.31.21.227’, ‘172.31.24.131’, ‘172.31.31.164’, ‘172.31.22.24’, ‘172.31.26.41’, ‘172.31.19.126’, ‘172.31.22.66’, ‘172.31.26.13’, ‘172.31.30.105’, ‘172.31.25.157’, ‘172.31.27.26’}
2021-02-17 14:55:40,744 ERROR autoscaler.py:266 – StandardAutoscaler: i-02b77234ffad2072c: Terminating failed to setup/initialize node.
2021-02-17 14:55:40,744 ERROR autoscaler.py:139 – StandardAutoscaler: Error during autoscaling.
Traceback (most recent call last):
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/autoscaler.py”, line 137, in update
self._update()
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/autoscaler.py”, line 270, in _update
self._get_node_type(node_id) + " (launch failed).",
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/autoscaler.py”, line 598, in _get_node_type
node_tags = self.provider.node_tags(node_id)
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/aws/node_provider.py”, line 170, in node_tags
d1 = self.tag_cache[node_id]
KeyError: ‘i-02b77234ffad2072c’
2021-02-17 14:55:46,909 ERROR autoscaler.py:266 – StandardAutoscaler: i-02b77234ffad2072c: Terminating failed to setup/initialize node.
2021-02-17 14:55:46,909 ERROR autoscaler.py:139 – StandardAutoscaler: Error during autoscaling.
Traceback (most recent call last):
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/autoscaler.py”, line 137, in update
self._update()
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/autoscaler.py”, line 270, in _update
self._get_node_type(node_id) + " (launch failed).",
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/autoscaler.py”, line 598, in _get_node_type
node_tags = self.provider.node_tags(node_id)
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/aws/node_provider.py”, line 170, in node_tags
d1 = self.tag_cache[node_id]
KeyError: ‘i-02b77234ffad2072c’
2021-02-17 14:55:47,082 INFO monitor.py:207 – :event_summary:Resized to 724 CPUs.
2021-02-17 14:55:52,997 ERROR autoscaler.py:266 – StandardAutoscaler: i-02b77234ffad2072c: Terminating failed to setup/initialize node.
2021-02-17 14:55:52,998 ERROR autoscaler.py:139 – StandardAutoscaler: Error during autoscaling.
Traceback (most recent call last):
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/autoscaler.py”, line 137, in update
self._update()
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/autoscaler.py”, line 270, in _update
self._get_node_type(node_id) + " (launch failed).",
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/autoscaler.py”, line 598, in _get_node_type
node_tags = self.provider.node_tags(node_id)
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/aws/node_provider.py”, line 170, in node_tags
d1 = self.tag_cache[node_id]
KeyError: ‘i-02b77234ffad2072c’
2021-02-17 14:55:58,965 ERROR autoscaler.py:266 – StandardAutoscaler: i-02b77234ffad2072c: Terminating failed to setup/initialize node.
2021-02-17 14:55:58,965 ERROR autoscaler.py:139 – StandardAutoscaler: Error during autoscaling.
Traceback (most recent call last):
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/autoscaler.py”, line 137, in update
self._update()
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/autoscaler.py”, line 270, in _update
self._get_node_type(node_id) + " (launch failed).",
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/autoscaler.py”, line 598, in _get_node_type
node_tags = self.provider.node_tags(node_id)
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/aws/node_provider.py”, line 170, in node_tags
d1 = self.tag_cache[node_id]
KeyError: ‘i-02b77234ffad2072c’
2021-02-17 14:56:05,002 ERROR autoscaler.py:266 – StandardAutoscaler: i-02b77234ffad2072c: Terminating failed to setup/initialize node.
2021-02-17 14:56:05,003 ERROR autoscaler.py:139 – StandardAutoscaler: Error during autoscaling.
Traceback (most recent call last):
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/autoscaler.py”, line 137, in update
self._update()
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/autoscaler.py”, line 270, in _update
self._get_node_type(node_id) + " (launch failed).",
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/autoscaler.py”, line 598, in _get_node_type
node_tags = self.provider.node_tags(node_id)
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/aws/node_provider.py”, line 170, in node_tags
d1 = self.tag_cache[node_id]
KeyError: ‘i-02b77234ffad2072c’
2021-02-17 14:56:10,999 ERROR autoscaler.py:266 – StandardAutoscaler: i-02b77234ffad2072c: Terminating failed to setup/initialize node.
2021-02-17 14:56:11,000 ERROR autoscaler.py:139 – StandardAutoscaler: Error during autoscaling.
Traceback (most recent call last):
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/autoscaler.py”, line 137, in update
self._update()
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/autoscaler.py”, line 270, in _update
self._get_node_type(node_id) + " (launch failed).",
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/autoscaler.py”, line 598, in _get_node_type
node_tags = self.provider.node_tags(node_id)
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/aws/node_provider.py”, line 170, in node_tags
d1 = self.tag_cache[node_id]
KeyError: ‘i-02b77234ffad2072c’
2021-02-17 14:56:11,001 CRITICAL autoscaler.py:152 – StandardAutoscaler: Too many errors, abort.
2021-02-17 14:56:11,001 ERROR monitor.py:271 – Error in monitor loop
Traceback (most recent call last):
File “/home/centos/.local/lib/python3.7/site-packages/ray/monitor.py”, line 269, in run
self._run()
File “/home/centos/.local/lib/python3.7/site-packages/ray/monitor.py”, line 202, in _run
self.autoscaler.update()
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/autoscaler.py”, line 154, in update
raise e
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/autoscaler.py”, line 137, in update
self._update()
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/autoscaler.py”, line 270, in _update
self._get_node_type(node_id) + " (launch failed).",
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/autoscaler.py”, line 598, in _get_node_type
node_tags = self.provider.node_tags(node_id)
File “/home/centos/.local/lib/python3.7/site-packages/ray/autoscaler/_private/aws/node_provider.py”, line 170, in node_tags
d1 = self.tag_cache[node_id]
KeyError: ‘i-02b77234ffad2072c’
2021-02-17 14:56:11,002 ERROR autoscaler.py:724 – StandardAutoscaler: kill_workers triggered
2021-02-17 14:56:11,453 ERROR autoscaler.py:729 – StandardAutoscaler: terminated 16 node(s)
2021-02-17 14:56:11,453 INFO monitor.py:250 – Monitor: Exception caught. Taking down workers…
2021-02-17 14:56:11,680 INFO monitor.py:262 – Monitor: Workers taken down.

# Fixed code
def terminate_node(self, node_id):
    node = self._get_cached_node(node_id)
    if self.cache_stopped_nodes:
        if node.spot_instance_request_id:
            cli_logger.print(
                "Terminating instance {} "
                + cf.dimmed("(cannot stop spot instances, only terminate)"),
                node_id,
            )  # todo: show node name?
            node.terminate()
        else:
            cli_logger.print(
                "Stopping instance {} "
                + cf.dimmed(
                    "(to terminate instead, "
                    "set `cache_stopped_nodes: False` "
                    "under `provider` in the cluster configuration)"
                ),
                node_id,
            )  # todo: show node name?
            node.stop()
    else:
        node.terminate()

    # TODO (Alex): We are leaking the tag cache here. Naively, we would
    # want to just remove the cache entry here, but terminating can be
    # asyncrhonous or error, which would result in a use after free error.
    # If this leak becomes bad, we can garbage collect the tag cache when
    # the node cache is updated.
    pass