# Buggy code
def recover_if_needed(self, node_id, now):
    if not self.can_update(node_id):
        return
    key = self.provider.internal_ip(node_id)
    if key not in self.load_metrics.last_heartbeat_time_by_ip:
        self.load_metrics.last_heartbeat_time_by_ip[key] = now
    last_heartbeat_time = self.load_metrics.last_heartbeat_time_by_ip[key]
    delta = now - last_heartbeat_time
    if delta < AUTOSCALER_HEARTBEAT_TIMEOUT_S:
        return
    logger.warning(
        "StandardAutoscaler: "
        "{}: No heartbeat in {}s, "
        "restarting Ray to recover...".format(node_id, delta)
    )
    updater = NodeUpdaterThread(
        node_id=node_id,
        provider_config=self.config["provider"],
        provider=self.provider,
        auth_config=self.config["auth"],
        cluster_name=self.config["cluster_name"],
        file_mounts={},
        initialization_commands=[],
        setup_commands=[],
        ray_start_commands=with_head_node_ip(self.config["worker_start_ray_commands"]),
        runtime_hash=self.runtime_hash,
        process_runner=self.process_runner,
        use_internal_ip=True,
    )
    updater.start()
    self.updaters[node_id] = updater


# Traceback (AssertionError)
ray up config/example_full.yaml
2020-06-08 01:13:10,526 INFO config.py:143 -- _configure_iam_role: Role not specified for head node, using arn:aws:iam::<redacted>:instance-profile/ray-autoscaler-v1
2020-06-08 01:13:11,089 INFO config.py:194 -- _configure_key_pair: KeyName not specified for nodes, using ray-autoscaler_us-west-2
2020-06-08 01:13:11,365 INFO config.py:235 -- _configure_subnet: SubnetIds not specified for head node, using [('subnet-<redacted>', 'us-west-2b'), ('subnet-<redacted>', 'us-west-2a')]
2020-06-08 01:13:11,366 INFO config.py:241 -- _configure_subnet: SubnetId not specified for workers, using [('subnet-<redacted>', 'us-west-2b'), ('subnet-<redacted>', 'us-west-2a')]
2020-06-08 01:13:11,725 INFO config.py:261 -- _configure_security_group: SecurityGroupIds not specified for head node, using ray-autoscaler-default (sg-<redacted>)
2020-06-08 01:13:11,725 INFO config.py:268 -- _configure_security_group: SecurityGroupIds not specified for workers, using ray-autoscaler-default (sg-<redacted>)
This will restart cluster services [y/N]: y
2020-06-08 01:13:15,214 INFO commands.py:238 -- get_or_create_head_node: Updating files on head node...
2020-06-08 01:13:15,215 INFO updater.py:379 -- NodeUpdater: i-<redacted>: Updating to <redacted>
2020-06-08 01:13:15,216 INFO updater.py:423 -- NodeUpdater: i-<redacted>: Waiting for remote shell...
2020-06-08 01:13:15,422 INFO log_timer.py:22 -- AWSNodeProvider: Set tag ray-node-status=waiting-for-ssh on ['i-<redacted>
Exception in thread Thread-2:
Traceback (most recent call last):
File "/usr/lib/python3.6/threading.py", line 916, in _bootstrap_inner
self.run()
File "/home/richard/improbable/ray/python/ray/autoscaler/updater.py", line 383, in run
self.do_update()
File "/home/richard/improbable/ray/python/ray/autoscaler/updater.py", line 450, in do_update
self.wait_ready(deadline)
File "/home/richard/improbable/ray/python/ray/autoscaler/updater.py", line 444, in wait_ready
assert False, "Unable to connect to node"
AssertionError: Unable to connect to node

2020-06-08 01:11:59,679 ERROR commands.py:304 -- get_or_create_head_node: Updating <redacted> failed
2020-06-08 01:11:59,701 INFO log_timer.py:22 -- AWSNodeProvider: Set tag ray-node-status=update-failed on ['i-<redacted>']  [LogTimer=333ms]

# Fixed code
def recover_if_needed(self, node_id, now):
    if not self.can_update(node_id):
        return
    key = self.provider.internal_ip(node_id)
    if key not in self.load_metrics.last_heartbeat_time_by_ip:
        self.load_metrics.last_heartbeat_time_by_ip[key] = now
    last_heartbeat_time = self.load_metrics.last_heartbeat_time_by_ip[key]
    delta = now - last_heartbeat_time
    if delta < AUTOSCALER_HEARTBEAT_TIMEOUT_S:
        return
    logger.warning(
        "StandardAutoscaler: "
        "{}: No heartbeat in {}s, "
        "restarting Ray to recover...".format(node_id, delta)
    )
    updater = NodeUpdaterThread(
        node_id=node_id,
        provider_config=self.config["provider"],
        provider=self.provider,
        auth_config=self.config["auth"],
        cluster_name=self.config["cluster_name"],
        file_mounts={},
        initialization_commands=[],
        setup_commands=[],
        ray_start_commands=with_head_node_ip(self.config["worker_start_ray_commands"]),
        runtime_hash=self.runtime_hash,
        process_runner=self.process_runner,
        use_internal_ip=True,
        docker_config=self.config["docker"],
    )
    updater.start()
    self.updaters[node_id] = updater