# Buggy code
def wait_ready(self, deadline):
    with cli_logger.group(
        "Waiting for SSH to become available", _numbered=("[]", 1, 6)
    ):
        with LogTimer(self.log_prefix + "Got remote shell"):
            cli_logger.old_info(
                logger, "{}Waiting for remote shell...", self.log_prefix
            )

            cli_logger.print("Running `{}` as a test.", cf.bold("uptime"))
            first_conn_refused_time = None
            while time.time() < deadline and not self.provider.is_terminated(
                self.node_id
            ):
                try:
                    cli_logger.old_debug(
                        logger, "{}Waiting for remote shell...", self.log_prefix
                    )

                    self.cmd_runner.run("uptime")
                    cli_logger.old_debug(logger, "Uptime succeeded.")
                    cli_logger.success("Success.")
                    return True
                except ProcessRunnerError as e:
                    first_conn_refused_time = cmd_output_util.handle_ssh_fails(
                        e, first_conn_refused_time, retry_interval=READY_CHECK_INTERVAL
                    )
                    time.sleep(READY_CHECK_INTERVAL)
                except Exception as e:
                    # TODO(maximsmol): we should not be ignoring
                    # exceptions if they get filtered properly
                    # (new style log + non-interactive shells)
                    #
                    # however threading this configuration state
                    # is a pain and I'm leaving it for later

                    retry_str = str(e)
                    if hasattr(e, "cmd"):
                        retry_str = "(Exit Status {}): {}".format(
                            e.returncode, " ".join(e.cmd)
                        )

                    cli_logger.print(
                        "SSH still not available {}, retrying in {} seconds.",
                        cf.gray(retry_str),
                        cf.bold(str(READY_CHECK_INTERVAL)),
                    )
                    cli_logger.old_debug(
                        logger,
                        "{}Node not up, retrying: {}",
                        self.log_prefix,
                        retry_str,
                    )

                    time.sleep(READY_CHECK_INTERVAL)

    assert False, "Unable to connect to node"


# Traceback (AssertionError)
(vanilla_ray_venv) richard@richard-desktop:~/improbable/vanillas/ray/python/ray/autoscaler/aws$ ray up aws_gpu_dummy.yaml
2020-08-12 20:12:39,383	INFO config.py:268 -- _configure_iam_role: Role not specified for head node, using arn:aws:iam::179622923911:instance-profile/ray-autoscaler-v1
2020-08-12 20:12:39,612	INFO config.py:346 -- _configure_key_pair: KeyName not specified for nodes, using ray-autoscaler_us-east-1
2020-08-12 20:12:39,745	INFO config.py:407 -- _configure_subnet: SubnetIds not specified for head node, using [('subnet-f737f791', 'us-east-1a')]
2020-08-12 20:12:39,746	INFO config.py:417 -- _configure_subnet: SubnetId not specified for workers, using [('subnet-f737f791', 'us-east-1a')]
2020-08-12 20:12:40,358	INFO config.py:590 -- _create_security_group: Created new security group ray-autoscaler-richard_cluster_gpu_dummy (sg-0061ca6aff182c1bf)
2020-08-12 20:12:40,739	INFO config.py:444 -- _configure_security_group: SecurityGroupIds not specified for head node, using ray-autoscaler-richard_cluster_gpu_dummy (sg-0061ca6aff182c1bf)
2020-08-12 20:12:40,739	INFO config.py:454 -- _configure_security_group: SecurityGroupIds not specified for workers, using ray-autoscaler-richard_cluster_gpu_dummy (sg-0061ca6aff182c1bf)
This will create a new cluster [y/N]: y
2020-08-12 20:12:42,619	INFO commands.py:531 -- get_or_create_head_node: Launching new head node...
2020-08-12 20:12:42,620	INFO node_provider.py:326 -- NodeProvider: calling create_instances with subnet-f737f791 (count=1).
2020-08-12 20:12:44,032	INFO node_provider.py:354 -- NodeProvider: Created instance [id=i-0729c7a86355d5ff8, name=pending, info=pending]
2020-08-12 20:12:44,223	INFO commands.py:570 -- get_or_create_head_node: Updating files on head node...
2020-08-12 20:12:44,320	INFO command_runner.py:331 -- NodeUpdater: i-0729c7a86355d5ff8: Waiting for IP...
2020-08-12 20:12:54,409	INFO command_runner.py:331 -- NodeUpdater: i-0729c7a86355d5ff8: Waiting for IP...
2020-08-12 20:12:54,534	INFO log_timer.py:27 -- NodeUpdater: i-0729c7a86355d5ff8: Got IP  [LogTimer=10310ms]
2020-08-12 20:12:54,534	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; command -v docker'
Warning: Permanently added '3.226.253.119' (ECDSA) to the list of known hosts.
/usr/bin/docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:14:04,587	INFO updater.py:71 -- NodeUpdater: i-0729c7a86355d5ff8: Updating to 6b5fc8ee8c5dcdf3cfabe0bf90ba4e844f65a7c9
2020-08-12 20:14:04,587	INFO updater.py:180 -- NodeUpdater: i-0729c7a86355d5ff8: Waiting for remote shell...
2020-08-12 20:14:04,587	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
2020-08-12 20:14:04,950	INFO log_timer.py:27 -- AWSNodeProvider: Set tag ray-node-status=waiting-for-ssh on ['i-0729c7a86355d5ff8']  [LogTimer=361ms]
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:14:21,222	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:14:26,417	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:14:31,610	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:14:36,798	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:14:41,986	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:14:47,170	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:14:52,358	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:14:57,554	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:15:02,750	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:15:07,938	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:15:13,126	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:15:18,307	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:15:23,494	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
Shared connection to 3.226.253.119 closed.
2020-08-12 20:19:01,502	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:19:06,689	INFO log_timer.py:27 -- NodeUpdater: i-0729c7a86355d5ff8: Got remote shell  [LogTimer=302102ms]
2020-08-12 20:19:06,690	INFO log_timer.py:27 -- NodeUpdater: i-0729c7a86355d5ff8: Applied config 6b5fc8ee8c5dcdf3cfabe0bf90ba4e844f65a7c9  [LogTimer=302103ms]
2020-08-12 20:19:06,690	ERROR updater.py:88 -- NodeUpdater: i-0729c7a86355d5ff8: Error executing: Unable to connect to node

Exception in thread Thread-2:
Traceback (most recent call last):
File "/usr/lib/python3.6/threading.py", line 916, in _bootstrap_inner
self.run()
File "/home/richard/improbable/vanillas/ray/python/ray/autoscaler/updater.py", line 76, in run
self.do_update()
File "/home/richard/improbable/vanillas/ray/python/ray/autoscaler/updater.py", line 232, in do_update
self.wait_ready(deadline)
File "/home/richard/improbable/vanillas/ray/python/ray/autoscaler/updater.py", line 224, in wait_ready
assert False, "Unable to connect to node"
AssertionError: Unable to connect to node

2020-08-12 20:19:06,962	ERROR commands.py:650 -- get_or_create_head_node: Updating 3.226.253.119 failed
2020-08-12 20:19:07,002	INFO log_timer.py:27 -- AWSNodeProvider: Set tag ray-node-status=update-failed on ['i-0729c7a86355d5ff8']  [LogTimer=312ms]

# Fixed code
def wait_ready(self, deadline):
    with cli_logger.group(
        "Waiting for SSH to become available", _numbered=("[]", 1, 6)
    ):
        with LogTimer(self.log_prefix + "Got remote shell"):
            cli_logger.old_info(
                logger, "{}Waiting for remote shell...", self.log_prefix
            )

            cli_logger.print("Running `{}` as a test.", cf.bold("uptime"))
            first_conn_refused_time = None
            while time.time() < deadline and not self.provider.is_terminated(
                self.node_id
            ):
                try:
                    cli_logger.old_debug(
                        logger, "{}Waiting for remote shell...", self.log_prefix
                    )

                    self.cmd_runner.run("uptime", run_env="host")
                    cli_logger.old_debug(logger, "Uptime succeeded.")
                    cli_logger.success("Success.")
                    return True
                except ProcessRunnerError as e:
                    first_conn_refused_time = cmd_output_util.handle_ssh_fails(
                        e, first_conn_refused_time, retry_interval=READY_CHECK_INTERVAL
                    )
                    time.sleep(READY_CHECK_INTERVAL)
                except Exception as e:
                    # TODO(maximsmol): we should not be ignoring
                    # exceptions if they get filtered properly
                    # (new style log + non-interactive shells)
                    #
                    # however threading this configuration state
                    # is a pain and I'm leaving it for later

                    retry_str = str(e)
                    if hasattr(e, "cmd"):
                        retry_str = "(Exit Status {}): {}".format(
                            e.returncode, " ".join(e.cmd)
                        )

                    cli_logger.print(
                        "SSH still not available {}, retrying in {} seconds.",
                        cf.gray(retry_str),
                        cf.bold(str(READY_CHECK_INTERVAL)),
                    )
                    cli_logger.old_debug(
                        logger,
                        "{}Node not up, retrying: {}",
                        self.log_prefix,
                        retry_str,
                    )

                    time.sleep(READY_CHECK_INTERVAL)

    assert False, "Unable to connect to node"