# Buggy code
def sync_file_mounts(self, sync_cmd, step_numbers=(0, 2)):
    # step_numbers is (# of previous steps, total steps)
    previous_steps, total_steps = step_numbers

    nolog_paths = []
    if cli_logger.verbosity == 0:
        nolog_paths = ["~/ray_bootstrap_key.pem", "~/ray_bootstrap_config.yaml"]

    def do_sync(remote_path, local_path, allow_non_existing_paths=False):
        if allow_non_existing_paths and not os.path.exists(local_path):
            # Ignore missing source files. In the future we should support
            # the --delete-missing-args command to delete files that have
            # been removed
            return

        assert os.path.exists(local_path), local_path

        if os.path.isdir(local_path):
            if not local_path.endswith("/"):
                local_path += "/"
            if not remote_path.endswith("/"):
                remote_path += "/"

        with LogTimer(
            self.log_prefix + "Synced {} to {}".format(local_path, remote_path)
        ):
            self.cmd_runner.run("mkdir -p {}".format(os.path.dirname(remote_path)))
            sync_cmd(local_path, remote_path)

            if remote_path not in nolog_paths:
                # todo: timed here?
                cli_logger.print(
                    "{} from {}", cf.bold(remote_path), cf.bold(local_path)
                )

    # Rsync file mounts
    with cli_logger.group(
        "Processing file mounts", _numbered=("[]", previous_steps + 1, total_steps)
    ):
        for remote_path, local_path in self.file_mounts.items():
            do_sync(remote_path, local_path)

    if self.cluster_synced_files:
        with cli_logger.group(
            "Processing worker file mounts",
            _numbered=("[]", previous_steps + 2, total_steps),
        ):
            for path in self.cluster_synced_files:
                do_sync(path, path, allow_non_existing_paths=True)
    else:
        cli_logger.print(
            "No worker file mounts to sync",
            _numbered=("[]", previous_steps + 2, total_steps),
        )


# Traceback (AssertionError)
(vanilla_ray_venv) richard@richard-desktop:~/improbable/vanillas/ray/python/ray/autoscaler/aws$ ray up aws_gpu_dummy.yaml
2020-08-12 20:12:39,383	INFO config.py:268 -- _configure_iam_role: Role not specified for head node, using arn:aws:iam::179622923911:instance-profile/ray-autoscaler-v1
2020-08-12 20:12:39,612	INFO config.py:346 -- _configure_key_pair: KeyName not specified for nodes, using ray-autoscaler_us-east-1
2020-08-12 20:12:39,745	INFO config.py:407 -- _configure_subnet: SubnetIds not specified for head node, using [('subnet-f737f791', 'us-east-1a')]
2020-08-12 20:12:39,746	INFO config.py:417 -- _configure_subnet: SubnetId not specified for workers, using [('subnet-f737f791', 'us-east-1a')]
2020-08-12 20:12:40,358	INFO config.py:590 -- _create_security_group: Created new security group ray-autoscaler-richard_cluster_gpu_dummy (sg-0061ca6aff182c1bf)
2020-08-12 20:12:40,739	INFO config.py:444 -- _configure_security_group: SecurityGroupIds not specified for head node, using ray-autoscaler-richard_cluster_gpu_dummy (sg-0061ca6aff182c1bf)
2020-08-12 20:12:40,739	INFO config.py:454 -- _configure_security_group: SecurityGroupIds not specified for workers, using ray-autoscaler-richard_cluster_gpu_dummy (sg-0061ca6aff182c1bf)
This will create a new cluster [y/N]: y
2020-08-12 20:12:42,619	INFO commands.py:531 -- get_or_create_head_node: Launching new head node...
2020-08-12 20:12:42,620	INFO node_provider.py:326 -- NodeProvider: calling create_instances with subnet-f737f791 (count=1).
2020-08-12 20:12:44,032	INFO node_provider.py:354 -- NodeProvider: Created instance [id=i-0729c7a86355d5ff8, name=pending, info=pending]
2020-08-12 20:12:44,223	INFO commands.py:570 -- get_or_create_head_node: Updating files on head node...
2020-08-12 20:12:44,320	INFO command_runner.py:331 -- NodeUpdater: i-0729c7a86355d5ff8: Waiting for IP...
2020-08-12 20:12:54,409	INFO command_runner.py:331 -- NodeUpdater: i-0729c7a86355d5ff8: Waiting for IP...
2020-08-12 20:12:54,534	INFO log_timer.py:27 -- NodeUpdater: i-0729c7a86355d5ff8: Got IP  [LogTimer=10310ms]
2020-08-12 20:12:54,534	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; command -v docker'
Warning: Permanently added '3.226.253.119' (ECDSA) to the list of known hosts.
/usr/bin/docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:14:04,587	INFO updater.py:71 -- NodeUpdater: i-0729c7a86355d5ff8: Updating to 6b5fc8ee8c5dcdf3cfabe0bf90ba4e844f65a7c9
2020-08-12 20:14:04,587	INFO updater.py:180 -- NodeUpdater: i-0729c7a86355d5ff8: Waiting for remote shell...
2020-08-12 20:14:04,587	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
2020-08-12 20:14:04,950	INFO log_timer.py:27 -- AWSNodeProvider: Set tag ray-node-status=waiting-for-ssh on ['i-0729c7a86355d5ff8']  [LogTimer=361ms]
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:14:21,222	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:14:26,417	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:14:31,610	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:14:36,798	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:14:41,986	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:14:47,170	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:14:52,358	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:14:57,554	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:15:02,750	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:15:07,938	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:15:13,126	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:15:18,307	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:15:23,494	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
Shared connection to 3.226.253.119 closed.
2020-08-12 20:19:01,502	INFO command_runner.py:468 -- NodeUpdater: i-0729c7a86355d5ff8: Running ssh -tt -i /home/richard/.ssh/ray-autoscaler_us-east-1.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=5 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPath=/tmp/ray_ssh_6ae199a93c/cfde1c79f1/%C -o ControlPersist=10s -o ConnectTimeout=120s ubuntu@3.226.253.119 bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; docker exec -it  pytorch_docker /bin/bash -c '"'"'bash --login -c -i '"'"'"'"'"'"'"'"'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; uptime'"'"'"'"'"'"'"'"''"'"' '
Error: No such container: pytorch_docker
Shared connection to 3.226.253.119 closed.
2020-08-12 20:19:06,689	INFO log_timer.py:27 -- NodeUpdater: i-0729c7a86355d5ff8: Got remote shell  [LogTimer=302102ms]
2020-08-12 20:19:06,690	INFO log_timer.py:27 -- NodeUpdater: i-0729c7a86355d5ff8: Applied config 6b5fc8ee8c5dcdf3cfabe0bf90ba4e844f65a7c9  [LogTimer=302103ms]
2020-08-12 20:19:06,690	ERROR updater.py:88 -- NodeUpdater: i-0729c7a86355d5ff8: Error executing: Unable to connect to node

Exception in thread Thread-2:
Traceback (most recent call last):
File "/usr/lib/python3.6/threading.py", line 916, in _bootstrap_inner
self.run()
File "/home/richard/improbable/vanillas/ray/python/ray/autoscaler/updater.py", line 76, in run
self.do_update()
File "/home/richard/improbable/vanillas/ray/python/ray/autoscaler/updater.py", line 232, in do_update
self.wait_ready(deadline)
File "/home/richard/improbable/vanillas/ray/python/ray/autoscaler/updater.py", line 224, in wait_ready
assert False, "Unable to connect to node"
AssertionError: Unable to connect to node

2020-08-12 20:19:06,962	ERROR commands.py:650 -- get_or_create_head_node: Updating 3.226.253.119 failed
2020-08-12 20:19:07,002	INFO log_timer.py:27 -- AWSNodeProvider: Set tag ray-node-status=update-failed on ['i-0729c7a86355d5ff8']  [LogTimer=312ms]

# Fixed code
def sync_file_mounts(self, sync_cmd, step_numbers=(0, 2)):
    # step_numbers is (# of previous steps, total steps)
    previous_steps, total_steps = step_numbers

    nolog_paths = []
    if cli_logger.verbosity == 0:
        nolog_paths = ["~/ray_bootstrap_key.pem", "~/ray_bootstrap_config.yaml"]

    def do_sync(remote_path, local_path, allow_non_existing_paths=False):
        if allow_non_existing_paths and not os.path.exists(local_path):
            # Ignore missing source files. In the future we should support
            # the --delete-missing-args command to delete files that have
            # been removed
            return

        assert os.path.exists(local_path), local_path

        if os.path.isdir(local_path):
            if not local_path.endswith("/"):
                local_path += "/"
            if not remote_path.endswith("/"):
                remote_path += "/"

        with LogTimer(
            self.log_prefix + "Synced {} to {}".format(local_path, remote_path)
        ):
            self.cmd_runner.run(
                "mkdir -p {}".format(os.path.dirname(remote_path)), run_env="host"
            )
            sync_cmd(local_path, remote_path)

            if remote_path not in nolog_paths:
                # todo: timed here?
                cli_logger.print(
                    "{} from {}", cf.bold(remote_path), cf.bold(local_path)
                )

    # Rsync file mounts
    with cli_logger.group(
        "Processing file mounts", _numbered=("[]", previous_steps + 1, total_steps)
    ):
        for remote_path, local_path in self.file_mounts.items():
            do_sync(remote_path, local_path)

    if self.cluster_synced_files:
        with cli_logger.group(
            "Processing worker file mounts",
            _numbered=("[]", previous_steps + 2, total_steps),
        ):
            for path in self.cluster_synced_files:
                do_sync(path, path, allow_non_existing_paths=True)
    else:
        cli_logger.print(
            "No worker file mounts to sync",
            _numbered=("[]", previous_steps + 2, total_steps),
        )