# Buggy code
def _home(self):
    # TODO (Dmitri): Think about how to use the node's HOME variable
    # without making an extra kubectl exec call.
    if self._home_cached is None:
        cmd = self.kubectl + ["exec", "-it", self.node_id, "--", "printenv", "HOME"]
        joined_cmd = " ".join(cmd)
        raw_out = self.process_runner.check_output(joined_cmd, shell=True)
        self._home_cached = raw_out.decode().strip("\n\r")
    return self._home_cached


# Traceback (subprocess.CalledProcessError)
Updating cluster configuration. [hash=0194a452ebd82e0ab6eade7b4dd3a4f4f775d5de]
New status: syncing-files
[2/7] Processing file mounts
2020-12-15 09:03:24,116	INFO command_runner.py:169 -- NodeUpdater: ray-head-m5nvd: Running kubectl -n ray exec -it ray-head-m5nvd -- bash --login -c -i 'true &amp;&amp; source ~/.bashrc &amp;&amp; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore &amp;&amp; (mkdir -p ~)'
Error from server: error dialing backend: EOF
New status: update-failed
!!!
Setup command `kubectl -n ray exec -it ray-head-m5nvd -- printenv HOME` failed with exit code 1. stderr:
!!!

Exception in thread Thread-1:
Traceback (most recent call last):
File "/Users/rliaw/miniconda3/lib/python3.7/threading.py", line 926, in _bootstrap_inner
self.run()
File "/Users/rliaw/miniconda3/lib/python3.7/site-packages/ray/autoscaler/_private/updater.py", line 124, in run
self.do_update()
File "/Users/rliaw/miniconda3/lib/python3.7/site-packages/ray/autoscaler/_private/updater.py", line 312, in do_update
self.rsync_up, step_numbers=(1, NUM_SETUP_STEPS))
File "/Users/rliaw/miniconda3/lib/python3.7/site-packages/ray/autoscaler/_private/updater.py", line 210, in sync_file_mounts
do_sync(remote_path, local_path)
File "/Users/rliaw/miniconda3/lib/python3.7/site-packages/ray/autoscaler/_private/updater.py", line 198, in do_sync
local_path, remote_path, docker_mount_if_possible=True)
File "/Users/rliaw/miniconda3/lib/python3.7/site-packages/ray/autoscaler/_private/updater.py", line 446, in rsync_up
self.cmd_runner.run_rsync_up(source, target, options=options)
File "/Users/rliaw/miniconda3/lib/python3.7/site-packages/ray/autoscaler/_private/command_runner.py", line 197, in run_rsync_up
target = self._home + target[1:]
File "/Users/rliaw/miniconda3/lib/python3.7/site-packages/ray/autoscaler/_private/command_runner.py", line 258, in _home
raw_out = self.process_runner.check_output(joined_cmd, shell=True)
File "/Users/rliaw/miniconda3/lib/python3.7/subprocess.py", line 395, in check_output
**kwargs).stdout
File "/Users/rliaw/miniconda3/lib/python3.7/subprocess.py", line 487, in run
output=stdout, stderr=stderr)
subprocess.CalledProcessError: Command 'kubectl -n ray exec -it ray-head-m5nvd -- printenv HOME' returned non-zero exit status 1.

Failed to setup head node.

# Fixed code
def _home(self):
    if self._home_cached is not None:
        return self._home_cached
    for _ in range(MAX_HOME_RETRIES - 1):
        try:
            self._home_cached = self._try_to_get_home()
            return self._home_cached
        except Exception:
            # TODO (Dmitri): Identify the exception we're trying to avoid.
            logger.info(
                "Error reading container's home directory. "
                f"Retrying in {HOME_RETRY_DELAY_S} seconds."
            )
            time.sleep(HOME_RETRY_DELAY_S)
    # Last try
    self._home_cached = self._try_to_get_home()
    return self._home_cached