Repository: skypilot-org/skypilot
Pull request #1191 (2022-10-04 03:26:32)

UX/reliability changes.
Many changes from a few weeks ago.

UX
- Make entrypoint optional -- no more `sky launch <flags> ''`, simply do `sky launch <flags>`
- Several reworded logging messages
- After a `spot launch`, no longer print the first few entries from `spot status` as it's too verbose

Reliability
- Handle rate limit exceeded errors from GCP (triggered when launching a bunch of things)
- Fail the runtime setup if ray patching somehow failed


Patch:
diff --git a/sky/authentication.py b/sky/authentication.py
index 10a0877bf..8378801f5 100644
--- a/sky/authentication.py
+++ b/sky/authentication.py
@@ -12,6 +12,7 @@ from cryptography.hazmat.primitives import serialization
 from cryptography.hazmat.primitives.asymmetric import rsa
 from cryptography.hazmat.backends import default_backend
 
+from sky import clouds
 from sky import sky_logging
 from sky.adaptors import aws, gcp
 from sky.utils import common_utils
@@ -30,11 +31,6 @@ logger = sky_logging.init_logger(__name__)
 MAX_TRIALS = 64
 PRIVATE_SSH_KEY_PATH = '~/.ssh/sky-key'
 
-GCP_CONFIGURE_PATH = '~/.config/gcloud/configurations/config_default'
-# Do not place the backup under the gcloud config directory, as ray
-# autoscaler can overwrite that directory on the remote nodes.
-GCP_CONFIGURE_SKY_BACKUP_PATH = '~/.sky/.sky_gcp_config_default'
-
 
 def generate_rsa_key_pair():
     key = rsa.generate_private_key(backend=default_backend(),
@@ -212,27 +208,13 @@ def setup_gcp_authentication(config):
             'additional authentication steps.')
         # Read the account information from the credential file, since the user
         # should be set according the account, when the oslogin is enabled.
-        config_path = os.path.expanduser(GCP_CONFIGURE_PATH)
+        config_path = os.path.expanduser(clouds.gcp.GCP_CONFIG_PATH)
         sky_backup_config_path = os.path.expanduser(
-            GCP_CONFIGURE_SKY_BACKUP_PATH)
-        if not os.path.exists(sky_backup_config_path):
-            if not os.path.exists(config_path):
-                with ux_utils.print_exception_no_traceback():
-                    raise RuntimeError(
-                        'GCP authentication failed, as the oslogin is enabled '
-                        f'but the file {config_path} is not found.')
-
-            # Create a backup of the config_default file in the same folder (the
-            # folder will be uploaded by `sky launch`), as the original file can
-            # be modified on the remote cluster by ray causing failure of
-            # launching GCP cluster on a remote cluster.
-            subprocess.run(f'cp {config_path} {sky_backup_config_path}',
-                           shell=True,
-                           check=True)
-        new_file_mounts = config.get('file_mounts', {})
-        new_file_mounts[
-            GCP_CONFIGURE_SKY_BACKUP_PATH] = GCP_CONFIGURE_SKY_BACKUP_PATH
-        config['file_mounts'] = new_file_mounts
+            clouds.gcp.GCP_CONFIG_SKY_BACKUP_PATH)
+        assert os.path.exists(sky_backup_config_path), (
+            'GCP credential backup file '
+            f'{sky_backup_config_path!r} does not exist.')
+
         with open(sky_backup_config_path, 'r') as infile:
             for line in infile:
                 if line.startswith('account'):
diff --git a/sky/backends/backend_utils.py b/sky/backends/backend_utils.py
index 488c2b15b..be0160092 100644
--- a/sky/backends/backend_utils.py
+++ b/sky/backends/backend_utils.py
@@ -1740,7 +1740,7 @@ def get_clusters(
         yellow = colorama.Fore.YELLOW
         reset = colorama.Style.RESET_ALL
         logger.warning(f'{yellow}The following cluster{plural} terminated on '
-                       'the cloud and removed from Sky\'s cluster table: '
+                       'the cloud and removed from the cluster table: '
                        f'{cluster_str}{reset}')
     updated_records = [
         record for record in updated_records if record is not None
@@ -1910,7 +1910,8 @@ def interrupt_handler(signum, frame):
     logger.warning(f'{colorama.Fore.LIGHTBLACK_EX}The job will keep '
                    f'running after Ctrl-C.{colorama.Style.RESET_ALL}')
     kill_children_processes()
-    raise KeyboardInterrupt(exceptions.KEYBOARD_INTERRUPT_CODE)
+    with ux_utils.print_exception_no_traceback():
+        raise KeyboardInterrupt(exceptions.KEYBOARD_INTERRUPT_CODE)
 
 
 # Handle ctrl-z
@@ -1919,7 +1920,8 @@ def stop_handler(signum, frame):
     logger.warning(f'{colorama.Fore.LIGHTBLACK_EX}The job will keep '
                    f'running after Ctrl-Z.{colorama.Style.RESET_ALL}')
     kill_children_processes()
-    raise KeyboardInterrupt(exceptions.SIGTSTP_CODE)
+    with ux_utils.print_exception_no_traceback():
+        raise KeyboardInterrupt(exceptions.SIGTSTP_CODE)
 
 
 def validate_schema(obj, schema, err_msg_prefix=''):
diff --git a/sky/backends/cloud_vm_ray_backend.py b/sky/backends/cloud_vm_ray_backend.py
index f00f99a95..d7f9db185 100644
--- a/sky/backends/cloud_vm_ray_backend.py
+++ b/sky/backends/cloud_vm_ray_backend.py
@@ -1187,11 +1187,19 @@ class RetryingVmProvisioner(object):
                         'Retrying head node provisioning due to head fetching '
                         'timeout.')
                     return True
+
+            if isinstance(to_provision_cloud, clouds.GCP):
+                if ('Quota exceeded for quota metric \'List requests\' and '
+                        'limit \'List requests per minute\'' in stderr):
+                    logger.info(
+                        'Retrying due to list request rate limit exceeded.')
+                    return True
+
             if ('Processing file mounts' in stdout and
                     'Running setup commands' not in stdout and
                     'Failed to setup head node.' in stderr):
                 logger.info(
-                    'Retrying sky runtime setup due to ssh connection issue.')
+                    'Retrying runtime setup due to ssh connection issue.')
                 return True
 
             if ('ConnectionResetError: [Errno 54] Connection reset by peer'
@@ -1202,9 +1210,18 @@ class RetryingVmProvisioner(object):
 
         retry_cnt = 0
         ray_up_return_value = None
+        # 5 seconds to 180 seconds. We need backoff for e.g., rate limit per
+        # minute errors.
+        backoff = common_utils.Backoff(initial_backoff=5,
+                                       max_backoff_factor=180 // 5)
         while (retry_cnt < _MAX_RAY_UP_RETRY and
                need_ray_up(ray_up_return_value)):
             retry_cnt += 1
+            if retry_cnt > 1:
+                sleep = backoff.current_backoff()
+                logger.info(
+                    'Retrying launching in {:.1f} seconds.'.format(sleep))
+                time.sleep(sleep)
             ray_up_return_value = ray_up()
 
         assert ray_up_return_value is not None
@@ -1672,11 +1689,11 @@ class CloudVmRayBackend(backends.Backend):
                 # RetryingVmProvisioner will retry within a cloud's regions
                 # first (if a region is not explicitly requested), then
                 # optionally retry on all other clouds (if
-                # backend.register_info() has been called).
-                # After this "round" of optimization across clouds, provisioning
-                # may still have not succeeded. This while loop will then kick
-                # in if retry_until_up is set, which will kick off new "rounds"
-                # of optimization infinitely.
+                # backend.register_info() has been called).  After this "round"
+                # of optimization across clouds, provisioning may still have
+                # not succeeded. This while loop will then kick in if
+                # retry_until_up is set, which will kick off new "rounds" of
+                # optimization infinitely.
                 try:
                     provisioner = RetryingVmProvisioner(self.log_dir, self._dag,
                                                         self._optimize_target,
@@ -2483,6 +2500,9 @@ class CloudVmRayBackend(backends.Backend):
                 with backend_utils.safe_console_status(
                         f'[bold cyan]{teardown_verb} '
                         f'[green]{cluster_name}'):
+                    # FIXME(zongheng): support retries. This call can fail for
+                    # example due to GCP returning list requests per limit
+                    # exceeded.
                     returncode, stdout, stderr = log_lib.run_with_log(
                         ['ray', 'down', '-y', f.name],
                         log_abs_path,
diff --git a/sky/cli.py b/sky/cli.py
index 037c521c9..202181c14 100644
--- a/sky/cli.py
+++ b/sky/cli.py
@@ -334,6 +334,7 @@ _EXTRA_RESOURCES_OPTIONS = [
          'Passing "none" resets the config.')),
     click.option(
         '--instance-type',
+        '-t',
         required=False,
         type=str,
         help=('The instance type to use. If specified, overrides the '
@@ -946,7 +947,7 @@ def cli():
 
 @cli.command(cls=_DocumentedCodeCommand)
 @click.argument('entrypoint',
-                required=True,
+                required=False,
                 type=str,
                 nargs=-1,
                 shell_complete=_complete_file_name)
@@ -1821,7 +1822,7 @@ def _terminate_or_stop_clusters(
                        f'{reserved_clusters_str} is not supported.')
                 if terminate:
                     msg += (
-                        '\nPlease specify --purge to force termination of the '
+                        '\nPlease specify --purge (-p) to force-terminate the '
                         'reserved cluster(s).')
                 raise click.UsageError(msg)
             if len(names) != 0:
diff --git a/sky/cloud_stores.py b/sky/cloud_stores.py
index dc8e69574..b4f8cdd59 100644
--- a/sky/cloud_stores.py
+++ b/sky/cloud_stores.py
@@ -10,6 +10,7 @@ TODO:
 import subprocess
 import urllib.parse
 
+from sky.clouds import gcp
 from sky.data import data_utils
 from sky.adaptors import aws
 
@@ -93,26 +94,18 @@ class GcsCloudStorage(CloudStorage):
     # We use gsutil as a basic implementation.  One pro is that its -m
     # multi-threaded download is nice, which frees us from implementing
     # parellel workers on our end.
-    _GET_GSUTIL = [
-        # Skip if gsutil already exists.
-        'pushd /tmp &>/dev/null',
-        '(test -f ~/google-cloud-sdk/bin/gsutil || (wget --quiet '
-        'https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/'
-        'google-cloud-sdk-367.0.0-linux-x86_64.tar.gz && '
-        'tar xzf google-cloud-sdk-367.0.0-linux-x86_64.tar.gz && '
-        'mv google-cloud-sdk ~/ && '
-        '~/google-cloud-sdk/install.sh -q ))',
-        'popd &>/dev/null',
-    ]
+    # The gsutil command is part of the Google Cloud SDK, and we reuse
+    # the installation logic here.
+    _GET_GSUTIL = gcp.GCLOUD_INSTALLATION_COMMAND
 
-    _GSUTIL = '~/google-cloud-sdk/bin/gsutil'
+    _GSUTIL = 'gsutil'
 
     def is_directory(self, url: str) -> bool:
         """Returns whether 'url' is a directory.
         In cloud object stores, a "directory" refers to a regular object whose
         name is a prefix of other objects.
         """
-        commands = list(self._GET_GSUTIL)
+        commands = [self._GET_GSUTIL]
         commands.append(f'{self._GSUTIL} ls -d {url}')
         command = ' && '.join(commands)
         p = subprocess.run(command,
@@ -142,14 +135,14 @@ class GcsCloudStorage(CloudStorage):
         """Downloads a directory using gsutil."""
         download_via_gsutil = (
             f'{self._GSUTIL} -m rsync -r {source} {destination}')
-        all_commands = list(self._GET_GSUTIL)
+        all_commands = [self._GET_GSUTIL]
         all_commands.append(download_via_gsutil)
         return ' && '.join(all_commands)
 
     def make_sync_file_command(self, source: str, destination: str) -> str:
         """Downloads a file using gsutil."""
         download_via_gsutil = f'{self._GSUTIL} -m cp {source} {destination}'
-        all_commands = list(self._GET_GSUTIL)
+        all_commands = [self._GET_GSUTIL]
         all_commands.append(download_via_gsutil)
         return ' && '.join(all_commands)
 
diff --git a/sky/clouds/gcp.py b/sky/clouds/gcp.py
index 46ab4e2eb..fcbc54133 100644
--- a/sky/clouds/gcp.py
+++ b/sky/clouds/gcp.py
@@ -18,6 +18,11 @@ DEFAULT_GCP_APPLICATION_CREDENTIAL_PATH = os.path.expanduser(
     '~/.config/gcloud/'
     'application_default_credentials.json')
 
+GCP_CONFIG_PATH = '~/.config/gcloud/configurations/config_default'
+# Do not place the backup under the gcloud config directory, as ray
+# autoscaler can overwrite that directory on the remote nodes.
+GCP_CONFIG_SKY_BACKUP_PATH = '~/.sky/.sky_gcp_config_default'
+
 # Minimum set of files under ~/.config/gcloud that grant GCP access.
 _CREDENTIAL_FILES = [
     'credentials.db',
@@ -29,6 +34,23 @@ _CREDENTIAL_FILES = [
 
 _IMAGE_ID_PREFIX = ('projects/deeplearning-platform-release/global/images/')
 
+_GCLOUD_INSTALLATION_LOG = '~/.sky/logs/gcloud_installation.log'
+# Need to be run with /bin/bash
+# We factor out the installation logic to keep it align in both spot
+# controller and cloud stores.
+GCLOUD_INSTALLATION_COMMAND = f'pushd /tmp &>/dev/null && \
+    gcloud --help > /dev/null 2>&1 || \
+    {{ mkdir -p {os.path.dirname(_GCLOUD_INSTALLATION_LOG)} && \
+    wget --quiet https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-382.0.0-linux-x86_64.tar.gz > {_GCLOUD_INSTALLATION_LOG} && \
+    tar xzf google-cloud-sdk-382.0.0-linux-x86_64.tar.gz >> {_GCLOUD_INSTALLATION_LOG} && \
+    rm -rf ~/google-cloud-sdk >> {_GCLOUD_INSTALLATION_LOG}  && \
+    mv google-cloud-sdk ~/ && \
+    ~/google-cloud-sdk/install.sh -q >> {_GCLOUD_INSTALLATION_LOG} 2>&1 && \
+    echo "source ~/google-cloud-sdk/path.bash.inc > /dev/null 2>&1" >> ~/.bashrc && \
+    source ~/google-cloud-sdk/path.bash.inc >> {_GCLOUD_INSTALLATION_LOG} 2>&1; }} && \
+    {{ cp {GCP_CONFIG_SKY_BACKUP_PATH} {GCP_CONFIG_PATH} > /dev/null 2>&1 || true; }} && \
+    popd &>/dev/null'
+
 
 def _run_output(cmd):
     proc = subprocess.run(cmd,
@@ -316,7 +338,7 @@ class GCP(clouds.Cloud):
                 'Run the following commands:\n    '
                 # Install the Google Cloud SDK:
                 '  $ pip install google-api-python-client\n    '
-                '  $ conda install -c conda-forge google-cloud-sdk\n    '
+                '  $ conda install -c conda-forge google-cloud-sdk -y\n    '
                 # This authenticates the CLI to make `gsutil` work:
                 '  $ gcloud init\n    '
                 # This will generate
@@ -328,13 +350,30 @@ class GCP(clouds.Cloud):
         return True, None
 
     def get_credential_file_mounts(self) -> Dict[str, str]:
+        # Create a backup of the config_default file, as the original file can
+        # be modified on the remote cluster by ray causing authentication
+        # problems. The backup file will be updated to the remote cluster
+        # whenever the original file is not empty and will be applied
+        # appropriately on the remote cluster when neccessary.
+        if (os.path.exists(os.path.expanduser(GCP_CONFIG_PATH)) and
+                os.path.getsize(os.path.expanduser(GCP_CONFIG_PATH)) > 0):
+            subprocess.run(f'cp {GCP_CONFIG_PATH} {GCP_CONFIG_SKY_BACKUP_PATH}',
+                           shell=True,
+                           check=True)
+        elif not os.path.exists(os.path.expanduser(GCP_CONFIG_SKY_BACKUP_PATH)):
+            raise RuntimeError(
+                'GCP credential file is empty. Please make sure you '
+                'have run: gcloud init')
+
         # Excluding the symlink to the python executable created by the gcp
         # credential, which causes problem for ray up multiple nodes, tracked
         # in #494, #496, #483.
-        return {
+        credentials = {
             f'~/.config/gcloud/{filename}': f'~/.config/gcloud/{filename}'
             for filename in _CREDENTIAL_FILES
         }
+        credentials[GCP_CONFIG_SKY_BACKUP_PATH] = GCP_CONFIG_SKY_BACKUP_PATH
+        return credentials
 
     def instance_type_exists(self, instance_type):
         return service_catalog.instance_type_exists(instance_type, 'gcp')
diff --git a/sky/execution.py b/sky/execution.py
index 8dbf23111..5ba46e4a1 100644
--- a/sky/execution.py
+++ b/sky/execution.py
@@ -15,7 +15,6 @@ Current task launcher:
 import enum
 import tempfile
 import os
-import time
 from typing import Any, List, Optional
 
 import colorama
@@ -28,6 +27,7 @@ from sky import optimizer
 from sky import sky_logging
 from sky import spot
 from sky.backends import backend_utils
+from sky.clouds import gcp
 from sky.data import data_utils
 from sky.data import storage
 from sky.usage import usage_lib
@@ -225,18 +225,17 @@ def _execute(
                 backend.teardown_ephemeral_storage(task)
                 backend.teardown(handle, terminate=True)
     finally:
-        # UX: print live clusters to make users aware (to save costs).
-        # Needed because this finally doesn't always get executed on errors.
-        # Disable the usage collection for this status command.
-        env = dict(os.environ,
-                   **{env_options.Options.DISABLE_LOGGING.value: '1'})
-        if cluster_name == spot.SPOT_CONTROLLER_NAME:
-            # For spot controller task, it requires a while to have the
-            # managed spot status shown in the status table.
-            time.sleep(0.5)
-            subprocess_utils.run(
-                f'sky spot status | head -n {_MAX_SPOT_JOB_LENGTH}')
-        else:
+        if cluster_name != spot.SPOT_CONTROLLER_NAME:
+            # UX: print live clusters to make users aware (to save costs).
+            #
+            # Don't print if this job is launched by the spot controller,
+            # because spot jobs are serverless, there can be many of them, and
+            # users tend to continuously monitor spot jobs using `sky spot
+            # status`.
+            #
+            # Disable the usage collection for this status command.
+            env = dict(os.environ,
+                       **{env_options.Options.DISABLE_LOGGING.value: '1'})
             subprocess_utils.run('sky status', env=env)
         print()
         print('\x1b[?25h', end='')  # Show cursor.
@@ -369,11 +368,8 @@ def spot_launch(
 
     change_default_value = dict()
     if not resources.use_spot_specified:
-        logger.info('Field use_spot not specified; defaulting to True.')
         change_default_value['use_spot'] = True
     if resources.spot_recovery is None:
-        logger.info('No spot recovery strategy specified; defaulting to '
-                    f'{spot.SPOT_DEFAULT_STRATEGY}.')
         change_default_value['spot_recovery'] = spot.SPOT_DEFAULT_STRATEGY
 
     new_resources = resources.copy(**change_default_value)
@@ -446,6 +442,7 @@ def spot_launch(
                 'user_yaml_path': f.name,
                 'spot_controller': controller_name,
                 'cluster_name': name,
+                'gcloud_installation_commands': gcp.GCLOUD_INSTALLATION_COMMAND,
                 'is_dev': env_options.Options.IS_DEVELOPER.get(),
                 'disable_logging': env_options.Options.DISABLE_LOGGING.get(),
                 'logging_user_hash': common_utils.get_user_hash(),
diff --git a/sky/resources.py b/sky/resources.py
index 8a81584b2..1cbc95fc3 100644
--- a/sky/resources.py
+++ b/sky/resources.py
@@ -252,7 +252,7 @@ class Resources:
         if self._cloud is None:
             with ux_utils.print_exception_no_traceback():
                 raise ValueError(
-                    'Cloud must be specified together with region/zone.')
+                    'Cloud must be specified when region/zone are specified.')
 
         # Validate whether region and zone exist in the catalog.
         self._cloud.validate_region_zone(region, zone)
diff --git a/sky/spot/spot_utils.py b/sky/spot/spot_utils.py
index d67fb8b67..4c0e0b36c 100644
--- a/sky/spot/spot_utils.py
+++ b/sky/spot/spot_utils.py
@@ -250,8 +250,8 @@ def stream_logs_by_id(job_id: int) -> str:
                         f'(status: {job_status.value}).')
             break
         logger.info(
-            f'INFO: The return code is {returncode}. '
-            f'Check the job status in {JOB_STATUS_CHECK_GAP_SECONDS} seconds.')
+            f'INFO: (Log streaming) Got return code {returncode}. Retrying '
+            f'in {JOB_STATUS_CHECK_GAP_SECONDS} seconds.')
         # If the tailing fails, it is likely that the cluster fails, so we wait
         # a while to make sure the spot state is updated by the controller, and
         # check the spot status again.
diff --git a/sky/templates/aws-ray.yml.j2 b/sky/templates/aws-ray.yml.j2
index 2b95e3cd3..da3fac071 100644
--- a/sky/templates/aws-ray.yml.j2
+++ b/sky/templates/aws-ray.yml.j2
@@ -105,7 +105,7 @@ setup_commands:
     sudo bash -c 'rm -rf /etc/security/limits.d; echo "* soft nofile 1048576" >> /etc/security/limits.conf; echo "* hard nofile 1048576" >> /etc/security/limits.conf';
     sudo grep -e '^DefaultTasksMax' /etc/systemd/system.conf || (sudo bash -c 'echo "DefaultTasksMax=infinity" >> /etc/systemd/system.conf'); sudo systemctl set-property user-$(id -u $(whoami)).slice TasksMax=infinity; sudo systemctl daemon-reload;
     mkdir -p ~/.ssh; (grep -Pzo -q "Host \*\n  StrictHostKeyChecking no" ~/.ssh/config) || printf "Host *\n  StrictHostKeyChecking no\n" >> ~/.ssh/config;
-    python3 -c "from sky.skylet.ray_patches import patch; patch()";
+    python3 -c "from sky.skylet.ray_patches import patch; patch()" || exit 1;
     [ -f /etc/fuse.conf ] && sudo sed -i 's/#user_allow_other/user_allow_other/g' /etc/fuse.conf || (sudo sh -c 'echo "user_allow_other" > /etc/fuse.conf'); # This is needed for `-o allow_other` option for `goofys`;
 
 # Command to start ray on the head node. You don't need to change this.
diff --git a/sky/templates/azure-ray.yml.j2 b/sky/templates/azure-ray.yml.j2
index 9a7802fc9..0e136f058 100644
--- a/sky/templates/azure-ray.yml.j2
+++ b/sky/templates/azure-ray.yml.j2
@@ -111,7 +111,7 @@ setup_commands:
     sudo bash -c 'rm -rf /etc/security/limits.d; echo "* soft nofile 1048576" >> /etc/security/limits.conf; echo "* hard nofile 1048576" >> /etc/security/limits.conf';
     sudo grep -e '^DefaultTasksMax' /etc/systemd/system.conf || (sudo bash -c 'echo "DefaultTasksMax=infinity" >> /etc/systemd/system.conf'); sudo systemctl set-property user-$(id -u $(whoami)).slice TasksMax=infinity; sudo systemctl daemon-reload;
     mkdir -p ~/.ssh; (grep -Pzo -q "Host \*\n  StrictHostKeyChecking no" ~/.ssh/config) || printf "Host *\n  StrictHostKeyChecking no\n" >> ~/.ssh/config;
-    python3 -c "from sky.skylet.ray_patches import patch; patch()";
+    python3 -c "from sky.skylet.ray_patches import patch; patch()" || exit 1;
     [ -f /etc/fuse.conf ] && sudo sed -i 's/#user_allow_other/user_allow_other/g' /etc/fuse.conf || (sudo sh -c 'echo "user_allow_other" > /etc/fuse.conf');
 
 # Command to start ray on the head node. You don't need to change this.
diff --git a/sky/templates/gcp-ray.yml.j2 b/sky/templates/gcp-ray.yml.j2
index d1ff6299c..1994094a6 100644
--- a/sky/templates/gcp-ray.yml.j2
+++ b/sky/templates/gcp-ray.yml.j2
@@ -147,7 +147,7 @@ setup_commands:
     sudo bash -c 'rm -rf /etc/security/limits.d; echo "* soft nofile 1048576" >> /etc/security/limits.conf; echo "* hard nofile 1048576" >> /etc/security/limits.conf';
     sudo grep -e '^DefaultTasksMax' /etc/systemd/system.conf || (sudo bash -c 'echo "DefaultTasksMax=infinity" >> /etc/systemd/system.conf'); sudo systemctl set-property user-$(id -u $(whoami)).slice TasksMax=infinity; sudo systemctl daemon-reload;
     mkdir -p ~/.ssh; (grep -Pzo -q "Host \*\n  StrictHostKeyChecking no" ~/.ssh/config) || printf "Host *\n  StrictHostKeyChecking no\n" >> ~/.ssh/config;
-    python3 -c "from sky.skylet.ray_patches import patch; patch()";
+    python3 -c "from sky.skylet.ray_patches import patch; patch()" || exit 1;
     [ -f /etc/fuse.conf ] && sudo sed -i 's/#user_allow_other/user_allow_other/g' /etc/fuse.conf || (sudo sh -c 'echo "user_allow_other" > /etc/fuse.conf'); # This is needed for `-o allow_other` option for `gcsfuse`;
 
 # Command to start ray on the head node. You don't need to change this.
diff --git a/sky/templates/spot-controller.yaml.j2 b/sky/templates/spot-controller.yaml.j2
index 00e6e4878..54d64c1c9 100644
--- a/sky/templates/spot-controller.yaml.j2
+++ b/sky/templates/spot-controller.yaml.j2
@@ -25,13 +25,12 @@ setup: |
   # pip list | grep azure-cli > /dev/null 2>&1 || \
   #  pip3 install azure-cli==2.31.0 azure-core
 
-  gcloud --help > /dev/null 2>&1 || \
-  (wget --quiet https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-382.0.0-linux-x86_64.tar.gz && \
-    tar xzf google-cloud-sdk-382.0.0-linux-x86_64.tar.gz && \
-    mv google-cloud-sdk ~/ && \
-    ~/google-cloud-sdk/install.sh -q > /dev/null 2>&1 && \
-    source ~/google-cloud-sdk/path.bash.inc > /dev/null 2>&1 && \
-    echo 'source ~/google-cloud-sdk/path.bash.inc > /dev/null 2>&1' >> ~/.bashrc)
+  {{gcloud_installation_commands}}
+
+  # Internal: disable logging for manually logging into the spot controller for debugging.
+  {% if is_dev %}
+  echo 'export SKYPILOT_DEV=1' >> ~/.bashrc
+  {% endif %}
 
 run: |
   python -u -m sky.spot.controller \


Test patch:
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 1ea5ff76d..59160d278 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -57,6 +57,10 @@ def test_accelerator_mismatch(monkeypatch):
         lambda: enabled_clouds,
     )
     monkeypatch.setattr('sky.check.check', lambda *_args, **_kwargs: None)
+    config_file_backup = tempfile.NamedTemporaryFile(
+        prefix='tmp_backup_config_default', delete=False)
+    monkeypatch.setattr('sky.clouds.gcp.GCP_CONFIG_SKY_BACKUP_PATH',
+                        config_file_backup.name)
 
     spec = textwrap.dedent("""\
         resources:
diff --git a/tests/test_optimizer_dryruns.py b/tests/test_optimizer_dryruns.py
index edd7e88e1..079d28232 100644
--- a/tests/test_optimizer_dryruns.py
+++ b/tests/test_optimizer_dryruns.py
@@ -38,6 +38,12 @@ def _make_resources(
         lambda: enabled_clouds,
     )
     monkeypatch.setattr('sky.check.check', lambda *_args, **_kwargs: None)
+
+    config_file_backup = tempfile.NamedTemporaryFile(
+        prefix='tmp_backup_config_default', delete=False)
+    monkeypatch.setattr('sky.clouds.gcp.GCP_CONFIG_SKY_BACKUP_PATH',
+                        config_file_backup.name)
+
     # Should create Resources here, since it uses the enabled clouds.
     return sky.Resources(*resources_args, **resources_kwargs)