Repository: skypilot-org/skypilot
Pull request #1139 (2022-08-30 06:38:29)

Install sky only when neccessary and add retry for spot cluster termination
Tested:
- [x] `sky launch -c min ./examples/minimal.yaml`; `sky launch -c min ./examples/minimal.yaml`. There is no SkyPilot installation in the `provision.log`
- [x] `tests/run_smoke_tests.sh`


Patch:
diff --git a/docs/source/reference/quota.rst b/docs/source/reference/quota.rst
index c6eb791a6..cdf58abf8 100644
--- a/docs/source/reference/quota.rst
+++ b/docs/source/reference/quota.rst
@@ -5,7 +5,7 @@ Requesting Quota Increase
 
 
 Most cloud providers enforce a quota policy to limit the number of VM instances that can exist in a given region.
-Users may encourter `QuotaExceeded` or `VcpuLimitExceeded` errors during resources provisioning, especially for high end GPUs such as V100/A100.
+Users may encounter `QuotaExceeded` or `VcpuLimitExceeded` errors during resources provisioning, especially for high end GPUs such as V100/A100.
 To check or increase your quota limits, please follow the below instructions.
 After submitting the request, it will usually take a few days for the support team to review.
 To increase chances of being approved, you may respond their inquiry emails on how the requested resources will be used your projects.
diff --git a/sky/backends/backend_utils.py b/sky/backends/backend_utils.py
index ebfeb629e..842e946cc 100644
--- a/sky/backends/backend_utils.py
+++ b/sky/backends/backend_utils.py
@@ -662,6 +662,7 @@ def write_cluster_config(to_provision: 'resources.Resources',
                          cluster_config_template: str,
                          cluster_name: str,
                          local_wheel_path: pathlib.Path,
+                         wheel_hash: str,
                          region: Optional[clouds.Region] = None,
                          zones: Optional[List[clouds.Zone]] = None,
                          auth_config: Optional[Dict[str, str]] = None,
@@ -727,6 +728,7 @@ def write_cluster_config(to_provision: 'resources.Resources',
                 'sky_remote_path': SKY_REMOTE_PATH,
                 'sky_local_path': str(local_wheel_path),
                 'sky_version': str(version.parse(sky.__version__)),
+                'sky_wheel_hash': wheel_hash,
                 # Local IP handling (optional).
                 'head_ip': None if ip_list is None else ip_list[0],
                 'worker_ips': None if ip_list is None else ip_list[1:],
diff --git a/sky/backends/cloud_vm_ray_backend.py b/sky/backends/cloud_vm_ray_backend.py
index f367b157d..dd37bb12d 100644
--- a/sky/backends/cloud_vm_ray_backend.py
+++ b/sky/backends/cloud_vm_ray_backend.py
@@ -439,7 +439,7 @@ class RetryingVmProvisioner(object):
 
     def __init__(self, log_dir: str, dag: 'dag.Dag',
                  optimize_target: OptimizeTarget,
-                 local_wheel_path: pathlib.Path):
+                 local_wheel_path: pathlib.Path, wheel_hash: str):
         self._blocked_regions = set()
         self._blocked_zones = set()
         self._blocked_launchable_resources = set()
@@ -448,6 +448,7 @@ class RetryingVmProvisioner(object):
         self._dag = dag
         self._optimize_target = optimize_target
         self._local_wheel_path = local_wheel_path
+        self._wheel_hash = wheel_hash
 
         colorama.init()
 
@@ -919,6 +920,7 @@ class RetryingVmProvisioner(object):
                 _get_cluster_config_template(to_provision.cloud),
                 cluster_name,
                 self._local_wheel_path,
+                self._wheel_hash,
                 region=region,
                 zones=zones,
                 dryrun=dryrun)
@@ -1654,7 +1656,7 @@ class CloudVmRayBackend(backends.Backend):
             with timeline.Event('backend.provision.wheel_build'):
                 # TODO(suquark): once we have sky on PyPI, we should directly
                 # install sky from PyPI.
-                local_wheel_path = wheel_utils.build_sky_wheel()
+                local_wheel_path, wheel_hash = wheel_utils.build_sky_wheel()
             backoff = common_utils.Backoff(_RETRY_UNTIL_UP_INIT_GAP_SECONDS)
             attempt_cnt = 1
             while True:
@@ -1669,7 +1671,8 @@ class CloudVmRayBackend(backends.Backend):
                 try:
                     provisioner = RetryingVmProvisioner(self.log_dir, self._dag,
                                                         self._optimize_target,
-                                                        local_wheel_path)
+                                                        local_wheel_path,
+                                                        wheel_hash)
                     config_dict = provisioner.provision_with_retries(
                         task, to_provision_config, dryrun, stream_logs)
                     break
diff --git a/sky/backends/wheel_utils.py b/sky/backends/wheel_utils.py
index 33e79ba01..3eee3fb9d 100644
--- a/sky/backends/wheel_utils.py
+++ b/sky/backends/wheel_utils.py
@@ -5,7 +5,7 @@ import pathlib
 import shutil
 import subprocess
 import tempfile
-from typing import Optional
+from typing import Optional, Tuple
 
 import filelock
 from packaging import version
@@ -83,14 +83,16 @@ def _build_sky_wheel() -> pathlib.Path:
     return WHEEL_DIR.absolute()
 
 
-def build_sky_wheel() -> pathlib.Path:
+def build_sky_wheel() -> Tuple[pathlib.Path, str]:
     """Build a wheel for Sky, or reuse a cached wheel.
 
     Caller is responsible for removing the wheel.
 
     Returns:
-        A temporary path to a directory holding the wheel; path is guaranteed
-        unique per wheel content hash.
+        A tuple of (wheel path, wheel hash):
+        - wheel_path: A temporary path to a directory holding the wheel; path
+        is guaranteed unique per wheel content hash.
+        - wheel_hash: The wheel content hash.
     """
 
     def _get_latest_modification_time(path: pathlib.Path) -> float:
@@ -130,4 +132,4 @@ def build_sky_wheel() -> pathlib.Path:
         temp_wheel_dir.mkdir(parents=True, exist_ok=True)
         shutil.copy(latest_wheel_path, temp_wheel_dir)
 
-    return temp_wheel_dir.absolute()
+    return temp_wheel_dir.absolute(), hash_of_latest_wheel
diff --git a/sky/spot/recovery_strategy.py b/sky/spot/recovery_strategy.py
index 90b41e546..fc7f86b31 100644
--- a/sky/spot/recovery_strategy.py
+++ b/sky/spot/recovery_strategy.py
@@ -97,17 +97,24 @@ class StrategyExecutor:
         """
         raise NotImplementedError
 
-    def terminate_cluster(self):
+    def terminate_cluster(self, max_retry: int = 3) -> None:
         """Terminate the spot cluster."""
         handle = global_user_state.get_handle_from_cluster_name(
             self.cluster_name)
-        if handle is not None:
+        if handle is None:
+            return
+        retry_cnt = 0
+        while True:
             try:
                 self.backend.teardown(handle, terminate=True)
-            except RuntimeError:
-                logger.error(
-                    f'Failed to terminate the spot cluster {self.cluster_name}.'
-                )
+                return
+            except Exception as e:  # pylint: disable=broad-except
+                retry_cnt += 1
+                if retry_cnt >= max_retry:
+                    raise RuntimeError('Failed to terminate the spot cluster '
+                                       f'{self.cluster_name}.') from e
+                logger.error('Failed to terminate the spot cluster '
+                             f'{self.cluster_name}. Retrying.')
 
     def _launch(self, max_retry=3, raise_on_failure=True) -> Optional[float]:
         """Implementation of launch().
diff --git a/sky/templates/aws-ray.yml.j2 b/sky/templates/aws-ray.yml.j2
index 938222b12..5fe500799 100644
--- a/sky/templates/aws-ray.yml.j2
+++ b/sky/templates/aws-ray.yml.j2
@@ -63,7 +63,7 @@ head_node_type: ray.head.default
 # Format: `REMOTE_PATH : LOCAL_PATH`
 file_mounts: {
   "{{sky_ray_yaml_remote_path}}": "{{sky_ray_yaml_local_path}}",
-  "{{sky_remote_path}}": "{{sky_local_path}}",
+  "{{sky_remote_path}}/{{sky_wheel_hash}}": "{{sky_local_path}}",
 {%- for remote_path, local_path in credentials.items() %}
   "{{remote_path}}": "{{local_path}}",
 {%- endfor %}
@@ -96,8 +96,7 @@ setup_commands:
     (which conda > /dev/null 2>&1 && conda init > /dev/null && conda config --set auto_activate_base false) || (wget -nc https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && bash Miniconda3-latest-Linux-x86_64.sh -b && eval "$(~/miniconda3/bin/conda shell.bash hook)" && conda init && conda config --set auto_activate_base true);
     source ~/.bashrc;
     (pip3 list | grep ray | grep {{ray_version}} 2>&1 > /dev/null || pip3 install -U ray[default]=={{ray_version}}) && mkdir -p ~/sky_workdir && mkdir -p ~/.sky/sky_app;
-    pip3 uninstall skypilot -y &> /dev/null;
-    pip3 install "$(echo {{sky_remote_path}}/skypilot-{{sky_version}}*.whl)[aws]";
+    [ "$(cat {{sky_remote_path}}/current_sky_wheel_hash)" == "{{sky_wheel_hash}}" ] || (pip3 uninstall skypilot -y; pip3 install "$(echo {{sky_remote_path}}/{{sky_wheel_hash}}/skypilot-{{sky_version}}*.whl)[aws]" || exit 1; echo "{{sky_wheel_hash}}" > {{sky_remote_path}}/current_sky_wheel_hash);
     sudo systemctl stop unattended-upgrades;
     sudo kill -9 `sudo lsof /var/lib/dpkg/lock-frontend | awk '{print $2}' | tail -n 1` || true;
     sudo pkill -9 apt-get;
diff --git a/sky/templates/azure-ray.yml.j2 b/sky/templates/azure-ray.yml.j2
index 2b9be5e3a..094dc59f6 100644
--- a/sky/templates/azure-ray.yml.j2
+++ b/sky/templates/azure-ray.yml.j2
@@ -70,7 +70,7 @@ head_node_type: ray.head.default
 # Format: `REMOTE_PATH : LOCAL_PATH`
 file_mounts: {
   "{{sky_ray_yaml_remote_path}}": "{{sky_ray_yaml_local_path}}",
-  "{{sky_remote_path}}": "{{sky_local_path}}",
+  "{{sky_remote_path}}/{{sky_wheel_hash}}": "{{sky_local_path}}",
 {%- for remote_path, local_path in credentials.items() %}
   "{{remote_path}}": "{{local_path}}",
 {%- endfor %}
@@ -102,8 +102,7 @@ setup_commands:
     which conda > /dev/null 2>&1 || (wget -nc https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && bash Miniconda3-latest-Linux-x86_64.sh -b && eval "$(/home/azureuser/miniconda3/bin/conda shell.bash hook)" && conda init && conda config --set auto_activate_base true);
     source ~/.bashrc;
     (pip3 list | grep ray | grep {{ray_version}} 2>&1 > /dev/null || pip3 install -U ray[default]=={{ray_version}}) && mkdir -p ~/sky_workdir && mkdir -p ~/.sky/sky_app && touch ~/.sudo_as_admin_successful;
-    pip3 uninstall skypilot -y &> /dev/null;
-    pip3 install "$(echo {{sky_remote_path}}/skypilot-{{sky_version}}*.whl)[azure]";
+    [ "$(cat {{sky_remote_path}}/current_sky_wheel_hash)" == "{{sky_wheel_hash}}" ] || (pip3 uninstall skypilot -y; pip3 install "$(echo {{sky_remote_path}}/{{sky_wheel_hash}}/skypilot-{{sky_version}}*.whl)[azure]" || exit 1; echo "{{sky_wheel_hash}}" > {{sky_remote_path}}/current_sky_wheel_hash);
     sudo systemctl stop unattended-upgrades;
     sudo kill -9 `sudo lsof /var/lib/dpkg/lock-frontend | awk '{print $2}' | tail -n 1` || true;
     sudo pkill -9 apt-get;
diff --git a/sky/templates/gcp-ray.yml.j2 b/sky/templates/gcp-ray.yml.j2
index e0c782982..38b7551ba 100644
--- a/sky/templates/gcp-ray.yml.j2
+++ b/sky/templates/gcp-ray.yml.j2
@@ -105,7 +105,7 @@ head_node_type: ray_head_default
 # Format: `REMOTE_PATH : LOCAL_PATH`
 file_mounts: {
   "{{sky_ray_yaml_remote_path}}": "{{sky_ray_yaml_local_path}}",
-  "{{sky_remote_path}}": "{{sky_local_path}}",
+  "{{sky_remote_path}}/{{sky_wheel_hash}}": "{{sky_local_path}}",
 {%- for remote_path, local_path in credentials.items() %}
   "{{remote_path}}": "{{local_path}}",
 {%- endfor %}
@@ -143,8 +143,7 @@ setup_commands:
     pip3 install --upgrade google-api-python-client;
   {%- endif %}
     (pip3 list | grep ray | grep {{ray_version}} 2>&1 > /dev/null || pip3 install -U ray[default]=={{ray_version}}) && mkdir -p ~/sky_workdir && mkdir -p ~/.sky/sky_app;
-    pip3 uninstall skypilot -y &> /dev/null;
-    pip3 install "$(echo {{sky_remote_path}}/skypilot-{{sky_version}}*.whl)[gcp]" || exit 1;
+    [ "$(cat {{sky_remote_path}}/current_sky_wheel_hash)" == "{{sky_wheel_hash}}" ] || (pip3 uninstall skypilot -y; pip3 install "$(echo {{sky_remote_path}}/{{sky_wheel_hash}}/skypilot-{{sky_version}}*.whl)[gcp]" || exit 1; echo "{{sky_wheel_hash}}" > {{sky_remote_path}}/current_sky_wheel_hash);
     sudo bash -c 'rm -rf /etc/security/limits.d; echo "* soft nofile 1048576" >> /etc/security/limits.conf; echo "* hard nofile 1048576" >> /etc/security/limits.conf';
     sudo grep -e '^DefaultTasksMax' /etc/systemd/system.conf || (sudo bash -c 'echo "DefaultTasksMax=infinity" >> /etc/systemd/system.conf'); sudo systemctl set-property user-$(id -u $(whoami)).slice TasksMax=infinity; sudo systemctl daemon-reload;
     mkdir -p ~/.ssh; (grep -Pzo -q "Host \*\n  StrictHostKeyChecking no" ~/.ssh/config) || printf "Host *\n  StrictHostKeyChecking no\n" >> ~/.ssh/config;


Test patch:
diff --git a/tests/test_wheels.py b/tests/test_wheels.py
index bbf07b413..1d2a18ce7 100644
--- a/tests/test_wheels.py
+++ b/tests/test_wheels.py
@@ -8,11 +8,13 @@ from sky.backends import wheel_utils
 def test_build_wheels():
     shutil.rmtree(wheel_utils.WHEEL_DIR, ignore_errors=True)
     start = time.time()
-    assert wheel_utils.build_sky_wheel().exists()
+    wheel_path, _ = wheel_utils.build_sky_wheel()
+    assert wheel_path.exists()
     duration = time.time() - start
 
     start = time.time()
-    assert wheel_utils.build_sky_wheel().exists()
+    wheel_path, _ = wheel_utils.build_sky_wheel()
+    assert wheel_path.exists()
     duration_cached = time.time() - start
 
     assert duration_cached < duration
@@ -25,6 +27,7 @@ def test_build_wheels():
     for root, _, _ in os.walk(str(wheel_utils.WHEEL_DIR)):
         # set file date to 1970-01-01 00:00 UTC
         os.utime(root, (0, 0))
-    assert wheel_utils.build_sky_wheel().exists()
+    wheel_path, _ = wheel_utils.build_sky_wheel()
+    assert wheel_path.exists()
 
     shutil.rmtree(wheel_utils.WHEEL_DIR, ignore_errors=True)