Repository: skypilot-org/skypilot
Pull request #5129: Trigger a smoke test on aws before release the nightly build
Author: login: zpoint
Created: 2025-04-07T08:07:51Z | Merged: 2025-04-09T05:24:57Z

<!-- Describe the changes in this PR -->

Add two more steps:
* Trigger `--aws` smoke test.
* Wait for it to succeed with a 2-hour timeout.
We need a buildkite API token with `read_builds` permission. @Michaelvll 

<img width="494" alt="image" src="https://github.com/user-attachments/assets/097ce352-a1ef-42d3-9130-317cb25e0431" />


<!-- Describe the tests ran -->
<!-- Unit tests (tests/test_*.py) are part of GitHub CI; below are tests that launch on the cloud. -->

Tested (run the relevant ones):

- [x] Code formatting: install pre-commit (auto-check on commit) or `bash format.sh`
- [x] A [failure case](https://github.com/zpoint/skypilot/actions/runs/14326027805/job/40151560119#step:2:49) on my personal repo
- [x] A [success case](https://github.com/zpoint/skypilot/actions/runs/14325310741) on my personal repo

<!-- CI commands (/-prefixed) can only be triggered by repo members -->



Diff:
diff --git a/.github/workflows/pypi-nightly-build.yml b/.github/workflows/pypi-nightly-build.yml
index c4c6bfa2409..f3d9531654b 100644
--- a/.github/workflows/pypi-nightly-build.yml
+++ b/.github/workflows/pypi-nightly-build.yml
@@ -24,6 +24,8 @@ jobs:
     runs-on: ubuntu-latest
     needs: check-date
     if: ${{ needs.check_date.outputs.should_run != 'false' }}
+    outputs:
+      build_number: ${{ steps.trigger_buildkite.outputs.build_number }}
     steps:
       - name: Clone repository
         uses: actions/checkout@v3
@@ -52,12 +54,61 @@ jobs:
           --wheel
           --outdir dist/
           .
+      - name: Upload distribution artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+      # Trigger Buildkite smoke tests
+      - name: Trigger Buildkite Smoke Tests
+        id: trigger_buildkite
+        uses: buildkite/trigger-pipeline-action@v2.3.0
+        with:
+          buildkite_api_access_token: ${{ secrets.BUILDKITE_TOKEN }}
+          pipeline: "skypilot-1/smoke-tests"
+          branch: "master"
+          commit: "${{ github.sha }}"
+          message: "nightly-build-pypi"
+          ignore_pipeline_branch_filter: true
+          build_env_vars: '{"ARGS": "--aws"}'
+
+      # Extract build number from response JSON
+      - name: Extract build number
+        id: extract_build_number
+        run: |
+          build_number=$(echo '${{ steps.trigger_buildkite.outputs.json }}' | jq -r '.number')
+          echo "build_number=$build_number" >> $GITHUB_OUTPUT
+
+  wait-for-buildkite:
+    needs: nightly-build-pypi
+    uses: ./.github/workflows/wait-for-buildkite.yml
+    with:
+      organization: "skypilot-1"
+      pipeline: "smoke-tests"
+      build_number: ${{ needs.nightly-build-pypi.outputs.build_number }}
+      timeout_minutes: 120
+    secrets:
+      BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }}
+
+  publish-to-pypi:
+    needs: wait-for-buildkite
+    if: needs.wait-for-buildkite.outputs.build_status == 'success'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download distribution artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
       - name: Publish distribution to Test PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           password: ${{ secrets.TEST_PYPI_API_TOKEN }}
           repository-url: https://test.pypi.org/legacy/
           skip-existing: true
+
       - name: Publish distribution to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
diff --git a/.github/workflows/wait-for-buildkite.yml b/.github/workflows/wait-for-buildkite.yml
new file mode 100644
index 00000000000..e9ae34d3f44
--- /dev/null
+++ b/.github/workflows/wait-for-buildkite.yml
@@ -0,0 +1,84 @@
+name: Wait for Buildkite Build
+
+on:
+  workflow_call:
+    inputs:
+      organization:
+        required: true
+        type: string
+        description: "Buildkite organization name"
+      pipeline:
+        required: true
+        type: string
+        description: "Buildkite pipeline name"
+      build_number:
+        required: true
+        type: string
+        description: "Buildkite build number to wait for"
+      timeout_minutes:
+        required: false
+        type: number
+        default: 120
+        description: "Timeout in minutes (default: 2 hours)"
+    secrets:
+      BUILDKITE_TOKEN:
+        required: true
+        description: "Buildkite API token with read access"
+    outputs:
+      build_status:
+        description: "Final status of the build (success, failure, timeout)"
+        value: ${{ jobs.wait.outputs.build_status }}
+
+jobs:
+  wait:
+    runs-on: ubuntu-latest
+    outputs:
+      build_status: ${{ steps.wait_buildkite.outputs.build_status }}
+    steps:
+      - name: Wait for Buildkite build completion
+        id: wait_buildkite
+        env:
+          BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }}
+        run: |
+          organization="${{ inputs.organization }}"
+          pipeline="${{ inputs.pipeline }}"
+          build_number="${{ inputs.build_number }}"
+          max_attempts=$((2 * ${{ inputs.timeout_minutes }}))  # Check every 30 seconds
+          attempt=0
+
+          while [ $attempt -lt $max_attempts ]; do
+            response=$(curl -s -H "Authorization: Bearer $BUILDKITE_TOKEN" \
+              "https://api.buildkite.com/v2/organizations/$organization/pipelines/$pipeline/builds/$build_number")
+
+            # Check if response contains only an error message
+            if [ $(echo "$response" | jq 'has("message") and length == 1') == "true" ]; then
+              error_message=$(echo "$response" | jq -r '.message')
+              echo "Error from Buildkite API: $error_message"
+              echo "build_status=failure" >> $GITHUB_OUTPUT
+              exit 1
+            fi
+
+            status=$(echo "$response" | jq -r '.state')
+
+            case $status in
+              "passed")
+                echo "Buildkite build succeeded"
+                echo "build_status=success" >> $GITHUB_OUTPUT
+                exit 0
+                ;;
+              "failed"|"canceled"|"blocked")
+                echo "Buildkite build failed with status: $status"
+                echo "build_status=failure" >> $GITHUB_OUTPUT
+                exit 1
+                ;;
+              *)
+                echo "Buildkite build status: $status - waiting..."
+                sleep 30
+                attempt=$((attempt+1))
+                ;;
+            esac
+          done
+
+          echo "Timeout waiting for Buildkite build to complete (${{ inputs.timeout_minutes }} minute limit reached)"
+          echo "build_status=timeout" >> $GITHUB_OUTPUT
+          exit 1
diff --git a/.github/workflows/weekly-smoke-tests-trigger.yaml b/.github/workflows/weekly-smoke-tests-trigger.yaml
index 8c6c81e696c..2db3606e6bd 100644
--- a/.github/workflows/weekly-smoke-tests-trigger.yaml
+++ b/.github/workflows/weekly-smoke-tests-trigger.yaml
@@ -18,43 +18,24 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Trigger Smoke Tests
-        env:
-          BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }}
-          BUILDKITE_API_URL: ${{ github.event_name == 'workflow_dispatch'
-            && 'https://api.buildkite.com/v2/organizations/skypilot-1/pipelines/smoke-tests/builds'
-            || 'https://api.buildkite.com/v2/organizations/skypilot-1/pipelines/full-smoke-tests-run/builds' }}
-        run: |
-          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
-            data='{
-              "commit": "HEAD",
-              "branch": "master",
-              "message": "Manual Smoke Tests: ${{ github.event.inputs.param }}",
-              "ignore_pipeline_branch_filters": true,
-              "env": {
-                "ARGS": "${{ github.event.inputs.param }}"
-              }
-            }'
-          else
-            data='{
-              "commit": "HEAD",
-              "branch": "master",
-              "message": "Biweekly Smoke Tests",
-              "ignore_pipeline_branch_filters": true
-            }'
-          fi
+        uses: buildkite/trigger-pipeline-action@v2.3.0
+        if: github.event_name == 'workflow_dispatch'
+        with:
+          buildkite_api_access_token: ${{ secrets.BUILDKITE_TOKEN }}
+          pipeline: "skypilot-1/smoke-tests"
+          branch: "master"
+          commit: "HEAD"
+          message: "Manual Smoke Tests: ${{ github.event.inputs.param }}"
+          ignore_pipeline_branch_filter: true
+          build_env_vars: '{"ARGS": "${{ github.event.inputs.param }}"}'
 
-          response=$(curl -w "%{http_code}" -H "Authorization: Bearer $BUILDKITE_TOKEN" \
-            -X POST "$BUILDKITE_API_URL" \
-            -d "$data")
-
-          http_code=$(echo "$response" | tail -n 1)
-          response_body=$(echo "$response" | head -n -1)
-
-          if [ "$http_code" != "201" ]; then
-            echo "Error: Buildkite API returned HTTP status code $http_code"
-            echo "Response body:"
-            echo "$response_body"
-            exit 1
-          fi
-
-          echo "Build triggered successfully (HTTP status code: $http_code)"
+      - name: Trigger Full Smoke Tests
+        uses: buildkite/trigger-pipeline-action@v2.3.0
+        if: github.event_name != 'workflow_dispatch'
+        with:
+          buildkite_api_access_token: ${{ secrets.BUILDKITE_TOKEN }}
+          pipeline: "skypilot-1/full-smoke-tests-run"
+          branch: "master"
+          commit: "HEAD"
+          message: "Biweekly Smoke Tests"
+          ignore_pipeline_branch_filter: true


Reviews:
user: login: zpoint
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-07T08:09:23Z
user: login: Michaelvll
body: Thanks @zpoint! It looks mostly good look to me.
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-07T16:59:18Z
user: login: zpoint
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-08T04:17:04Z
user: login: Michaelvll
body: Thanks @zpoint! Please see the race condition below.
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-09T02:11:25Z
user: login: zpoint
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-09T02:47:42Z
user: login: Michaelvll
body: Nice @zpoint! Thanks for adding the trigger. Let's get it in today and make our nightly build robust!
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-09T04:34:19Z

Review comments:
diff_hunk: @@ -0,0 +1,152 @@
+name: test-pypi-publish-nightly
path: .github/workflows/test-pypi-nightly-build.yml
user: login: zpoint
body: This file is for testing the GitHub Action without triggering the actual PyPI publish on my personal repo. I'll remove this file after the PR is approved.
created_at: 2025-04-07T08:09:16Z
author_association: COLLABORATOR
diff_hunk: @@ -52,15 +52,104 @@ jobs:
           --wheel
           --outdir dist/
           .
+
+      # Trigger Buildkite smoke tests
+      - name: Trigger Buildkite Smoke Tests
+        id: trigger_buildkite
+        env:
+          BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }}
+        run: |
+          data='{
+            "commit": "${{ github.sha }}",
+            "branch": "master",
+            "message": "nightly-build-pypi",
+            "ignore_pipeline_branch_filters": true,
+            "env": {
+              "ARGS": "--aws"
+            }
+          }'
+
+          response=$(curl -w "%{http_code}" -H "Authorization: Bearer $BUILDKITE_TOKEN" \
+            -X POST "https://api.buildkite.com/v2/organizations/skypilot-1/pipelines/smoke-tests/builds" \
+            -d "$data")
+
+          http_code=$(echo "$response" | tail -n 1)
+          response_body=$(echo "$response" | head -n -1)
+
+          if [ "$http_code" != "201" ]; then
+            echo "Error: Buildkite API returned HTTP status code $http_code"
+            echo "Response body:"
+            echo "$response_body"
+            exit 1
+          fi
+
+          build_url=$(echo "$response_body" | jq -r '.web_url')
+          build_number=$(echo "$response_body" | jq -r '.number')
+
+          echo "Build triggered successfully (HTTP status code: $http_code)"
+          echo "Build URL: $build_url"
+          echo "::set-output name=build_url::$build_url"
+          echo "::set-output name=build_number::$build_number"
path: .github/workflows/pypi-nightly-build.yml
user: login: Michaelvll
body: Can we just use the existing GH action we have? Check here for calling other workflows: https://github.com/skypilot-org/skypilot/blob/c5039370280815a3f347e76622dc154ede36d6c3/.github/workflows/helm-docker-release.yaml#L27-L34
created_at: 2025-04-07T16:54:14Z
author_association: COLLABORATOR
diff_hunk: @@ -52,15 +52,104 @@ jobs:
           --wheel
           --outdir dist/
           .
+
+      # Trigger Buildkite smoke tests
+      - name: Trigger Buildkite Smoke Tests
+        id: trigger_buildkite
+        env:
+          BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }}
+        run: |
+          data='{
+            "commit": "${{ github.sha }}",
+            "branch": "master",
+            "message": "nightly-build-pypi",
+            "ignore_pipeline_branch_filters": true,
+            "env": {
+              "ARGS": "--aws"
+            }
+          }'
+
+          response=$(curl -w "%{http_code}" -H "Authorization: Bearer $BUILDKITE_TOKEN" \
+            -X POST "https://api.buildkite.com/v2/organizations/skypilot-1/pipelines/smoke-tests/builds" \
+            -d "$data")
+
+          http_code=$(echo "$response" | tail -n 1)
+          response_body=$(echo "$response" | head -n -1)
+
+          if [ "$http_code" != "201" ]; then
+            echo "Error: Buildkite API returned HTTP status code $http_code"
+            echo "Response body:"
+            echo "$response_body"
+            exit 1
+          fi
+
+          build_url=$(echo "$response_body" | jq -r '.web_url')
+          build_number=$(echo "$response_body" | jq -r '.number')
+
+          echo "Build triggered successfully (HTTP status code: $http_code)"
+          echo "Build URL: $build_url"
+          echo "::set-output name=build_url::$build_url"
+          echo "::set-output name=build_number::$build_number"
+
+      # Wait for Buildkite build to complete (2 hour timeout)
+      - name: Wait for Buildkite build completion
+        id: wait_buildkite
+        env:
+          BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }}
+        run: |
+          build_number=${{ steps.trigger_buildkite.outputs.build_number }}
+          max_attempts=240  # 2 hours with 30-second intervals (240 attempts = 120 minutes)
+          attempt=0
+
+          while [ $attempt -lt $max_attempts ]; do
+            response=$(curl -s -H "Authorization: Bearer $BUILDKITE_TOKEN" \
+              "https://api.buildkite.com/v2/organizations/skypilot-1/pipelines/smoke-tests/builds/$build_number")
+
+            # Check if response contains only an error message
+            if [ $(echo "$response" | jq 'has("message") and length == 1') == "true" ]; then
+              error_message=$(echo "$response" | jq -r '.message')
+              echo "Error from Buildkite API: $error_message"
+              echo "::set-output name=build_status::failure"
+              exit 1
+            fi
+
+            status=$(echo "$response" | jq -r '.state')
+
+            case $status in
+              "passed")
+                echo "Buildkite build succeeded"
+                echo "::set-output name=build_status::success"
+                exit 0
+                ;;
+              "failed"|"canceled"|"blocked")
+                echo "Buildkite build failed with status: $status"
+                echo "::set-output name=build_status::failure"
+                exit 1
+                ;;
+              *)
+                echo "Buildkite build status: $status - waiting..."
+                sleep 30
+                attempt=$((attempt+1))
+                ;;
+            esac
+          done
+
+          echo "Timeout waiting for Buildkite build to complete (2 hour limit reached)"
+          echo "::set-output name=build_status::timeout"
+          exit 1
+
+      # Publish to PyPI only if Buildkite build succeeded
       - name: Publish distribution to Test PyPI
+        if: steps.wait_buildkite.outputs.build_status == 'success'
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           password: ${{ secrets.TEST_PYPI_API_TOKEN }}
           repository-url: https://test.pypi.org/legacy/
           skip-existing: true
+
       - name: Publish distribution to PyPI
+        if: steps.wait_buildkite.outputs.build_status == 'success'
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           password: ${{ secrets.PYPI_API_TOKEN }}
           skip-existing: true
-
path: .github/workflows/pypi-nightly-build.yml
user: login: Michaelvll
body: nit: EOF new line
created_at: 2025-04-07T16:55:11Z
author_association: COLLABORATOR
diff_hunk: @@ -52,15 +52,104 @@ jobs:
           --wheel
           --outdir dist/
           .
+
+      # Trigger Buildkite smoke tests
+      - name: Trigger Buildkite Smoke Tests
path: .github/workflows/pypi-nightly-build.yml
user: login: Michaelvll
body: Have we checked this? https://buildkite.com/docs/pipelines/tutorials/github-actions
created_at: 2025-04-07T16:57:20Z
author_association: COLLABORATOR
diff_hunk: @@ -52,15 +52,104 @@ jobs:
           --wheel
           --outdir dist/
           .
+
+      # Trigger Buildkite smoke tests
+      - name: Trigger Buildkite Smoke Tests
path: .github/workflows/pypi-nightly-build.yml
user: login: zpoint
body: Good catch, I will use this instead
created_at: 2025-04-08T04:17:04Z
author_association: COLLABORATOR
diff_hunk: @@ -52,12 +54,67 @@ jobs:
           --wheel
           --outdir dist/
           .
+
+      # Trigger Buildkite smoke tests
+      - name: Trigger Buildkite Smoke Tests
+        id: trigger_buildkite
+        uses: buildkite/trigger-pipeline-action@v2.3.0
+        with:
+          buildkite_api_access_token: ${{ secrets.BUILDKITE_TOKEN }}
+          pipeline: "skypilot-1/smoke-tests"
+          branch: "master"
+          commit: "${{ github.sha }}"
+          message: "nightly-build-pypi"
+          ignore_pipeline_branch_filter: true
+          build_env_vars: '{"ARGS": "--aws"}'
+
+      # Extract build number from response JSON
+      - name: Extract build number
+        id: extract_build_number
+        run: |
+          build_number=$(echo '${{ steps.trigger_buildkite.outputs.json }}' | jq -r '.number')
+          echo "build_number=$build_number" >> $GITHUB_OUTPUT
+
+  wait-for-buildkite:
+    needs: nightly-build-pypi
+    uses: ./.github/workflows/wait-for-buildkite.yml
+    with:
+      organization: "skypilot-1"
+      pipeline: "smoke-tests"
+      build_number: ${{ needs.nightly-build-pypi.outputs.build_number }}
+      timeout_minutes: 120
+    secrets:
+      BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }}
+
+  publish-to-pypi:
+    needs: wait-for-buildkite
+    if: needs.wait-for-buildkite.outputs.build_status == 'success'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v3
path: .github/workflows/pypi-nightly-build.yml
user: login: Michaelvll
body: Cloning again will cause race condition with other commits merged during the test, can we use the same commit used for testing?
created_at: 2025-04-09T02:06:59Z
author_association: COLLABORATOR
diff_hunk: @@ -52,12 +54,67 @@ jobs:
           --wheel
           --outdir dist/
           .
+
+      # Trigger Buildkite smoke tests
+      - name: Trigger Buildkite Smoke Tests
+        id: trigger_buildkite
+        uses: buildkite/trigger-pipeline-action@v2.3.0
+        with:
+          buildkite_api_access_token: ${{ secrets.BUILDKITE_TOKEN }}
+          pipeline: "skypilot-1/smoke-tests"
+          branch: "master"
+          commit: "${{ github.sha }}"
+          message: "nightly-build-pypi"
+          ignore_pipeline_branch_filter: true
+          build_env_vars: '{"ARGS": "--aws"}'
+
+      # Extract build number from response JSON
+      - name: Extract build number
+        id: extract_build_number
+        run: |
+          build_number=$(echo '${{ steps.trigger_buildkite.outputs.json }}' | jq -r '.number')
+          echo "build_number=$build_number" >> $GITHUB_OUTPUT
+
+  wait-for-buildkite:
+    needs: nightly-build-pypi
+    uses: ./.github/workflows/wait-for-buildkite.yml
+    with:
+      organization: "skypilot-1"
+      pipeline: "smoke-tests"
+      build_number: ${{ needs.nightly-build-pypi.outputs.build_number }}
+      timeout_minutes: 120
+    secrets:
+      BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }}
+
+  publish-to-pypi:
+    needs: wait-for-buildkite
+    if: needs.wait-for-buildkite.outputs.build_status == 'success'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v3
path: .github/workflows/pypi-nightly-build.yml
user: login: zpoint
body: Use upload and download now instead of git clone twice, tested [here](https://github.com/zpoint/skypilot/actions/runs/14347636802)
created_at: 2025-04-09T02:47:42Z
author_association: COLLABORATOR

PR comments:
user: login: zpoint
created_at: 2025-04-09T06:12:18Z
body: @Michaelvll merged, please make sure your buildkite token have `read builds` access
author_association: COLLABORATOR
user: login: Michaelvll
created_at: 2025-04-09T07:06:08Z
body: Added the `read_builds` access. Please let me know if there is any other permissions needed. Also, please help monitor today's nightly action : )
author_association: COLLABORATOR