Repository: nebari-dev/nebari (language: python)

Issue:
Add nebari upgrade basic test workflow
From an offline convo I had with @dcmcand:

- We want the workflow to be triggered by pre-releases or optionally manually
- We want to use local provider

The steps should be

1. Deploy the most recent released version of nebari
2. Perform a health check (`curl` some endpoints for now)
3. Run `nebari upgrade` for the version that triggered the workflow
4. Perform the health check again

Depending on the scope of the health checks, which we want to keep minimal for the initial version of this test, this might or might not detect whether the upgrade does not break anything.

In any case, it will not be able to prevent the data loss described in the top comment that happened because a resource was deleted and recreated. To test that we would need to add some state in between step 1. and 2. above and verify its integrity after step 4.

_Originally posted by @pmeier in https://github.com/nebari-dev/nebari/issues/2701#issuecomment-2419405929_
            


Discussion / hints:

Fix patch:
diff --git a/.github/actions/health-check/action.yml b/.github/actions/health-check/action.yml
new file mode 100644
index 000000000..14602b3d7
--- /dev/null
+++ b/.github/actions/health-check/action.yml
@@ -0,0 +1,19 @@
+name: health-check
+description: "Check health of Nebari deployment"
+
+inputs:
+  domain:
+    description: Domain name
+    required: true
+
+runs:
+  using: composite
+
+  steps:
+    - name: List kubernetes components
+      shell: bash
+      run: kubectl get --all-namespaces all,cm,secret,pv,pvc,ing
+
+    - name: Check if JupyterHub login page is accessible
+      shell: bash
+      run: curl --insecure --include 'https://${{ inputs.domain }}/hub/home'
diff --git a/.github/actions/init-local/action.yml b/.github/actions/init-local/action.yml
new file mode 100644
index 000000000..306876973
--- /dev/null
+++ b/.github/actions/init-local/action.yml
@@ -0,0 +1,81 @@
+name: init-local
+description: "Initialize Nebari config for local deployment"
+
+inputs:
+  directory:
+    description: "Path to directory to initialize in"
+    required: false
+    default: './local-deployment'
+
+outputs:
+  directory:
+    description: "Path to config directory"
+    value: ${{ steps.metadata.outputs.directory }}
+  config:
+    description: "Path to Nebari config"
+    value: ${{ steps.metadata.outputs.config }}
+  project:
+    description: "Project name"
+    value: ${{ steps.metadata.outputs.project }}
+  domain:
+    description: "Domain name"
+    value: ${{ steps.metadata.outputs.domain }}
+
+runs:
+  using: composite
+
+  steps:
+    - shell: bash
+      id: metadata
+      run: |
+        # Setup metadata
+        DIRECTORY=$(realpath '${{ inputs.directory }}')
+        mkdir --parents "${DIRECTORY}"
+        echo "directory=${DIRECTORY}" | tee --append "${GITHUB_OUTPUT}"
+
+        CONFIG="${DIRECTORY}/nebari-config.yaml"
+        echo "config=${CONFIG}" | tee --append "${GITHUB_OUTPUT}"
+
+        PROJECT='github-actions'
+        echo "project=${PROJECT}" | tee --append "${GITHUB_OUTPUT}"
+
+        DOMAIN='github-actions.nebari.dev'
+        nslookup "${DOMAIN}"
+        echo "domain=${DOMAIN}" | tee --append "${GITHUB_OUTPUT}"
+
+    - shell: bash -l {0}
+      id: init
+      working-directory: ${{ steps.metadata.outputs.directory }}
+      run: |
+        nebari init local \
+          --project-name '${{ steps.metadata.outputs.project }}' \
+          --domain-name '${{ steps.metadata.outputs.domain }}' \
+          --auth-provider password \
+          --output '${{ steps.metadata.outputs.config }}'
+
+    - shell: bash
+      run: |
+        # Update nebari config for CI
+
+        # Change default JupyterLab theme
+        cat >> '${{ steps.metadata.outputs.config }}' <<- EOM
+        jupyterlab:
+          default_settings:
+            "@jupyterlab/apputils-extension:themes":
+              theme: JupyterLab Dark
+        EOM
+
+        # Change default value for minio persistence size
+        cat >> '${{ steps.metadata.outputs.config }}' <<- EOM
+        monitoring:
+          enabled: true
+          overrides:
+            minio:
+              persistence:
+                size: 1Gi
+        EOM
+
+    - shell: bash
+      run: |
+        # Display Nebari config
+        cat '${{ steps.metadata.outputs.config }}'
diff --git a/.github/actions/setup-local/action.yml b/.github/actions/setup-local/action.yml
new file mode 100644
index 000000000..87a12f45d
--- /dev/null
+++ b/.github/actions/setup-local/action.yml
@@ -0,0 +1,30 @@
+name: setup-local
+description: "Setup runner for local deployment"
+
+inputs:
+  kubectl-version:
+    description: "Version of kubectl to install"
+    required: false
+    default: "1.19.16"
+
+runs:
+  using: composite
+
+  steps:
+    - uses: azure/setup-kubectl@v4
+      with:
+        version: v${{ inputs.kubectl-version }}
+
+    - shell: bash
+      run: |
+        # Enable docker permissions for user
+        sudo docker ps
+        sudo usermod -aG docker $USER && newgrp docker
+
+        docker info
+        docker ps
+
+    - shell: bash
+      run: |
+        # Get routing table for docker pods
+        ip route


Test patch:
diff --git a/.github/workflows/test_local_integration.yaml b/.github/workflows/test_local_integration.yaml
index ea547ede8..64509142c 100644
--- a/.github/workflows/test_local_integration.yaml
+++ b/.github/workflows/test_local_integration.yaml
@@ -53,6 +53,9 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: Setup runner for local deployment
+        uses: ./.github/actions/setup-local
+
       - name: Checkout the branch from the PR that triggered the job
         if: ${{ github.event_name == 'issue_comment' }}
         run: |
@@ -75,82 +78,29 @@ jobs:
           pip install .[dev]
           playwright install
 
-      - uses: azure/setup-kubectl@v4.0.0
-        with:
-          version: v1.19.16
-
-      - name: Enable docker permissions for user
-        run: |
-          sudo docker ps
-          sudo usermod -aG docker $USER && newgrp docker
-
-          docker info
-          docker ps
-
-      - name: Get routing table for docker pods
-        run: |
-          ip route
-
-      - name: Initialize Nebari Cloud
-        run: |
-          mkdir -p local-deployment
-          cd local-deployment
-          nebari init local --project=thisisatest --domain github-actions.nebari.dev --auth-provider=password
-
-          # Need smaller profiles on Local Kind
-          sed -i -E 's/(cpu_guarantee):\s+[0-9\.]+/\1: 0.25/g' "nebari-config.yaml"
-          sed -i -E 's/(mem_guarantee):\s+[A-Za-z0-9\.]+/\1: 0.25G/g' "nebari-config.yaml"
-
-          # Change default JupyterLab theme
-          cat >> nebari-config.yaml <<- EOM
-          jupyterlab:
-            default_settings:
-              "@jupyterlab/apputils-extension:themes":
-                theme: JupyterLab Dark
-          EOM
-
-          # Change default value for minio persistence size
-          cat >> nebari-config.yaml <<- EOM
-          monitoring:
-            enabled: true
-            overrides:
-              minio:
-                persistence:
-                  size: 1Gi
-          EOM
-
-          cat nebari-config.yaml
+      - name: Initialize Nebari config for local deployment
+        id: init
+        uses: ./.github/actions/init-local
 
       - name: Deploy Nebari
-        working-directory: local-deployment
-        run: |
-          nebari deploy --config nebari-config.yaml --disable-prompt
+        working-directory: ${{ steps.init.outputs.directory }}
+        run: nebari deploy --config ${{ steps.init.outputs.config }} --disable-prompt
 
-      - name: Basic kubectl checks after deployment
-        if: always()
-        run: |
-          kubectl get all,cm,secret,pv,pvc,ing -A
-
-      - name: Check github-actions.nebari.dev resolves
-        run: |
-          nslookup github-actions.nebari.dev
-
-      - name: Curl jupyterhub login page
-        run: |
-          curl -k https://github-actions.nebari.dev/hub/home -i
+      - name: Health check
+        uses: ./.github/actions/health-check
+        with:
+          domain: ${{ steps.init.outputs.domain }}
 
       - name: Create example-user
-        working-directory: local-deployment
+        working-directory: ${{ steps.init.outputs.directory }}
         run: |
-          nebari keycloak adduser --user "${TEST_USERNAME}" "${TEST_PASSWORD}" --config nebari-config.yaml
-          nebari keycloak listusers --config nebari-config.yaml
-
-      - name: Get nebari-config.yaml full path
-        run: echo "NEBARI_CONFIG_PATH=`realpath ./local-deployment/nebari-config.yaml`" >> "$GITHUB_ENV"
+          nebari keycloak adduser --user "${TEST_USERNAME}" "${TEST_PASSWORD}" --config ${{ steps.init.outputs.config }}
+          nebari keycloak listusers --config ${{ steps.init.outputs.config }}
 
       ### DEPLOYMENT TESTS
       - name: Deployment Pytests
         env:
+          NEBARI_CONFIG_PATH: ${{ steps.init.outputs.config }}
           KEYCLOAK_USERNAME: ${{ env.TEST_USERNAME }}
           KEYCLOAK_PASSWORD: ${{ env.TEST_PASSWORD }}
         run: |
@@ -165,7 +115,7 @@ jobs:
         env:
           KEYCLOAK_USERNAME: ${{ env.TEST_USERNAME }}
           KEYCLOAK_PASSWORD: ${{ env.TEST_PASSWORD }}
-          NEBARI_FULL_URL: https://github-actions.nebari.dev/
+          NEBARI_FULL_URL: "https://${{ steps.init.outputs.domain }}/"
         working-directory: tests/tests_e2e/playwright
         run: |
           # create environment file
@@ -187,6 +137,5 @@ jobs:
         # in the CI, it makes sense to only run on merge to main or workflow_dispatch to speed
         # up feedback cycle
         if: github.ref_name == 'main' || github.event_name == 'workflow_dispatch'
-        working-directory: local-deployment
-        run: |
-          nebari destroy --config nebari-config.yaml --disable-prompt
+        working-directory: ${{ steps.init.outputs.directory }}
+        run: nebari destroy --config ${{ steps.init.outputs.config }} --disable-prompt
diff --git a/.github/workflows/test_local_upgrade.yaml b/.github/workflows/test_local_upgrade.yaml
new file mode 100644
index 000000000..06be364eb
--- /dev/null
+++ b/.github/workflows/test_local_upgrade.yaml
@@ -0,0 +1,85 @@
+name: "Local Upgrade Tests"
+
+on:
+  pull_request:
+    paths:
+      - ".github/actions/**"
+      - ".github/workflows/test_local_upgrade.yaml"
+  release:
+    types:
+      - prereleased
+  workflow_dispatch:
+
+
+concurrency:
+  group: ${{ github.ref_name }}
+  cancel-in-progress: true
+
+env:
+  NEBARI_IMAGE_TAG: "main"
+
+jobs:
+  test-local-upgrade:
+    runs-on: "cirun-runner--${{ github.run_id }}"
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup runner for local deployment
+        uses: ./.github/actions/setup-local
+
+      - name: Setup Python
+        uses: conda-incubator/setup-miniconda@v3
+        env:
+          CONDA: /home/runnerx/miniconda3
+        with:
+          auto-update-conda: true
+          python-version: "3.11"
+          miniconda-version: "latest"
+
+      - name: Install latest stable Nebari release
+        run: pip install nebari
+
+      - name: Initialize Nebari config for local deployment
+        id: init
+        uses: ./.github/actions/init-local
+
+      - name: Extract old Nebari version
+        run: |
+          OLD_NEBARI_VERSION=$(grep 'nebari_version: ' ${{ steps.init.outputs.config }} | sed 's/nebari_version: //')
+          echo "OLD_NEBARI_VERSION=${OLD_NEBARI_VERSION}" | tee --append "${GITHUB_ENV}"
+
+      - name: Deploy Nebari
+        working-directory: ${{ steps.init.outputs.directory }}
+        run: nebari deploy --config ${{ steps.init.outputs.config }} --disable-prompt
+
+      - name: Health check before upgrade
+        id: health-check-before
+        uses: ./.github/actions/health-check
+        with:
+          domain: ${{ steps.init.outputs.domain }}
+
+      - name: Install current Nebari
+        run: pip install --upgrade .
+
+      - name: Upgrade Nebari config
+        run: |
+          git add --force ${{ steps.init.outputs.config }}
+          nebari upgrade --config ${{ steps.init.outputs.config }} --attempt-fixes
+          git diff
+          nebari validate --config ${{ steps.init.outputs.config }}
+
+      - name: Redeploy Nebari
+        working-directory: ${{ steps.init.outputs.directory }}
+        run: nebari deploy --config ${{ steps.init.outputs.config }} --disable-prompt
+
+      - name: Health check after upgrade
+        id: health-check-after
+        uses: ./.github/actions/health-check
+        with:
+          domain: ${{ steps.init.outputs.domain }}