Repository: substratusai/kubeai (go)

Issue:
Remove request body logging
https://github.com/substratusai/kubeai/blob/884aa62c81e2f5575a92c60a62f6e8734418a17b/internal/modelproxy/request.go#L131

This becomes especially problematic with binary requests (Speech to Text).

Pull request description:
Refactor e2e tests
* Add central bash script to run e2e test cases
* Separate e2e tests out into separate cases
* Add python openai client test case
* Fix #262 

Patch:
diff --git a/.gitignore b/.gitignore
index 3b22c52..3a5d940 100644
--- a/.gitignore
+++ b/.gitignore
@@ -37,3 +37,5 @@ Chart.lock
 .venv
 *__pycache__
 site
+
+/tmp
diff --git a/Makefile b/Makefile
index 5403d90..301b0b3 100644
--- a/Makefile
+++ b/Makefile
@@ -3,6 +3,7 @@ IMG ?= controller:latest
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION = 1.30.0
 CRD_REF_DOCS_VERSION = v0.1.0
+SKAFFOLD_VERSION = v2.13.2
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -66,14 +67,25 @@ fmt: ## Run go fmt against code.
 vet: ## Run go vet against code.
 	go vet ./...
 
-.PHONY: test
-test: manifests generate fmt vet envtest ## Run tests.
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -v $$(go list ./... | grep -v /e2e) -coverprofile cover.out
+.PHONY: test-unit
+test-unit: fmt vet
+	go test -v ./internal/... -coverprofile cover.unit.out
 
-# Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors.
-.PHONY: test-e2e  # Run the e2e tests against a Kind k8s instance that is spun up.
-test-e2e:
-	go test ./test/e2e/ -v -ginkgo.v
+.PHONY: test-integration
+test-integration: fmt vet envtest
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -v ./test/integration -coverprofile cover.integration.out
+
+.PHONY: test-e2e-quickstart
+test-e2e-quickstart: skaffold
+	./test/e2e/run.sh quickstart
+
+.PHONY: test-e2e-faster-whisper
+test-e2e-faster-whisper: skaffold
+	./test/e2e/run.sh faster-whisper --profile kubeai-only
+
+.PHONY: test-e2e-openai-python-client
+test-e2e-openai-python-client: skaffold
+	./test/e2e/run.sh openai-python-client --profile kubeai-only
 
 .PHONY: lint
 lint: golangci-lint ## Run golangci-lint linter
@@ -141,6 +153,7 @@ CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
 ENVTEST ?= $(LOCALBIN)/setup-envtest
 GOLANGCI_LINT = $(LOCALBIN)/golangci-lint
 CRD_REF_DOCS ?= $(LOCALBIN)/crd-ref-docs
+SKAFFOLD ?= $(LOCALBIN)/skaffold
 YQ ?= $(LOCALBIN)/yq
 
 ## Tool Versions
@@ -198,8 +211,14 @@ generate-kubernetes-api-reference: crd-ref-docs
 .PHONY: crd-ref-docs
 crd-ref-docs: $(CRD_REF_DOCS) ## Download crd-ref-docs.
 $(CRD_REF_DOCS): $(LOCALBIN)
-	test -s $(LOCALBIN)/crd-ref-docs || \
-	GOBIN=$(LOCALBIN) go install github.com/elastic/crd-ref-docs@$(CRD_REF_DOCS_VERSION)
+	$(call go-install-tool,$(CRD_REF_DOCS),github.com/elastic/crd-ref-docs,$(CRD_REF_DOCS_VERSION))
+
+SKAFFOLD_PLATFORM ?= $(shell go env GOOS)-$(shell go env GOARCH)
+.PHONY: skaffold
+skaffold: $(SKAFFOLD) ## Download skaffold.
+$(SKAFFOLD): $(LOCALBIN)
+	curl -Lo $(SKAFFOLD) https://storage.googleapis.com/skaffold/releases/$(SKAFFOLD_VERSION)/skaffold-$(SKAFFOLD_PLATFORM)
+	chmod +x $(SKAFFOLD)
 
 # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist
 # $1 - target path with name of binary
diff --git a/charts/kubeai/values-gke.yaml b/charts/kubeai/values-gke.yaml
index bad91d2..10d1e37 100644
--- a/charts/kubeai/values-gke.yaml
+++ b/charts/kubeai/values-gke.yaml
@@ -7,6 +7,14 @@ resourceProfiles:
     nodeSelector:
       cloud.google.com/gke-accelerator: "nvidia-h100-80gb"
       cloud.google.com/gke-spot: "true"
+  nvidia-gpu-a100-80gb:
+    nodeSelector:
+      cloud.google.com/gke-accelerator: "nvidia-a100-80gb"
+      cloud.google.com/gke-spot: "true"
+  nvidia-gpu-a100-40gb:
+    nodeSelector:
+      cloud.google.com/gke-accelerator: "nvidia-tesla-a100"
+      cloud.google.com/gke-spot: "true"
   google-tpu-v5e-1x1:
     imageName: google-tpu
     limits:
diff --git a/charts/kubeai/values.yaml b/charts/kubeai/values.yaml
index 62001ac..2e72f73 100644
--- a/charts/kubeai/values.yaml
+++ b/charts/kubeai/values.yaml
@@ -81,17 +81,24 @@ resourceProfiles:
         operator: "Equal"
         value: "present"
         effect: "NoSchedule"
-
-  # nvidia-gpu-v100:
-  # nvidia-gpu-a100:
-  # google-tpu-v5e:
-  #   imageName: "google-tpu"
-  #   requests:
-  #     google.com/tpu: 4
-  #   limits:
-  #     google.com/tpu: 4
-  #   nodeSelector:
-  #     cloud.google.com/gke-accelerator-type: tpu-v5-lite-podslice
+  nvidia-gpu-a100-80gb:
+    imageName: "nvidia-gpu"
+    limits:
+      nvidia.com/gpu: "1"
+    tolerations:
+      - key: "nvidia.com/gpu"
+        operator: "Equal"
+        value: "present"
+        effect: "NoSchedule"
+  nvidia-gpu-a100-40gb:
+    imageName: "nvidia-gpu"
+    limits:
+      nvidia.com/gpu: "1"
+    tolerations:
+      - key: "nvidia.com/gpu"
+        operator: "Equal"
+        value: "present"
+        effect: "NoSchedule"
 
 modelAutoscaling:
   # Interval that the autoscaler will scrape model server metrics.
diff --git a/charts/models/values.yaml b/charts/models/values.yaml
index 97df836..cf6c477 100644
--- a/charts/models/values.yaml
+++ b/charts/models/values.yaml
@@ -161,6 +161,28 @@ catalog:
     # You can also use nvidia-gpu-a100-80gb:8
     resourceProfile: nvidia-gpu-h100:8
     targetRequests: 500
+  llama-3.1-405b-instruct-fp8-a100-80b:
+    features: [TextGeneration]
+    url: hf://neuralmagic/Meta-Llama-3.1-405B-Instruct-FP8
+    engine: VLLM
+    env:
+      VLLM_ATTENTION_BACKEND: FLASHINFER
+    args:
+      - --max-model-len=65536
+      - --max-num-batched-token=65536
+      - --gpu-memory-utilization=0.98
+      - --tensor-parallel-size=8
+      - --enable-prefix-caching
+      - --disable-log-requests
+      - --max-num-seqs=128
+      - --kv-cache-dtype=fp8
+      - --enforce-eager
+      - --enable-chunked-prefill=false
+      - --num-scheduler-steps=8
+    targetRequests: 128
+    minReplicas: 1
+    maxReplicas: 1
+    resourceProfile: nvidia-gpu-a100-80gb:8
   nomic-embed-text-cpu:
     enabled: false
     features: ["TextEmbedding"]
diff --git a/internal/modelproxy/request.go b/internal/modelproxy/request.go
index 54fb799..f47152a 100644
--- a/internal/modelproxy/request.go
+++ b/internal/modelproxy/request.go
@@ -128,7 +128,6 @@ func (pr *proxyRequest) parseModel() error {
 			return fmt.Errorf("closing multipart writer: %w", err)
 		}
 		pr.body = buf.Bytes()
-		fmt.Println("pr.body:", string(pr.body))
 		// Set a new content length based on the new body - which had the "model" field removed.
 		pr.r.ContentLength = int64(len(pr.body))
 
diff --git a/manifests/models/llama-3.1-405b-instruct-fp8-a100-80b.yaml b/manifests/models/llama-3.1-405b-instruct-fp8-a100-80b.yaml
new file mode 100644
index 0000000..da2940e
--- /dev/null
+++ b/manifests/models/llama-3.1-405b-instruct-fp8-a100-80b.yaml
@@ -0,0 +1,28 @@
+# Source: models/templates/models.yaml
+apiVersion: kubeai.org/v1
+kind: Model
+metadata:
+  name: llama-3.1-405b-instruct-fp8-a100-80b
+spec:
+  features: [TextGeneration]
+  owner:
+  url: hf://neuralmagic/Meta-Llama-3.1-405B-Instruct-FP8
+  engine: VLLM
+  args:
+    - --max-model-len=65536
+    - --max-num-batched-token=65536
+    - --gpu-memory-utilization=0.98
+    - --tensor-parallel-size=8
+    - --enable-prefix-caching
+    - --disable-log-requests
+    - --max-num-seqs=128
+    - --kv-cache-dtype=fp8
+    - --enforce-eager
+    - --enable-chunked-prefill=false
+    - --num-scheduler-steps=8
+  env:
+    VLLM_ATTENTION_BACKEND: FLASHINFER
+  minReplicas: 1
+  maxReplicas: 1
+  targetRequests: 128
+  resourceProfile: nvidia-gpu-a100-80gb:8
diff --git a/skaffold-build.json b/skaffold-build.json
new file mode 100644
index 0000000..d558a6b
--- /dev/null
+++ b/skaffold-build.json
@@ -0,0 +1,1 @@
+{"builds":null}
\ No newline at end of file
diff --git a/skaffold-tags.json b/skaffold-tags.json
new file mode 100644
index 0000000..d558a6b
--- /dev/null
+++ b/skaffold-tags.json
@@ -0,0 +1,1 @@
+{"builds":null}
\ No newline at end of file
diff --git a/skaffold.yaml b/skaffold.yaml
index 4ada207..419657f 100644
--- a/skaffold.yaml
+++ b/skaffold.yaml
@@ -5,20 +5,38 @@ metadata:
 build:
   artifacts:
   - image: substratusai/kubeai
+    # NOTE: If you run into this error:
+    #
+    # Checking cache...
+    #  - substratusai/kubeai: Error checking cache.
+    # getting hash for artifact "substratusai/kubeai": getting dependencies for "substratusai/kubeai": parsing ONBUILD instructions: retrieving image "gcr.io/distroless/static:nonroot": GET https://gcr.io/v2/token?scope=repository%3Adistroless%2Fstatic%3Apull&service=gcr.io: UNAUTHORIZED: authentication failed
+    #
+    # Then run "docker pull gcr.io/distroless/static:nonroot" and retry.
+    # Disabling cache for the image that uses distroless/static:nonroot does not appear to work:
+    #
+    # docker:
+    #   noCache: true
+    #
   local:
     push: false
 deploy:
   helm:
     releases:
     - name: kubeai
-      chartPath: charts/kubeai
-      valuesFiles:
-      - charts/kubeai/values.yaml
-      setValueTemplates:
-        image.tag: "{{.DIGEST_HEX}}"
+      chartPath: ./charts/kubeai
 portForward:
 - resourceType: service
   resourceName: kubeai
   namespace: default
   port: 80
-  localPort: 8000
\ No newline at end of file
+  localPort: 8000
+
+profiles:
+- name: kubeai-only
+  deploy:
+    helm:
+      releases:
+      - name: kubeai
+        chartPath: ./charts/kubeai
+        setValueTemplates:
+          openwebui.enabled: false


Test patch:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 5e7eeee..646e14a 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -8,38 +8,44 @@ on:
   pull_request:
 
 jobs:
-  unit-integration:
+  unit-and-integration:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
 
-      - name: Setup Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: ">=1.21.0"
+      - name: Run unit tests
+        run: make test-unit
 
-      - name: Run unit & integration tests
-        run: make test
-  e2e-quickstart:
+      - name: Run integration tests
+        run: make test-integration
+
+  e2e:
     runs-on: ubuntu-latest
+    # NOTE: Uncomment if we start getting limited on number of concurrent jobs
+    # (due to rapid pushes, etc).
+    #needs: unit-and-integration # No use in running e2e tests if integration tests fail.
+    strategy:
+      matrix:
+        testcase: ["quickstart", "openai-python-client", "faster-whisper"]
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
+
       - name: Install kind
         run: |
           curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.24.0/kind-linux-amd64
           chmod +x ./kind
           sudo mv ./kind /usr/local/bin/kind
+
       - name: Install helm
         run: |
           curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
           chmod 700 get_helm.sh
           ./get_helm.sh
-      - name: Install skaffold
-        run: |
-          curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-amd64
-          chmod +x skaffold
-          sudo mv skaffold /usr/local/bin
-      - name: Run the quickstart tests
-        run: test/quickstart.sh
+
+      - name: Start kind cluster
+        run: kind create cluster
+
+      - name: Run the e2e testcase
+        run: make test-e2e-${{ matrix.testcase }}
\ No newline at end of file
diff --git a/test/e2e/common.sh b/test/e2e/common.sh
new file mode 100644
index 0000000..49adacd
--- /dev/null
+++ b/test/e2e/common.sh
@@ -0,0 +1,23 @@
+set -e
+
+retry() {
+  local retries=$1
+  shift
+  local count=0
+
+  set +x
+  until "$@"; do
+    exit_code=$?
+    count=$((count + 1))
+    if [ "$count" -lt "$retries" ]; then
+      echo -n "."
+      sleep 1  # Optional delay between retries
+    else
+      echo "Command failed after $count attempts."
+      set -x
+      return $exit_code
+    fi
+  done || true  # Prevent 'set -e' from exiting on failed command
+  echo ""
+  set -x
+}
\ No newline at end of file
diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go
deleted file mode 100644
index 1e527cd..0000000
--- a/test/e2e/e2e_suite_test.go
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
-Copyright 2024.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package e2e
-
-import (
-	"fmt"
-	"testing"
-
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
-)
-
-// Run e2e tests using the Ginkgo runner.
-func TestE2E(t *testing.T) {
-	RegisterFailHandler(Fail)
-	_, _ = fmt.Fprintf(GinkgoWriter, "Starting kubeai suite\n")
-	RunSpecs(t, "e2e suite")
-}
diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go
deleted file mode 100644
index 7335b0b..0000000
--- a/test/e2e/e2e_test.go
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
-Copyright 2024.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package e2e
-
-import (
-	"fmt"
-	"os/exec"
-	"time"
-
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
-
-	"github.com/substratusai/kubeai/test/utils"
-)
-
-const namespace = "kubeai-system"
-
-var _ = Describe("controller", Ordered, func() {
-	BeforeAll(func() {
-		By("installing prometheus operator")
-		Expect(utils.InstallPrometheusOperator()).To(Succeed())
-
-		By("installing the cert-manager")
-		Expect(utils.InstallCertManager()).To(Succeed())
-
-		By("creating manager namespace")
-		cmd := exec.Command("kubectl", "create", "ns", namespace)
-		_, _ = utils.Run(cmd)
-	})
-
-	AfterAll(func() {
-		By("uninstalling the Prometheus manager bundle")
-		utils.UninstallPrometheusOperator()
-
-		By("uninstalling the cert-manager bundle")
-		utils.UninstallCertManager()
-
-		By("removing manager namespace")
-		cmd := exec.Command("kubectl", "delete", "ns", namespace)
-		_, _ = utils.Run(cmd)
-	})
-
-	Context("Operator", func() {
-		It("should run successfully", func() {
-			var controllerPodName string
-			var err error
-
-			// projectimage stores the name of the image used in the example
-			var projectimage = "example.com/kubeai:v0.0.1"
-
-			By("building the manager(Operator) image")
-			cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage))
-			_, err = utils.Run(cmd)
-			ExpectWithOffset(1, err).NotTo(HaveOccurred())
-
-			By("loading the the manager(Operator) image on Kind")
-			err = utils.LoadImageToKindClusterWithName(projectimage)
-			ExpectWithOffset(1, err).NotTo(HaveOccurred())
-
-			By("installing CRDs")
-			cmd = exec.Command("make", "install")
-			_, err = utils.Run(cmd)
-			ExpectWithOffset(1, err).NotTo(HaveOccurred())
-
-			By("deploying the controller-manager")
-			cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage))
-			_, err = utils.Run(cmd)
-			ExpectWithOffset(1, err).NotTo(HaveOccurred())
-
-			By("validating that the controller-manager pod is running as expected")
-			verifyControllerUp := func() error {
-				// Get pod name
-
-				cmd = exec.Command("kubectl", "get",
-					"pods", "-l", "control-plane=controller-manager",
-					"-o", "go-template={{ range .items }}"+
-						"{{ if not .metadata.deletionTimestamp }}"+
-						"{{ .metadata.name }}"+
-						"{{ \"\\n\" }}{{ end }}{{ end }}",
-					"-n", namespace,
-				)
-
-				podOutput, err := utils.Run(cmd)
-				ExpectWithOffset(2, err).NotTo(HaveOccurred())
-				podNames := utils.GetNonEmptyLines(string(podOutput))
-				if len(podNames) != 1 {
-					return fmt.Errorf("expect 1 controller pods running, but got %d", len(podNames))
-				}
-				controllerPodName = podNames[0]
-				ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("controller-manager"))
-
-				// Validate pod status
-				cmd = exec.Command("kubectl", "get",
-					"pods", controllerPodName, "-o", "jsonpath={.status.phase}",
-					"-n", namespace,
-				)
-				status, err := utils.Run(cmd)
-				ExpectWithOffset(2, err).NotTo(HaveOccurred())
-				if string(status) != "Running" {
-					return fmt.Errorf("controller pod in %s status", status)
-				}
-				return nil
-			}
-			EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed())
-
-		})
-	})
-})
diff --git a/test/e2e/faster-whisper/test.sh b/test/e2e/faster-whisper/test.sh
new file mode 100755
index 0000000..733e0bc
--- /dev/null
+++ b/test/e2e/faster-whisper/test.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+source $REPO_DIR/test/e2e/common.sh
+
+model=faster-whisper-medium-en-cpu
+
+cleanup() {
+    echo "Running faster-whisper test case cleanup..."
+    kubectl delete -f $REPO_DIR/manifests/models/$model.yaml
+}
+trap cleanup EXIT
+
+kubectl apply -f $REPO_DIR/manifests/models/$model.yaml
+
+audio_file=$TMP_DIR/kubeai.mp4
+curl -L -o $audio_file https://github.com/user-attachments/assets/711d1279-6af9-4c6c-a052-e59e7730b757
+
+transcription_file=$TMP_DIR/transcription.json
+curl http://localhost:8000/openai/v1/audio/transcriptions \
+  -F "file=@$audio_file" \
+  -F "language=en" \
+  -F "model=$model" > $transcription_file
+  
+result_contains_kubernetes=$(cat $transcription_file | jq '.text | ascii_downcase | contains("kubernetes")')
+if [ "$result_contains_kubernetes" = "true" ]; then
+  echo "The transcript contains 'kubernetes'."
+else
+  echo "The text does not contain 'kubernetes':"
+  cat $transcription_file
+  exit 1
+fi
diff --git a/test/e2e/openai-python-client/.gitignore b/test/e2e/openai-python-client/.gitignore
new file mode 100644
index 0000000..efa407c
--- /dev/null
+++ b/test/e2e/openai-python-client/.gitignore
@@ -0,0 +1,162 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
\ No newline at end of file
diff --git a/test/e2e/openai-python-client/requirements.txt b/test/e2e/openai-python-client/requirements.txt
new file mode 100644
index 0000000..a0ff474
--- /dev/null
+++ b/test/e2e/openai-python-client/requirements.txt
@@ -0,0 +1,2 @@
+pytest==8.3.3
+openai==1.51.0
diff --git a/test/e2e/openai-python-client/test.py b/test/e2e/openai-python-client/test.py
new file mode 100644
index 0000000..7eba1b9
--- /dev/null
+++ b/test/e2e/openai-python-client/test.py
@@ -0,0 +1,37 @@
+from openai import OpenAI
+import pytest
+
+base_url = "http://localhost:8000/openai/v1"
+model = "opt-125m-cpu"
+client = OpenAI(base_url=base_url, api_key="ignored-by-kubeai")
+
+
+def test_list_models():
+    response = client.models.list()
+    model_ids = [model.id for model in response.data]
+    assert model in model_ids
+
+
+## TODO: FIX: This test is failing b/c the model does not have a chat template:
+## E openai.BadRequestError: Error code: 400 - {'object': 'error', 'message': 'As of transformers v4.44, default chat template is no longer allowed, so you must provide a chat template if the tokenizer does not define one.', 'type': 'BadRequestError', 'param': None, 'code': 400}
+# def test_chat_completion():
+#    # Define the test parameters
+#    messages = [
+#        {"role": "system", "content": "You are a helpful assistant."},
+#        {"role": "user", "content": "Hello, how are you?"},
+#    ]
+#
+#    response = client.chat.completions.create(model=model, messages=messages)
+#
+#    # Assert that the response contains at least one "choices"
+#    assert len(response.choices) > 0
+
+
+def test_completion():
+    response = client.completions.create(
+        model=model, prompt="How are you?", max_tokens=50
+    )
+
+    print(response)
+    # Assert that the response contains at least one "choices"
+    assert len(response.choices) > 0
diff --git a/test/e2e/openai-python-client/test.sh b/test/e2e/openai-python-client/test.sh
new file mode 100755
index 0000000..db22b70
--- /dev/null
+++ b/test/e2e/openai-python-client/test.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+source $REPO_DIR/test/e2e/common.sh
+
+kubectl apply -f $REPO_DIR/manifests/models/opt-125m-cpu.yaml
+
+python -m venv $TEST_DIR/venv
+
+cleanup() {
+    echo "Running openai-python-client test case cleanup..."
+    kubectl delete -f $REPO_DIR/manifests/models/opt-125m-cpu.yaml
+    deactivate
+}
+trap cleanup EXIT
+
+source $TEST_DIR/venv/bin/activate
+
+which pip
+pip install -r $TEST_DIR/requirements.txt
+
+# Wait for models to sync.
+sleep 3
+
+pytest $TEST_DIR/test.py
diff --git a/test/e2e/quickstart/test.sh b/test/e2e/quickstart/test.sh
new file mode 100755
index 0000000..1a05af3
--- /dev/null
+++ b/test/e2e/quickstart/test.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+source $REPO_DIR/test/e2e/common.sh
+
+models_release="kubeai-models"
+
+cleanup() {
+    echo "Running quickstart cleanup..."
+    helm uninstall --ignore-not-found $models_release
+}
+trap cleanup EXIT
+
+helm install $models_release $REPO_DIR/charts/models -f - <<EOF
+catalog:
+  gemma2-2b-cpu:
+    enabled: true
+    minReplicas: 1
+  qwen2-500m-cpu:
+    enabled: true
+  nomic-embed-text-cpu:
+    enabled: true
+EOF
+
+curl http://localhost:8000/openai/v1/completions \
+  -H "Content-Type: application/json" \
+  -d '{"model": "gemma2-2b-cpu", "prompt": "Who was the first president of the United States?", "max_tokens": 40}'
diff --git a/test/e2e/run.sh b/test/e2e/run.sh
new file mode 100755
index 0000000..d3421e6
--- /dev/null
+++ b/test/e2e/run.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+# run.sh <testcase> [skaffold flags]
+
+testcase=$1
+shift # All additional arguments are passed to skaffold.
+skaffold_flags=$@
+
+export REPO_DIR=$(git rev-parse --show-toplevel)
+export TEST_DIR=$REPO_DIR/test/e2e/$testcase
+export TMP_DIR=$REPO_DIR/tmp
+export PATH=$REPO_DIR/bin:$PATH
+export DOCKER_BUILDKIT=1
+
+mkdir -p $REPO_DIR/tmp
+
+source $REPO_DIR/test/e2e/common.sh
+
+skaffold_log_file=$TMP_DIR/skaffold.log
+
+# Avoid using an unintended kubectl context.
+expected_kubectl_context=${TEST_KUBECTL_CONTEXT:-kind-kind}
+current_kubectl_context=$(kubectl config current-context)
+if [ "$current_kubectl_context" != "$expected_kubectl_context" ]; then
+    output "Current kubectl context is $current_kubectl_context, expected $expected_kubectl_context"
+    output "Set TEST_KUBECTL_CONTEXT to override the expected context."
+    exit 1
+fi
+
+# Function to handle errors
+error_handler() {
+    echo "Tests failed. Printing logs..."
+    if [ -f $skaffold_log_file ]; then
+        echo "--- Skaffold Logs ---"
+        cat $skaffold_log_file
+    fi
+    echo "--- Nodes ---"
+    kubectl get nodes -owide
+    echo "--- Pods ---"
+    kubectl get pods -owide
+    echo "--- Events ---"
+    kubectl get events
+    echo "--- Models ---"
+    kubectl get crds models.kubeai.org && kubectl get models -oyaml
+    echo "!!! FAIL !!!"
+    exit 1
+}
+trap 'error_handler' ERR
+
+skaffold_pid=9999999999999999
+cleanup() {
+    echo "Running test framework cleanup..."
+    if [ $skaffold_pid != 9999999999999999 ]; then
+        kill $skaffold_pid
+        wait $skaffold_pid
+    fi
+    skaffold delete -f $REPO_DIR/skaffold.yaml
+}
+trap cleanup EXIT
+
+# Assert that PATH is configured correctly.
+if [ "$(which skaffold)" != "$REPO_DIR/bin/skaffold" ]; then
+    echo "Tools are not set up correctly. Probably need to run via Makefile."
+    exit 1
+fi
+
+skaffold_build_file=$TMP_DIR/skaffold-build.json
+skaffold build -f $REPO_DIR/skaffold.yaml --file-output=$skaffold_build_file $skaffold_flags
+skaffold deploy -f $REPO_DIR/skaffold.yaml --tail --port-forward --load-images --build-artifacts=$skaffold_build_file $skaffold_flags > $skaffold_log_file &
+skaffold_pid=$!
+
+echo "Waiting for KubeAI API on localhost:8000"
+retry 600 curl -s http://localhost:8000/openai/v1/models
+echo "KubeAI API is ready"
+
+$REPO_DIR/test/e2e/$testcase/test.sh
+
+echo "!!! PASS !!!"
\ No newline at end of file
diff --git a/test/quickstart.sh b/test/quickstart.sh
deleted file mode 100755
index 6110a8e..0000000
--- a/test/quickstart.sh
+++ /dev/null
@@ -1,106 +0,0 @@
-#!/usr/bin/env bash
-
-set -xe
-
-KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME:-kubeai-tests}
-
-if kind get clusters | grep -q ${KIND_CLUSTER_NAME}; then
-  echo "Cluster ${KIND_CLUSTER_NAME} already exists.. reusing it"
-  else
-  kind create cluster --name ${KIND_CLUSTER_NAME}
-fi
-
-# Capture PID and run skaffold in background.
-skaffold run --tail --port-forward &
-skaffold_pid=$!
-
-error_handler() {
-  local exit_status=$?  # Capture the exit status of the last command
-  if [ $exit_status -ne 0 ]; then
-    echo "An error occurred. Exiting with status $exit_status. Leaving kind cluster intact for debugging"
-  elif [ "$TEST_CLEANUP" != "false" ]; then
-    echo "Exiting normally. Deleting kind cluster"
-    kind delete cluster --name=${KIND_CLUSTER_NAME}
-  fi
-  # Send exit signal to skaffold and wait for it to exit.
-  kill "$skaffold_pid"
-  wait "$skaffold_pid"
-}
-
-trap 'error_handler' ERR EXIT
-
-function wait_for_pod_ready() {
-  local label="$1"
-  local start_time=$SECONDS
-
-  while ! kubectl get pod -l "$label" | grep -q Running; do
-    sleep 5
-    if (( SECONDS - start_time >= 300 )); then
-      echo "Pods with label $label did not start in time."
-      kubectl describe pod -l "$label" || true
-      kubectl logs -l "$label" || true
-      exit 1
-    fi
-  done
-
-  kubectl wait --for=condition=ready pod -l "$label" --timeout=1200s
-  exit_code=$?
-  if [ $exit_code -ne 0 ]; then
-    kubectl describe pod -l "$label"
-    kubectl logs -l "$label"
-    echo "Pods with label $label did not get ready in time."
-    exit 1
-  fi
-}
-
-# wait for kubeai pod to be ready.
-wait_for_pod_ready app.kubernetes.io/name=kubeai
-
-kubectl get pods -w | awk '{print "[pods -w] " $0}' &
-kubectl get events -w | awk '{print "[events -w] " $0}' &
-
-# Ensure the model count is 0.
-curl -s -X GET "http://localhost:8000/openai/v1/models" | jq '. | length == 0'
-
-# By using the --reuse-values flag we can just append models to the previous install
-# while avoiding overriding the image that skaffold originally built and set in the
-# first install.
-helm install kubeai-models ./charts/models -f - <<EOF
-catalog:
-  gemma2-2b-cpu:
-    enabled: true
-    minReplicas: 1
-  qwen2-500m-cpu:
-    enabled: true
-  nomic-embed-text-cpu:
-    enabled: true
-  faster-whisper-medium-en-cpu:
-    enabled: true
-    minReplicas: 1
-EOF
-
-
-# wait for kubeai pod to be ready after helm upgrade.
-sleep 10
-wait_for_pod_ready app.kubernetes.io/name=kubeai
-wait_for_pod_ready model=gemma2-2b-cpu
-
-curl -s -X GET "http://localhost:8000/openai/v1/models" | jq '. | length == 4'
-
-curl http://localhost:8000/openai/v1/completions \
-  -H "Content-Type: application/json" \
-  -d '{"model": "gemma2-2b-cpu", "prompt": "Who was the first president of the United States?", "max_tokens": 40}'
-
-# Test the speech to text endpoint
-wait_for_pod_ready model=faster-whisper-medium-en-cpu
-curl -L -o kubeai.mp4 https://github.com/user-attachments/assets/711d1279-6af9-4c6c-a052-e59e7730b757
-result=$(curl http://localhost:8000/openai/v1/audio/transcriptions \
-  -F "file=@kubeai.mp4" \
-  -F "language=en" \
-  -F "model=faster-whisper-medium-en-cpu" | jq '.text | ascii_downcase | contains("kubernetes")')
-if [ "$result" = "true" ]; then
-  echo "The transcript contains 'kubernetes'."
-else
-  echo "The text does not contain 'kubernetes'."
-  exit 1
-fi