Repository: kserve/kserve

Issue:
No matches for kind \"HorizontalPodAutoscaler\" in version \"autoscaling/v2beta2\
/kind bug

**What steps did you take and what happened:**
Deploy kserve in raw mode on kubernetes 1.26 where autoscaling/v2beta2 is no longer available


**What did you expect to happen:**
Kserve should support v2 of the api



Discussion:

Resolving patch:
diff --git a/hack/python-sdk/update_release_version_helper.py b/hack/python-sdk/update_release_version_helper.py
--- a/hack/python-sdk/update_release_version_helper.py
+++ b/hack/python-sdk/update_release_version_helper.py
@@ -24,7 +24,6 @@
 toml_files = [
     "python/kserve/pyproject.toml",
     "python/aiffairness/pyproject.toml",
-    "python/aixexplainer/pyproject.toml",
     "python/alibiexplainer/pyproject.toml",
     "python/artexplainer/pyproject.toml",
     "python/custom_model/pyproject.toml",


Test patch:
diff --git a/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go b/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go
--- a/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go
+++ b/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go
@@ -32,7 +32,7 @@ import (
 	"github.com/onsi/gomega"
 	. "github.com/onsi/gomega"
 	appsv1 "k8s.io/api/apps/v1"
-	v2beta2 "k8s.io/api/autoscaling/v2beta2"
+	autoscalingv2 "k8s.io/api/autoscaling/v2"
 	v1 "k8s.io/api/core/v1"
 	"knative.dev/pkg/apis"
 	duckv1 "knative.dev/pkg/apis/duck/v1"
@@ -436,38 +436,38 @@ var _ = Describe("v1beta1 inference service controller", func() {
 			var maxReplicas int32 = 3
 			var cpuUtilization int32 = 75
 			var stabilizationWindowSeconds int32 = 0
-			selectPolicy := v2beta2.MaxPolicySelect
-			actualHPA := &v2beta2.HorizontalPodAutoscaler{}
+			selectPolicy := autoscalingv2.MaxChangePolicySelect
+			actualHPA := &autoscalingv2.HorizontalPodAutoscaler{}
 			predictorHPAKey := types.NamespacedName{Name: constants.PredictorServiceName(serviceKey.Name),
 				Namespace: serviceKey.Namespace}
 			Eventually(func() error { return k8sClient.Get(context.TODO(), predictorHPAKey, actualHPA) }, timeout).
 				Should(Succeed())
-			expectedHPA := &v2beta2.HorizontalPodAutoscaler{
-				Spec: v2beta2.HorizontalPodAutoscalerSpec{
-					ScaleTargetRef: v2beta2.CrossVersionObjectReference{
+			expectedHPA := &autoscalingv2.HorizontalPodAutoscaler{
+				Spec: autoscalingv2.HorizontalPodAutoscalerSpec{
+					ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{
 						APIVersion: "apps/v1",
 						Kind:       "Deployment",
 						Name:       constants.PredictorServiceName(serviceKey.Name),
 					},
 					MinReplicas: &minReplicas,
 					MaxReplicas: maxReplicas,
-					Metrics: []v2beta2.MetricSpec{
+					Metrics: []autoscalingv2.MetricSpec{
 						{
-							Type: v2beta2.ResourceMetricSourceType,
-							Resource: &v2beta2.ResourceMetricSource{
+							Type: autoscalingv2.ResourceMetricSourceType,
+							Resource: &autoscalingv2.ResourceMetricSource{
 								Name: v1.ResourceCPU,
-								Target: v2beta2.MetricTarget{
+								Target: autoscalingv2.MetricTarget{
 									Type:               "Utilization",
 									AverageUtilization: &cpuUtilization,
 								},
 							},
 						},
 					},
-					Behavior: &v2beta2.HorizontalPodAutoscalerBehavior{
-						ScaleUp: &v2beta2.HPAScalingRules{
+					Behavior: &autoscalingv2.HorizontalPodAutoscalerBehavior{
+						ScaleUp: &autoscalingv2.HPAScalingRules{
 							StabilizationWindowSeconds: &stabilizationWindowSeconds,
 							SelectPolicy:               &selectPolicy,
-							Policies: []v2beta2.HPAScalingPolicy{
+							Policies: []autoscalingv2.HPAScalingPolicy{
 								{
 									Type:          "Pods",
 									Value:         4,
@@ -480,10 +480,10 @@ var _ = Describe("v1beta1 inference service controller", func() {
 								},
 							},
 						},
-						ScaleDown: &v2beta2.HPAScalingRules{
+						ScaleDown: &autoscalingv2.HPAScalingRules{
 							StabilizationWindowSeconds: nil,
 							SelectPolicy:               &selectPolicy,
-							Policies: []v2beta2.HPAScalingPolicy{
+							Policies: []autoscalingv2.HPAScalingPolicy{
 								{
 									Type:          "Percent",
 									Value:         100,
@@ -868,38 +868,38 @@ var _ = Describe("v1beta1 inference service controller", func() {
 			var maxReplicas int32 = 3
 			var cpuUtilization int32 = 75
 			var stabilizationWindowSeconds int32 = 0
-			selectPolicy := v2beta2.MaxPolicySelect
-			actualHPA := &v2beta2.HorizontalPodAutoscaler{}
+			selectPolicy := autoscalingv2.MaxChangePolicySelect
+			actualHPA := &autoscalingv2.HorizontalPodAutoscaler{}
 			predictorHPAKey := types.NamespacedName{Name: constants.PredictorServiceName(serviceKey.Name),
 				Namespace: serviceKey.Namespace}
 			Eventually(func() error { return k8sClient.Get(context.TODO(), predictorHPAKey, actualHPA) }, timeout).
 				Should(Succeed())
-			expectedHPA := &v2beta2.HorizontalPodAutoscaler{
-				Spec: v2beta2.HorizontalPodAutoscalerSpec{
-					ScaleTargetRef: v2beta2.CrossVersionObjectReference{
+			expectedHPA := &autoscalingv2.HorizontalPodAutoscaler{
+				Spec: autoscalingv2.HorizontalPodAutoscalerSpec{
+					ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{
 						APIVersion: "apps/v1",
 						Kind:       "Deployment",
 						Name:       constants.PredictorServiceName(serviceKey.Name),
 					},
 					MinReplicas: &minReplicas,
 					MaxReplicas: maxReplicas,
-					Metrics: []v2beta2.MetricSpec{
+					Metrics: []autoscalingv2.MetricSpec{
 						{
-							Type: v2beta2.ResourceMetricSourceType,
-							Resource: &v2beta2.ResourceMetricSource{
+							Type: autoscalingv2.ResourceMetricSourceType,
+							Resource: &autoscalingv2.ResourceMetricSource{
 								Name: v1.ResourceCPU,
-								Target: v2beta2.MetricTarget{
+								Target: autoscalingv2.MetricTarget{
 									Type:               "Utilization",
 									AverageUtilization: &cpuUtilization,
 								},
 							},
 						},
 					},
-					Behavior: &v2beta2.HorizontalPodAutoscalerBehavior{
-						ScaleUp: &v2beta2.HPAScalingRules{
+					Behavior: &autoscalingv2.HorizontalPodAutoscalerBehavior{
+						ScaleUp: &autoscalingv2.HPAScalingRules{
 							StabilizationWindowSeconds: &stabilizationWindowSeconds,
 							SelectPolicy:               &selectPolicy,
-							Policies: []v2beta2.HPAScalingPolicy{
+							Policies: []autoscalingv2.HPAScalingPolicy{
 								{
 									Type:          "Pods",
 									Value:         4,
@@ -912,10 +912,10 @@ var _ = Describe("v1beta1 inference service controller", func() {
 								},
 							},
 						},
-						ScaleDown: &v2beta2.HPAScalingRules{
+						ScaleDown: &autoscalingv2.HPAScalingRules{
 							StabilizationWindowSeconds: nil,
 							SelectPolicy:               &selectPolicy,
-							Policies: []v2beta2.HPAScalingPolicy{
+							Policies: []autoscalingv2.HPAScalingPolicy{
 								{
 									Type:          "Percent",
 									Value:         100,
@@ -1301,38 +1301,38 @@ var _ = Describe("v1beta1 inference service controller", func() {
 			var maxReplicas int32 = 3
 			var cpuUtilization int32 = 75
 			var stabilizationWindowSeconds int32 = 0
-			selectPolicy := v2beta2.MaxPolicySelect
-			actualHPA := &v2beta2.HorizontalPodAutoscaler{}
+			selectPolicy := autoscalingv2.MaxChangePolicySelect
+			actualHPA := &autoscalingv2.HorizontalPodAutoscaler{}
 			predictorHPAKey := types.NamespacedName{Name: constants.PredictorServiceName(serviceKey.Name),
 				Namespace: serviceKey.Namespace}
 			Eventually(func() error { return k8sClient.Get(context.TODO(), predictorHPAKey, actualHPA) }, timeout).
 				Should(Succeed())
-			expectedHPA := &v2beta2.HorizontalPodAutoscaler{
-				Spec: v2beta2.HorizontalPodAutoscalerSpec{
-					ScaleTargetRef: v2beta2.CrossVersionObjectReference{
+			expectedHPA := &autoscalingv2.HorizontalPodAutoscaler{
+				Spec: autoscalingv2.HorizontalPodAutoscalerSpec{
+					ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{
 						APIVersion: "apps/v1",
 						Kind:       "Deployment",
 						Name:       constants.PredictorServiceName(serviceKey.Name),
 					},
 					MinReplicas: &minReplicas,
 					MaxReplicas: maxReplicas,
-					Metrics: []v2beta2.MetricSpec{
+					Metrics: []autoscalingv2.MetricSpec{
 						{
-							Type: v2beta2.ResourceMetricSourceType,
-							Resource: &v2beta2.ResourceMetricSource{
+							Type: autoscalingv2.ResourceMetricSourceType,
+							Resource: &autoscalingv2.ResourceMetricSource{
 								Name: v1.ResourceCPU,
-								Target: v2beta2.MetricTarget{
+								Target: autoscalingv2.MetricTarget{
 									Type:               "Utilization",
 									AverageUtilization: &cpuUtilization,
 								},
 							},
 						},
 					},
-					Behavior: &v2beta2.HorizontalPodAutoscalerBehavior{
-						ScaleUp: &v2beta2.HPAScalingRules{
+					Behavior: &autoscalingv2.HorizontalPodAutoscalerBehavior{
+						ScaleUp: &autoscalingv2.HPAScalingRules{
 							StabilizationWindowSeconds: &stabilizationWindowSeconds,
 							SelectPolicy:               &selectPolicy,
-							Policies: []v2beta2.HPAScalingPolicy{
+							Policies: []autoscalingv2.HPAScalingPolicy{
 								{
 									Type:          "Pods",
 									Value:         4,
@@ -1345,10 +1345,10 @@ var _ = Describe("v1beta1 inference service controller", func() {
 								},
 							},
 						},
-						ScaleDown: &v2beta2.HPAScalingRules{
+						ScaleDown: &autoscalingv2.HPAScalingRules{
 							StabilizationWindowSeconds: nil,
 							SelectPolicy:               &selectPolicy,
-							Policies: []v2beta2.HPAScalingPolicy{
+							Policies: []autoscalingv2.HPAScalingPolicy{
 								{
 									Type:          "Percent",
 									Value:         100,
diff --git a/test/e2e/qpext/test_qpext.py b/test/e2e/qpext/test_qpext.py
--- a/test/e2e/qpext/test_qpext.py
+++ b/test/e2e/qpext/test_qpext.py
@@ -33,7 +33,7 @@
 
 ENABLE_METRIC_AGG = "serving.kserve.io/enable-metric-aggregation"
 METRICS_AGG_PORT = 9088
-METRICS_PATH = "/metrics"
+METRICS_PATH = "metrics"
 
 
 def test_qpext_kserve():
@@ -86,6 +86,7 @@ def send_metrics_request(kserve_client, service_name):
 
     url = f"http://localhost:{METRICS_AGG_PORT}/{METRICS_PATH}"
     with portforward.forward(KSERVE_TEST_NAMESPACE, pod_name, METRICS_AGG_PORT, METRICS_AGG_PORT):
+        logging.info(f"metrics request url: {url}")
         response = requests.get(url)
         logging.info(f"response: {response}, content: {response.content}")
         logging.info("Got response code %s, content %s", response.status_code, response.content)
diff --git a/test/scripts/gh-actions/run-e2e-tests.sh b/test/scripts/gh-actions/run-e2e-tests.sh
--- a/test/scripts/gh-actions/run-e2e-tests.sh
+++ b/test/scripts/gh-actions/run-e2e-tests.sh
@@ -23,5 +23,5 @@ set -o pipefail
 echo "Starting E2E functional tests ..."
 source python/kserve/.venv/bin/activate
 pushd test/e2e >/dev/null
-  pytest -m "$1" --ignore=qpext
+  pytest -m "$1" --ignore=qpext --log-level=INFO
 popd
diff --git a/test/scripts/gh-actions/run-qpext-test.sh b/test/scripts/gh-actions/run-qpext-test.sh
--- a/test/scripts/gh-actions/run-qpext-test.sh
+++ b/test/scripts/gh-actions/run-qpext-test.sh
@@ -23,5 +23,5 @@ set -o pipefail
 echo "Starting E2E queue proxy extension tests ..."
 source python/kserve/.venv/bin/activate
 pushd test/e2e >/dev/null
-  pytest qpext
+  pytest --log-level=INFO qpext
 popd
diff --git a/test/scripts/gh-actions/setup-deps.sh b/test/scripts/gh-actions/setup-deps.sh
--- a/test/scripts/gh-actions/setup-deps.sh
+++ b/test/scripts/gh-actions/setup-deps.sh
@@ -23,8 +23,8 @@ set -o pipefail
 
 SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )";
 
-ISTIO_VERSION="1.12.0"
-KNATIVE_VERSION="knative-v1.4.0"
+ISTIO_VERSION="1.17.2"
+KNATIVE_VERSION="knative-v1.9.0"
 CERT_MANAGER_VERSION="v1.5.0"
 YQ_VERSION="v4.28.1"
 
@@ -58,7 +58,7 @@ spec:
 EOF
 
 echo "Installing Knative serving ..."
-pushd ${SCRIPT_DIR}/../../overlays/knative >/dev/null
+pushd "${SCRIPT_DIR}"/../../overlays/knative >/dev/null
   curl -s -O -L https://github.com/knative/serving/releases/download/${KNATIVE_VERSION}/serving-core.yaml
   curl -s -O -L https://github.com/knative/net-istio/releases/download/${KNATIVE_VERSION}/release.yaml
 
@@ -74,14 +74,19 @@ kubectl wait --for=condition=Ready pods --all --timeout=300s -n knative-serving
 # echo "Add knative hpa..."
 # kubectl apply -f https://github.com/knative/serving/releases/download/knative-v1.0.0/serving-hpa.yaml
 
-# Skip tag resolution for certain domains
 # sleep to avoid knative webhook timeout error
 sleep 5
 # Retry if configmap patch fails
 for i in 1 2 3; do
+  # Skip tag resolution for certain domains
   kubectl patch cm config-deployment --patch '{"data":{"registries-skipping-tag-resolving":"nvcr.io,index.docker.io"}}' -n knative-serving && break || sleep 15
 done
 
+echo "Patching knative external domain ..."
+# Patch the external domain as the default domain svc.cluster.local is not exposed on ingress (from knative 1.8)
+for i in 1 2 3; do kubectl patch cm config-domain --patch '{"data":{"example.com":""}}' -n knative-serving && break || sleep 15; done
+kubectl describe cm config-domain -n knative-serving
+
 echo "Installing cert-manager ..."
 kubectl create namespace cert-manager
 sleep 2
diff --git a/test/scripts/gh-actions/setup-kserve.sh b/test/scripts/gh-actions/setup-kserve.sh
--- a/test/scripts/gh-actions/setup-kserve.sh
+++ b/test/scripts/gh-actions/setup-kserve.sh
@@ -46,5 +46,5 @@ export PATH="$PATH:$POETRY_HOME/bin"
 echo "Installing KServe Python SDK ..."
 pushd python/kserve >/dev/null
     poetry config virtualenvs.in-project true
-    poetry version $(cat ${../python/VERSION}) && poetry install --with=test --no-interaction
+    poetry version $(cat ../VERSION) && poetry install --with=test --no-interaction
 popd