Repository: kserve/kserve

Issue:
Update ModelMesh version to `v0.10.0`
#### Proposed Changes

Update to the latest ModelMesh version (`v0.10.0`)

> If your changes should also be in the most recent release, add the corresponding "cherrypick-0.X"
label to the original PR; for example, "cherrypick-0.10".

#### TODOs:

> - [ ] Best practice is to open a PR for the cherry-pick yourself after your original PR has been merged
into the main branch.
> - [ ] After the cherry-pick PR has merged, remove the cherry-pick label from the original PR.


**Type of changes**

- [x] This change requires a documentation update --> https://github.com/kserve/website/pull/214

**Special notes for your reviewer**:

1. Please confirm that if this PR changes any image versions, then that's the sole change this PR makes.

YES

**Checklist**:

- [x] Have you made corresponding changes to the documentation?

**Release note**:
<!--  Write your release note:
1. Enter your extended release note in the below block. If the PR requires additional action from users switching to the new release, include the string "action required".
3. If no release note is required, just write "NONE".
-->
```release-note
???
```
Not sure, maybe, 
- "Updgrade to ModelMesh v0.10.0" 
or
- point to ModelMesh release notes here
https://github.com/kserve/modelmesh-serving/releases/tag/v0.10.0


---

/cc @yuzisun @rachitchauhan43 @njhill 



Discussion:

Resolving patch:
diff --git a/python/kserve/setup.py b/python/kserve/setup.py
--- a/python/kserve/setup.py
+++ b/python/kserve/setup.py
@@ -21,7 +21,8 @@
     'pytest-cov',
     'pytest-asyncio',
     'pytest-tornasync',
-    'mypy'
+    'mypy',
+    'portforward',
 ]
 
 with open('requirements.txt') as f:


Test patch:
diff --git a/config/overlays/test/configmap/inferenceservice.yaml b/config/overlays/test/configmap/inferenceservice.yaml
--- a/config/overlays/test/configmap/inferenceservice.yaml
+++ b/config/overlays/test/configmap/inferenceservice.yaml
@@ -71,3 +71,8 @@ data:
         "cpuRequest": "100m",
         "cpuLimit": "300m"
     }
+  metricsAggregator: |-
+    {
+      "enableMetricAggregation": "false",
+      "enablePrometheusScraping" : "false"
+    }
diff --git a/pkg/webhook/admission/pod/metrics_aggregate_injector_test.go b/pkg/webhook/admission/pod/metrics_aggregate_injector_test.go
--- a/pkg/webhook/admission/pod/metrics_aggregate_injector_test.go
+++ b/pkg/webhook/admission/pod/metrics_aggregate_injector_test.go
@@ -95,7 +95,7 @@ func TestInjectMetricsAggregator(t *testing.T) {
 					Name:      "deployment",
 					Namespace: "default",
 					Annotations: map[string]string{
-						constants.EnableMetricAggregation: "true",
+						constants.EnableMetricAggregation: "false",
 					},
 				},
 				Spec: v1.PodSpec{
@@ -147,14 +147,14 @@ func TestInjectMetricsAggregator(t *testing.T) {
 				},
 			},
 		},
-		"setPromAnnotationTrue": {
+		"setPromAnnotationTrueWithAggTrue": {
 			original: &v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      "deployment",
 					Namespace: "default",
 					Annotations: map[string]string{
-						constants.EnableMetricAggregation:          "true",
-						constants.SetPrometheusAggregateAnnotation: "true",
+						constants.EnableMetricAggregation: "true",
+						constants.SetPrometheusAnnotation: "true",
 					},
 				},
 				Spec: v1.PodSpec{
@@ -172,10 +172,10 @@ func TestInjectMetricsAggregator(t *testing.T) {
 					Name:      "deployment",
 					Namespace: "default",
 					Annotations: map[string]string{
-						constants.EnableMetricAggregation:          "true",
-						constants.SetPrometheusAggregateAnnotation: "true",
-						constants.PrometheusPortAnnotationKey:      constants.QueueProxyAggregatePrometheusMetricsPort,
-						constants.PrometheusPathAnnotationKey:      constants.DefaultPrometheusPath,
+						constants.EnableMetricAggregation:     "true",
+						constants.SetPrometheusAnnotation:     "true",
+						constants.PrometheusPortAnnotationKey: constants.QueueProxyAggregatePrometheusMetricsPort,
+						constants.PrometheusPathAnnotationKey: constants.DefaultPrometheusPath,
 					},
 				},
 				Spec: v1.PodSpec{
@@ -194,14 +194,56 @@ func TestInjectMetricsAggregator(t *testing.T) {
 				},
 			},
 		},
+		"setPromAnnotationTrueWithAggFalse": {
+			original: &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "deployment",
+					Namespace: "default",
+					Annotations: map[string]string{
+						constants.EnableMetricAggregation: "false",
+						constants.SetPrometheusAnnotation: "true",
+					},
+				},
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{{
+						Name: "sklearn",
+					},
+						{
+							Name: "queue-proxy",
+						},
+					},
+				},
+			},
+			expected: &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "deployment",
+					Namespace: "default",
+					Annotations: map[string]string{
+						constants.EnableMetricAggregation:     "false",
+						constants.SetPrometheusAnnotation:     "true",
+						constants.PrometheusPortAnnotationKey: constants.DefaultPodPrometheusPort,
+						constants.PrometheusPathAnnotationKey: constants.DefaultPrometheusPath,
+					},
+				},
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{{
+						Name: "sklearn",
+					},
+						{
+							Name: "queue-proxy",
+						},
+					},
+				},
+			},
+		},
 		"SetPromAnnotationFalse": {
 			original: &v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      "deployment",
 					Namespace: "default",
 					Annotations: map[string]string{
-						constants.EnableMetricAggregation:          "true",
-						constants.SetPrometheusAggregateAnnotation: "false",
+						constants.EnableMetricAggregation: "true",
+						constants.SetPrometheusAnnotation: "false",
 					},
 				},
 				Spec: v1.PodSpec{
@@ -219,8 +261,8 @@ func TestInjectMetricsAggregator(t *testing.T) {
 					Name:      "deployment",
 					Namespace: "default",
 					Annotations: map[string]string{
-						constants.EnableMetricAggregation:          "true",
-						constants.SetPrometheusAggregateAnnotation: "false",
+						constants.EnableMetricAggregation: "true",
+						constants.SetPrometheusAnnotation: "false",
 					},
 				},
 				Spec: v1.PodSpec{
@@ -241,11 +283,14 @@ func TestInjectMetricsAggregator(t *testing.T) {
 		},
 	}
 
+	cfgMap := v1.ConfigMap{Data: map[string]string{"enableMetricAggregation": "false", "enablePrometheusScraping": "false"}}
+	ma, err := newMetricsAggregator(&cfgMap)
+	if err != nil {
+		t.Errorf("Error creating the metrics aggregator %v", err)
+	}
+
 	for name, scenario := range scenarios {
-		err := InjectMetricsAggregator(scenario.original)
-		if err != nil {
-			t.Errorf("Test %q unexpected error %e", name, err)
-		}
+		ma.InjectMetricsAggregator(scenario.original)
 		if diff, _ := kmp.SafeDiff(scenario.expected.Spec, scenario.original.Spec); diff != "" {
 			t.Errorf("Test %q unexpected result (-want +got): %v", name, diff)
 		}
diff --git a/pkg/webhook/admission/pod/mutator_test.go b/pkg/webhook/admission/pod/mutator_test.go
--- a/pkg/webhook/admission/pod/mutator_test.go
+++ b/pkg/webhook/admission/pod/mutator_test.go
@@ -15,6 +15,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+	"sort"
 	"testing"
 )
 
@@ -255,8 +256,16 @@ func TestMutator_Handle(t *testing.T) {
 					},
 				},
 			},
-			matcher: gomega.Equal(admission.Response{
+			matcher: gomega.BeEquivalentTo(admission.Response{
 				Patches: []jsonpatch.JsonPatchOperation{
+					{
+						Operation: "add",
+						Path:      "/metadata/annotations",
+						Value: map[string]interface{}{
+							"serving.kserve.io/enable-metric-aggregation":  "",
+							"serving.kserve.io/enable-prometheus-scraping": "",
+						},
+					},
 					{
 						Operation: "add",
 						Path:      "/metadata/namespace",
@@ -287,11 +296,21 @@ func TestMutator_Handle(t *testing.T) {
 			}
 			tc.request.Object.Raw = byteData
 			res := mutator.Handle(context.TODO(), tc.request)
+			sortPatches(res.Patches)
 			g.Expect(res).Should(tc.matcher)
 			if err := c.Delete(context.TODO(), &tc.configMap); err != nil {
 				t.Errorf("failed to delete configmap %v", err)
 			}
 		})
 	}
+}
 
+// sortPatches sorts the slice of patches by Path so that the comparison works
+// when there are > 1 patches. Note: make sure the matcher Patches are sorted.
+func sortPatches(patches []jsonpatch.JsonPatchOperation) {
+	if len(patches) > 1 {
+		sort.Slice(patches, func(i, j int) bool {
+			return patches[i].Path < patches[j].Path
+		})
+	}
 }
diff --git a/qpext/cmd/qpext/main_test.go b/qpext/cmd/qpext/main_test.go
new file mode 100644
--- /dev/null
+++ b/qpext/cmd/qpext/main_test.go
@@ -0,0 +1,233 @@
+/*
+Copyright 2022 The KServe Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	logger "github.com/kserve/kserve/qpext"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/expfmt"
+	"github.com/stretchr/testify/assert"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+func TestGetHeaderTimeout(t *testing.T) {
+	inputs := []string{"1.23", "100", "notvalid", "12.wrong"}
+	errIsNil := []bool{true, true, false, false}
+
+	for i, input := range inputs {
+		_, err := getHeaderTimeout(input)
+		if errIsNil[i] == true {
+			assert.NoError(t, err)
+		} else {
+			assert.Error(t, err)
+		}
+	}
+}
+
+func TestNegotiateMetricsFromat(t *testing.T) {
+	contentTypes := []string{"", "random", "text/plain;version=0.0.4;q=0.5,*/*;q=0.1", `application/openmetrics-text; version=1.0.0; charset=utf-8`}
+	expected := []expfmt.Format{expfmt.FmtText, expfmt.FmtText, expfmt.FmtText, expfmt.FmtOpenMetrics}
+
+	for i, contentType := range contentTypes {
+		result := negotiateMetricsFormat(contentType)
+		assert.Equal(t, expected[i], result)
+	}
+}
+
+func TestScrapeHeaders(t *testing.T) {
+	metricExample := `# TYPE my_metric counter
+	my_metric{} 0
+	`
+	timeoutHeader := "X-Prometheus-Scrape-Timeout-Seconds"
+	tests := []struct {
+		name            string
+		headerVal       string
+		expectNilCancel bool
+	}{
+		{
+			name:      "timeout header parses",
+			headerVal: "10",
+		},
+		{
+			name:            "timeout header invalid",
+			headerVal:       "invalid",
+			expectNilCancel: true,
+		},
+	}
+
+	zapLogger := logger.InitializeLogger()
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			promRegistry = prometheus.NewRegistry()
+			qp := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				_, err := w.Write([]byte(metricExample))
+				assert.NoError(t, err)
+			}))
+			defer qp.Close()
+
+			url := getURL(strings.Split(qp.URL, ":")[2], "/metrics")
+
+			req := &http.Request{
+				Header: map[string][]string{timeoutHeader: {test.headerVal}},
+			}
+			queueProxy, queueProxyCancel, _, err := scrape(url, req.Header, zapLogger)
+			assert.NoError(t, err)
+			assert.NotNil(t, queueProxy)
+			if test.expectNilCancel {
+				assert.Nil(t, queueProxyCancel)
+			} else {
+				assert.NotNil(t, queueProxyCancel)
+			}
+		})
+	}
+}
+
+func TestScrapeErr(t *testing.T) {
+	metricExample := `# TYPE my_metric counter
+	my_metric{} 0
+	`
+	zapLogger := logger.InitializeLogger()
+	promRegistry = prometheus.NewRegistry()
+	qp := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, err := w.Write([]byte(metricExample))
+		assert.NoError(t, err)
+	}))
+	defer qp.Close()
+
+	url := "not-a-real-url"
+
+	req := &http.Request{}
+	queueProxy, _, _, err := scrape(url, req.Header, zapLogger)
+	assert.Error(t, err)
+	assert.Nil(t, queueProxy)
+}
+
+func TestHandleStats(t *testing.T) {
+	metricExample := `# TYPE my_metric counter
+	my_metric{} 0
+	`
+	otherMetricExample := `# TYPE my_other_metric counter
+	my_other_metric{} 0
+	`
+	tests := []struct {
+		name             string
+		queueproxy       string
+		app              string
+		output           string
+		expectParseError bool
+	}{
+		{
+			name:       "queueproxy metric only",
+			queueproxy: metricExample,
+			output:     metricExample,
+		},
+		{
+			name:   "app metric only",
+			app:    metricExample,
+			output: metricExample,
+		},
+		{
+			name:       "multiple metric",
+			queueproxy: metricExample,
+			app:        otherMetricExample,
+			output:     metricExample + otherMetricExample,
+		},
+		// when app and queueproxy share a metric, Prometheus will fail.
+		{
+			name:             "conflict metric",
+			queueproxy:       metricExample + otherMetricExample,
+			app:              metricExample,
+			output:           metricExample + otherMetricExample + metricExample,
+			expectParseError: true,
+		},
+	}
+
+	zapLogger := logger.InitializeLogger()
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			promRegistry = prometheus.NewRegistry()
+			rec := httptest.NewRecorder()
+			qp := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				_, err := w.Write([]byte(test.queueproxy))
+				assert.NoError(t, err)
+			}))
+			defer qp.Close()
+
+			app := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				_, err := w.Write([]byte(test.app))
+				assert.NoError(t, err)
+			}))
+			defer app.Close()
+
+			psc := &ScrapeConfigurations{
+				logger:         zapLogger,
+				QueueProxyPort: strings.Split(qp.URL, ":")[2],
+				AppPort:        strings.Split(app.URL, ":")[2],
+			}
+			req := &http.Request{}
+			psc.handleStats(rec, req)
+			assert.Equal(t, rec.Code, 200)
+			assert.Contains(t, rec.Body.String(), test.output)
+
+			parser := expfmt.TextParser{}
+			mfMap, err := parser.TextToMetricFamilies(strings.NewReader(rec.Body.String()))
+			if !test.expectParseError {
+				assert.NoErrorf(t, err, "failed to parse metrics: %v", err)
+			} else if err == nil && test.expectParseError {
+				assert.False(t, test.expectParseError, "expected a prse error, got %+v", mfMap)
+			}
+		})
+	}
+
+}
+
+func TestHandleStatsErr(t *testing.T) {
+	zapLogger := logger.InitializeLogger()
+	fail := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusInternalServerError)
+	}))
+	defer fail.Close()
+	pass := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer pass.Close()
+	failPort := strings.Split(fail.URL, ":")[2]
+	passPort := strings.Split(pass.URL, ":")[2]
+
+	tests := []struct {
+		name       string
+		queueproxy string
+		app        string
+	}{
+		{"both pass", passPort, passPort},
+		{"queue proxy pass", passPort, failPort},
+		{"app pass", failPort, passPort},
+		{"both fail", failPort, failPort},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			sc := NewScrapeConfigs(zapLogger, test.queueproxy, test.app, DefaultQueueProxyMetricsPath)
+			req := &http.Request{}
+			rec := httptest.NewRecorder()
+			sc.handleStats(rec, req)
+			assert.Equal(t, 200, rec.Code)
+		})
+	}
+}
diff --git a/test/e2e/qpext/__init__.py b/test/e2e/qpext/__init__.py
new file mode 100644
diff --git a/test/e2e/qpext/test_qpext.py b/test/e2e/qpext/test_qpext.py
new file mode 100644
--- /dev/null
+++ b/test/e2e/qpext/test_qpext.py
@@ -0,0 +1,94 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import logging
+import time
+import requests
+import portforward
+from kubernetes import client
+from kserve import (
+    constants,
+    KServeClient,
+    V1beta1InferenceService,
+    V1beta1InferenceServiceSpec,
+    V1beta1PredictorSpec,
+    V1beta1SKLearnSpec,
+)
+from kubernetes.client import V1ResourceRequirements
+from ..common.utils import KSERVE_TEST_NAMESPACE
+from ..common.utils import predict
+
+logging.basicConfig(level=logging.INFO)
+
+ENABLE_METRIC_AGG = "serving.kserve.io/enable-metric-aggregation"
+METRICS_AGG_PORT = 9088
+METRICS_PATH = "/metrics"
+
+
+def test_qpext_kserve():
+    # test the qpext using the sklearn predictor
+    service_name = "isvc-sklearn-v2"
+    predictor = V1beta1PredictorSpec(
+        min_replicas=1,
+        sklearn=V1beta1SKLearnSpec(
+            storage_uri="gs://seldon-models/sklearn/mms/lr_model",
+            protocol_version="v2",
+            resources=V1ResourceRequirements(
+                requests={"cpu": "50m", "memory": "128Mi"},
+                limits={"cpu": "100m", "memory": "512Mi"},
+            ),
+        ),
+    )
+
+    isvc = V1beta1InferenceService(
+        api_version=constants.KSERVE_V1BETA1,
+        kind=constants.KSERVE_KIND,
+        metadata=client.V1ObjectMeta(
+            name=service_name, namespace=KSERVE_TEST_NAMESPACE,
+            # set the metric aggregation annotation to true
+            annotations={ENABLE_METRIC_AGG: "true"},
+        ),
+        spec=V1beta1InferenceServiceSpec(predictor=predictor),
+    )
+
+    kserve_client = KServeClient(config_file=os.environ.get("KUBECONFIG", "~/.kube/config"))
+    kserve_client.create(isvc)
+    kserve_client.wait_isvc_ready(service_name, namespace=KSERVE_TEST_NAMESPACE)
+
+    res = predict(service_name, "./data/iris_input_v2.json", protocol_version="v2")
+    assert res["outputs"][0]["data"] == [1, 1]
+
+    send_metrics_request(kserve_client, service_name)
+    kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE)
+
+
+def send_metrics_request(kserve_client, service_name):
+    time.sleep(10)
+    pods = kserve_client.core_api.list_namespaced_pod(KSERVE_TEST_NAMESPACE,
+                                                      label_selector='serving.kserve.io/inferenceservice={}'.
+                                                      format(service_name))
+    pod_name = ""
+    for pod in pods.items:
+        # get a pod name
+        pod_name = pod.metadata.name
+        break
+
+    url = f"http://localhost:{METRICS_AGG_PORT}/{METRICS_PATH}"
+    with portforward.forward(KSERVE_TEST_NAMESPACE, pod_name, METRICS_AGG_PORT, METRICS_AGG_PORT):
+        response = requests.get(url)
+        logging.info(f"response: {response}, content: {response.content}")
+        logging.info("Got response code %s, content %s", response.status_code, response.content)
+
+        assert response.status_code == 200
+        assert len(response.content) > 0
diff --git a/test/scripts/gh-actions/build-qpext-image.sh b/test/scripts/gh-actions/build-qpext-image.sh
new file mode 100755
--- /dev/null
+++ b/test/scripts/gh-actions/build-qpext-image.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Copyright 2022 The KServe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The script is used to build all the queue-proxy extension image.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+echo "Github SHA ${GITHUB_SHA}"
+export QPEXT_IMG=kserve/qpext:${GITHUB_SHA}
+
+pushd qpext >/dev/null
+echo "Building queue proxy extension image"
+docker build -t ${QPEXT_IMG} -f qpext.Dockerfile .
+popd
+echo "Done building image"
diff --git a/test/scripts/gh-actions/run-e2e-tests.sh b/test/scripts/gh-actions/run-e2e-tests.sh
--- a/test/scripts/gh-actions/run-e2e-tests.sh
+++ b/test/scripts/gh-actions/run-e2e-tests.sh
@@ -22,5 +22,5 @@ set -o pipefail
 
 echo "Starting E2E functional tests ..."
 pushd test/e2e >/dev/null
-  pytest -m "$1"
+  pytest -m "$1" --ignore=qpext
 popd
diff --git a/test/scripts/gh-actions/run-qpext-test.sh b/test/scripts/gh-actions/run-qpext-test.sh
new file mode 100755
--- /dev/null
+++ b/test/scripts/gh-actions/run-qpext-test.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Copyright 2022 The KServe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The script is used to deploy knative and kserve, and run e2e tests.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+echo "Starting E2E queue proxy extension tests ..."
+pushd test/e2e >/dev/null
+  pytest qpext
+popd