Repository: kubernetes-sigs/gateway-api-inference-extension

Issue:
Docs: Getting Started Guide for istio resulting in a broken example
**What happened**:
Following the Istio setup in the [Getting Started Guide ](https://gateway-api-inference-extension.sigs.k8s.io/guides/#__tabbed_4_1) results in a non working setup (istio/istio#56055) as the two code bases have diverged. 

**What you expected to happen**:
A working example.

**How to reproduce it (as minimally and precisely as possible)**:
Using any "older" version of istio will trigger this bug as GIE has moved to not support BUFFERED processing mode.

**Anything else we need to know?**:


**Environment**:
*

Maybe some form of "testable docs" could be handy to catch regressions on both sides and as a light sanity check?

Discussion:
@nirrozenbaum: @aslakknutsen can you add a link to the fix PR you created in Istio?
just to add more context for future readers of this issue.@nirrozenbaum: once that PR is merged in Istio, we should make sure Istio has a new tag with the fix and update our getting started guide accordingly.

CC @LiorLieberman @kfswain @LiorLieberman: Thanks! @nirrozenbaum lets keep this open until this is released. 

For ref this is the PR that broke it - https://github.com/kubernetes-sigs/gateway-api-inference-extension/pull/672@LiorLieberman: /assign

Solution patch:
diff --git a/pkg/epp/backend/metrics/pod_metrics.go b/pkg/epp/backend/metrics/pod_metrics.go
index bdeb28b..735f297 100644
--- a/pkg/epp/backend/metrics/pod_metrics.go
+++ b/pkg/epp/backend/metrics/pod_metrics.go
@@ -75,6 +75,7 @@ func toInternalPod(in *corev1.Pod) *backend.Pod {
 			Namespace: in.Namespace,
 		},
 		Address: in.Status.PodIP,
+		Labels:  in.Labels,
 	}
 }
 
diff --git a/pkg/epp/backend/pod.go b/pkg/epp/backend/pod.go
index a63a0a8..57f034b 100644
--- a/pkg/epp/backend/pod.go
+++ b/pkg/epp/backend/pod.go
@@ -22,6 +22,7 @@ import (
 type Pod struct {
 	NamespacedName types.NamespacedName
 	Address        string
+	Labels         map[string]string
 }
 
 func (p *Pod) String() string {
@@ -41,5 +42,6 @@ func (p *Pod) Clone() *Pod {
 			Namespace: p.NamespacedName.Namespace,
 		},
 		Address: p.Address,
+		Labels:  p.Labels,
 	}
 }
diff --git a/site-src/guides/index.md b/site-src/guides/index.md
index bcd1068..be1b972 100644
--- a/site-src/guides/index.md
+++ b/site-src/guides/index.md
@@ -135,7 +135,7 @@ This quickstart guide is intended for engineers familiar with k8s and model serv
       2. Install Istio
       
          ```
-         TAG=1.26-alpha.80c74f7f43482c226f4f4b10b4dda6261b67a71f
+         TAG=1.26-alpha.9befed2f1439d883120f8de70fd70d84ca0ebc3d
          # on Linux
          wget https://storage.googleapis.com/istio-build/dev/$TAG/istioctl-$TAG-linux-amd64.tar.gz
          tar -xvf istioctl-$TAG-linux-amd64.tar.gz



Test patch:
diff --git a/test/integration/epp/hermetic_test.go b/test/integration/epp/hermetic_test.go
index c63fd01..68787da 100644
--- a/test/integration/epp/hermetic_test.go
+++ b/test/integration/epp/hermetic_test.go
@@ -98,7 +98,7 @@ func TestFullDuplexStreamed_KubeInferenceModelRequest(t *testing.T) {
 	tests := []struct {
 		name              string
 		requests          []*extProcPb.ProcessingRequest
-		pods              map[backend.Pod]*backendmetrics.Metrics
+		pods              map[*backend.Pod]*backendmetrics.Metrics
 		wantResponses     []*extProcPb.ProcessingResponse
 		wantMetrics       map[string]string
 		wantErr           bool
@@ -109,7 +109,7 @@ func TestFullDuplexStreamed_KubeInferenceModelRequest(t *testing.T) {
 			name:     "select lower queue and kv cache, no active lora",
 			requests: integrationutils.GenerateStreamedRequestSet(logger, "test1", "my-model"),
 			// pod-1 will be picked because it has relatively low queue size and low KV cache.
-			pods: map[backend.Pod]*backendmetrics.Metrics{
+			pods: map[*backend.Pod]*backendmetrics.Metrics{
 				fakePod(0): {
 					WaitingQueueSize:    3,
 					KVCacheUsagePercent: 0.2,
@@ -184,7 +184,7 @@ func TestFullDuplexStreamed_KubeInferenceModelRequest(t *testing.T) {
 			requests: integrationutils.GenerateStreamedRequestSet(logger, "test2", "sql-lora"),
 			// pod-1 will be picked because it has relatively low queue size, with the requested
 			// model being active, and has low KV cache.
-			pods: map[backend.Pod]*backendmetrics.Metrics{
+			pods: map[*backend.Pod]*backendmetrics.Metrics{
 				fakePod(0): {
 					WaitingQueueSize:    0,
 					KVCacheUsagePercent: 0.2,
@@ -269,7 +269,7 @@ func TestFullDuplexStreamed_KubeInferenceModelRequest(t *testing.T) {
 			// pod-2 will be picked despite it NOT having the requested model being active
 			// as it's above the affinity for queue size. Also is critical, so we should
 			// still honor request despite all queues > 5
-			pods: map[backend.Pod]*backendmetrics.Metrics{
+			pods: map[*backend.Pod]*backendmetrics.Metrics{
 				fakePod(0): {
 					WaitingQueueSize:    10,
 					KVCacheUsagePercent: 0.2,
@@ -352,7 +352,7 @@ func TestFullDuplexStreamed_KubeInferenceModelRequest(t *testing.T) {
 			requests: integrationutils.GenerateStreamedRequestSet(logger, "test4", "sql-lora-sheddable"),
 			// no pods will be picked as all models are either above kv threshold,
 			// queue threshold, or both.
-			pods: map[backend.Pod]*backendmetrics.Metrics{
+			pods: map[*backend.Pod]*backendmetrics.Metrics{
 				fakePod(0): {
 					WaitingQueueSize:    6,
 					KVCacheUsagePercent: 0.2,
@@ -400,7 +400,7 @@ func TestFullDuplexStreamed_KubeInferenceModelRequest(t *testing.T) {
 			name:     "noncritical, but one server has capacity, do not shed",
 			requests: integrationutils.GenerateStreamedRequestSet(logger, "test5", "sql-lora-sheddable"),
 			// pod 0 will be picked as all other models are above threshold
-			pods: map[backend.Pod]*backendmetrics.Metrics{
+			pods: map[*backend.Pod]*backendmetrics.Metrics{
 				fakePod(0): {
 					WaitingQueueSize:    4,
 					KVCacheUsagePercent: 0.2,
@@ -511,7 +511,7 @@ func TestFullDuplexStreamed_KubeInferenceModelRequest(t *testing.T) {
 
 			//
 			// pod 0 will be picked as all other models are above threshold
-			pods: map[backend.Pod]*backendmetrics.Metrics{
+			pods: map[*backend.Pod]*backendmetrics.Metrics{
 				fakePod(0): {
 					WaitingQueueSize:    4,
 					KVCacheUsagePercent: 0.2,
@@ -622,7 +622,7 @@ func TestFullDuplexStreamed_KubeInferenceModelRequest(t *testing.T) {
 
 			//
 			// pod 0 will be picked as all other models are above threshold
-			pods: map[backend.Pod]*backendmetrics.Metrics{
+			pods: map[*backend.Pod]*backendmetrics.Metrics{
 				fakePod(0): {
 					WaitingQueueSize:    4,
 					KVCacheUsagePercent: 0.2,
@@ -734,7 +734,7 @@ func TestFullDuplexStreamed_KubeInferenceModelRequest(t *testing.T) {
 
 			//
 			// pod 0 will be picked as all other models are above threshold
-			pods: map[backend.Pod]*backendmetrics.Metrics{
+			pods: map[*backend.Pod]*backendmetrics.Metrics{
 				fakePod(0): {
 					WaitingQueueSize:    4,
 					KVCacheUsagePercent: 0.2,
@@ -833,7 +833,7 @@ func TestFullDuplexStreamed_KubeInferenceModelRequest(t *testing.T) {
 
 			//
 			// pod 0 will be picked as all other models are above threshold
-			pods: map[backend.Pod]*backendmetrics.Metrics{
+			pods: map[*backend.Pod]*backendmetrics.Metrics{
 				fakePod(0): {
 					WaitingQueueSize:    4,
 					KVCacheUsagePercent: 0.2,
@@ -1181,7 +1181,7 @@ func TestFullDuplexStreamed_KubeInferenceModelRequest(t *testing.T) {
 					DynamicMetadata: makeMetadata("192.168.1.1:8000"),
 				},
 			},
-			pods: map[backend.Pod]*backendmetrics.Metrics{
+			pods: map[*backend.Pod]*backendmetrics.Metrics{
 				fakePod(0): {
 					WaitingQueueSize:    4,
 					KVCacheUsagePercent: 0.2,
@@ -1227,7 +1227,7 @@ func TestFullDuplexStreamed_KubeInferenceModelRequest(t *testing.T) {
 	}
 }
 
-func setUpHermeticServer(t *testing.T, podAndMetrics map[backend.Pod]*backendmetrics.Metrics, streamed bool) (client extProcPb.ExternalProcessor_ProcessClient, cleanup func()) {
+func setUpHermeticServer(t *testing.T, podAndMetrics map[*backend.Pod]*backendmetrics.Metrics, streamed bool) (client extProcPb.ExternalProcessor_ProcessClient, cleanup func()) {
 	// Reconfigure the TestPodMetricsClient.
 	res := map[types.NamespacedName]*backendmetrics.Metrics{}
 	for pod, metrics := range podAndMetrics {
@@ -1305,10 +1305,11 @@ func setUpHermeticServer(t *testing.T, podAndMetrics map[backend.Pod]*backendmet
 	}
 }
 
-func fakePod(index int) backend.Pod {
-	return backend.Pod{
+func fakePod(index int) *backend.Pod {
+	return &backend.Pod{
 		NamespacedName: types.NamespacedName{Name: fmt.Sprintf("pod-%v", index), Namespace: "default"},
 		Address:        fmt.Sprintf("192.168.1.%d", index+1),
+		Labels:         make(map[string]string, 0),
 	}
 }