Repository: kubernetes-sigs/gateway-api-inference-extension

Issue:
gpu deployment cannot be applied when using k8s version older than 1.30
```
kubectl version
Client Version: v1.29.11
Kustomize Version: v5.0.4-0.20230601165947-6ce0bf390ce3
Server Version: v1.29.11
```

according to the documentation in gpu deployment file, when using version older than 1.30, it's needed to update the `lifecycle.preStop` section, comment the the `sleep` and uncomment the `exec` instead.
that's currently not working since sleep is expected to get a string and not an integer number.

```
gateway-api-inference-extension nirro$ kubectl apply -f ./config/manifests/vllm/gpu-deployment.yaml 
configmap/vllm-llama3-8b-adapters created
Error from server (BadRequest): error when creating "./config/manifests/vllm/gpu-deployment.yaml": Deployment in version "v1" cannot be handled as a Deployment: json: cannot unmarshal number into Go struct field ExecAction.spec.template.spec.containers.lifecycle.preStop.exec.command of type string
```

Discussion:
@nirrozenbaum: /assign

Solution patch:
diff --git a/config/charts/body-based-routing/README.md b/config/charts/body-based-routing/README.md
index 062f2b5c..a6b8d3cd 100644
--- a/config/charts/body-based-routing/README.md
+++ b/config/charts/body-based-routing/README.md
@@ -47,7 +47,7 @@ The following table list the configurable parameters of the chart.
 | `bbr.image.tag`              | Image tag.                                                                                                        |
 | `bbr.image.pullPolicy`       | Image pull policy for the container. Possible values: `Always`, `IfNotPresent`, or `Never`. Defaults to `Always`. |
 | `provider.name`              | Name of the Inference Gateway implementation being used. Possible values: `istio`, `gke`. Defaults to `none`.     |
-| `inference-gateway.name`     | The name of the Gateway. Defaults to `inference-gateway`.                                                         |                        
+| `inferenceGateway.name`      | The name of the Gateway. Defaults to `inference-gateway`.                                                         |                        
 
 ## Notes
 
diff --git a/config/charts/body-based-routing/templates/gke.yaml b/config/charts/body-based-routing/templates/gke.yaml
index 937bfa0b..77b776a4 100644
--- a/config/charts/body-based-routing/templates/gke.yaml
+++ b/config/charts/body-based-routing/templates/gke.yaml
@@ -9,7 +9,7 @@ spec:
   targetRefs:
   - group: "gateway.networking.k8s.io"
     kind: Gateway
-    name: {{ .Values.inference-gateway.name }}
+    name: {{ .Values.inferenceGateway.name }}
   extensionChains:
   - name: chain1
     extensions:
diff --git a/config/charts/body-based-routing/values.yaml b/config/charts/body-based-routing/values.yaml
index b77d7542..0b88dc43 100644
--- a/config/charts/body-based-routing/values.yaml
+++ b/config/charts/body-based-routing/values.yaml
@@ -12,5 +12,5 @@ bbr:
 provider:
   name: none
 
-inference-gateway:
+inferenceGateway:
   name: inference-gateway
diff --git a/config/charts/inferencepool/templates/epp-deployment.yaml b/config/charts/inferencepool/templates/epp-deployment.yaml
index 9faace73..d925a38e 100644
--- a/config/charts/inferencepool/templates/epp-deployment.yaml
+++ b/config/charts/inferencepool/templates/epp-deployment.yaml
@@ -16,6 +16,8 @@ spec:
         {{- include "gateway-api-inference-extension.selectorLabels" . | nindent 8 }}
     spec:
       serviceAccountName: {{ include "gateway-api-inference-extension.name" . }}
+      # Conservatively, this timeout should mirror the longest grace period of the pods within the pool
+      terminationGracePeriodSeconds: 130
       containers:
       - name: epp
         image: {{ .Values.inferenceExtension.image.hub }}/{{ .Values.inferenceExtension.image.name }}:{{ .Values.inferenceExtension.image.tag }}
diff --git a/config/manifests/gateway/gke/healthcheck.yaml b/config/manifests/gateway/gke/healthcheck.yaml
index 95f4f2d2..93b6cd7f 100644
--- a/config/manifests/gateway/gke/healthcheck.yaml
+++ b/config/manifests/gateway/gke/healthcheck.yaml
@@ -7,7 +7,7 @@ spec:
   targetRef:
     group: "inference.networking.x-k8s.io"
     kind: InferencePool
-    name: vllm-llama2-7b
+    name: vllm-llama3-8b-instruct
   default:
     config:
       type: HTTP
diff --git a/config/manifests/gateway/httproute.yaml b/config/manifests/gateway/gke/httproute.yaml
similarity index 100%
rename from config/manifests/gateway/httproute.yaml
rename to config/manifests/gateway/gke/httproute.yaml
diff --git a/config/manifests/gateway/httproute-with-timeout.yaml b/config/manifests/gateway/istio/httproute.yaml
similarity index 100%
rename from config/manifests/gateway/httproute-with-timeout.yaml
rename to config/manifests/gateway/istio/httproute.yaml
diff --git a/config/manifests/gateway/kgateway/httproute.yaml b/config/manifests/gateway/kgateway/httproute.yaml
new file mode 100644
index 00000000..03967729
--- /dev/null
+++ b/config/manifests/gateway/kgateway/httproute.yaml
@@ -0,0 +1,21 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: llm-route
+spec:
+  parentRefs:
+  - group: gateway.networking.k8s.io
+    kind: Gateway
+    name: inference-gateway
+  rules:
+  - backendRefs:
+    - group: inference.networking.x-k8s.io
+      kind: InferencePool
+      name: vllm-llama3-8b-instruct
+      port: 8000 # Remove when https://github.com/kgateway-dev/kgateway/issues/10987 is fixed.
+    matches:
+    - path:
+        type: PathPrefix
+        value: /
+    timeouts:
+      request: 300s
diff --git a/config/manifests/inferencepool-resources.yaml b/config/manifests/inferencepool-resources.yaml
index d0f36e83..cef70d7f 100644
--- a/config/manifests/inferencepool-resources.yaml
+++ b/config/manifests/inferencepool-resources.yaml
@@ -42,6 +42,8 @@ spec:
       labels:
         app: vllm-llama3-8b-instruct-epp
     spec:
+      # Conservatively, this timeout should mirror the longest grace period of the pods within the pool
+      terminationGracePeriodSeconds: 130
       containers:
       - name: epp
         image: us-central1-docker.pkg.dev/k8s-staging-images/gateway-api-inference-extension/epp:main
diff --git a/config/manifests/vllm/gpu-deployment.yaml b/config/manifests/vllm/gpu-deployment.yaml
index 3386a791..e7cb193e 100644
--- a/config/manifests/vllm/gpu-deployment.yaml
+++ b/config/manifests/vllm/gpu-deployment.yaml
@@ -77,7 +77,7 @@ spec:
               #exec:
               #  command:
               #  - /usr/bin/sleep
-              #  - 30
+              #  - "30"
           livenessProbe:
             httpGet:
               path: /health
@@ -133,7 +133,6 @@ spec:
               path: /health
               port: http
               scheme: HTTP
-
           resources:
             limits:
               nvidia.com/gpu: 1
@@ -235,12 +234,12 @@ spec:
           emptyDir: {}
         - name: config-volume
           configMap:
-            name: vllm-llama3-8b-adapters
+            name: vllm-llama3-8b-instruct-adapters
 ---
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: vllm-llama3-8b-adapters
+  name: vllm-llama3-8b-instruct-adapters
 data:
   configmap.yaml: |
       vLLMLoRAConfig:
diff --git a/hack/push-chart.sh b/hack/push-chart.sh
index e0938af4..36ed92cd 100755
--- a/hack/push-chart.sh
+++ b/hack/push-chart.sh
@@ -30,7 +30,7 @@ CHART=${CHART:-inferencepool}
 
 HELM=${HELM:-./bin/helm}
 
-readonly semver_regex='^v([0-9]+)(\.[0-9]+){1,2}$'
+readonly semver_regex='^v([0-9]+)(\.[0-9]+){1,2}(-rc.[0-9]+)?$'
 
 chart_version=${CHART_VERSION}
 if [[ ${EXTRA_TAG} =~ ${semver_regex} ]]
diff --git a/hack/release-quickstart.sh b/hack/release-quickstart.sh
index 832bd872..c2c0f74d 100755
--- a/hack/release-quickstart.sh
+++ b/hack/release-quickstart.sh
@@ -36,19 +36,26 @@ sed -i.bak -E "s|(releases/download/)v[0-9]+\.[0-9]+\.0-rc\.?[0-9]+|\1${RELEASE_
 sed -i.bak "s|kubectl apply -k https://github.com/kubernetes-sigs/gateway-api-inference-extension/config/crd|kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/releases/download/${RELEASE_TAG}/manifests.yaml|g" "$README"
 
 # -----------------------------------------------------------------------------
-# Update config/manifests/ext_proc.yaml
+# Update image references
 # -----------------------------------------------------------------------------
-EXT_PROC="config/manifests/ext_proc.yaml"
-echo "Updating ${EXT_PROC} ..."
+EPP="config/manifests/inferencepool-resources.yaml"
+#TODO: Put all helm values files into an array to loop over
+EPP_HELM="config/charts/inferencepool/values.yaml"
+BBR_HELM="config/charts/body-based-routing/values.yaml"
+echo "Updating ${EPP} & ${EPP_HELM} ..."
 
-# Update the EPP container tag.
-sed -i.bak -E "s|(us-central1-docker\.pkg\.dev/k8s-staging-images/gateway-api-inference-extension/epp:)[^\"[:space:]]+|\1${RELEASE_TAG}|g" "$EXT_PROC"
+# Update the container tag.
+sed -i.bak -E "s|(us-central1-docker\.pkg\.dev/k8s-staging-images/gateway-api-inference-extension/epp:)[^\"[:space:]]+|\1${RELEASE_TAG}|g" "$EPP"
+sed -i.bak -E "s|(tag: )[^\"[:space:]]+|\1${RELEASE_TAG}|g" "$EPP_HELM"
+sed -i.bak -E "s|(tag: )[^\"[:space:]]+|\1${RELEASE_TAG}|g" "$BBR_HELM"
 
-# Update the EPP container image pull policy.
-sed -i.bak '/us-central1-docker.pkg.dev\/k8s-staging-images\/gateway-api-inference-extension\/epp/ { n; s/Always/IfNotPresent/ }' "$EXT_PROC"
+# Update the container image pull policy.
+sed -i.bak '/us-central1-docker.pkg.dev\/k8s-staging-images\/gateway-api-inference-extension\/epp/ { n; s/Always/IfNotPresent/ }' "$EPP"
 
-# Update the EPP container registry.
-sed -i.bak -E "s|us-central1-docker\.pkg\.dev/k8s-staging-images|registry.k8s.io|g" "$EXT_PROC"
+# Update the container registry.
+sed -i.bak -E "s|us-central1-docker\.pkg\.dev/k8s-staging-images|registry.k8s.io|g" "$EPP"
+sed -i.bak -E "s|us-central1-docker\.pkg\.dev/k8s-staging-images|registry.k8s.io|g" "$EPP_HELM"
+sed -i.bak -E "s|us-central1-docker\.pkg\.dev/k8s-staging-images|registry.k8s.io|g" "$BBR_HELM"
 
 # -----------------------------------------------------------------------------
 # Update config/manifests/vllm/gpu-deployment.yaml
@@ -65,8 +72,8 @@ sed -i.bak '/vllm\/vllm-openai/ { n; s/Always/IfNotPresent/ }' "$VLLM_DEPLOY"
 # -----------------------------------------------------------------------------
 # Stage the changes
 # -----------------------------------------------------------------------------
-echo "Staging $README $EXT_PROC $VLLM_DEPLOY files..."
-git add $README $EXT_PROC $VLLM_DEPLOY
+echo "Staging $README $EPP $EPP_HELM $BBR_HELM $VLLM_DEPLOY files..."
+git add $README $EPP $EPP_HELM $BBR_HELM $VLLM_DEPLOY
 
 # -----------------------------------------------------------------------------
 # Cleanup backup files and finish
diff --git a/pkg/epp/backend/metrics/logger.go b/pkg/epp/backend/metrics/logger.go
index 8c73d488..d71dc3fa 100644
--- a/pkg/epp/backend/metrics/logger.go
+++ b/pkg/epp/backend/metrics/logger.go
@@ -110,4 +110,5 @@ func flushPrometheusMetricsOnce(logger logr.Logger, datastore Datastore) {
 	podTotalCount := len(podMetrics)
 	metrics.RecordInferencePoolAvgKVCache(pool.Name, kvCacheTotal/float64(podTotalCount))
 	metrics.RecordInferencePoolAvgQueueSize(pool.Name, float64(queueTotal/podTotalCount))
+	metrics.RecordinferencePoolReadyPods(pool.Name, float64(podTotalCount))
 }
diff --git a/pkg/epp/handlers/server.go b/pkg/epp/handlers/server.go
index cd354c2f..a92f091c 100644
--- a/pkg/epp/handlers/server.go
+++ b/pkg/epp/handlers/server.go
@@ -228,6 +228,7 @@ type RequestContext struct {
 	ResponseSize              int
 	ResponseComplete          bool
 	ResponseStatusCode        string
+	RequestRunning            bool
 
 	RequestState         StreamRequestState
 	modelServerStreaming bool
diff --git a/pkg/epp/handlers/streamingserver.go b/pkg/epp/handlers/streamingserver.go
index d704578a..874dd734 100644
--- a/pkg/epp/handlers/streamingserver.go
+++ b/pkg/epp/handlers/streamingserver.go
@@ -81,13 +81,16 @@ func (s *StreamingServer) Process(srv extProcPb.ExternalProcessor_ProcessServer)
 	// error metrics. This doesn't cover the error "Cannot receive stream request" because
 	// such errors might happen even though response is processed.
 	var err error
-	defer func(error) {
+	defer func(error, *RequestContext) {
 		if reqCtx.ResponseStatusCode != "" {
 			metrics.RecordRequestErrCounter(reqCtx.Model, reqCtx.ResolvedTargetModel, reqCtx.ResponseStatusCode)
 		} else if err != nil {
 			metrics.RecordRequestErrCounter(reqCtx.Model, reqCtx.ResolvedTargetModel, errutil.CanonicalCode(err))
 		}
-	}(err)
+		if reqCtx.RequestRunning {
+			metrics.DecRunningRequests(reqCtx.Model)
+		}
+	}(err, reqCtx)
 
 	for {
 		select {
@@ -269,6 +272,8 @@ func (r *RequestContext) updateStateAndSendIfNeeded(srv extProcPb.ExternalProces
 			return status.Errorf(codes.Unknown, "failed to send response back to Envoy: %v", err)
 		}
 		r.RequestState = BodyRequestResponsesComplete
+		metrics.IncRunningRequests(r.Model)
+		r.RequestRunning = true
 		// Dump the response so a new stream message can begin
 		r.reqBodyResp = nil
 	}
diff --git a/pkg/epp/metrics/metrics.go b/pkg/epp/metrics/metrics.go
index e86ca901..434b8381 100644
--- a/pkg/epp/metrics/metrics.go
+++ b/pkg/epp/metrics/metrics.go
@@ -121,6 +121,16 @@ var (
 		[]string{"model_name", "target_model_name"},
 	)
 
+	runningRequests = compbasemetrics.NewGaugeVec(
+		&compbasemetrics.GaugeOpts{
+			Subsystem:      InferenceModelComponent,
+			Name:           "running_requests",
+			Help:           "Inference model number of running requests in each model.",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"model_name"},
+	)
+
 	// Inference Pool Metrics
 	inferencePoolAvgKVCache = compbasemetrics.NewGaugeVec(
 		&compbasemetrics.GaugeOpts{
@@ -141,6 +151,16 @@ var (
 		},
 		[]string{"name"},
 	)
+
+	inferencePoolReadyPods = compbasemetrics.NewGaugeVec(
+		&compbasemetrics.GaugeOpts{
+			Subsystem:      InferencePoolComponent,
+			Name:           "ready_pods",
+			Help:           "The number of ready pods in the inference server pool.",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"name"},
+	)
 )
 
 var registerMetrics sync.Once
@@ -155,9 +175,11 @@ func Register() {
 		legacyregistry.MustRegister(responseSizes)
 		legacyregistry.MustRegister(inputTokens)
 		legacyregistry.MustRegister(outputTokens)
+		legacyregistry.MustRegister(runningRequests)
 
 		legacyregistry.MustRegister(inferencePoolAvgKVCache)
 		legacyregistry.MustRegister(inferencePoolAvgQueueSize)
+		legacyregistry.MustRegister(inferencePoolReadyPods)
 	})
 }
 
@@ -209,6 +231,20 @@ func RecordOutputTokens(modelName, targetModelName string, size int) {
 	}
 }
 
+// IncRunningRequests increases the current running requests.
+func IncRunningRequests(modelName string) {
+	if modelName != "" {
+		runningRequests.WithLabelValues(modelName).Inc()
+	}
+}
+
+// DecRunningRequests decreases the current running requests.
+func DecRunningRequests(modelName string) {
+	if modelName != "" {
+		runningRequests.WithLabelValues(modelName).Dec()
+	}
+}
+
 func RecordInferencePoolAvgKVCache(name string, utilization float64) {
 	inferencePoolAvgKVCache.WithLabelValues(name).Set(utilization)
 }
@@ -216,3 +252,7 @@ func RecordInferencePoolAvgKVCache(name string, utilization float64) {
 func RecordInferencePoolAvgQueueSize(name string, queueSize float64) {
 	inferencePoolAvgQueueSize.WithLabelValues(name).Set(queueSize)
 }
+
+func RecordinferencePoolReadyPods(name string, runningPods float64) {
+	inferencePoolReadyPods.WithLabelValues(name).Set(runningPods)
+}
diff --git a/site-src/guides/index.md b/site-src/guides/index.md
index 4548d5cd..f1545438 100644
--- a/site-src/guides/index.md
+++ b/site-src/guides/index.md
@@ -7,11 +7,12 @@
 This quickstart guide is intended for engineers familiar with k8s and model servers (vLLM in this instance). The goal of this guide is to get an Inference Gateway up and running! 
 
 ## **Prerequisites**
- - A cluster with:
-    - Support for services of type `LoadBalancer`. (This can be validated by ensuring your Envoy Gateway is up and running).
-   For example, with Kind, you can follow [these steps](https://kind.sigs.k8s.io/docs/user/loadbalancer).
-    - Support for [sidecar containers](https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/) (enabled by default since Kubernetes v1.29)
-   to run the model server deployment.
+
+- A cluster with:
+  - Support for services of type `LoadBalancer`. For kind clusters, follow [this guide](https://kind.sigs.k8s.io/docs/user/loadbalancer)
+  to get services of type LoadBalancer working.
+  - Support for [sidecar containers](https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/) (enabled by default since Kubernetes v1.29)
+  to run the model server deployment.
 
 ## **Steps**
 
@@ -105,6 +106,24 @@ This quickstart guide is intended for engineers familiar with k8s and model serv
          inference-gateway   inference-gateway   <MY_ADDRESS>    True         22s
          ```
 
+      3. Deploy the HTTPRoute
+
+         ```bash
+         kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/gateway/gke/httproute.yaml
+         ```
+
+      4. Confirm that the HTTPRoute status conditions include `Accepted=True` and `ResolvedRefs=True`:
+
+         ```bash
+         kubectl get httproute llm-route -o yaml
+         ```
+
+      5. Given that the default connection timeout may be insufficient for most inference workloads, it is recommended to configure a timeout appropriate for your intended use case.
+
+      ```bash
+      kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/gateway/gke/gcp-backend-policy.yaml
+      ```
+
 === "Istio"
 
       Please note that this feature is currently in an experimental phase and is not intended for production use. 
@@ -114,7 +133,7 @@ This quickstart guide is intended for engineers familiar with k8s and model serv
 
          - Gateway API [CRDs](https://gateway-api.sigs.k8s.io/guides/#installing-gateway-api) installed.
 
-      1. Install Istio
+      2. Install Istio
       
          ```
          TAG=1.26-alpha.80c74f7f43482c226f4f4b10b4dda6261b67a71f
@@ -131,19 +150,19 @@ This quickstart guide is intended for engineers familiar with k8s and model serv
          ./istioctl install --set tag=$TAG --set hub=gcr.io/istio-testing
          ```
 
-      1. If you run the Endpoint Picker (EPP) with the `--secureServing` flag set to `true` (the default mode), it is currently using a self-signed certificate. As a security measure, Istio does not trust self-signed certificates by default. As a temporary workaround, you can apply the destination rule to bypass TLS verification for EPP. A more secure TLS implementation in EPP is being discussed in [Issue 582](https://github.com/kubernetes-sigs/gateway-api-inference-extension/issues/582).
+      3. If you run the Endpoint Picker (EPP) with the `--secureServing` flag set to `true` (the default mode), it is currently using a self-signed certificate. As a security measure, Istio does not trust self-signed certificates by default. As a temporary workaround, you can apply the destination rule to bypass TLS verification for EPP. A more secure TLS implementation in EPP is being discussed in [Issue 582](https://github.com/kubernetes-sigs/gateway-api-inference-extension/issues/582).
 
          ```bash
          kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/gateway/istio/destination-rule.yaml
          ```
 
-      1. Deploy Gateway
+      4. Deploy Gateway
 
          ```bash
          kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/gateway/istio/gateway.yaml
          ```
 
-      1. Label the gateway
+      5. Label the gateway
 
          ```bash
          kubectl label gateway llm-gateway istio.io/enable-inference-extproc=true
@@ -156,9 +175,21 @@ This quickstart guide is intended for engineers familiar with k8s and model serv
          inference-gateway   inference-gateway   <MY_ADDRESS>    True         22s
          ```
 
+      6. Deploy the HTTPRoute
+
+         ```bash
+         kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/gateway/istio/httproute.yaml
+         ```
+
+      7. Confirm that the HTTPRoute status conditions include `Accepted=True` and `ResolvedRefs=True`:
+
+         ```bash
+         kubectl get httproute llm-route -o yaml
+         ```
+
 === "Kgateway"
 
-      [Kgateway](https://kgateway.dev/) v2.0.0 adds support for inference extension as a **technical preview**. This means do not
+      [Kgateway](https://kgateway.dev/) recently added support for inference extension as a **technical preview**. This means do not
       run Kgateway with inference extension in production environments. Refer to [Issue 10411](https://github.com/kgateway-dev/kgateway/issues/10411)
       for the list of caveats, supported features, etc.
 
@@ -167,20 +198,20 @@ This quickstart guide is intended for engineers familiar with k8s and model serv
          - [Helm](https://helm.sh/docs/intro/install/) installed.
          - Gateway API [CRDs](https://gateway-api.sigs.k8s.io/guides/#installing-gateway-api) installed.
 
-      1. Install Kgateway CRDs
+      2. Set the Kgateway version and install the Kgateway CRDs.
 
          ```bash
-         helm upgrade -i --create-namespace --namespace kgateway-system --version $VERSION kgateway-crds oci://cr.kgateway.dev/kgateway-dev/charts/kgateway-crds
+         KGTW_VERSION=v2.0.0-rc.2
+         helm upgrade -i --create-namespace --namespace kgateway-system --version $KGTW_VERSION kgateway-crds oci://cr.kgateway.dev/kgateway-dev/charts/kgateway-crds
          ```
 
-      1. Install Kgateway
+      3. Install Kgateway
 
          ```bash
-         helm upgrade -i --namespace kgateway-system --version $VERSION kgateway oci://cr.kgateway.dev/kgateway-dev/charts/kgateway
-         --set inferenceExtension.enabled=true
+         helm upgrade -i --namespace kgateway-system --version $KGTW_VERSION kgateway oci://cr.kgateway.dev/kgateway-dev/charts/kgateway --set inferenceExtension.enabled=true
          ```
 
-      1. Deploy Gateway
+      4. Deploy the Gateway
 
          ```bash
          kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/gateway/kgateway/gateway.yaml
@@ -193,50 +224,50 @@ This quickstart guide is intended for engineers familiar with k8s and model serv
          inference-gateway   kgateway            <MY_ADDRESS>    True         22s
          ```
 
-### Deploy the HTTPRoute
+      5. Deploy the HTTPRoute
 
-   ```bash
-   kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/gateway/httproute.yaml
-   ```
+         ```bash
+         kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/gateway/kgateway/httproute.yaml
+         ```
 
-### Configure Timeouts
+      6. Confirm that the HTTPRoute status conditions include `Accepted=True` and `ResolvedRefs=True`:
 
-   Given that default timeouts for above implementations may be insufficient for most inference workloads, it is recommended to configure a timeout appropriate for your intended use case.
+         ```bash
+         kubectl get httproute llm-route -o yaml
+         ```
 
-=== "GKE"
+### Try it out
 
-      ```bash
-      kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/gateway/gke/gcp-backend-policy.yaml
-      ```
+   Wait until the gateway is ready.
 
-=== "Istio"
+=== "GPU-Based Model Server"
 
       ```bash
-      kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/gateway/httproute-with-timeout.yaml
+      IP=$(kubectl get gateway/inference-gateway -o jsonpath='{.status.addresses[0].value}')
+      PORT=80
+
+      curl -i ${IP}:${PORT}/v1/completions -H 'Content-Type: application/json' -d '{
+      "model": "food-review",
+      "prompt": "Write as if you were a critic: San Francisco",
+      "max_tokens": 100,
+      "temperature": 0
+      }'
       ```
 
-=== "Kgateway"
+=== "CPU-Based Model Server"
 
       ```bash
-      kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/gateway/httproute-with-timeout.yaml
+      IP=$(kubectl get gateway/inference-gateway -o jsonpath='{.status.addresses[0].value}')
+      PORT=80
+
+      curl -i ${IP}:${PORT}/v1/completions -H 'Content-Type: application/json' -d '{
+      "model": "Qwen/Qwen2.5-1.5B-Instruct",
+      "prompt": "Write as if you were a critic: San Francisco",
+      "max_tokens": 100,
+      "temperature": 0
+      }'
       ```
 
-### Try it out
-
-   Wait until the gateway is ready.
-
-   ```bash
-   IP=$(kubectl get gateway/inference-gateway -o jsonpath='{.status.addresses[0].value}')
-   PORT=80
-
-   curl -i ${IP}:${PORT}/v1/completions -H 'Content-Type: application/json' -d '{
-   "model": "food-review",
-   "prompt": "Write as if you were a critic: San Francisco",
-   "max_tokens": 100,
-   "temperature": 0
-   }'
-   ```
-
 ### Cleanup
 
    The following cleanup assumes you would like to clean ALL resources that were created in this quickstart guide.  
@@ -258,10 +289,12 @@ This quickstart guide is intended for engineers familiar with k8s and model serv
       kubectl delete -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/gateway/gke/gateway.yaml --ignore-not-found
       kubectl delete -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/gateway/gke/healthcheck.yaml --ignore-not-found
       kubectl delete -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/gateway/gke/gcp-backend-policy.yaml --ignore-not-found
+      kubectl delete -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/gateway/gke/httproute.yaml --ignore-not-found
       kubectl delete -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/gateway/istio/gateway.yaml --ignore-not-found
       kubectl delete -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/gateway/istio/destination-rule.yaml --ignore-not-found
+      kubectl delete -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/gateway/istio/httproute.yaml --ignore-not-found
       kubectl delete -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/gateway/kgateway/gateway.yaml --ignore-not-found
-      kubectl delete -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/gateway/httproute.yaml --ignore-not-found
+      kubectl delete -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/gateway/kgateway/httproute.yaml --ignore-not-found
       ```
 
    1. Uninstall the CRDs
diff --git a/site-src/guides/metrics.md b/site-src/guides/metrics.md
index fca43dd6..a781f721 100644
--- a/site-src/guides/metrics.md
+++ b/site-src/guides/metrics.md
@@ -30,8 +30,10 @@ curl -i ${IP}:${PORT}/v1/completions -H 'Content-Type: application/json' -d '{
 | inference_model_response_sizes               | Distribution     | Distribution of response size in bytes.                           | `model_name`=&lt;model-name&gt; <br> `target_model_name`=&lt;target-model-name&gt; | ALPHA       |
 | inference_model_input_tokens                 | Distribution     | Distribution of input token count.                                | `model_name`=&lt;model-name&gt; <br> `target_model_name`=&lt;target-model-name&gt; | ALPHA       |
 | inference_model_output_tokens                | Distribution     | Distribution of output token count.                               | `model_name`=&lt;model-name&gt; <br> `target_model_name`=&lt;target-model-name&gt; | ALPHA       |
+| inference_model_running_requests                | Gauge     | Number of running requests for each model.             | `model_name`=&lt;model-name&gt;  | ALPHA       |
 | inference_pool_average_kv_cache_utilization  | Gauge            | The average kv cache utilization for an inference server pool.    | `name`=&lt;inference-pool-name&gt;                                                 | ALPHA       |
 | inference_pool_average_queue_size            | Gauge            | The average number of requests pending in the model server queue. | `name`=&lt;inference-pool-name&gt;                                                 | ALPHA       |
+| inference_pool_ready_pods                    | Gauge            | The number of ready pods for an inference server pool.            | `name`=&lt;inference-pool-name&gt;                                                 | ALPHA       |
 
 ## Scrape Metrics
 
diff --git a/site-src/images/logo/logo-text-xl-dark.png b/site-src/images/logo/logo-text-xl-dark.png
new file mode 100644
index 00000000..4d878e5c
Binary files /dev/null and b/site-src/images/logo/logo-text-xl-dark.png differ
diff --git a/site-src/implementations.md b/site-src/implementations.md
index 89acb436..8a95119d 100644
--- a/site-src/implementations.md
+++ b/site-src/implementations.md
@@ -54,3 +54,12 @@ Issue](https://github.com/GoogleCloudPlatform/gke-gateway-api/issues/20).
 [gke-gateway]:https://cloud.google.com/kubernetes-engine/docs/concepts/gateway-api
 [gke-gateway-deploy]:https://cloud.google.com/kubernetes-engine/docs/how-to/deploying-gateways
 [gke-multi-cluster-gateway]:https://cloud.google.com/kubernetes-engine/docs/how-to/deploying-multi-cluster-gateways
+
+## Istio
+
+[Istio](https://istio.io/) is an open source service mesh and gateway implementation.
+It provides a fully compliant implementation of the Kubernetes Gateway API for cluster ingress traffic control. 
+For service mesh users, Istio also fully supports east-west (including [GAMMA](https://gateway-api.sigs.k8s.io/mesh/)) traffic management within the mesh.
+
+Gateway API Inference Extension support is being tracked by this [GitHub
+Issue](https://github.com/istio/istio/issues/55768).
diff --git a/tools/dynamic-lora-sidecar/README.md b/tools/dynamic-lora-sidecar/README.md
index 4e85fd92..65dc0d78 100644
--- a/tools/dynamic-lora-sidecar/README.md
+++ b/tools/dynamic-lora-sidecar/README.md
@@ -83,7 +83,7 @@ In this example, both adapters will use `meta-llama/Llama-3.1-8B-Instruct` as th
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: vllm-llama3-8b-adapters
+  name: vllm-llama3-8b-instruct-adapters
 data:
   configmap.yaml: |
       vLLMLoRAConfig:



Test patch:
diff --git a/pkg/epp/metrics/metrics_test.go b/pkg/epp/metrics/metrics_test.go
index c2436bab..dc4c7044 100644
--- a/pkg/epp/metrics/metrics_test.go
+++ b/pkg/epp/metrics/metrics_test.go
@@ -36,6 +36,7 @@ const (
 	ResponseSizesMetric     = InferenceModelComponent + "_response_sizes"
 	InputTokensMetric       = InferenceModelComponent + "_input_tokens"
 	OutputTokensMetric      = InferenceModelComponent + "_output_tokens"
+	RunningRequestsMetric   = InferenceModelComponent + "_running_requests"
 	KVCacheAvgUsageMetric   = InferencePoolComponent + "_average_kv_cache_utilization"
 	QueueAvgSizeMetric      = InferencePoolComponent + "_average_queue_size"
 )
@@ -345,6 +346,66 @@ func TestRecordResponseMetrics(t *testing.T) {
 	}
 }
 
+func TestRunningRequestsMetrics(t *testing.T) {
+	type request struct {
+		modelName string
+		complete  bool // true -> request is completed, false -> running request
+	}
+
+	scenarios := []struct {
+		name     string
+		requests []request
+	}{
+		{
+			name: "basic test",
+			requests: []request{
+				{
+					modelName: "m1",
+					complete:  false,
+				},
+				{
+					modelName: "m1",
+					complete:  false,
+				},
+				{
+					modelName: "m1",
+					complete:  true,
+				},
+				{
+					modelName: "m2",
+					complete:  false,
+				},
+			},
+		},
+	}
+
+	Register()
+	for _, scenario := range scenarios {
+		t.Run(scenario.name, func(t *testing.T) {
+			for _, req := range scenario.requests {
+				if req.complete {
+					DecRunningRequests(req.modelName)
+				} else {
+					IncRunningRequests(req.modelName)
+				}
+			}
+
+			wantRunningRequests, err := os.Open("testdata/running_requests_metrics")
+			defer func() {
+				if err := wantRunningRequests.Close(); err != nil {
+					t.Error(err)
+				}
+			}()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, wantRunningRequests, RunningRequestsMetric); err != nil {
+				t.Error(err)
+			}
+		})
+	}
+}
+
 func TestInferencePoolMetrics(t *testing.T) {
 	scenarios := []struct {
 		name         string
diff --git a/pkg/epp/metrics/testdata/running_requests_metrics b/pkg/epp/metrics/testdata/running_requests_metrics
new file mode 100644
index 00000000..a880e499
--- /dev/null
+++ b/pkg/epp/metrics/testdata/running_requests_metrics
@@ -0,0 +1,4 @@
+# HELP inference_model_running_requests [ALPHA] Inference model number of running requests in each model.
+# TYPE inference_model_running_requests gauge
+inference_model_running_requests{model_name="m1"} 1
+inference_model_running_requests{model_name="m2"} 1
diff --git a/test/e2e/epp/e2e_test.go b/test/e2e/epp/e2e_test.go
index 09c8835a..e86b2d49 100644
--- a/test/e2e/epp/e2e_test.go
+++ b/test/e2e/epp/e2e_test.go
@@ -94,11 +94,11 @@ var _ = ginkgo.Describe("InferencePool", func() {
 func newInferenceModel(ns string) *v1alpha2.InferenceModel {
 	targets := []v1alpha2.TargetModel{
 		{
-			Name:   modelName + "-0",
+			Name:   modelName,
 			Weight: ptr.To(int32(50)),
 		},
 		{
-			Name:   modelName + "-1",
+			Name:   "cad-fabricator",
 			Weight: ptr.To(int32(50)),
 		},
 	}
diff --git a/test/integration/epp/hermetic_test.go b/test/integration/epp/hermetic_test.go
index 8e02aca4..2acdacf8 100644
--- a/test/integration/epp/hermetic_test.go
+++ b/test/integration/epp/hermetic_test.go
@@ -430,7 +430,13 @@ func TestFullDuplexStreamed_KubeInferenceModelRequest(t *testing.T) {
 					# HELP inference_model_request_total [ALPHA] Counter of inference model requests broken out for each model and target model.
 					# TYPE inference_model_request_total counter
 					inference_model_request_total{model_name="my-model",target_model_name="my-model-12345"} 1
-					`},
+					`,
+				`inference_pool_ready_pods`: `
+					# HELP inference_pool_ready_pods [ALPHA] The number of ready pods in the inference server pool.
+					# TYPE inference_pool_ready_pods gauge
+					inference_pool_ready_pods{name="vllm-llama3-8b-instruct-pool"} 3
+					`,
+			},
 			wantErr: false,
 			wantResponses: []*extProcPb.ProcessingResponse{
 				{
@@ -1465,6 +1471,11 @@ func TestFullDuplexStreamed_KubeInferenceModelRequest(t *testing.T) {
 					},
 				},
 			},
+			wantMetrics: map[string]string{`inference_pool_ready_pods`: `
+				# HELP inference_pool_ready_pods [ALPHA] The number of ready pods in the inference server pool.
+				# TYPE inference_pool_ready_pods gauge
+				inference_pool_ready_pods{name="vllm-llama3-8b-instruct-pool"} 1
+				`},
 		},
 	}
 
diff --git a/test/testdata/envoy.yaml b/test/testdata/envoy.yaml
index fc32b5aa..62e6b4c5 100644
--- a/test/testdata/envoy.yaml
+++ b/test/testdata/envoy.yaml
@@ -104,10 +104,11 @@ data:
                             timeout: 10s
                           processing_mode:
                             request_header_mode: SEND
-                            response_header_mode: SKIP
-                            request_body_mode: BUFFERED
-                            request_trailer_mode: SKIP
-                            response_trailer_mode: SKIP
+                            response_header_mode: SEND
+                            request_body_mode: FULL_DUPLEX_STREAMED
+                            response_body_mode: FULL_DUPLEX_STREAMED
+                            request_trailer_mode: SEND
+                            response_trailer_mode: SEND
                           message_timeout: 1000s
                         # Mark it as disabled if needed for troubleshooting:
                         # disabled: true
@@ -221,7 +222,7 @@ spec:
     spec:
       containers:
       - name: envoy
-        image: docker.io/envoyproxy/envoy:distroless-v1.32.2
+        image: docker.io/envoyproxy/envoy:distroless-v1.33.2
         args:
           - "--service-cluster" 
           - "default/inference-gateway"