Repository: kubernetes-sigs/gateway-api-inference-extension

Issue:
Add support for PrepareRequestData and AdmitRequest plugins
This is to evolve the EPP datalayer based on the suggestions in [Evolving the EPP framework](https://docs.google.com/document/d/1EQwXL2pCuUyM1B917FUgP_8pFS3VF8F_bUfjy8IE7gM/edit?tab=t.vmaefhinvkl5#heading=h.s9mr6kynb3ls) doc.

Hints from issue comments:
@rahulgurnani: /assign@rahulgurnani: /assign@rahulgurnani: /milestone Data Layer Rework@rahulgurnani: /milestone Data Layer Rework


Solution patch:
diff --git a/README.md b/README.md
index 19e801b..01d7502 100644
--- a/README.md
+++ b/README.md
@@ -112,7 +112,7 @@ Follow this [README](./test/e2e/epp/README.md) to learn more about running the i
 
 ## Contributing
 
-Our community meeting is weekly at Thursday 10AM PDT ([Zoom](https://zoom.us/j/9955436256?pwd=Z2FQWU1jeDZkVC9RRTN4TlZyZTBHZz09), [Meeting Notes](https://www.google.com/url?q=https://docs.google.com/document/d/1frfPE5L1sI3737rdQV04IcDGeOcGJj2ItjMg6z2SRH0/edit?usp%3Dsharing&sa=D&source=calendar&usd=2&usg=AOvVaw1pUVy7UN_2PMj8qJJcFm1U)).
+Our community meeting is weekly at Thursday 10AM PDT ([Zoom](https://zoom.us/j/96271651417?pwd=NViXawg6lMsRjgXbu2YmW8DxWqbjta.1), [Meeting Notes](https://www.google.com/url?q=https://docs.google.com/document/d/1frfPE5L1sI3737rdQV04IcDGeOcGJj2ItjMg6z2SRH0/edit?usp%3Dsharing&sa=D&source=calendar&usd=2&usg=AOvVaw1pUVy7UN_2PMj8qJJcFm1U)).
 
 We currently utilize the [#gateway-api-inference-extension](https://kubernetes.slack.com/?redir=%2Fmessages%2Fgateway-api-inference-extension) channel in Kubernetes Slack workspace for communications.
 
diff --git a/pkg/epp/datalayer/metrics/extractor.go b/pkg/epp/datalayer/metrics/extractor.go
index 562f222..ab69912 100644
--- a/pkg/epp/datalayer/metrics/extractor.go
+++ b/pkg/epp/datalayer/metrics/extractor.go
@@ -28,6 +28,7 @@ import (
 	dto "github.com/prometheus/client_model/go"
 
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/datalayer"
+	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/metrics"
 )
 
 const (
@@ -47,6 +48,17 @@ type Extractor struct {
 	mapping *Mapping
 }
 
+func Produces() map[string]any {
+	return map[string]any{
+		metrics.WaitingQueueSizeKey:    int(0),
+		metrics.KVCacheUsagePercentKey: float64(0),
+		metrics.ActiveModelsKey:        map[string]int{},
+		metrics.WaitingModelsKey:       map[string]int{},
+		metrics.MaxActiveModelsKey:     int(0),
+		metrics.UpdateTimeKey:          time.Time{},
+	}
+}
+
 // NewExtractor returns a new model server protocol (MSP) metrics extractor,
 // configured with the given metrics' specifications.
 // These are mandatory metrics per the MSP specification, and are used
diff --git a/pkg/epp/metrics/metrics.go b/pkg/epp/metrics/metrics.go
index db361a4..df50794 100644
--- a/pkg/epp/metrics/metrics.go
+++ b/pkg/epp/metrics/metrics.go
@@ -34,6 +34,13 @@ const (
 	InferenceObjectiveComponent = "inference_objective"
 	InferencePoolComponent      = "inference_pool"
 	InferenceExtension          = "inference_extension"
+
+	KVCacheUsagePercentKey = "KVCacheUsagePercent"
+	WaitingQueueSizeKey    = "WaitingQueueSize"
+	MaxActiveModelsKey     = "MaxActiveModels"
+	ActiveModelsKey        = "ActiveModels"
+	WaitingModelsKey       = "WaitingModels"
+	UpdateTimeKey          = "UpdateTime"
 )
 
 var (
diff --git a/pkg/epp/plugins/plugins.go b/pkg/epp/plugins/plugins.go
index 93ed23d..3f212c7 100644
--- a/pkg/epp/plugins/plugins.go
+++ b/pkg/epp/plugins/plugins.go
@@ -22,3 +22,21 @@ type Plugin interface {
 	// TypedName returns the type and name tuple of this plugin instance.
 	TypedName() TypedName
 }
+
+// ConsumerPlugin defines the interface for a consumer.
+type ConsumerPlugin interface {
+	Plugin
+	// Consumes returns data consumed by the plugin.
+	// This is a map from data key (string) produced to
+	// the data type of the key (represented as data with default value casted as any field).
+	Consumes() map[string]any
+}
+
+// ProducerPlugin defines the interface for a producer.
+type ProducerPlugin interface {
+	Plugin
+	// Produces returns data produced by the producer.
+	// This is a map from data key (string) produced to
+	// the data type of the key (represented as data with default value casted as any field).
+	Produces() map[string]any
+}
diff --git a/pkg/epp/scheduling/framework/plugins/scorer/kvcache_utilization.go b/pkg/epp/scheduling/framework/plugins/scorer/kvcache_utilization.go
index c58f635..d78bcc9 100644
--- a/pkg/epp/scheduling/framework/plugins/scorer/kvcache_utilization.go
+++ b/pkg/epp/scheduling/framework/plugins/scorer/kvcache_utilization.go
@@ -20,6 +20,7 @@ import (
 	"context"
 	"encoding/json"
 
+	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/metrics"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/plugins"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/types"
@@ -54,6 +55,13 @@ func (s *KVCacheUtilizationScorer) TypedName() plugins.TypedName {
 	return s.typedName
 }
 
+// Consumes returns the list of data that is consumed by the plugin.
+func (s *KVCacheUtilizationScorer) Consumes() map[string]any {
+	return map[string]any{
+		metrics.KVCacheUsagePercentKey: float64(0),
+	}
+}
+
 // WithName sets the name of the scorer.
 func (s *KVCacheUtilizationScorer) WithName(name string) *KVCacheUtilizationScorer {
 	s.typedName.Name = name
diff --git a/pkg/epp/scheduling/framework/plugins/scorer/lora_affinity.go b/pkg/epp/scheduling/framework/plugins/scorer/lora_affinity.go
index d995bae..8052479 100644
--- a/pkg/epp/scheduling/framework/plugins/scorer/lora_affinity.go
+++ b/pkg/epp/scheduling/framework/plugins/scorer/lora_affinity.go
@@ -20,6 +20,7 @@ import (
 	"context"
 	"encoding/json"
 
+	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/metrics"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/plugins"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/types"
@@ -54,6 +55,14 @@ func (s *LoraAffinityScorer) TypedName() plugins.TypedName {
 	return s.tn
 }
 
+// Consumes returns the list of data that is consumed by the plugin.
+func (s *LoraAffinityScorer) Consumes() map[string]any {
+	return map[string]any{
+		metrics.ActiveModelsKey:  map[string]int{},
+		metrics.WaitingModelsKey: map[string]int{},
+	}
+}
+
 // WithName sets the name of the scorer.
 func (s *LoraAffinityScorer) WithName(name string) *LoraAffinityScorer {
 	s.tn.Name = name
diff --git a/pkg/epp/scheduling/framework/plugins/scorer/queue.go b/pkg/epp/scheduling/framework/plugins/scorer/queue.go
index e2a1349..e2a07b0 100644
--- a/pkg/epp/scheduling/framework/plugins/scorer/queue.go
+++ b/pkg/epp/scheduling/framework/plugins/scorer/queue.go
@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"math"
 
+	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/metrics"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/plugins"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/types"
@@ -56,6 +57,13 @@ func (s *QueueScorer) TypedName() plugins.TypedName {
 	return s.typedName
 }
 
+// Consumes returns the list of data that is consumed by the plugin.
+func (s *QueueScorer) Consumes() map[string]any {
+	return map[string]any{
+		metrics.WaitingQueueSizeKey: int(0),
+	}
+}
+
 // WithName sets the name of the scorer.
 func (s *QueueScorer) WithName(name string) *QueueScorer {
 	s.typedName.Name = name
diff --git a/pkg/epp/scheduling/types/cycle_state.go b/pkg/epp/scheduling/types/cycle_state.go
index 4ac617a..f70ada3 100644
--- a/pkg/epp/scheduling/types/cycle_state.go
+++ b/pkg/epp/scheduling/types/cycle_state.go
@@ -33,6 +33,7 @@ func NewCycleState() *CycleState {
 // CycleState does not provide any data protection, as all plugins are assumed to be
 // trusted.
 // Note: CycleState uses a sync.Map to back the storage, because it is thread safe. It's aimed to optimize for the "write once and read many times" scenarios.
+// TODO: Perhaps, deprecate CycleState once datalayer producer-consumer changes are made.
 type CycleState struct {
 	// key: StateKey, value: StateData
 	storage sync.Map
diff --git a/pkg/epp/scheduling/types/types.go b/pkg/epp/scheduling/types/types.go
index c93f0c5..6f9bec8 100644
--- a/pkg/epp/scheduling/types/types.go
+++ b/pkg/epp/scheduling/types/types.go
@@ -17,7 +17,10 @@ limitations under the License.
 package types
 
 import (
+	"encoding/json"
+	"errors"
 	"fmt"
+	"strings"
 
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/backend"
 	backendmetrics "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/backend/metrics"
@@ -68,8 +71,8 @@ func (r *LLMRequestBody) CacheSalt() string {
 	return r.Completions.CacheSalt
 }
 
-// CompletionsRequest is a structured representation of the fields we parse out of the
-// /v1/completions request body.
+// CompletionsRequest is a structured representation of the fields we parse out of the /v1/completions request
+// body. For detailed body fields, please refer to https://platform.openai.com/docs/api-reference/completions.
 // This struct includes fields usable for plugins and scheduling decisions - and not the entire
 // API spec.
 type CompletionsRequest struct {
@@ -87,8 +90,8 @@ func (r *CompletionsRequest) String() string {
 	return fmt.Sprintf("{PromptLength: %d}", len(r.Prompt))
 }
 
-// ChatCompletionsRequest is a structured representation of the fields we parse out of the
-// /v1/chat/completions request body.
+// ChatCompletionsRequest is a structured representation of the fields we parse out of the v1/chat/completions
+// request body. For detailed body fields, please refer to https://platform.openai.com/docs/api-reference/chat.
 // This struct includes fields usable for plugins and scheduling decisions - and not the entire
 // API spec.
 type ChatCompletionsRequest struct {
@@ -113,16 +116,75 @@ func (r *ChatCompletionsRequest) String() string {
 
 	messagesLen := 0
 	for _, msg := range r.Messages {
-		messagesLen += len(msg.Content)
+		messagesLen += len(msg.Content.PlainText())
 	}
-
 	return fmt.Sprintf("{MessagesLength: %d}", messagesLen)
 }
 
 // Message represents a single message in a chat-completions request.
 type Message struct {
-	Role    string
-	Content string // TODO: support multi-modal content
+	// Role is the message Role, optional values are 'user', 'assistant', ...
+	Role string `json:"role,omitempty"`
+	// Content defines text of this message
+	Content Content `json:"content,omitempty"`
+}
+
+type Content struct {
+	Raw        string
+	Structured []ContentBlock
+}
+
+type ContentBlock struct {
+	Type     string     `json:"type"`
+	Text     string     `json:"text,omitempty"`
+	ImageURL ImageBlock `json:"image_url,omitempty"`
+}
+
+type ImageBlock struct {
+	Url string `json:"url,omitempty"`
+}
+
+// UnmarshalJSON allow use both format
+func (mc *Content) UnmarshalJSON(data []byte) error {
+	// Raw format
+	var str string
+	if err := json.Unmarshal(data, &str); err == nil {
+		mc.Raw = str
+		return nil
+	}
+
+	// Block format
+	var blocks []ContentBlock
+	if err := json.Unmarshal(data, &blocks); err == nil {
+		mc.Structured = blocks
+		return nil
+	}
+
+	return errors.New("content format not supported")
+}
+
+func (mc Content) MarshalJSON() ([]byte, error) {
+	if mc.Raw != "" {
+		return json.Marshal(mc.Raw)
+	}
+	if mc.Structured != nil {
+		return json.Marshal(mc.Structured)
+	}
+	return json.Marshal("")
+}
+
+func (mc Content) PlainText() string {
+	if mc.Raw != "" {
+		return mc.Raw
+	}
+	var sb strings.Builder
+	for _, block := range mc.Structured {
+		if block.Type == "text" {
+			sb.WriteString(block.Text)
+			sb.WriteString(" ")
+		}
+	}
+	return sb.String()
 }
 
 type Pod interface {
diff --git a/site-src/_includes/epp-latest.md b/site-src/_includes/epp-latest.md
new file mode 100644
index 0000000..9f9a4e2
--- /dev/null
+++ b/site-src/_includes/epp-latest.md
@@ -0,0 +1,43 @@
+=== "GKE"
+
+      ```bash
+      export GATEWAY_PROVIDER=gke
+      helm install vllm-llama3-8b-instruct \
+      --set inferencePool.modelServers.matchLabels.app=vllm-llama3-8b-instruct \
+      --set provider.name=$GATEWAY_PROVIDER \
+      --version $IGW_CHART_VERSION \
+      oci://us-central1-docker.pkg.dev/k8s-staging-images/gateway-api-inference-extension/charts/inferencepool
+      ```
+
+=== "Istio"
+
+      ```bash
+      export GATEWAY_PROVIDER=istio
+      helm install vllm-llama3-8b-instruct \
+      --set inferencePool.modelServers.matchLabels.app=vllm-llama3-8b-instruct \
+      --set provider.name=$GATEWAY_PROVIDER \
+      --version $IGW_CHART_VERSION \
+      oci://us-central1-docker.pkg.dev/k8s-staging-images/gateway-api-inference-extension/charts/inferencepool
+      ```
+
+=== "Kgateway"
+
+      ```bash
+      export GATEWAY_PROVIDER=none
+      helm install vllm-llama3-8b-instruct \
+      --set inferencePool.modelServers.matchLabels.app=vllm-llama3-8b-instruct \
+      --set provider.name=$GATEWAY_PROVIDER \
+      --version $IGW_CHART_VERSION \
+      oci://us-central1-docker.pkg.dev/k8s-staging-images/gateway-api-inference-extension/charts/inferencepool
+      ```
+
+=== "Agentgateway"
+
+      ```bash
+      export GATEWAY_PROVIDER=none
+      helm install vllm-llama3-8b-instruct \
+      --set inferencePool.modelServers.matchLabels.app=vllm-llama3-8b-instruct \
+      --set provider.name=$GATEWAY_PROVIDER \
+      --version $IGW_CHART_VERSION \
+      oci://us-central1-docker.pkg.dev/k8s-staging-images/gateway-api-inference-extension/charts/inferencepool
+      ```
diff --git a/site-src/guides/getting-started-latest.md b/site-src/guides/getting-started-latest.md
index a11e70f..e436a5c 100644
--- a/site-src/guides/getting-started-latest.md
+++ b/site-src/guides/getting-started-latest.md
@@ -26,13 +26,13 @@
 --8<-- "site-src/_includes/model-server-cpu.md"
 
     ```bash
-    kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/vllm/sim-deployment.yaml
+    kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/vllm/cpu-deployment.yaml
     ```
 
 --8<-- "site-src/_includes/model-server-sim.md"
 
     ```bash
-    kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/gateway-api-inference-extension/refs/tags/v1.0.0/config/manifests/vllm/sim-deployment.yaml
+    kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/vllm/sim-deployment.yaml
     ```
 
 ### Install the Inference Extension CRDs
@@ -51,7 +51,7 @@ kubectl apply -k https://github.com/kubernetes-sigs/gateway-api-inference-extens
    export IGW_CHART_VERSION=v0
    ```
 
---8<-- "site-src/_includes/epp.md"
+--8<-- "site-src/_includes/epp-latest.md"
 
 ### Deploy an Inference Gateway
 
@@ -147,8 +147,9 @@ kubectl apply -k https://github.com/kubernetes-sigs/gateway-api-inference-extens
 
 === "Kgateway"
 
-      [Kgateway](https://kgateway.dev/) added Inference Gateway support as a **technical preview** in the
-      [v2.0.0 release](https://github.com/kgateway-dev/kgateway/releases/tag/v2.0.0). InferencePool v1.0.1 is currently supported in the latest [rolling release](https://github.com/kgateway-dev/kgateway/releases/tag/v2.1.0-main), which includes the latest changes but may be unstable until the [v2.1.0 release](https://github.com/kgateway-dev/kgateway/milestone/58) is published.
+      [Kgateway](https://kgateway.dev/) is a Gateway API and Inference Gateway
+      [conformant](https://github.com/kubernetes-sigs/gateway-api-inference-extension/tree/main/conformance/reports/v1.0.0/gateway/kgateway)
+      gateway. Follow these steps to run Kgateway:
 
       1. Requirements
 
@@ -158,7 +159,7 @@ kubectl apply -k https://github.com/kubernetes-sigs/gateway-api-inference-extens
       2. Set the Kgateway version and install the Kgateway CRDs.
 
          ```bash
-         KGTW_VERSION=v2.1.0-main
+         KGTW_VERSION=v2.2.0-main
          helm upgrade -i --create-namespace --namespace kgateway-system --version $KGTW_VERSION kgateway-crds oci://cr.kgateway.dev/kgateway-dev/charts/kgateway-crds
          ```
 
@@ -195,7 +196,9 @@ kubectl apply -k https://github.com/kubernetes-sigs/gateway-api-inference-extens
 
 === "Agentgateway"
 
-      [Agentgateway](https://agentgateway.dev/) is a purpose-built proxy designed for AI workloads, and comes with native support for Inference Gateway. Agentgateway integrates with [Kgateway](https://kgateway.dev/) as it's control plane. InferencePool v1.0.0 is currently supported in the latest [rolling release](https://github.com/kgateway-dev/kgateway/releases/tag/v2.1.0-main), which includes the latest changes but may be unstable until the [v2.1.0 release](https://github.com/kgateway-dev/kgateway/milestone/58) is published.
+      [Agentgateway](https://agentgateway.dev/) is a purpose-built proxy designed for AI workloads, and comes with native support for Inference Gateway.
+      Agentgateway integrates with [Kgateway](https://kgateway.dev/) as it's control plane. Follow these steps to run Kgateway with the agentgateway
+      data plane:
 
       1. Requirements
 
@@ -205,14 +208,14 @@ kubectl apply -k https://github.com/kubernetes-sigs/gateway-api-inference-extens
       2. Set the Kgateway version and install the Kgateway CRDs.
 
          ```bash
-         KGTW_VERSION=v2.1.0-main
+         KGTW_VERSION=v2.2.0-main
          helm upgrade -i --create-namespace --namespace kgateway-system --version $KGTW_VERSION kgateway-crds oci://cr.kgateway.dev/kgateway-dev/charts/kgateway-crds
          ```
 
       3. Install Kgateway
 
          ```bash
-         helm upgrade -i --namespace kgateway-system --version $KGTW_VERSION kgateway oci://cr.kgateway.dev/kgateway-dev/charts/kgateway --set inferenceExtension.enabled=true --set agentGateway.enabled=true
+         helm upgrade -i --namespace kgateway-system --version $KGTW_VERSION kgateway oci://cr.kgateway.dev/kgateway-dev/charts/kgateway --set inferenceExtension.enabled=true --set agentgateway.enabled=true
          ```
 
       4. Deploy the Gateway
diff --git a/site-src/guides/index.md b/site-src/guides/index.md
index 5577dc4..21b2384 100644
--- a/site-src/guides/index.md
+++ b/site-src/guides/index.md
@@ -34,7 +34,7 @@
 ### Install the Inference Extension CRDs
 
 ```bash
-kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/releases/download/v1.0.2/v1-manifests.yaml
+kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/releases/download/v1.0.2/manifests.yaml
 ```
 
 ### Install the Gateway
@@ -77,7 +77,7 @@ kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extens
       2. Set the Kgateway version and install the Kgateway CRDs.
 
          ```bash
-         KGTW_VERSION=v2.1.0
+         KGTW_VERSION=v2.2.0-main
          helm upgrade -i --create-namespace --namespace kgateway-system --version $KGTW_VERSION kgateway-crds oci://cr.kgateway.dev/kgateway-dev/charts/kgateway-crds
          ```
 
@@ -97,7 +97,7 @@ kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extens
       2. Set the Kgateway version and install the Kgateway CRDs.
 
          ```bash
-         KGTW_VERSION=v2.1.0
+         KGTW_VERSION=v2.2.0-main
          helm upgrade -i --create-namespace --namespace kgateway-system --version $KGTW_VERSION kgateway-crds oci://cr.kgateway.dev/kgateway-dev/charts/kgateway-crds
          ```
 
diff --git a/site-src/guides/metrics-and-observability.md b/site-src/guides/metrics-and-observability.md
index 8bcd952..d9dab80 100644
--- a/site-src/guides/metrics-and-observability.md
+++ b/site-src/guides/metrics-and-observability.md
@@ -17,7 +17,7 @@ This guide describes the current state of exposed metrics and how to scrape them
       "max_tokens": 10,
       "temperature": 0,
       "stream": true,
-      "stream_options": {"include_usage": "true"}
+      "stream_options": {"include_usage": true}
       }'
       ```
 



Test patch:
diff --git a/pkg/epp/scheduling/framework/plugins/multi/prefix/plugin_test.go b/pkg/epp/scheduling/framework/plugins/multi/prefix/plugin_test.go
index 108815d..59a09db 100644
--- a/pkg/epp/scheduling/framework/plugins/multi/prefix/plugin_test.go
+++ b/pkg/epp/scheduling/framework/plugins/multi/prefix/plugin_test.go
@@ -217,8 +217,8 @@ func TestPrefixPluginChatCompletions(t *testing.T) {
 		Body: &types.LLMRequestBody{
 			ChatCompletions: &types.ChatCompletionsRequest{
 				Messages: []types.Message{
-					{Role: "user", Content: "hello world"},
-					{Role: "assistant", Content: "hi there"},
+					{Role: "user", Content: types.Content{Raw: "hello world"}},
+					{Role: "assistant", Content: types.Content{Raw: "hi there"}},
 				},
 			},
 		},
@@ -252,8 +252,8 @@ func TestPrefixPluginChatCompletionsGrowth(t *testing.T) {
 		Body: &types.LLMRequestBody{
 			ChatCompletions: &types.ChatCompletionsRequest{
 				Messages: []types.Message{
-					{Role: "system", Content: "You are a helpful assistant"},
-					{Role: "user", Content: "Hello, how are you?"},
+					{Role: "system", Content: types.Content{Raw: "You are a helpful assistant"}},
+					{Role: "user", Content: types.Content{Raw: "Hello, how are you?"}},
 				},
 			},
 		},
@@ -285,10 +285,10 @@ func TestPrefixPluginChatCompletionsGrowth(t *testing.T) {
 		Body: &types.LLMRequestBody{
 			ChatCompletions: &types.ChatCompletionsRequest{
 				Messages: []types.Message{
-					{Role: "system", Content: "You are a helpful assistant"},
-					{Role: "user", Content: "Hello, how are you?"},
-					{Role: "assistant", Content: "I'm doing well, thank you! How can I help you today?"},
-					{Role: "user", Content: "Can you explain how prefix caching works?"},
+					{Role: "system", Content: types.Content{Raw: "You are a helpful assistant"}},
+					{Role: "user", Content: types.Content{Raw: "Hello, how are you?"}},
+					{Role: "assistant", Content: types.Content{Raw: "I'm doing well, thank you! How can I help you today?"}},
+					{Role: "user", Content: types.Content{Raw: "Can you explain how prefix caching works?"}},
 				},
 			},
 		},
@@ -318,12 +318,12 @@ func TestPrefixPluginChatCompletionsGrowth(t *testing.T) {
 		Body: &types.LLMRequestBody{
 			ChatCompletions: &types.ChatCompletionsRequest{
 				Messages: []types.Message{
-					{Role: "system", Content: "You are a helpful assistant"},
-					{Role: "user", Content: "Hello, how are you?"},
-					{Role: "assistant", Content: "I'm doing well, thank you! How can I help you today?"},
-					{Role: "user", Content: "Can you explain how prefix caching works?"},
-					{Role: "assistant", Content: "Prefix caching is a technique where..."},
-					{Role: "user", Content: "That's very helpful, thank you!"},
+					{Role: "system", Content: types.Content{Raw: "You are a helpful assistant"}},
+					{Role: "user", Content: types.Content{Raw: "Hello, how are you?"}},
+					{Role: "assistant", Content: types.Content{Raw: "I'm doing well, thank you! How can I help you today?"}},
+					{Role: "user", Content: types.Content{Raw: "Can you explain how prefix caching works?"}},
+					{Role: "assistant", Content: types.Content{Raw: "Prefix caching is a technique where..."}},
+					{Role: "user", Content: types.Content{Raw: "That's very helpful, thank you!"}},
 				},
 			},
 		},
@@ -437,7 +437,7 @@ func BenchmarkPrefixPluginChatCompletionsStress(b *testing.B) {
 		b.Run(fmt.Sprintf("messages_%d_length_%d", scenario.messageCount, scenario.messageLength), func(b *testing.B) {
 			// Generate messages for this scenario
 			messages := make([]types.Message, scenario.messageCount)
-			messages[0] = types.Message{Role: "system", Content: "You are a helpful assistant."}
+			messages[0] = types.Message{Role: "system", Content: types.Content{Raw: "You are a helpful assistant."}}
 
 			for i := 1; i < scenario.messageCount; i++ {
 				role := "user"
@@ -445,7 +445,7 @@ func BenchmarkPrefixPluginChatCompletionsStress(b *testing.B) {
 					role = "assistant"
 				}
 				content := randomPrompt(scenario.messageLength)
-				messages[i] = types.Message{Role: role, Content: content}
+				messages[i] = types.Message{Role: role, Content: types.Content{Raw: content}}
 			}
 
 			pod := &types.PodMetrics{
diff --git a/pkg/epp/util/request/body_test.go b/pkg/epp/util/request/body_test.go
index 51389e5..5f5ec5c 100644
--- a/pkg/epp/util/request/body_test.go
+++ b/pkg/epp/util/request/body_test.go
@@ -58,8 +58,58 @@ func TestExtractRequestData(t *testing.T) {
 			want: &types.LLMRequestBody{
 				ChatCompletions: &types.ChatCompletionsRequest{
 					Messages: []types.Message{
-						{Role: "system", Content: "this is a system message"},
-						{Role: "user", Content: "hello"},
+						{Role: "system", Content: types.Content{Raw: "this is a system message"}},
+						{Role: "user", Content: types.Content{Raw: "hello"}},
+					},
+				},
+			},
+		},
+		{
+			name: "chat completions request body with multi-modal content",
+			body: map[string]any{
+				"model": "test",
+				"messages": []any{
+					map[string]any{
+						"role": "system",
+						"content": []map[string]any{
+							{
+								"type": "text",
+								"text": "Describe this image in one sentence.",
+							},
+						},
+					},
+					map[string]any{
+						"role": "user",
+						"content": []map[string]any{
+							{
+								"type": "image_url",
+								"image_url": map[string]any{
+									"url": "https://example.com/images/dui.jpg.",
+								},
+							},
+						},
+					},
+				},
+			},
+			want: &types.LLMRequestBody{
+				ChatCompletions: &types.ChatCompletionsRequest{
+					Messages: []types.Message{
+						{Role: "system", Content: types.Content{
+							Structured: []types.ContentBlock{
+								{
+									Text: "Describe this image in one sentence.",
+									Type: "text",
+								},
+							},
+						}},
+						{Role: "user", Content: types.Content{
+							Structured: []types.ContentBlock{
+								{
+									Type:     "image_url",
+									ImageURL: types.ImageBlock{Url: "https://example.com/images/dui.jpg."},
+								},
+							},
+						}},
 					},
 				},
 			},
@@ -81,7 +131,7 @@ func TestExtractRequestData(t *testing.T) {
 			},
 			want: &types.LLMRequestBody{
 				ChatCompletions: &types.ChatCompletionsRequest{
-					Messages:                  []types.Message{{Role: "user", Content: "hello"}},
+					Messages:                  []types.Message{{Role: "user", Content: types.Content{Raw: "hello"}}},
 					Tools:                     []any{map[string]any{"type": "function"}},
 					Documents:                 []any{map[string]any{"content": "doc"}},
 					ChatTemplate:              "custom template",
@@ -256,8 +306,8 @@ func TestExtractRequestData(t *testing.T) {
 			want: &types.LLMRequestBody{
 				ChatCompletions: &types.ChatCompletionsRequest{
 					Messages: []types.Message{
-						{Role: "system", Content: "this is a system message"},
-						{Role: "user", Content: "hello"},
+						{Role: "system", Content: types.Content{Raw: "this is a system message"}},
+						{Role: "user", Content: types.Content{Raw: "hello"}},
 					},
 					CacheSalt: "Z3V2bmV3aGxza3ZubGFoZ3Zud3V3ZWZ2bmd0b3V2bnZmc2xpZ3RoZ2x2aQ==",
 				},