Repository: envoyproxy/ai-gateway
Pull request #459 (2025-03-05 10:01:00)

extproc: add genai metrics to track token usage and latency
**Commit Message**

extproc: add GenAI metrics to track token usage and latency

Adds GenAI metrics according to the OpenTelemetry Semantic Conventions for Generative AI Metrics [1].
Note those metrics are still in experimental phase and may still be subject to change.

1: https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-metrics/

**Related Issues/PRs (if applicable)**

This is a follow-up of https://github.com/envoyproxy/ai-gateway/pull/432, implementing the remaining review comments.

**Special notes for reviewers (if applicable)**

This PR contains all the commits in the original PR intact, and the only added piece are the last two commits: https://github.com/envoyproxy/ai-gateway/pull/459/commits/8a0982630526d2459cb8c71f7ba6d82c67b21841 and https://github.com/envoyproxy/ai-gateway/pull/459/commits/3f5dde2e2d46f3848a801d10d1c875343cd32457

Thr first commit contains:

* Addresses recording the metrics in a deferred function to make the recording less error-prone.
* Creates interfaces for the metrics to decouple the metrics logic from the processor and tests.
* Removes the global state on metrics and injects the metrics instance in the processor at startup time.
* Refactors the metrics to align with the OpenTelemetry Semantic Conventions for GenAI.

The second commit contains:

A refactoring of the above to use the OpenTelemetry SDK instead of the Prometheus one, to decouple the core from Prometheus. I left this in a separate commit because I don't know if we really care about this?

Example metrics:

```
# HELP gen_ai_client_token_usage Number of tokens processed.
# TYPE gen_ai_client_token_usage histogram
gen_ai_client_token_usage_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",gen_ai_token_type="input",le="1"} 1
gen_ai_client_token_usage_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",gen_ai_token_type="input",le="4"} 1
gen_ai_client_token_usage_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",gen_ai_token_type="input",le="16"} 2
gen_ai_client_token_usage_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",gen_ai_token_type="input",le="64"} 3
gen_ai_client_token_usage_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",gen_ai_token_type="input",le="256"} 3
gen_ai_client_token_usage_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",gen_ai_token_type="input",le="1024"} 3
gen_ai_client_token_usage_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",gen_ai_token_type="input",le="4096"} 3
gen_ai_client_token_usage_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",gen_ai_token_type="input",le="16384"} 3
gen_ai_client_token_usage_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",gen_ai_token_type="input",le="65536"} 3
gen_ai_client_token_usage_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",gen_ai_token_type="input",le="262144"} 3
gen_ai_client_token_usage_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",gen_ai_token_type="input",le="1.048576e+06"} 3
gen_ai_client_token_usage_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",gen_ai_token_type="input",le="4.194304e+06"} 3
gen_ai_client_token_usage_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",gen_ai_token_type="input",le="1.6777216e+07"} 3
gen_ai_client_token_usage_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",gen_ai_token_type="input",le="6.7108864e+07"} 3
gen_ai_client_token_usage_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",gen_ai_token_type="input",le="+Inf"} 3
gen_ai_client_token_usage_sum{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",gen_ai_token_type="input"} 51
gen_ai_client_token_usage_count{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",gen_ai_token_type="input"} 3
# HELP gen_ai_server_request_duration Time spent processing request.
# TYPE gen_ai_server_request_duration histogram
gen_ai_server_request_duration_bucket{error_type="",gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="0.01"} 2
gen_ai_server_request_duration_bucket{error_type="",gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="0.02"} 2
gen_ai_server_request_duration_bucket{error_type="",gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="0.04"} 2
gen_ai_server_request_duration_bucket{error_type="",gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="0.08"} 2
gen_ai_server_request_duration_bucket{error_type="",gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="0.16"} 2
gen_ai_server_request_duration_bucket{error_type="",gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="0.32"} 2
gen_ai_server_request_duration_bucket{error_type="",gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="0.64"} 2
gen_ai_server_request_duration_bucket{error_type="",gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="1.28"} 2
gen_ai_server_request_duration_bucket{error_type="",gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="2.56"} 3
gen_ai_server_request_duration_bucket{error_type="",gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="5.12"} 3
gen_ai_server_request_duration_bucket{error_type="",gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="10.24"} 3
gen_ai_server_request_duration_bucket{error_type="",gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="20.48"} 3
gen_ai_server_request_duration_bucket{error_type="",gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="40.96"} 3
gen_ai_server_request_duration_bucket{error_type="",gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="81.92"} 3
gen_ai_server_request_duration_bucket{error_type="",gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="+Inf"} 3
gen_ai_server_request_duration_sum{error_type="",gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock"} 1.413312417
gen_ai_server_request_duration_count{error_type="",gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock"} 3
# HELP gen_ai_server_time_to_first_token Time to receive first token in streaming responses.
# TYPE gen_ai_server_time_to_first_token histogram
gen_ai_server_time_to_first_token_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="0.001"} 0
gen_ai_server_time_to_first_token_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="0.005"} 2
gen_ai_server_time_to_first_token_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="0.01"} 2
gen_ai_server_time_to_first_token_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="0.02"} 2
gen_ai_server_time_to_first_token_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="0.04"} 2
gen_ai_server_time_to_first_token_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="0.06"} 2
gen_ai_server_time_to_first_token_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="0.08"} 2
gen_ai_server_time_to_first_token_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="0.1"} 2
gen_ai_server_time_to_first_token_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="0.25"} 2
gen_ai_server_time_to_first_token_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="0.5"} 2
gen_ai_server_time_to_first_token_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="0.75"} 2
gen_ai_server_time_to_first_token_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="1"} 2
gen_ai_server_time_to_first_token_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="2.5"} 3
gen_ai_server_time_to_first_token_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="5"} 3
gen_ai_server_time_to_first_token_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="7.5"} 3
gen_ai_server_time_to_first_token_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="10"} 3
gen_ai_server_time_to_first_token_bucket{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock",le="+Inf"} 3
gen_ai_server_time_to_first_token_sum{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock"} 1.4131814169999999
gen_ai_server_time_to_first_token_count{gen_ai_operation_name="chat",gen_ai_request_model="something",gen_ai_response_model="something",gen_ai_system="aws.bedrock"} 3
```


Patch:
diff --git a/cmd/extproc/mainlib/main.go b/cmd/extproc/mainlib/main.go
index 6d6689e..b5f4fbd 100644
--- a/cmd/extproc/mainlib/main.go
+++ b/cmd/extproc/mainlib/main.go
@@ -13,6 +13,7 @@ import (
 	"log"
 	"log/slog"
 	"net"
+	"net/http"
 	"os"
 	"os/signal"
 	"strings"
@@ -20,10 +21,16 @@ import (
 	"time"
 
 	extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	otelprom "go.opentelemetry.io/otel/exporters/prometheus"
+	"go.opentelemetry.io/otel/metric"
+	metricsdk "go.opentelemetry.io/otel/sdk/metric"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/health/grpc_health_v1"
 
 	"github.com/envoyproxy/ai-gateway/internal/extproc"
+	"github.com/envoyproxy/ai-gateway/internal/metrics"
 	"github.com/envoyproxy/ai-gateway/internal/version"
 )
 
@@ -32,6 +39,7 @@ type extProcFlags struct {
 	configPath  string     // path to the configuration file.
 	extProcAddr string     // gRPC address for the external processor.
 	logLevel    slog.Level // log level for the external processor.
+	metricsAddr string     // HTTP address for the metrics server.
 }
 
 // parseAndValidateFlags parses and validates the flas passed to the external processor.
@@ -51,13 +59,14 @@ func parseAndValidateFlags(args []string) (extProcFlags, error) {
 	fs.StringVar(&flags.extProcAddr,
 		"extProcAddr",
 		":1063",
-		"gRPC address for the external processor. For example, :1063 or unix:///tmp/ext_proc.sock",
+		"gRPC address for the external processor. For example, :1063 or unix:///tmp/ext_proc.sock.",
 	)
 	logLevelPtr := fs.String(
 		"logLevel",
 		"info",
 		"log level for the external processor. One of 'debug', 'info', 'warn', or 'error'.",
 	)
+	fs.StringVar(&flags.metricsAddr, "metricsAddr", ":9190", "HTTP address for the metrics server.")
 
 	if err := fs.Parse(args); err != nil {
 		return extProcFlags{}, fmt.Errorf("failed to parse extProcFlags: %w", err)
@@ -102,11 +111,13 @@ func Main() {
 		log.Fatalf("failed to listen: %v", err)
 	}
 
+	metricsServer, meter := startMetricsServer(flags.metricsAddr, l)
+
 	server, err := extproc.NewServer(l)
 	if err != nil {
 		log.Fatalf("failed to create external processor server: %v", err)
 	}
-	server.Register("/v1/chat/completions", extproc.NewChatCompletionProcessor)
+	server.Register("/v1/chat/completions", extproc.ChatCompletionProcessorFactory(metrics.NewChatCompletion(meter)))
 	server.Register("/v1/models", extproc.NewModelsProcessor)
 
 	if err := extproc.StartConfigWatcher(ctx, flags.configPath, server, l, time.Second*5); err != nil {
@@ -116,10 +127,18 @@ func Main() {
 	s := grpc.NewServer()
 	extprocv3.RegisterExternalProcessorServer(s, server)
 	grpc_health_v1.RegisterHealthServer(s, server)
+
 	go func() {
 		<-ctx.Done()
 		s.GracefulStop()
+
+		shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		if err := metricsServer.Shutdown(shutdownCtx); err != nil {
+			l.Error("Failed to shutdown metrics server gracefully", "error", err)
+		}
 	}()
+
 	_ = s.Serve(lis)
 }
 
@@ -130,3 +149,46 @@ func listenAddress(addrFlag string) (string, string) {
 	}
 	return "tcp", addrFlag
 }
+
+// startMetricsServer starts the HTTP server for Prometheus metrics.
+func startMetricsServer(addr string, logger *slog.Logger) (*http.Server, metric.Meter) {
+	registry := prometheus.NewRegistry()
+	exporter, err := otelprom.New(otelprom.WithRegisterer(registry))
+	if err != nil {
+		log.Fatal("failed to create metrics exporter")
+	}
+	provider := metricsdk.NewMeterProvider(metricsdk.WithReader(exporter))
+	meter := provider.Meter("envoyproxy/ai-gateway")
+
+	// Create a new HTTP server for metrics.
+	mux := http.NewServeMux()
+
+	// Register the metrics handler.
+	mux.Handle("/metrics", promhttp.HandlerFor(
+		registry,
+		promhttp.HandlerOpts{
+			EnableOpenMetrics: true,
+		},
+	))
+
+	// Add a simple health check endpoint.
+	mux.HandleFunc("/health", func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("OK"))
+	})
+
+	server := &http.Server{
+		Addr:              addr,
+		Handler:           mux,
+		ReadHeaderTimeout: 5 * time.Second,
+	}
+
+	go func() {
+		logger.Info("Starting metrics server", "address", addr)
+		if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+			logger.Error("Metrics server failed", "error", err)
+		}
+	}()
+
+	return server, meter
+}
diff --git a/go.mod b/go.mod
index 419d6d1..5fb533d 100644
--- a/go.mod
+++ b/go.mod
@@ -15,7 +15,12 @@ require (
 	github.com/google/cel-go v0.24.1
 	github.com/google/go-cmp v0.7.0
 	github.com/openai/openai-go v0.1.0-alpha.61
+	github.com/prometheus/client_golang v1.20.5
 	github.com/stretchr/testify v1.10.0
+	go.opentelemetry.io/otel v1.34.0
+	go.opentelemetry.io/otel/exporters/prometheus v0.56.0
+	go.opentelemetry.io/otel/metric v1.34.0
+	go.opentelemetry.io/otel/sdk/metric v1.34.0
 	go.uber.org/goleak v1.3.0
 	go.uber.org/zap v1.27.0
 	golang.org/x/exp v0.0.0-20250228200357-dead58393ab7
@@ -272,7 +277,6 @@ require (
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/polyfloyd/go-errorlint v1.7.1 // indirect
-	github.com/prometheus/client_golang v1.20.5 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.62.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
@@ -344,8 +348,7 @@ require (
 	go-simpler.org/sloglint v0.9.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect
-	go.opentelemetry.io/otel v1.34.0 // indirect
-	go.opentelemetry.io/otel/metric v1.34.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.34.0 // indirect
 	go.opentelemetry.io/otel/trace v1.34.0 // indirect
 	go.uber.org/automaxprocs v1.6.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
diff --git a/internal/extproc/chatcompletion_processor.go b/internal/extproc/chatcompletion_processor.go
index 2613173..8295ee6 100644
--- a/internal/extproc/chatcompletion_processor.go
+++ b/internal/extproc/chatcompletion_processor.go
@@ -25,18 +25,22 @@ import (
 	"github.com/envoyproxy/ai-gateway/internal/apischema/openai"
 	"github.com/envoyproxy/ai-gateway/internal/extproc/translator"
 	"github.com/envoyproxy/ai-gateway/internal/llmcostcel"
+	"github.com/envoyproxy/ai-gateway/internal/metrics"
 )
 
-// NewChatCompletionProcessor implements [Processor] for the /chat/completions endpoint.
-func NewChatCompletionProcessor(config *processorConfig, requestHeaders map[string]string, logger *slog.Logger) (Processor, error) {
-	if config.schema.Name != filterapi.APISchemaOpenAI {
-		return nil, fmt.Errorf("unsupported API schema: %s", config.schema.Name)
+// ChatCompletionProcessorFactory returns a factory method to instantiate the chat completion processor.
+func ChatCompletionProcessorFactory(ccm metrics.ChatCompletion) ProcessorFactory {
+	return func(config *processorConfig, requestHeaders map[string]string, logger *slog.Logger) (Processor, error) {
+		if config.schema.Name != filterapi.APISchemaOpenAI {
+			return nil, fmt.Errorf("unsupported API schema: %s", config.schema.Name)
+		}
+		return &chatCompletionProcessor{
+			config:         config,
+			requestHeaders: requestHeaders,
+			logger:         logger,
+			metrics:        ccm,
+		}, nil
 	}
-	return &chatCompletionProcessor{
-		config:         config,
-		requestHeaders: requestHeaders,
-		logger:         logger,
-	}, nil
 }
 
 // chatCompletionProcessor handles the processing of the request and response messages for a single stream.
@@ -49,6 +53,8 @@ type chatCompletionProcessor struct {
 	translator       translator.OpenAIChatCompletionTranslator
 	// cost is the cost of the request that is accumulated during the processing of the response.
 	costs translator.LLMTokenUsage
+	// metrics tracking.
+	metrics metrics.ChatCompletion
 }
 
 // selectTranslator selects the translator based on the output schema.
@@ -70,7 +76,10 @@ func (c *chatCompletionProcessor) selectTranslator(out filterapi.VersionedAPISch
 
 // ProcessRequestHeaders implements [Processor.ProcessRequestHeaders].
 func (c *chatCompletionProcessor) ProcessRequestHeaders(_ context.Context, _ *corev3.HeaderMap) (res *extprocv3.ProcessingResponse, err error) {
-	// The request headers have already been at the time the processor was created
+	// Start tracking metrics for this request.
+	c.metrics.StartRequest()
+
+	// The request headers have already been at the time the processor was created.
 	return &extprocv3.ProcessingResponse{Response: &extprocv3.ProcessingResponse_RequestHeaders{
 		RequestHeaders: &extprocv3.HeadersResponse{},
 	}}, nil
@@ -78,16 +87,23 @@ func (c *chatCompletionProcessor) ProcessRequestHeaders(_ context.Context, _ *co
 
 // ProcessRequestBody implements [Processor.ProcessRequestBody].
 func (c *chatCompletionProcessor) ProcessRequestBody(ctx context.Context, rawBody *extprocv3.HttpBody) (res *extprocv3.ProcessingResponse, err error) {
+	defer func() {
+		if err != nil {
+			c.metrics.RecordRequestCompletion(ctx, false)
+		}
+	}()
 	model, body, err := parseOpenAIChatCompletionBody(rawBody)
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse request body: %w", err)
 	}
 	c.logger.Info("Processing request", "path", c.requestHeaders[":path"], "model", model)
 
+	c.metrics.SetModel(model)
 	c.requestHeaders[c.config.modelNameHeaderKey] = model
 	b, err := c.config.router.Calculate(c.requestHeaders)
 	if err != nil {
 		if errors.Is(err, x.ErrNoMatchingRule) {
+			c.metrics.RecordRequestCompletion(ctx, false)
 			return &extprocv3.ProcessingResponse{
 				Response: &extprocv3.ProcessingResponse_ImmediateResponse{
 					ImmediateResponse: &extprocv3.ImmediateResponse{
@@ -101,6 +117,7 @@ func (c *chatCompletionProcessor) ProcessRequestBody(ctx context.Context, rawBod
 		return nil, fmt.Errorf("failed to calculate route: %w", err)
 	}
 	c.logger.Info("Selected backend", "backend", b.Name)
+	c.metrics.SetBackend(*b)
 
 	if err = c.selectTranslator(b.Schema); err != nil {
 		return nil, fmt.Errorf("failed to select translator: %w", err)
@@ -143,7 +160,12 @@ func (c *chatCompletionProcessor) ProcessRequestBody(ctx context.Context, rawBod
 }
 
 // ProcessResponseHeaders implements [Processor.ProcessResponseHeaders].
-func (c *chatCompletionProcessor) ProcessResponseHeaders(_ context.Context, headers *corev3.HeaderMap) (res *extprocv3.ProcessingResponse, err error) {
+func (c *chatCompletionProcessor) ProcessResponseHeaders(ctx context.Context, headers *corev3.HeaderMap) (res *extprocv3.ProcessingResponse, err error) {
+	defer func() {
+		if err != nil {
+			c.metrics.RecordRequestCompletion(ctx, false)
+		}
+	}()
 	c.responseHeaders = headersToMap(headers)
 	if enc := c.responseHeaders["content-encoding"]; enc != "" {
 		c.responseEncoding = enc
@@ -167,7 +189,10 @@ func (c *chatCompletionProcessor) ProcessResponseHeaders(_ context.Context, head
 }
 
 // ProcessResponseBody implements [Processor.ProcessResponseBody].
-func (c *chatCompletionProcessor) ProcessResponseBody(_ context.Context, body *extprocv3.HttpBody) (res *extprocv3.ProcessingResponse, err error) {
+func (c *chatCompletionProcessor) ProcessResponseBody(ctx context.Context, body *extprocv3.HttpBody) (res *extprocv3.ProcessingResponse, err error) {
+	defer func() {
+		c.metrics.RecordRequestCompletion(ctx, err == nil)
+	}()
 	var br io.Reader
 	switch c.responseEncoding {
 	case "gzip":
@@ -204,12 +229,18 @@ func (c *chatCompletionProcessor) ProcessResponseBody(_ context.Context, body *e
 	c.costs.InputTokens += tokenUsage.InputTokens
 	c.costs.OutputTokens += tokenUsage.OutputTokens
 	c.costs.TotalTokens += tokenUsage.TotalTokens
+
+	// Update metrics with token usage.
+	c.metrics.RecordTokenUsage(ctx, tokenUsage.InputTokens, tokenUsage.OutputTokens, tokenUsage.TotalTokens)
+	c.metrics.RecordTokenLatency(ctx, tokenUsage.OutputTokens)
+
 	if body.EndOfStream && len(c.config.requestCosts) > 0 {
 		resp.DynamicMetadata, err = c.maybeBuildDynamicMetadata()
 		if err != nil {
 			return nil, fmt.Errorf("failed to build dynamic metadata: %w", err)
 		}
 	}
+
 	return resp, nil
 }
 
diff --git a/internal/metrics/chat_completion_metrics.go b/internal/metrics/chat_completion_metrics.go
new file mode 100644
index 0000000..1fcbd3f
--- /dev/null
+++ b/internal/metrics/chat_completion_metrics.go
@@ -0,0 +1,139 @@
+// Copyright Envoy AI Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
+package metrics
+
+import (
+	"context"
+	"time"
+
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+
+	"github.com/envoyproxy/ai-gateway/filterapi"
+)
+
+// ChatCompletion is the interface for the chat completion AI Gateway metrics.
+type ChatCompletion interface {
+	// StartRequest initializes timing for a new request.
+	StartRequest()
+	// SetModel sets the model the request. This is usually called after parsing the request body .
+	SetModel(model string)
+	// SetBackend sets the selected backend when the routing decision has been made. This is usually called
+	// after parsing the request body to determine the model and invoke the routing logic.
+	SetBackend(backend filterapi.Backend)
+
+	// RecordTokenUsage records token usage metrics.
+	RecordTokenUsage(ctx context.Context, inputTokens, outputTokens, totalTokens uint32)
+	// RecordRequestCompletion records latency metrics for the entire request
+	RecordRequestCompletion(ctx context.Context, success bool)
+	// RecordTokenLatency records latency metrics for token generation.
+	RecordTokenLatency(ctx context.Context, tokens uint32)
+}
+
+// chatCompletion is the implementation for the chat completion AI Gateway metrics.
+type chatCompletion struct {
+	metrics        *genAI
+	firstTokenSent bool
+	requestStart   time.Time
+	lastTokenTime  time.Time
+	model          string
+	backend        string
+}
+
+// NewChatCompletion creates a new ChatCompletion instance.
+func NewChatCompletion(meter metric.Meter) ChatCompletion {
+	return &chatCompletion{
+		metrics: newGenAI(meter),
+		model:   "unknown",
+		backend: "unknown",
+	}
+}
+
+// StartRequest initializes timing for a new request.
+func (c *chatCompletion) StartRequest() {
+	c.requestStart = time.Now()
+	c.firstTokenSent = false
+}
+
+// SetModel sets the model for the request.
+func (c *chatCompletion) SetModel(model string) {
+	c.model = model
+}
+
+// SetBackend sets the name of the backend to be reported in the metrics according to:
+// https://opentelemetry.io/docs/specs/semconv/attributes-registry/gen-ai/#gen-ai-system
+func (c *chatCompletion) SetBackend(backend filterapi.Backend) {
+	switch backend.Schema.Name {
+	case filterapi.APISchemaOpenAI:
+		c.backend = genaiSystemOpenAI
+	case filterapi.APISchemaAWSBedrock:
+		c.backend = genAISystemAWSBedrock
+	default:
+		c.backend = backend.Name
+	}
+}
+
+// RecordTokenUsage implements [ChatCompletion.RecordTokenUsage].
+func (c *chatCompletion) RecordTokenUsage(ctx context.Context, inputTokens, outputTokens, totalTokens uint32) {
+	attrs := []attribute.KeyValue{
+		attribute.Key(genaiAttributeOperationName).String(genaiOperationChat),
+		attribute.Key(genaiAttributeSystemName).String(c.backend),
+		attribute.Key(genaiAttributeRequestModel).String(c.model),
+	}
+
+	c.metrics.tokenUsage.Record(ctx, float64(inputTokens),
+		metric.WithAttributes(attrs...),
+		metric.WithAttributes(attribute.Key(genaiAttributeTokenType).String(genaiTokenTypeInput)),
+	)
+	c.metrics.tokenUsage.Record(ctx, float64(outputTokens),
+		metric.WithAttributes(attrs...),
+		metric.WithAttributes(attribute.Key(genaiAttributeTokenType).String(genaiTokenTypeOutput)),
+	)
+	c.metrics.tokenUsage.Record(ctx, float64(totalTokens),
+		metric.WithAttributes(attrs...),
+		metric.WithAttributes(attribute.Key(genaiAttributeTokenType).String(genaiTokenTypeTotal)),
+	)
+}
+
+// RecordRequestCompletion implements [ChatCompletion.RecordRequestCompletion].
+func (c *chatCompletion) RecordRequestCompletion(ctx context.Context, success bool) {
+	attrs := []attribute.KeyValue{
+		attribute.Key(genaiAttributeOperationName).String(genaiOperationChat),
+		attribute.Key(genaiAttributeSystemName).String(c.backend),
+		attribute.Key(genaiAttributeRequestModel).String(c.model),
+	}
+
+	if success {
+		// According to the semantic conventions, the error attribute should not be added for successful operations
+		c.metrics.requestLatency.Record(ctx, time.Since(c.requestStart).Seconds(), metric.WithAttributes(attrs...))
+	} else {
+		// We don't have a set of typed errors yet, or a set of low-cardinality values, so we can just set the value to the
+		// placeholder one. See: https://opentelemetry.io/docs/specs/semconv/attributes-registry/error/#error-type
+		c.metrics.requestLatency.Record(ctx, time.Since(c.requestStart).Seconds(),
+			metric.WithAttributes(attrs...),
+			metric.WithAttributes(attribute.Key(genaiAttributeErrorType).String(genaiErrorTypeFallback)),
+		)
+	}
+}
+
+// RecordTokenLatency implements [ChatCompletion.RecordTokenLatency].
+func (c *chatCompletion) RecordTokenLatency(ctx context.Context, tokens uint32) {
+	attrs := []attribute.KeyValue{
+		attribute.Key(genaiAttributeOperationName).String(genaiOperationChat),
+		attribute.Key(genaiAttributeSystemName).String(c.backend),
+		attribute.Key(genaiAttributeRequestModel).String(c.model),
+	}
+
+	if !c.firstTokenSent {
+		c.firstTokenSent = true
+		c.metrics.firstTokenLatency.Record(ctx, time.Since(c.requestStart).Seconds(), metric.WithAttributes(attrs...))
+	} else if tokens > 0 {
+		// Calculate time between tokens.
+		itl := time.Since(c.lastTokenTime).Seconds() / float64(tokens)
+		c.metrics.outputTokenLatency.Record(ctx, itl, metric.WithAttributes(attrs...))
+	}
+	c.lastTokenTime = time.Now()
+}
diff --git a/internal/metrics/genai.go b/internal/metrics/genai.go
new file mode 100644
index 0000000..6f850e1
--- /dev/null
+++ b/internal/metrics/genai.go
@@ -0,0 +1,90 @@
+// Copyright Envoy AI Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
+package metrics
+
+import "go.opentelemetry.io/otel/metric"
+
+const (
+	// Metric names, attributes and values according to the Semantic Conventions for Generative AI Metrics.
+	// See: https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-metrics/
+
+	genaiMetricClientTokenUsage         = "gen_ai.client.token.usage" // #nosec G101: Potential hardcoded credentials
+	genaiMetricServerRequestDuration    = "gen_ai.server.request.duration"
+	genaiMetricServerTimeToFirstToken   = "gen_ai.server.time_to_first_token"   // #nosec G101: Potential hardcoded credentials
+	genaiMetricServerTimePerOutputToken = "gen_ai.server.time_per_output_token" // #nosec G101: Potential hardcoded credentials
+
+	genaiAttributeOperationName = "gen_ai.operation.name"
+	genaiAttributeSystemName    = "gen_ai.system.name"
+	genaiAttributeRequestModel  = "gen_ai.request.model"
+	genaiAttributeTokenType     = "gen_ai.token.type" // #nosec G101: Potential hardcoded credentials
+	genaiAttributeErrorType     = "error.type"
+
+	genaiOperationChat     = "chat"
+	genaiSystemOpenAI      = "openai"
+	genAISystemAWSBedrock  = "aws.bedrock"
+	genaiTokenTypeInput    = "input"
+	genaiTokenTypeOutput   = "output"
+	genaiTokenTypeTotal    = "total"
+	genaiErrorTypeFallback = "_OTHER"
+)
+
+// genAI holds metrics according to the Semantic Conventions for Generative AI Metrics.
+// See: https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-metrics/.
+type genAI struct {
+	// Number of tokens processed.
+	// See: https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-metrics/#metric-gen_aiclienttokenusage
+	tokenUsage metric.Float64Histogram
+	// requestLatency is the total latency of the request.
+	// Measured from the start of the received request headers in extproc to the end of the processed response body in extproc.
+	// See: https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-metrics/#metric-gen_aiserverrequestduration
+	requestLatency metric.Float64Histogram
+	// firstTokenLatency is the latency to receive the first token.
+	// Measured from the start of the received request headers in extproc to the receiving of the first token in the response body in extproc.
+	// See: https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-metrics/#metric-gen_aiservertime_to_first_token
+	firstTokenLatency metric.Float64Histogram
+	// outputTokenLatency is the latency between consecutive tokens, if supported, or by chunks/tokens otherwise, by backend, model.
+	// See: https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-metrics/#metric-gen_aiservertime_per_output_token
+	outputTokenLatency metric.Float64Histogram
+}
+
+// newGenAI creates a new genAI metrics instance.
+func newGenAI(meter metric.Meter) *genAI {
+	return &genAI{
+		tokenUsage: mustRegisterHistogram(meter,
+			genaiMetricClientTokenUsage,
+			metric.WithDescription("Number of tokens processed."),
+			metric.WithUnit("{token}"),
+			metric.WithExplicitBucketBoundaries(1, 4, 16, 64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, 67108864),
+		),
+		requestLatency: mustRegisterHistogram(meter,
+			genaiMetricServerRequestDuration,
+			metric.WithDescription("Time spent processing request."),
+			metric.WithUnit("s"),
+			metric.WithExplicitBucketBoundaries(0.01, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64, 1.28, 2.56, 5.12, 10.24, 20.48, 40.96, 81.92),
+		),
+		firstTokenLatency: mustRegisterHistogram(meter,
+			genaiMetricServerTimeToFirstToken,
+			metric.WithDescription("Time to receive first token in streaming responses."),
+			metric.WithUnit("s"),
+			metric.WithExplicitBucketBoundaries(0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5, 0.75, 1.0, 2.5, 5.0, 7.5, 10.0),
+		),
+		outputTokenLatency: mustRegisterHistogram(meter,
+			genaiMetricServerTimePerOutputToken,
+			metric.WithDescription("Time between consecutive tokens in streaming responses."),
+			metric.WithUnit("s"),
+			metric.WithExplicitBucketBoundaries(0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75, 1.0, 2.5),
+		),
+	}
+}
+
+// mustRegisterHistogram registers a histogram with the meter and panics if it fails.
+func mustRegisterHistogram(meter metric.Meter, name string, options ...metric.Float64HistogramOption) metric.Float64Histogram {
+	h, err := meter.Float64Histogram(name, options...)
+	if err != nil {
+		panic(err)
+	}
+	return h
+}


Test patch:
diff --git a/cmd/extproc/mainlib/main_test.go b/cmd/extproc/mainlib/main_test.go
index 4f193af..8b2e41f 100644
--- a/cmd/extproc/mainlib/main_test.go
+++ b/cmd/extproc/mainlib/main_test.go
@@ -6,7 +6,9 @@
 package mainlib
 
 import (
+	"io"
 	"log/slog"
+	"net/http"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -104,3 +106,19 @@ func TestListenAddress(t *testing.T) {
 		})
 	}
 }
+
+func TestStartMetricsServer(t *testing.T) {
+	s, m := startMetricsServer("127.0.0.1:", slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{})))
+	t.Cleanup(func() { _ = s.Shutdown(t.Context()) })
+
+	require.NotNil(t, s)
+	require.NotNil(t, m)
+
+	require.HTTPStatusCode(t, s.Handler.ServeHTTP, http.MethodGet, "/", nil, http.StatusNotFound)
+
+	require.HTTPSuccess(t, s.Handler.ServeHTTP, http.MethodGet, "/health", nil)
+	require.HTTPBodyContains(t, s.Handler.ServeHTTP, http.MethodGet, "/health", nil, "OK")
+
+	require.HTTPSuccess(t, s.Handler.ServeHTTP, http.MethodGet, "/metrics", nil)
+	require.HTTPBodyContains(t, s.Handler.ServeHTTP, http.MethodGet, "/metrics", nil, "target_info{")
+}
diff --git a/internal/extproc/chatcompletion_processor_test.go b/internal/extproc/chatcompletion_processor_test.go
index a03d2c1..c103532 100644
--- a/internal/extproc/chatcompletion_processor_test.go
+++ b/internal/extproc/chatcompletion_processor_test.go
@@ -24,15 +24,15 @@ import (
 	"github.com/envoyproxy/ai-gateway/internal/llmcostcel"
 )
 
-func TestChatCompletion_Scheema(t *testing.T) {
+func TestChatCompletion_Schema(t *testing.T) {
 	t.Run("unsupported", func(t *testing.T) {
 		cfg := &processorConfig{schema: filterapi.VersionedAPISchema{Name: "Foo", Version: "v123"}}
-		_, err := NewChatCompletionProcessor(cfg, nil, nil)
+		_, err := ChatCompletionProcessorFactory(nil)(cfg, nil, nil)
 		require.ErrorContains(t, err, "unsupported API schema: Foo")
 	})
 	t.Run("supported openai", func(t *testing.T) {
 		cfg := &processorConfig{schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI, Version: "v123"}}
-		_, err := NewChatCompletionProcessor(cfg, nil, nil)
+		_, err := ChatCompletionProcessorFactory(nil)(cfg, nil, nil)
 		require.NoError(t, err)
 	})
 }
@@ -56,49 +56,71 @@ func TestChatCompletion_SelectTranslator(t *testing.T) {
 }
 
 func TestChatCompletion_ProcessRequestHeaders(t *testing.T) {
-	p := &chatCompletionProcessor{}
+	mm := &mockChatCompletionMetrics{}
+	p := &chatCompletionProcessor{
+		metrics: mm,
+	}
 	res, err := p.ProcessRequestHeaders(t.Context(), &corev3.HeaderMap{
 		Headers: []*corev3.HeaderValue{{Key: "foo", Value: "bar"}},
 	})
 	require.NoError(t, err)
 	_, ok := res.Response.(*extprocv3.ProcessingResponse_RequestHeaders)
 	require.True(t, ok)
+	require.NotZero(t, mm.requestStart)
+	mm.RequireRequestNotCompleted(t)
 }
 
 func TestChatCompletion_ProcessResponseHeaders(t *testing.T) {
 	t.Run("error translation", func(t *testing.T) {
+		mm := &mockChatCompletionMetrics{}
 		mt := &mockTranslator{t: t, expHeaders: make(map[string]string)}
-		p := &chatCompletionProcessor{translator: mt}
+		p := &chatCompletionProcessor{
+			translator: mt,
+			metrics:    mm,
+		}
 		mt.retErr = errors.New("test error")
 		_, err := p.ProcessResponseHeaders(t.Context(), nil)
 		require.ErrorContains(t, err, "test error")
+		mm.RequireRequestFailure(t)
 	})
 	t.Run("ok", func(t *testing.T) {
 		inHeaders := &corev3.HeaderMap{
 			Headers: []*corev3.HeaderValue{{Key: "foo", Value: "bar"}, {Key: "dog", RawValue: []byte("cat")}},
 		}
 		expHeaders := map[string]string{"foo": "bar", "dog": "cat"}
+		mm := &mockChatCompletionMetrics{}
 		mt := &mockTranslator{t: t, expHeaders: expHeaders}
-		p := &chatCompletionProcessor{translator: mt}
+		p := &chatCompletionProcessor{
+			translator: mt,
+			metrics:    mm,
+		}
 		res, err := p.ProcessResponseHeaders(t.Context(), inHeaders)
 		require.NoError(t, err)
 		commonRes := res.Response.(*extprocv3.ProcessingResponse_ResponseHeaders).ResponseHeaders.Response
 		require.Equal(t, mt.retHeaderMutation, commonRes.HeaderMutation)
+		mm.RequireRequestNotCompleted(t)
 	})
 }
 
 func TestChatCompletion_ProcessResponseBody(t *testing.T) {
 	t.Run("error translation", func(t *testing.T) {
+		mm := &mockChatCompletionMetrics{}
 		mt := &mockTranslator{t: t}
-		p := &chatCompletionProcessor{translator: mt}
+		p := &chatCompletionProcessor{
+			translator: mt,
+			metrics:    mm,
+		}
 		mt.retErr = errors.New("test error")
 		_, err := p.ProcessResponseBody(t.Context(), &extprocv3.HttpBody{})
 		require.ErrorContains(t, err, "test error")
+		mm.RequireRequestFailure(t)
+		mm.RequireTokensRecorded(t, 0)
 	})
 	t.Run("ok", func(t *testing.T) {
 		inBody := &extprocv3.HttpBody{Body: []byte("some-body"), EndOfStream: true}
 		expBodyMut := &extprocv3.BodyMutation{}
 		expHeadMut := &extprocv3.HeaderMutation{}
+		mm := &mockChatCompletionMetrics{}
 		mt := &mockTranslator{
 			t: t, expResponseBody: inBody,
 			retBodyMutation: expBodyMut, retHeaderMutation: expHeadMut,
@@ -109,26 +131,33 @@ func TestChatCompletion_ProcessResponseBody(t *testing.T) {
 		require.NoError(t, err)
 		celProgUint, err := llmcostcel.NewProgram("uint(9999)")
 		require.NoError(t, err)
-		p := &chatCompletionProcessor{translator: mt, logger: slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{})), config: &processorConfig{
-			metadataNamespace: "ai_gateway_llm_ns",
-			requestCosts: []processorConfigRequestCost{
-				{LLMRequestCost: &filterapi.LLMRequestCost{Type: filterapi.LLMRequestCostTypeOutputToken, MetadataKey: "output_token_usage"}},
-				{LLMRequestCost: &filterapi.LLMRequestCost{Type: filterapi.LLMRequestCostTypeInputToken, MetadataKey: "input_token_usage"}},
-				{
-					celProg:        celProgInt,
-					LLMRequestCost: &filterapi.LLMRequestCost{Type: filterapi.LLMRequestCostTypeCEL, MetadataKey: "cel_int"},
-				},
-				{
-					celProg:        celProgUint,
-					LLMRequestCost: &filterapi.LLMRequestCost{Type: filterapi.LLMRequestCostTypeCEL, MetadataKey: "cel_uint"},
+		p := &chatCompletionProcessor{
+			translator: mt,
+			logger:     slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{})),
+			metrics:    mm,
+			config: &processorConfig{
+				metadataNamespace: "ai_gateway_llm_ns",
+				requestCosts: []processorConfigRequestCost{
+					{LLMRequestCost: &filterapi.LLMRequestCost{Type: filterapi.LLMRequestCostTypeOutputToken, MetadataKey: "output_token_usage"}},
+					{LLMRequestCost: &filterapi.LLMRequestCost{Type: filterapi.LLMRequestCostTypeInputToken, MetadataKey: "input_token_usage"}},
+					{
+						celProg:        celProgInt,
+						LLMRequestCost: &filterapi.LLMRequestCost{Type: filterapi.LLMRequestCostTypeCEL, MetadataKey: "cel_int"},
+					},
+					{
+						celProg:        celProgUint,
+						LLMRequestCost: &filterapi.LLMRequestCost{Type: filterapi.LLMRequestCostTypeCEL, MetadataKey: "cel_uint"},
+					},
 				},
 			},
-		}}
+		}
 		res, err := p.ProcessResponseBody(t.Context(), inBody)
 		require.NoError(t, err)
 		commonRes := res.Response.(*extprocv3.ProcessingResponse_ResponseBody).ResponseBody.Response
 		require.Equal(t, expBodyMut, commonRes.BodyMutation)
 		require.Equal(t, expHeadMut, commonRes.HeaderMutation)
+		mm.RequireRequestSuccess(t)
+		mm.RequireTokensRecorded(t, 1)
 
 		md := res.DynamicMetadata
 		require.NotNil(t, md)
@@ -152,21 +181,42 @@ func TestChatCompletion_ProcessRequestBody(t *testing.T) {
 		return bytes
 	}
 	t.Run("body parser error", func(t *testing.T) {
-		p := &chatCompletionProcessor{}
+		mm := &mockChatCompletionMetrics{}
+		p := &chatCompletionProcessor{
+			metrics: mm,
+		}
 		_, err := p.ProcessRequestBody(t.Context(), &extprocv3.HttpBody{Body: []byte("nonjson")})
 		require.ErrorContains(t, err, "invalid character 'o' in literal null")
+		mm.RequireRequestFailure(t)
+		mm.RequireTokensRecorded(t, 0)
+		mm.RequireSelected(t, "", "")
 	})
 	t.Run("router error", func(t *testing.T) {
 		headers := map[string]string{":path": "/foo"}
 		rt := mockRouter{t: t, expHeaders: headers, retErr: errors.New("test error")}
-		p := &chatCompletionProcessor{config: &processorConfig{router: rt}, requestHeaders: headers, logger: slog.Default()}
+		mm := &mockChatCompletionMetrics{}
+		p := &chatCompletionProcessor{
+			config:         &processorConfig{router: rt},
+			requestHeaders: headers,
+			logger:         slog.Default(),
+			metrics:        mm,
+		}
 		_, err := p.ProcessRequestBody(t.Context(), &extprocv3.HttpBody{Body: bodyFromModel(t, "some-model")})
 		require.ErrorContains(t, err, "failed to calculate route: test error")
+		mm.RequireRequestFailure(t)
+		mm.RequireTokensRecorded(t, 0)
+		mm.RequireSelected(t, "some-model", "")
 	})
 	t.Run("router error 404", func(t *testing.T) {
 		headers := map[string]string{":path": "/foo"}
 		rt := mockRouter{t: t, expHeaders: headers, retErr: x.ErrNoMatchingRule}
-		p := &chatCompletionProcessor{config: &processorConfig{router: rt}, requestHeaders: headers, logger: slog.Default()}
+		mm := &mockChatCompletionMetrics{}
+		p := &chatCompletionProcessor{
+			config:         &processorConfig{router: rt},
+			requestHeaders: headers,
+			logger:         slog.Default(),
+			metrics:        mm,
+		}
 		resp, err := p.ProcessRequestBody(t.Context(), &extprocv3.HttpBody{Body: bodyFromModel(t, "some-model")})
 		require.NoError(t, err)
 		require.NotNil(t, resp)
@@ -174,6 +224,9 @@ func TestChatCompletion_ProcessRequestBody(t *testing.T) {
 		require.NotNil(t, ir)
 		require.Equal(t, typev3.StatusCode_NotFound, ir.GetStatus().GetCode())
 		require.Equal(t, x.ErrNoMatchingRule.Error(), string(ir.GetBody()))
+		mm.RequireRequestFailure(t)
+		mm.RequireTokensRecorded(t, 0)
+		mm.RequireSelected(t, "some-model", "")
 	})
 	t.Run("translator not found", func(t *testing.T) {
 		headers := map[string]string{":path": "/foo"}
@@ -181,9 +234,18 @@ func TestChatCompletion_ProcessRequestBody(t *testing.T) {
 			t: t, expHeaders: headers, retBackendName: "some-backend",
 			retVersionedAPISchema: filterapi.VersionedAPISchema{Name: "some-schema", Version: "v10.0"},
 		}
-		p := &chatCompletionProcessor{config: &processorConfig{router: rt}, requestHeaders: headers, logger: slog.Default()}
+		mm := &mockChatCompletionMetrics{}
+		p := &chatCompletionProcessor{
+			config:         &processorConfig{router: rt},
+			requestHeaders: headers,
+			logger:         slog.Default(),
+			metrics:        mm,
+		}
 		_, err := p.ProcessRequestBody(t.Context(), &extprocv3.HttpBody{Body: bodyFromModel(t, "some-model")})
 		require.ErrorContains(t, err, "unsupported API schema: backend={some-schema v10.0}")
+		mm.RequireRequestFailure(t)
+		mm.RequireTokensRecorded(t, 0)
+		mm.RequireSelected(t, "some-model", "some-backend")
 	})
 	t.Run("translator error", func(t *testing.T) {
 		headers := map[string]string{":path": "/foo"}
@@ -195,12 +257,19 @@ func TestChatCompletion_ProcessRequestBody(t *testing.T) {
 		var body openai.ChatCompletionRequest
 		require.NoError(t, json.Unmarshal(someBody, &body))
 		tr := mockTranslator{t: t, retErr: errors.New("test error"), expRequestBody: &body}
+		mm := &mockChatCompletionMetrics{}
 		p := &chatCompletionProcessor{
 			config:         &processorConfig{router: rt},
-			requestHeaders: headers, logger: slog.Default(), translator: tr,
+			requestHeaders: headers,
+			logger:         slog.Default(),
+			metrics:        mm,
+			translator:     tr,
 		}
 		_, err := p.ProcessRequestBody(t.Context(), &extprocv3.HttpBody{Body: someBody})
 		require.ErrorContains(t, err, "failed to transform request: test error")
+		mm.RequireRequestFailure(t)
+		mm.RequireTokensRecorded(t, 0)
+		mm.RequireSelected(t, "some-model", "some-backend")
 	})
 	t.Run("ok", func(t *testing.T) {
 		someBody := bodyFromModel(t, "some-model")
@@ -215,11 +284,18 @@ func TestChatCompletion_ProcessRequestBody(t *testing.T) {
 		var expBody openai.ChatCompletionRequest
 		require.NoError(t, json.Unmarshal(someBody, &expBody))
 		mt := mockTranslator{t: t, expRequestBody: &expBody, retHeaderMutation: headerMut, retBodyMutation: bodyMut}
-		p := &chatCompletionProcessor{config: &processorConfig{
-			router:                   rt,
-			selectedBackendHeaderKey: "x-ai-gateway-backend-key",
-			modelNameHeaderKey:       "x-ai-gateway-model-key",
-		}, requestHeaders: headers, logger: slog.Default(), translator: mt}
+		mm := &mockChatCompletionMetrics{}
+		p := &chatCompletionProcessor{
+			config: &processorConfig{
+				router:                   rt,
+				selectedBackendHeaderKey: "x-ai-gateway-backend-key",
+				modelNameHeaderKey:       "x-ai-gateway-model-key",
+			},
+			requestHeaders: headers,
+			logger:         slog.Default(),
+			metrics:        mm,
+			translator:     mt,
+		}
 		resp, err := p.ProcessRequestBody(t.Context(), &extprocv3.HttpBody{Body: someBody})
 		require.NoError(t, err)
 		require.Equal(t, mt, p.translator)
@@ -228,6 +304,9 @@ func TestChatCompletion_ProcessRequestBody(t *testing.T) {
 		require.Equal(t, headerMut, commonRes.HeaderMutation)
 		require.Equal(t, bodyMut, commonRes.BodyMutation)
 
+		mm.RequireRequestNotCompleted(t)
+		mm.RequireSelected(t, "some-model", "some-backend")
+
 		// Check the model and backend headers are set in headerMut.
 		hdrs := headerMut.SetHeaders
 		require.Len(t, hdrs, 2)
diff --git a/internal/extproc/mocks_test.go b/internal/extproc/mocks_test.go
index 6b94e28..80ba8aa 100644
--- a/internal/extproc/mocks_test.go
+++ b/internal/extproc/mocks_test.go
@@ -10,6 +10,7 @@ import (
 	"io"
 	"log/slog"
 	"testing"
+	"time"
 
 	corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	extprocv3http "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_proc/v3"
@@ -21,6 +22,7 @@ import (
 	"github.com/envoyproxy/ai-gateway/filterapi/x"
 	"github.com/envoyproxy/ai-gateway/internal/apischema/openai"
 	"github.com/envoyproxy/ai-gateway/internal/extproc/translator"
+	"github.com/envoyproxy/ai-gateway/internal/metrics"
 )
 
 var (
@@ -168,3 +170,74 @@ func (m mockExternalProcessingStream) SendMsg(any) error { panic("TODO") }
 func (m mockExternalProcessingStream) RecvMsg(any) error { panic("TODO") }
 
 var _ extprocv3.ExternalProcessor_ProcessServer = &mockExternalProcessingStream{}
+
+// mockChatCompletionMetrics implements [metrics.ChatCompletion] for testing.
+type mockChatCompletionMetrics struct {
+	requestStart        time.Time
+	model               string
+	backend             string
+	requestSuccessCount int
+	requestErrorCount   int
+	tokenUsageCount     int
+	tokenLatencyCount   int
+}
+
+// StartRequest implements [metrics.ChatCompletion].
+func (m *mockChatCompletionMetrics) StartRequest() { m.requestStart = time.Now() }
+
+// SetModel implements [metrics.ChatCompletion].
+func (m *mockChatCompletionMetrics) SetModel(model string) { m.model = model }
+
+// SetBackend implements [metrics.ChatCompletion].
+func (m *mockChatCompletionMetrics) SetBackend(backend filterapi.Backend) { m.backend = backend.Name }
+
+// RecordTokenUsage implements [metrics.ChatCompletion].
+func (m *mockChatCompletionMetrics) RecordTokenUsage(_ context.Context, _, _, _ uint32) {
+	m.tokenUsageCount++
+}
+
+// RecordTokenLatency implements [metrics.ChatCompletion].
+func (m *mockChatCompletionMetrics) RecordTokenLatency(_ context.Context, _ uint32) {
+	m.tokenLatencyCount++
+}
+
+// RecordRequestCompletion implements [metrics.ChatCompletion].
+func (m *mockChatCompletionMetrics) RecordRequestCompletion(_ context.Context, success bool) {
+	if success {
+		m.requestSuccessCount++
+	} else {
+		m.requestErrorCount++
+	}
+}
+
+// RequireModelAndBackendSet asserts the model and backend set on the metrics.
+func (m *mockChatCompletionMetrics) RequireSelected(t *testing.T, model, backend string) {
+	require.Equal(t, model, m.model)
+	require.Equal(t, backend, m.backend)
+}
+
+// RequireRequestFailure asserts the request was marked as a failure.
+func (m *mockChatCompletionMetrics) RequireRequestFailure(t *testing.T) {
+	require.Equal(t, 0, m.requestSuccessCount)
+	require.Equal(t, 1, m.requestErrorCount)
+}
+
+// RequireRequestNotCompleted asserts the request was not completed.
+func (m *mockChatCompletionMetrics) RequireRequestNotCompleted(t *testing.T) {
+	require.Equal(t, 0, m.requestSuccessCount)
+	require.Equal(t, 0, m.requestErrorCount)
+}
+
+// RequireRequestSuccess asserts the request was marked as a success.
+func (m *mockChatCompletionMetrics) RequireRequestSuccess(t *testing.T) {
+	require.Equal(t, 1, m.requestSuccessCount)
+	require.Equal(t, 0, m.requestErrorCount)
+}
+
+// RequireTokensRecorded asserts the number of tokens recorded.
+func (m *mockChatCompletionMetrics) RequireTokensRecorded(t *testing.T, count int) {
+	require.Equal(t, count, m.tokenUsageCount)
+	require.Equal(t, count, m.tokenLatencyCount)
+}
+
+var _ metrics.ChatCompletion = &mockChatCompletionMetrics{}
diff --git a/internal/metrics/chat_completion_metrics_test.go b/internal/metrics/chat_completion_metrics_test.go
new file mode 100644
index 0000000..8c24948
--- /dev/null
+++ b/internal/metrics/chat_completion_metrics_test.go
@@ -0,0 +1,177 @@
+// Copyright Envoy AI Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
+package metrics
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/metric/metricdata"
+
+	"github.com/envoyproxy/ai-gateway/filterapi"
+)
+
+func TestNewProcessorMetrics(t *testing.T) {
+	var (
+		mr    = metric.NewManualReader()
+		meter = metric.NewMeterProvider(metric.WithReader(mr)).Meter("test")
+		pm    = NewChatCompletion(meter).(*chatCompletion)
+	)
+
+	assert.NotNil(t, pm)
+	assert.False(t, pm.firstTokenSent)
+}
+
+func TestStartRequest(t *testing.T) {
+	var (
+		mr    = metric.NewManualReader()
+		meter = metric.NewMeterProvider(metric.WithReader(mr)).Meter("test")
+		pm    = NewChatCompletion(meter).(*chatCompletion)
+	)
+
+	before := time.Now()
+	pm.StartRequest()
+	after := time.Now()
+
+	assert.False(t, pm.firstTokenSent)
+	assert.GreaterOrEqual(t, pm.requestStart, before)
+	assert.LessOrEqual(t, pm.requestStart, after)
+}
+
+func TestRecordTokenUsage(t *testing.T) {
+	var (
+		mr    = metric.NewManualReader()
+		meter = metric.NewMeterProvider(metric.WithReader(mr)).Meter("test")
+		pm    = NewChatCompletion(meter).(*chatCompletion)
+
+		attrs = []attribute.KeyValue{
+			attribute.Key(genaiAttributeOperationName).String(genaiOperationChat),
+			attribute.Key(genaiAttributeSystemName).String(genaiSystemOpenAI),
+			attribute.Key(genaiAttributeRequestModel).String("test-model"),
+		}
+		inputAttrs  = attribute.NewSet(append(attrs, attribute.Key(genaiAttributeTokenType).String(genaiTokenTypeInput))...)
+		outputAttrs = attribute.NewSet(append(attrs, attribute.Key(genaiAttributeTokenType).String(genaiTokenTypeOutput))...)
+		totalAttrs  = attribute.NewSet(append(attrs, attribute.Key(genaiAttributeTokenType).String(genaiTokenTypeTotal))...)
+	)
+
+	pm.SetModel("test-model")
+	pm.SetBackend(filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}})
+	pm.RecordTokenUsage(t.Context(), 10, 5, 15)
+
+	count, sum := getHistogramValues(t, mr, genaiMetricClientTokenUsage, inputAttrs)
+	assert.Equal(t, uint64(1), count)
+	assert.Equal(t, 10.0, sum)
+
+	count, sum = getHistogramValues(t, mr, genaiMetricClientTokenUsage, outputAttrs)
+	assert.Equal(t, uint64(1), count)
+	assert.Equal(t, 5.0, sum)
+
+	count, sum = getHistogramValues(t, mr, genaiMetricClientTokenUsage, totalAttrs)
+	assert.Equal(t, uint64(1), count)
+	assert.Equal(t, 15.0, sum)
+}
+
+func TestRecordTokenLatency(t *testing.T) {
+	var (
+		mr    = metric.NewManualReader()
+		meter = metric.NewMeterProvider(metric.WithReader(mr)).Meter("test")
+		pm    = NewChatCompletion(meter).(*chatCompletion)
+
+		attrs = attribute.NewSet(
+			attribute.Key(genaiAttributeOperationName).String(genaiOperationChat),
+			attribute.Key(genaiAttributeSystemName).String(genAISystemAWSBedrock),
+			attribute.Key(genaiAttributeRequestModel).String("test-model"),
+		)
+	)
+
+	pm.StartRequest()
+	pm.SetModel("test-model")
+	pm.SetBackend(filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaAWSBedrock}})
+
+	// Test first token.
+	time.Sleep(10 * time.Millisecond)
+	pm.RecordTokenLatency(t.Context(), 1)
+	assert.True(t, pm.firstTokenSent)
+	count, sum := getHistogramValues(t, mr, genaiMetricServerTimeToFirstToken, attrs)
+	assert.Equal(t, uint64(1), count)
+	assert.Greater(t, sum, 0.0)
+
+	// Test subsequent tokens.
+	time.Sleep(10 * time.Millisecond)
+	pm.RecordTokenLatency(t.Context(), 5)
+	count, sum = getHistogramValues(t, mr, genaiMetricServerTimePerOutputToken, attrs)
+	assert.Equal(t, uint64(1), count)
+	assert.Greater(t, sum, 0.0)
+
+	// Test zero tokens case.
+	time.Sleep(10 * time.Millisecond)
+	pm.RecordTokenLatency(t.Context(), 0)
+	count, sum = getHistogramValues(t, mr, genaiMetricServerTimePerOutputToken, attrs)
+	assert.Equal(t, uint64(1), count)
+	assert.Greater(t, sum, 0.0)
+}
+
+func TestRecordRequestCompletion(t *testing.T) {
+	var (
+		mr    = metric.NewManualReader()
+		meter = metric.NewMeterProvider(metric.WithReader(mr)).Meter("test")
+		pm    = NewChatCompletion(meter).(*chatCompletion)
+
+		attrs = []attribute.KeyValue{
+			attribute.Key(genaiAttributeOperationName).String(genaiOperationChat),
+			attribute.Key(genaiAttributeSystemName).String("custom"),
+			attribute.Key(genaiAttributeRequestModel).String("test-model"),
+		}
+		attrsSuccess = attribute.NewSet(attrs...)
+		attrsFailure = attribute.NewSet(append(attrs, attribute.Key(genaiAttributeErrorType).String(genaiErrorTypeFallback))...)
+	)
+
+	pm.StartRequest()
+	pm.SetModel("test-model")
+	pm.SetBackend(filterapi.Backend{Name: "custom"})
+
+	time.Sleep(10 * time.Millisecond)
+	pm.RecordRequestCompletion(t.Context(), true)
+	count, sum := getHistogramValues(t, mr, genaiMetricServerRequestDuration, attrsSuccess)
+	assert.Equal(t, uint64(1), count)
+	assert.Greater(t, sum, 0.0)
+
+	// Test some failed requests.
+	pm.RecordRequestCompletion(t.Context(), false)
+	pm.RecordRequestCompletion(t.Context(), false)
+	count, sum = getHistogramValues(t, mr, genaiMetricServerRequestDuration, attrsFailure)
+	assert.Equal(t, uint64(2), count)
+	assert.Greater(t, sum, 0.0)
+}
+
+// getHistogramValues returns the count and sum of a histogram metric with the given attributes.
+func getHistogramValues(t *testing.T, reader metric.Reader, metric string, attrs attribute.Set) (uint64, float64) {
+	var data metricdata.ResourceMetrics
+	require.NoError(t, reader.Collect(t.Context(), &data))
+
+	var datapoints []metricdata.HistogramDataPoint[float64]
+	for _, sm := range data.ScopeMetrics {
+		for _, m := range sm.Metrics {
+			if m.Name != metric {
+				continue
+			}
+			data := m.Data.(metricdata.Histogram[float64])
+			for _, dp := range data.DataPoints {
+				if dp.Attributes.Equals(&attrs) {
+					datapoints = append(datapoints, dp)
+				}
+			}
+		}
+	}
+
+	require.Len(t, datapoints, 1, "found %d datapoints for attributes: %v", len(datapoints), attrs)
+
+	return datapoints[0].Count, datapoints[0].Sum
+}