Repository: envoyproxy/ai-gateway
Pull request #525 (2025-03-26 16:50:46)

extproc: pass backend to SetBackend  by pointer
**Commit Message**

Previously, the recently introduced SetBackend API of x.ChatCompletionMetrics was accepting filterapi.Backend by value while the existing Router API returns it by pointer. Since the backend object is allocated prior to the call to the method at config loading time, there's no reason to pass it by value considering the size of the struct. 


Patch:
diff --git a/examples/extproc_custom_metrics/main.go b/examples/extproc_custom_metrics/main.go
index 56515c5..893e66f 100644
--- a/examples/extproc_custom_metrics/main.go
+++ b/examples/extproc_custom_metrics/main.go
@@ -61,7 +61,7 @@ func (m *myCustomChatCompletionMetrics) SetModel(model string) {
 	m.logger.Info("SetModel", "model", model)
 }
 
-func (m *myCustomChatCompletionMetrics) SetBackend(backend filterapi.Backend) {
+func (m *myCustomChatCompletionMetrics) SetBackend(backend *filterapi.Backend) {
 	m.logger.Info("SetBackend", "backend", backend.Name)
 }
 
diff --git a/filterapi/x/x.go b/filterapi/x/x.go
index 603b066..276a4c5 100644
--- a/filterapi/x/x.go
+++ b/filterapi/x/x.go
@@ -59,7 +59,7 @@ type ChatCompletionMetrics interface {
 	SetModel(model string)
 	// SetBackend sets the selected backend when the routing decision has been made. This is usually called
 	// after parsing the request body to determine the model and invoke the routing logic.
-	SetBackend(backend filterapi.Backend)
+	SetBackend(backend *filterapi.Backend)
 
 	// RecordTokenUsage records token usage metrics.
 	RecordTokenUsage(ctx context.Context, inputTokens, outputTokens, totalTokens uint32, extraAttrs ...attribute.KeyValue)
diff --git a/internal/extproc/chatcompletion_processor.go b/internal/extproc/chatcompletion_processor.go
index 8ba087f..4a021f8 100644
--- a/internal/extproc/chatcompletion_processor.go
+++ b/internal/extproc/chatcompletion_processor.go
@@ -117,11 +117,10 @@ func (c *chatCompletionProcessor) ProcessRequestBody(ctx context.Context, rawBod
 				},
 			}, nil
 		}
-
 		return nil, fmt.Errorf("failed to calculate route: %w", err)
 	}
 	c.logger.Info("Selected backend", "backend", b.Name, "schema", b.Schema)
-	c.metrics.SetBackend(*b)
+	c.metrics.SetBackend(b)
 
 	if err = c.selectTranslator(b.Schema); err != nil {
 		return nil, fmt.Errorf("failed to select translator: %w", err)
diff --git a/internal/metrics/chat_completion_metrics.go b/internal/metrics/chat_completion_metrics.go
index 0f4c027..f3b6b2d 100644
--- a/internal/metrics/chat_completion_metrics.go
+++ b/internal/metrics/chat_completion_metrics.go
@@ -56,7 +56,7 @@ func (c *chatCompletion) SetModel(model string) {
 
 // SetBackend sets the name of the backend to be reported in the metrics according to:
 // https://opentelemetry.io/docs/specs/semconv/attributes-registry/gen-ai/#gen-ai-system
-func (c *chatCompletion) SetBackend(backend filterapi.Backend) {
+func (c *chatCompletion) SetBackend(backend *filterapi.Backend) {
 	switch backend.Schema.Name {
 	case filterapi.APISchemaOpenAI:
 		c.backend = genaiSystemOpenAI


Test patch:
diff --git a/internal/extproc/mocks_test.go b/internal/extproc/mocks_test.go
index 62229d0..51bce55 100644
--- a/internal/extproc/mocks_test.go
+++ b/internal/extproc/mocks_test.go
@@ -187,7 +187,7 @@ func (m *mockChatCompletionMetrics) StartRequest(_ map[string]string) { m.reques
 func (m *mockChatCompletionMetrics) SetModel(model string) { m.model = model }
 
 // SetBackend implements [metrics.ChatCompletion].
-func (m *mockChatCompletionMetrics) SetBackend(backend filterapi.Backend) { m.backend = backend.Name }
+func (m *mockChatCompletionMetrics) SetBackend(backend *filterapi.Backend) { m.backend = backend.Name }
 
 // RecordTokenUsage implements [metrics.ChatCompletion].
 func (m *mockChatCompletionMetrics) RecordTokenUsage(_ context.Context, _, _, _ uint32, _ ...attribute.KeyValue) {
diff --git a/internal/metrics/chat_completion_metrics_test.go b/internal/metrics/chat_completion_metrics_test.go
index c0f29d3..99893c3 100644
--- a/internal/metrics/chat_completion_metrics_test.go
+++ b/internal/metrics/chat_completion_metrics_test.go
@@ -64,7 +64,7 @@ func TestRecordTokenUsage(t *testing.T) {
 	)
 
 	pm.SetModel("test-model")
-	pm.SetBackend(filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}})
+	pm.SetBackend(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}})
 	pm.RecordTokenUsage(t.Context(), 10, 5, 15, extra)
 
 	count, sum := getHistogramValues(t, mr, genaiMetricClientTokenUsage, inputAttrs)
@@ -97,7 +97,7 @@ func TestRecordTokenLatency(t *testing.T) {
 
 	pm.StartRequest(nil)
 	pm.SetModel("test-model")
-	pm.SetBackend(filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaAWSBedrock}})
+	pm.SetBackend(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaAWSBedrock}})
 
 	// Test first token.
 	time.Sleep(10 * time.Millisecond)
@@ -141,7 +141,7 @@ func TestRecordRequestCompletion(t *testing.T) {
 
 	pm.StartRequest(nil)
 	pm.SetModel("test-model")
-	pm.SetBackend(filterapi.Backend{Name: "custom"})
+	pm.SetBackend(&filterapi.Backend{Name: "custom"})
 
 	time.Sleep(10 * time.Millisecond)
 	pm.RecordRequestCompletion(t.Context(), true, extra)