Repository: kubernetes-sigs/gateway-api-inference-extension
Pull request #545: Add some more unit tests for BBR
Author: login: rramkumar1
Labels: name: cncf-cla: yes
description: Indicates the PR's author has signed the CNCF CLA.
name: size/L
description: Denotes a PR that changes 100-499 lines, ignoring generated files.
name: approved
description: Indicates a PR has been approved by an approver from all required OWNERS files.
name: lgtm
description: "Looks good to me", indicates that a PR is ready to be merged.
name: ok-to-test
description: Indicates a non-member PR verified by an org member that is safe to test.
Created: 2025-03-20T04:06:37Z | Merged: 2025-03-20T17:24:33Z

Added unit tests verify the logic to handle stream body buffer.


Diff:
diff --git a/pkg/body-based-routing/handlers/server.go b/pkg/body-based-routing/handlers/server.go
index fee8f78c21..24664f9889 100644
--- a/pkg/body-based-routing/handlers/server.go
+++ b/pkg/body-based-routing/handlers/server.go
@@ -46,7 +46,7 @@ func (s *Server) Process(srv extProcPb.ExternalProcessor_ProcessServer) error {
 	loggerVerbose := logger.V(logutil.VERBOSE)
 	loggerVerbose.Info("Processing")
 
-	var streamedBody []byte
+	streamedBody := &streamedBody{}
 
 	for {
 		select {
@@ -105,17 +105,22 @@ func (s *Server) Process(srv extProcPb.ExternalProcessor_ProcessServer) error {
 	}
 }
 
-func (s *Server) processRequestBody(ctx context.Context, body *extProcPb.HttpBody, streamedBody []byte, logger logr.Logger) ([]*extProcPb.ProcessingResponse, error) {
+type streamedBody struct {
+	body []byte
+}
+
+func (s *Server) processRequestBody(ctx context.Context, body *extProcPb.HttpBody, streamedBody *streamedBody, logger logr.Logger) ([]*extProcPb.ProcessingResponse, error) {
 	loggerVerbose := logger.V(logutil.VERBOSE)
 
 	var requestBody map[string]interface{}
 	if s.streaming {
 		// In the stream case, we can receive multiple request bodies.
-		streamedBody = append(streamedBody, body.Body...)
-
-		if body.EndOfStream {
+		if !body.EndOfStream {
+			streamedBody.body = append(streamedBody.body, body.Body...)
+			return nil, nil
+		} else {
 			loggerVerbose.Info("Flushing stream buffer")
-			err := json.Unmarshal(streamedBody, &requestBody)
+			err := json.Unmarshal(streamedBody.body, &requestBody)
 			if err != nil {
 				logger.V(logutil.DEFAULT).Error(err, "Error unmarshaling request body")
 			}
diff --git a/pkg/body-based-routing/handlers/server_test.go b/pkg/body-based-routing/handlers/server_test.go
new file mode 100644
index 0000000000..f4e8e25418
--- /dev/null
+++ b/pkg/body-based-routing/handlers/server_test.go
@@ -0,0 +1,145 @@
+/*
+Copyright 2025 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package handlers
+
+import (
+	"context"
+	"testing"
+
+	basepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	extProcPb "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3"
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/protobuf/testing/protocmp"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	logutil "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/logging"
+)
+
+func TestProcessRequestBody(t *testing.T) {
+	ctx := logutil.NewTestLoggerIntoContext(context.Background())
+
+	cases := []struct {
+		desc      string
+		streaming bool
+		bodys     []*extProcPb.HttpBody
+		want      []*extProcPb.ProcessingResponse
+	}{
+		{
+			desc: "no-streaming",
+			bodys: []*extProcPb.HttpBody{
+				{
+					Body: mapToBytes(t, map[string]any{
+						"model": "foo",
+					}),
+				},
+			},
+			want: []*extProcPb.ProcessingResponse{
+				{
+					Response: &extProcPb.ProcessingResponse_RequestBody{
+						RequestBody: &extProcPb.BodyResponse{
+							Response: &extProcPb.CommonResponse{
+								// Necessary so that the new headers are used in the routing decision.
+								ClearRouteCache: true,
+								HeaderMutation: &extProcPb.HeaderMutation{
+									SetHeaders: []*basepb.HeaderValueOption{
+										{
+											Header: &basepb.HeaderValue{
+												Key:      modelHeader,
+												RawValue: []byte("foo"),
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			desc:      "streaming",
+			streaming: true,
+			bodys: []*extProcPb.HttpBody{
+				{
+					Body: mapToBytes(t, map[string]any{
+						"model": "foo",
+					}),
+				},
+				{
+					EndOfStream: true,
+				},
+			},
+			want: []*extProcPb.ProcessingResponse{
+				{
+					Response: &extProcPb.ProcessingResponse_RequestHeaders{
+						RequestHeaders: &extProcPb.HeadersResponse{
+							Response: &extProcPb.CommonResponse{
+								ClearRouteCache: true,
+								HeaderMutation: &extProcPb.HeaderMutation{
+									SetHeaders: []*basepb.HeaderValueOption{
+										{
+											Header: &basepb.HeaderValue{
+												Key:      modelHeader,
+												RawValue: []byte("foo"),
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+				{
+					Response: &extProcPb.ProcessingResponse_RequestBody{
+						RequestBody: &extProcPb.BodyResponse{
+							Response: &extProcPb.CommonResponse{
+								BodyMutation: &extProcPb.BodyMutation{
+									Mutation: &extProcPb.BodyMutation_StreamedResponse{
+										StreamedResponse: &extProcPb.StreamedBodyResponse{
+											Body: mapToBytes(t, map[string]any{
+												"model": "foo",
+											}),
+											EndOfStream: true,
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.desc, func(t *testing.T) {
+			srv := NewServer(tc.streaming)
+			streamedBody := &streamedBody{}
+			for i, body := range tc.bodys {
+				got, err := srv.processRequestBody(context.Background(), body, streamedBody, log.FromContext(ctx))
+				if err != nil {
+					t.Fatalf("processRequestBody(): %v", err)
+				}
+
+				if i == len(tc.bodys)-1 {
+					if diff := cmp.Diff(tc.want, got, protocmp.Transform()); diff != "" {
+						t.Errorf("processRequestBody returned unexpected response, diff(-want, +got): %v", diff)
+					}
+				}
+			}
+		})
+	}
+}


Reviews:
user: login: danehans
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T16:18:12Z
user: login: rramkumar1
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T16:25:34Z

Review comments:
diff_hunk: @@ -0,0 +1,146 @@
+/*
+Copyright 2025 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package handlers
+
+import (
+	"context"
+	"testing"
+
+	basepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	eppb "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3"
+	extProcPb "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3"
path: pkg/body-based-routing/handlers/server_test.go
user: login: danehans
body: Duplicate imports.
created_at: 2025-03-20T16:18:12Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,146 @@
+/*
+Copyright 2025 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package handlers
+
+import (
+	"context"
+	"testing"
+
+	basepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	eppb "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3"
+	extProcPb "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3"
path: pkg/body-based-routing/handlers/server_test.go
user: login: rramkumar1
body: Done
created_at: 2025-03-20T16:25:34Z
author_association: CONTRIBUTOR

PR comments:
user: login: k8s-ci-robot
created_at: 2025-03-20T04:06:47Z
body: Hi @rramkumar1. Thanks for your PR.

I'm waiting for a [kubernetes-sigs](https://github.com/orgs/kubernetes-sigs/people) member to verify that this patch is reasonable to test. If it is, they should reply with `/ok-to-test` on its own line. Until that is done, I will not automatically test new commits in this PR, but the usual testing commands by org members will still work. Regular contributors should [join the org](https://git.k8s.io/community/community-membership.md#member) to skip this step.

Once the patch is verified, the new status will be reflected by the `ok-to-test` label.

I understand the commands that are listed [here](https://go.k8s.io/bot-commands?repo=kubernetes-sigs%2Fgateway-api-inference-extension).

<details>

Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository.
</details>

author_association: CONTRIBUTOR
user: login: netlify[bot]
created_at: 2025-03-20T04:06:58Z
body: ### <span aria-hidden="true">✅</span> Deploy Preview for *gateway-api-inference-extension* ready!


|  Name | Link |
|:-:|------------------------|
|<span aria-hidden="true">🔨</span> Latest commit | bef797dcc123416e2223d552d6e8b8aa3f21e8d7 |
|<span aria-hidden="true">🔍</span> Latest deploy log | https://app.netlify.com/sites/gateway-api-inference-extension/deploys/67dc417aca172a000815fe53 |
|<span aria-hidden="true">😎</span> Deploy Preview | [https://deploy-preview-545--gateway-api-inference-extension.netlify.app](https://deploy-preview-545--gateway-api-inference-extension.netlify.app) |
|<span aria-hidden="true">📱</span> Preview on mobile | <details><summary> Toggle QR Code... </summary><br /><br />![QR Code](https://app.netlify.com/qr-code/eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1cmwiOiJodHRwczovL2RlcGxveS1wcmV2aWV3LTU0NS0tZ2F0ZXdheS1hcGktaW5mZXJlbmNlLWV4dGVuc2lvbi5uZXRsaWZ5LmFwcCJ9.cEWL_xS5PFRSiYqDcaTK-JG9s4rmrOBaaLkBsVcyIuQ)<br /><br />_Use your smartphone camera to open QR code link._</details> |
---
<!-- [gateway-api-inference-extension Preview](https://deploy-preview-545--gateway-api-inference-extension.netlify.app) -->
_To edit notification comments on pull requests, go to your [Netlify site configuration](https://app.netlify.com/sites/gateway-api-inference-extension/configuration/notifications#deploy-webhooks)._
author_association: NONE
user: login: rramkumar1
created_at: 2025-03-20T04:10:41Z
body: /assign @kfswain 
author_association: CONTRIBUTOR
user: login: ahg-g
created_at: 2025-03-20T15:47:14Z
body: /ok-to-test
author_association: CONTRIBUTOR
user: login: kfswain
created_at: 2025-03-20T16:13:10Z
body: /retest

author_association: COLLABORATOR
user: login: danehans
created_at: 2025-03-20T16:27:49Z
body: /lgtm
author_association: CONTRIBUTOR
user: login: ahg-g
created_at: 2025-03-20T17:23:29Z
body: /lgtm
/approve
author_association: CONTRIBUTOR
user: login: k8s-ci-robot
created_at: 2025-03-20T17:23:36Z
body: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/kubernetes-sigs/gateway-api-inference-extension/pull/545#issuecomment-2741177911" title="Approved">ahg-g</a>*, *<a href="https://github.com/kubernetes-sigs/gateway-api-inference-extension/pull/545#" title="Author self-approved">rramkumar1</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=kubernetes-sigs%2Fgateway-api-inference-extension).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/kubernetes-sigs/gateway-api-inference-extension/blob/main/OWNERS)~~ [ahg-g]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
author_association: CONTRIBUTOR