Repository: kubernetes-sigs/gateway-api-inference-extension

Issue:
Panic when there are no pods
```
panic: invalid argument to Intn

goroutine 254 [running]:
math/rand.(*Rand).Intn(0x945e60?, 0xc0000007e0?)
        /usr/local/go/src/math/rand/rand.go:180 +0x4c
math/rand.Intn(0x0)
        /usr/local/go/src/math/rand/rand.go:464 +0x25
sigs.k8s.io/gateway-api-inference-extension/pkg/epp/handlers.GetRandomPod({0x252f350?, 0xc0006ee000?})
        /src/pkg/epp/handlers/server.go:452 +0x34
sigs.k8s.io/gateway-api-inference-extension/pkg/epp/handlers.(*StreamingServer).HandleRequestHeaders(0xc0007bd9c0, {0x0?, 0x0?}, 0xc000b958c0, 0xc0006ae118)
        /src/pkg/epp/handlers/request.go:140 +0x78
sigs.k8s.io/gateway-api-inference-extension/pkg/epp/handlers.(*StreamingServer).Process(0xc0007bd9c0, {0x2526288, 0xc00090eba0})
        /src/pkg/epp/handlers/server.go:161 +0x3d7
github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3._ExternalProcessor_Process_Handler({0x1f96f40?, 0xc0007bd9c0}, {0x2521598, 0xc0000007e0})
        /go/pkg/mod/github.com/envoyproxy/go-control-plane/envoy@v1.32.4/service/ext_proc/v3/external_processor_grpc.pb.go:106 +0xd8
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000253000, {0x251a070, 0xc00090bc20}, 0xc000a86b40, 0xc00082faa0, 0x37c80e0, 0x0)
        /go/pkg/mod/google.golang.org/grpc@v1.71.1/server.go:1695 +0x1252
google.golang.org/grpc.(*Server).handleStream(0xc000253000, {0x251ab10, 0xc000830340}, 0xc000a86b40)
        /go/pkg/mod/google.golang.org/grpc@v1.71.1/server.go:1819 +0xb47
google.golang.org/grpc.(*Server).serveStreams.func2.1()
        /go/pkg/mod/google.golang.org/grpc@v1.71.1/server.go:1035 +0x7f
created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 253
        /go/pkg/mod/google.golang.org/grpc@v1.71.1/server.go:1046 +0x11d
```

Looks like we don't handle the case of there being no pods available

Discussion:

Solution patch:
diff --git a/pkg/epp/handlers/request.go b/pkg/epp/handlers/request.go
index 9121b59a..8d30e543 100644
--- a/pkg/epp/handlers/request.go
+++ b/pkg/epp/handlers/request.go
@@ -138,6 +138,9 @@ func (s *StreamingServer) HandleRequestHeaders(ctx context.Context, reqCtx *Requ
 		// The above PR will address endpoint admission, but currently any request without a body will be
 		// routed to a random upstream pod.
 		pod := GetRandomPod(s.datastore)
+		if pod == nil {
+			return errutil.Error{Code: errutil.Internal, Msg: "no pods available in datastore"}
+		}
 		pool, err := s.datastore.PoolGet()
 		if err != nil {
 			return err
diff --git a/pkg/epp/handlers/server.go b/pkg/epp/handlers/server.go
index 2e3a35fe..5e23c7a0 100644
--- a/pkg/epp/handlers/server.go
+++ b/pkg/epp/handlers/server.go
@@ -449,6 +449,9 @@ func RandomWeightedDraw(logger logr.Logger, model *v1alpha2.InferenceModel, seed
 
 func GetRandomPod(ds datastore.Datastore) *backendmetrics.Pod {
 	pods := ds.PodGetAll()
+	if len(pods) == 0 {
+		return nil
+	}
 	number := rand.Intn(len(pods))
 	pod := pods[number]
 	return pod.GetPod()



Test patch:
diff --git a/pkg/epp/handlers/streamingserver_test.go b/pkg/epp/handlers/streamingserver_test.go
index 72f7031a..23d2b68f 100644
--- a/pkg/epp/handlers/streamingserver_test.go
+++ b/pkg/epp/handlers/streamingserver_test.go
@@ -18,8 +18,14 @@ package handlers
 
 import (
 	"testing"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"sigs.k8s.io/gateway-api-inference-extension/api/v1alpha2"
+	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/backend/metrics"
+	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/datastore"
 	logutil "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/logging"
 )
 
@@ -126,6 +132,55 @@ func TestRandomWeightedDraw(t *testing.T) {
 	}
 }
 
+func TestGetRandomPod(t *testing.T) {
+	tests := []struct {
+		name      string
+		storePods []*corev1.Pod
+		expectNil bool
+	}{
+		{
+			name:      "No pods available",
+			storePods: []*corev1.Pod{},
+			expectNil: true,
+		},
+		{
+			name: "Single pod available",
+			storePods: []*corev1.Pod{
+				{ObjectMeta: metav1.ObjectMeta{Name: "pod1"}},
+			},
+			expectNil: false,
+		},
+		{
+			name: "Multiple pods available",
+			storePods: []*corev1.Pod{
+				{ObjectMeta: metav1.ObjectMeta{Name: "pod1"}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "pod2"}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "pod3"}},
+			},
+			expectNil: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			pmf := metrics.NewPodMetricsFactory(&metrics.FakePodMetricsClient{}, time.Millisecond)
+			ds := datastore.NewDatastore(t.Context(), pmf)
+			for _, pod := range test.storePods {
+				ds.PodUpdateOrAddIfNotExist(pod)
+			}
+
+			gotPod := GetRandomPod(ds)
+
+			if test.expectNil && gotPod != nil {
+				t.Errorf("expected nil pod, got: %v", gotPod)
+			}
+			if !test.expectNil && gotPod == nil {
+				t.Errorf("expected non-nil pod, got nil")
+			}
+		})
+	}
+}
+
 func pointer(v int32) *int32 {
 	return &v
 }