Repository: kubernetes-sigs/gateway-api-inference-extension

Issue:
integration tests flakes
I got a test run failure: https://prow.k8s.io/view/gs/kubernetes-ci-logs/pr-logs/pull/kubernetes-sigs_gateway-api-inference-extension/842/pull-gateway-api-inference-extension-test-unit-main/1923088072233193472

In a purely log level change focused PR: https://github.com/kubernetes-sigs/gateway-api-inference-extension/pull/842

Discussion:
@LukeAVanDrie: /assign @LukeAVanDrie 

Solution patch:
diff --git a/pkg/epp/scheduling/plugins/filter/decision_tree_filter.go b/pkg/epp/scheduling/plugins/filter/decision_tree_filter.go
index 399780b..066a90d 100644
--- a/pkg/epp/scheduling/plugins/filter/decision_tree_filter.go
+++ b/pkg/epp/scheduling/plugins/filter/decision_tree_filter.go
@@ -22,6 +22,9 @@ import (
 	logutil "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/logging"
 )
 
+// compile-time type validation
+var _ plugins.Filter = &DecisionTreeFilter{}
+
 // DecisionTreeFilter applies current fitler, and then recursively applies next filters
 // depending success or failure of the current filter.
 // It can be used to construct a flow chart algorithm.
diff --git a/pkg/epp/scheduling/plugins/filter/least_kvcache_filter.go b/pkg/epp/scheduling/plugins/filter/least_kvcache_filter.go
index eed6476..ee7d824 100644
--- a/pkg/epp/scheduling/plugins/filter/least_kvcache_filter.go
+++ b/pkg/epp/scheduling/plugins/filter/least_kvcache_filter.go
@@ -26,7 +26,7 @@ import (
 // compile-time type validation
 var _ plugins.Filter = &LeastKVCacheFilter{}
 
-// NewLeastKVCacheFilter returns a new LeastKVCacheFilter.
+// NewLeastKVCacheFilter initializes a new LeastKVCacheFilter and returns its pointer.
 func NewLeastKVCacheFilter() *LeastKVCacheFilter {
 	return &LeastKVCacheFilter{}
 }
diff --git a/pkg/epp/scheduling/plugins/filter/least_queue_filter.go b/pkg/epp/scheduling/plugins/filter/least_queue_filter.go
index 7a71d51..7e0bfc3 100644
--- a/pkg/epp/scheduling/plugins/filter/least_queue_filter.go
+++ b/pkg/epp/scheduling/plugins/filter/least_queue_filter.go
@@ -26,7 +26,7 @@ import (
 // compile-time type validation
 var _ plugins.Filter = &LeastQueueFilter{}
 
-// NewLeastQueueFilter returns a new LeastQueueFilter.
+// NewLeastQueueFilter initializes a new LeastQueueFilter and returns its pointer.
 func NewLeastQueueFilter() *LeastQueueFilter {
 	return &LeastQueueFilter{}
 }
@@ -35,8 +35,7 @@ func NewLeastQueueFilter() *LeastQueueFilter {
 // (max-min) by the number of pods, and finds the pods that fall into the first range.
 // The intuition is that if there are multiple pods that share similar queue size in the low range,
 // we should consider them all instead of the absolute minimum one. This worked better than picking
-// the least one as it gives more choices for the next filter, which on aggregate gave better
-// results.
+// the least one as it gives more choices for the next filter, which on aggregate gave better results.
 type LeastQueueFilter struct{}
 
 // Name returns the name of the filter.
diff --git a/pkg/epp/scheduling/plugins/filter/lora_affinity_filter.go b/pkg/epp/scheduling/plugins/filter/lora_affinity_filter.go
index bc744a8..bc1e55b 100644
--- a/pkg/epp/scheduling/plugins/filter/lora_affinity_filter.go
+++ b/pkg/epp/scheduling/plugins/filter/lora_affinity_filter.go
@@ -28,7 +28,7 @@ import (
 // compile-time type validation
 var _ plugins.Filter = &LoraAffinityFilter{}
 
-// NewLoraAffinityFilter returns a new LoraAffinityFilter.
+// NewLoraAffinityFilter initializes a new LoraAffinityFilter and returns its pointer.
 func NewLoraAffinityFilter() *LoraAffinityFilter {
 	return &LoraAffinityFilter{
 		loraAffinityThreshold: config.Conf.LoraAffinityThreshold,
diff --git a/pkg/epp/scheduling/plugins/filter/low_queue_filter.go b/pkg/epp/scheduling/plugins/filter/low_queue_filter.go
index feb599b..ce7d352 100644
--- a/pkg/epp/scheduling/plugins/filter/low_queue_filter.go
+++ b/pkg/epp/scheduling/plugins/filter/low_queue_filter.go
@@ -25,7 +25,7 @@ import (
 // compile-time type validation
 var _ plugins.Filter = &LowQueueFilter{}
 
-// NewLowQueueFilter returns a new LowQueueFilter.
+// NewLowQueueFilter initializes a new LowQueueFilter and returns its pointer.
 func NewLowQueueFilter() *LowQueueFilter {
 	return &LowQueueFilter{
 		queueingThresholdLoRA: config.Conf.QueueingThresholdLoRA,
diff --git a/pkg/epp/scheduling/plugins/filter/sheddable_capacity_filter.go b/pkg/epp/scheduling/plugins/filter/sheddable_capacity_filter.go
index 5a298a0..cdc3355 100644
--- a/pkg/epp/scheduling/plugins/filter/sheddable_capacity_filter.go
+++ b/pkg/epp/scheduling/plugins/filter/sheddable_capacity_filter.go
@@ -25,7 +25,7 @@ import (
 // compile-time type validation
 var _ plugins.Filter = &SheddableCapacityFilter{}
 
-// NewSheddableCapacityFilter returns a new SheddableCapacityFilter.
+// NewSheddableCapacityFilter initializes a new SheddableCapacityFilter and returns its pointer.
 func NewSheddableCapacityFilter() *SheddableCapacityFilter {
 	return &SheddableCapacityFilter{
 		queueThreshold:   config.Conf.QueueThresholdCritical,
diff --git a/pkg/epp/scheduling/plugins/multi/prefix/plugin.go b/pkg/epp/scheduling/plugins/multi/prefix/plugin.go
index de2cf70..b07f085 100644
--- a/pkg/epp/scheduling/plugins/multi/prefix/plugin.go
+++ b/pkg/epp/scheduling/plugins/multi/prefix/plugin.go
@@ -23,6 +23,7 @@ import (
 	"github.com/cespare/xxhash/v2"
 	k8stypes "k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/metrics"
+	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/plugins"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/types"
 	logutil "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/logging"
 )
@@ -87,6 +88,7 @@ func (s ServerID) String() string {
 	return k8stypes.NamespacedName(s).String()
 }
 
+// compile-time type validation
 var _ types.StateData = &schedulingContextState{}
 
 // This is the state of this plugin to be used during a scheduling cycle.
@@ -111,6 +113,12 @@ func (s *schedulingContextState) Clone() types.StateData {
 	}
 }
 
+// compile-time type validation
+var _ plugins.PreSchedule = &Plugin{}
+var _ plugins.Scorer = &Plugin{}
+var _ plugins.PostSchedule = &Plugin{}
+
+// New initializes a new prefix Plugin and returns its pointer.
 func New(config Config) *Plugin {
 	m := &Plugin{
 		Config:  config,
@@ -124,6 +132,7 @@ func (m *Plugin) Name() string {
 	return "prefix-cache"
 }
 
+// PreSchedule initializes the prefix plugin state for the current scheduling cycle.
 func (m *Plugin) PreSchedule(ctx *types.SchedulingContext) {
 	hashes := hashPrompt(ctx, m.HashBlockSize, m.MaxPrefixBlocksToMatch)
 	state := &schedulingContextState{
@@ -135,7 +144,7 @@ func (m *Plugin) PreSchedule(ctx *types.SchedulingContext) {
 	ctx.Logger.V(logutil.TRACE).Info(fmt.Sprintf("PreSchedule, cached servers: %+v", state.PrefixCacheServers), "hashes", state.PrefixHashes)
 }
 
-// If a request was routed to a server, record it in the cache:
+// PostSchedule records in the plugin cache the result of the scheduling selection.
 func (m *Plugin) PostSchedule(ctx *types.SchedulingContext, res *types.Result) {
 	targetPod := res.TargetPod.GetPod()
 	state, err := m.getPrefixState(ctx.CycleState)
@@ -199,6 +208,7 @@ func (m *Plugin) matchLongestPrefix(ctx *types.SchedulingContext, hashes []Block
 	return res
 }
 
+// getPrefixState returns the cycle state as a schedulingContextState.
 func (m *Plugin) getPrefixState(cycleState *types.CycleState) (*schedulingContextState, error) {
 	prefixStateKey := types.StateKey(m.Name())
 	state, err := cycleState.Read(prefixStateKey)
diff --git a/pkg/epp/scheduling/plugins/picker/max_score_picker.go b/pkg/epp/scheduling/plugins/picker/max_score_picker.go
index a6d7b39..e85aee5 100644
--- a/pkg/epp/scheduling/plugins/picker/max_score_picker.go
+++ b/pkg/epp/scheduling/plugins/picker/max_score_picker.go
@@ -24,11 +24,13 @@ import (
 	logutil "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/logging"
 )
 
+// compile-time type validation
 var _ plugins.Picker = &MaxScorePicker{}
 
-func NewMaxScorePicker() plugins.Picker {
+// NewMaxScorePicker initializes a new MaxScorePicker and returns its pointer.
+func NewMaxScorePicker() *MaxScorePicker {
 	return &MaxScorePicker{
-		random: &RandomPicker{},
+		random: NewRandomPicker(),
 	}
 }
 
diff --git a/pkg/epp/scheduling/plugins/picker/random_picker.go b/pkg/epp/scheduling/plugins/picker/random_picker.go
index fb9f9a2..1d12198 100644
--- a/pkg/epp/scheduling/plugins/picker/random_picker.go
+++ b/pkg/epp/scheduling/plugins/picker/random_picker.go
@@ -25,15 +25,23 @@ import (
 	logutil "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/logging"
 )
 
+// compile-time type validation
 var _ plugins.Picker = &RandomPicker{}
 
+// NewRandomPicker initializes a new RandomPicker and returns its pointer.
+func NewRandomPicker() *RandomPicker {
+	return &RandomPicker{}
+}
+
 // RandomPicker picks a random pod from the list of candidates.
 type RandomPicker struct{}
 
+// Name returns the name of the picker.
 func (p *RandomPicker) Name() string {
 	return "random"
 }
 
+// Pick selects a random pod from the list of candidates.
 func (p *RandomPicker) Pick(ctx *types.SchedulingContext, scoredPods []*types.ScoredPod) *types.Result {
 	ctx.Logger.V(logutil.DEBUG).Info(fmt.Sprintf("Selecting a random pod from %d candidates: %+v", len(scoredPods), scoredPods))
 	i := rand.Intn(len(scoredPods))



Test patch:
diff --git a/test/integration/epp/hermetic_test.go b/test/integration/epp/hermetic_test.go
index 9233925..3035a9b 100644
--- a/test/integration/epp/hermetic_test.go
+++ b/test/integration/epp/hermetic_test.go
@@ -55,7 +55,6 @@ import (
 	metricsutils "k8s.io/component-base/metrics/testutil"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
-	"sigs.k8s.io/controller-runtime/pkg/client"
 	k8sclient "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/config"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
@@ -66,7 +65,6 @@ import (
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/datastore"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/metrics"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling"
-	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/server"
 	runserver "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/server"
 	logutil "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/logging"
 	epptestutil "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/testing"
@@ -1270,7 +1268,12 @@ func TestFullDuplexStreamed_KubeInferenceModelRequest(t *testing.T) {
 			if err != nil && !test.wantErr {
 				t.Errorf("Unexpected error, got: %v, want error: %v", err, test.wantErr)
 			}
-			if diff := cmp.Diff(test.wantResponses, responses, protocmp.Transform()); diff != "" {
+			if diff := cmp.Diff(test.wantResponses, responses,
+				protocmp.Transform(),
+				protocmp.SortRepeated(func(a, b *configPb.HeaderValueOption) bool {
+					return a.GetHeader().GetKey() < b.GetHeader().GetKey()
+				}),
+			); diff != "" {
 				t.Errorf("Unexpected response, (-want +got): %v", diff)
 			}
 
@@ -1398,7 +1401,7 @@ func BeforeSuite() func() {
 	// Init runtime.
 	ctrl.SetLogger(logger)
 
-	mgr, err := server.NewManagerWithOptions(cfg, managerTestOptions("default", "vllm-llama3-8b-instruct-pool"))
+	mgr, err := runserver.NewManagerWithOptions(cfg, managerTestOptions("default", "vllm-llama3-8b-instruct-pool"))
 	if err != nil {
 		logutil.Fatal(logger, err, "Failed to create controller manager")
 	}
@@ -1537,7 +1540,7 @@ func managerTestOptions(namespace, name string) ctrl.Options {
 	return ctrl.Options{
 		Scheme: scheme,
 		Cache: cache.Options{
-			ByObject: map[client.Object]cache.ByObject{
+			ByObject: map[k8sclient.Object]cache.ByObject{
 				&corev1.Pod{}: {
 					Namespaces: map[string]cache.Config{
 						namespace: {},