Repository: kubernetes-sigs/gateway-api-inference-extension
Pull request #809: merge has capacity filter with sheddable filter.
Author: login: nirrozenbaum
Labels: name: cncf-cla: yes
description: Indicates the PR's author has signed the CNCF CLA.
name: approved
description: Indicates a PR has been approved by an approver from all required OWNERS files.
name: lgtm
description: "Looks good to me", indicates that a PR is ready to be merged.
name: size/M
description: Denotes a PR that changes 30-99 lines, ignoring generated files.
Created: 2025-05-09T05:48:08Z | Merged: 2025-05-09T20:23:15Z

has capacity only use was for sheddable requests (passthrough for critical ones).


Diff:
diff --git a/pkg/epp/scheduling/plugins/filter/filter_test.go b/pkg/epp/scheduling/plugins/filter/filter_test.go
index 1818c52099..d78452a620 100644
--- a/pkg/epp/scheduling/plugins/filter/filter_test.go
+++ b/pkg/epp/scheduling/plugins/filter/filter_test.go
@@ -131,8 +131,9 @@ func TestFilter(t *testing.T) {
 			},
 		},
 		{
-			name:   "lowQueueAndLessThanKVCacheThresholdPredicate",
-			filter: &HasCapacityFilter{queueThreshold: 0, kvCacheThreshold: 0.8},
+			name:   "SheddableCapacityFilter, sheddable request",
+			req:    &types.LLMRequest{Critical: false},
+			filter: &SheddableCapacityFilter{queueThreshold: 0, kvCacheThreshold: 0.8},
 			input: []types.Pod{
 				&types.PodMetrics{
 					// This pod should be returned.
diff --git a/pkg/epp/scheduling/plugins/filter/has_capacity_filter.go b/pkg/epp/scheduling/plugins/filter/sheddable_capacity_filter.go
similarity index 67%
rename from pkg/epp/scheduling/plugins/filter/has_capacity_filter.go
rename to pkg/epp/scheduling/plugins/filter/sheddable_capacity_filter.go
index e6ff2ebfb3..5a298a0226 100644
--- a/pkg/epp/scheduling/plugins/filter/has_capacity_filter.go
+++ b/pkg/epp/scheduling/plugins/filter/sheddable_capacity_filter.go
@@ -23,29 +23,33 @@ import (
 )
 
 // compile-time type validation
-var _ plugins.Filter = &HasCapacityFilter{}
+var _ plugins.Filter = &SheddableCapacityFilter{}
 
-// NewHasCapacityFilter returns a new HasCapacityFilter.
-func NewHasCapacityFilter() *HasCapacityFilter {
-	return &HasCapacityFilter{
+// NewSheddableCapacityFilter returns a new SheddableCapacityFilter.
+func NewSheddableCapacityFilter() *SheddableCapacityFilter {
+	return &SheddableCapacityFilter{
 		queueThreshold:   config.Conf.QueueThresholdCritical,
 		kvCacheThreshold: config.Conf.KVCacheThreshold,
 	}
 }
 
-// HasCapacityFilter filters only pods that has capacity for sheddable requests.
-type HasCapacityFilter struct {
+// SheddableCapacityFilter filters only pods that has capacity for sheddable requests.
+type SheddableCapacityFilter struct {
 	queueThreshold   int
 	kvCacheThreshold float64
 }
 
 // Name returns the name of the filter.
-func (f *HasCapacityFilter) Name() string {
-	return "has-capacity"
+func (f *SheddableCapacityFilter) Name() string {
+	return "sheddable-capacity"
 }
 
 // Filter filters out pods that doesn't meet the filter criteria.
-func (f *HasCapacityFilter) Filter(ctx *types.SchedulingContext, pods []types.Pod) []types.Pod {
+func (f *SheddableCapacityFilter) Filter(ctx *types.SchedulingContext, pods []types.Pod) []types.Pod {
+	if ctx.Req.Critical {
+		return pods // // Allow all pods to passthrough if the request is critical, even if all pods reach their capacity.
+	}
+
 	filteredPods := []types.Pod{}
 
 	for _, pod := range pods {
diff --git a/pkg/epp/scheduling/plugins/filter/sheddable_request_filter.go b/pkg/epp/scheduling/plugins/filter/sheddable_request_filter.go
deleted file mode 100644
index 7eb6326bec..0000000000
--- a/pkg/epp/scheduling/plugins/filter/sheddable_request_filter.go
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
-Copyright 2025 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package filter
-
-import (
-	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/plugins"
-	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/types"
-)
-
-// compile-time type validation
-var _ plugins.Filter = &SheddableRequestFilter{}
-
-// NewSheddableRequestFilter returns a new SheddableRequestFilter.
-func NewSheddableRequestFilter() *SheddableRequestFilter {
-	return &SheddableRequestFilter{
-		hasCapacityFilter: NewHasCapacityFilter(),
-	}
-}
-
-// LowQueueFilter returns pods that their waiting queue size is less than a configured threshold
-type SheddableRequestFilter struct {
-	hasCapacityFilter *HasCapacityFilter
-}
-
-// Name returns the name of the filter.
-func (f *SheddableRequestFilter) Name() string {
-	return "sheddable-request"
-}
-
-// Filter filters out pods that doesn't meet the filter criteria.
-func (f *SheddableRequestFilter) Filter(ctx *types.SchedulingContext, pods []types.Pod) []types.Pod {
-	if ctx.Req.Critical {
-		// Allow all pods to pass through if the request is critical, even if all pods reach their capacity.
-		return pods
-	}
-
-	// Only allow pods that have enough capacity to handle the request.
-	return f.hasCapacityFilter.Filter(ctx, pods)
-}
diff --git a/pkg/epp/scheduling/scheduler.go b/pkg/epp/scheduling/scheduler.go
index b967304372..2e85619af0 100644
--- a/pkg/epp/scheduling/scheduler.go
+++ b/pkg/epp/scheduling/scheduler.go
@@ -66,7 +66,7 @@ func NewScheduler(datastore Datastore) *Scheduler {
 
 	defaultConfig := &SchedulerConfig{
 		preSchedulePlugins:  []plugins.PreSchedule{},
-		filters:             []plugins.Filter{filter.NewSheddableRequestFilter(), lowLatencyFilter},
+		filters:             []plugins.Filter{filter.NewSheddableCapacityFilter(), lowLatencyFilter},
 		scorers:             map[plugins.Scorer]int{},
 		picker:              &picker.RandomPicker{},
 		postSchedulePlugins: []plugins.PostSchedule{},


Reviews:
user: login: liu-cong
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-09T16:30:52Z
user: login: LukeAVanDrie
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-09T19:25:56Z

Review comments:
diff_hunk: @@ -132,7 +132,8 @@ func TestFilter(t *testing.T) {
 		},
 		{
 			name:   "lowQueueAndLessThanKVCacheThresholdPredicate",
path: pkg/epp/scheduling/plugins/filter/filter_test.go
user: login: liu-cong
body: ```suggestion
			name:   "SheddableCapacityFilter, sheddable request",
```
created_at: 2025-05-09T16:30:08Z
author_association: CONTRIBUTOR
diff_hunk: @@ -132,7 +132,8 @@ func TestFilter(t *testing.T) {
 		},
 		{
 			name:   "lowQueueAndLessThanKVCacheThresholdPredicate",
-			filter: &HasCapacityFilter{queueThreshold: 0, kvCacheThreshold: 0.8},
+			req:    &types.LLMRequest{Critical: false},
path: pkg/epp/scheduling/plugins/filter/filter_test.go
user: login: liu-cong
body: nit: Can you also add a test case on Critical: true?
created_at: 2025-05-09T16:30:49Z
author_association: CONTRIBUTOR
diff_hunk: @@ -132,7 +132,8 @@ func TestFilter(t *testing.T) {
 		},
 		{
 			name:   "lowQueueAndLessThanKVCacheThresholdPredicate",
-			filter: &HasCapacityFilter{queueThreshold: 0, kvCacheThreshold: 0.8},
+			req:    &types.LLMRequest{Critical: false},
path: pkg/epp/scheduling/plugins/filter/filter_test.go
user: login: LukeAVanDrie
body: This is maybe not worth the effort considering #808 --> #805.
created_at: 2025-05-09T19:25:56Z
author_association: CONTRIBUTOR

PR comments:
user: login: netlify[bot]
created_at: 2025-05-09T05:48:14Z
body: ### <span aria-hidden="true">✅</span> Deploy Preview for *gateway-api-inference-extension* ready!


|  Name | Link |
|:-:|------------------------|
|<span aria-hidden="true">🔨</span> Latest commit | 17b1c815bb3cfe85d6ac604a5e9cb3639afeda0c |
|<span aria-hidden="true">🔍</span> Latest deploy log | https://app.netlify.com/sites/gateway-api-inference-extension/deploys/681e5a0577a27100087ba26e |
|<span aria-hidden="true">😎</span> Deploy Preview | [https://deploy-preview-809--gateway-api-inference-extension.netlify.app](https://deploy-preview-809--gateway-api-inference-extension.netlify.app) |
|<span aria-hidden="true">📱</span> Preview on mobile | <details><summary> Toggle QR Code... </summary><br /><br />![QR Code](https://app.netlify.com/qr-code/eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1cmwiOiJodHRwczovL2RlcGxveS1wcmV2aWV3LTgwOS0tZ2F0ZXdheS1hcGktaW5mZXJlbmNlLWV4dGVuc2lvbi5uZXRsaWZ5LmFwcCJ9.QG7d1Om72Lh0uZYvUvqJH_6j0YiaI9pAzpdSsRHtYa0)<br /><br />_Use your smartphone camera to open QR code link._</details> |
---
<!-- [gateway-api-inference-extension Preview](https://deploy-preview-809--gateway-api-inference-extension.netlify.app) -->
_To edit notification comments on pull requests, go to your [Netlify site configuration](https://app.netlify.com/sites/gateway-api-inference-extension/configuration/notifications#deploy-webhooks)._
author_association: NONE
user: login: nirrozenbaum
created_at: 2025-05-09T05:48:31Z
body: cc @liu-cong @ahg-g 
author_association: CONTRIBUTOR
user: login: kfswain
created_at: 2025-05-09T16:54:02Z
body: /approve
will leave lgtm to others
author_association: COLLABORATOR
user: login: k8s-ci-robot
created_at: 2025-05-09T16:54:09Z
body: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/kubernetes-sigs/gateway-api-inference-extension/pull/809#issuecomment-2867290808" title="Approved">kfswain</a>*, *<a href="https://github.com/kubernetes-sigs/gateway-api-inference-extension/pull/809#" title="Author self-approved">nirrozenbaum</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=kubernetes-sigs%2Fgateway-api-inference-extension).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/kubernetes-sigs/gateway-api-inference-extension/blob/main/OWNERS)~~ [kfswain]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
author_association: CONTRIBUTOR
user: login: LukeAVanDrie
created_at: 2025-05-09T19:21:52Z
body: Just a heads up, I delete these files anyways in #805. Capacity decisions should not be a responsibility of the scheduler per the architecture proposal. Admission control (and criticality based service differentiation) should happen outside the scheduler (long term in the flow controller). The scheduler should then decide the optimal pod to route approved requests to. No reason not to submit this though.
author_association: CONTRIBUTOR
user: login: nirrozenbaum
created_at: 2025-05-09T19:28:29Z
body: > Just a heads up, I delete these files anyways in #805. Capacity decisions should not be a responsibility of the scheduler per the architecture proposal. Admission control (and criticality based service differentiation) should happen outside the scheduler (long term in the flow controller). The scheduler should then decide the optimal pod to route approved requests to. No reason not to submit this though.

@LukeAVanDrie yeah, sounds good. 
Current PR doesn’t change anything but merging two filters that are actually the same (one is calling the other). once criticality handling is done in flow control this can be removed 
author_association: CONTRIBUTOR
user: login: LukeAVanDrie
created_at: 2025-05-09T19:35:38Z
body: > once criticality handling is done in flow control this can be removed

This is unrelated to this PR, but I guess long term, we need to also decide if this separation of responsibilities (specifically, request shedding) is a hard rule or just for the reference implementation. I can see instances where implementers would have custom scheduling plugins that may want to drop requests still.
author_association: CONTRIBUTOR
user: login: liu-cong
created_at: 2025-05-09T20:22:48Z
body: /lgtm
author_association: CONTRIBUTOR