Repository: kubernetes-sigs/gateway-api-inference-extension

Issue:
[InferenceModel update] Document all new changes
This is related to: https://github.com/kubernetes-sigs/gateway-api-inference-extension/tree/main/docs/proposals/1199-inferencemodel-api-evolution#structure-change

We need to document all the API changes in our site & give examples consuming these new configuration levels.

Discussion:
@ahg-g: As part of this, we also need to update Criticality to Priority@danehans: FYI- the API docs generator is broken: https://github.com/kubernetes-sigs/gateway-api-inference-extension/issues/1421@nirrozenbaum: @kfswain can we close this issue as completed?

Solution patch:
diff --git a/Makefile b/Makefile
index a2563a6..0ce7cdf 100644
--- a/Makefile
+++ b/Makefile
@@ -211,7 +211,7 @@ image-build: ## Build the EPP image using Docker Buildx.
 
 .PHONY: image-push
 image-push: PUSH=--push ## Build the EPP image and push it to $IMAGE_REPO.
-image-push: MULTI=true image-build
+image-push: image-build
 
 .PHONY: image-load
 image-load: LOAD=--load ## Build the EPP image and load it in the local Docker registry.
diff --git a/cloudbuild.yaml b/cloudbuild.yaml
index 3666674..5971d20 100644
--- a/cloudbuild.yaml
+++ b/cloudbuild.yaml
@@ -13,6 +13,7 @@ steps:
     - EXTRA_TAG=$_PULL_BASE_REF
     - DOCKER_BUILDX_CMD=/buildx-entrypoint
     - GIT_COMMIT_SHA=$_PULL_BASE_SHA
+    - MULTI=true
   - name: gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20240718-5ef92b5c36
     entrypoint: make
     args:
diff --git a/cmd/epp/runner/runner.go b/cmd/epp/runner/runner.go
index c9c5f60..d138ee5 100644
--- a/cmd/epp/runner/runner.go
+++ b/cmd/epp/runner/runner.go
@@ -387,6 +387,7 @@ func (r *Runner) registerInTreePlugins() {
 	plugins.Register(profile.SingleProfileHandlerType, profile.SingleProfileHandlerFactory)
 	plugins.Register(scorer.KvCacheUtilizationScorerType, scorer.KvCacheUtilizationScorerFactory)
 	plugins.Register(scorer.QueueScorerType, scorer.QueueScorerFactory)
+	plugins.Register(scorer.LoraAffinityScorerType, scorer.LoraAffinityScorerFactory)
 	// register filter for test purpose only (used in conformance tests)
 	plugins.Register(testfilter.HeaderBasedTestingFilterType, testfilter.HeaderBasedTestingFilterFactory)
 }
diff --git a/conformance/conformance.go b/conformance/conformance.go
index eb18e8f..6aebd69 100644
--- a/conformance/conformance.go
+++ b/conformance/conformance.go
@@ -30,7 +30,6 @@ import (
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -39,7 +38,6 @@ import (
 	clientsetscheme "k8s.io/client-go/kubernetes/scheme"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	k8sconfig "sigs.k8s.io/controller-runtime/pkg/client/config"
-
 	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
 	confflags "sigs.k8s.io/gateway-api/conformance/utils/flags"
 	apikubernetes "sigs.k8s.io/gateway-api/conformance/utils/kubernetes"
@@ -49,7 +47,6 @@ import (
 
 	inferencev1 "sigs.k8s.io/gateway-api-inference-extension/api/v1"
 	inferencev1alpha2 "sigs.k8s.io/gateway-api-inference-extension/apix/v1alpha2"
-
 	"sigs.k8s.io/gateway-api-inference-extension/conformance/resources"
 	"sigs.k8s.io/gateway-api-inference-extension/conformance/tests"
 	inferenceconfig "sigs.k8s.io/gateway-api-inference-extension/conformance/utils/config"
diff --git a/pkg/epp/flowcontrol/controller/internal/processor.go b/pkg/epp/flowcontrol/controller/internal/processor.go
index 0154dc7..cef5b35 100644
--- a/pkg/epp/flowcontrol/controller/internal/processor.go
+++ b/pkg/epp/flowcontrol/controller/internal/processor.go
@@ -26,8 +26,8 @@ import (
 	"time"
 
 	"github.com/go-logr/logr"
-
 	"sigs.k8s.io/controller-runtime/pkg/log"
+
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/flowcontrol/contracts"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/flowcontrol/framework"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/flowcontrol/types"
diff --git a/pkg/epp/scheduling/framework/plugins/scorer/lora_affinity.go b/pkg/epp/scheduling/framework/plugins/scorer/lora_affinity.go
new file mode 100644
index 0000000..ed8822a
--- /dev/null
+++ b/pkg/epp/scheduling/framework/plugins/scorer/lora_affinity.go
@@ -0,0 +1,90 @@
+/*
+Copyright 2025 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package scorer
+
+import (
+	"context"
+	"encoding/json"
+
+	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/plugins"
+	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework"
+	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/types"
+)
+
+const (
+	DefaultLoraAffinityScorerWeight = 1
+	LoraAffinityScorerType          = "lora-affinity"
+)
+
+// compile-time type assertion
+var _ framework.Scorer = &LoraAffinityScorer{}
+
+// LoraAffinityScorerFactory defines the factory function for LoraAffinityScorer.
+func LoraAffinityScorerFactory(name string, _ json.RawMessage, _ plugins.Handle) (plugins.Plugin, error) {
+	return NewLoraAffinityScorer().WithName(name), nil
+}
+
+// NewLoraAffinityScorer initializes a new LoraAffinityScorer and returns its pointer.
+func NewLoraAffinityScorer() *LoraAffinityScorer {
+	return &LoraAffinityScorer{
+		tn: plugins.TypedName{Type: LoraAffinityScorerType, Name: LoraAffinityScorerType},
+	}
+}
+
+// LoraAffinityScorer scores list of candidate pods based on Lora affinity and availability.
+type LoraAffinityScorer struct {
+	tn plugins.TypedName
+}
+
+// TypedName returns the type and name tuple of this plugin instance.
+func (s *LoraAffinityScorer) TypedName() plugins.TypedName {
+	return s.tn
+}
+
+// WithName sets the name of the scorer.
+func (s *LoraAffinityScorer) WithName(name string) *LoraAffinityScorer {
+	s.tn.Name = name
+	return s
+}
+
+func (s *LoraAffinityScorer) Score(_ context.Context, _ *types.CycleState, request *types.LLMRequest, pods []types.Pod) map[types.Pod]float64 {
+	scores := make(map[types.Pod]float64, len(pods))
+
+	// Assign a score to each pod for loading the target adapter.
+	for _, pod := range pods {
+		_, active := pod.GetMetrics().ActiveModels[request.TargetModel]
+		_, waiting := pod.GetMetrics().WaitingModels[request.TargetModel]
+
+		// Determine the model server's suitability score based on adapter load status and capacity.
+		switch {
+		// Ideal: The adapter is already active on this model server.
+		case active:
+			scores[pod] = 1.0
+		// Good: The model server has capacity to load at least one more adapter.
+		case len(pod.GetMetrics().ActiveModels)+len(pod.GetMetrics().WaitingModels) < pod.GetMetrics().MaxActiveModels:
+			scores[pod] = 0.8
+		// Moderate: The adapter is already in the queue to be loaded on this model server.
+		case waiting:
+			scores[pod] = 0.6
+		// Unsuitable: The model server has reached its maximum capacity and cannot load the adapter.
+		default:
+			scores[pod] = 0.0
+		}
+	}
+
+	return scores
+}
diff --git a/pkg/epp/scheduling/scheduler.go b/pkg/epp/scheduling/scheduler.go
index ef0c522..12c1883 100644
--- a/pkg/epp/scheduling/scheduler.go
+++ b/pkg/epp/scheduling/scheduler.go
@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"sigs.k8s.io/controller-runtime/pkg/log"
+
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/metrics"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/types"
diff --git a/site-src/guides/epp-configuration/config-text.md b/site-src/guides/epp-configuration/config-text.md
index ceb0ed6..7c33aae 100644
--- a/site-src/guides/epp-configuration/config-text.md
+++ b/site-src/guides/epp-configuration/config-text.md
@@ -261,3 +261,13 @@ available to serve new request).
 
 - *Type*: queue-scorer
 - *Parameters*: none
+
+
+#### **LoraAffinityScorer**
+
+Scores list of candidate pods based on the LoRA adapters loaded on the pod. 
+Pods with the adapter already loaded or able to be actively loaded will be 
+scored higher (since it's more available to serve new request).
+
+- *Type*: lora-affinity-scorer
+- *Parameters*: none
diff --git a/site-src/guides/index.md b/site-src/guides/index.md
index 884e497..a1b10ed 100644
--- a/site-src/guides/index.md
+++ b/site-src/guides/index.md
@@ -88,7 +88,7 @@ This quickstart guide is intended for engineers familiar with k8s and model serv
    Deploy the sample InferenceModel which is configured to forward traffic to the `food-review-1` [LoRA adapter](https://docs.vllm.ai/en/latest/features/lora.html) of the sample model server.
 
    ```bash
-   kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/inferencemodel.yaml
+   kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/inferenceobjective.yaml
    ```
 
 ### Deploy the InferencePool and Endpoint Picker Extension
@@ -269,7 +269,7 @@ This quickstart guide is intended for engineers familiar with k8s and model serv
 
       ```bash
       kubectl delete -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/inferencepool-resources.yaml --ignore-not-found
-      kubectl delete -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/inferencemodel.yaml --ignore-not-found
+      kubectl delete -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/inferenceobjective.yaml --ignore-not-found
       kubectl delete -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/vllm/cpu-deployment.yaml --ignore-not-found
       kubectl delete -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/vllm/gpu-deployment.yaml --ignore-not-found
       kubectl delete secret hf-token --ignore-not-found
diff --git a/site-src/performance/regression-testing/index.md b/site-src/performance/regression-testing/index.md
index 83e7a3b..e4974c1 100644
--- a/site-src/performance/regression-testing/index.md
+++ b/site-src/performance/regression-testing/index.md
@@ -71,7 +71,7 @@ Optionally, you can also run benchmarks against the `ShareGPT` dataset for addit
 
 Update deployments for multi-LoRA support:  
 - vLLM Deployment: `./config/manifests/regression-testing/vllm/multi-lora-deployment.yaml`  
-- InferenceModel: `./config/manifests/inferencemodel.yaml`
+- InferenceObjective.yaml: `./config/manifests/inferenceobjective.yaml.yaml`
 
 Refer to example manifest:  
 `./config/manifests/regression-testing/multi-lora-regression.yaml`



Test patch:
diff --git a/conformance/tests/epp_unavailable_fail_open.go b/conformance/tests/epp_unavailable_fail_open.go
index 3d0c560..be72005 100644
--- a/conformance/tests/epp_unavailable_fail_open.go
+++ b/conformance/tests/epp_unavailable_fail_open.go
@@ -22,13 +22,13 @@ import (
 
 	"github.com/stretchr/testify/require"
 	"k8s.io/apimachinery/pkg/types"
-	testfilter "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework/plugins/test/filter"
 	"sigs.k8s.io/gateway-api/conformance/utils/suite"
 	"sigs.k8s.io/gateway-api/pkg/features"
 
 	"sigs.k8s.io/gateway-api-inference-extension/conformance/resources"
 	k8sutils "sigs.k8s.io/gateway-api-inference-extension/conformance/utils/kubernetes"
 	trafficutils "sigs.k8s.io/gateway-api-inference-extension/conformance/utils/traffic"
+	testfilter "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework/plugins/test/filter"
 )
 
 func init() {
diff --git a/conformance/tests/httproute_invalid_inferencepool_ref.go b/conformance/tests/httproute_invalid_inferencepool_ref.go
index 97841e4..1e5eaf4 100644
--- a/conformance/tests/httproute_invalid_inferencepool_ref.go
+++ b/conformance/tests/httproute_invalid_inferencepool_ref.go
@@ -21,11 +21,12 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
-	"sigs.k8s.io/gateway-api-inference-extension/conformance/resources"
 	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
 	"sigs.k8s.io/gateway-api/conformance/utils/kubernetes"
 	"sigs.k8s.io/gateway-api/conformance/utils/suite"
 	"sigs.k8s.io/gateway-api/pkg/features"
+
+	"sigs.k8s.io/gateway-api-inference-extension/conformance/resources"
 )
 
 func init() {
diff --git a/pkg/epp/flowcontrol/controller/internal/processor_test.go b/pkg/epp/flowcontrol/controller/internal/processor_test.go
index aeb46b6..09f1206 100644
--- a/pkg/epp/flowcontrol/controller/internal/processor_test.go
+++ b/pkg/epp/flowcontrol/controller/internal/processor_test.go
@@ -53,9 +53,9 @@ import (
 	"github.com/go-logr/logr"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/flowcontrol/contracts"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/flowcontrol/contracts/mocks"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/flowcontrol/framework"
diff --git a/pkg/epp/scheduling/framework/plugins/scorer/lora_affinity_test.go b/pkg/epp/scheduling/framework/plugins/scorer/lora_affinity_test.go
new file mode 100644
index 0000000..418bafd
--- /dev/null
+++ b/pkg/epp/scheduling/framework/plugins/scorer/lora_affinity_test.go
@@ -0,0 +1,174 @@
+/*
+Copyright 2025 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package scorer
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	k8stypes "k8s.io/apimachinery/pkg/types"
+
+	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/backend"
+	backendmetrics "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/backend/metrics"
+	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/types"
+)
+
+func TestLoraAffinityScorer(t *testing.T) {
+	tests := []struct {
+		name              string
+		request           *types.LLMRequest
+		pods              []types.Pod
+		expectedScoresPod map[string]float64 // Map of pod name to expected score
+	}{
+		{
+			name:    "Target model is active",
+			request: &types.LLMRequest{TargetModel: "active-model-1"},
+			pods: []types.Pod{
+				&types.PodMetrics{
+					Pod: &backend.Pod{NamespacedName: k8stypes.NamespacedName{Name: "pod1"}},
+					MetricsState: &backendmetrics.MetricsState{
+						ActiveModels:    map[string]int{"active-model-1": 1},
+						WaitingModels:   map[string]int{},
+						MaxActiveModels: 5,
+					},
+				},
+			},
+			expectedScoresPod: map[string]float64{
+				"pod1": 1.0,
+			},
+		},
+		{
+			name:    "Target model is waiting",
+			request: &types.LLMRequest{TargetModel: "active-model-1"},
+			pods: []types.Pod{
+				&types.PodMetrics{
+					Pod: &backend.Pod{NamespacedName: k8stypes.NamespacedName{Name: "pod1"}},
+					MetricsState: &backendmetrics.MetricsState{
+						ActiveModels:    map[string]int{"active-model-2": 2},
+						WaitingModels:   map[string]int{"active-model-1": 1},
+						MaxActiveModels: 2,
+					},
+				},
+			},
+			expectedScoresPod: map[string]float64{
+				"pod1": 0.6,
+			},
+		},
+		{
+			name:    "Pods have no space for new model",
+			request: &types.LLMRequest{TargetModel: "active-model-1"},
+			pods: []types.Pod{
+				&types.PodMetrics{
+					Pod: &backend.Pod{NamespacedName: k8stypes.NamespacedName{Name: "pod1"}},
+					MetricsState: &backendmetrics.MetricsState{
+						ActiveModels:    map[string]int{"active-model-2": 2},
+						WaitingModels:   map[string]int{"active-model-3": 1},
+						MaxActiveModels: 2,
+					},
+				},
+				&types.PodMetrics{
+					Pod: &backend.Pod{NamespacedName: k8stypes.NamespacedName{Name: "pod2"}},
+					MetricsState: &backendmetrics.MetricsState{
+						ActiveModels:    map[string]int{},
+						WaitingModels:   map[string]int{},
+						MaxActiveModels: 0,
+					},
+				},
+			},
+			expectedScoresPod: map[string]float64{
+				"pod1": 0.0,
+				"pod2": 0.0,
+			},
+		},
+		{
+			name:    "Multiple pods with mixed active and waiting models",
+			request: &types.LLMRequest{TargetModel: "active-model-1"},
+			pods: []types.Pod{
+				&types.PodMetrics{
+					Pod: &backend.Pod{NamespacedName: k8stypes.NamespacedName{Name: "pod1"}},
+					MetricsState: &backendmetrics.MetricsState{
+						ActiveModels:    map[string]int{"active-model-1": 1},
+						WaitingModels:   map[string]int{},
+						MaxActiveModels: 5,
+					},
+				},
+				&types.PodMetrics{
+					Pod: &backend.Pod{NamespacedName: k8stypes.NamespacedName{Name: "pod2"}},
+					MetricsState: &backendmetrics.MetricsState{
+						ActiveModels:    map[string]int{"active-model-2": 4},
+						WaitingModels:   map[string]int{"active-model-1": 1},
+						MaxActiveModels: 5,
+					},
+				},
+				&types.PodMetrics{
+					Pod: &backend.Pod{NamespacedName: k8stypes.NamespacedName{Name: "pod3"}},
+					MetricsState: &backendmetrics.MetricsState{
+						ActiveModels:    map[string]int{"active-model-2": 1},
+						WaitingModels:   map[string]int{},
+						MaxActiveModels: 2,
+					},
+				},
+				&types.PodMetrics{
+					Pod: &backend.Pod{NamespacedName: k8stypes.NamespacedName{Name: "pod4"}},
+					MetricsState: &backendmetrics.MetricsState{
+						ActiveModels:    map[string]int{"active-model-3": 1},
+						WaitingModels:   map[string]int{"active-model-1": 1},
+						MaxActiveModels: 2,
+					},
+				},
+				&types.PodMetrics{
+					Pod: &backend.Pod{NamespacedName: k8stypes.NamespacedName{Name: "pod5"}},
+					MetricsState: &backendmetrics.MetricsState{
+						ActiveModels:    map[string]int{"active-model-4": 1, "active-model-5": 1},
+						WaitingModels:   map[string]int{},
+						MaxActiveModels: 2,
+					},
+				},
+			},
+			expectedScoresPod: map[string]float64{
+				"pod1": 1.0,
+				"pod2": 0.8,
+				"pod3": 0.8,
+				"pod4": 0.6,
+				"pod5": 0.0,
+			},
+		},
+		{
+			name:              "Empty pods slice",
+			request:           &types.LLMRequest{TargetModel: "modelA"},
+			pods:              []types.Pod{},
+			expectedScoresPod: map[string]float64{}, // No pods, no scores
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			scorer := &LoraAffinityScorer{}
+			scores := scorer.Score(context.Background(), types.NewCycleState(), test.request, test.pods)
+
+			for _, pod := range test.pods {
+				expectedScore, ok := test.expectedScoresPod[pod.GetPod().NamespacedName.Name]
+				if !ok {
+					t.Fatalf("Expected score not found for pod %s in test %s", pod.GetPod().NamespacedName, test.name)
+				}
+				assert.InDelta(t, expectedScore, scores[pod], 0.0001, "Pod %s should have score %f", pod.GetPod().NamespacedName.Name, expectedScore)
+			}
+			assert.Len(t, scores, len(test.expectedScoresPod), "Number of scored pods should match expected")
+		})
+	}
+}