Repository: kubeai-project/kubeai

Issue:
Vllm container version and oss models
I'm trying to deploy a gpt-oss model, which, to my understanding, requires a cmpatible version of vllm. 
vllm does have current docker conainers for it, but they are specific to oss, so I cannot use them to deploy other models and just alter 
the modelServers images value (since then, other models won't work any more).

Any suggestions on how to proceed here? And/or is there a plan to make them available any time soon? 


Hints from issue comments:
@m4oc: You can handle this by overriding the image directly in your model manifest.  For example, for GPT-OSS you could specify:

`image: vllm/vllm-openai:gptoss`

@tpfau: Can I do this for individual models? 
I know that I can update the model for general "nvidia/default" pods, but is the image option for a model deployment documented somewhere?@kishan2910: I recently deployed gpt oss, and yes you need to upgrade the vLLM version. If kubeai is deployed using helm, the version can be updated from the repo. 

helm pull kubeai/kubeai --untar

Now in values.yaml file,  change the version of vllm.

modelServers:
  VLLM:
    images:
      # The key is the image name (referenced from resourceProfiles) and the value is the image.
      # The "default" image should always be specified.
      # "default" is used when no imageName is specified or if a specific image is not found.
      default: "vllm/vllm-openai:v0.10.1"
      nvidia-gpu: "vllm/vllm-openai:v0.10.1"
      cpu: "substratusai/vllm:v0.6.3.post1-cpu"
      google-tpu: "substratusai/vllm:v0.6.4.post1-tpu"
      # Source: https://github.com/substratusai/vllm-docker/blob/main/Dockerfile.cuda-arm
      gh200: "substratusai/vllm-gh200:v0.10.1"
      # upstream vLLM seems to have broken ROCm support, so we are using a fork from AMD.
      # Source: https://hub.docker.com/r/rocm/vllm-dev
      # Source: https://github.com/ROCm/vllm
      amd-gpu: substratusai/vllm-rocm:nightly_main_20250120
  OLlama:
    images:
      default: "ollama/ollama:latest"
  FasterWhisper:
    images:
      default: "fedirz/faster-whisper-server:latest-cpu"
      nvidia-gpu: "fedirz/faster-whisper-server:latest-cuda"

After that you will need to update this kubeai chart.
helm dependency update ./kubeai
helm upgrade --install kubeai ./kubeai
helm upgrade --install kubeai-models kubeai/models -f ./llm-rwx.yaml

Solution patch:
diff --git a/api/k8s/v1/model_types.go b/api/k8s/v1/model_types.go
index e09dc85..3310f6d 100644
--- a/api/k8s/v1/model_types.go
+++ b/api/k8s/v1/model_types.go
@@ -142,12 +142,13 @@ type ModelSpec struct {
 	PriorityClassName string `json:"priorityClassName,omitempty"`
 }
 
-// +kubebuilder:validation:Enum=TextGeneration;TextEmbedding;SpeechToText
+// +kubebuilder:validation:Enum=TextGeneration;TextEmbedding;Reranking;SpeechToText
 type ModelFeature string
 
 const (
 	ModelFeatureTextGeneration = "TextGeneration"
 	ModelFeatureTextEmbedding  = "TextEmbedding"
+	ModelFeatureReranking      = "Reranking"
 	// TODO (samos123): Add validation that Speech to Text only supports Faster Whisper.
 	ModelFeatureSpeechToText = "SpeechToText"
 )
diff --git a/api/openai/v1/rerank.go b/api/openai/v1/rerank.go
new file mode 100644
index 0000000..6167c1b
--- /dev/null
+++ b/api/openai/v1/rerank.go
@@ -0,0 +1,38 @@
+package v1
+
+import (
+	"github.com/go-json-experiment/json/jsontext"
+)
+
+// RerankRequest represents a request to rerank a list of documents based on a query.
+type RerankRequest struct {
+	Query     string   `json:"query"`
+	Documents []string `json:"documents"`
+	Model     string   `json:"model"`
+	TopN      int      `json:"top_n,omitzero"`
+
+	// Unknown fields should be preserved to fully support engines that accept
+	// extended parameters.
+	Unknown jsontext.Value `json:",unknown"`
+}
+
+func (r *RerankRequest) GetModel() string  { return r.Model }
+func (r *RerankRequest) SetModel(m string) { r.Model = m }
+
+// RerankResult contains a single reranked document and its relevance score.
+type RerankResult struct {
+	DocumentIndex  int     `json:"document_index"`
+	Document       string  `json:"document,omitzero"`
+	RelevanceScore float32 `json:"relevance_score"`
+}
+
+// RerankResponse represents the response from a reranking request.
+type RerankResponse struct {
+	Object string         `json:"object"`
+	Data   []RerankResult `json:"data"`
+	Model  string         `json:"model"`
+
+	// Unknown fields should be preserved to fully support engines that return
+	// extended parameters.
+	Unknown jsontext.Value `json:",unknown"`
+}
diff --git a/charts/kubeai/templates/crds/kubeai.org_models.yaml b/charts/kubeai/templates/crds/kubeai.org_models.yaml
index 9ce7ef7..1889c7c 100644
--- a/charts/kubeai/templates/crds/kubeai.org_models.yaml
+++ b/charts/kubeai/templates/crds/kubeai.org_models.yaml
@@ -149,6 +149,7 @@ spec:
                   enum:
                   - TextGeneration
                   - TextEmbedding
+                  - Reranking
                   - SpeechToText
                   type: string
                 type: array
diff --git a/charts/kubeai/values.yaml b/charts/kubeai/values.yaml
index 9f462b7..5d93b8a 100644
--- a/charts/kubeai/values.yaml
+++ b/charts/kubeai/values.yaml
@@ -42,16 +42,14 @@ modelServers:
       # The key is the image name (referenced from resourceProfiles) and the value is the image.
       # The "default" image should always be specified.
       # "default" is used when no imageName is specified or if a specific image is not found.
-      default: "vllm/vllm-openai:v0.8.3"
-      nvidia-gpu: "vllm/vllm-openai:v0.8.3"
-      cpu: "substratusai/vllm:v0.6.3.post1-cpu"
-      google-tpu: "substratusai/vllm:v0.6.4.post1-tpu"
-      # Source: https://github.com/substratusai/vllm-docker/blob/main/Dockerfile.cuda-arm
-      gh200: "substratusai/vllm-gh200:v0.8.3"
-      # upstream vLLM seems to have broken ROCm support, so we are using a fork from AMD.
-      # Source: https://hub.docker.com/r/rocm/vllm-dev
-      # Source: https://github.com/ROCm/vllm
-      amd-gpu: substratusai/vllm-rocm:nightly_main_20250120
+      default: "vllm/vllm-openai:v0.10.2"
+      nvidia-gpu: "vllm/vllm-openai:v0.10.2"
+      cpu: "public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.10.2"
+      google-tpu: "vllm/vllm-tpu:23194d83e8f2a6783b0d8c275f5f8a22faab9aec"
+      # VLLM support arm
+      gh200: "vllm/vllm-openai:v0.10.2"
+      # AMD use last ROCM driver version
+      amd-gpu: "rocm/vllm/rocm6.4.1_vllm_0.10.1_20250909"
   OLlama:
     images:
       default: "ollama/ollama:latest"
@@ -70,7 +68,7 @@ modelServerPods:
   # Security Context for the model pods
   # Needed for OpenShift
   securityContext:
-    runAsUser: 0
+    # runAsUser: 0
     readOnlyRootFilesystem: false
     allowPrivilegeEscalation: false
     capabilities:
@@ -245,7 +243,7 @@ open-webui:
   # Security Context for the openwebui pod
   # Needed for OpenShift
   containerSecurityContext:
-    runAsUser: 0
+    # runAsUser: 0
     readOnlyRootFilesystem: false
     allowPrivilegeEscalation: false
     capabilities:
diff --git a/charts/models/values.yaml b/charts/models/values.yaml
index ad64266..9d6bb00 100644
--- a/charts/models/values.yaml
+++ b/charts/models/values.yaml
@@ -589,6 +589,12 @@ catalog:
     url: "hf://BAAI/bge-small-en-v1.5"
     engine: Infinity
     resourceProfile: cpu:1
+  bge-rerank-base-cpu:
+    enabled: false
+    features: ["Reranking"]
+    url: "hf://BAAI/bge-reranker-base"
+    engine: Infinity
+    resourceProfile: cpu:1
   # Opt #
   opt-125m-cpu:
     enabled: false
diff --git a/docs/README.md b/docs/README.md
index af69b6d..b5913a0 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -1,9 +1,5 @@
 # KubeAI: AI Inferencing Operator
 
-> **🚨 Looking for New Maintainers** 
-> 
-> We are looking for new maintainers to take over this project! If you're interested in becoming a maintainer and helping drive the future of KubeAI, please reach out on our [Discord channel](https://discord.gg/JeXhcmjZVm).
-
 Deploy and scale machine learning models on Kubernetes. Built for LLMs, embeddings, and speech-to-text.
 
 ## Highlights
@@ -13,6 +9,7 @@ What is it for?
 🚀 **LLM Inferencing** - Operate vLLM and Ollama servers  
 🎙️ **Speech Processing** - Transcribe audio with FasterWhisper  
 🔢 **Vector Embeddings** - Generate embeddings with Infinity  
+📚 **Reranking** - Reorder search results with cross-encoder models  
 
 What do you get?
 
@@ -58,6 +55,7 @@ No need to change your client libraries, KubeAI supports the following endpoints
 /v1/chat/completions
 /v1/completions
 /v1/embeddings
+/v1/rerank
 /v1/models
 /v1/audio/transcriptions
 ```
diff --git a/docs/how-to/configure-reranking-models.md b/docs/how-to/configure-reranking-models.md
new file mode 100644
index 0000000..454e306
--- /dev/null
+++ b/docs/how-to/configure-reranking-models.md
@@ -0,0 +1,43 @@
+# Configure reranking models
+
+KubeAI supports reranking models via the Infinity and vLLM (Recommended for GPU) engine.
+
+## Install BAAI/bge-reranker-base model using Infinity
+
+Create a file named `kubeai-rerank-models.yaml` with the following content:
+
+```yaml
+catalog:
+  bge-rerank-base-cpu:
+    enabled: true
+    features: ["Reranking"]
+    owner: baai
+    url: "hf://BAAI/bge-reranker-base"
+    engine: Infinity
+    #engine: VLLM
+    resourceProfile: cpu:1
+    # resourceProfile: nvidia-gpu-l4:1
+    minReplicas: 1
+```
+
+Apply the kubeai-models helm chart:
+
+```bash
+helm install kubeai-models kubeai/models -f ./kubeai-rerank-models.yaml.yaml
+```
+
+Once the pod is ready, you can call the rerank endpoint:
+
+```python
+import requests
+resp = requests.post(
+    "http://localhost:8000/vllm/v1/rerank",
+    json={
+        "model": "bge-rerank-base-cpu",
+        "query": "Which document talks about apples?",
+        "documents": ["An apple a day keeps the doctor away", "Oranges are tasty"],
+    },
+)
+print(resp.json())
+```
+
diff --git a/docs/installation/aks.md b/docs/installation/aks.md
index ffc34e9..e590ed3 100644
--- a/docs/installation/aks.md
+++ b/docs/installation/aks.md
@@ -137,4 +137,5 @@ Take a look at the following how-to guides to deploy models:
 
 - [Configure Text Generation Models](../how-to/configure-text-generation-models.md)
 - [Configure Embedding Models](../how-to/configure-embedding-models.md)
+- [Configure Reranking Models](../how-to/configure-reranking-models.md)
 - [Configure Speech to Text Models](../how-to/configure-speech-to-text.md)
diff --git a/docs/installation/any.md b/docs/installation/any.md
index c7deb4c..e0b47ac 100644
--- a/docs/installation/any.md
+++ b/docs/installation/any.md
@@ -98,5 +98,6 @@ Take a look at the following how-to guides to deploy models:
 
 * [Configure Text Generation Models](../how-to/configure-text-generation-models.md)
 * [Configure Embedding Models](../how-to/configure-embedding-models.md)
+* [Configure Reranking Models](../how-to/configure-reranking-models.md)
 * [Configure Speech to Text Models](../how-to/configure-speech-to-text.md)
 
diff --git a/docs/installation/eks.md b/docs/installation/eks.md
index 748c856..9b791d0 100644
--- a/docs/installation/eks.md
+++ b/docs/installation/eks.md
@@ -151,4 +151,5 @@ Take a look at the following how-to guides to deploy models:
 
 * [Configure Text Generation Models](../how-to/configure-text-generation-models.md)
 * [Configure Embedding Models](../how-to/configure-embedding-models.md)
+* [Configure Reranking Models](../how-to/configure-reranking-models.md)
 * [Configure Speech to Text Models](../how-to/configure-speech-to-text.md)
diff --git a/docs/installation/gke.md b/docs/installation/gke.md
index beaa181..17094ef 100644
--- a/docs/installation/gke.md
+++ b/docs/installation/gke.md
@@ -68,4 +68,5 @@ Take a look at the following how-to guides to deploy models:
 
 * [Configure Text Generation Models](../how-to/configure-text-generation-models.md)
 * [Configure Embedding Models](../how-to/configure-embedding-models.md)
+* [Configure Reranking Models](../how-to/configure-reranking-models.md)
 * [Configure Speech to Text Models](../how-to/configure-speech-to-text.md)
\ No newline at end of file
diff --git a/docs/reference/kubernetes-api.md b/docs/reference/kubernetes-api.md
index a488670..91172e3 100644
--- a/docs/reference/kubernetes-api.md
+++ b/docs/reference/kubernetes-api.md
@@ -108,7 +108,7 @@ _Underlying type:_ _string_
 
 
 _Validation:_
-- Enum: [TextGeneration TextEmbedding SpeechToText]
+- Enum: [TextGeneration TextEmbedding Reranking SpeechToText]
 
 _Appears in:_
 - [ModelSpec](#modelspec)
@@ -130,7 +130,7 @@ _Appears in:_
 | --- | --- | --- | --- |
 | `url` _string_ | URL of the model to be served.<br />Currently the following formats are supported:<br /><br />For VLLM, FasterWhisper, Infinity engines:<br /><br />"hf://<repo>/<model>"<br />"pvc://<pvcName>"<br />"pvc://<pvcName>/<pvcSubpath>"<br />"gs://<bucket>/<path>" (only with cacheProfile)<br />"oss://<bucket>/<path>" (only with cacheProfile)<br />"s3://<bucket>/<path>" (only with cacheProfile)<br /><br />For OLlama engine:<br /><br />"ollama://<model>" |  | Required: \{\} <br /> |
 | `adapters` _[Adapter](#adapter) array_ |  |  |  |
-| `features` _[ModelFeature](#modelfeature) array_ | Features that the model supports.<br />Dictates the APIs that are available for the model. |  | Enum: [TextGeneration TextEmbedding SpeechToText] <br /> |
+| `features` _[ModelFeature](#modelfeature) array_ | Features that the model supports.<br />Dictates the APIs that are available for the model. |  | Enum: [TextGeneration TextEmbedding Reranking SpeechToText] <br /> |
 | `engine` _string_ | Engine to be used for the server process. |  | Enum: [OLlama VLLM FasterWhisper Infinity] <br />Required: \{\} <br /> |
 | `resourceProfile` _string_ | ResourceProfile required to serve the model.<br />Use the format "<resource-profile-name>:<count>".<br />Example: "nvidia-gpu-l4:2" - 2x NVIDIA L4 GPUs.<br />Must be a valid ResourceProfile defined in the system config. |  |  |
 | `cacheProfile` _string_ | CacheProfile to be used for caching model artifacts.<br />Must be a valid CacheProfile defined in the system config. |  |  |
diff --git a/docs/reference/openai-api-compatibility.md b/docs/reference/openai-api-compatibility.md
index 49bc08d..138b1c0 100644
--- a/docs/reference/openai-api-compatibility.md
+++ b/docs/reference/openai-api-compatibility.md
@@ -32,6 +32,14 @@ POST /v1/embeddings
 
 * Supported for  Models with `.spec.features: ["TextEmbedding"]`.
 
+### Reranking
+
+```
+POST /v1/vllm/rerank
+```
+
+* Supported for  Models with `.spec.features: ["Reranking"]`.
+
 ### Speech-to-Text
 
 ```
diff --git a/hack/dev-models/kind-cpu-rerank.yaml b/hack/dev-models/kind-cpu-rerank.yaml
new file mode 100644
index 0000000..031b748
--- /dev/null
+++ b/hack/dev-models/kind-cpu-rerank.yaml
@@ -0,0 +1,34 @@
+---
+apiVersion: kubeai.org/v1
+kind: Model
+metadata:
+  name: dev-rerank
+  annotations:
+    # Have the controller send requests to localhost to allow for
+    # running the controller locally (assuming a port-forward is in place).
+    model-pod-ip: "127.0.0.1"
+    model-pod-port: "7000" 
+spec:
+  features: ["Reranking"]
+  url: hf://BAAI/bge-reranker-base
+  engine: Infinity
+  resourceProfile: cpu:1
+  minReplicas: 1
+  maxReplicas: 3
+
+---
+# Service for port-fowarding to the model:
+#
+# while true; do kubectl port-forward service/dev-rerank 7000:7000; done
+#
+apiVersion: v1
+kind: Service
+metadata:
+  name: dev-rerank
+spec:
+  selector:
+    model: dev-rerank
+  ports:
+    - protocol: TCP
+      port: 7000
+      targetPort: 8000
diff --git a/hack/dev-models/vllm-rerank.yaml b/hack/dev-models/vllm-rerank.yaml
new file mode 100644
index 0000000..5bb4cce
--- /dev/null
+++ b/hack/dev-models/vllm-rerank.yaml
@@ -0,0 +1,10 @@
+apiVersion: kubeai.org/v1
+kind: Model
+metadata:
+  name: bge-rerank-base-cpu
+spec:
+  features: [Reranking]
+  url: hf://BAAI/bge-reranker-base
+  engine: Infinity
+  minReplicas: 1
+  resourceProfile: cpu:1
diff --git a/internal/apiutils/request.go b/internal/apiutils/request.go
index 59b4757..7fad5a2 100644
--- a/internal/apiutils/request.go
+++ b/internal/apiutils/request.go
@@ -172,6 +172,8 @@ func (r *Request) readJSONBody(body io.Reader, path string) error {
 		r.modelRequest = &openaiv1.ChatCompletionRequest{}
 	case "/v1/embeddings":
 		r.modelRequest = &openaiv1.EmbeddingRequest{}
+	case "/v1/rerank":
+		r.modelRequest = &openaiv1.RerankRequest{}
 	default:
 		return fmt.Errorf("unknown path: %q", path)
 	}
diff --git a/internal/modelcontroller/adapters.go b/internal/modelcontroller/adapters.go
index 73a1443..5a2c75f 100644
--- a/internal/modelcontroller/adapters.go
+++ b/internal/modelcontroller/adapters.go
@@ -205,6 +205,7 @@ func (r *ModelReconciler) patchServerAdapterLoader(podSpec *corev1.PodSpec, m *v
 		Image:           image,
 		ImagePullPolicy: corev1.PullIfNotPresent,
 		Env:             env,
+		SecurityContext: r.ModelServerPods.ModelContainerSecurityContext,
 		Command:         []string{"sleep", "infinity"},
 		VolumeMounts: []corev1.VolumeMount{
 			{
diff --git a/internal/modelcontroller/cache.go b/internal/modelcontroller/cache.go
index 6807825..db3320f 100644
--- a/internal/modelcontroller/cache.go
+++ b/internal/modelcontroller/cache.go
@@ -332,11 +332,13 @@ func (r *ModelReconciler) loadCacheJobForModel(m *kubeaiv1.Model, c ModelConfig)
 			Completions:             ptr.To[int32](1),
 			Template: corev1.PodTemplateSpec{
 				Spec: corev1.PodSpec{
-					RestartPolicy: corev1.RestartPolicyOnFailure,
+					RestartPolicy:   corev1.RestartPolicyOnFailure,
+					SecurityContext: r.ModelServerPods.ModelPodSecurityContext,
 					Containers: []corev1.Container{
 						{
-							Name: "loader",
-							Env:  env,
+							Name:            "loader",
+							Env:             env,
+							SecurityContext: r.ModelServerPods.ModelContainerSecurityContext,
 							VolumeMounts: []corev1.VolumeMount{
 								{
 									Name:      "model",
@@ -383,10 +385,12 @@ func (r *ModelReconciler) evictCacheJobForModel(m *kubeaiv1.Model, c ModelConfig
 			Completions:             ptr.To[int32](1),
 			Template: corev1.PodTemplateSpec{
 				Spec: corev1.PodSpec{
-					RestartPolicy: corev1.RestartPolicyOnFailure,
+					RestartPolicy:   corev1.RestartPolicyOnFailure,
+					SecurityContext: r.ModelServerPods.ModelPodSecurityContext,
 					Containers: []corev1.Container{
 						{
-							Name: "evictor",
+							Name:            "evictor",
+							SecurityContext: r.ModelServerPods.ModelContainerSecurityContext,
 							VolumeMounts: []corev1.VolumeMount{
 								{
 									Name:      "model",
diff --git a/internal/openaiserver/handler.go b/internal/openaiserver/handler.go
index c27eff7..4fd7d8e 100644
--- a/internal/openaiserver/handler.go
+++ b/internal/openaiserver/handler.go
@@ -38,6 +38,7 @@ func NewHandler(k8sClient client.Client, modelProxy *modelproxy.Handler) *Handle
 	handle("/openai/v1/chat/completions", http.StripPrefix("/openai", modelProxy))
 	handle("/openai/v1/completions", http.StripPrefix("/openai", modelProxy))
 	handle("/openai/v1/embeddings", http.StripPrefix("/openai", modelProxy))
+	handle("/vllm/v1/rerank", http.StripPrefix("/vllm", modelProxy))
 	handle("/openai/v1/audio/transcriptions", http.StripPrefix("/openai", modelProxy))
 	handle("/openai/v1/models", http.HandlerFunc(h.getModels))
 
diff --git a/manifests/models/bge-rerank-base-cpu.yaml b/manifests/models/bge-rerank-base-cpu.yaml
new file mode 100644
index 0000000..6487060
--- /dev/null
+++ b/manifests/models/bge-rerank-base-cpu.yaml
@@ -0,0 +1,11 @@
+# Source: models/templates/models.yaml
+apiVersion: kubeai.org/v1
+kind: Model
+metadata:
+  name: bge-rerank-base-cpu
+spec:
+  features: [Reranking]
+  url: hf://BAAI/bge-reranker-base
+  engine: Infinity
+  minReplicas: 0
+  resourceProfile: cpu:1



Test patch:
diff --git a/internal/apiutils/request_test.go b/internal/apiutils/request_test.go
index 5d1b3a9..1c2f9dd 100644
--- a/internal/apiutils/request_test.go
+++ b/internal/apiutils/request_test.go
@@ -59,6 +59,12 @@ func TestParseRequest(t *testing.T) {
 			expModel:  "test-model",
 			expPrefix: "test-prefi", // "test-prefix" (max 10) --> "test-prefi"
 		},
+		{
+			name:     "rerank request",
+			body:     `{"model": "test-model", "query": "q", "documents": ["d1", "d2"]}`,
+			path:     "/v1/rerank",
+			expModel: "test-model",
+		},
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {