Repository: kaito-project/kaito

Issue:
Consolidate inference perf benchmarks
To help users choose the optimal GPU sku to balance cost/perf of inference with KAITO + vLLM, we should publish benchmarking methods and results across a range of GPU types, including (but not limited to): T4, A10, A100, H100.

Short term goal: Publish results on Azure NC and ND series VM sizes, in a new directory in `kaito-project/kaito`.
Long term goal: Publish results on these skus across additional cloud services.


Discussion:
@chewong: 1. Create a Kubernetes cluster + 1 worker node per GPU SKU (Let's test A10, A100 and H100 for now. T4 will no longer work with vLLM 0.11.0).
2. Create a KAITO workspace. Start off with two small models first (phi-4-mini-instruct and llama-3.1-8b-instruct) and we can do the same for larger models (llama-3.1-70b-instruct) later
3. Use a LLM benchmark tool (e.g. https://github.com/vllm-project/guidellm) to measure TTFT and ITL vs different RPS and GPU SKUs.

Solution patch:
diff --git a/.github/actions/e2e-base-setup/action.yaml b/.github/actions/e2e-base-setup/action.yaml
index b8939d9..79e9be6 100644
--- a/.github/actions/e2e-base-setup/action.yaml
+++ b/.github/actions/e2e-base-setup/action.yaml
@@ -26,7 +26,7 @@ inputs:
   k8s_version:
     description: "Kubernetes version"
     required: false
-    default: "1.30.0"
+    default: "1.31.10"
 
 outputs:
   cluster_name:
@@ -166,6 +166,7 @@ runs:
         REGISTRY: ${{ env.REGISTRY }}
         VERSION: ${{ env.VERSION }}
         TEST_SUITE: ${{ inputs.node_provisioner }}
+        HELM_INSTALL_EXTRA_ARGS: "--set featureGates.gatewayAPIInferenceExtension=true"
 
     - name: Set up E2E ACR Credentials and Secret
       shell: bash
@@ -196,4 +197,4 @@ runs:
             --docker-username="$E2E_ACR_AMRT_USERNAME" \
             --docker-password="$E2E_ACR_AMRT_PASSWORD"
           echo "Secret $E2E_AMRT_SECRET_NAME created successfully."
-        fi
\ No newline at end of file
+        fi
diff --git a/.github/workflows/e2e-workflow.yaml b/.github/workflows/e2e-workflow.yaml
index 9ec48b4..99e8437 100644
--- a/.github/workflows/e2e-workflow.yaml
+++ b/.github/workflows/e2e-workflow.yaml
@@ -23,7 +23,7 @@ on:
         default: "swedencentral"
       k8s_version:
         type: string
-        default: "1.30.0"
+        default: "1.31.10"
 
 jobs:
   e2e-tests:
@@ -33,7 +33,7 @@ jobs:
     permissions:
       contents: read
       id-token: write # This is required for requesting the JWT
-    
+
     env:
       TEST_SUITE: ${{ inputs.node_provisioner }}
       GO_VERSION: "1.24"
@@ -84,10 +84,10 @@ jobs:
           make docker-build-dataset
         env:
           REGISTRY: ${{ env.CLUSTER_NAME }}.azurecr.io
-      
+
       - name: Remove existing Go modules directory
         run: sudo rm -rf ~/go/pkg/mod
-      
+
       - name: Set up Go ${{ env.GO_VERSION }}
         uses: actions/setup-go@v5.5.0
         with:
diff --git a/.github/workflows/ragengine-e2e-workflow.yaml b/.github/workflows/ragengine-e2e-workflow.yaml
index 7171a21..18571f3 100644
--- a/.github/workflows/ragengine-e2e-workflow.yaml
+++ b/.github/workflows/ragengine-e2e-workflow.yaml
@@ -23,7 +23,7 @@ on:
         default: "swedencentral"
       k8s_version:
         type: string
-        default: "1.30.0"
+        default: "1.31.10"
 
 jobs:
   e2e-tests:
@@ -32,9 +32,9 @@ jobs:
     permissions:
       contents: read
       id-token: write
-    
+
     env:
-      TEST_SUITE: ${{ inputs.node_provisioner }}      
+      TEST_SUITE: ${{ inputs.node_provisioner }}
       GO_VERSION: "1.24"
       KARPENTER_NAMESPACE: "karpenter"
       GPU_PROVISIONER_NAMESPACE: "gpu-provisioner"
@@ -51,7 +51,7 @@ jobs:
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           ref: ${{ inputs.git_sha }}
-      
+
       - name: Run e2e-base-setup composite action
         id: base-setup
         uses: ./.github/actions/e2e-base-setup
@@ -69,7 +69,7 @@ jobs:
         run: |
           echo "REGISTRY=${{ inputs.registry }}" >> $GITHUB_ENV
           echo "VERSION=$(echo ${{ inputs.tag }} | tr -d v)" >> $GITHUB_ENV
-      
+
       - name: build kaito RAG Engine image
         if: ${{ !inputs.isRelease }}
         shell: bash
@@ -77,7 +77,7 @@ jobs:
           make docker-build-ragengine
         env:
           REGISTRY: ${{ env.CLUSTER_NAME }}.azurecr.io
-      
+
       - name: build kaito RAG Engine service image
         if: ${{ !inputs.isRelease }}
         shell: bash
@@ -85,10 +85,10 @@ jobs:
           make docker-build-ragservice
         env:
           REGISTRY: ${{ env.CLUSTER_NAME }}.azurecr.io
-      
+
       - name: Remove existing Go modules directory
         run: sudo rm -rf ~/go/pkg/mod
-      
+
       - name: Set up Go ${{ env.GO_VERSION }}
         uses: actions/setup-go@v5.5.0
         with:
@@ -96,7 +96,7 @@ jobs:
 
       - name: Install Helm
         uses: azure/setup-helm@v4
-      
+
       - name: Install KAITO RAG Engine helm chart
         if: ${{ !inputs.isRelease }}
         shell: bash
diff --git a/Makefile b/Makefile
index cd22fed..96f98d5 100644
--- a/Makefile
+++ b/Makefile
@@ -30,7 +30,7 @@ TEST_SUITE ?= gpuprovisioner
 
 AZURE_SUBSCRIPTION_ID ?= $(AZURE_SUBSCRIPTION_ID)
 AZURE_LOCATION ?= eastus
-AKS_K8S_VERSION ?= 1.30.0
+AKS_K8S_VERSION ?= 1.31.10
 AZURE_CLUSTER_NAME ?= kaito-demo
 AZURE_RESOURCE_GROUP ?= demo
 AZURE_RESOURCE_GROUP_MC=MC_$(AZURE_RESOURCE_GROUP)_$(AZURE_CLUSTER_NAME)_$(AZURE_LOCATION)
@@ -60,6 +60,9 @@ GO_INSTALL := ./hack/go-install.sh
 
 BUILD_FLAGS ?=
 
+# Extra arguments for commands
+HELM_INSTALL_EXTRA_ARGS ?=
+
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(shell go env GOPATH)/bin
@@ -392,7 +395,7 @@ az-patch-install-helm: ## Install Kaito workspace Helm chart and set Azure clien
 	yq -i '(.image.tag)                                                     = "$(IMG_TAG)"'                               ./charts/kaito/workspace/values.yaml
 	yq -i '(.clusterName)                                                   = "$(AZURE_CLUSTER_NAME)"'                    ./charts/kaito/workspace/values.yaml
 
-	helm install kaito-workspace ./charts/kaito/workspace --namespace $(KAITO_NAMESPACE) --create-namespace
+	helm install kaito-workspace ./charts/kaito/workspace --namespace $(KAITO_NAMESPACE) --create-namespace $(HELM_INSTALL_EXTRA_ARGS)
 
 .PHONY: az-patch-install-ragengine-helm
 az-patch-install-ragengine-helm: ## Install Kaito RAG Engine Helm chart and set Azure client env vars and settings in Helm values.
@@ -402,7 +405,7 @@ az-patch-install-ragengine-helm: ## Install Kaito RAG Engine Helm chart and set
 	yq -i '(.image.tag)                                                     = "$(IMG_TAG)"'                               ./charts/kaito/ragengine/values.yaml
 	yq -i '(.clusterName)                                                   = "$(AZURE_CLUSTER_NAME)"'                    ./charts/kaito/ragengine/values.yaml
 
-	helm install kaito-ragengine ./charts/kaito/ragengine --namespace $(KAITO_RAGENGINE_NAMESPACE) --create-namespace
+	helm install kaito-ragengine ./charts/kaito/ragengine --namespace $(KAITO_RAGENGINE_NAMESPACE) --create-namespace $(HELM_INSTALL_EXTRA_ARGS)
 
 .PHONY: az-patch-install-ragengine-helm-e2e
 az-patch-install-ragengine-helm-e2e: ## Install Kaito RAG Engine Helm chart for e2e tests and set Azure client env vars and settings in Helm values.
@@ -415,7 +418,7 @@ az-patch-install-ragengine-helm-e2e: ## Install Kaito RAG Engine Helm chart for
 	yq -i '(.presetRagImageName)                                         	= "$(RAGENGINE_SERVICE_IMG_NAME)"'            ./charts/kaito/ragengine/values.yaml
 	yq -i '(.presetRagImageTag)                                         	= "$(RAGENGINE_SERVICE_IMG_TAG)"'             ./charts/kaito/ragengine/values.yaml
 
-	helm install kaito-ragengine ./charts/kaito/ragengine --namespace $(KAITO_RAGENGINE_NAMESPACE) --create-namespace
+	helm install kaito-ragengine ./charts/kaito/ragengine --namespace $(KAITO_RAGENGINE_NAMESPACE) --create-namespace $(HELM_INSTALL_EXTRA_ARGS)
 
 .PHONY: aws-patch-install-helm
 aws-patch-install-helm: ## Install Kaito workspace Helm chart and set AWS env vars and settings in Helm values.
@@ -424,7 +427,7 @@ aws-patch-install-helm: ## Install Kaito workspace Helm chart and set AWS env va
 	yq -i '(.clusterName)                                                   = "$(AWS_CLUSTER_NAME)"'                    		./charts/kaito/workspace/values.yaml
 	yq -i '(.cloudProviderName)                                             = "aws"'                                        ./charts/kaito/workspace/values.yaml
 
-	helm install kaito-workspace ./charts/kaito/workspace --namespace $(KAITO_NAMESPACE) --create-namespace
+	helm install kaito-workspace ./charts/kaito/workspace --namespace $(KAITO_NAMESPACE) --create-namespace $(HELM_INSTALL_EXTRA_ARGS)
 
 generate-identities: ## Create identities for the provisioner component.
 	./hack/deploy/generate-identities.sh \
@@ -446,7 +449,7 @@ gpu-provisioner-helm: ## Install GPU provisioner Helm chart for Azure cluster an
 	--values gpu-provisioner-values.yaml \
 	--set settings.azure.clusterName=$(AZURE_CLUSTER_NAME) \
 	--namespace $(GPU_PROVISIONER_NAMESPACE) --create-namespace \
-	https://github.com/Azure/gpu-provisioner/raw/gh-pages/charts/gpu-provisioner-$(GPU_PROVISIONER_VERSION).tgz
+	https://github.com/Azure/gpu-provisioner/raw/gh-pages/charts/gpu-provisioner-$(GPU_PROVISIONER_VERSION).tgz $(HELM_INSTALL_EXTRA_ARGS)
 
 	kubectl wait --for=condition=available deploy "gpu-provisioner" -n gpu-provisioner --timeout=300s
 
diff --git a/Tiltfile b/Tiltfile
index 6b5d45a..bc8ec36 100644
--- a/Tiltfile
+++ b/Tiltfile
@@ -11,7 +11,15 @@ if 'allowed_contexts' in settings:
     allow_k8s_contexts(settings['allowed_contexts'])
 if 'default_registry' in settings:
     # Set the default registry for tilt to use
-    default_registry(settings['default_registry'].removesuffix('/') + '/kaito/workspace')
+    default_registry(settings['default_registry'].removesuffix('/'))
+if 'feature_gates' in settings:
+    # Set the feature gates for the controller
+    feature_gates = settings['feature_gates']
+else:
+    feature_gates = {
+        'vLLM': True,
+        'gatewayAPIInferenceExtension': True,
+    }
 
 def main(IMG='controller:latest', DISABLE_SECURITY_CONTEXT=True):
 
@@ -25,7 +33,7 @@ def main(IMG='controller:latest', DISABLE_SECURITY_CONTEXT=True):
 
     def yaml():
         cluster_name = k8s_context() if not settings.get('cluster_name') else settings['cluster_name']
-        helm_template = 'helm template kaito-workspace ./charts/kaito/workspace --namespace kaito-workspace --set clusterName={} --set image.repository=controller --set image.tag=latest'.format(k8s_context())
+        helm_template = 'helm template kaito-workspace ./charts/kaito/workspace --namespace kaito-workspace --set clusterName={} --set image.repository=controller --set image.tag=latest --set featureGates.gatewayAPIInferenceExtension={}'.format(cluster_name, feature_gates['gatewayAPIInferenceExtension'])
         # Set the image name and tag to controller:latest for Tilt to
         # substitute later during docker_build_with_restart
         data = local(helm_template, quiet=True)
@@ -34,6 +42,10 @@ def main(IMG='controller:latest', DISABLE_SECURITY_CONTEXT=True):
             decoded = decode_yaml_stream(data)
             if decoded:
                 for d in decoded:
+                    # Workaround: Force the namespace to "kaito-workspace" because `helm template` does not correctly render
+                    # the subchart namespace. See: https://github.com/fluxcd-community/helm-charts/issues/239.
+                    if "namespace" not in d["metadata"]:
+                        d["metadata"]["namespace"] = "kaito-workspace"
                     if d["kind"] == "Deployment":
                         if "securityContext" in d['spec']['template']['spec']:
                             d['spec']['template']['spec'].pop('securityContext')
@@ -106,7 +118,9 @@ def main(IMG='controller:latest', DISABLE_SECURITY_CONTEXT=True):
     # Tilt will replace the old one in the running container with the new one.
     docker_build_with_restart(IMG, '.',
      dockerfile_contents=DOCKERFILE,
-     entrypoint='/manager',
+     entrypoint='/manager --feature-gates={}'.format(
+        ','.join(['{}={}'.format(k, str(v).lower()) for k, v in feature_gates.items()])
+     ),
      only=['./tilt_bin/manager', './presets/workspace/models/supported_models.yaml'],
      live_update=[
            sync('./tilt_bin/manager', '/manager'),
diff --git a/api/v1beta1/workspace_types.go b/api/v1beta1/workspace_types.go
index 01897fa..f116ead 100644
--- a/api/v1beta1/workspace_types.go
+++ b/api/v1beta1/workspace_types.go
@@ -24,6 +24,7 @@ import (
 // will provision new nodes before deploying the workload.
 // The final list of nodes used to run the workload is presented in workspace Status.
 type ResourceSpec struct {
+	// Deprecated: Count is deprecated in v1beta1 and will be removed in a future version.
 	// Count is the required number of GPU nodes.
 	// +optional
 	// +kubebuilder:default:=1
@@ -84,6 +85,8 @@ type PresetSpec struct {
 }
 
 type InferenceSpec struct {
+	// Number of desired inference workloads, Default value is 1.
+	Replicas int32 `json:"replicas,omitempty"`
 	// Preset describes the base model that will be deployed with preset configurations.
 	// +optional
 	Preset *PresetSpec `json:"preset,omitempty"`
@@ -104,6 +107,26 @@ type InferenceSpec struct {
 	Adapters []AdapterSpec `json:"adapters,omitempty"`
 }
 
+type InferenceStatus struct {
+	// Total number of running inference workloads of the workspace.
+	// This field is dynamic and changes based on scaling actions. It will set to Deployement.Status.Replicas
+	// when underlay workload is Deployment for Workspace.
+	Replicas int32 `json:"replicas,omitempty"`
+
+	// Selector is used to select the pods that provide metrics for making scaling action decisions.
+	// This field must be set when HPA and VPA is used for scaling.
+	Selector string `json:"selector,omitempty"`
+
+	// PerReplicaNodeCount is used for recording the number of gpu nodes for one replica of workspace workload.
+	// This field remains immutable after being set.
+	PerReplicaNodeCount int32 `json:"perReplicaNodeCount,omitempty"`
+
+	// TargetNodeCount is used for recording the desired number of gpu nodes that needed for all replicas of workspace.
+	// This field is dynamic and changes based on scaling actions. It has a specific calculation formula:
+	// TargetNodeCount = Resource.Replicas * InferenceStatus.PerReplicaNodeCount
+	TargetNodeCount int32 `json:"targetNodeCount,omitempty"`
+}
+
 type AdapterSpec struct {
 	// Source describes where to obtain the adapter data.
 	// +optional
@@ -184,6 +207,11 @@ type WorkspaceStatus struct {
 	// Conditions report the current conditions of the workspace.
 	// +optional
 	Conditions []metav1.Condition `json:"conditions,omitempty"`
+
+	// Inference provides the status of the inference service.
+	// It is only set when the InferenceSpec is specified in the Workspace.
+	// +optional
+	Inference *InferenceStatus `json:"inference,omitempty"`
 }
 
 // Workspace is the Schema for the workspaces API
@@ -197,6 +225,7 @@ type WorkspaceStatus struct {
 // +kubebuilder:printcolumn:name="JobStarted",type="string",JSONPath=".status.conditions[?(@.type==\"JobStarted\")].status",description=""
 // +kubebuilder:printcolumn:name="WorkspaceSucceeded",type="string",JSONPath=".status.conditions[?(@.type==\"WorkspaceSucceeded\")].status",description=""
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description=""
+// +kubebuilder:subresource:scale:specpath=.spec.inference.replicas,statuspath=.status.inference.replicas,selectorpath=.status.inference.selector
 type Workspace struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go
index a280362..693e494 100644
--- a/api/v1beta1/zz_generated.deepcopy.go
+++ b/api/v1beta1/zz_generated.deepcopy.go
@@ -1,3 +1,5 @@
+//go:build !ignore_autogenerated
+
 // Copyright (c) KAITO authors.
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -166,6 +168,21 @@ func (in *InferenceSpec) DeepCopy() *InferenceSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *InferenceStatus) DeepCopyInto(out *InferenceStatus) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InferenceStatus.
+func (in *InferenceStatus) DeepCopy() *InferenceStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(InferenceStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *PresetMeta) DeepCopyInto(out *PresetMeta) {
 	*out = *in
@@ -419,6 +436,11 @@ func (in *WorkspaceStatus) DeepCopyInto(out *WorkspaceStatus) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.Inference != nil {
+		in, out := &in.Inference, &out.Inference
+		*out = new(InferenceStatus)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkspaceStatus.
diff --git a/charts/kaito/workspace/Chart.yaml b/charts/kaito/workspace/Chart.yaml
index c0a2df1..31d2484 100644
--- a/charts/kaito/workspace/Chart.yaml
+++ b/charts/kaito/workspace/Chart.yaml
@@ -36,4 +36,4 @@ dependencies:
 - name: flux2
   version: "2.16.3"
   repository: https://fluxcd-community.github.io/helm-charts
-  condition: flux2.enabled
+  condition: featureGates.gatewayAPIInferenceExtension
diff --git a/charts/kaito/workspace/crds/inference.networking.x-k8s.io_inferencemodels.yaml b/charts/kaito/workspace/crds/inference.networking.x-k8s.io_inferencemodels.yaml
new file mode 100644
index 0000000..d76c93f
--- /dev/null
+++ b/charts/kaito/workspace/crds/inference.networking.x-k8s.io_inferencemodels.yaml
@@ -0,0 +1,256 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.16.1
+  name: inferencemodels.inference.networking.x-k8s.io
+spec:
+  group: inference.networking.x-k8s.io
+  names:
+    kind: InferenceModel
+    listKind: InferenceModelList
+    plural: inferencemodels
+    singular: inferencemodel
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - jsonPath: .spec.modelName
+      name: Model Name
+      type: string
+    - jsonPath: .spec.poolRef.name
+      name: Inference Pool
+      type: string
+    - jsonPath: .spec.criticality
+      name: Criticality
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1alpha2
+    schema:
+      openAPIV3Schema:
+        description: InferenceModel is the Schema for the InferenceModels API.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: |-
+              InferenceModelSpec represents the desired state of a specific model use case. This resource is
+              managed by the "Inference Workload Owner" persona.
+
+              The Inference Workload Owner persona is someone that trains, verifies, and
+              leverages a large language model from a model frontend, drives the lifecycle
+              and rollout of new versions of those models, and defines the specific
+              performance and latency goals for the model. These workloads are
+              expected to operate within an InferencePool sharing compute capacity with other
+              InferenceModels, defined by the Inference Platform Admin.
+
+              InferenceModel's modelName (not the ObjectMeta name) is unique for a given InferencePool,
+              if the name is reused, an error will be shown on the status of a
+              InferenceModel that attempted to reuse. The oldest InferenceModel, based on
+              creation timestamp, will be selected to remain valid. In the event of a race
+              condition, one will be selected at random.
+            properties:
+              criticality:
+                description: |-
+                  Criticality defines how important it is to serve the model compared to other models referencing the same pool.
+                  Criticality impacts how traffic is handled in resource constrained situations. It handles this by
+                  queuing or rejecting requests of lower criticality. InferenceModels of an equivalent Criticality will
+                  fairly share resources over throughput of tokens. In the future, the metric used to calculate fairness,
+                  and the proportionality of fairness will be configurable.
+
+                  Default values for this field will not be set, to allow for future additions of new field that may 'one of' with this field.
+                  Any implementations that may consume this field may treat an unset value as the 'Standard' range.
+                enum:
+                - Critical
+                - Standard
+                - Sheddable
+                type: string
+              modelName:
+                description: |-
+                  ModelName is the name of the model as it will be set in the "model" parameter for an incoming request.
+                  ModelNames must be unique for a referencing InferencePool
+                  (names can be reused for a different pool in the same cluster).
+                  The modelName with the oldest creation timestamp is retained, and the incoming
+                  InferenceModel's Ready status is set to false with a corresponding reason.
+                  In the rare case of a race condition, one Model will be selected randomly to be considered valid, and the other rejected.
+                  Names can be reserved without an underlying model configured in the pool.
+                  This can be done by specifying a target model and setting the weight to zero,
+                  an error will be returned specifying that no valid target model is found.
+                maxLength: 256
+                type: string
+                x-kubernetes-validations:
+                - message: modelName is immutable
+                  rule: self == oldSelf
+              poolRef:
+                description: PoolRef is a reference to the inference pool, the pool
+                  must exist in the same namespace.
+                properties:
+                  group:
+                    default: inference.networking.x-k8s.io
+                    description: Group is the group of the referent.
+                    maxLength: 253
+                    pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
+                    type: string
+                  kind:
+                    default: InferencePool
+                    description: Kind is kind of the referent. For example "InferencePool".
+                    maxLength: 63
+                    minLength: 1
+                    pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$
+                    type: string
+                  name:
+                    description: Name is the name of the referent.
+                    maxLength: 253
+                    minLength: 1
+                    type: string
+                required:
+                - name
+                type: object
+              targetModels:
+                description: |-
+                  TargetModels allow multiple versions of a model for traffic splitting.
+                  If not specified, the target model name is defaulted to the modelName parameter.
+                  modelName is often in reference to a LoRA adapter.
+                items:
+                  description: |-
+                    TargetModel represents a deployed model or a LoRA adapter. The
+                    Name field is expected to match the name of the LoRA adapter
+                    (or base model) as it is registered within the model server. Inference
+                    Gateway assumes that the model exists on the model server and it's the
+                    responsibility of the user to validate a correct match. Should a model fail
+                    to exist at request time, the error is processed by the Inference Gateway
+                    and emitted on the appropriate InferenceModel object.
+                  properties:
+                    name:
+                      description: Name is the name of the adapter or base model,
+                        as expected by the ModelServer.
+                      maxLength: 253
+                      type: string
+                    weight:
+                      description: |-
+                        Weight is used to determine the proportion of traffic that should be
+                        sent to this model when multiple target models are specified.
+
+                        Weight defines the proportion of requests forwarded to the specified
+                        model. This is computed as weight/(sum of all weights in this
+                        TargetModels list). For non-zero values, there may be some epsilon from
+                        the exact proportion defined here depending on the precision an
+                        implementation supports. Weight is not a percentage and the sum of
+                        weights does not need to equal 100.
+
+                        If a weight is set for any targetModel, it must be set for all targetModels.
+                        Conversely weights are optional, so long as ALL targetModels do not specify a weight.
+                      format: int32
+                      maximum: 1000000
+                      minimum: 1
+                      type: integer
+                  required:
+                  - name
+                  type: object
+                maxItems: 10
+                type: array
+                x-kubernetes-validations:
+                - message: Weights should be set for all models, or none of the models.
+                  rule: self.all(model, has(model.weight)) || self.all(model, !has(model.weight))
+            required:
+            - modelName
+            - poolRef
+            type: object
+          status:
+            description: InferenceModelStatus defines the observed state of InferenceModel
+            properties:
+              conditions:
+                default:
+                - lastTransitionTime: "1970-01-01T00:00:00Z"
+                  message: Waiting for controller
+                  reason: Pending
+                  status: Unknown
+                  type: Ready
+                description: |-
+                  Conditions track the state of the InferenceModel.
+
+                  Known condition types are:
+
+                  * "Accepted"
+                items:
+                  description: Condition contains details for one aspect of the current
+                    state of this API Resource.
+                  properties:
+                    lastTransitionTime:
+                      description: |-
+                        lastTransitionTime is the last time the condition transitioned from one status to another.
+                        This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                      format: date-time
+                      type: string
+                    message:
+                      description: |-
+                        message is a human readable message indicating details about the transition.
+                        This may be an empty string.
+                      maxLength: 32768
+                      type: string
+                    observedGeneration:
+                      description: |-
+                        observedGeneration represents the .metadata.generation that the condition was set based upon.
+                        For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                        with respect to the current state of the instance.
+                      format: int64
+                      minimum: 0
+                      type: integer
+                    reason:
+                      description: |-
+                        reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                        Producers of specific condition types may define expected values and meanings for this field,
+                        and whether the values are considered a guaranteed API.
+                        The value should be a CamelCase string.
+                        This field may not be empty.
+                      maxLength: 1024
+                      minLength: 1
+                      pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                      type: string
+                    status:
+                      description: status of the condition, one of True, False, Unknown.
+                      enum:
+                      - "True"
+                      - "False"
+                      - Unknown
+                      type: string
+                    type:
+                      description: type of condition in CamelCase or in foo.example.com/CamelCase.
+                      maxLength: 316
+                      pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                      type: string
+                  required:
+                  - lastTransitionTime
+                  - message
+                  - reason
+                  - status
+                  - type
+                  type: object
+                maxItems: 8
+                type: array
+                x-kubernetes-list-map-keys:
+                - type
+                x-kubernetes-list-type: map
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
diff --git a/charts/kaito/workspace/crds/inference.networking.x-k8s.io_inferencepools.yaml b/charts/kaito/workspace/crds/inference.networking.x-k8s.io_inferencepools.yaml
new file mode 100644
index 0000000..0c5f1b7
--- /dev/null
+++ b/charts/kaito/workspace/crds/inference.networking.x-k8s.io_inferencepools.yaml
@@ -0,0 +1,285 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.16.1
+  name: inferencepools.inference.networking.x-k8s.io
+spec:
+  group: inference.networking.x-k8s.io
+  names:
+    kind: InferencePool
+    listKind: InferencePoolList
+    plural: inferencepools
+    singular: inferencepool
+  scope: Namespaced
+  versions:
+  - name: v1alpha2
+    schema:
+      openAPIV3Schema:
+        description: InferencePool is the Schema for the InferencePools API.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: InferencePoolSpec defines the desired state of InferencePool
+            properties:
+              extensionRef:
+                description: Extension configures an endpoint picker as an extension
+                  service.
+                properties:
+                  failureMode:
+                    default: FailClose
+                    description: |-
+                      Configures how the gateway handles the case when the extension is not responsive.
+                      Defaults to failClose.
+                    enum:
+                    - FailOpen
+                    - FailClose
+                    type: string
+                  group:
+                    default: ""
+                    description: |-
+                      Group is the group of the referent.
+                      The default value is "", representing the Core API group.
+                    maxLength: 253
+                    pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
+                    type: string
+                  kind:
+                    default: Service
+                    description: |-
+                      Kind is the Kubernetes resource kind of the referent. For example
+                      "Service".
+
+                      Defaults to "Service" when not specified.
+
+                      ExternalName services can refer to CNAME DNS records that may live
+                      outside of the cluster and as such are difficult to reason about in
+                      terms of conformance. They also may not be safe to forward to (see
+                      CVE-2021-25740 for more information). Implementations MUST NOT
+                      support ExternalName Services.
+                    maxLength: 63
+                    minLength: 1
+                    pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$
+                    type: string
+                  name:
+                    description: Name is the name of the referent.
+                    maxLength: 253
+                    minLength: 1
+                    type: string
+                  portNumber:
+                    description: |-
+                      The port number on the service running the extension. When unspecified,
+                      implementations SHOULD infer a default value of 9002 when the Kind is
+                      Service.
+                    format: int32
+                    maximum: 65535
+                    minimum: 1
+                    type: integer
+                required:
+                - name
+                type: object
+              selector:
+                additionalProperties:
+                  description: |-
+                    LabelValue is the value of a label. This is used for validation
+                    of maps. This matches the Kubernetes label validation rules:
+                    * must be 63 characters or less (can be empty),
+                    * unless empty, must begin and end with an alphanumeric character ([a-z0-9A-Z]),
+                    * could contain dashes (-), underscores (_), dots (.), and alphanumerics between.
+
+                    Valid values include:
+
+                    * MyValue
+                    * my.name
+                    * 123-my-value
+                  maxLength: 63
+                  minLength: 0
+                  pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$
+                  type: string
+                description: |-
+                  Selector defines a map of labels to watch model server pods
+                  that should be included in the InferencePool.
+                  In some cases, implementations may translate this field to a Service selector, so this matches the simple
+                  map used for Service selectors instead of the full Kubernetes LabelSelector type.
+                  If sepecified, it will be applied to match the model server pods in the same namespace as the InferencePool.
+                  Cross namesoace selector is not supported.
+                type: object
+              targetPortNumber:
+                description: |-
+                  TargetPortNumber defines the port number to access the selected model servers.
+                  The number must be in the range 1 to 65535.
+                format: int32
+                maximum: 65535
+                minimum: 1
+                type: integer
+            required:
+            - extensionRef
+            - selector
+            - targetPortNumber
+            type: object
+          status:
+            default:
+              parent:
+              - conditions:
+                - lastTransitionTime: "1970-01-01T00:00:00Z"
+                  message: Waiting for controller
+                  reason: Pending
+                  status: Unknown
+                  type: Accepted
+                parentRef:
+                  kind: Status
+                  name: default
+            description: Status defines the observed state of InferencePool.
+            properties:
+              parent:
+                description: |-
+                  Parents is a list of parent resources (usually Gateways) that are
+                  associated with the InferencePool, and the status of the InferencePool with respect to
+                  each parent.
+
+                  A maximum of 32 Gateways will be represented in this list. When the list contains
+                  `kind: Status, name: default`, it indicates that the InferencePool is not
+                  associated with any Gateway and a controller must perform the following:
+
+                   - Remove the parent when setting the "Accepted" condition.
+                   - Add the parent when the controller will no longer manage the InferencePool
+                     and no other parents exist.
+                items:
+                  description: PoolStatus defines the observed state of InferencePool
+                    from a Gateway.
+                  properties:
+                    conditions:
+                      default:
+                      - lastTransitionTime: "1970-01-01T00:00:00Z"
+                        message: Waiting for controller
+                        reason: Pending
+                        status: Unknown
+                        type: Accepted
+                      description: |-
+                        Conditions track the state of the InferencePool.
+
+                        Known condition types are:
+
+                        * "Accepted"
+                        * "ResolvedRefs"
+                      items:
+                        description: Condition contains details for one aspect of
+                          the current state of this API Resource.
+                        properties:
+                          lastTransitionTime:
+                            description: |-
+                              lastTransitionTime is the last time the condition transitioned from one status to another.
+                              This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                            format: date-time
+                            type: string
+                          message:
+                            description: |-
+                              message is a human readable message indicating details about the transition.
+                              This may be an empty string.
+                            maxLength: 32768
+                            type: string
+                          observedGeneration:
+                            description: |-
+                              observedGeneration represents the .metadata.generation that the condition was set based upon.
+                              For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                              with respect to the current state of the instance.
+                            format: int64
+                            minimum: 0
+                            type: integer
+                          reason:
+                            description: |-
+                              reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                              Producers of specific condition types may define expected values and meanings for this field,
+                              and whether the values are considered a guaranteed API.
+                              The value should be a CamelCase string.
+                              This field may not be empty.
+                            maxLength: 1024
+                            minLength: 1
+                            pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                            type: string
+                          status:
+                            description: status of the condition, one of True, False,
+                              Unknown.
+                            enum:
+                            - "True"
+                            - "False"
+                            - Unknown
+                            type: string
+                          type:
+                            description: type of condition in CamelCase or in foo.example.com/CamelCase.
+                            maxLength: 316
+                            pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                            type: string
+                        required:
+                        - lastTransitionTime
+                        - message
+                        - reason
+                        - status
+                        - type
+                        type: object
+                      maxItems: 8
+                      type: array
+                      x-kubernetes-list-map-keys:
+                      - type
+                      x-kubernetes-list-type: map
+                    parentRef:
+                      description: GatewayRef indicates the gateway that observed
+                        state of InferencePool.
+                      properties:
+                        group:
+                          default: gateway.networking.k8s.io
+                          description: Group is the group of the referent.
+                          maxLength: 253
+                          pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
+                          type: string
+                        kind:
+                          default: Gateway
+                          description: Kind is kind of the referent. For example "Gateway".
+                          maxLength: 63
+                          minLength: 1
+                          pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$
+                          type: string
+                        name:
+                          description: Name is the name of the referent.
+                          maxLength: 253
+                          minLength: 1
+                          type: string
+                        namespace:
+                          description: |-
+                            Namespace is the namespace of the referent.  If not present,
+                            the namespace of the referent is assumed to be the same as
+                            the namespace of the referring object.
+                          maxLength: 63
+                          minLength: 1
+                          pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$
+                          type: string
+                      required:
+                      - name
+                      type: object
+                  required:
+                  - parentRef
+                  type: object
+                maxItems: 32
+                type: array
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
diff --git a/charts/kaito/workspace/crds/kaito.sh_workspaces.yaml b/charts/kaito/workspace/crds/kaito.sh_workspaces.yaml
index 9a4dc38..6f7b5a3 100644
--- a/charts/kaito/workspace/crds/kaito.sh_workspaces.yaml
+++ b/charts/kaito/workspace/crds/kaito.sh_workspaces.yaml
@@ -523,6 +523,11 @@ spec:
                 required:
                 - name
                 type: object
+              replicas:
+                description: Number of desired inference workloads, Default value
+                  is 1.
+                format: int32
+                type: integer
               template:
                 description: |-
                   Template specifies the Pod template used to run the inference service. Users can specify custom Pod settings
@@ -550,7 +555,9 @@ spec:
             properties:
               count:
                 default: 1
-                description: Count is the required number of GPU nodes.
+                description: |-
+                  Deprecated: Count is deprecated in v1beta1 and will be removed in a future version.
+                  Count is the required number of GPU nodes.
                 type: integer
               instanceType:
                 default: Standard_NC24ads_A100_v4
@@ -689,6 +696,31 @@ spec:
                   - type
                   type: object
                 type: array
+              inference:
+                description: |-
+                  Inference provides the status of the inference service.
+                  It is only set when the InferenceSpec is specified in the Workspace.
+                properties:
+                  perReplicaNodeCount:
+                    description: PerReplicaNodeCount is used for recording the number
+                      of gpu nodes for one replica of workspace workload.
+                    format: int32
+                    type: integer
+                  replicas:
+                    description: Total number of running inference workloads of the
+                      workspace.
+                    format: int32
+                    type: integer
+                  selector:
+                    description: Selector is used to select the pods for the inference
+                      workload.
+                    type: string
+                  targetNodeCount:
+                    description: TargetNodeCount is used for recording the total number
+                      of gpu nodes for the workspace.
+                    format: int32
+                    type: integer
+                type: object
               workerNodes:
                 description: WorkerNodes is the list of nodes chosen to run the workload
                   based on the workspace resource requirement.
@@ -801,4 +833,8 @@ spec:
     served: true
     storage: true
     subresources:
+      scale:
+        labelSelectorPath: .status.inference.selector
+        specReplicasPath: .spec.inference.replicas
+        statusReplicasPath: .status.inference.replicas
       status: {}
diff --git a/charts/kaito/workspace/templates/_helpers.tpl b/charts/kaito/workspace/templates/_helpers.tpl
index b1d92af..f6bb305 100644
--- a/charts/kaito/workspace/templates/_helpers.tpl
+++ b/charts/kaito/workspace/templates/_helpers.tpl
@@ -57,7 +57,7 @@ Utils function
 {{- define "utils.joinKeyValuePairs" -}}
 {{- $pairs := list -}}
 {{- range $key, $value := . -}}
-{{- $pairs = append $pairs (printf "%s=%s" $key $value) -}}
+{{- $pairs = append $pairs (printf "%s=%t" $key $value) -}}
 {{- end -}}
 {{- join "," $pairs -}}
 {{- end -}}
diff --git a/charts/kaito/workspace/templates/clusterrole.yaml b/charts/kaito/workspace/templates/clusterrole.yaml
index 77ad044..2170f52 100644
--- a/charts/kaito/workspace/templates/clusterrole.yaml
+++ b/charts/kaito/workspace/templates/clusterrole.yaml
@@ -66,4 +66,12 @@ rules:
     verbs: ["get", "list", "watch", "patch", "delete"]
   - apiGroups: ["storage.k8s.io"]
     resources: ["storageclasses"]
-    verbs: ["get", "list", "watch"]
\ No newline at end of file
+    verbs: ["get", "list", "watch"]
+  {{- if .Values.featureGates.gatewayAPIInferenceExtension }}
+  - apiGroups: ["source.toolkit.fluxcd.io"]
+    resources: ["ocirepositories"]
+    verbs: ["get", "list", "watch", "create", "update"]
+  - apiGroups: ["helm.toolkit.fluxcd.io"]
+    resources: ["helmreleases"]
+    verbs: ["get", "list", "watch", "create", "update"]
+  {{- end }}
diff --git a/charts/kaito/workspace/templates/local-csi-driver-ds.yaml b/charts/kaito/workspace/templates/local-csi-driver-ds.yaml
index c86d580..b5994b9 100644
--- a/charts/kaito/workspace/templates/local-csi-driver-ds.yaml
+++ b/charts/kaito/workspace/templates/local-csi-driver-ds.yaml
@@ -193,6 +193,7 @@ spec:
         - --trace-sample-rate=1000000
         - --trace-service-id=$(POD_NAME)
         - --v=2
+        - --enable-cleanup=true
         command:
         - /local-csi-driver
         env:
@@ -208,7 +209,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
-        image: "mcr.microsoft.com/acstor/local-csi-driver:v0.2.3"
+        image: "mcr.microsoft.com/acstor/local-csi-driver:v0.2.4"
         imagePullPolicy: IfNotPresent
         livenessProbe:
           httpGet:
diff --git a/charts/kaito/workspace/values.yaml b/charts/kaito/workspace/values.yaml
index 88b2158..ac4169a 100644
--- a/charts/kaito/workspace/values.yaml
+++ b/charts/kaito/workspace/values.yaml
@@ -16,8 +16,9 @@ securityContext:
     drop:
       - "ALL"
 featureGates:
-  vLLM: "true"
-  disableNodeAutoProvisioning: "false"
+  vLLM: true
+  disableNodeAutoProvisioning: false
+  gatewayAPIInferenceExtension: false
 webhook:
   port: 9443
 # Knative logging configuration
@@ -39,11 +40,10 @@ cloudProviderName: "azure"
 clusterName: "kaito"
 # Set values for the Flux controller to only install the Helm controller
 flux2:
-  enabled: false
   helmController:
-    create: false
+    create: true
   sourceController:
-    create: false
+    create: true
   imageAutomationController:
     create: false
   imageReflectionController:
@@ -51,4 +51,4 @@ flux2:
   kustomizeController:
     create: false
   notificationController:
-    create: false
\ No newline at end of file
+    create: false
diff --git a/cmd/workspace/main.go b/cmd/workspace/main.go
index 3a09451..e687e9d 100644
--- a/cmd/workspace/main.go
+++ b/cmd/workspace/main.go
@@ -24,6 +24,8 @@ import (
 
 	//+kubebuilder:scaffold:imports
 	azurev1alpha2 "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2"
+	helmv2 "github.com/fluxcd/helm-controller/api/v2"
+	sourcev1 "github.com/fluxcd/source-controller/api/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
@@ -69,6 +71,8 @@ func init() {
 	utilruntime.Must(kaitoutils.KarpenterSchemeBuilder.AddToScheme(scheme))
 	utilruntime.Must(azurev1alpha2.SchemeBuilder.AddToScheme(scheme))
 	utilruntime.Must(kaitoutils.AwsSchemeBuilder.AddToScheme(scheme))
+	utilruntime.Must(helmv2.AddToScheme(scheme))
+	utilruntime.Must(sourcev1.AddToScheme(scheme))
 
 	//+kubebuilder:scaffold:scheme
 	klog.InitFlags(nil)
@@ -94,6 +98,11 @@ func main() {
 	opts.BindFlags(flag.CommandLine)
 	flag.Parse()
 
+	if err := featuregates.ParseAndValidateFeatureGates(featureGates); err != nil {
+		klog.ErrorS(err, "unable to set `feature-gates` flag")
+		exitWithErrorFunc()
+	}
+
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
 	ctx := withShutdownSignal(context.Background())
@@ -181,11 +190,6 @@ func main() {
 		time.Sleep(2 * time.Second)
 	}
 
-	if err := featuregates.ParseAndValidateFeatureGates(featureGates); err != nil {
-		klog.ErrorS(err, "unable to set `feature-gates` flag")
-		exitWithErrorFunc()
-	}
-
 	klog.InfoS("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		klog.ErrorS(err, "problem running manager")
diff --git a/config/crd/bases/kaito.sh_workspaces.yaml b/config/crd/bases/kaito.sh_workspaces.yaml
index 9a4dc38..ac4a1b1 100644
--- a/config/crd/bases/kaito.sh_workspaces.yaml
+++ b/config/crd/bases/kaito.sh_workspaces.yaml
@@ -523,6 +523,11 @@ spec:
                 required:
                 - name
                 type: object
+              replicas:
+                description: Number of desired inference workloads, Default value
+                  is 1.
+                format: int32
+                type: integer
               template:
                 description: |-
                   Template specifies the Pod template used to run the inference service. Users can specify custom Pod settings
@@ -550,7 +555,9 @@ spec:
             properties:
               count:
                 default: 1
-                description: Count is the required number of GPU nodes.
+                description: |-
+                  Deprecated: Count is deprecated in v1beta1 and will be removed in a future version.
+                  Count is the required number of GPU nodes.
                 type: integer
               instanceType:
                 default: Standard_NC24ads_A100_v4
@@ -689,6 +696,37 @@ spec:
                   - type
                   type: object
                 type: array
+              inference:
+                description: |-
+                  Inference provides the status of the inference service.
+                  It is only set when the InferenceSpec is specified in the Workspace.
+                properties:
+                  perReplicaNodeCount:
+                    description: |-
+                      PerReplicaNodeCount is used for recording the number of gpu nodes for one replica of workspace workload.
+                      This field remains immutable after being set.
+                    format: int32
+                    type: integer
+                  replicas:
+                    description: |-
+                      Total number of running inference workloads of the workspace.
+                      This field is dynamic and changes based on scaling actions. It will set to Deployement.Status.Replicas
+                      when underlay workload is Deployment for Workspace.
+                    format: int32
+                    type: integer
+                  selector:
+                    description: |-
+                      Selector is used to select the pods that provide metrics for making scaling action decisions.
+                      This field must be set when HPA and VPA is used for scaling.
+                    type: string
+                  targetNodeCount:
+                    description: |-
+                      TargetNodeCount is used for recording the desired number of gpu nodes that needed for all replicas of workspace.
+                      This field is dynamic and changes based on scaling actions. It has a specific calculation formula:
+                      TargetNodeCount = Resource.Replicas * InferenceStatus.PerReplicaNodeCount
+                    format: int32
+                    type: integer
+                type: object
               workerNodes:
                 description: WorkerNodes is the list of nodes chosen to run the workload
                   based on the workspace resource requirement.
@@ -801,4 +839,8 @@ spec:
     served: true
     storage: true
     subresources:
+      scale:
+        labelSelectorPath: .status.inference.selector
+        specReplicasPath: .spec.inference.replicas
+        statusReplicasPath: .status.inference.replicas
       status: {}
diff --git a/go.mod b/go.mod
index 12d06a0..22d29a1 100644
--- a/go.mod
+++ b/go.mod
@@ -9,6 +9,8 @@ require (
 	github.com/aws/karpenter-provider-aws v1.0.6
 	github.com/awslabs/operatorpkg v0.0.0-20240805231134-67d0acfb6306
 	github.com/distribution/reference v0.6.0
+	github.com/fluxcd/helm-controller/api v1.3.0
+	github.com/fluxcd/source-controller/api v1.6.0
 	github.com/go-logr/logr v1.4.3
 	github.com/onsi/ginkgo/v2 v2.23.4
 	github.com/onsi/gomega v1.37.0
@@ -18,6 +20,7 @@ require (
 	gopkg.in/yaml.v2 v2.4.0
 	gotest.tools v2.2.0+incompatible
 	k8s.io/api v0.33.2
+	k8s.io/apiextensions-apiserver v0.33.2
 	k8s.io/apimachinery v0.33.2
 	k8s.io/client-go v0.33.2
 	k8s.io/component-base v0.33.2
@@ -26,6 +29,7 @@ require (
 	k8s.io/utils v0.0.0-20250321185631-1f6e0b77f77e
 	knative.dev/pkg v0.0.0-20240910170930-fdbc0b5adde7
 	sigs.k8s.io/controller-runtime v0.21.0
+	sigs.k8s.io/gateway-api-inference-extension v0.5.1
 	sigs.k8s.io/karpenter v1.0.4
 )
 
@@ -38,19 +42,19 @@ require (
 	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/emicklei/go-restful/v3 v3.12.1 // indirect
+	github.com/emicklei/go-restful/v3 v3.12.2 // indirect
 	github.com/evanphx/json-patch/v5 v5.9.11 // indirect
-	github.com/fluxcd/helm-controller/api v1.3.0 // indirect
+	github.com/fluxcd/pkg/apis/acl v0.7.0 // indirect
 	github.com/fluxcd/pkg/apis/kustomize v1.10.0 // indirect
 	github.com/fluxcd/pkg/apis/meta v1.12.0 // indirect
-	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/fsnotify/fsnotify v1.8.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.8.0 // indirect
 	github.com/go-kit/log v0.2.1 // indirect
 	github.com/go-logfmt/logfmt v0.6.0 // indirect
 	github.com/go-logr/zapr v1.3.0 // indirect
-	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonpointer v0.21.1 // indirect
 	github.com/go-openapi/jsonreference v0.21.0 // indirect
-	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/go-openapi/swag v0.23.1 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/gobuffalo/flect v1.0.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
@@ -68,22 +72,23 @@ require (
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kelseyhightower/envconfig v1.4.0 // indirect
-	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mailru/easyjson v0.9.0 // indirect
 	github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
 	github.com/moby/spdystream v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
-	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/go-digest v1.0.1-0.20231025023718-d50d2fec9c98 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
-	github.com/prometheus/common v0.64.0 // indirect
+	github.com/prometheus/common v0.65.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/prometheus/statsd_exporter v0.24.0 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
+	github.com/shopspring/decimal v1.4.0 // indirect
 	github.com/spf13/cobra v1.9.1 // indirect
 	github.com/spf13/pflag v1.0.6 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
@@ -92,14 +97,15 @@ require (
 	go.uber.org/automaxprocs v1.6.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
+	go.yaml.in/yaml/v2 v2.4.2 // indirect
 	golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect
 	golang.org/x/net v0.40.0 // indirect
 	golang.org/x/oauth2 v0.30.0 // indirect
-	golang.org/x/sync v0.14.0 // indirect
+	golang.org/x/sync v0.15.0 // indirect
 	golang.org/x/sys v0.33.0 // indirect
 	golang.org/x/term v0.32.0 // indirect
 	golang.org/x/text v0.25.0 // indirect
-	golang.org/x/time v0.9.0 // indirect
+	golang.org/x/time v0.11.0 // indirect
 	golang.org/x/tools v0.32.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/api v0.183.0 // indirect
@@ -110,14 +116,13 @@ require (
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apiextensions-apiserver v0.33.2 // indirect
 	k8s.io/cloud-provider v0.33.2 // indirect
 	k8s.io/csi-translation-lib v0.33.2 // indirect
 	k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
 	sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
 	sigs.k8s.io/randfill v1.0.0 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.7.0 // indirect
-	sigs.k8s.io/yaml v1.4.0 // indirect
+	sigs.k8s.io/yaml v1.5.0 // indirect
 )
 
 replace (
diff --git a/go.sum b/go.sum
index ddc7614..c204c32 100644
--- a/go.sum
+++ b/go.sum
@@ -150,8 +150,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
 github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
-github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU=
-github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU=
+github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -164,14 +164,16 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/fluxcd/helm-controller/api v1.3.0 h1:PupXPuQbksmU0g2Lc6NjIYal2HJGL+6xohsf82eGVjo=
 github.com/fluxcd/helm-controller/api v1.3.0/go.mod h1:4b8PfdH0e/9Pfol2ogdMYbQ1nLjcVu9gAv27cQzIPK4=
+github.com/fluxcd/pkg/apis/acl v0.7.0 h1:dMhZJH+g6ZRPjs4zVOAN9vHBd1DcavFgcIFkg5ooOE0=
+github.com/fluxcd/pkg/apis/acl v0.7.0/go.mod h1:uv7pXXR/gydiX4MUwlQa7vS8JONEDztynnjTvY3JxKQ=
 github.com/fluxcd/pkg/apis/kustomize v1.10.0 h1:47EeSzkQvlQZdH92vHMe2lK2iR8aOSEJq95avw5idts=
 github.com/fluxcd/pkg/apis/kustomize v1.10.0/go.mod h1:UsqMV4sqNa1Yg0pmTsdkHRJr7bafBOENIJoAN+3ezaQ=
 github.com/fluxcd/pkg/apis/meta v1.12.0 h1:XW15TKZieC2b7MN8VS85stqZJOx+/b8jATQ/xTUhVYg=
 github.com/fluxcd/pkg/apis/meta v1.12.0/go.mod h1:+son1Va60x2eiDcTwd7lcctbI6C+K3gM7R+ULmEq1SI=
-github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
-github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
-github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
-github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
+github.com/fluxcd/source-controller/api v1.6.0 h1:IxfjUczJ2pzbXIef6iQ0RHEH4AYA9anJfTGK8dzwODM=
+github.com/fluxcd/source-controller/api v1.6.0/go.mod h1:ZJcAi0nemsnBxjVgmJl0WQzNvB0rMETxQMTdoFosmMw=
+github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
+github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/fxamacker/cbor/v2 v2.8.0 h1:fFtUGXUzXPHTIUdne5+zzMPTfffl3RD5qYnkY40vtxU=
 github.com/fxamacker/cbor/v2 v2.8.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
 github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
@@ -202,8 +204,8 @@ github.com/go-openapi/analysis v0.23.0 h1:aGday7OWupfMs+LbmLZG4k0MYXIANxcuBTYUC0
 github.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE8yA5v22ohupo=
 github.com/go-openapi/errors v0.22.0 h1:c4xY/OLxUBSTiepAg3j/MHuAv5mJhnf53LLMWFB+u/w=
 github.com/go-openapi/errors v0.22.0/go.mod h1:J3DmZScxCDufmIMsdOuDHxJbdOGC0xtUynjIx092vXE=
-github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
-github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
+github.com/go-openapi/jsonpointer v0.21.1 h1:whnzv/pNXtK2FbX/W9yJfRmE2gsmkfahjMKB0fZvcic=
+github.com/go-openapi/jsonpointer v0.21.1/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk=
 github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ=
 github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4=
 github.com/go-openapi/loads v0.22.0 h1:ECPGd4jX1U6NApCGG1We+uEozOAvXvJSF4nnwHZ8Aco=
@@ -214,8 +216,8 @@ github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9Z
 github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk=
 github.com/go-openapi/strfmt v0.23.0 h1:nlUS6BCqcnAk0pyhi9Y+kdDVZdZMHfEKQiS4HaMgO/c=
 github.com/go-openapi/strfmt v0.23.0/go.mod h1:NrtIpfKtWIygRkKVsxh7XQMDQW5HKQl6S5ik2elW+K4=
-github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
-github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
+github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZU=
+github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0=
 github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58=
 github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ=
 github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
@@ -357,8 +359,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
-github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
-github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
+github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4=
 github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE=
@@ -385,8 +387,8 @@ github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus
 github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8=
 github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y=
 github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0=
-github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
-github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/go-digest v1.0.1-0.20231025023718-d50d2fec9c98 h1:H55sU3giNgBkIvmAo0vI/AAFwVTwfWsf6MN3+9H6U8o=
+github.com/opencontainers/go-digest v1.0.1-0.20231025023718-d50d2fec9c98/go.mod h1:RqnyioA3pIEZMkSbOIcrw32YSgETfn/VrLuEikEdPNU=
 github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
@@ -425,8 +427,8 @@ github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9
 github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
 github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
 github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
-github.com/prometheus/common v0.64.0 h1:pdZeA+g617P7oGv1CzdTzyeShxAGrTBsolKNOLQPGO4=
-github.com/prometheus/common v0.64.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8=
+github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE=
+github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
@@ -447,8 +449,8 @@ github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWN
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/samber/lo v1.49.1 h1:4BIFyVfuQSEpluc7Fua+j1NolZHiEHEpaSEKdsH0tew=
 github.com/samber/lo v1.49.1/go.mod h1:dO6KHFzUKXgP8LDhU0oI8d2hekjXnGOu0DB8Jecxd6o=
-github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
-github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
+github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
@@ -515,6 +517,10 @@ go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN8
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
+go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
+go.yaml.in/yaml/v3 v3.0.3 h1:bXOww4E/J3f66rav3pX3m8w6jDE4knZjGOw8b5Y6iNE=
+go.yaml.in/yaml/v3 v3.0.3/go.mod h1:tBHosrYAkRZjRAOREWbDnBXUf08JOwYq++0QNwQiWzI=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -613,8 +619,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
-golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
+golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -674,8 +680,8 @@ golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY=
-golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
+golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -718,8 +724,7 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU=
-golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ=
+golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU=
 golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -871,8 +876,6 @@ k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
 k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
 k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4=
 k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8=
-k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJJI8IUa1AmH/qa0=
-k8s.io/utils v0.0.0-20241210054802-24370beab758/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 k8s.io/utils v0.0.0-20250321185631-1f6e0b77f77e h1:KqK5c/ghOm8xkHYhlodbp6i6+r+ChV2vuAuVRdFbLro=
 k8s.io/utils v0.0.0-20250321185631-1f6e0b77f77e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 knative.dev/pkg v0.0.0-20240910170930-fdbc0b5adde7 h1:ClGJ7Q7iQC9qxov1lNys7xC0MqDGaw/ysGEuyqdIwAM=
@@ -888,8 +891,8 @@ sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.1 h1:Lp0nALZmvMJ
 sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.1/go.mod h1:pPkJPx/eMVWP3R+LhPoOYGoY7lywcMJev5L2uSfH+Jo=
 sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8=
 sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM=
-sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8=
-sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo=
+sigs.k8s.io/gateway-api-inference-extension v0.5.1 h1:OMpt4gKlPWkD+h5kHcZZVh4926kix2DSBPI7X5ntuCA=
+sigs.k8s.io/gateway-api-inference-extension v0.5.1/go.mod h1:lki0jx1qysZSZT4Ai2BxuAcpx6G8g5oBgOGuuJzjy/k=
 sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE=
 sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
 sigs.k8s.io/karpenter v1.0.4 h1:tE4gbYjXZ2uny35QmrKFSuMhxkIfGCqCulXhwyN2D3c=
@@ -899,5 +902,6 @@ sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
 sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
 sigs.k8s.io/structured-merge-diff/v4 v4.7.0 h1:qPeWmscJcXP0snki5IYF79Z8xrl8ETFxgMd7wez1XkI=
 sigs.k8s.io/structured-merge-diff/v4 v4.7.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps=
-sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
 sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
+sigs.k8s.io/yaml v1.5.0 h1:M10b2U7aEUY6hRtU870n2VTPgR5RZiL/I6Lcc2F4NUQ=
+sigs.k8s.io/yaml v1.5.0/go.mod h1:wZs27Rbxoai4C0f8/9urLZtZtF3avA3gKvGyPdDqTO4=
diff --git a/hack/local-csi-driver-charts-sync/local-csi-driver-charts-sync.sh b/hack/local-csi-driver-charts-sync/local-csi-driver-charts-sync.sh
index 9e8b37d..8287a61 100755
--- a/hack/local-csi-driver-charts-sync/local-csi-driver-charts-sync.sh
+++ b/hack/local-csi-driver-charts-sync/local-csi-driver-charts-sync.sh
@@ -12,7 +12,7 @@
 SCRIPT_DIR="$(realpath "$(dirname "$0")")"
 
 REPO=https://github.com/Azure/local-csi-driver.git
-REVISION="a9a2fd1af2d0fbfe4262f66105dc2569b6394a9c"
+REVISION="38cdc94deda8548076d670ea1beba5d69b349a81"
 
 TEMP_DIR=$(mktemp -d)
 REPO_DIR="$TEMP_DIR/local-csi-driver"
@@ -36,7 +36,7 @@ echo "Checking out revision: $REVISION"
 
 helm template --release-name local-csi-driver \
   --set image.driver.tag="0.0.1-latest" \
-  --set webhook.ephemeral.enabled=false \
+  --set webhook.enforceEphemeral.enabled=false \
   --set webhook.hyperconverged.enabled=false \
   --set observability.metrics.enabled=false \
   "$REPO_DIR/charts/latest/" > "$REPO_DIR/local-csi-driver-charts.yaml"
diff --git a/hack/local-csi-driver-charts-sync/local-csi-driver-charts.diff b/hack/local-csi-driver-charts-sync/local-csi-driver-charts.diff
index 906ce35..be1e8d7 100644
--- a/hack/local-csi-driver-charts-sync/local-csi-driver-charts.diff
+++ b/hack/local-csi-driver-charts-sync/local-csi-driver-charts.diff
@@ -1,5 +1,5 @@
 diff --git a/./local-csi-driver-charts.yaml b/./charts/kaito/workspace/templates/local-csi-driver-ds.yaml
-index 93d0b571..cd8173e2 100644
+index e7365b6..82e259f 100644
 --- a/./local-csi-driver-charts.yaml
 +++ b/./charts/kaito/workspace/templates/local-csi-driver-ds.yaml
 @@ -4,13 +4,9 @@ apiVersion: v1
@@ -80,33 +80,21 @@ index 93d0b571..cd8173e2 100644
        annotations:
          kubectl.kubernetes.io/default-container: driver
      spec:
-@@ -221,7 +201,7 @@ spec:
+@@ -222,7 +202,7 @@ spec:
            valueFrom:
              fieldRef:
                fieldPath: spec.nodeName
 -        image: "localcsidriver.azurecr.io/acstor/local-csi-driver:0.0.1-latest"
-+        image: "mcr.microsoft.com/acstor/local-csi-driver:v0.2.3"
++        image: "mcr.microsoft.com/acstor/local-csi-driver:v0.2.4"
          imagePullPolicy: IfNotPresent
          livenessProbe:
            httpGet:
-@@ -262,9 +242,6 @@ spec:
-         volumeMounts:
-         - mountPath: /dev
-           name: device
--        - mountPath: /etc/kubernetes/
--          name: k8s-cfg
--          readOnly: true
-         - mountPath: /csi
-           name: csi-socket-dir
-         - mountPath: /var/lib/kubelet/
-@@ -404,10 +381,6 @@ spec:
-           path: /dev
-           type: Directory
-         name: device
--      - hostPath:
--          path: /etc/kubernetes/
--          type: DirectoryOrCreate
--        name: k8s-cfg
-       - hostPath:
-           path: /var/lib/kubelet/plugins/localdisk.csi.acstor.io/
-           type: DirectoryOrCreate
+@@ -287,8 +267,6 @@ spec:
+         - --capacity-poll-interval=15s
+         - --v=2
+         env:
+-        # POD_NAME and POD_NAMESPACE are used for capacity tracking support
+-        # More info: https://github.com/kubernetes-csi/external-provisioner?tab=readme-ov-file#capacity-support
+         - name: NAMESPACE
+           valueFrom:
+             fieldRef:
diff --git a/pkg/featuregates/featuregates.go b/pkg/featuregates/featuregates.go
index 9885910..1ae194c 100644
--- a/pkg/featuregates/featuregates.go
+++ b/pkg/featuregates/featuregates.go
@@ -25,9 +25,10 @@ import (
 var (
 	// FeatureGates is a map that holds the feature gate names and their default values for Kaito.
 	FeatureGates = map[string]bool{
-		consts.FeatureFlagVLLM:                        true,
-		consts.FeatureFlagEnsureNodeClass:             false,
-		consts.FeatureFlagDisableNodeAutoProvisioning: false,
+		consts.FeatureFlagVLLM:                         true,
+		consts.FeatureFlagEnsureNodeClass:              false,
+		consts.FeatureFlagDisableNodeAutoProvisioning:  false,
+		consts.FeatureFlagGatewayAPIInferenceExtension: false,
 		//	Add more feature gates here
 	}
 )
diff --git a/pkg/utils/common.go b/pkg/utils/common.go
index 98f8803..dc19494 100644
--- a/pkg/utils/common.go
+++ b/pkg/utils/common.go
@@ -25,7 +25,9 @@ import (
 	awsv1beta1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1"
 	"gopkg.in/yaml.v2"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/equality"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
@@ -346,3 +348,38 @@ func DedupVolumeMounts(mounts []corev1.VolumeMount) []corev1.VolumeMount {
 
 	return result
 }
+
+// InferencePoolName returns the name of the inference pool for the given workspace.
+func InferencePoolName(workspaceName string) string {
+	return fmt.Sprintf("%s-inferencepool", workspaceName)
+}
+
+// ClientObjectSpecEqual compares the spec field of two client.Objects for equality.
+// For example:
+//
+//	a:   {"apiVersion": "apps/v1", "kind": "Deployment", "spec": {"replicas": 2}}
+//	b:   {"apiVersion": "apps/v1", "kind": "Deployment", "spec": {"replicas": 2}}
+//	result: true
+//
+//	c:   {"apiVersion": "apps/v1", "kind": "Deployment", "spec": {"replicas": 3}}
+//	d:   {"apiVersion": "apps/v1", "kind": "Deployment", "spec": {"replicas": 2}}
+//	result: false
+func ClientObjectSpecEqual(a, b client.Object) (bool, error) {
+	aUnstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(a)
+	if err != nil {
+		return false, err
+	}
+	bUnstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(b)
+	if err != nil {
+		return false, err
+	}
+	aSpec, aOK, err := unstructured.NestedMap(aUnstructured, "spec")
+	if err != nil {
+		return false, err
+	}
+	bSpec, bOK, err := unstructured.NestedMap(bUnstructured, "spec")
+	if err != nil {
+		return false, err
+	}
+	return aOK && bOK && equality.Semantic.DeepEqual(aSpec, bSpec), nil
+}
diff --git a/pkg/utils/consts/consts.go b/pkg/utils/consts/consts.go
index 80544d7..5ad8000 100644
--- a/pkg/utils/consts/consts.go
+++ b/pkg/utils/consts/consts.go
@@ -31,9 +31,10 @@ const (
 	NvidiaGPU                     = "nvidia.com/gpu"
 
 	// Feature flags
-	FeatureFlagVLLM                        = "vLLM"
-	FeatureFlagEnsureNodeClass             = "ensureNodeClass"
-	FeatureFlagDisableNodeAutoProvisioning = "disableNodeAutoProvisioning"
+	FeatureFlagVLLM                         = "vLLM"
+	FeatureFlagEnsureNodeClass              = "ensureNodeClass"
+	FeatureFlagDisableNodeAutoProvisioning  = "disableNodeAutoProvisioning"
+	FeatureFlagGatewayAPIInferenceExtension = "gatewayAPIInferenceExtension"
 
 	// Nodeclaim related consts
 	KaitoNodePoolName             = "kaito"
@@ -50,6 +51,22 @@ const (
 	GpuSkuPrefix = "Standard_N"
 
 	NodePluginInstallTimeout = 60 * time.Second
+
+	// PortInferenceServer is the default port for the inference server.
+	PortInferenceServer = int32(5000)
+
+	// InferencePoolChartURL is the OCI registry URL for the Gateway API Inference Extension inferencepool chart.
+	InferencePoolChartURL = "oci://registry.k8s.io/gateway-api-inference-extension/charts/inferencepool"
+
+	// InferencePoolChartVersion is the tag/version of the inferencepool chart to deploy.
+	// MUST KEEP IN SYNC with the version in go.mod.
+	InferencePoolChartVersion = "v0.5.1"
+
+	// GatewayAPIInferenceExtensionImageRepository is the image repository for the Gateway API Inference Extension components.
+	GatewayAPIInferenceExtensionImageRepository = "mcr.microsoft.com/oss/v2/gateway-api-inference-extension"
+
+	// ConditionReady is the condition type for a ready condition.
+	ConditionReady = "Ready"
 )
 
 var (
diff --git a/pkg/utils/crd.go b/pkg/utils/crd.go
new file mode 100644
index 0000000..56433fa
--- /dev/null
+++ b/pkg/utils/crd.go
@@ -0,0 +1,42 @@
+// Copyright (c) KAITO authors.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// EnsureKindExists checks if a specific GroupVersionKind (GVK) exists in the cluster.
+func EnsureKindExists(restConfig *rest.Config, gvk schema.GroupVersionKind) (bool, error) {
+	discoveryClient, err := discovery.NewDiscoveryClientForConfig(restConfig)
+	if err != nil {
+		return false, err
+	}
+
+	resources, err := discoveryClient.ServerResourcesForGroupVersion(gvk.GroupVersion().String())
+	if client.IgnoreNotFound(err) != nil {
+		return false, err
+	}
+
+	for _, r := range resources.APIResources {
+		if r.Kind == gvk.Kind {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
diff --git a/pkg/utils/resources/resources.go b/pkg/utils/resources/resources.go
index fee9eaa..6dfb5a9 100644
--- a/pkg/utils/resources/resources.go
+++ b/pkg/utils/resources/resources.go
@@ -18,15 +18,19 @@ import (
 	"fmt"
 	"time"
 
+	helmv2 "github.com/fluxcd/helm-controller/api/v2"
+	sourcev1 "github.com/fluxcd/source-controller/api/v1"
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/util/retry"
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/kaito-project/kaito/pkg/utils"
+	"github.com/kaito-project/kaito/pkg/utils/consts"
 )
 
 func CreateResource(ctx context.Context, resource client.Object, kubeClient client.Client) error {
@@ -39,6 +43,10 @@ func CreateResource(ctx context.Context, resource client.Object, kubeClient clie
 		klog.InfoS("CreateService", "service", klog.KObj(r))
 	case *corev1.ConfigMap:
 		klog.InfoS("CreateConfigMap", "configmap", klog.KObj(r))
+	case *helmv2.HelmRelease:
+		klog.InfoS("CreateHelmRelease", "helmrelease", klog.KObj(r))
+	case *sourcev1.OCIRepository:
+		klog.InfoS("CreateOCIRepository", "ocirepository", klog.KObj(r))
 	}
 
 	// Create the resource.
@@ -110,6 +118,20 @@ func CheckResourceStatus(obj client.Object, kubeClient client.Client, timeoutDur
 					klog.InfoS("job status is active/succeeded", "name", k8sResource.Name)
 					return nil
 				}
+			case *helmv2.HelmRelease:
+				for _, condition := range k8sResource.Status.Conditions {
+					if condition.Type == consts.ConditionReady && condition.Status == metav1.ConditionTrue {
+						klog.InfoS("helmrelease status is ready", "helmrelease", k8sResource.Name)
+						return nil
+					}
+				}
+			case *sourcev1.OCIRepository:
+				for _, condition := range k8sResource.Status.Conditions {
+					if condition.Type == consts.ConditionReady && condition.Status == metav1.ConditionTrue {
+						klog.InfoS("ocirepository status is ready", "ocirepository", k8sResource.Name)
+						return nil
+					}
+				}
 			default:
 				return fmt.Errorf("unsupported resource type")
 			}
diff --git a/pkg/workspace/controllers/workspace_controller.go b/pkg/workspace/controllers/workspace_controller.go
index 1d573a9..233f4d0 100644
--- a/pkg/workspace/controllers/workspace_controller.go
+++ b/pkg/workspace/controllers/workspace_controller.go
@@ -24,6 +24,8 @@ import (
 	"strconv"
 	"time"
 
+	helmv2 "github.com/fluxcd/helm-controller/api/v2"
+	sourcev1 "github.com/fluxcd/source-controller/api/v1"
 	"github.com/go-logr/logr"
 	"github.com/samber/lo"
 	appsv1 "k8s.io/api/apps/v1"
@@ -32,6 +34,7 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/tools/record"
@@ -44,10 +47,12 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	gaiev1alpha2 "sigs.k8s.io/gateway-api-inference-extension/api/v1alpha2"
 	karpenterv1 "sigs.k8s.io/karpenter/pkg/apis/v1"
 
 	kaitov1beta1 "github.com/kaito-project/kaito/api/v1beta1"
 	"github.com/kaito-project/kaito/pkg/featuregates"
+	pkgmodel "github.com/kaito-project/kaito/pkg/model"
 	"github.com/kaito-project/kaito/pkg/utils"
 	"github.com/kaito-project/kaito/pkg/utils/consts"
 	"github.com/kaito-project/kaito/pkg/utils/nodeclaim"
@@ -210,6 +215,13 @@ func (c *WorkspaceReconciler) addOrUpdateWorkspace(ctx context.Context, wObj *ka
 			}
 			return reconcile.Result{}, err
 		}
+		if err = c.ensureGatewayAPIInferenceExtension(ctx, wObj); err != nil {
+			if updateErr := c.updateStatusConditionIfNotMatch(ctx, wObj, kaitov1beta1.WorkspaceConditionTypeSucceeded, metav1.ConditionFalse,
+				"workspaceFailed", err.Error()); updateErr != nil {
+				klog.ErrorS(updateErr, "failed to update workspace status", "workspace", klog.KObj(wObj))
+				return reconcile.Result{}, updateErr
+			}
+		}
 
 		if err = c.updateStatusConditionIfNotMatch(ctx, wObj, kaitov1beta1.WorkspaceConditionTypeSucceeded, metav1.ConditionTrue,
 			"workspaceSucceeded", "workspace succeeds"); err != nil {
@@ -356,13 +368,16 @@ func (c *WorkspaceReconciler) applyWorkspaceResource(ctx context.Context, wObj *
 		return err
 	}
 
+	//nolint:staticcheck //SA1019: deprecate Resource.Count field
 	selectedNodes := utils.SelectNodes(validNodes, wObj.Resource.PreferredNodes, wObj.Status.WorkerNodes, lo.FromPtr(wObj.Resource.Count))
 
+	//nolint:staticcheck //SA1019: deprecate Resource.Count field
 	newNodesCount := lo.FromPtr(wObj.Resource.Count) - len(selectedNodes)
 
 	if newNodesCount > 0 {
 		// Check if node auto-provisioning is disabled
 		if featuregates.FeatureGates[consts.FeatureFlagDisableNodeAutoProvisioning] {
+			//nolint:staticcheck //SA1019: deprecate Resource.Count field
 			return fmt.Errorf("node auto-provisioning is disabled but insufficient nodes available: need %d nodes, have %d selected nodes", lo.FromPtr(wObj.Resource.Count), len(selectedNodes))
 		}
 
@@ -467,8 +482,9 @@ func (c *WorkspaceReconciler) getAllQualifiedNodes(ctx context.Context, wObj *ka
 				}
 			}
 		}
-
+		//nolint:staticcheck //SA1019: deprecate Resource.Count field
 		if len(qualifiedNodes) < *wObj.Resource.Count {
+			//nolint:staticcheck //SA1019: deprecate Resource.Count field
 			return nil, fmt.Errorf("when node auto-provisioning is disabled, at least %d preferred nodes must match the label selector and be ready and not deleting, only have %d", *wObj.Resource.Count, len(qualifiedNodes))
 		}
 	} else {
@@ -863,6 +879,93 @@ func updateWorkspaceWithRetry(ctx context.Context, c client.Client, wObj *kaitov
 	})
 }
 
+// ensureGatewayAPIInferenceExtension reconciles Gateway API Inference Extension components for a Workspace.
+//
+// How it works:
+// 1) Dry-runs preset inference generation to determine if the target workload is a StatefulSet.
+// 2) Renders a Flux OCIRepository and a HelmRelease for the InferencePool chart.
+// 3) Creates the resources if absent; updates them if the desired spec differs.
+// 4) Waits for resources to become ready using the model's inference readiness timeout.
+// 5) Aggregates and returns any errors.
+//
+// Idempotent and safe to call on every reconcile; no-op if preconditions are not met.
+func (c *WorkspaceReconciler) ensureGatewayAPIInferenceExtension(ctx context.Context, wObj *kaitov1beta1.Workspace) error {
+	runtimeName := kaitov1beta1.GetWorkspaceRuntimeName(wObj)
+	isPresetInference := wObj.Inference != nil && wObj.Inference.Preset != nil
+
+	// Gateway API Inference Extension is specifically designed to work with vLLM and preset-based inference workloads.
+	if !featuregates.FeatureGates[consts.FeatureFlagGatewayAPIInferenceExtension] ||
+		runtimeName != pkgmodel.RuntimeNameVLLM || !isPresetInference {
+		return nil
+	}
+
+	model := plugin.KaitoModelRegister.MustGet(string(wObj.Inference.Preset.Name))
+
+	// Dry-run the inference workload generation to determine if it will be a StatefulSet or not.
+	workloadObj, _ := inference.GeneratePresetInference(ctx, wObj, "", model, c.Client)
+	_, isStatefulSet := workloadObj.(*appsv1.StatefulSet)
+
+	ociRepository := manifests.GenerateInferencePoolOCIRepository(wObj)
+	helmRelease, err := manifests.GenerateInferencePoolHelmRelease(wObj, isStatefulSet)
+	if err != nil {
+		return err
+	}
+
+	// Create or update OCIRepository
+	existingOCIRepo := &sourcev1.OCIRepository{}
+	err = resources.GetResource(ctx, ociRepository.Name, ociRepository.Namespace, c.Client, existingOCIRepo)
+	if err != nil {
+		if !apierrors.IsNotFound(err) {
+			return err
+		}
+		if err := resources.CreateResource(ctx, ociRepository, c.Client); client.IgnoreAlreadyExists(err) != nil {
+			return err
+		}
+	} else {
+		equal, err := utils.ClientObjectSpecEqual(ociRepository, existingOCIRepo)
+		if err != nil {
+			return err
+		}
+		if !equal {
+			existingOCIRepo.Spec = ociRepository.Spec
+			if err := c.Update(ctx, existingOCIRepo); err != nil {
+				return err
+			}
+		}
+	}
+
+	// Check if HelmRelease exists
+	existingHelmRelease := &helmv2.HelmRelease{}
+	err = resources.GetResource(ctx, helmRelease.Name, helmRelease.Namespace, c.Client, existingHelmRelease)
+	if err != nil {
+		if !apierrors.IsNotFound(err) {
+			return err
+		}
+		if err := resources.CreateResource(ctx, helmRelease, c.Client); client.IgnoreAlreadyExists(err) != nil {
+			return err
+		}
+	} else {
+		equal, err := utils.ClientObjectSpecEqual(helmRelease, existingHelmRelease)
+		if err != nil {
+			return err
+		}
+		if !equal {
+			existingHelmRelease.Spec = helmRelease.Spec
+			if err := c.Update(ctx, existingHelmRelease); err != nil {
+				return err
+			}
+		}
+	}
+
+	for _, resource := range []client.Object{ociRepository, helmRelease} {
+		if err := resources.CheckResourceStatus(resource, c.Client, 5*time.Minute); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // SetupWithManager sets up the controller with the Manager.
 func (c *WorkspaceReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	c.Recorder = mgr.GetEventRecorderFor("Workspace")
@@ -884,6 +987,33 @@ func (c *WorkspaceReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		).
 		WithOptions(controller.Options{MaxConcurrentReconciles: 5})
 
+	if featuregates.FeatureGates[consts.FeatureFlagGatewayAPIInferenceExtension] {
+		// Verify that all prerequisite CRDs exist before configuring watches that depend on them.
+		// - FluxCD HelmRelease / OCIRepository: required for installing and reconciling the InferencePool Helm chart.
+		// - Gateway API Inference Extension InferencePool / InferenceModel: required runtime CRDs that the Workspace
+		//   controller indirectly relies on (Helm chart renders resources referencing them).
+		// Failing fast here provides a clear, actionable error instead of deferred reconcile failures later.
+		for _, gvk := range []schema.GroupVersionKind{
+			helmv2.GroupVersion.WithKind(helmv2.HelmReleaseKind),
+			sourcev1.GroupVersion.WithKind(sourcev1.OCIRepositoryKind),
+			gaiev1alpha2.SchemeGroupVersion.WithKind("InferencePool"),
+			gaiev1alpha2.SchemeGroupVersion.WithKind("InferenceModel"),
+		} {
+			found, err := utils.EnsureKindExists(mgr.GetConfig(), gvk)
+			if err != nil {
+				return fmt.Errorf("failed to ensure kind %s exists: %w", gvk.Kind, err)
+			}
+			if !found {
+				return fmt.Errorf("%s not found in the cluster, please ensure the Gateway API Inference Extension is installed", gvk.String())
+			}
+		}
+
+		// We don't need to own InferencePool and InferenceModel because they are managed by Flux's HelmRelease
+		builder = builder.
+			Owns(&helmv2.HelmRelease{}).
+			Owns(&sourcev1.OCIRepository{})
+	}
+
 	go monitorWorkspaces(context.Background(), c.Client)
 
 	return builder.Complete(c)
diff --git a/pkg/workspace/inference/preset_inferences.go b/pkg/workspace/inference/preset_inferences.go
index 1355144..1925645 100644
--- a/pkg/workspace/inference/preset_inferences.go
+++ b/pkg/workspace/inference/preset_inferences.go
@@ -41,20 +41,17 @@ import (
 
 const (
 	ProbePath = "/health"
-
-	// PortInferenceServer is the default port for the inference server.
-	PortInferenceServer = 5000
 )
 
 var (
 	containerPorts = []corev1.ContainerPort{{
-		ContainerPort: int32(PortInferenceServer),
+		ContainerPort: int32(consts.PortInferenceServer),
 	}}
 
 	defaultLivenessProbe = &corev1.Probe{
 		ProbeHandler: corev1.ProbeHandler{
 			HTTPGet: &corev1.HTTPGetAction{
-				Port: intstr.FromInt(PortInferenceServer),
+				Port: intstr.FromInt32(consts.PortInferenceServer),
 				Path: ProbePath,
 			},
 		},
@@ -65,7 +62,7 @@ var (
 	defaultReadinessProbe = &corev1.Probe{
 		ProbeHandler: corev1.ProbeHandler{
 			HTTPGet: &corev1.HTTPGetAction{
-				Port: intstr.FromInt(PortInferenceServer),
+				Port: intstr.FromInt32(consts.PortInferenceServer),
 				Path: ProbePath,
 			},
 		},
@@ -141,6 +138,8 @@ func GeneratePresetInference(ctx context.Context, workspaceObj *v1beta1.Workspac
 
 	gpuConfig := getGPUConfig(gctx)
 	// initially respect the user setting by deploying the model on the same number of nodes as the user requested
+
+	//nolint:staticcheck //SA1019: deprecate Resource.Count field
 	numNodes := *workspaceObj.Resource.Count
 	// if gpu mem is known, we can setup the distributed correctly
 	if gpuConfig.GPUMemGB > 0 && gpuConfig.GPUCount > 0 {
@@ -265,7 +264,7 @@ func getDistributedInferenceProbe(probeType probeType, wObj *v1beta1.Workspace,
 	case probeTypeLiveness:
 		args["ray-port"] = strconv.Itoa(pkgmodel.PortRayCluster)
 	case probeTypeReadiness:
-		args["vllm-port"] = strconv.Itoa(PortInferenceServer)
+		args["vllm-port"] = strconv.FormatInt(int64(consts.PortInferenceServer), 10)
 	}
 
 	// for distributed inference, we cannot use the default http probe since only the leader pod
diff --git a/pkg/workspace/manifests/manifests.go b/pkg/workspace/manifests/manifests.go
index 813bd7a..dc082e2 100644
--- a/pkg/workspace/manifests/manifests.go
+++ b/pkg/workspace/manifests/manifests.go
@@ -15,21 +15,25 @@ package manifests
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"path"
 
+	helmv2 "github.com/fluxcd/helm-controller/api/v2"
+	sourcev1 "github.com/fluxcd/source-controller/api/v1"
 	"github.com/samber/lo"
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/ptr"
 
-	"github.com/kaito-project/kaito/api/v1beta1"
 	kaitov1beta1 "github.com/kaito-project/kaito/api/v1beta1"
 	pkgmodel "github.com/kaito-project/kaito/pkg/model"
 	"github.com/kaito-project/kaito/pkg/utils"
+	"github.com/kaito-project/kaito/pkg/utils/consts"
 	"github.com/kaito-project/kaito/pkg/utils/generator"
 	"github.com/kaito-project/kaito/pkg/workspace/image"
 )
@@ -41,11 +45,11 @@ func GenerateHeadlessServiceManifest(workspaceObj *kaitov1beta1.Workspace) *core
 	}
 
 	return &corev1.Service{
-		ObjectMeta: v1.ObjectMeta{
+		ObjectMeta: metav1.ObjectMeta{
 			Name:      serviceName,
 			Namespace: workspaceObj.Namespace,
-			OwnerReferences: []v1.OwnerReference{
-				*v1.NewControllerRef(workspaceObj, kaitov1beta1.GroupVersion.WithKind("Workspace")),
+			OwnerReferences: []metav1.OwnerReference{
+				*metav1.NewControllerRef(workspaceObj, kaitov1beta1.GroupVersion.WithKind("Workspace")),
 			},
 		},
 		Spec: corev1.ServiceSpec{
@@ -68,11 +72,11 @@ func GenerateServiceManifest(workspaceObj *kaitov1beta1.Workspace, serviceType c
 	}
 
 	return &corev1.Service{
-		ObjectMeta: v1.ObjectMeta{
+		ObjectMeta: metav1.ObjectMeta{
 			Name:      workspaceObj.Name,
 			Namespace: workspaceObj.Namespace,
-			OwnerReferences: []v1.OwnerReference{
-				*v1.NewControllerRef(workspaceObj, kaitov1beta1.GroupVersion.WithKind("Workspace")),
+			OwnerReferences: []metav1.OwnerReference{
+				*metav1.NewControllerRef(workspaceObj, kaitov1beta1.GroupVersion.WithKind("Workspace")),
 			},
 		},
 		Spec: corev1.ServiceSpec{
@@ -83,7 +87,7 @@ func GenerateServiceManifest(workspaceObj *kaitov1beta1.Workspace, serviceType c
 					Name:       "http",
 					Protocol:   corev1.ProtocolTCP,
 					Port:       80,
-					TargetPort: intstr.FromInt32(5000),
+					TargetPort: intstr.FromInt32(consts.PortInferenceServer),
 				},
 				{
 					Name:       "ray",
@@ -111,18 +115,18 @@ func GenerateStatefulSetManifest(revisionNum string, replicas int) func(*generat
 		selector := map[string]string{
 			kaitov1beta1.LabelWorkspaceName: ctx.Workspace.Name,
 		}
-		labelselector := &v1.LabelSelector{
+		labelselector := &metav1.LabelSelector{
 			MatchLabels: selector,
 		}
 
-		ss.ObjectMeta = v1.ObjectMeta{
+		ss.ObjectMeta = metav1.ObjectMeta{
 			Name:      ctx.Workspace.Name,
 			Namespace: ctx.Workspace.Namespace,
 			Annotations: map[string]string{
 				kaitov1beta1.WorkspaceRevisionAnnotation: revisionNum,
 			},
-			OwnerReferences: []v1.OwnerReference{
-				*v1.NewControllerRef(ctx.Workspace, kaitov1beta1.GroupVersion.WithKind("Workspace")),
+			OwnerReferences: []metav1.OwnerReference{
+				*metav1.NewControllerRef(ctx.Workspace, kaitov1beta1.GroupVersion.WithKind("Workspace")),
 			},
 		}
 		ss.Spec = appsv1.StatefulSetSpec{
@@ -134,7 +138,7 @@ func GenerateStatefulSetManifest(revisionNum string, replicas int) func(*generat
 			},
 			Selector: labelselector,
 			Template: corev1.PodTemplateSpec{
-				ObjectMeta: v1.ObjectMeta{
+				ObjectMeta: metav1.ObjectMeta{
 					Labels: selector,
 				},
 			},
@@ -165,21 +169,21 @@ func GenerateTuningJobManifest(revisionNum string) func(*generator.WorkspaceGene
 			kaitov1beta1.LabelWorkspaceName: ctx.Workspace.Name,
 		}
 
-		j.ObjectMeta = v1.ObjectMeta{
+		j.ObjectMeta = metav1.ObjectMeta{
 			Name:      ctx.Workspace.Name,
 			Namespace: ctx.Workspace.Namespace,
 			Labels:    labels,
 			Annotations: map[string]string{
 				kaitov1beta1.WorkspaceRevisionAnnotation: revisionNum,
 			},
-			OwnerReferences: []v1.OwnerReference{
-				*v1.NewControllerRef(ctx.Workspace, kaitov1beta1.GroupVersion.WithKind("Workspace")),
+			OwnerReferences: []metav1.OwnerReference{
+				*metav1.NewControllerRef(ctx.Workspace, kaitov1beta1.GroupVersion.WithKind("Workspace")),
 			},
 		}
 
 		j.Spec = batchv1.JobSpec{
 			Template: corev1.PodTemplateSpec{
-				ObjectMeta: v1.ObjectMeta{
+				ObjectMeta: metav1.ObjectMeta{
 					Labels: labels,
 				},
 			},
@@ -203,15 +207,15 @@ func GenerateDeploymentManifest(revisionNum string, replicas int) func(*generato
 		selector := map[string]string{
 			kaitov1beta1.LabelWorkspaceName: ctx.Workspace.Name,
 		}
-		labelselector := &v1.LabelSelector{
+		labelselector := &metav1.LabelSelector{
 			MatchLabels: selector,
 		}
 
-		d.ObjectMeta = v1.ObjectMeta{
+		d.ObjectMeta = metav1.ObjectMeta{
 			Name:      ctx.Workspace.Name,
 			Namespace: ctx.Workspace.Namespace,
-			OwnerReferences: []v1.OwnerReference{
-				*v1.NewControllerRef(ctx.Workspace, kaitov1beta1.GroupVersion.WithKind("Workspace")),
+			OwnerReferences: []metav1.OwnerReference{
+				*metav1.NewControllerRef(ctx.Workspace, kaitov1beta1.GroupVersion.WithKind("Workspace")),
 			},
 			Annotations: map[string]string{
 				kaitov1beta1.WorkspaceRevisionAnnotation: revisionNum,
@@ -234,7 +238,7 @@ func GenerateDeploymentManifest(revisionNum string, replicas int) func(*generato
 			},
 			Selector: labelselector,
 			Template: corev1.PodTemplateSpec{
-				ObjectMeta: v1.ObjectMeta{
+				ObjectMeta: metav1.ObjectMeta{
 					Labels: selector,
 				},
 			},
@@ -302,7 +306,7 @@ func GenerateDeploymentManifestWithPodTemplate(workspaceObj *kaitov1beta1.Worksp
 		templateCopy.ObjectMeta.Labels = make(map[string]string)
 	}
 	templateCopy.ObjectMeta.Labels[kaitov1beta1.LabelWorkspaceName] = workspaceObj.Name
-	labelselector := &v1.LabelSelector{
+	labelselector := &metav1.LabelSelector{
 		MatchLabels: map[string]string{
 			kaitov1beta1.LabelWorkspaceName: workspaceObj.Name,
 		},
@@ -328,14 +332,15 @@ func GenerateDeploymentManifestWithPodTemplate(workspaceObj *kaitov1beta1.Worksp
 	}
 
 	return &appsv1.Deployment{
-		ObjectMeta: v1.ObjectMeta{
+		ObjectMeta: metav1.ObjectMeta{
 			Name:      workspaceObj.Name,
 			Namespace: workspaceObj.Namespace,
-			OwnerReferences: []v1.OwnerReference{
-				*v1.NewControllerRef(workspaceObj, kaitov1beta1.GroupVersion.WithKind("Workspace")),
+			OwnerReferences: []metav1.OwnerReference{
+				*metav1.NewControllerRef(workspaceObj, kaitov1beta1.GroupVersion.WithKind("Workspace")),
 			},
 		},
 		Spec: appsv1.DeploymentSpec{
+			//nolint:staticcheck //SA1019: deprecate Resource.Count field
 			Replicas: lo.ToPtr(int32(*workspaceObj.Resource.Count)),
 			Selector: labelselector,
 			Template: *templateCopy,
@@ -348,7 +353,7 @@ func GetModelImageName(presetObj *pkgmodel.PresetParam) string {
 }
 
 // GenerateModelPullerContainer creates an init container that pulls model images using ORAS
-func GenerateModelPullerContainer(ctx context.Context, workspaceObj *v1beta1.Workspace, presetObj *pkgmodel.PresetParam) []corev1.Container {
+func GenerateModelPullerContainer(ctx context.Context, workspaceObj *kaitov1beta1.Workspace, presetObj *pkgmodel.PresetParam) []corev1.Container {
 	if presetObj.DownloadAtRuntime {
 		// If the preset is set to download at runtime, we don't need to pull the model weights.
 		return nil
@@ -374,3 +379,80 @@ func GenerateModelPullerContainer(ctx context.Context, workspaceObj *v1beta1.Wor
 
 	return []corev1.Container{puller}
 }
+
+// GenerateInferencePoolOCIRepository generates a Flux OCIRepository for the inference pool.
+func GenerateInferencePoolOCIRepository(workspaceObj *kaitov1beta1.Workspace) *sourcev1.OCIRepository {
+	return &sourcev1.OCIRepository{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      utils.InferencePoolName(workspaceObj.Name),
+			Namespace: workspaceObj.Namespace,
+			OwnerReferences: []metav1.OwnerReference{
+				*metav1.NewControllerRef(workspaceObj, kaitov1beta1.GroupVersion.WithKind("Workspace")),
+			},
+		},
+		Spec: sourcev1.OCIRepositorySpec{
+			// Chart source for Gateway API Inference Extension inference pool;
+			// keep in sync with consts.InferencePoolChartVersion when upgrading.
+			URL: consts.InferencePoolChartURL,
+			Reference: &sourcev1.OCIRepositoryRef{
+				Tag: consts.InferencePoolChartVersion,
+			},
+		},
+	}
+}
+
+// GenerateInferencePoolHelmRelease generates a Flux HelmRelease for the inference pool.
+func GenerateInferencePoolHelmRelease(workspaceObj *kaitov1beta1.Workspace, isStatefulSet bool) (*helmv2.HelmRelease, error) {
+	matchLabels := map[string]string{
+		kaitov1beta1.LabelWorkspaceName: workspaceObj.Name,
+	}
+	if isStatefulSet {
+		// Endpoint Picker from Gateway API Inference Extension expects to pick an endpoint that can serve traffic.
+		// In a multi-node inference environment, this means we need to select the leader pod (with pod index 0)
+		// since only the leader pod is capable of serving traffic.
+		matchLabels[appsv1.PodIndexLabel] = "0"
+	}
+
+	// Based on https://github.com/kubernetes-sigs/gateway-api-inference-extension/blob/v0.5.1/config/charts/inferencepool/values.yaml
+	helmValues := map[string]any{
+		"inferenceExtension": map[string]any{
+			"image": map[string]string{
+				"hub":        consts.GatewayAPIInferenceExtensionImageRepository,
+				"tag":        consts.InferencePoolChartVersion,
+				"pullPolicy": string(corev1.PullIfNotPresent),
+			},
+			"pluginsConfigFile": "plugins-v2.yaml",
+		},
+		"inferencePool": map[string]any{
+			"targetPortNumber": consts.PortInferenceServer,
+			"modelServers": map[string]any{
+				"matchLabels": matchLabels,
+			},
+		},
+	}
+	rawHelmValues, err := json.Marshal(helmValues)
+	if err != nil {
+		return nil, err
+	}
+
+	return &helmv2.HelmRelease{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      utils.InferencePoolName(workspaceObj.Name),
+			Namespace: workspaceObj.Namespace,
+			OwnerReferences: []metav1.OwnerReference{
+				*metav1.NewControllerRef(workspaceObj, kaitov1beta1.GroupVersion.WithKind("Workspace")),
+			},
+		},
+		Spec: helmv2.HelmReleaseSpec{
+			// Referencing the OCIRepository created above
+			ChartRef: &helmv2.CrossNamespaceSourceReference{
+				Kind:      sourcev1.OCIRepositoryKind,
+				Namespace: workspaceObj.Namespace,
+				Name:      utils.InferencePoolName(workspaceObj.Name),
+			},
+			Values: &apiextensionsv1.JSON{
+				Raw: rawHelmValues,
+			},
+		},
+	}, nil
+}
diff --git a/pkg/workspace/tuning/preset_tuning.go b/pkg/workspace/tuning/preset_tuning.go
index d5d02da..9fd1391 100644
--- a/pkg/workspace/tuning/preset_tuning.go
+++ b/pkg/workspace/tuning/preset_tuning.go
@@ -39,15 +39,13 @@ import (
 )
 
 const (
-	PortInferenceServer = int32(5000)
-
 	DefaultBaseDir          = "/mnt"
 	DefaultOutputVolumePath = "/mnt/output"
 )
 
 var (
 	containerPorts = []corev1.ContainerPort{{
-		ContainerPort: PortInferenceServer,
+		ContainerPort: consts.PortInferenceServer,
 	}}
 
 	// Come up with valid liveness and readiness probes for fine-tuning
diff --git a/presets/ragengine/config.py b/presets/ragengine/config.py
index 5aa2aa0..d10cae7 100644
--- a/presets/ragengine/config.py
+++ b/presets/ragengine/config.py
@@ -77,3 +77,13 @@ LLM_CONTEXT_WINDOW = int(
 # Vector database configuration
 VECTOR_DB_IMPLEMENTATION = os.getenv("VECTOR_DB_IMPLEMENTATION", "faiss")
 DEFAULT_VECTOR_DB_PERSIST_DIR = os.getenv("DEFAULT_VECTOR_DB_PERSIST_DIR", "storage")
+
+"""
+=========================================================================
+"""
+
+# RAG configurations
+RAG_SIMILARITY_THRESHOLD = float(os.getenv("RAG_SIMILARITY_THRESHOLD", 0.85))
+RAG_DEFAULT_CONTEXT_TOKEN_FILL_RATIO = float(
+    os.getenv("RAG_CONTEXT_TOKEN_FILL_RATIO", 0.5)
+)
diff --git a/presets/ragengine/inference/inference.py b/presets/ragengine/inference/inference.py
index 2225da9..6f935ee 100644
--- a/presets/ragengine/inference/inference.py
+++ b/presets/ragengine/inference/inference.py
@@ -22,6 +22,7 @@ from urllib.parse import urljoin, urlparse
 
 import httpx
 import requests
+import tiktoken
 from fastapi import HTTPException
 from llama_index.core.llms import (
     ChatMessage,
@@ -65,6 +66,7 @@ class Inference(CustomLLM):
     _default_max_model_len: int = None
     _model_retrieval_attempted: bool = False
     _async_http_client: httpx.AsyncClient = PrivateAttr(default=None)
+    _token_encoder: Any = None
 
     async def _get_httpx_client(self):
         """Lazily initializes the HTTP client on first request."""
@@ -162,9 +164,64 @@ class Inference(CustomLLM):
         """Perform an asynchronous chat completion request."""
         try:
             base_model, base_max_len = self._get_default_model_info()
+
+            # 3 characters per token is a conservative approximation for English text and code. For other languages, this may vary.
+            content_token_approximation = (
+                sum(
+                    self.count_tokens(message.content)
+                    for message in messages
+                    if message.content
+                )
+                if messages
+                else 0
+            )
+            logger.info(
+                f"Content token approximation: {content_token_approximation} tokens for messages"
+            )
+
+            if content_token_approximation > LLM_CONTEXT_WINDOW:
+                logger.error(
+                    f"Content length exceeds context window size ({LLM_CONTEXT_WINDOW}). Please reduce the length of the messages."
+                )
+                raise HTTPException(
+                    status_code=400,
+                    detail=f"Content length exceeds context window size ({LLM_CONTEXT_WINDOW}). Please reduce the length of the messages.",
+                )
+
+            # if max_tokens is not provided but content length is less than the context window, we can pass None for max_tokens to allow the model to decide
+            max_tokens = kwargs.get("max_tokens")
+            if max_tokens is not None:
+                logger.info(f"Using provided max_tokens: {max_tokens} tokens")
+                if max_tokens > LLM_CONTEXT_WINDOW:
+                    logger.error(
+                        f"Provided max_tokens ({max_tokens}) exceeds context window size ({LLM_CONTEXT_WINDOW}). Adjusting to fit within context window."
+                    )
+                    raise HTTPException(
+                        status_code=400,
+                        detail=f"Provided max_tokens ({max_tokens}) exceeds context window size ({LLM_CONTEXT_WINDOW}). Adjusting to fit within context window.",
+                    )
+                else:
+                    available_tokens_for_completion = (
+                        LLM_CONTEXT_WINDOW - content_token_approximation
+                    )
+                    if max_tokens > available_tokens_for_completion:
+                        logger.warning(
+                            f"Provided max_tokens ({max_tokens}) plus content length ({content_token_approximation}) exceeds context window size ({LLM_CONTEXT_WINDOW}). Adjusting max_tokens to {available_tokens_for_completion}."
+                        )
+                        max_tokens = available_tokens_for_completion
+
+            if max_tokens and max_tokens < 0:
+                logger.error(
+                    f"max_tokens ({max_tokens}) is negative after content token approximation."
+                )
+                raise HTTPException(
+                    status_code=400,
+                    detail=f"Provided content length exceeds max_tokens limit ({max_tokens}). Please reduce the length of the messages or increase max_tokens.",
+                )
+
             req = {
                 "model": self.get_param("model", base_model),
-                "max_tokens": self.get_param("max_tokens", base_max_len),
+                "max_tokens": max_tokens,
                 "messages": [
                     {
                         "role": message.role,
@@ -402,3 +459,26 @@ class Inference(CustomLLM):
         """Closes the HTTP client when shutting down."""
         if self._async_http_client:
             await self._async_http_client.aclose()
+
+    def count_tokens(self, prompt):
+        """Counts the tokens in the input prompt."""
+        if self._token_encoder is None:
+            try:
+                default_model, default_max_len = self._get_default_model_info()
+                if default_model and "gpt" in default_model:
+                    self._token_encoder = tiktoken.encoding_for_model(default_model)
+            except Exception as e:
+                logger.warning(
+                    f"failed to get token encoder during initialization: {e}"
+                )
+
+        try:
+            if self._token_encoder is None:
+                logger.info("falling back to o200k_base tokenizer")
+                self._token_encoder = tiktoken.get_encoding("o200k_base")
+            return len(self._token_encoder.encode(prompt))
+        except Exception as e:
+            logger.error(f"Error during tokenization: {e}")
+            return int(
+                len(prompt) / 3
+            )  # Fallback to character count if tokenization fails
diff --git a/presets/ragengine/main.py b/presets/ragengine/main.py
index ceaa427..71303df 100644
--- a/presets/ragengine/main.py
+++ b/presets/ragengine/main.py
@@ -252,7 +252,6 @@ async def index_documents(request: IndexRequest):
     {
       "index_name": "example_index",
       "query": "What is RAG?",
-      "top_k": 5,
       "llm_params": {"temperature": 0.7, "max_tokens": 2048},
       "rerank_params": {"top_n": 3}  # ⚠️ Experimental Feature
     }
@@ -348,8 +347,7 @@ async def query_index(request: QueryRequest):
       ],
       "temperature": 0.7,
       "max_tokens": 2048,
-      "top_k": 5,
-      "rerank_params": {"top_n": 3}
+      "context_token_ratio": 0.5
     }
     ```
 
diff --git a/presets/ragengine/requirements.txt b/presets/ragengine/requirements.txt
index b6c513a..226c076 100644
--- a/presets/ragengine/requirements.txt
+++ b/presets/ragengine/requirements.txt
@@ -18,6 +18,7 @@ aiorwlock==1.5.0
 httpx==0.27.0
 openai==1.97.1
 
+tiktoken==0.11.0
 tree-sitter==0.23.2
 tree_sitter_languages==1.10.2
 tree-sitter-language-pack==0.7.3
diff --git a/presets/ragengine/vector_store/base.py b/presets/ragengine/vector_store/base.py
index 7642928..9239178 100644
--- a/presets/ragengine/vector_store/base.py
+++ b/presets/ragengine/vector_store/base.py
@@ -33,10 +33,17 @@ from llama_index.vector_stores.faiss import FaissMapVectorStore
 from openai.types.chat import ChatCompletionContentPartTextParam, CompletionCreateParams
 from pydantic import ValidationError
 
-from ragengine.config import LLM_RERANKER_BATCH_SIZE, LLM_RERANKER_TOP_N
+from ragengine.config import (
+    LLM_RERANKER_BATCH_SIZE,
+    LLM_RERANKER_TOP_N,
+    RAG_DEFAULT_CONTEXT_TOKEN_FILL_RATIO,
+)
 from ragengine.embedding.base import BaseEmbeddingModel
 from ragengine.inference.inference import Inference
 from ragengine.models import ChatCompletionResponse, Document, messages_to_prompt
+from ragengine.vector_store.node_processors.contex_selection_node_processor import (
+    ContextSelectionProcessor,
+)
 from ragengine.vector_store.transformers.custom_transformer import CustomTransformer
 
 # Configure logging
@@ -250,6 +257,15 @@ class BaseVectorStore(ABC):
                 detail=f"No such index: '{request.get('index_name')}' exists.",
             )
 
+        if request.get("context_token_ratio") and (
+            request.get("context_token_ratio") < 0.2
+            or request.get("context_token_ratio") > 0.8
+        ):
+            raise HTTPException(
+                status_code=400,
+                detail=f"Invalid context_token_ratio: {request.get('context_token_ratio')}. Must be between 0.2 and 0.8.",
+            )
+
         llm_params = {}
         if request.get("model") is not None:
             llm_params["model"] = request.get("model")
@@ -342,16 +358,46 @@ class BaseVectorStore(ABC):
                         detail="Invalid request format: user messages must contain 'content'.",
                     )
 
+        max_tokens = request.get("max_tokens")
         prompt = messages_to_prompt(request.get("messages", []))
+        prompt_len = self.llm.count_tokens(prompt)
+        if prompt_len > self.llm.metadata.context_window:
+            logger.error(
+                f"Prompt length ({prompt_len}) exceeds context window ({self.llm.metadata.context_window})."
+            )
+            raise HTTPException(
+                status_code=400, detail="Prompt length exceeds context window."
+            )
+
+        if max_tokens and max_tokens > self.llm.metadata.context_window - prompt_len:
+            # max_tokens is greater than the available tokens
+            # this edit will make sure we dont add more context in the rag than we should
+            # we also handle updating the max_tokens within the inference code before shipping to LLM based off added context
+            logger.warning(
+                f"max_tokens ({max_tokens}) is greater than available context after prompt consideration. Setting to {self.llm.metadata.context_window - prompt_len}."
+            )
+            max_tokens = self.llm.metadata.context_window - prompt_len
 
         logger.info(
             f"Creating chat engine for index '{request.get('index_name')}' with prompt: {prompt}"
         )
-        chat_engine = self.index_map[request.get("index_name")].as_chat_engine(
+        chat_engine = self.index_map[
+            request.get("index_name")
+        ].as_chat_engine(
             llm=self.llm,
-            similarity_top_k=request.get("top_k", 5),
+            similarity_top_k=100,  # Might want to make this a function of avg doc node size in an index but this should be a wide enough default
             chat_mode=ChatMode.CONDENSE_PLUS_CONTEXT,
             verbose=True,
+            node_postprocessors=[
+                ContextSelectionProcessor(
+                    rag_context_token_fill_ratio=request.get(
+                        "context_token_ratio", RAG_DEFAULT_CONTEXT_TOKEN_FILL_RATIO
+                    ),
+                    llm=self.llm,
+                    max_tokens=max_tokens,
+                    similarity_threshold=0.8,
+                )
+            ],
         )
 
         logger.info("Processing chat completion request with prompt.")
diff --git a/presets/ragengine/vector_store/node_processors/__init__.py b/presets/ragengine/vector_store/node_processors/__init__.py
new file mode 100644
index 0000000..0f759ee
--- /dev/null
+++ b/presets/ragengine/vector_store/node_processors/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) KAITO authors.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
diff --git a/presets/ragengine/vector_store/node_processors/contex_selection_node_processor.py b/presets/ragengine/vector_store/node_processors/contex_selection_node_processor.py
new file mode 100644
index 0000000..07a22a7
--- /dev/null
+++ b/presets/ragengine/vector_store/node_processors/contex_selection_node_processor.py
@@ -0,0 +1,121 @@
+# Copyright (c) KAITO authors.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from llama_index.core.bridge.pydantic import Field, PrivateAttr
+from llama_index.core.llms.llm import LLM
+from llama_index.core.postprocessor.types import BaseNodePostprocessor
+from llama_index.core.schema import NodeWithScore, QueryBundle
+from llama_index.core.settings import Settings
+
+ADDITION_PROMPT_TOKENS = (
+    150  # Accounts for the addition prompt added by llamaindex after node processing
+)
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class ContextSelectionProcessor(BaseNodePostprocessor):
+    """
+    Context selection processor.
+    This processor selects nodes based on their relevance to the query and the available context window.
+    """
+
+    llm: LLM = Field(description="The llm to get metadata from.")
+
+    _max_tokens: int = PrivateAttr()
+    _rag_context_token_fill_ratio: float = PrivateAttr()
+    _similarity_threshold: float | None = PrivateAttr()
+
+    def __init__(
+        self,
+        rag_context_token_fill_ratio: float,
+        llm: LLM | None = None,
+        max_tokens: int | None = None,
+        similarity_threshold: float | None = None,
+    ) -> None:
+        llm = llm or Settings.llm
+
+        super().__init__(
+            llm=llm,
+        )
+
+        self._rag_context_token_fill_ratio = rag_context_token_fill_ratio
+
+        # Use the passed in max_tokens or the context window size if not provided
+        self._max_tokens = max_tokens or llm.metadata.context_window
+
+        self._similarity_threshold = similarity_threshold or None
+
+    @classmethod
+    def class_name(cls) -> str:
+        return "ContextSelectionProcessor"
+
+    def _postprocess_nodes(
+        self,
+        nodes: list[NodeWithScore],
+        query_bundle: QueryBundle | None = None,
+    ) -> list[NodeWithScore]:
+        if query_bundle is None:
+            raise ValueError("Query bundle must be provided.")
+        if len(nodes) == 0:
+            return []
+
+        query_token_aproximation = self.llm.count_tokens(query_bundle.query_str)
+        # Total tokens available for context after accounting only for query
+        available_tokens_for_context = (
+            self.llm.metadata.context_window
+            - query_token_aproximation
+            - ADDITION_PROMPT_TOKENS
+        )
+        # Take the lesser of the max_tokens param and available context
+        available_tokens_for_context = min(
+            self._max_tokens, available_tokens_for_context
+        )
+        # Apply the RAG context fill ratio
+        available_tokens_for_context = int(
+            available_tokens_for_context * self._rag_context_token_fill_ratio
+        )
+
+        if available_tokens_for_context <= 0:
+            logger.warning(
+                "No available tokens for context after accounting for query and addition prompt."
+            )
+            return []
+
+        # the scores from faiss are distances and we want to rerank based on relevance
+        ranked_nodes = sorted(nodes, key=lambda x: x.score or 0.0)
+
+        result: list[NodeWithScore] = []
+        for idx in range(len(ranked_nodes)):
+            node = ranked_nodes[idx]
+            if (
+                self._similarity_threshold is not None
+                and node.score > self._similarity_threshold
+            ):
+                continue
+
+            node_token_approximation = self.llm.count_tokens(node.text)
+            if node_token_approximation > available_tokens_for_context:
+                continue
+
+            available_tokens_for_context -= node_token_approximation
+            result.append(node)
+
+        logger.info(
+            f"Selected {len(result)} nodes out of {len(nodes)} based on context window and similarity threshold."
+        )
+        return result
diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl
index 24d0b78..87c59ea 100644
--- a/terraform/.terraform.lock.hcl
+++ b/terraform/.terraform.lock.hcl
@@ -2,22 +2,22 @@
 # Manual edits may be lost in future updates.
 
 provider "registry.terraform.io/hashicorp/azurerm" {
-  version     = "4.35.0"
-  constraints = "4.35.0"
+  version     = "4.40.0"
+  constraints = "4.40.0"
   hashes = [
-    "h1:sxwmvyk9CQ9stspIPIz7ySbpiqDCEcl2gVEGM7omQoY=",
-    "zh:05a7fac1291517527d272dd2e22a8b3223775d399dc608c8250435e63d437386",
-    "zh:3519dae43473cb5a82c026d664572859e2fc886fe2a54ecc59007186ef4aa166",
-    "zh:57cd9d04334d296a9af3aa54d776bc3b4709ecf5d511fee94dc7420a547364ab",
-    "zh:7d57cbd44ebf959fc496743b4e3587a51dd7546a687bab404dc3f5899d3ef764",
-    "zh:8c1e722c84aa8d769cdef0be933d6f096ff0ec8224eaf1ae2c7bdb93b0365c6f",
-    "zh:9519f72c937003ecc97a0a5a60c0ad1063dafa3d431cf04067bdc4b40ebfd023",
-    "zh:9bf108a2840ecfb3d2e76b1ac59d0d9bed418b1b28df8c2c5dd09e34cb1e3699",
-    "zh:b870884accad83c437a12a577933f378a21bb6663a6034a3fb219b4c1f4b9188",
-    "zh:cf3292dd535133afe16614ac84241065485d581ab67b4abf857db5bfbba7109e",
-    "zh:d48aa50227ca3701a16d2818b5d059c5f236344e3a90162fb930ecc6f136c9b0",
-    "zh:e63733db32189c1e849b5dbf7c10dd852f48ed4186c8af81240ee643d2df3425",
-    "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
+    "h1:/TNkrn3b7z0ucgKX6Y3QqSedGxFJfrksNc482wOfQvc=",
+    "zh:035ff0ed9fe359e9ec36a79d22ce47eaa9d6b2ec6ba834d6d2fdec3d216d6b52",
+    "zh:1a22d900c453cca39e9a0c423a517593295a07619f52386565c6b96e6c9f6b27",
+    "zh:1f46331d0ef6fae6bae357e574e9d2a0b74986fd271c80d21e8b64b8a493d07c",
+    "zh:38f5cb8264c27962e64e41987d5e607591807c6db64f9f02c4b97f195ecc7c26",
+    "zh:64bbc0f4f19b38caa1d0590a1fc58800779a0d00396100040a118f91ec43c707",
+    "zh:6b1695c75ee5c9b632c1b80c4df56bc9a5c612e6952d563807492742c13dca49",
+    "zh:6d9038d784601386e731f5bbe82d9bf405b3b147158ca09c055ac3ed1632d14b",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:b5b80bf352a3fe130024a7bf68531549f0fc3c1b96945440942a851128b7bf8b",
+    "zh:ce82cc82a3f483e69dde55818e28be625d81c5ff2b881e13ab3a6cd4ed06ff04",
+    "zh:f4ad5295d44c612490d2ae25454f7220fc961d609abc82b1b9a8fd718691b060",
+    "zh:f6fd85bfca8516f9f3840acc3f0e5e335f6195b66231c2db05c0fb43dd8aebe1",
   ]
 }
 
@@ -26,6 +26,7 @@ provider "registry.terraform.io/hashicorp/helm" {
   constraints = "3.0.2"
   hashes = [
     "h1:+tHGl509bhyUrvvj9GQTBsdK+ImHJnRuo6ppDZPavqY=",
+    "h1:tOye2RnjFNXH236AsqGaIWtz4j6PZrpPuJhOSBt0KxU=",
     "zh:2778de76c7dfb2e85c75fe6de3c11172a25551ed499bfb9e9f940a5be81167b0",
     "zh:3b4c436a41e4fbae5f152852a9bd5c97db4460af384e26977477a40adf036690",
     "zh:617a372f5bb2288f3faf5fd4c878a68bf08541cf418a3dbb8a19bc41ad4a0bf2",
@@ -42,22 +43,22 @@ provider "registry.terraform.io/hashicorp/helm" {
 }
 
 provider "registry.terraform.io/hashicorp/kubernetes" {
-  version     = "2.37.1"
-  constraints = "2.37.1"
+  version     = "2.38.0"
+  constraints = "2.38.0"
   hashes = [
-    "h1:qo9Ue/rIEnvxOpiK9qizwRFV7rvb5gCziKVytIcZHyk=",
-    "zh:0ed097413c7fc804479e325966886b405dc0b75ad2b4f54ce4df1d8e4802b397",
-    "zh:17dcf4a685a00d2d048671124e8a1a8e836b58ecd2ef628a1c666fe0ced2e598",
-    "zh:36891284e5bced57c438f12d0b27856b0d4b70b562bd200b01919a6a89545be9",
-    "zh:3e49d86b508e641ba122d1b0af24cdc4d8ffa2ec1b30022436fb1d7c6ba696ea",
-    "zh:40be623e116708bdcb0fac32989db43720f031c5fe9a4dc63395078185d24403",
-    "zh:44fc0ac3bc39e289b67f9dde7ee9fef29eb8192197e5e68fee69098573021722",
-    "zh:957aa451573bcde5d57f6f8338ea3139010c7f61fefe8f6a140a8c267f056511",
-    "zh:c55fd85b7e8acaac17e30670ac3574b88b3530820dd004bcd2a5daa8624a46e9",
-    "zh:c743f06843a1f5ecde2b8ef639f4d3db654a334ef248dee57261c719ea843f3a",
-    "zh:c93cc71c64b838d89522ac5fb60f68e0e1e7f2fc39db6b0ead7afd78795e79ed",
-    "zh:eda1163c2266905adc54bc78cc3e7b606a164fbc6b59be607db933b302015ccd",
+    "h1:soK8Lt0SZ6dB+HsypFRDzuX/npqlMU6M0fvyaR1yW0k=",
+    "zh:0af928d776eb269b192dc0ea0f8a3f0f5ec117224cd644bdacdc682300f84ba0",
+    "zh:1be998e67206f7cfc4ffe77c01a09ac91ce725de0abaec9030b22c0a832af44f",
+    "zh:326803fe5946023687d603f6f1bab24de7af3d426b01d20e51d4e6fbe4e7ec1b",
+    "zh:4a99ec8d91193af961de1abb1f824be73df07489301d62e6141a656b3ebfff12",
+    "zh:5136e51765d6a0b9e4dbcc3b38821e9736bd2136cf15e9aac11668f22db117d2",
+    "zh:63fab47349852d7802fb032e4f2b6a101ee1ce34b62557a9ad0f0f0f5b6ecfdc",
+    "zh:924fb0257e2d03e03e2bfe9c7b99aa73c195b1f19412ca09960001bee3c50d15",
+    "zh:b63a0be5e233f8f6727c56bed3b61eb9456ca7a8bb29539fba0837f1badf1396",
+    "zh:d39861aa21077f1bc899bc53e7233262e530ba8a3a2d737449b100daeb303e4d",
+    "zh:de0805e10ebe4c83ce3b728a67f6b0f9d18be32b25146aa89116634df5145ad4",
     "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
+    "zh:faf23e45f0090eef8ba28a8aac7ec5d4fdf11a36c40a8d286304567d71c1e7db",
   ]
 }
 
@@ -66,6 +67,7 @@ provider "registry.terraform.io/hashicorp/random" {
   constraints = "3.7.2"
   hashes = [
     "h1:356j/3XnXEKr9nyicLUufzoF4Yr6hRy481KIxRVpK0c=",
+    "h1:KG4NuIBl1mRWU0KD/BGfCi1YN/j3F7H4YgeeM7iSdNs=",
     "zh:14829603a32e4bc4d05062f059e545a91e27ff033756b48afbae6b3c835f508f",
     "zh:1527fb07d9fea400d70e9e6eb4a2b918d5060d604749b6f1c361518e7da546dc",
     "zh:1e86bcd7ebec85ba336b423ba1db046aeaa3c0e5f921039b3f1a6fc2f978feab",
diff --git a/terraform/kubernetes.tf b/terraform/kubernetes.tf
index 93c968f..2c7ab66 100644
--- a/terraform/kubernetes.tf
+++ b/terraform/kubernetes.tf
@@ -8,8 +8,8 @@ resource "azurerm_kubernetes_cluster" "example" {
 
   default_node_pool {
     name       = "default"
-    node_count = 1
-    vm_size    = "Standard_D2_v2"
+    node_count = var.node_pool_count
+    vm_size    = var.node_pool_vm_size
 
     upgrade_settings {
       drain_timeout_in_minutes      = 0
diff --git a/terraform/main.tf b/terraform/main.tf
index 5fb4fa2..dd54571 100644
--- a/terraform/main.tf
+++ b/terraform/main.tf
@@ -2,7 +2,7 @@ terraform {
   required_providers {
     azurerm = {
       source  = "hashicorp/azurerm"
-      version = "=4.35.0"
+      version = "=4.40.0"
     }
 
     random = {
@@ -12,7 +12,7 @@ terraform {
 
     kubernetes = {
       source  = "hashicorp/kubernetes"
-      version = "=2.37.1"
+      version = "=2.38.0"
     }
 
     helm = {
diff --git a/terraform/variables.tf b/terraform/variables.tf
index 958e211..21c7b13 100644
--- a/terraform/variables.tf
+++ b/terraform/variables.tf
@@ -4,6 +4,18 @@ variable "location" {
   description = "value of location"
 }
 
+variable "node_pool_count" {
+  type        = number
+  default     = 1
+  description = "Number of nodes in the AKS node pool"
+}
+
+variable "node_pool_vm_size" {
+  type        = string
+  default     = "standard_d2s_v6"
+  description = "VM size for the AKS node pool"
+}
+
 variable "kaito_gpu_provisioner_version" {
   type        = string
   default     = "0.3.5"
diff --git a/website/docs/benchmarks/a10/benchmark-phi4-p1.csv b/website/docs/benchmarks/a10/benchmark-phi4-p1.csv
new file mode 100644
index 0000000..795a982
--- /dev/null
+++ b/website/docs/benchmarks/a10/benchmark-phi4-p1.csv
@@ -0,0 +1,2 @@
+Type,Run Id,Id,Name,Start Time,End Time,Duration,Successful Requests,Successful Requests per second mean,Successful Requests per second median,Successful Requests per second std dev,"Successful Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Request concurrency mean,Successful Request concurrency median,Successful Request concurrency std dev,"Successful Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Request latency mean,Successful Request latency median,Successful Request latency std dev,"Successful Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Prompt token count mean,Successful Prompt token count median,Successful Prompt token count std dev,"Successful Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Output token count mean,Successful Output token count median,Successful Output token count std dev,"Successful Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Time to first token ms mean,Successful Time to first token ms median,Successful Time to first token ms std dev,"Successful Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Time per output token ms mean,Successful Time per output token ms median,Successful Time per output token ms std dev,"Successful Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Inter token latency ms mean,Successful Inter token latency ms median,Successful Inter token latency ms std dev,"Successful Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Output tokens per second mean,Successful Output tokens per second median,Successful Output tokens per second std dev,"Successful Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Tokens per second mean,Successful Tokens per second median,Successful Tokens per second std dev,"Successful Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Requests,Errored Requests per second mean,Errored Requests per second median,Errored Requests per second std dev,"Errored Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Request concurrency mean,Errored Request concurrency median,Errored Request concurrency std dev,"Errored Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Request latency mean,Errored Request latency median,Errored Request latency std dev,"Errored Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Prompt token count mean,Errored Prompt token count median,Errored Prompt token count std dev,"Errored Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Output token count mean,Errored Output token count median,Errored Output token count std dev,"Errored Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Time to first token ms mean,Errored Time to first token ms median,Errored Time to first token ms std dev,"Errored Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Time per output token ms mean,Errored Time per output token ms median,Errored Time per output token ms std dev,"Errored Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Inter token latency ms mean,Errored Inter token latency ms median,Errored Inter token latency ms std dev,"Errored Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Output tokens per second mean,Errored Output tokens per second median,Errored Output tokens per second std dev,"Errored Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Tokens per second mean,Errored Tokens per second median,Errored Tokens per second std dev,"Errored Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Requests,Incomplete Requests per second mean,Incomplete Requests per second median,Incomplete Requests per second std dev,"Incomplete Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Request concurrency mean,Incomplete Request concurrency median,Incomplete Request concurrency std dev,"Incomplete Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Request latency mean,Incomplete Request latency median,Incomplete Request latency std dev,"Incomplete Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Prompt token count mean,Incomplete Prompt token count median,Incomplete Prompt token count std dev,"Incomplete Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Output token count mean,Incomplete Output token count median,Incomplete Output token count std dev,"Incomplete Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Time to first token ms mean,Incomplete Time to first token ms median,Incomplete Time to first token ms std dev,"Incomplete Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Time per output token ms mean,Incomplete Time per output token ms median,Incomplete Time per output token ms std dev,"Incomplete Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Inter token latency ms mean,Incomplete Inter token latency ms median,Incomplete Inter token latency ms std dev,"Incomplete Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Output tokens per second mean,Incomplete Output tokens per second median,Incomplete Output tokens per second std dev,"Incomplete Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Tokens per second mean,Incomplete Tokens per second median,Incomplete Tokens per second std dev,"Incomplete Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Requests,Total Requests per second mean,Total Requests per second median,Total Requests per second std dev,"Total Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Request concurrency mean,Total Request concurrency median,Total Request concurrency std dev,"Total Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Request latency mean,Total Request latency median,Total Request latency std dev,"Total Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Prompt token count mean,Total Prompt token count median,Total Prompt token count std dev,"Total Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Output token count mean,Total Output token count median,Total Output token count std dev,"Total Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Time to first token ms mean,Total Time to first token ms median,Total Time to first token ms std dev,"Total Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Time per output token ms mean,Total Time per output token ms median,Total Time per output token ms std dev,"Total Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Inter token latency ms mean,Total Inter token latency ms median,Total Inter token latency ms std dev,"Total Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Output tokens per second mean,Total Output tokens per second median,Total Output tokens per second std dev,"Total Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Tokens per second mean,Total Tokens per second median,Total Tokens per second std dev,"Total Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Args,Worker,Request Loader,Extras
+generative_benchmark,012dc94a-0a19-4874-ae5b-83ef11a1740c,a7f1ba55-5242-4ec2-ad51-7d2bd81bc130,poisson@1.00,2025-08-15 17:38:09,2025-08-15 17:55:46,1056.8281016349792,1000,0.9462276773800178,0.5500336304007808,22.577654770599537,"[0.10840483101069426, 0.10840483101069426, 0.12668640181361668, 0.17535461269402633, 0.22849596919613818, 0.3367201467524596, 0.979371412413205, 1.7739116018267342, 2.7760944889185994, 6.182331663802227, 246723.76470588235]",4.15359085625389,4.0,2.132836152895644,"[0.0, 0.0, 0.0, 1.0, 2.0, 3.0, 5.0, 7.0, 8.0, 10.0, 13.0]",4.389631539583206,4.380148887634277,0.10593887255282987,"[4.1574180126190186, 4.1574180126190186, 4.188790559768677, 4.234158515930176, 4.260356664657593, 4.31402325630188, 4.450855731964111, 4.531518936157227, 4.5984132289886475, 4.687419891357422, 4.738286018371582]",200.05400000000003,200.0,0.25900579144104097,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 201.0, 202.0]",200.00000000000003,200.0,2.8421709430404014e-14,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0]",369.7241015434265,367.66791343688965,14.322467962037605,"[342.34046936035156, 342.34046936035156, 350.9666919708252, 355.12757301330566, 357.44643211364746, 361.76252365112305, 373.6283779144287, 379.5344829559326, 391.9100761413574, 424.24917221069336, 491.7619228363037]",20.099078699350358,20.056315660476685,0.5187243326236758,"[18.91164779663086, 18.91164779663086, 19.126254320144653, 19.31639552116394, 19.478856325149536, 19.72736954689026, 20.39910912513733, 20.75339436531067, 21.173570156097412, 21.571906805038452, 21.823430061340332]",20.200079094824478,20.157101166308227,0.5213309875614832,"[19.00668120264408, 19.00668120264408, 19.22236615089915, 19.413462835340642, 19.576740025275917, 19.82650205717614, 20.501617211193295, 20.8576827792067, 21.279970006128053, 21.680308346772314, 21.93309553903551]",189.24458924832618,109.98279840570589,638.6434009875273,"[0.0, 0.1874615209929395, 0.30104765460034444, 50.59535096925174, 51.80839447614812, 68.55454218561015, 190.94527906765, 351.8121120617346, 532.8130081300814, 1300.5593798449613, 838860.8]",566.8916251121104,222.5921562383909,14103.590527430668,"[0.0, 0.1874615209929395, 0.30104765460034444, 101.48328090975079, 103.80912780912782, 139.3919574609505, 392.3944241743849, 716.73000683527, 1088.8639667705088, 2962.0790960451977, 167772160.0]",0,0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0,0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",1000,0.9462276773800178,0.5500336304007808,22.577654770599537,"[0.10840483101069426, 0.10840483101069426, 0.12668640181361668, 0.17535461269402633, 0.22849596919613818, 0.3367201467524596, 0.979371412413205, 1.7739116018267342, 2.7760944889185994, 6.182331663802227, 246723.76470588235]",4.15359085625389,4.0,2.132836152895644,"[0.0, 0.0, 0.0, 1.0, 2.0, 3.0, 5.0, 7.0, 8.0, 10.0, 13.0]",4.389631539583206,4.380148887634277,0.10593887255282987,"[4.1574180126190186, 4.1574180126190186, 4.188790559768677, 4.234158515930176, 4.260356664657593, 4.31402325630188, 4.450855731964111, 4.531518936157227, 4.5984132289886475, 4.687419891357422, 4.738286018371582]",200.05400000000003,200.0,0.25900579144104097,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 201.0, 202.0]",200.00000000000003,200.0,2.8421709430404014e-14,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0]",369.7241015434265,367.66791343688965,14.322467962037605,"[342.34046936035156, 342.34046936035156, 350.9666919708252, 355.12757301330566, 357.44643211364746, 361.76252365112305, 373.6283779144287, 379.5344829559326, 391.9100761413574, 424.24917221069336, 491.7619228363037]",20.099078699350358,20.056315660476685,0.5187243326236758,"[18.91164779663086, 18.91164779663086, 19.126254320144653, 19.31639552116394, 19.478856325149536, 19.72736954689026, 20.39910912513733, 20.75339436531067, 21.173570156097412, 21.571906805038452, 21.823430061340332]",20.200079094824478,20.157101166308227,0.5213309875614832,"[19.00668120264408, 19.00668120264408, 19.22236615089915, 19.413462835340642, 19.576740025275917, 19.82650205717614, 20.501617211193295, 20.8576827792067, 21.279970006128053, 21.680308346772314, 21.93309553903551]",189.24458924832618,109.98279840570589,638.6434009875273,"[0.0, 0.1874615209929395, 0.30104765460034444, 50.59535096925174, 51.80839447614812, 68.55454218561015, 190.94527906765, 351.8121120617346, 532.8130081300814, 1300.5593798449613, 838860.8]",566.8916251121104,222.5921562383909,14103.590527430668,"[0.0, 0.1874615209929395, 0.30104765460034444, 101.48328090975079, 103.80912780912782, 139.3919574609505, 392.3944241743849, 716.73000683527, 1088.8639667705088, 2962.0790960451977, 167772160.0]","{""profile"": {""type_"": ""async"", ""completed_strategies"": 1, ""measured_rates"": [0.9462276773800178], ""measured_concurrencies"": [4.15359085625389], ""max_concurrency"": null, ""strategy_type"": ""poisson"", ""rate"": [1.0], ""initial_burst"": true, ""random_seed"": 42, ""strategy_types"": [""poisson""]}, ""strategy_index"": 0, ""strategy"": {""type_"": ""poisson"", ""max_concurrency"": null, ""rate"": 1.0, ""initial_burst"": true, ""random_seed"": 42}, ""max_number"": 1000, ""max_duration"": null, ""warmup_number"": null, ""warmup_duration"": null, ""cooldown_number"": null, ""cooldown_duration"": null}","{""type_"": ""generative_requests_worker"", ""backend_type"": ""openai_http"", ""backend_target"": ""http://127.0.0.1:8000"", ""backend_model"": ""phi-4-mini-instruct"", ""backend_info"": {""max_output_tokens"": 16384, ""timeout"": 300, ""http2"": true, ""authorization"": false, ""organization"": null, ""project"": null, ""text_completions_path"": ""/v1/completions"", ""chat_completions_path"": ""/v1/chat/completions""}}","{""type_"": ""generative_request_loader"", ""data"": ""prompt_tokens=200,output_tokens=200"", ""data_args"": null, ""processor"": ""microsoft/phi-4-mini-instruct"", ""processor_args"": null}",{}
diff --git a/website/docs/benchmarks/a10/benchmark-phi4-p16.csv b/website/docs/benchmarks/a10/benchmark-phi4-p16.csv
new file mode 100644
index 0000000..c137a56
--- /dev/null
+++ b/website/docs/benchmarks/a10/benchmark-phi4-p16.csv
@@ -0,0 +1,2 @@
+Type,Run Id,Id,Name,Start Time,End Time,Duration,Successful Requests,Successful Requests per second mean,Successful Requests per second median,Successful Requests per second std dev,"Successful Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Request concurrency mean,Successful Request concurrency median,Successful Request concurrency std dev,"Successful Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Request latency mean,Successful Request latency median,Successful Request latency std dev,"Successful Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Prompt token count mean,Successful Prompt token count median,Successful Prompt token count std dev,"Successful Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Output token count mean,Successful Output token count median,Successful Output token count std dev,"Successful Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Time to first token ms mean,Successful Time to first token ms median,Successful Time to first token ms std dev,"Successful Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Time per output token ms mean,Successful Time per output token ms median,Successful Time per output token ms std dev,"Successful Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Inter token latency ms mean,Successful Inter token latency ms median,Successful Inter token latency ms std dev,"Successful Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Output tokens per second mean,Successful Output tokens per second median,Successful Output tokens per second std dev,"Successful Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Tokens per second mean,Successful Tokens per second median,Successful Tokens per second std dev,"Successful Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Requests,Errored Requests per second mean,Errored Requests per second median,Errored Requests per second std dev,"Errored Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Request concurrency mean,Errored Request concurrency median,Errored Request concurrency std dev,"Errored Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Request latency mean,Errored Request latency median,Errored Request latency std dev,"Errored Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Prompt token count mean,Errored Prompt token count median,Errored Prompt token count std dev,"Errored Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Output token count mean,Errored Output token count median,Errored Output token count std dev,"Errored Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Time to first token ms mean,Errored Time to first token ms median,Errored Time to first token ms std dev,"Errored Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Time per output token ms mean,Errored Time per output token ms median,Errored Time per output token ms std dev,"Errored Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Inter token latency ms mean,Errored Inter token latency ms median,Errored Inter token latency ms std dev,"Errored Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Output tokens per second mean,Errored Output tokens per second median,Errored Output tokens per second std dev,"Errored Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Tokens per second mean,Errored Tokens per second median,Errored Tokens per second std dev,"Errored Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Requests,Incomplete Requests per second mean,Incomplete Requests per second median,Incomplete Requests per second std dev,"Incomplete Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Request concurrency mean,Incomplete Request concurrency median,Incomplete Request concurrency std dev,"Incomplete Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Request latency mean,Incomplete Request latency median,Incomplete Request latency std dev,"Incomplete Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Prompt token count mean,Incomplete Prompt token count median,Incomplete Prompt token count std dev,"Incomplete Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Output token count mean,Incomplete Output token count median,Incomplete Output token count std dev,"Incomplete Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Time to first token ms mean,Incomplete Time to first token ms median,Incomplete Time to first token ms std dev,"Incomplete Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Time per output token ms mean,Incomplete Time per output token ms median,Incomplete Time per output token ms std dev,"Incomplete Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Inter token latency ms mean,Incomplete Inter token latency ms median,Incomplete Inter token latency ms std dev,"Incomplete Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Output tokens per second mean,Incomplete Output tokens per second median,Incomplete Output tokens per second std dev,"Incomplete Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Tokens per second mean,Incomplete Tokens per second median,Incomplete Tokens per second std dev,"Incomplete Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Requests,Total Requests per second mean,Total Requests per second median,Total Requests per second std dev,"Total Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Request concurrency mean,Total Request concurrency median,Total Request concurrency std dev,"Total Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Request latency mean,Total Request latency median,Total Request latency std dev,"Total Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Prompt token count mean,Total Prompt token count median,Total Prompt token count std dev,"Total Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Output token count mean,Total Output token count median,Total Output token count std dev,"Total Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Time to first token ms mean,Total Time to first token ms median,Total Time to first token ms std dev,"Total Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Time per output token ms mean,Total Time per output token ms median,Total Time per output token ms std dev,"Total Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Inter token latency ms mean,Total Inter token latency ms median,Total Inter token latency ms std dev,"Total Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Output tokens per second mean,Total Output tokens per second median,Total Output tokens per second std dev,"Total Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Tokens per second mean,Total Tokens per second median,Total Tokens per second std dev,"Total Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Args,Worker,Request Loader,Extras
+generative_benchmark,c4701419-1e0e-4d9f-9f04-db211edbd344,5e68022c-0279-4724-9d09-d3c569ab1ea6,poisson@16.00,2025-08-15 19:19:04,2025-08-15 19:20:59,114.99309754371643,1000,8.696174130102323,3.3917704178837016,194.96581140994965,"[0.05116303795273293, 0.05116303795273293, 0.05116303795273293, 0.05116303795273293, 0.05116303795273293, 1.2816290237071961, 5.61278347921855, 7.204714846923789, 8.478668415913837, 39.22513069419895, 838860.8]",348.5695351406012,375.0,141.98778391415715,"[1.0, 1.0, 3.0, 81.0, 130.0, 242.0, 483.0, 512.0, 512.0, 512.0, 989.0]",40.082778706789014,40.8013482093811,10.960925672811488,"[9.70338487625122, 9.70338487625122, 19.913882970809937, 20.613262176513672, 24.536616802215576, 28.91778540611267, 50.49473834037781, 53.31632447242737, 54.17862391471863, 56.15079665184021, 56.65515398979187]",200.05400000000003,200.0,0.25900579144104097,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 201.0, 202.0]",200.00000000000003,200.0,2.8421709430404014e-14,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0]",14181.425513029099,12345.425605773926,11054.362875965435,"[397.5718021392822, 397.5718021392822, 442.18969345092773, 501.48987770080566, 544.3127155303955, 1125.7097721099854, 24915.17162322998, 28834.81478691101, 28873.327255249023, 29246.623992919922, 29614.11738395691]",129.50638111233712,139.88537669181824,18.927785776838483,"[45.87083101272583, 45.87083101272583, 74.15640473365784, 88.87372255325317, 96.48467183113098, 121.21659755706787, 142.27177619934082, 142.61401772499084, 142.84520030021667, 145.31147122383118, 149.28679823875427]",130.15716694707248,140.5883182832344,19.02290027822963,"[46.10133770123199, 46.10133770123199, 74.52904998357572, 89.3203241741238, 96.96951942827235, 121.8257261880079, 142.98670974808124, 143.33067108039282, 143.56301537710218, 146.0416796219409, 150.0369831545269]",1739.2261298463347,909.4327840416305,4401.262821112297,"[0.0, 0.0, 21.690898653855104, 147.06535764375877, 247.93426730507773, 466.0337777777778, 1805.554885923375, 3527.5895710681243, 5489.926701570681, 13486.508038585209, 1198372.5714285714]",5209.938794563213,1951.746859004188,83411.86793918996,"[0.0, 0.0, 43.490880434669904, 353.8004217629692, 607.2540900535688, 1082.4010322580646, 3705.21554770318, 7073.025295109612, 10922.666666666666, 31068.91851851852, 167772160.0]",0,0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0,0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",1000,8.696174130102323,3.3917704178837016,194.96581140994965,"[0.05116303795273293, 0.05116303795273293, 0.05116303795273293, 0.05116303795273293, 0.05116303795273293, 1.2816290237071961, 5.61278347921855, 7.204714846923789, 8.478668415913837, 39.22513069419895, 838860.8]",348.5695351406012,375.0,141.98778391415715,"[1.0, 1.0, 3.0, 81.0, 130.0, 242.0, 483.0, 512.0, 512.0, 512.0, 989.0]",40.082778706789014,40.8013482093811,10.960925672811488,"[9.70338487625122, 9.70338487625122, 19.913882970809937, 20.613262176513672, 24.536616802215576, 28.91778540611267, 50.49473834037781, 53.31632447242737, 54.17862391471863, 56.15079665184021, 56.65515398979187]",200.05400000000003,200.0,0.25900579144104097,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 201.0, 202.0]",200.00000000000003,200.0,2.8421709430404014e-14,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0]",14181.425513029099,12345.425605773926,11054.362875965435,"[397.5718021392822, 397.5718021392822, 442.18969345092773, 501.48987770080566, 544.3127155303955, 1125.7097721099854, 24915.17162322998, 28834.81478691101, 28873.327255249023, 29246.623992919922, 29614.11738395691]",129.50638111233712,139.88537669181824,18.927785776838483,"[45.87083101272583, 45.87083101272583, 74.15640473365784, 88.87372255325317, 96.48467183113098, 121.21659755706787, 142.27177619934082, 142.61401772499084, 142.84520030021667, 145.31147122383118, 149.28679823875427]",130.15716694707248,140.5883182832344,19.02290027822963,"[46.10133770123199, 46.10133770123199, 74.52904998357572, 89.3203241741238, 96.96951942827235, 121.8257261880079, 142.98670974808124, 143.33067108039282, 143.56301537710218, 146.0416796219409, 150.0369831545269]",1739.2261298463347,909.4327840416305,4401.262821112297,"[0.0, 0.0, 21.690898653855104, 147.06535764375877, 247.93426730507773, 466.0337777777778, 1805.554885923375, 3527.5895710681243, 5489.926701570681, 13486.508038585209, 1198372.5714285714]",5209.938794563213,1951.746859004188,83411.86793918996,"[0.0, 0.0, 43.490880434669904, 353.8004217629692, 607.2540900535688, 1082.4010322580646, 3705.21554770318, 7073.025295109612, 10922.666666666666, 31068.91851851852, 167772160.0]","{""profile"": {""type_"": ""async"", ""completed_strategies"": 1, ""measured_rates"": [8.696174130102323], ""measured_concurrencies"": [348.5695351406012], ""max_concurrency"": null, ""strategy_type"": ""poisson"", ""rate"": [16.0], ""initial_burst"": true, ""random_seed"": 42, ""strategy_types"": [""poisson""]}, ""strategy_index"": 0, ""strategy"": {""type_"": ""poisson"", ""max_concurrency"": null, ""rate"": 16.0, ""initial_burst"": true, ""random_seed"": 42}, ""max_number"": 1000, ""max_duration"": null, ""warmup_number"": null, ""warmup_duration"": null, ""cooldown_number"": null, ""cooldown_duration"": null}","{""type_"": ""generative_requests_worker"", ""backend_type"": ""openai_http"", ""backend_target"": ""http://127.0.0.1:8000"", ""backend_model"": ""phi-4-mini-instruct"", ""backend_info"": {""max_output_tokens"": 16384, ""timeout"": 300, ""http2"": true, ""authorization"": false, ""organization"": null, ""project"": null, ""text_completions_path"": ""/v1/completions"", ""chat_completions_path"": ""/v1/chat/completions""}}","{""type_"": ""generative_request_loader"", ""data"": ""prompt_tokens=200,output_tokens=200"", ""data_args"": null, ""processor"": ""microsoft/phi-4-mini-instruct"", ""processor_args"": null}",{}
diff --git a/website/docs/benchmarks/a10/benchmark-phi4-p2.csv b/website/docs/benchmarks/a10/benchmark-phi4-p2.csv
new file mode 100644
index 0000000..0af7b70
--- /dev/null
+++ b/website/docs/benchmarks/a10/benchmark-phi4-p2.csv
@@ -0,0 +1,2 @@
+Type,Run Id,Id,Name,Start Time,End Time,Duration,Successful Requests,Successful Requests per second mean,Successful Requests per second median,Successful Requests per second std dev,"Successful Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Request concurrency mean,Successful Request concurrency median,Successful Request concurrency std dev,"Successful Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Request latency mean,Successful Request latency median,Successful Request latency std dev,"Successful Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Prompt token count mean,Successful Prompt token count median,Successful Prompt token count std dev,"Successful Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Output token count mean,Successful Output token count median,Successful Output token count std dev,"Successful Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Time to first token ms mean,Successful Time to first token ms median,Successful Time to first token ms std dev,"Successful Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Time per output token ms mean,Successful Time per output token ms median,Successful Time per output token ms std dev,"Successful Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Inter token latency ms mean,Successful Inter token latency ms median,Successful Inter token latency ms std dev,"Successful Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Output tokens per second mean,Successful Output tokens per second median,Successful Output tokens per second std dev,"Successful Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Tokens per second mean,Successful Tokens per second median,Successful Tokens per second std dev,"Successful Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Requests,Errored Requests per second mean,Errored Requests per second median,Errored Requests per second std dev,"Errored Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Request concurrency mean,Errored Request concurrency median,Errored Request concurrency std dev,"Errored Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Request latency mean,Errored Request latency median,Errored Request latency std dev,"Errored Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Prompt token count mean,Errored Prompt token count median,Errored Prompt token count std dev,"Errored Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Output token count mean,Errored Output token count median,Errored Output token count std dev,"Errored Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Time to first token ms mean,Errored Time to first token ms median,Errored Time to first token ms std dev,"Errored Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Time per output token ms mean,Errored Time per output token ms median,Errored Time per output token ms std dev,"Errored Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Inter token latency ms mean,Errored Inter token latency ms median,Errored Inter token latency ms std dev,"Errored Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Output tokens per second mean,Errored Output tokens per second median,Errored Output tokens per second std dev,"Errored Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Tokens per second mean,Errored Tokens per second median,Errored Tokens per second std dev,"Errored Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Requests,Incomplete Requests per second mean,Incomplete Requests per second median,Incomplete Requests per second std dev,"Incomplete Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Request concurrency mean,Incomplete Request concurrency median,Incomplete Request concurrency std dev,"Incomplete Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Request latency mean,Incomplete Request latency median,Incomplete Request latency std dev,"Incomplete Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Prompt token count mean,Incomplete Prompt token count median,Incomplete Prompt token count std dev,"Incomplete Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Output token count mean,Incomplete Output token count median,Incomplete Output token count std dev,"Incomplete Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Time to first token ms mean,Incomplete Time to first token ms median,Incomplete Time to first token ms std dev,"Incomplete Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Time per output token ms mean,Incomplete Time per output token ms median,Incomplete Time per output token ms std dev,"Incomplete Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Inter token latency ms mean,Incomplete Inter token latency ms median,Incomplete Inter token latency ms std dev,"Incomplete Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Output tokens per second mean,Incomplete Output tokens per second median,Incomplete Output tokens per second std dev,"Incomplete Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Tokens per second mean,Incomplete Tokens per second median,Incomplete Tokens per second std dev,"Incomplete Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Requests,Total Requests per second mean,Total Requests per second median,Total Requests per second std dev,"Total Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Request concurrency mean,Total Request concurrency median,Total Request concurrency std dev,"Total Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Request latency mean,Total Request latency median,Total Request latency std dev,"Total Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Prompt token count mean,Total Prompt token count median,Total Prompt token count std dev,"Total Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Output token count mean,Total Output token count median,Total Output token count std dev,"Total Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Time to first token ms mean,Total Time to first token ms median,Total Time to first token ms std dev,"Total Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Time per output token ms mean,Total Time per output token ms median,Total Time per output token ms std dev,"Total Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Inter token latency ms mean,Total Inter token latency ms median,Total Inter token latency ms std dev,"Total Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Output tokens per second mean,Total Output tokens per second median,Total Output tokens per second std dev,"Total Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Tokens per second mean,Total Tokens per second median,Total Tokens per second std dev,"Total Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Args,Worker,Request Loader,Extras
+generative_benchmark,ec7ffeef-d871-4080-9fd6-8e93d3f7d2f2,64208d5d-b6ef-40da-82c8-c2edd2c9b31c,poisson@2.00,2025-08-15 18:57:45,2025-08-15 19:06:34,528.9625027179718,1000,1.890493172695026,1.1118390881989777,34.27714979674303,"[0.2119836154707914, 0.2119836154707914, 0.21965135989895723, 0.33000270340273113, 0.4362195753958791, 0.6760243750854237, 1.9232903948833362, 3.4319818872568475, 5.617045439327182, 12.466388665146873, 220752.84210526315]",8.854546806018572,9.0,3.279258956044617,"[0.0, 1.0, 2.0, 4.0, 5.0, 7.0, 11.0, 13.0, 15.0, 18.0, 23.0]",4.683723238945007,4.660579204559326,0.16946826229087464,"[4.229826927185059, 4.229826927185059, 4.363725185394287, 4.452978134155273, 4.490201711654663, 4.568145036697388, 4.780122756958008, 4.896669864654541, 4.967177391052246, 5.2661073207855225, 5.348655939102173]",200.05400000000003,200.0,0.25900579144104097,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 201.0, 202.0]",200.00000000000003,200.0,2.8421709430404014e-14,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0]",391.5707309246063,392.99869537353516,25.191047285217405,"[344.41351890563965, 344.41351890563965, 356.3952445983887, 361.386775970459, 364.47596549987793, 371.8833923339844, 405.32469749450684, 414.30211067199707, 426.3617992401123, 469.5141315460205, 635.9431743621826]",21.46033608198166,21.370692253112793,0.8204700770093298,"[19.18872833251953, 19.18872833251953, 19.80484127998352, 20.29750347137451, 20.54035782814026, 20.87454319000244, 21.950119733810425, 22.475367784500122, 22.838857173919678, 24.278961420059204, 24.62005615234375]",21.568176966815734,21.47808266644502,0.824593042220432,"[19.285154103034706, 19.285154103034706, 19.904363095460823, 20.39950097625579, 20.643575706673627, 20.979440391962253, 22.06042184302555, 22.5883093311559, 22.953625300421788, 24.400966251315783, 24.743775027481156]",378.0967440458326,216.25697344676462,1017.2422775814739,"[0.0, 1.744990277189546, 51.33849863523422, 75.01616826441551, 93.05786297479588, 135.874307557744, 382.7618178499726, 705.6366083445491, 1074.0855313700383, 2699.037323037323, 838860.8]",1132.607693213799,441.55216338561956,19987.193802154274,"[0.0, 1.744990277189546, 102.9352835791592, 152.60338366381663, 188.77954811414168, 277.08951575609433, 776.7229629629629, 1439.8571918983866, 2207.5284210526315, 6186.289085545723, 59918628.571428575]",0,0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0,0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",1000,1.890493172695026,1.1118390881989777,34.27714979674303,"[0.2119836154707914, 0.2119836154707914, 0.21965135989895723, 0.33000270340273113, 0.4362195753958791, 0.6760243750854237, 1.9232903948833362, 3.4319818872568475, 5.617045439327182, 12.466388665146873, 220752.84210526315]",8.854546806018572,9.0,3.279258956044617,"[0.0, 1.0, 2.0, 4.0, 5.0, 7.0, 11.0, 13.0, 15.0, 18.0, 23.0]",4.683723238945007,4.660579204559326,0.16946826229087464,"[4.229826927185059, 4.229826927185059, 4.363725185394287, 4.452978134155273, 4.490201711654663, 4.568145036697388, 4.780122756958008, 4.896669864654541, 4.967177391052246, 5.2661073207855225, 5.348655939102173]",200.05400000000003,200.0,0.25900579144104097,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 201.0, 202.0]",200.00000000000003,200.0,2.8421709430404014e-14,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0]",391.5707309246063,392.99869537353516,25.191047285217405,"[344.41351890563965, 344.41351890563965, 356.3952445983887, 361.386775970459, 364.47596549987793, 371.8833923339844, 405.32469749450684, 414.30211067199707, 426.3617992401123, 469.5141315460205, 635.9431743621826]",21.46033608198166,21.370692253112793,0.8204700770093298,"[19.18872833251953, 19.18872833251953, 19.80484127998352, 20.29750347137451, 20.54035782814026, 20.87454319000244, 21.950119733810425, 22.475367784500122, 22.838857173919678, 24.278961420059204, 24.62005615234375]",21.568176966815734,21.47808266644502,0.824593042220432,"[19.285154103034706, 19.285154103034706, 19.904363095460823, 20.39950097625579, 20.643575706673627, 20.979440391962253, 22.06042184302555, 22.5883093311559, 22.953625300421788, 24.400966251315783, 24.743775027481156]",378.0967440458326,216.25697344676462,1017.2422775814739,"[0.0, 1.744990277189546, 51.33849863523422, 75.01616826441551, 93.05786297479588, 135.874307557744, 382.7618178499726, 705.6366083445491, 1074.0855313700383, 2699.037323037323, 838860.8]",1132.607693213799,441.55216338561956,19987.193802154274,"[0.0, 1.744990277189546, 102.9352835791592, 152.60338366381663, 188.77954811414168, 277.08951575609433, 776.7229629629629, 1439.8571918983866, 2207.5284210526315, 6186.289085545723, 59918628.571428575]","{""profile"": {""type_"": ""async"", ""completed_strategies"": 1, ""measured_rates"": [1.890493172695026], ""measured_concurrencies"": [8.854546806018572], ""max_concurrency"": null, ""strategy_type"": ""poisson"", ""rate"": [2.0], ""initial_burst"": true, ""random_seed"": 42, ""strategy_types"": [""poisson""]}, ""strategy_index"": 0, ""strategy"": {""type_"": ""poisson"", ""max_concurrency"": null, ""rate"": 2.0, ""initial_burst"": true, ""random_seed"": 42}, ""max_number"": 1000, ""max_duration"": null, ""warmup_number"": null, ""warmup_duration"": null, ""cooldown_number"": null, ""cooldown_duration"": null}","{""type_"": ""generative_requests_worker"", ""backend_type"": ""openai_http"", ""backend_target"": ""http://127.0.0.1:8000"", ""backend_model"": ""phi-4-mini-instruct"", ""backend_info"": {""max_output_tokens"": 16384, ""timeout"": 300, ""http2"": true, ""authorization"": false, ""organization"": null, ""project"": null, ""text_completions_path"": ""/v1/completions"", ""chat_completions_path"": ""/v1/chat/completions""}}","{""type_"": ""generative_request_loader"", ""data"": ""prompt_tokens=200,output_tokens=200"", ""data_args"": null, ""processor"": ""microsoft/phi-4-mini-instruct"", ""processor_args"": null}",{}
diff --git a/website/docs/benchmarks/a10/benchmark-phi4-p4.csv b/website/docs/benchmarks/a10/benchmark-phi4-p4.csv
new file mode 100644
index 0000000..e48ec47
--- /dev/null
+++ b/website/docs/benchmarks/a10/benchmark-phi4-p4.csv
@@ -0,0 +1,2 @@
+Type,Run Id,Id,Name,Start Time,End Time,Duration,Successful Requests,Successful Requests per second mean,Successful Requests per second median,Successful Requests per second std dev,"Successful Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Request concurrency mean,Successful Request concurrency median,Successful Request concurrency std dev,"Successful Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Request latency mean,Successful Request latency median,Successful Request latency std dev,"Successful Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Prompt token count mean,Successful Prompt token count median,Successful Prompt token count std dev,"Successful Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Output token count mean,Successful Output token count median,Successful Output token count std dev,"Successful Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Time to first token ms mean,Successful Time to first token ms median,Successful Time to first token ms std dev,"Successful Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Time per output token ms mean,Successful Time per output token ms median,Successful Time per output token ms std dev,"Successful Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Inter token latency ms mean,Successful Inter token latency ms median,Successful Inter token latency ms std dev,"Successful Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Output tokens per second mean,Successful Output tokens per second median,Successful Output tokens per second std dev,"Successful Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Tokens per second mean,Successful Tokens per second median,Successful Tokens per second std dev,"Successful Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Requests,Errored Requests per second mean,Errored Requests per second median,Errored Requests per second std dev,"Errored Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Request concurrency mean,Errored Request concurrency median,Errored Request concurrency std dev,"Errored Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Request latency mean,Errored Request latency median,Errored Request latency std dev,"Errored Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Prompt token count mean,Errored Prompt token count median,Errored Prompt token count std dev,"Errored Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Output token count mean,Errored Output token count median,Errored Output token count std dev,"Errored Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Time to first token ms mean,Errored Time to first token ms median,Errored Time to first token ms std dev,"Errored Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Time per output token ms mean,Errored Time per output token ms median,Errored Time per output token ms std dev,"Errored Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Inter token latency ms mean,Errored Inter token latency ms median,Errored Inter token latency ms std dev,"Errored Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Output tokens per second mean,Errored Output tokens per second median,Errored Output tokens per second std dev,"Errored Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Tokens per second mean,Errored Tokens per second median,Errored Tokens per second std dev,"Errored Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Requests,Incomplete Requests per second mean,Incomplete Requests per second median,Incomplete Requests per second std dev,"Incomplete Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Request concurrency mean,Incomplete Request concurrency median,Incomplete Request concurrency std dev,"Incomplete Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Request latency mean,Incomplete Request latency median,Incomplete Request latency std dev,"Incomplete Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Prompt token count mean,Incomplete Prompt token count median,Incomplete Prompt token count std dev,"Incomplete Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Output token count mean,Incomplete Output token count median,Incomplete Output token count std dev,"Incomplete Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Time to first token ms mean,Incomplete Time to first token ms median,Incomplete Time to first token ms std dev,"Incomplete Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Time per output token ms mean,Incomplete Time per output token ms median,Incomplete Time per output token ms std dev,"Incomplete Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Inter token latency ms mean,Incomplete Inter token latency ms median,Incomplete Inter token latency ms std dev,"Incomplete Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Output tokens per second mean,Incomplete Output tokens per second median,Incomplete Output tokens per second std dev,"Incomplete Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Tokens per second mean,Incomplete Tokens per second median,Incomplete Tokens per second std dev,"Incomplete Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Requests,Total Requests per second mean,Total Requests per second median,Total Requests per second std dev,"Total Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Request concurrency mean,Total Request concurrency median,Total Request concurrency std dev,"Total Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Request latency mean,Total Request latency median,Total Request latency std dev,"Total Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Prompt token count mean,Total Prompt token count median,Total Prompt token count std dev,"Total Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Output token count mean,Total Output token count median,Total Output token count std dev,"Total Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Time to first token ms mean,Total Time to first token ms median,Total Time to first token ms std dev,"Total Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Time per output token ms mean,Total Time per output token ms median,Total Time per output token ms std dev,"Total Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Inter token latency ms mean,Total Inter token latency ms median,Total Inter token latency ms std dev,"Total Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Output tokens per second mean,Total Output tokens per second median,Total Output tokens per second std dev,"Total Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Tokens per second mean,Total Tokens per second median,Total Tokens per second std dev,"Total Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Args,Worker,Request Loader,Extras
+generative_benchmark,93801663-d39f-4207-ab89-c61a5e45c225,232c9b4f-928c-416a-9062-b9e0b02351b3,poisson@4.00,2025-08-15 19:07:44,2025-08-15 19:12:11,266.51418232917786,1000,3.752145537849376,2.1577968813503516,46.220638742764535,"[0.1800342340716663, 0.1800342340716663, 0.1800342340716663, 0.6001393928042579, 0.8300544206297705, 1.2717955062237354, 3.758680452872933, 6.703689319146964, 11.090497079502576, 28.489848594969466, 135300.12903225806]",20.280369129898677,20.0,5.515843058291248,"[1.0, 2.0, 9.0, 12.0, 13.0, 16.0, 24.0, 28.0, 30.0, 34.0, 36.0]",5.405005995988846,5.365562438964844,0.29223685071337696,"[4.729373931884766, 4.729373931884766, 4.8349409103393555, 4.964627265930176, 5.041317701339722, 5.184744358062744, 5.600762367248535, 5.82323694229126, 5.943918228149414, 6.101513147354126, 6.126415967941284]",200.05400000000003,200.0,0.25900579144104097,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 201.0, 202.0]",200.00000000000003,200.0,2.8421709430404014e-14,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0]",384.2136287689209,378.85570526123047,21.723545961129528,"[345.1721668243408, 345.1721668243408, 358.6556911468506, 363.4612560272217, 366.4844036102295, 371.537446975708, 388.7045383453369, 408.65206718444824, 426.206111907959, 471.36497497558594, 516.7863368988037]",25.103554602861404,24.93818759918213,1.4490794975788281,"[21.6083824634552, 21.6083824634552, 22.30905771255493, 22.84892201423645, 23.29678177833557, 24.051921367645264, 26.064733266830444, 27.17718482017517, 27.750428915023804, 28.5881769657135, 28.77230167388916]",25.22970311845367,25.06350512480616,1.4563613040993246,"[21.716967299954977, 21.716967299954977, 22.421163530205963, 22.963740717825576, 23.413851033503086, 24.172785294115844, 26.195711825960245, 27.31375358811575, 27.889878306556586, 28.73183614644573, 28.916886104411216]",750.4253554243376,432.3133374561946,1882.9303016050042,"[0.0, 0.0, 95.25148748694191, 145.3680379856514, 180.77338160503405, 266.88114023924663, 768.3282652500458, 1414.6050590219224, 2147.621095750128, 5262.614805520702, 838860.8]",2247.9366567443267,872.5408778864156,35360.1556119851,"[0.0, 0.0, 187.74861235452104, 296.0197614510551, 366.73113578735683, 536.2188698542573, 1554.5974796145292, 2886.6510667584307, 4415.056842105263, 12264.04678362573, 41943040.0]",0,0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0,0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",1000,3.752145537849376,2.1577968813503516,46.220638742764535,"[0.1800342340716663, 0.1800342340716663, 0.1800342340716663, 0.6001393928042579, 0.8300544206297705, 1.2717955062237354, 3.758680452872933, 6.703689319146964, 11.090497079502576, 28.489848594969466, 135300.12903225806]",20.280369129898677,20.0,5.515843058291248,"[1.0, 2.0, 9.0, 12.0, 13.0, 16.0, 24.0, 28.0, 30.0, 34.0, 36.0]",5.405005995988846,5.365562438964844,0.29223685071337696,"[4.729373931884766, 4.729373931884766, 4.8349409103393555, 4.964627265930176, 5.041317701339722, 5.184744358062744, 5.600762367248535, 5.82323694229126, 5.943918228149414, 6.101513147354126, 6.126415967941284]",200.05400000000003,200.0,0.25900579144104097,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 201.0, 202.0]",200.00000000000003,200.0,2.8421709430404014e-14,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0]",384.2136287689209,378.85570526123047,21.723545961129528,"[345.1721668243408, 345.1721668243408, 358.6556911468506, 363.4612560272217, 366.4844036102295, 371.537446975708, 388.7045383453369, 408.65206718444824, 426.206111907959, 471.36497497558594, 516.7863368988037]",25.103554602861404,24.93818759918213,1.4490794975788281,"[21.6083824634552, 21.6083824634552, 22.30905771255493, 22.84892201423645, 23.29678177833557, 24.051921367645264, 26.064733266830444, 27.17718482017517, 27.750428915023804, 28.5881769657135, 28.77230167388916]",25.22970311845367,25.06350512480616,1.4563613040993246,"[21.716967299954977, 21.716967299954977, 22.421163530205963, 22.963740717825576, 23.413851033503086, 24.172785294115844, 26.195711825960245, 27.31375358811575, 27.889878306556586, 28.73183614644573, 28.916886104411216]",750.4253554243376,432.3133374561946,1882.9303016050042,"[0.0, 0.0, 95.25148748694191, 145.3680379856514, 180.77338160503405, 266.88114023924663, 768.3282652500458, 1414.6050590219224, 2147.621095750128, 5262.614805520702, 838860.8]",2247.9366567443267,872.5408778864156,35360.1556119851,"[0.0, 0.0, 187.74861235452104, 296.0197614510551, 366.73113578735683, 536.2188698542573, 1554.5974796145292, 2886.6510667584307, 4415.056842105263, 12264.04678362573, 41943040.0]","{""profile"": {""type_"": ""async"", ""completed_strategies"": 1, ""measured_rates"": [3.752145537849376], ""measured_concurrencies"": [20.280369129898677], ""max_concurrency"": null, ""strategy_type"": ""poisson"", ""rate"": [4.0], ""initial_burst"": true, ""random_seed"": 42, ""strategy_types"": [""poisson""]}, ""strategy_index"": 0, ""strategy"": {""type_"": ""poisson"", ""max_concurrency"": null, ""rate"": 4.0, ""initial_burst"": true, ""random_seed"": 42}, ""max_number"": 1000, ""max_duration"": null, ""warmup_number"": null, ""warmup_duration"": null, ""cooldown_number"": null, ""cooldown_duration"": null}","{""type_"": ""generative_requests_worker"", ""backend_type"": ""openai_http"", ""backend_target"": ""http://127.0.0.1:8000"", ""backend_model"": ""phi-4-mini-instruct"", ""backend_info"": {""max_output_tokens"": 16384, ""timeout"": 300, ""http2"": true, ""authorization"": false, ""organization"": null, ""project"": null, ""text_completions_path"": ""/v1/completions"", ""chat_completions_path"": ""/v1/chat/completions""}}","{""type_"": ""generative_request_loader"", ""data"": ""prompt_tokens=200,output_tokens=200"", ""data_args"": null, ""processor"": ""microsoft/phi-4-mini-instruct"", ""processor_args"": null}",{}
diff --git a/website/docs/benchmarks/a10/benchmark-phi4-p8.csv b/website/docs/benchmarks/a10/benchmark-phi4-p8.csv
new file mode 100644
index 0000000..83227db
--- /dev/null
+++ b/website/docs/benchmarks/a10/benchmark-phi4-p8.csv
@@ -0,0 +1,2 @@
+Type,Run Id,Id,Name,Start Time,End Time,Duration,Successful Requests,Successful Requests per second mean,Successful Requests per second median,Successful Requests per second std dev,"Successful Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Request concurrency mean,Successful Request concurrency median,Successful Request concurrency std dev,"Successful Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Request latency mean,Successful Request latency median,Successful Request latency std dev,"Successful Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Prompt token count mean,Successful Prompt token count median,Successful Prompt token count std dev,"Successful Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Output token count mean,Successful Output token count median,Successful Output token count std dev,"Successful Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Time to first token ms mean,Successful Time to first token ms median,Successful Time to first token ms std dev,"Successful Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Time per output token ms mean,Successful Time per output token ms median,Successful Time per output token ms std dev,"Successful Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Inter token latency ms mean,Successful Inter token latency ms median,Successful Inter token latency ms std dev,"Successful Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Output tokens per second mean,Successful Output tokens per second median,Successful Output tokens per second std dev,"Successful Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Tokens per second mean,Successful Tokens per second median,Successful Tokens per second std dev,"Successful Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Requests,Errored Requests per second mean,Errored Requests per second median,Errored Requests per second std dev,"Errored Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Request concurrency mean,Errored Request concurrency median,Errored Request concurrency std dev,"Errored Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Request latency mean,Errored Request latency median,Errored Request latency std dev,"Errored Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Prompt token count mean,Errored Prompt token count median,Errored Prompt token count std dev,"Errored Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Output token count mean,Errored Output token count median,Errored Output token count std dev,"Errored Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Time to first token ms mean,Errored Time to first token ms median,Errored Time to first token ms std dev,"Errored Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Time per output token ms mean,Errored Time per output token ms median,Errored Time per output token ms std dev,"Errored Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Inter token latency ms mean,Errored Inter token latency ms median,Errored Inter token latency ms std dev,"Errored Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Output tokens per second mean,Errored Output tokens per second median,Errored Output tokens per second std dev,"Errored Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Tokens per second mean,Errored Tokens per second median,Errored Tokens per second std dev,"Errored Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Requests,Incomplete Requests per second mean,Incomplete Requests per second median,Incomplete Requests per second std dev,"Incomplete Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Request concurrency mean,Incomplete Request concurrency median,Incomplete Request concurrency std dev,"Incomplete Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Request latency mean,Incomplete Request latency median,Incomplete Request latency std dev,"Incomplete Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Prompt token count mean,Incomplete Prompt token count median,Incomplete Prompt token count std dev,"Incomplete Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Output token count mean,Incomplete Output token count median,Incomplete Output token count std dev,"Incomplete Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Time to first token ms mean,Incomplete Time to first token ms median,Incomplete Time to first token ms std dev,"Incomplete Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Time per output token ms mean,Incomplete Time per output token ms median,Incomplete Time per output token ms std dev,"Incomplete Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Inter token latency ms mean,Incomplete Inter token latency ms median,Incomplete Inter token latency ms std dev,"Incomplete Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Output tokens per second mean,Incomplete Output tokens per second median,Incomplete Output tokens per second std dev,"Incomplete Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Tokens per second mean,Incomplete Tokens per second median,Incomplete Tokens per second std dev,"Incomplete Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Requests,Total Requests per second mean,Total Requests per second median,Total Requests per second std dev,"Total Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Request concurrency mean,Total Request concurrency median,Total Request concurrency std dev,"Total Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Request latency mean,Total Request latency median,Total Request latency std dev,"Total Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Prompt token count mean,Total Prompt token count median,Total Prompt token count std dev,"Total Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Output token count mean,Total Output token count median,Total Output token count std dev,"Total Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Time to first token ms mean,Total Time to first token ms median,Total Time to first token ms std dev,"Total Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Time per output token ms mean,Total Time per output token ms median,Total Time per output token ms std dev,"Total Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Inter token latency ms mean,Total Inter token latency ms median,Total Inter token latency ms std dev,"Total Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Output tokens per second mean,Total Output tokens per second median,Total Output tokens per second std dev,"Total Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Tokens per second mean,Total Tokens per second median,Total Tokens per second std dev,"Total Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Args,Worker,Request Loader,Extras
+generative_benchmark,1237a03f-e66c-4c50-9db9-534475b15bca,f64faad6-4b2b-4250-9f28-4007757c3ebf,poisson@8.00,2025-08-15 19:14:42,2025-08-15 19:16:58,136.48799967765808,1000,7.326651444534955,3.685502018795225,120.94244360864955,"[0.12111009307851699, 0.12111009307851699, 0.12111009307851699, 0.12111009307851699, 1.1124052574710608, 2.022811649084496, 6.840193416342539, 14.467457478622897, 20.364356704845044, 27.641750912757516, 599186.2857142857]",84.86652048966091,81.0,22.086866333957218,"[1.0, 6.0, 19.0, 44.0, 65.0, 72.0, 103.0, 115.0, 120.0, 123.0, 127.0]",11.583261621236801,11.198832273483276,1.7665856271835376,"[6.4426586627960205, 6.4426586627960205, 7.096365690231323, 8.378738403320312, 9.88152265548706, 10.358792543411255, 13.348541021347046, 13.960595607757568, 14.117420196533203, 14.27633547782898, 14.33666706085205]",200.05400000000003,200.0,0.25900579144104097,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 201.0, 202.0]",200.00000000000003,200.0,2.8421709430404014e-14,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0]",484.6444962024689,478.41930389404297,37.86530087357088,"[389.13750648498535, 389.13750648498535, 416.0499572753906, 437.30926513671875, 444.71120834350586, 460.88719367980957, 502.1486282348633, 529.4029712677002, 549.0798950195312, 627.3975372314453, 690.3095245361328]",55.49226576089859,53.54759335517883,8.778863844532966,"[29.721224308013916, 29.721224308013916, 33.06289076805115, 39.02484893798828, 47.07970976829529, 49.44548964500427, 64.41624641418457, 67.24563002586365, 67.98963904380798, 68.99959087371826, 69.14482355117798]",55.77112136773728,53.8166767388732,8.822978738224087,"[29.870577193983834, 29.870577193983834, 33.229035947790095, 39.22095370652089, 47.31629122441738, 49.69395944221535, 64.73994614490911, 67.58354776468708, 68.33129552141506, 69.346322486149, 69.49228497605827]",1465.3229622555461,834.8535031847134,3416.798664364896,"[0.0, 0.0, 102.4950882166072, 242.96495394774954, 319.9316552250191, 495.60486825002954, 1522.433393829401, 2837.8240866035185, 4310.692702980473, 10485.76, 838860.8]",4389.448166981003,1715.4617586912066,66139.11911008079,"[0.0, 0.0, 183.80752881370788, 531.3280972890803, 680.8935064935065, 1031.0481809242872, 3086.316409124356, 5737.761969904241, 8793.090146750525, 23696.632768361582, 167772160.0]",0,0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0,0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",1000,7.326651444534955,3.685502018795225,120.94244360864955,"[0.12111009307851699, 0.12111009307851699, 0.12111009307851699, 0.12111009307851699, 1.1124052574710608, 2.022811649084496, 6.840193416342539, 14.467457478622897, 20.364356704845044, 27.641750912757516, 599186.2857142857]",84.86652048966091,81.0,22.086866333957218,"[1.0, 6.0, 19.0, 44.0, 65.0, 72.0, 103.0, 115.0, 120.0, 123.0, 127.0]",11.583261621236801,11.198832273483276,1.7665856271835376,"[6.4426586627960205, 6.4426586627960205, 7.096365690231323, 8.378738403320312, 9.88152265548706, 10.358792543411255, 13.348541021347046, 13.960595607757568, 14.117420196533203, 14.27633547782898, 14.33666706085205]",200.05400000000003,200.0,0.25900579144104097,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 201.0, 202.0]",200.00000000000003,200.0,2.8421709430404014e-14,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0]",484.6444962024689,478.41930389404297,37.86530087357088,"[389.13750648498535, 389.13750648498535, 416.0499572753906, 437.30926513671875, 444.71120834350586, 460.88719367980957, 502.1486282348633, 529.4029712677002, 549.0798950195312, 627.3975372314453, 690.3095245361328]",55.49226576089859,53.54759335517883,8.778863844532966,"[29.721224308013916, 29.721224308013916, 33.06289076805115, 39.02484893798828, 47.07970976829529, 49.44548964500427, 64.41624641418457, 67.24563002586365, 67.98963904380798, 68.99959087371826, 69.14482355117798]",55.77112136773728,53.8166767388732,8.822978738224087,"[29.870577193983834, 29.870577193983834, 33.229035947790095, 39.22095370652089, 47.31629122441738, 49.69395944221535, 64.73994614490911, 67.58354776468708, 68.33129552141506, 69.346322486149, 69.49228497605827]",1465.3229622555461,834.8535031847134,3416.798664364896,"[0.0, 0.0, 102.4950882166072, 242.96495394774954, 319.9316552250191, 495.60486825002954, 1522.433393829401, 2837.8240866035185, 4310.692702980473, 10485.76, 838860.8]",4389.448166981003,1715.4617586912066,66139.11911008079,"[0.0, 0.0, 183.80752881370788, 531.3280972890803, 680.8935064935065, 1031.0481809242872, 3086.316409124356, 5737.761969904241, 8793.090146750525, 23696.632768361582, 167772160.0]","{""profile"": {""type_"": ""async"", ""completed_strategies"": 1, ""measured_rates"": [7.326651444534955], ""measured_concurrencies"": [84.86652048966091], ""max_concurrency"": null, ""strategy_type"": ""poisson"", ""rate"": [8.0], ""initial_burst"": true, ""random_seed"": 42, ""strategy_types"": [""poisson""]}, ""strategy_index"": 0, ""strategy"": {""type_"": ""poisson"", ""max_concurrency"": null, ""rate"": 8.0, ""initial_burst"": true, ""random_seed"": 42}, ""max_number"": 1000, ""max_duration"": null, ""warmup_number"": null, ""warmup_duration"": null, ""cooldown_number"": null, ""cooldown_duration"": null}","{""type_"": ""generative_requests_worker"", ""backend_type"": ""openai_http"", ""backend_target"": ""http://127.0.0.1:8000"", ""backend_model"": ""phi-4-mini-instruct"", ""backend_info"": {""max_output_tokens"": 16384, ""timeout"": 300, ""http2"": true, ""authorization"": false, ""organization"": null, ""project"": null, ""text_completions_path"": ""/v1/completions"", ""chat_completions_path"": ""/v1/chat/completions""}}","{""type_"": ""generative_request_loader"", ""data"": ""prompt_tokens=200,output_tokens=200"", ""data_args"": null, ""processor"": ""microsoft/phi-4-mini-instruct"", ""processor_args"": null}",{}
diff --git a/website/docs/benchmarks/a100/benchmark-phi4-p1.csv b/website/docs/benchmarks/a100/benchmark-phi4-p1.csv
new file mode 100644
index 0000000..0d6eca9
--- /dev/null
+++ b/website/docs/benchmarks/a100/benchmark-phi4-p1.csv
@@ -0,0 +1,28 @@
+A100,TTFT,,,ITL,,
+,mean,median,p99,mean,median,p99
+1,288.7,286.6,332.3,8,8,8.3
+2,284.6,282,340.5,8.1,8.1,8.4
+4,283.7,282.9,304.6,8.4,8.4,9.2
+8,287.7,285.1,340.4,9.2,9.1,9.9
+16,292.8,290.6,345.6,10.7,10.7,12.1
+,,,,,,
+,,,,,,
+,,,,,,
+32,342.9,334.7,527.2,25.6,25.1,32.6
+64,4716.8,3953.2,8046.3,37.6,39.6,39.9
+,,,,,,
+A10,,,,,,
+,mean,median,p99,mean,median,p99
+1,369.7,367.7,424.2,20.2,20.2,21.7
+2,391.6,393,469.5,21.6,21.5,24.4
+4,384.2,378.9,471.4,25.2,25.1,28.7
+8,484.6,478.4,627.4,55.8,53.8,69.3
+16,14181.4,12345.4,29246.6,130.2,140.6,129.5
+,,,,,,
+,TTFT,,ITL,,,
+,Standard_NV36ads_A10_v5,Standard_NC24ads_A100_v4,Standard_NV36ads_A10_v5,Standard_NC24ads_A100_v4,,
+1,369.7,288.7,20.2,8,,
+2,391.6,284.6,21.6,8.1,,
+4,384.2,283.7,25.2,8.4,,
+8,484.6,287.7,55.8,9.2,,
+16,14181.4,292.8,130.2,10.7,,
diff --git a/website/docs/benchmarks/a100/benchmark-phi4-p16.csv b/website/docs/benchmarks/a100/benchmark-phi4-p16.csv
new file mode 100644
index 0000000..2812a12
--- /dev/null
+++ b/website/docs/benchmarks/a100/benchmark-phi4-p16.csv
@@ -0,0 +1,2 @@
+Type,Run Id,Id,Name,Start Time,End Time,Duration,Successful Requests,Successful Requests per second mean,Successful Requests per second median,Successful Requests per second std dev,"Successful Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Request concurrency mean,Successful Request concurrency median,Successful Request concurrency std dev,"Successful Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Request latency mean,Successful Request latency median,Successful Request latency std dev,"Successful Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Prompt token count mean,Successful Prompt token count median,Successful Prompt token count std dev,"Successful Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Output token count mean,Successful Output token count median,Successful Output token count std dev,"Successful Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Time to first token ms mean,Successful Time to first token ms median,Successful Time to first token ms std dev,"Successful Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Time per output token ms mean,Successful Time per output token ms median,Successful Time per output token ms std dev,"Successful Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Inter token latency ms mean,Successful Inter token latency ms median,Successful Inter token latency ms std dev,"Successful Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Output tokens per second mean,Successful Output tokens per second median,Successful Output tokens per second std dev,"Successful Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Tokens per second mean,Successful Tokens per second median,Successful Tokens per second std dev,"Successful Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Requests,Errored Requests per second mean,Errored Requests per second median,Errored Requests per second std dev,"Errored Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Request concurrency mean,Errored Request concurrency median,Errored Request concurrency std dev,"Errored Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Request latency mean,Errored Request latency median,Errored Request latency std dev,"Errored Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Prompt token count mean,Errored Prompt token count median,Errored Prompt token count std dev,"Errored Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Output token count mean,Errored Output token count median,Errored Output token count std dev,"Errored Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Time to first token ms mean,Errored Time to first token ms median,Errored Time to first token ms std dev,"Errored Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Time per output token ms mean,Errored Time per output token ms median,Errored Time per output token ms std dev,"Errored Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Inter token latency ms mean,Errored Inter token latency ms median,Errored Inter token latency ms std dev,"Errored Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Output tokens per second mean,Errored Output tokens per second median,Errored Output tokens per second std dev,"Errored Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Tokens per second mean,Errored Tokens per second median,Errored Tokens per second std dev,"Errored Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Requests,Incomplete Requests per second mean,Incomplete Requests per second median,Incomplete Requests per second std dev,"Incomplete Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Request concurrency mean,Incomplete Request concurrency median,Incomplete Request concurrency std dev,"Incomplete Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Request latency mean,Incomplete Request latency median,Incomplete Request latency std dev,"Incomplete Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Prompt token count mean,Incomplete Prompt token count median,Incomplete Prompt token count std dev,"Incomplete Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Output token count mean,Incomplete Output token count median,Incomplete Output token count std dev,"Incomplete Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Time to first token ms mean,Incomplete Time to first token ms median,Incomplete Time to first token ms std dev,"Incomplete Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Time per output token ms mean,Incomplete Time per output token ms median,Incomplete Time per output token ms std dev,"Incomplete Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Inter token latency ms mean,Incomplete Inter token latency ms median,Incomplete Inter token latency ms std dev,"Incomplete Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Output tokens per second mean,Incomplete Output tokens per second median,Incomplete Output tokens per second std dev,"Incomplete Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Tokens per second mean,Incomplete Tokens per second median,Incomplete Tokens per second std dev,"Incomplete Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Requests,Total Requests per second mean,Total Requests per second median,Total Requests per second std dev,"Total Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Request concurrency mean,Total Request concurrency median,Total Request concurrency std dev,"Total Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Request latency mean,Total Request latency median,Total Request latency std dev,"Total Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Prompt token count mean,Total Prompt token count median,Total Prompt token count std dev,"Total Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Output token count mean,Total Output token count median,Total Output token count std dev,"Total Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Time to first token ms mean,Total Time to first token ms median,Total Time to first token ms std dev,"Total Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Time per output token ms mean,Total Time per output token ms median,Total Time per output token ms std dev,"Total Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Inter token latency ms mean,Total Inter token latency ms median,Total Inter token latency ms std dev,"Total Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Output tokens per second mean,Total Output tokens per second median,Total Output tokens per second std dev,"Total Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Tokens per second mean,Total Tokens per second median,Total Tokens per second std dev,"Total Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Args,Worker,Request Loader,Extras
+generative_benchmark,c1884b74-2310-4322-ae23-12201ec1257b,4df9a01e-6d05-469d-8c8a-7d7405258091,poisson@16.00,2025-08-15 17:03:29,2025-08-15 17:04:35,66.52685260772705,1000,15.03152427631682,8.828143614267132,110.51736154975002,"[0.383744431923283, 0.383744431923283, 0.383744431923283, 2.0943100213010943, 3.098727279309257, 5.0812147996312325, 15.448918027956315, 27.475706658805805, 41.56191721910085, 90.531059788474, 190650.18181818182]",36.50362227744675,35.0,8.587002732895554,"[1.0, 3.0, 12.0, 26.0, 28.0, 31.0, 42.0, 47.0, 50.0, 61.0, 73.0]",2.4284710988998417,2.41560959815979,0.12225364284040219,"[2.137021541595459, 2.137021541595459, 2.2211432456970215, 2.255568027496338, 2.276679039001465, 2.3321940898895264, 2.5178065299987793, 2.598365068435669, 2.6484103202819824, 2.699416399002075, 2.7175817489624023]",200.05400000000003,200.0,0.25900579144104097,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 201.0, 202.0]",200.00000000000003,200.0,2.8421709430404014e-14,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0]",292.7789876461029,290.57836532592773,12.079789751296756,"[274.7788429260254, 274.7788429260254, 277.91905403137207, 280.3952693939209, 282.1223735809326, 285.9795093536377, 295.7479953765869, 303.295373916626, 312.1519088745117, 345.6146717071533, 380.60641288757324]",10.678066830635071,10.634708404541016,0.5854422815190098,"[9.22131896018982, 9.22131896018982, 9.634584188461304, 9.839798212051392, 9.950255155563354, 10.213592052459717, 11.124186515808105, 11.494894027709961, 11.73763394355774, 12.023042440414429, 12.138100862503052]",10.731725457924695,10.688149150292478,0.5883842025316681,"[9.26765724642193, 9.26765724642193, 9.68299918438322, 9.889244434222505, 10.000256437752116, 10.264916635637906, 11.18008695056091, 11.552657314281367, 11.796617028701245, 12.083459739109978, 12.199096344224172]",3006.2898237390878,1716.163666121113,6503.616715463182,"[0.0, 0.0, 278.4877498174092, 510.3801411535653, 660.5203149606299, 1015.5699757869249, 3125.412816691505, 5849.796373779637, 8867.450317124736, 21399.51020408163, 1677721.6]",9005.49141461134,3557.509754028838,128070.78117045686,"[0.0, 0.0, 673.675554127851, 1165.7320733740967, 1463.4696441032797, 2172.089073019161, 6345.391830559758, 11650.844444444445, 17772.474576271186, 48210.3908045977, 139810133.33333334]",0,0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0,0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",1000,15.03152427631682,8.828143614267132,110.51736154975002,"[0.383744431923283, 0.383744431923283, 0.383744431923283, 2.0943100213010943, 3.098727279309257, 5.0812147996312325, 15.448918027956315, 27.475706658805805, 41.56191721910085, 90.531059788474, 190650.18181818182]",36.50362227744675,35.0,8.587002732895554,"[1.0, 3.0, 12.0, 26.0, 28.0, 31.0, 42.0, 47.0, 50.0, 61.0, 73.0]",2.4284710988998417,2.41560959815979,0.12225364284040219,"[2.137021541595459, 2.137021541595459, 2.2211432456970215, 2.255568027496338, 2.276679039001465, 2.3321940898895264, 2.5178065299987793, 2.598365068435669, 2.6484103202819824, 2.699416399002075, 2.7175817489624023]",200.05400000000003,200.0,0.25900579144104097,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 201.0, 202.0]",200.00000000000003,200.0,2.8421709430404014e-14,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0]",292.7789876461029,290.57836532592773,12.079789751296756,"[274.7788429260254, 274.7788429260254, 277.91905403137207, 280.3952693939209, 282.1223735809326, 285.9795093536377, 295.7479953765869, 303.295373916626, 312.1519088745117, 345.6146717071533, 380.60641288757324]",10.678066830635071,10.634708404541016,0.5854422815190098,"[9.22131896018982, 9.22131896018982, 9.634584188461304, 9.839798212051392, 9.950255155563354, 10.213592052459717, 11.124186515808105, 11.494894027709961, 11.73763394355774, 12.023042440414429, 12.138100862503052]",10.731725457924695,10.688149150292478,0.5883842025316681,"[9.26765724642193, 9.26765724642193, 9.68299918438322, 9.889244434222505, 10.000256437752116, 10.264916635637906, 11.18008695056091, 11.552657314281367, 11.796617028701245, 12.083459739109978, 12.199096344224172]",3006.2898237390878,1716.163666121113,6503.616715463182,"[0.0, 0.0, 278.4877498174092, 510.3801411535653, 660.5203149606299, 1015.5699757869249, 3125.412816691505, 5849.796373779637, 8867.450317124736, 21399.51020408163, 1677721.6]",9005.49141461134,3557.509754028838,128070.78117045686,"[0.0, 0.0, 673.675554127851, 1165.7320733740967, 1463.4696441032797, 2172.089073019161, 6345.391830559758, 11650.844444444445, 17772.474576271186, 48210.3908045977, 139810133.33333334]","{""profile"": {""type_"": ""async"", ""completed_strategies"": 1, ""measured_rates"": [15.03152427631682], ""measured_concurrencies"": [36.50362227744675], ""max_concurrency"": null, ""strategy_type"": ""poisson"", ""rate"": [16.0], ""initial_burst"": true, ""random_seed"": 42, ""strategy_types"": [""poisson""]}, ""strategy_index"": 0, ""strategy"": {""type_"": ""poisson"", ""max_concurrency"": null, ""rate"": 16.0, ""initial_burst"": true, ""random_seed"": 42}, ""max_number"": 1000, ""max_duration"": null, ""warmup_number"": null, ""warmup_duration"": null, ""cooldown_number"": null, ""cooldown_duration"": null}","{""type_"": ""generative_requests_worker"", ""backend_type"": ""openai_http"", ""backend_target"": ""http://127.0.0.1:8000"", ""backend_model"": ""phi-4-mini-instruct"", ""backend_info"": {""max_output_tokens"": 16384, ""timeout"": 300, ""http2"": true, ""authorization"": false, ""organization"": null, ""project"": null, ""text_completions_path"": ""/v1/completions"", ""chat_completions_path"": ""/v1/chat/completions""}}","{""type_"": ""generative_request_loader"", ""data"": ""prompt_tokens=200,output_tokens=200"", ""data_args"": null, ""processor"": ""microsoft/phi-4-mini-instruct"", ""processor_args"": null}",{}
diff --git a/website/docs/benchmarks/a100/benchmark-phi4-p2.csv b/website/docs/benchmarks/a100/benchmark-phi4-p2.csv
new file mode 100644
index 0000000..b676c26
--- /dev/null
+++ b/website/docs/benchmarks/a100/benchmark-phi4-p2.csv
@@ -0,0 +1,2 @@
+Type,Run Id,Id,Name,Start Time,End Time,Duration,Successful Requests,Successful Requests per second mean,Successful Requests per second median,Successful Requests per second std dev,"Successful Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Request concurrency mean,Successful Request concurrency median,Successful Request concurrency std dev,"Successful Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Request latency mean,Successful Request latency median,Successful Request latency std dev,"Successful Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Prompt token count mean,Successful Prompt token count median,Successful Prompt token count std dev,"Successful Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Output token count mean,Successful Output token count median,Successful Output token count std dev,"Successful Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Time to first token ms mean,Successful Time to first token ms median,Successful Time to first token ms std dev,"Successful Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Time per output token ms mean,Successful Time per output token ms median,Successful Time per output token ms std dev,"Successful Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Inter token latency ms mean,Successful Inter token latency ms median,Successful Inter token latency ms std dev,"Successful Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Output tokens per second mean,Successful Output tokens per second median,Successful Output tokens per second std dev,"Successful Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Tokens per second mean,Successful Tokens per second median,Successful Tokens per second std dev,"Successful Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Requests,Errored Requests per second mean,Errored Requests per second median,Errored Requests per second std dev,"Errored Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Request concurrency mean,Errored Request concurrency median,Errored Request concurrency std dev,"Errored Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Request latency mean,Errored Request latency median,Errored Request latency std dev,"Errored Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Prompt token count mean,Errored Prompt token count median,Errored Prompt token count std dev,"Errored Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Output token count mean,Errored Output token count median,Errored Output token count std dev,"Errored Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Time to first token ms mean,Errored Time to first token ms median,Errored Time to first token ms std dev,"Errored Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Time per output token ms mean,Errored Time per output token ms median,Errored Time per output token ms std dev,"Errored Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Inter token latency ms mean,Errored Inter token latency ms median,Errored Inter token latency ms std dev,"Errored Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Output tokens per second mean,Errored Output tokens per second median,Errored Output tokens per second std dev,"Errored Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Tokens per second mean,Errored Tokens per second median,Errored Tokens per second std dev,"Errored Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Requests,Incomplete Requests per second mean,Incomplete Requests per second median,Incomplete Requests per second std dev,"Incomplete Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Request concurrency mean,Incomplete Request concurrency median,Incomplete Request concurrency std dev,"Incomplete Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Request latency mean,Incomplete Request latency median,Incomplete Request latency std dev,"Incomplete Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Prompt token count mean,Incomplete Prompt token count median,Incomplete Prompt token count std dev,"Incomplete Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Output token count mean,Incomplete Output token count median,Incomplete Output token count std dev,"Incomplete Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Time to first token ms mean,Incomplete Time to first token ms median,Incomplete Time to first token ms std dev,"Incomplete Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Time per output token ms mean,Incomplete Time per output token ms median,Incomplete Time per output token ms std dev,"Incomplete Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Inter token latency ms mean,Incomplete Inter token latency ms median,Incomplete Inter token latency ms std dev,"Incomplete Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Output tokens per second mean,Incomplete Output tokens per second median,Incomplete Output tokens per second std dev,"Incomplete Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Tokens per second mean,Incomplete Tokens per second median,Incomplete Tokens per second std dev,"Incomplete Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Requests,Total Requests per second mean,Total Requests per second median,Total Requests per second std dev,"Total Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Request concurrency mean,Total Request concurrency median,Total Request concurrency std dev,"Total Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Request latency mean,Total Request latency median,Total Request latency std dev,"Total Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Prompt token count mean,Total Prompt token count median,Total Prompt token count std dev,"Total Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Output token count mean,Total Output token count median,Total Output token count std dev,"Total Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Time to first token ms mean,Total Time to first token ms median,Total Time to first token ms std dev,"Total Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Time per output token ms mean,Total Time per output token ms median,Total Time per output token ms std dev,"Total Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Inter token latency ms mean,Total Inter token latency ms median,Total Inter token latency ms std dev,"Total Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Output tokens per second mean,Total Output tokens per second median,Total Output tokens per second std dev,"Total Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Tokens per second mean,Total Tokens per second median,Total Tokens per second std dev,"Total Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Args,Worker,Request Loader,Extras
+generative_benchmark,832f5b61-91d5-4886-96f6-8f250befd70a,7bded625-ade1-44ba-a34a-503d486ff020,poisson@2.00,2025-08-15 16:18:53,2025-08-15 16:27:39,526.5340964794159,1000,1.8992122384596486,1.1233681675427447,30.055049784655754,"[0.21437513665807764, 0.21437513665807764, 0.2555707366271215, 0.35237312483496835, 0.4579898476447881, 0.6902254441379192, 1.9930869510260756, 3.5679837078990997, 5.637573572798596, 12.473061745209936, 167772.16]",3.6010728624896617,3.0,1.9535525440848045,"[0.0, 0.0, 0.0, 1.0, 1.0, 2.0, 5.0, 6.0, 7.0, 9.0, 13.0]",1.8960876460075378,1.8963055610656738,0.02959840189219553,"[1.8172087669372559, 1.8172087669372559, 1.8322250843048096, 1.8476104736328125, 1.8578884601593018, 1.875415325164795, 1.9156973361968994, 1.9331169128417969, 1.9431052207946777, 1.9807205200195312, 2.0245254039764404]",200.05400000000003,200.0,0.25900579144104097,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 201.0, 202.0]",200.00000000000003,200.0,2.8421709430404014e-14,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0]",284.62650084495544,281.9640636444092,11.082842997136451,"[270.9815502166748, 270.9815502166748, 272.72987365722656, 274.83654022216797, 276.2734889984131, 278.81717681884766, 286.7157459259033, 293.8394546508789, 300.47130584716797, 340.4512405395508, 373.8696575164795]",8.056834528446197,8.050402402877808,0.14267749122340218,"[7.667185068130493, 7.667185068130493, 7.74356484413147, 7.829161882400513, 7.879680395126343, 7.954884767532349, 8.153313398361206, 8.230262994766235, 8.298704624176025, 8.403304815292358, 8.717594146728516]",8.09732113411678,8.090856686309355,0.14339446354110771,"[7.705713636312053, 7.705713636312053, 7.782477230282884, 7.868504404422626, 7.91927677902145, 7.994859062846581, 8.194284822473573, 8.271621100267573, 8.340406657463342, 8.445532477680763, 8.76140115249097]",379.8405484796913,207.0238894373149,1151.9039548487194,"[0.0, 0.3242626516469362, 0.4282604451625393, 2.367892040078495, 126.12551496015637, 137.1225317117824, 363.1746471555979, 713.5597141884995, 1106.384595093643, 2807.4323962516733, 838860.8]",1137.8313465468448,431.7792876261066,20753.336374994135,"[0.0, 0.3242626516469362, 0.4282604451625393, 2.367892040078495, 253.00422246350584, 278.09998673915925, 773.1435944700461, 1441.3415807560139, 2241.7445216461783, 6213.783703703703, 167772160.0]",0,0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0,0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",1000,1.8992122384596486,1.1233681675427447,30.055049784655754,"[0.21437513665807764, 0.21437513665807764, 0.2555707366271215, 0.35237312483496835, 0.4579898476447881, 0.6902254441379192, 1.9930869510260756, 3.5679837078990997, 5.637573572798596, 12.473061745209936, 167772.16]",3.6010728624896617,3.0,1.9535525440848045,"[0.0, 0.0, 0.0, 1.0, 1.0, 2.0, 5.0, 6.0, 7.0, 9.0, 13.0]",1.8960876460075378,1.8963055610656738,0.02959840189219553,"[1.8172087669372559, 1.8172087669372559, 1.8322250843048096, 1.8476104736328125, 1.8578884601593018, 1.875415325164795, 1.9156973361968994, 1.9331169128417969, 1.9431052207946777, 1.9807205200195312, 2.0245254039764404]",200.05400000000003,200.0,0.25900579144104097,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 201.0, 202.0]",200.00000000000003,200.0,2.8421709430404014e-14,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0]",284.62650084495544,281.9640636444092,11.082842997136451,"[270.9815502166748, 270.9815502166748, 272.72987365722656, 274.83654022216797, 276.2734889984131, 278.81717681884766, 286.7157459259033, 293.8394546508789, 300.47130584716797, 340.4512405395508, 373.8696575164795]",8.056834528446197,8.050402402877808,0.14267749122340218,"[7.667185068130493, 7.667185068130493, 7.74356484413147, 7.829161882400513, 7.879680395126343, 7.954884767532349, 8.153313398361206, 8.230262994766235, 8.298704624176025, 8.403304815292358, 8.717594146728516]",8.09732113411678,8.090856686309355,0.14339446354110771,"[7.705713636312053, 7.705713636312053, 7.782477230282884, 7.868504404422626, 7.91927677902145, 7.994859062846581, 8.194284822473573, 8.271621100267573, 8.340406657463342, 8.445532477680763, 8.76140115249097]",379.8405484796913,207.0238894373149,1151.9039548487194,"[0.0, 0.3242626516469362, 0.4282604451625393, 2.367892040078495, 126.12551496015637, 137.1225317117824, 363.1746471555979, 713.5597141884995, 1106.384595093643, 2807.4323962516733, 838860.8]",1137.8313465468448,431.7792876261066,20753.336374994135,"[0.0, 0.3242626516469362, 0.4282604451625393, 2.367892040078495, 253.00422246350584, 278.09998673915925, 773.1435944700461, 1441.3415807560139, 2241.7445216461783, 6213.783703703703, 167772160.0]","{""profile"": {""type_"": ""async"", ""completed_strategies"": 1, ""measured_rates"": [1.8992122384596486], ""measured_concurrencies"": [3.6010728624896617], ""max_concurrency"": null, ""strategy_type"": ""poisson"", ""rate"": [2.0], ""initial_burst"": true, ""random_seed"": 42, ""strategy_types"": [""poisson""]}, ""strategy_index"": 0, ""strategy"": {""type_"": ""poisson"", ""max_concurrency"": null, ""rate"": 2.0, ""initial_burst"": true, ""random_seed"": 42}, ""max_number"": 1000, ""max_duration"": null, ""warmup_number"": null, ""warmup_duration"": null, ""cooldown_number"": null, ""cooldown_duration"": null}","{""type_"": ""generative_requests_worker"", ""backend_type"": ""openai_http"", ""backend_target"": ""http://127.0.0.1:8000"", ""backend_model"": ""phi-4-mini-instruct"", ""backend_info"": {""max_output_tokens"": 16384, ""timeout"": 300, ""http2"": true, ""authorization"": false, ""organization"": null, ""project"": null, ""text_completions_path"": ""/v1/completions"", ""chat_completions_path"": ""/v1/chat/completions""}}","{""type_"": ""generative_request_loader"", ""data"": ""prompt_tokens=200,output_tokens=200"", ""data_args"": null, ""processor"": ""microsoft/phi-4-mini-instruct"", ""processor_args"": null}",{}
diff --git a/website/docs/benchmarks/a100/benchmark-phi4-p32.csv b/website/docs/benchmarks/a100/benchmark-phi4-p32.csv
new file mode 100644
index 0000000..7de7382
--- /dev/null
+++ b/website/docs/benchmarks/a100/benchmark-phi4-p32.csv
@@ -0,0 +1,2 @@
+Type,Run Id,Id,Name,Start Time,End Time,Duration,Successful Requests,Successful Requests per second mean,Successful Requests per second median,Successful Requests per second std dev,"Successful Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Request concurrency mean,Successful Request concurrency median,Successful Request concurrency std dev,"Successful Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Request latency mean,Successful Request latency median,Successful Request latency std dev,"Successful Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Prompt token count mean,Successful Prompt token count median,Successful Prompt token count std dev,"Successful Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Output token count mean,Successful Output token count median,Successful Output token count std dev,"Successful Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Time to first token ms mean,Successful Time to first token ms median,Successful Time to first token ms std dev,"Successful Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Time per output token ms mean,Successful Time per output token ms median,Successful Time per output token ms std dev,"Successful Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Inter token latency ms mean,Successful Inter token latency ms median,Successful Inter token latency ms std dev,"Successful Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Output tokens per second mean,Successful Output tokens per second median,Successful Output tokens per second std dev,"Successful Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Tokens per second mean,Successful Tokens per second median,Successful Tokens per second std dev,"Successful Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Requests,Errored Requests per second mean,Errored Requests per second median,Errored Requests per second std dev,"Errored Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Request concurrency mean,Errored Request concurrency median,Errored Request concurrency std dev,"Errored Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Request latency mean,Errored Request latency median,Errored Request latency std dev,"Errored Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Prompt token count mean,Errored Prompt token count median,Errored Prompt token count std dev,"Errored Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Output token count mean,Errored Output token count median,Errored Output token count std dev,"Errored Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Time to first token ms mean,Errored Time to first token ms median,Errored Time to first token ms std dev,"Errored Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Time per output token ms mean,Errored Time per output token ms median,Errored Time per output token ms std dev,"Errored Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Inter token latency ms mean,Errored Inter token latency ms median,Errored Inter token latency ms std dev,"Errored Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Output tokens per second mean,Errored Output tokens per second median,Errored Output tokens per second std dev,"Errored Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Tokens per second mean,Errored Tokens per second median,Errored Tokens per second std dev,"Errored Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Requests,Incomplete Requests per second mean,Incomplete Requests per second median,Incomplete Requests per second std dev,"Incomplete Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Request concurrency mean,Incomplete Request concurrency median,Incomplete Request concurrency std dev,"Incomplete Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Request latency mean,Incomplete Request latency median,Incomplete Request latency std dev,"Incomplete Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Prompt token count mean,Incomplete Prompt token count median,Incomplete Prompt token count std dev,"Incomplete Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Output token count mean,Incomplete Output token count median,Incomplete Output token count std dev,"Incomplete Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Time to first token ms mean,Incomplete Time to first token ms median,Incomplete Time to first token ms std dev,"Incomplete Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Time per output token ms mean,Incomplete Time per output token ms median,Incomplete Time per output token ms std dev,"Incomplete Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Inter token latency ms mean,Incomplete Inter token latency ms median,Incomplete Inter token latency ms std dev,"Incomplete Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Output tokens per second mean,Incomplete Output tokens per second median,Incomplete Output tokens per second std dev,"Incomplete Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Tokens per second mean,Incomplete Tokens per second median,Incomplete Tokens per second std dev,"Incomplete Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Requests,Total Requests per second mean,Total Requests per second median,Total Requests per second std dev,"Total Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Request concurrency mean,Total Request concurrency median,Total Request concurrency std dev,"Total Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Request latency mean,Total Request latency median,Total Request latency std dev,"Total Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Prompt token count mean,Total Prompt token count median,Total Prompt token count std dev,"Total Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Output token count mean,Total Output token count median,Total Output token count std dev,"Total Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Time to first token ms mean,Total Time to first token ms median,Total Time to first token ms std dev,"Total Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Time per output token ms mean,Total Time per output token ms median,Total Time per output token ms std dev,"Total Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Inter token latency ms mean,Total Inter token latency ms median,Total Inter token latency ms std dev,"Total Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Output tokens per second mean,Total Output tokens per second median,Total Output tokens per second std dev,"Total Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Tokens per second mean,Total Tokens per second median,Total Tokens per second std dev,"Total Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Args,Worker,Request Loader,Extras
+generative_benchmark,506dd9e4-4ba4-4d40-b23b-d1f27ddfb40f,2bc97891-b062-4eb9-b7f6-a7bc129fae99,poisson@32.00,2025-08-15 17:05:16,2025-08-15 17:05:51,34.974223613739014,1000,28.59248602754308,13.79396713892945,225.56305588832274,"[0.26281962713021834, 0.26281962713021834, 0.26281962713021834, 0.26281962713021834, 0.26281962713021834, 7.6268947659364, 25.272067772917346, 39.37278462000601, 52.55954186037769, 250.06283908662732, 149796.57142857142]",155.36898310688304,157.0,40.294160450031605,"[1.0, 3.0, 22.0, 68.0, 106.0, 137.0, 185.0, 204.0, 206.0, 208.0, 210.0]",5.433909557819367,5.323040008544922,0.8230430199897915,"[3.456366777420044, 3.456366777420044, 3.6860690116882324, 4.033002138137817, 4.288512468338013, 4.902594804763794, 6.212217807769775, 6.511993169784546, 6.713866710662842, 6.841138601303101, 6.88016676902771]",200.05400000000003,200.0,0.25900579144104097,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 201.0, 202.0]",200.00000000000003,200.0,2.8421709430404014e-14,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0]",342.89787101745605,334.70892906188965,41.14520586884624,"[277.33421325683594, 277.33421325683594, 300.7080554962158, 310.49633026123047, 315.39082527160645, 324.5377540588379, 348.452091217041, 375.05269050598145, 393.61047744750977, 527.1565914154053, 696.2161064147949]",25.454647389650347,24.93955969810486,4.105211873434645,"[15.614761114120483, 15.614761114120483, 16.567548513412476, 18.319313526153564, 19.89113450050354, 22.69835114479065, 29.351214170455933, 30.855705738067627, 31.613012552261353, 32.447437047958374, 32.49902129173279]",25.582560190603363,25.06488411869835,4.12584107882879,"[15.693227250372345, 15.693227250372345, 16.65080252604269, 18.411370378043785, 19.991089950254814, 22.812413210844873, 29.498707709000936, 31.01075953574636, 31.771871911820455, 32.61048949543555, 32.662332956515364]",5718.468613022587,3307.8107255520504,11394.542102570913,"[0.0, 0.0, 120.90118759368154, 834.5212892956625, 1166.056157909369, 1916.0822293284605, 6043.665706051873, 11214.716577540106, 16844.59437751004, 40329.846153846156, 1677721.6]",17129.95852650325,6842.257748776509,212756.78721244415,"[0.0, 0.0, 137.6174289651552, 1908.2365787079164, 2616.5339987523394, 4076.0971817298346, 12264.04678362573, 22671.913513513515, 34379.54098360656, 93206.75555555556, 167772160.0]",0,0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0,0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",1000,28.59248602754308,13.79396713892945,225.56305588832274,"[0.26281962713021834, 0.26281962713021834, 0.26281962713021834, 0.26281962713021834, 0.26281962713021834, 7.6268947659364, 25.272067772917346, 39.37278462000601, 52.55954186037769, 250.06283908662732, 149796.57142857142]",155.36898310688304,157.0,40.294160450031605,"[1.0, 3.0, 22.0, 68.0, 106.0, 137.0, 185.0, 204.0, 206.0, 208.0, 210.0]",5.433909557819367,5.323040008544922,0.8230430199897915,"[3.456366777420044, 3.456366777420044, 3.6860690116882324, 4.033002138137817, 4.288512468338013, 4.902594804763794, 6.212217807769775, 6.511993169784546, 6.713866710662842, 6.841138601303101, 6.88016676902771]",200.05400000000003,200.0,0.25900579144104097,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 201.0, 202.0]",200.00000000000003,200.0,2.8421709430404014e-14,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0]",342.89787101745605,334.70892906188965,41.14520586884624,"[277.33421325683594, 277.33421325683594, 300.7080554962158, 310.49633026123047, 315.39082527160645, 324.5377540588379, 348.452091217041, 375.05269050598145, 393.61047744750977, 527.1565914154053, 696.2161064147949]",25.454647389650347,24.93955969810486,4.105211873434645,"[15.614761114120483, 15.614761114120483, 16.567548513412476, 18.319313526153564, 19.89113450050354, 22.69835114479065, 29.351214170455933, 30.855705738067627, 31.613012552261353, 32.447437047958374, 32.49902129173279]",25.582560190603363,25.06488411869835,4.12584107882879,"[15.693227250372345, 15.693227250372345, 16.65080252604269, 18.411370378043785, 19.991089950254814, 22.812413210844873, 29.498707709000936, 31.01075953574636, 31.771871911820455, 32.61048949543555, 32.662332956515364]",5718.468613022587,3307.8107255520504,11394.542102570913,"[0.0, 0.0, 120.90118759368154, 834.5212892956625, 1166.056157909369, 1916.0822293284605, 6043.665706051873, 11214.716577540106, 16844.59437751004, 40329.846153846156, 1677721.6]",17129.95852650325,6842.257748776509,212756.78721244415,"[0.0, 0.0, 137.6174289651552, 1908.2365787079164, 2616.5339987523394, 4076.0971817298346, 12264.04678362573, 22671.913513513515, 34379.54098360656, 93206.75555555556, 167772160.0]","{""profile"": {""type_"": ""async"", ""completed_strategies"": 1, ""measured_rates"": [28.59248602754308], ""measured_concurrencies"": [155.36898310688304], ""max_concurrency"": null, ""strategy_type"": ""poisson"", ""rate"": [32.0], ""initial_burst"": true, ""random_seed"": 42, ""strategy_types"": [""poisson""]}, ""strategy_index"": 0, ""strategy"": {""type_"": ""poisson"", ""max_concurrency"": null, ""rate"": 32.0, ""initial_burst"": true, ""random_seed"": 42}, ""max_number"": 1000, ""max_duration"": null, ""warmup_number"": null, ""warmup_duration"": null, ""cooldown_number"": null, ""cooldown_duration"": null}","{""type_"": ""generative_requests_worker"", ""backend_type"": ""openai_http"", ""backend_target"": ""http://127.0.0.1:8000"", ""backend_model"": ""phi-4-mini-instruct"", ""backend_info"": {""max_output_tokens"": 16384, ""timeout"": 300, ""http2"": true, ""authorization"": false, ""organization"": null, ""project"": null, ""text_completions_path"": ""/v1/completions"", ""chat_completions_path"": ""/v1/chat/completions""}}","{""type_"": ""generative_request_loader"", ""data"": ""prompt_tokens=200,output_tokens=200"", ""data_args"": null, ""processor"": ""microsoft/phi-4-mini-instruct"", ""processor_args"": null}",{}
diff --git a/website/docs/benchmarks/a100/benchmark-phi4-p4.csv b/website/docs/benchmarks/a100/benchmark-phi4-p4.csv
new file mode 100644
index 0000000..cbe3696
--- /dev/null
+++ b/website/docs/benchmarks/a100/benchmark-phi4-p4.csv
@@ -0,0 +1,2 @@
+Type,Run Id,Id,Name,Start Time,End Time,Duration,Successful Requests,Successful Requests per second mean,Successful Requests per second median,Successful Requests per second std dev,"Successful Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Request concurrency mean,Successful Request concurrency median,Successful Request concurrency std dev,"Successful Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Request latency mean,Successful Request latency median,Successful Request latency std dev,"Successful Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Prompt token count mean,Successful Prompt token count median,Successful Prompt token count std dev,"Successful Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Output token count mean,Successful Output token count median,Successful Output token count std dev,"Successful Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Time to first token ms mean,Successful Time to first token ms median,Successful Time to first token ms std dev,"Successful Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Time per output token ms mean,Successful Time per output token ms median,Successful Time per output token ms std dev,"Successful Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Inter token latency ms mean,Successful Inter token latency ms median,Successful Inter token latency ms std dev,"Successful Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Output tokens per second mean,Successful Output tokens per second median,Successful Output tokens per second std dev,"Successful Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Tokens per second mean,Successful Tokens per second median,Successful Tokens per second std dev,"Successful Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Requests,Errored Requests per second mean,Errored Requests per second median,Errored Requests per second std dev,"Errored Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Request concurrency mean,Errored Request concurrency median,Errored Request concurrency std dev,"Errored Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Request latency mean,Errored Request latency median,Errored Request latency std dev,"Errored Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Prompt token count mean,Errored Prompt token count median,Errored Prompt token count std dev,"Errored Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Output token count mean,Errored Output token count median,Errored Output token count std dev,"Errored Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Time to first token ms mean,Errored Time to first token ms median,Errored Time to first token ms std dev,"Errored Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Time per output token ms mean,Errored Time per output token ms median,Errored Time per output token ms std dev,"Errored Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Inter token latency ms mean,Errored Inter token latency ms median,Errored Inter token latency ms std dev,"Errored Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Output tokens per second mean,Errored Output tokens per second median,Errored Output tokens per second std dev,"Errored Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Tokens per second mean,Errored Tokens per second median,Errored Tokens per second std dev,"Errored Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Requests,Incomplete Requests per second mean,Incomplete Requests per second median,Incomplete Requests per second std dev,"Incomplete Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Request concurrency mean,Incomplete Request concurrency median,Incomplete Request concurrency std dev,"Incomplete Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Request latency mean,Incomplete Request latency median,Incomplete Request latency std dev,"Incomplete Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Prompt token count mean,Incomplete Prompt token count median,Incomplete Prompt token count std dev,"Incomplete Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Output token count mean,Incomplete Output token count median,Incomplete Output token count std dev,"Incomplete Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Time to first token ms mean,Incomplete Time to first token ms median,Incomplete Time to first token ms std dev,"Incomplete Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Time per output token ms mean,Incomplete Time per output token ms median,Incomplete Time per output token ms std dev,"Incomplete Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Inter token latency ms mean,Incomplete Inter token latency ms median,Incomplete Inter token latency ms std dev,"Incomplete Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Output tokens per second mean,Incomplete Output tokens per second median,Incomplete Output tokens per second std dev,"Incomplete Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Tokens per second mean,Incomplete Tokens per second median,Incomplete Tokens per second std dev,"Incomplete Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Requests,Total Requests per second mean,Total Requests per second median,Total Requests per second std dev,"Total Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Request concurrency mean,Total Request concurrency median,Total Request concurrency std dev,"Total Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Request latency mean,Total Request latency median,Total Request latency std dev,"Total Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Prompt token count mean,Total Prompt token count median,Total Prompt token count std dev,"Total Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Output token count mean,Total Output token count median,Total Output token count std dev,"Total Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Time to first token ms mean,Total Time to first token ms median,Total Time to first token ms std dev,"Total Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Time per output token ms mean,Total Time per output token ms median,Total Time per output token ms std dev,"Total Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Inter token latency ms mean,Total Inter token latency ms median,Total Inter token latency ms std dev,"Total Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Output tokens per second mean,Total Output tokens per second median,Total Output tokens per second std dev,"Total Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Tokens per second mean,Total Tokens per second median,Total Tokens per second std dev,"Total Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Args,Worker,Request Loader,Extras
+generative_benchmark,c88dfbcb-9fe6-4aa6-a8c6-83b907ace8ff,d12ec2ce-37dd-4d0c-aaf7-20b5d8189fbf,poisson@4.00,2025-08-15 16:46:51,2025-08-15 16:51:15,263.77663683891296,1000,3.7910863220638262,2.267747868936485,37.79041180937987,"[0.4396655681065352, 0.4396655681065352, 0.4925104847577057, 0.666897747848878, 0.9008655758549441, 1.363418245530773, 3.980979246117545, 7.010017883107984, 11.04880721571272, 24.597570916682795, 55188.21052631579]",7.445592525957859,7.0,2.904053882224569,"[0.0, 0.0, 1.0, 3.0, 4.0, 6.0, 9.0, 11.0, 13.0, 15.0, 19.0]",1.9639733557701113,1.9535698890686035,0.049503781742412115,"[1.8340961933135986, 1.8340961933135986, 1.8739666938781738, 1.9006307125091553, 1.9116461277008057, 1.9309313297271729, 1.9860272407531738, 2.0377414226531982, 2.0681679248809814, 2.107847213745117, 2.1287474632263184]",200.05400000000003,200.0,0.25900579144104097,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 201.0, 202.0]",200.00000000000003,200.0,2.8421709430404014e-14,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0]",283.7414622306824,282.9470634460449,6.768858790788341,"[264.174222946167, 264.174222946167, 273.07939529418945, 274.9671936035156, 276.644229888916, 279.41131591796875, 286.67283058166504, 290.8475399017334, 294.15369033813477, 304.61597442626953, 342.50783920288086]",8.400718457698822,8.352411985397339,0.24436261704799178,"[7.7453577518463135, 7.7453577518463135, 7.952300310134888, 8.086087703704834, 8.1459641456604, 8.242254257202148, 8.517093658447266, 8.764212131500244, 8.916981220245361, 9.109491109848022, 9.185280799865723]",8.442933123315399,8.394383904921948,0.24559056989747916,"[7.784279147584234, 7.784279147584234, 7.992261618226018, 8.126721310256114, 8.186898638854673, 8.283672620303667, 8.559893124067603, 8.808253398492708, 8.961790171100866, 9.155267447083439, 9.231437989814797]",758.213473326443,409.0806593192236,2013.5571994182117,"[0.0, 0.0, 123.5180964160556, 146.905677559455, 173.0609011388018, 242.15137694128515, 764.6862351868733, 1452.824385174922, 2236.9621333333334, 5667.978378378379, 838860.8]",2271.266353152693,892.9750904832872,38688.96441334727,"[0.0, 0.0, 249.02357062281067, 299.65735514753163, 371.70365118752215, 539.5991251768944, 1582.159185213127, 2896.6187845303866, 4466.777422790203, 12228.291545189504, 167772160.0]",0,0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0,0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",1000,3.7910863220638262,2.267747868936485,37.79041180937987,"[0.4396655681065352, 0.4396655681065352, 0.4925104847577057, 0.666897747848878, 0.9008655758549441, 1.363418245530773, 3.980979246117545, 7.010017883107984, 11.04880721571272, 24.597570916682795, 55188.21052631579]",7.445592525957859,7.0,2.904053882224569,"[0.0, 0.0, 1.0, 3.0, 4.0, 6.0, 9.0, 11.0, 13.0, 15.0, 19.0]",1.9639733557701113,1.9535698890686035,0.049503781742412115,"[1.8340961933135986, 1.8340961933135986, 1.8739666938781738, 1.9006307125091553, 1.9116461277008057, 1.9309313297271729, 1.9860272407531738, 2.0377414226531982, 2.0681679248809814, 2.107847213745117, 2.1287474632263184]",200.05400000000003,200.0,0.25900579144104097,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 201.0, 202.0]",200.00000000000003,200.0,2.8421709430404014e-14,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0]",283.7414622306824,282.9470634460449,6.768858790788341,"[264.174222946167, 264.174222946167, 273.07939529418945, 274.9671936035156, 276.644229888916, 279.41131591796875, 286.67283058166504, 290.8475399017334, 294.15369033813477, 304.61597442626953, 342.50783920288086]",8.400718457698822,8.352411985397339,0.24436261704799178,"[7.7453577518463135, 7.7453577518463135, 7.952300310134888, 8.086087703704834, 8.1459641456604, 8.242254257202148, 8.517093658447266, 8.764212131500244, 8.916981220245361, 9.109491109848022, 9.185280799865723]",8.442933123315399,8.394383904921948,0.24559056989747916,"[7.784279147584234, 7.784279147584234, 7.992261618226018, 8.126721310256114, 8.186898638854673, 8.283672620303667, 8.559893124067603, 8.808253398492708, 8.961790171100866, 9.155267447083439, 9.231437989814797]",758.213473326443,409.0806593192236,2013.5571994182117,"[0.0, 0.0, 123.5180964160556, 146.905677559455, 173.0609011388018, 242.15137694128515, 764.6862351868733, 1452.824385174922, 2236.9621333333334, 5667.978378378379, 838860.8]",2271.266353152693,892.9750904832872,38688.96441334727,"[0.0, 0.0, 249.02357062281067, 299.65735514753163, 371.70365118752215, 539.5991251768944, 1582.159185213127, 2896.6187845303866, 4466.777422790203, 12228.291545189504, 167772160.0]","{""profile"": {""type_"": ""async"", ""completed_strategies"": 1, ""measured_rates"": [3.7910863220638262], ""measured_concurrencies"": [7.445592525957859], ""max_concurrency"": null, ""strategy_type"": ""poisson"", ""rate"": [4.0], ""initial_burst"": true, ""random_seed"": 42, ""strategy_types"": [""poisson""]}, ""strategy_index"": 0, ""strategy"": {""type_"": ""poisson"", ""max_concurrency"": null, ""rate"": 4.0, ""initial_burst"": true, ""random_seed"": 42}, ""max_number"": 1000, ""max_duration"": null, ""warmup_number"": null, ""warmup_duration"": null, ""cooldown_number"": null, ""cooldown_duration"": null}","{""type_"": ""generative_requests_worker"", ""backend_type"": ""openai_http"", ""backend_target"": ""http://127.0.0.1:8000"", ""backend_model"": ""phi-4-mini-instruct"", ""backend_info"": {""max_output_tokens"": 16384, ""timeout"": 300, ""http2"": true, ""authorization"": false, ""organization"": null, ""project"": null, ""text_completions_path"": ""/v1/completions"", ""chat_completions_path"": ""/v1/chat/completions""}}","{""type_"": ""generative_request_loader"", ""data"": ""prompt_tokens=200,output_tokens=200"", ""data_args"": null, ""processor"": ""microsoft/phi-4-mini-instruct"", ""processor_args"": null}",{}
diff --git a/website/docs/benchmarks/a100/benchmark-phi4-p64.csv b/website/docs/benchmarks/a100/benchmark-phi4-p64.csv
new file mode 100644
index 0000000..6255f6f
--- /dev/null
+++ b/website/docs/benchmarks/a100/benchmark-phi4-p64.csv
@@ -0,0 +1,2 @@
+Type,Run Id,Id,Name,Start Time,End Time,Duration,Successful Requests,Successful Requests per second mean,Successful Requests per second median,Successful Requests per second std dev,"Successful Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Request concurrency mean,Successful Request concurrency median,Successful Request concurrency std dev,"Successful Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Request latency mean,Successful Request latency median,Successful Request latency std dev,"Successful Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Prompt token count mean,Successful Prompt token count median,Successful Prompt token count std dev,"Successful Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Output token count mean,Successful Output token count median,Successful Output token count std dev,"Successful Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Time to first token ms mean,Successful Time to first token ms median,Successful Time to first token ms std dev,"Successful Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Time per output token ms mean,Successful Time per output token ms median,Successful Time per output token ms std dev,"Successful Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Inter token latency ms mean,Successful Inter token latency ms median,Successful Inter token latency ms std dev,"Successful Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Output tokens per second mean,Successful Output tokens per second median,Successful Output tokens per second std dev,"Successful Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Tokens per second mean,Successful Tokens per second median,Successful Tokens per second std dev,"Successful Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Requests,Errored Requests per second mean,Errored Requests per second median,Errored Requests per second std dev,"Errored Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Request concurrency mean,Errored Request concurrency median,Errored Request concurrency std dev,"Errored Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Request latency mean,Errored Request latency median,Errored Request latency std dev,"Errored Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Prompt token count mean,Errored Prompt token count median,Errored Prompt token count std dev,"Errored Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Output token count mean,Errored Output token count median,Errored Output token count std dev,"Errored Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Time to first token ms mean,Errored Time to first token ms median,Errored Time to first token ms std dev,"Errored Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Time per output token ms mean,Errored Time per output token ms median,Errored Time per output token ms std dev,"Errored Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Inter token latency ms mean,Errored Inter token latency ms median,Errored Inter token latency ms std dev,"Errored Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Output tokens per second mean,Errored Output tokens per second median,Errored Output tokens per second std dev,"Errored Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Tokens per second mean,Errored Tokens per second median,Errored Tokens per second std dev,"Errored Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Requests,Incomplete Requests per second mean,Incomplete Requests per second median,Incomplete Requests per second std dev,"Incomplete Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Request concurrency mean,Incomplete Request concurrency median,Incomplete Request concurrency std dev,"Incomplete Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Request latency mean,Incomplete Request latency median,Incomplete Request latency std dev,"Incomplete Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Prompt token count mean,Incomplete Prompt token count median,Incomplete Prompt token count std dev,"Incomplete Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Output token count mean,Incomplete Output token count median,Incomplete Output token count std dev,"Incomplete Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Time to first token ms mean,Incomplete Time to first token ms median,Incomplete Time to first token ms std dev,"Incomplete Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Time per output token ms mean,Incomplete Time per output token ms median,Incomplete Time per output token ms std dev,"Incomplete Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Inter token latency ms mean,Incomplete Inter token latency ms median,Incomplete Inter token latency ms std dev,"Incomplete Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Output tokens per second mean,Incomplete Output tokens per second median,Incomplete Output tokens per second std dev,"Incomplete Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Tokens per second mean,Incomplete Tokens per second median,Incomplete Tokens per second std dev,"Incomplete Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Requests,Total Requests per second mean,Total Requests per second median,Total Requests per second std dev,"Total Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Request concurrency mean,Total Request concurrency median,Total Request concurrency std dev,"Total Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Request latency mean,Total Request latency median,Total Request latency std dev,"Total Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Prompt token count mean,Total Prompt token count median,Total Prompt token count std dev,"Total Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Output token count mean,Total Output token count median,Total Output token count std dev,"Total Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Time to first token ms mean,Total Time to first token ms median,Total Time to first token ms std dev,"Total Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Time per output token ms mean,Total Time per output token ms median,Total Time per output token ms std dev,"Total Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Inter token latency ms mean,Total Inter token latency ms median,Total Inter token latency ms std dev,"Total Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Output tokens per second mean,Total Output tokens per second median,Total Output tokens per second std dev,"Total Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Tokens per second mean,Total Tokens per second median,Total Tokens per second std dev,"Total Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Args,Worker,Request Loader,Extras
+generative_benchmark,866360af-6091-4ab9-b31b-efd63f257766,87e47972-4c73-4ce9-be27-9e4e3b61f644,poisson@64.00,2025-08-15 17:06:26,2025-08-15 17:06:58,31.891711235046387,999,31.32475371538485,0.29273241506419967,496.928964245685,"[0.15390008073481265, 0.15390008073481265, 0.15390008073481265, 0.15390008073481265, 0.15390008073481265, 0.2713725317662069, 18.802214501849154, 29.90655058575228, 52.34243498227924, 356.77985709424973, 524288.0]",382.1333254791747,447.0,135.73950042560662,"[1.0, 8.0, 53.0, 124.0, 197.0, 253.0, 510.0, 511.0, 512.0, 512.0, 787.0]",12.19902212984927,11.836660385131836,3.250543970770784,"[6.497722387313843, 6.497722387313843, 6.675297021865845, 6.8625099658966064, 7.035495042800903, 8.310498714447021, 15.328571319580078, 15.6010103225708, 15.847218036651611, 15.918026447296143, 15.937759637832642]",200.054054054054,200.0,0.25912975381146697,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 201.0, 202.0]",199.99999999999994,200.0,5.684341886080802e-14,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0]",4716.8109746308655,3953.1517028808594,3132.1201900244564,"[281.693696975708, 281.693696975708, 301.56493186950684, 328.27091217041016, 340.2674198150635, 389.23144340515137, 7954.9853801727295, 8020.8680629730225, 8029.900550842285, 8046.334505081177, 8155.239582061768]",37.41063423581548,39.39435005187988,3.128019414277279,"[27.431107759475708, 27.431107759475708, 28.088079690933228, 30.958008766174316, 32.40635633468628, 36.138564348220825, 39.517091512680054, 39.628095626831055, 39.67329502105713, 39.74640727043152, 39.98524785041809]",37.59862737267888,39.59231160992953,3.143738104801285,"[27.56895252208614, 27.56895252208614, 28.229225820033395, 31.113576649421425, 32.56920234641837, 36.32016517409128, 39.71566986199, 39.82723178575986, 39.87265831262023, 39.94613796023268, 40.186178744138786]",6264.919386967145,3449.2631578947367,12902.739875385878,"[0.0, 0.0, 99.60351460460699, 751.802115074386, 1122.6723768736617, 1918.711802378774, 6574.144200626959, 12446.005934718101, 19152.07305936073, 47127.011235955055, 1677721.6]",18766.88257926682,7384.338028169014,253515.98625880873,"[0.0, 0.0, 190.16612259702575, 1928.415632183908, 2709.498708010336, 4332.9586776859505, 13400.332268370606, 24966.095238095237, 38479.85321100918, 104857.6, 167772160.0]",1,0.09514484764475951,0.09514484764475951,0.0,"[0.09514484764475951, 0.09514484764475951, 0.09514484764475951, 0.09514484764475951, 0.09514484764475951, 0.09514484764475951, 0.09514484764475951, 0.09514484764475951, 0.09514484764475951, 0.09514484764475951, 0.09514484764475951]",1.0,1.0,0.0,"[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]",10.510290622711182,10.510290622711182,0.0,"[10.510290622711182, 10.510290622711182, 10.510290622711182, 10.510290622711182, 10.510290622711182, 10.510290622711182, 10.510290622711182, 10.510290622711182, 10.510290622711182, 10.510290622711182, 10.510290622711182]",200.0,200.0,0.0,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0]",193.0,193.0,0.0,"[193.0, 193.0, 193.0, 193.0, 193.0, 193.0, 193.0, 193.0, 193.0, 193.0, 193.0]",2751.9195079803467,2751.9195079803467,0.0,"[2751.9195079803467, 2751.9195079803467, 2751.9195079803467, 2751.9195079803467, 2751.9195079803467, 2751.9195079803467, 2751.9195079803467, 2751.9195079803467, 2751.9195079803467, 2751.9195079803467, 2751.9195079803467]",39.625727450909395,39.625727450909395,0.0,"[39.625727450909395, 39.625727450909395, 39.625727450909395, 39.625727450909395, 39.625727450909395, 39.625727450909395, 39.625727450909395, 39.625727450909395, 39.625727450909395, 39.625727450909395, 39.625727450909395]",39.832111448049545,39.832111448049545,0.0,"[39.832111448049545, 39.832111448049545, 39.832111448049545, 39.832111448049545, 39.832111448049545, 39.832111448049545, 39.832111448049545, 39.832111448049545, 39.832111448049545, 39.832111448049545, 39.832111448049545]",18.26781074779383,24.747492388662057,10.87975441394911,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 24.747492388662057, 24.747492388662057, 24.747492388662057, 24.747492388662057, 24.747492388662057]",56.23060495805287,50.525869442135566,437.045555427447,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 50.525869442135566, 50.52647809955187, 50.52647809955187, 50.52647809955187, 10105.173888427114]",0,0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",1000,31.356109825210062,0.29273241506419967,497.0287436407293,"[0.15390008073481265, 0.15390008073481265, 0.15390008073481265, 0.15390008073481265, 0.15390008073481265, 0.2713725317662069, 18.802214501849154, 29.90655058575228, 52.34243498227924, 356.77985709424973, 524288.0]",382.46288730623525,448.0,135.97564775675932,"[1.0, 8.0, 53.0, 124.0, 197.0, 253.0, 510.0, 512.0, 512.0, 512.0, 787.0]",12.197333398342135,11.83504867553711,3.2493567105040158,"[6.497722387313843, 6.497722387313843, 6.675297021865845, 6.8625099658966064, 7.035495042800903, 8.310498714447021, 15.321745157241821, 15.60087537765503, 15.843006610870361, 15.91231918334961, 15.937759637832642]",200.05400000000003,200.0,0.25900579144104097,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 201.0, 202.0]",199.99300000000002,200.0,0.2212487288099076,"[193.0, 193.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0]",4714.846083164215,3950.896739959717,3131.1696928705924,"[281.693696975708, 281.693696975708, 301.56493186950684, 328.27091217041016, 340.2674198150635, 389.23144340515137, 7954.501628875732, 8020.813703536987, 8029.507637023926, 8044.428110122681, 8155.239582061768]",37.41277187558544,39.39435005187988,3.1272661510805504,"[27.431107759475708, 27.431107759475708, 28.088079690933228, 30.958008766174316, 32.40635633468628, 36.138564348220825, 39.517091512680054, 39.628095626831055, 39.67329502105713, 39.74640727043152, 39.98524785041809]",37.60078236779162,39.59231160992953,3.1429861591323918,"[27.56895252208614, 27.56895252208614, 28.229225820033395, 31.113576649421425, 32.56920234641837, 36.32016517409128, 39.71566986199, 39.82723178575986, 39.87265831262023, 39.94613796023268, 40.186178744138786]",6270.9711161634095,3454.9456342668864,12911.754564500663,"[0.0, 0.0, 99.60351460460699, 751.802115074386, 1123.8756698821007, 1922.2291475710358, 6584.464678178964, 12446.005934718101, 19152.07305936073, 47127.011235955055, 1677721.6]",18785.445396283347,7397.361552028218,253666.66568390664,"[0.0, 0.0, 190.16612259702575, 1928.415632183908, 2709.498708010336, 4337.439503619442, 13400.332268370606, 24966.095238095237, 38479.85321100918, 104857.6, 167772160.0]","{""profile"": {""type_"": ""async"", ""completed_strategies"": 1, ""measured_rates"": [31.32475371538485], ""measured_concurrencies"": [382.1333254791747], ""max_concurrency"": null, ""strategy_type"": ""poisson"", ""rate"": [64.0], ""initial_burst"": true, ""random_seed"": 42, ""strategy_types"": [""poisson""]}, ""strategy_index"": 0, ""strategy"": {""type_"": ""poisson"", ""max_concurrency"": null, ""rate"": 64.0, ""initial_burst"": true, ""random_seed"": 42}, ""max_number"": 1000, ""max_duration"": null, ""warmup_number"": null, ""warmup_duration"": null, ""cooldown_number"": null, ""cooldown_duration"": null}","{""type_"": ""generative_requests_worker"", ""backend_type"": ""openai_http"", ""backend_target"": ""http://127.0.0.1:8000"", ""backend_model"": ""phi-4-mini-instruct"", ""backend_info"": {""max_output_tokens"": 16384, ""timeout"": 300, ""http2"": true, ""authorization"": false, ""organization"": null, ""project"": null, ""text_completions_path"": ""/v1/completions"", ""chat_completions_path"": ""/v1/chat/completions""}}","{""type_"": ""generative_request_loader"", ""data"": ""prompt_tokens=200,output_tokens=200"", ""data_args"": null, ""processor"": ""microsoft/phi-4-mini-instruct"", ""processor_args"": null}",{}
diff --git a/website/docs/benchmarks/a100/benchmark-phi4-p8.csv b/website/docs/benchmarks/a100/benchmark-phi4-p8.csv
new file mode 100644
index 0000000..0880e36
--- /dev/null
+++ b/website/docs/benchmarks/a100/benchmark-phi4-p8.csv
@@ -0,0 +1,2 @@
+Type,Run Id,Id,Name,Start Time,End Time,Duration,Successful Requests,Successful Requests per second mean,Successful Requests per second median,Successful Requests per second std dev,"Successful Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Request concurrency mean,Successful Request concurrency median,Successful Request concurrency std dev,"Successful Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Request latency mean,Successful Request latency median,Successful Request latency std dev,"Successful Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Prompt token count mean,Successful Prompt token count median,Successful Prompt token count std dev,"Successful Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Output token count mean,Successful Output token count median,Successful Output token count std dev,"Successful Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Time to first token ms mean,Successful Time to first token ms median,Successful Time to first token ms std dev,"Successful Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Time per output token ms mean,Successful Time per output token ms median,Successful Time per output token ms std dev,"Successful Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Inter token latency ms mean,Successful Inter token latency ms median,Successful Inter token latency ms std dev,"Successful Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Output tokens per second mean,Successful Output tokens per second median,Successful Output tokens per second std dev,"Successful Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Successful Tokens per second mean,Successful Tokens per second median,Successful Tokens per second std dev,"Successful Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Requests,Errored Requests per second mean,Errored Requests per second median,Errored Requests per second std dev,"Errored Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Request concurrency mean,Errored Request concurrency median,Errored Request concurrency std dev,"Errored Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Request latency mean,Errored Request latency median,Errored Request latency std dev,"Errored Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Prompt token count mean,Errored Prompt token count median,Errored Prompt token count std dev,"Errored Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Output token count mean,Errored Output token count median,Errored Output token count std dev,"Errored Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Time to first token ms mean,Errored Time to first token ms median,Errored Time to first token ms std dev,"Errored Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Time per output token ms mean,Errored Time per output token ms median,Errored Time per output token ms std dev,"Errored Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Inter token latency ms mean,Errored Inter token latency ms median,Errored Inter token latency ms std dev,"Errored Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Output tokens per second mean,Errored Output tokens per second median,Errored Output tokens per second std dev,"Errored Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Errored Tokens per second mean,Errored Tokens per second median,Errored Tokens per second std dev,"Errored Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Requests,Incomplete Requests per second mean,Incomplete Requests per second median,Incomplete Requests per second std dev,"Incomplete Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Request concurrency mean,Incomplete Request concurrency median,Incomplete Request concurrency std dev,"Incomplete Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Request latency mean,Incomplete Request latency median,Incomplete Request latency std dev,"Incomplete Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Prompt token count mean,Incomplete Prompt token count median,Incomplete Prompt token count std dev,"Incomplete Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Output token count mean,Incomplete Output token count median,Incomplete Output token count std dev,"Incomplete Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Time to first token ms mean,Incomplete Time to first token ms median,Incomplete Time to first token ms std dev,"Incomplete Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Time per output token ms mean,Incomplete Time per output token ms median,Incomplete Time per output token ms std dev,"Incomplete Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Inter token latency ms mean,Incomplete Inter token latency ms median,Incomplete Inter token latency ms std dev,"Incomplete Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Output tokens per second mean,Incomplete Output tokens per second median,Incomplete Output tokens per second std dev,"Incomplete Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Incomplete Tokens per second mean,Incomplete Tokens per second median,Incomplete Tokens per second std dev,"Incomplete Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Requests,Total Requests per second mean,Total Requests per second median,Total Requests per second std dev,"Total Requests per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Request concurrency mean,Total Request concurrency median,Total Request concurrency std dev,"Total Request concurrency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Request latency mean,Total Request latency median,Total Request latency std dev,"Total Request latency [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Prompt token count mean,Total Prompt token count median,Total Prompt token count std dev,"Total Prompt token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Output token count mean,Total Output token count median,Total Output token count std dev,"Total Output token count [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Time to first token ms mean,Total Time to first token ms median,Total Time to first token ms std dev,"Total Time to first token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Time per output token ms mean,Total Time per output token ms median,Total Time per output token ms std dev,"Total Time per output token ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Inter token latency ms mean,Total Inter token latency ms median,Total Inter token latency ms std dev,"Total Inter token latency ms [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Output tokens per second mean,Total Output tokens per second median,Total Output tokens per second std dev,"Total Output tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Total Tokens per second mean,Total Tokens per second median,Total Tokens per second std dev,"Total Tokens per second [min, 0.1, 1, 5, 10, 25, 75, 90, 95, 99, max]",Args,Worker,Request Loader,Extras
+generative_benchmark,205dabb2-b02d-4f47-aead-3c01c2e7fd7a,1b68e56c-83eb-4ab7-af8a-3b9156025d27,poisson@8.00,2025-08-15 16:53:53,2025-08-15 16:56:05,132.22053456306458,1000,7.5631217443235705,4.465588501463934,86.499749397637,"[0.4521044399819344, 0.4521044399819344, 0.4521044399819344, 1.359512037324555, 1.7113838354905653, 2.765407093275946, 7.792685736368433, 13.667614923145605, 20.569287581346764, 49.7816602179125, 299593.14285714284]",15.972126726799129,16.0,4.549022280664887,"[1.0, 4.0, 6.0, 9.0, 11.0, 13.0, 19.0, 22.0, 24.0, 29.0, 36.0]",2.1118431339263917,2.107211112976074,0.05807149187903208,"[1.9498155117034912, 1.9498155117034912, 1.975759744644165, 2.0177552700042725, 2.0400140285491943, 2.0789759159088135, 2.1425089836120605, 2.1928727626800537, 2.2148988246917725, 2.2635509967803955, 2.276479482650757]",200.05400000000003,200.0,0.25900579144104097,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 201.0, 202.0]",200.00000000000003,200.0,2.8421709430404014e-14,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0]",287.6975531578064,285.1216793060303,12.499123746320844,"[271.04997634887695, 271.04997634887695, 274.33180809020996, 276.69525146484375, 278.19013595581055, 281.2933921813965, 289.43872451782227, 296.4813709259033, 306.3397407531738, 340.40379524230957, 372.7762699127197]",9.12031340122223,9.10202145576477,0.28676138402406615,"[8.319361209869385, 8.319361209869385, 8.449748754501343, 8.640646934509277, 8.767681121826172, 8.963372707366943, 9.278026819229126, 9.524905681610107, 9.6482515335083, 9.860951900482178, 9.909793138504028]",9.166144121831387,9.14776025705002,0.2882023960040866,"[8.36116704509486, 8.36116704509486, 8.492209803518938, 8.684067270863595, 8.811739820930827, 9.00841478127331, 9.324650069577011, 9.572769529256389, 9.696735209556081, 9.910504422595153, 9.959591093973899]",1512.6167857429703,828.7500493973523,3660.4820351629364,"[0.0, 0.0, 168.3512884322068, 248.6544937159118, 308.6089323817232, 477.65675891128575, 1546.0022115739034, 2976.794889992903, 4568.9586056644885, 11125.474801061007, 838860.8]",4531.119178876462,1757.1445328864684,65574.13131009998,"[0.0, 0.0, 369.5096467271606, 580.1250345781466, 726.0349662454561, 1074.6359210863438, 3137.100972326103, 5882.6143057503505, 9039.448275862069, 24818.366863905325, 93206755.55555555]",0,0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0,0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",0.0,0.0,0.0,"[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",1000,7.5631217443235705,4.465588501463934,86.499749397637,"[0.4521044399819344, 0.4521044399819344, 0.4521044399819344, 1.359512037324555, 1.7113838354905653, 2.765407093275946, 7.792685736368433, 13.667614923145605, 20.569287581346764, 49.7816602179125, 299593.14285714284]",15.972126726799129,16.0,4.549022280664887,"[1.0, 4.0, 6.0, 9.0, 11.0, 13.0, 19.0, 22.0, 24.0, 29.0, 36.0]",2.1118431339263917,2.107211112976074,0.05807149187903208,"[1.9498155117034912, 1.9498155117034912, 1.975759744644165, 2.0177552700042725, 2.0400140285491943, 2.0789759159088135, 2.1425089836120605, 2.1928727626800537, 2.2148988246917725, 2.2635509967803955, 2.276479482650757]",200.05400000000003,200.0,0.25900579144104097,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 201.0, 202.0]",200.00000000000003,200.0,2.8421709430404014e-14,"[200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0, 200.0]",287.6975531578064,285.1216793060303,12.499123746320844,"[271.04997634887695, 271.04997634887695, 274.33180809020996, 276.69525146484375, 278.19013595581055, 281.2933921813965, 289.43872451782227, 296.4813709259033, 306.3397407531738, 340.40379524230957, 372.7762699127197]",9.12031340122223,9.10202145576477,0.28676138402406615,"[8.319361209869385, 8.319361209869385, 8.449748754501343, 8.640646934509277, 8.767681121826172, 8.963372707366943, 9.278026819229126, 9.524905681610107, 9.6482515335083, 9.860951900482178, 9.909793138504028]",9.166144121831387,9.14776025705002,0.2882023960040866,"[8.36116704509486, 8.36116704509486, 8.492209803518938, 8.684067270863595, 8.811739820930827, 9.00841478127331, 9.324650069577011, 9.572769529256389, 9.696735209556081, 9.910504422595153, 9.959591093973899]",1512.6167857429703,828.7500493973523,3660.4820351629364,"[0.0, 0.0, 168.3512884322068, 248.6544937159118, 308.6089323817232, 477.65675891128575, 1546.0022115739034, 2976.794889992903, 4568.9586056644885, 11125.474801061007, 838860.8]",4531.119178876462,1757.1445328864684,65574.13131009998,"[0.0, 0.0, 369.5096467271606, 580.1250345781466, 726.0349662454561, 1074.6359210863438, 3137.100972326103, 5882.6143057503505, 9039.448275862069, 24818.366863905325, 93206755.55555555]","{""profile"": {""type_"": ""async"", ""completed_strategies"": 1, ""measured_rates"": [7.5631217443235705], ""measured_concurrencies"": [15.972126726799129], ""max_concurrency"": null, ""strategy_type"": ""poisson"", ""rate"": [8.0], ""initial_burst"": true, ""random_seed"": 42, ""strategy_types"": [""poisson""]}, ""strategy_index"": 0, ""strategy"": {""type_"": ""poisson"", ""max_concurrency"": null, ""rate"": 8.0, ""initial_burst"": true, ""random_seed"": 42}, ""max_number"": 1000, ""max_duration"": null, ""warmup_number"": null, ""warmup_duration"": null, ""cooldown_number"": null, ""cooldown_duration"": null}","{""type_"": ""generative_requests_worker"", ""backend_type"": ""openai_http"", ""backend_target"": ""http://127.0.0.1:8000"", ""backend_model"": ""phi-4-mini-instruct"", ""backend_info"": {""max_output_tokens"": 16384, ""timeout"": 300, ""http2"": true, ""authorization"": false, ""organization"": null, ""project"": null, ""text_completions_path"": ""/v1/completions"", ""chat_completions_path"": ""/v1/chat/completions""}}","{""type_"": ""generative_request_loader"", ""data"": ""prompt_tokens=200,output_tokens=200"", ""data_args"": null, ""processor"": ""microsoft/phi-4-mini-instruct"", ""processor_args"": null}",{}
diff --git a/website/docs/gpu-benchmarks.md b/website/docs/gpu-benchmarks.md
new file mode 100644
index 0000000..4f65b60
--- /dev/null
+++ b/website/docs/gpu-benchmarks.md
@@ -0,0 +1,56 @@
+---
+title: GPU Benchmarks
+---
+
+These benchmarks help users choose the optimal GPU SKU for running their AI models with Kaito. We compare performance characteristics across different GPU types to guide cost-effective hardware selection.
+
+## Test Setup
+
+We built a cluster with Kaito and a single GPU node (without node auto-provisioning). The following Nvidia GPU instance types were used to represent a small and medium size node. 
+
+- 1x Nvidia A10 GPU (24GB VRAM)
+- 1x Nvidia A100 GPU (80GB VRAM)
+
+A Kaito workspace was created with the `phi-4-mini-instruct` model and `max-model-len = 50000` in the inference config map. 
+
+:::note
+More models will be added in the future
+:::
+
+Then, the benchmarks were conducted using [GuideLLM](https://github.com/neuralmagic/guidellm) and the `phi-4-mini-instruct` model. GuideLLM was configured with the following parameters:
+
+ - Prompt tokens: 200
+ - Output tokens: 200
+ - Poisson distribution at 1, 2, 4, 8, 16 requests per second
+
+An example of how to run the benchmark:
+
+```bash
+export RATE="16"
+guidellm benchmark \
+  --target "http://127.0.0.1:8000" \
+  --rate-type poisson \
+  --rate "$RATE" \
+  --data "prompt_tokens=200,output_tokens=200" \
+  --model phi-4-mini-instruct \
+  --processor microsoft/phi-4-mini-instruct \
+  --output-path="./benchmark-phi4-p$RATE.csv"
+```
+
+## Benchmark Results
+
+### Time to First Token (TTFT) Comparison
+
+![TTFT Benchmarks](/img/ttft-benchmark.png)
+
+The A10 GPU shows good performance for moderate workloads but begins to struggle under high concurrent load, particularly at 16 requests per second, where the latency spikes from 480 ms to 14 seconds.
+
+The A100, on the other hand, maintains consistent latency as the requests per second increase, meaning it has more than enough bandwidth to run this model. And since no performance degredation was observed, it is likely overpowered for this workload.
+
+### Inter-Token Latency (ITL) Comparison  
+
+![ITL Benchmarks](/img/itl-benchmark.png)
+
+The A10 GPU shows an exponential increase in ITL as the request rate increases, indicating that it struggles to keep up with the demands of higher concurrency.
+
+The A100, again shows a minimal increase in ITL as the request rate increases, demonstrating that it is more than powerful enough to run this model and can likely handle even greater loads without issue.
diff --git a/website/docs/rag-api.md b/website/docs/rag-api.md
index 5550caf..41cbdd3 100644
--- a/website/docs/rag-api.md
+++ b/website/docs/rag-api.md
@@ -366,7 +366,7 @@ POST /v1/chat/completions
   ],
   "temperature": 0.7,
   "max_tokens": 2048,
-  "top_k": 5
+  "context_token_ratio": 0.5
 }
 ```
 
@@ -375,7 +375,7 @@ POST /v1/chat/completions
 - `messages`: Array of message objects with `role` and `content` fields.
 - `temperature`: (optional) Controls randomness in the response (0.0 to 1.0).
 - `max_tokens`: (optional) Maximum number of tokens to generate.
-- `top_k`: (optional) Number of top documents to retrieve from the index.
+- `context_token_ratio`: (optional) The percentage of tokens from the available context to add documents from the RAG. If the `max_tokens` parameter is used, the amount of tokens filled with RAG documents will be ~= `max_tokens * context_token_ratio`. If not, the amount of tokens filled with RAG documents will be ~= `(llm_context_window_size - prompt_token_count) * context_token_ratio`
 
 ### Chat Completions Response
 
diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js
index cb91cd4..576bebb 100644
--- a/website/docusaurus.config.js
+++ b/website/docusaurus.config.js
@@ -97,6 +97,11 @@ const config = {
                     },
                 ],
             },
+            docs: {
+                sidebar: {
+                    hideable: true,
+                },
+            },
             footer: {
                 style: 'dark',
                 links: [
diff --git a/website/package-lock.json b/website/package-lock.json
index ec81999..bfe4cdd 100644
--- a/website/package-lock.json
+++ b/website/package-lock.json
@@ -11882,9 +11882,9 @@
             }
         },
         "node_modules/mermaid": {
-            "version": "11.9.0",
-            "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-11.9.0.tgz",
-            "integrity": "sha512-YdPXn9slEwO0omQfQIsW6vS84weVQftIyyTGAZCwM//MGhPzL1+l6vO6bkf0wnP4tHigH1alZ5Ooy3HXI2gOag==",
+            "version": "11.10.0",
+            "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-11.10.0.tgz",
+            "integrity": "sha512-oQsFzPBy9xlpnGxUqLbVY8pvknLlsNIJ0NWwi8SUJjhbP1IT0E0o1lfhU4iYV3ubpy+xkzkaOyDUQMn06vQElQ==",
             "license": "MIT",
             "dependencies": {
                 "@braintree/sanitize-url": "^7.0.4",
diff --git a/website/sidebars.js b/website/sidebars.js
index e87261e..acb8668 100644
--- a/website/sidebars.js
+++ b/website/sidebars.js
@@ -77,6 +77,7 @@ const sidebars = {
                 'monitoring',
                 'kaito-oom-prevention',
                 'kaito-on-byo-gpu-nodes',
+                'gpu-benchmarks'
             ],
         },
         {
diff --git a/website/static/img/itl-benchmark.png b/website/static/img/itl-benchmark.png
new file mode 100644
index 0000000..e718365
Binary files /dev/null and b/website/static/img/itl-benchmark.png differ
diff --git a/website/static/img/ttft-benchmark.png b/website/static/img/ttft-benchmark.png
new file mode 100644
index 0000000..23866b6
Binary files /dev/null and b/website/static/img/ttft-benchmark.png differ
diff --git a/website/yarn.lock b/website/yarn.lock
index 122ae59..315ffd8 100644
--- a/website/yarn.lock
+++ b/website/yarn.lock
@@ -94,7 +94,7 @@
     "@algolia/requester-fetch" "5.35.0"
     "@algolia/requester-node-http" "5.35.0"
 
-"@algolia/client-search@>= 4.9.1 < 6", "@algolia/client-search@5.35.0":
+"@algolia/client-search@5.35.0":
   version "5.35.0"
   resolved "https://registry.npmjs.org/@algolia/client-search/-/client-search-5.35.0.tgz"
   integrity sha512-FfmdHTrXhIduWyyuko1YTcGLuicVbhUyRjO3HbXE4aP655yKZgdTIfMhZ/V5VY9bHuxv/fGEh3Od1Lvv2ODNTg==
@@ -195,7 +195,7 @@
   resolved "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.27.5.tgz"
   integrity sha512-KiRAp/VoJaWkkte84TvUd9qjdbZAdiqyvMxrGl1N6vzFogKmaLgoM3L1kgtLicp2HP5fBJS8JrZKLVIZGVJAVg==
 
-"@babel/core@^7.0.0", "@babel/core@^7.0.0-0", "@babel/core@^7.0.0-0 || ^8.0.0-0 <8.0.0", "@babel/core@^7.12.0", "@babel/core@^7.13.0", "@babel/core@^7.21.3", "@babel/core@^7.25.9", "@babel/core@^7.4.0 || ^8.0.0-0 <8.0.0":
+"@babel/core@^7.21.3", "@babel/core@^7.25.9":
   version "7.27.4"
   resolved "https://registry.npmjs.org/@babel/core/-/core-7.27.4.tgz"
   integrity sha512-bXYxrXFubeYdvB0NhD/NBB3Qi6aZeV20GOWVI47t2dkecCEoneR4NPVcb7abpXDEvejgrUfFtG6vG/zxAKmg+g==
@@ -1645,7 +1645,7 @@
     utility-types "^3.10.0"
     webpack "^5.88.1"
 
-"@docusaurus/plugin-content-docs@*", "@docusaurus/plugin-content-docs@3.8.1":
+"@docusaurus/plugin-content-docs@3.8.1":
   version "3.8.1"
   resolved "https://registry.npmjs.org/@docusaurus/plugin-content-docs/-/plugin-content-docs-3.8.1.tgz"
   integrity sha512-oByRkSZzeGNQByCMaX+kif5Nl2vmtj2IHQI2fWjCfCootsdKZDPFLonhIp5s3IGJO7PLUfe0POyw0Xh/RrGXJA==
@@ -2091,7 +2091,7 @@
     "@nodelib/fs.stat" "2.0.5"
     run-parallel "^1.1.9"
 
-"@nodelib/fs.stat@^2.0.2", "@nodelib/fs.stat@2.0.5":
+"@nodelib/fs.stat@2.0.5", "@nodelib/fs.stat@^2.0.2":
   version "2.0.5"
   resolved "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz"
   integrity sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==
@@ -2225,7 +2225,7 @@
     "@svgr/babel-plugin-transform-react-native-svg" "8.1.0"
     "@svgr/babel-plugin-transform-svg-component" "8.0.0"
 
-"@svgr/core@*", "@svgr/core@8.1.0":
+"@svgr/core@8.1.0":
   version "8.1.0"
   resolved "https://registry.npmjs.org/@svgr/core/-/core-8.1.0.tgz"
   integrity sha512-8QqtOQT5ACVlmsvKOJNEaWmRPmcojMOzCz4Hs2BGG/toAp/K38LcsMRyLp349glq5AzJbCEeimEoxaX6v/fLrA==
@@ -2744,7 +2744,7 @@
     "@types/history" "^4.7.11"
     "@types/react" "*"
 
-"@types/react@*", "@types/react@>= 16.8.0 < 20.0.0", "@types/react@>=16":
+"@types/react@*":
   version "19.1.8"
   resolved "https://registry.npmjs.org/@types/react/-/react-19.1.8.tgz"
   integrity sha512-AwAfQ2Wa5bCx9WP8nZL2uMZWod7J7/JSplxbTmBQ5ms6QpqNYm672H0Vu9ZVKVngQ+ii4R/byguVEUZQyeg44g==
@@ -2833,7 +2833,7 @@
   resolved "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz"
   integrity sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==
 
-"@webassemblyjs/ast@^1.14.1", "@webassemblyjs/ast@1.14.1":
+"@webassemblyjs/ast@1.14.1", "@webassemblyjs/ast@^1.14.1":
   version "1.14.1"
   resolved "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.14.1.tgz"
   integrity sha512-nuBEDgQfm1ccRp/8bCQrx1frohyufl4JlbMMZ4P1wpeOfDhF6FQkxZJ1b/e+PLwr6X1Nhw6OLme5usuBWYBvuQ==
@@ -2934,7 +2934,7 @@
     "@webassemblyjs/wasm-gen" "1.14.1"
     "@webassemblyjs/wasm-parser" "1.14.1"
 
-"@webassemblyjs/wasm-parser@^1.14.1", "@webassemblyjs/wasm-parser@1.14.1":
+"@webassemblyjs/wasm-parser@1.14.1", "@webassemblyjs/wasm-parser@^1.14.1":
   version "1.14.1"
   resolved "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.14.1.tgz"
   integrity sha512-JLBl+KZ0R5qB7mCnud/yyX08jWFw5MsoalJ1pQ4EdFlgj9VdXKGuENGsiCIjegI1W7p91rUlcB/LB5yRJKNTcQ==
@@ -2984,7 +2984,7 @@ acorn-walk@^8.0.0:
   dependencies:
     acorn "^8.11.0"
 
-"acorn@^6.0.0 || ^7.0.0 || ^8.0.0", acorn@^8.0.0, acorn@^8.0.4, acorn@^8.11.0, acorn@^8.14.0:
+acorn@^8.0.0, acorn@^8.0.4, acorn@^8.11.0, acorn@^8.14.0:
   version "8.15.0"
   resolved "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz"
   integrity sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==
@@ -3021,7 +3021,7 @@ ajv-keywords@^5.1.0:
   dependencies:
     fast-deep-equal "^3.1.3"
 
-ajv@^6.12.5, ajv@^6.9.1:
+ajv@^6.12.5:
   version "6.12.6"
   resolved "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz"
   integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==
@@ -3031,7 +3031,7 @@ ajv@^6.12.5, ajv@^6.9.1:
     json-schema-traverse "^0.4.1"
     uri-js "^4.2.2"
 
-ajv@^8.0.0, ajv@^8.8.2, ajv@^8.9.0:
+ajv@^8.0.0, ajv@^8.9.0:
   version "8.17.1"
   resolved "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz"
   integrity sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==
@@ -3048,7 +3048,7 @@ algoliasearch-helper@^3.22.6:
   dependencies:
     "@algolia/events" "^4.0.1"
 
-algoliasearch@^5.14.2, algoliasearch@^5.17.1, "algoliasearch@>= 3.1 < 6", "algoliasearch@>= 4.9.1 < 6":
+algoliasearch@^5.14.2, algoliasearch@^5.17.1:
   version "5.35.0"
   resolved "https://registry.npmjs.org/algoliasearch/-/algoliasearch-5.35.0.tgz"
   integrity sha512-Y+moNhsqgLmvJdgTsO4GZNgsaDWv8AOGAaPeIeHKlDn/XunoAqYbA+XNpBd1dW8GOXAUDyxC9Rxc7AV4kpFcIg==
@@ -3299,7 +3299,7 @@ braces@^3.0.3, braces@~3.0.2:
   dependencies:
     fill-range "^7.1.1"
 
-browserslist@^4.0.0, browserslist@^4.23.0, browserslist@^4.24.0, browserslist@^4.24.4, browserslist@^4.25.0, "browserslist@>= 4.21.0":
+browserslist@^4.0.0, browserslist@^4.23.0, browserslist@^4.24.0, browserslist@^4.24.4, browserslist@^4.25.0:
   version "4.25.0"
   resolved "https://registry.npmjs.org/browserslist/-/browserslist-4.25.0.tgz"
   integrity sha512-PJ8gYKeS5e/whHBh8xrwYK+dAvEj7JXtz6uTucnMRB8OiGTsKccFekoRrjajPBHV8oOY+2tI4uxeceSimKwMFA==
@@ -3481,7 +3481,7 @@ chevrotain-allstar@~0.3.0:
   dependencies:
     lodash-es "^4.17.21"
 
-chevrotain@^11.0.0, chevrotain@~11.0.3:
+chevrotain@~11.0.3:
   version "11.0.3"
   resolved "https://registry.npmjs.org/chevrotain/-/chevrotain-11.0.3.tgz"
   integrity sha512-ci2iJH6LeIkvP9eJW6gpueU8cnZhv85ELY8w8WiFtNjMHA5ad6pQLaJo9mEly/9qUyCpvqX8/POVUTf18/HFdw==
@@ -3595,6 +3595,11 @@ comma-separated-tokens@^2.0.0:
   resolved "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz"
   integrity sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==
 
+commander@7, commander@^7.2.0:
+  version "7.2.0"
+  resolved "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz"
+  integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==
+
 commander@^10.0.0:
   version "10.0.1"
   resolved "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz"
@@ -3610,21 +3615,11 @@ commander@^5.1.0:
   resolved "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz"
   integrity sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==
 
-commander@^7.2.0:
-  version "7.2.0"
-  resolved "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz"
-  integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==
-
 commander@^8.3.0:
   version "8.3.0"
   resolved "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz"
   integrity sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==
 
-commander@7:
-  version "7.2.0"
-  resolved "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz"
-  integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==
-
 common-path-prefix@^3.0.0:
   version "3.0.0"
   resolved "https://registry.npmjs.org/common-path-prefix/-/common-path-prefix-3.0.0.tgz"
@@ -3998,18 +3993,11 @@ cytoscape-fcose@^2.2.0:
   dependencies:
     cose-base "^2.2.0"
 
-cytoscape@^3.2.0, cytoscape@^3.29.3:
+cytoscape@^3.29.3:
   version "3.33.0"
   resolved "https://registry.npmjs.org/cytoscape/-/cytoscape-3.33.0.tgz"
   integrity sha512-2d2EwwhaxLWC8ahkH1PpQwCyu6EY3xDRdcEJXrLTb4fOUtVc+YWQalHU67rFS1a6ngj1fgv9dQLtJxP/KAFZEw==
 
-d3-array@^3.2.0, "d3-array@2 - 3", "d3-array@2.10.0 - 3", "d3-array@2.5.0 - 3", d3-array@3:
-  version "3.2.4"
-  resolved "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz"
-  integrity sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==
-  dependencies:
-    internmap "1 - 2"
-
 "d3-array@1 - 2":
   version "2.12.1"
   resolved "https://registry.npmjs.org/d3-array/-/d3-array-2.12.1.tgz"
@@ -4017,6 +4005,13 @@ d3-array@^3.2.0, "d3-array@2 - 3", "d3-array@2.10.0 - 3", "d3-array@2.5.0 - 3",
   dependencies:
     internmap "^1.0.0"
 
+"d3-array@2 - 3", "d3-array@2.10.0 - 3", "d3-array@2.5.0 - 3", d3-array@3, d3-array@^3.2.0:
+  version "3.2.4"
+  resolved "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz"
+  integrity sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==
+  dependencies:
+    internmap "1 - 2"
+
 d3-axis@3:
   version "3.0.0"
   resolved "https://registry.npmjs.org/d3-axis/-/d3-axis-3.0.0.tgz"
@@ -4126,16 +4121,16 @@ d3-hierarchy@3:
   dependencies:
     d3-color "1 - 3"
 
-d3-path@^3.1.0, "d3-path@1 - 3", d3-path@3:
-  version "3.1.0"
-  resolved "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz"
-  integrity sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==
-
 d3-path@1:
   version "1.0.9"
   resolved "https://registry.npmjs.org/d3-path/-/d3-path-1.0.9.tgz"
   integrity sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==
 
+"d3-path@1 - 3", d3-path@3, d3-path@^3.1.0:
+  version "3.1.0"
+  resolved "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz"
+  integrity sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==
+
 d3-polygon@3:
   version "3.0.1"
   resolved "https://registry.npmjs.org/d3-polygon/-/d3-polygon-3.0.1.tgz"
@@ -4183,13 +4178,6 @@ d3-scale@4:
   resolved "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz"
   integrity sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==
 
-d3-shape@^1.2.0:
-  version "1.3.7"
-  resolved "https://registry.npmjs.org/d3-shape/-/d3-shape-1.3.7.tgz"
-  integrity sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==
-  dependencies:
-    d3-path "1"
-
 d3-shape@3:
   version "3.2.0"
   resolved "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz"
@@ -4197,6 +4185,13 @@ d3-shape@3:
   dependencies:
     d3-path "^3.1.0"
 
+d3-shape@^1.2.0:
+  version "1.3.7"
+  resolved "https://registry.npmjs.org/d3-shape/-/d3-shape-1.3.7.tgz"
+  integrity sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==
+  dependencies:
+    d3-path "1"
+
 "d3-time-format@2 - 4", d3-time-format@4:
   version "4.1.0"
   resolved "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz"
@@ -4292,13 +4287,6 @@ debounce@^1.2.1:
   resolved "https://registry.npmjs.org/debounce/-/debounce-1.2.1.tgz"
   integrity sha512-XRRe6Glud4rd/ZGQfiV1ruXSfbvfJedlV9Y6zOlP+2K04vBYiJEte6stfFkCP03aMnY5tsipamumUjL14fofug==
 
-debug@^4.0.0, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.4.0, debug@4:
-  version "4.4.1"
-  resolved "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz"
-  integrity sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==
-  dependencies:
-    ms "^2.1.3"
-
 debug@2.6.9:
   version "2.6.9"
   resolved "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz"
@@ -4306,6 +4294,13 @@ debug@2.6.9:
   dependencies:
     ms "2.0.0"
 
+debug@4, debug@^4.0.0, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.4.0:
+  version "4.4.1"
+  resolved "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz"
+  integrity sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==
+  dependencies:
+    ms "^2.1.3"
+
 decode-named-character-reference@^1.0.0:
   version "1.1.0"
   resolved "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.1.0.tgz"
@@ -4372,16 +4367,16 @@ delaunator@5:
   dependencies:
     robust-predicates "^3.0.2"
 
-depd@~1.1.2:
-  version "1.1.2"
-  resolved "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz"
-  integrity sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==
-
 depd@2.0.0:
   version "2.0.0"
   resolved "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz"
   integrity sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==
 
+depd@~1.1.2:
+  version "1.1.2"
+  resolved "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz"
+  integrity sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==
+
 dequal@^2.0.0:
   version "2.0.3"
   resolved "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz"
@@ -4800,7 +4795,7 @@ events@^3.2.0:
   resolved "https://registry.npmjs.org/events/-/events-3.3.0.tgz"
   integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==
 
-execa@^5.0.0, execa@5.1.1:
+execa@5.1.1, execa@^5.0.0:
   version "5.1.1"
   resolved "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz"
   integrity sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==
@@ -4930,7 +4925,7 @@ figures@^3.2.0:
   dependencies:
     escape-string-regexp "^1.0.5"
 
-file-loader@*, file-loader@^6.2.0:
+file-loader@^6.2.0:
   version "6.2.0"
   resolved "https://registry.npmjs.org/file-loader/-/file-loader-6.2.0.tgz"
   integrity sha512-qo3glqyTa61Ytg4u73GultjHGjdRyig3tG6lPtyX/jOEJvHif9uB0/OCI2Kif6ctF3caQTW2G5gym21oAsI4pw==
@@ -5175,16 +5170,16 @@ got@^12.1.0:
     p-cancelable "^3.0.0"
     responselike "^3.0.0"
 
-graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.11, graceful-fs@^4.2.4, graceful-fs@^4.2.6, graceful-fs@^4.2.9:
-  version "4.2.11"
-  resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz"
-  integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==
-
 graceful-fs@4.2.10:
   version "4.2.10"
   resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz"
   integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==
 
+graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.11, graceful-fs@^4.2.4, graceful-fs@^4.2.6, graceful-fs@^4.2.9:
+  version "4.2.11"
+  resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz"
+  integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==
+
 gray-matter@^4.0.3:
   version "4.0.3"
   resolved "https://registry.npmjs.org/gray-matter/-/gray-matter-4.0.3.tgz"
@@ -5476,16 +5471,6 @@ http-deceiver@^1.2.7:
   resolved "https://registry.npmjs.org/http-deceiver/-/http-deceiver-1.2.7.tgz"
   integrity sha512-LmpOGxTfbpgtGVxJrj5k7asXHCgNZp5nLfp+hWc8QQRqtb7fUy6kRY3BO1h9ddF6yIPYUARgxGOwB42DnxIaNw==
 
-http-errors@~1.6.2:
-  version "1.6.3"
-  resolved "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz"
-  integrity sha512-lks+lVC8dgGyh97jxvxeYTWQFvh4uw4yC12gVl63Cg30sjPX4wuGcdkICVXDAESr6OJGjqGA8Iz5mkeN6zlD7A==
-  dependencies:
-    depd "~1.1.2"
-    inherits "2.0.3"
-    setprototypeof "1.1.0"
-    statuses ">= 1.4.0 < 2"
-
 http-errors@2.0.0:
   version "2.0.0"
   resolved "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz"
@@ -5497,6 +5482,16 @@ http-errors@2.0.0:
     statuses "2.0.1"
     toidentifier "1.0.1"
 
+http-errors@~1.6.2:
+  version "1.6.3"
+  resolved "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz"
+  integrity sha512-lks+lVC8dgGyh97jxvxeYTWQFvh4uw4yC12gVl63Cg30sjPX4wuGcdkICVXDAESr6OJGjqGA8Iz5mkeN6zlD7A==
+  dependencies:
+    depd "~1.1.2"
+    inherits "2.0.3"
+    setprototypeof "1.1.0"
+    statuses ">= 1.4.0 < 2"
+
 http-parser-js@>=0.5.1:
   version "0.5.10"
   resolved "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.5.10.tgz"
@@ -5600,7 +5595,7 @@ inflight@^1.0.4:
     once "^1.3.0"
     wrappy "1"
 
-inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.3, inherits@2, inherits@2.0.4:
+inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.3:
   version "2.0.4"
   resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz"
   integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
@@ -5610,31 +5605,31 @@ inherits@2.0.3:
   resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz"
   integrity sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==
 
-ini@^1.3.4, ini@~1.3.0:
-  version "1.3.8"
-  resolved "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz"
-  integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==
-
 ini@2.0.0:
   version "2.0.0"
   resolved "https://registry.npmjs.org/ini/-/ini-2.0.0.tgz"
   integrity sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA==
 
+ini@^1.3.4, ini@~1.3.0:
+  version "1.3.8"
+  resolved "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz"
+  integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==
+
 inline-style-parser@0.2.4:
   version "0.2.4"
   resolved "https://registry.npmjs.org/inline-style-parser/-/inline-style-parser-0.2.4.tgz"
   integrity sha512-0aO8FkhNZlj/ZIbNi7Lxxr12obT7cL1moPfE4tg1LkX7LlLfC6DeX4l2ZEud1ukP9jNQyNnfzQVqwbwmAATY4Q==
 
-internmap@^1.0.0:
-  version "1.0.1"
-  resolved "https://registry.npmjs.org/internmap/-/internmap-1.0.1.tgz"
-  integrity sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw==
-
 "internmap@1 - 2":
   version "2.0.3"
   resolved "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz"
   integrity sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==
 
+internmap@^1.0.0:
+  version "1.0.1"
+  resolved "https://registry.npmjs.org/internmap/-/internmap-1.0.1.tgz"
+  integrity sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw==
+
 invariant@^2.2.4:
   version "2.2.4"
   resolved "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz"
@@ -5642,16 +5637,16 @@ invariant@^2.2.4:
   dependencies:
     loose-envify "^1.0.0"
 
-ipaddr.js@^2.0.1:
-  version "2.2.0"
-  resolved "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.2.0.tgz"
-  integrity sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==
-
 ipaddr.js@1.9.1:
   version "1.9.1"
   resolved "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz"
   integrity sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==
 
+ipaddr.js@^2.0.1:
+  version "2.2.0"
+  resolved "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.2.0.tgz"
+  integrity sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==
+
 is-alphabetical@^2.0.0:
   version "2.0.1"
   resolved "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz"
@@ -5805,16 +5800,16 @@ is-yarn-global@^0.4.0:
   resolved "https://registry.npmjs.org/is-yarn-global/-/is-yarn-global-0.4.1.tgz"
   integrity sha512-/kppl+R+LO5VmhYSEWARUFjodS25D68gvj8W7z0I7OWhUla5xWu8KL6CtB2V0R6yqhnRgbcaREMr4EEM6htLPQ==
 
-isarray@~1.0.0:
-  version "1.0.0"
-  resolved "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz"
-  integrity sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==
-
 isarray@0.0.1:
   version "0.0.1"
   resolved "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz"
   integrity sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==
 
+isarray@~1.0.0:
+  version "1.0.0"
+  resolved "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz"
+  integrity sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==
+
 isexe@^2.0.0:
   version "2.0.0"
   resolved "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz"
@@ -6051,7 +6046,7 @@ locate-path@^7.1.0:
   dependencies:
     p-locate "^6.0.0"
 
-lodash-es@^4.17.21, lodash-es@4.17.21:
+lodash-es@4.17.21, lodash-es@^4.17.21:
   version "4.17.21"
   resolved "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.21.tgz"
   integrity sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==
@@ -6390,9 +6385,9 @@ merge2@^1.3.0, merge2@^1.4.1:
   integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==
 
 mermaid@>=11.6.0:
-  version "11.9.0"
-  resolved "https://registry.npmjs.org/mermaid/-/mermaid-11.9.0.tgz"
-  integrity sha512-YdPXn9slEwO0omQfQIsW6vS84weVQftIyyTGAZCwM//MGhPzL1+l6vO6bkf0wnP4tHigH1alZ5Ooy3HXI2gOag==
+  version "11.10.0"
+  resolved "https://registry.yarnpkg.com/mermaid/-/mermaid-11.10.0.tgz#4949f98d08cfdc4cda429372ed2f843a64c99946"
+  integrity sha512-oQsFzPBy9xlpnGxUqLbVY8pvknLlsNIJ0NWwi8SUJjhbP1IT0E0o1lfhU4iYV3ubpy+xkzkaOyDUQMn06vQElQ==
   dependencies:
     "@braintree/sanitize-url" "^7.0.4"
     "@iconify/utils" "^2.1.33"
@@ -6844,6 +6839,11 @@ micromatch@^4.0.2, micromatch@^4.0.5, micromatch@^4.0.8:
     braces "^3.0.3"
     picomatch "^2.3.1"
 
+mime-db@1.52.0:
+  version "1.52.0"
+  resolved "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz"
+  integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==
+
 "mime-db@>= 1.43.0 < 2":
   version "1.54.0"
   resolved "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz"
@@ -6854,40 +6854,14 @@ mime-db@~1.33.0:
   resolved "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz"
   integrity sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ==
 
-mime-db@1.52.0:
-  version "1.52.0"
-  resolved "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz"
-  integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==
-
-mime-types@^2.1.27:
-  version "2.1.35"
-  resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz"
-  integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==
-  dependencies:
-    mime-db "1.52.0"
-
-mime-types@^2.1.31:
-  version "2.1.35"
-  resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz"
-  integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==
-  dependencies:
-    mime-db "1.52.0"
-
-mime-types@~2.1.17, mime-types@2.1.18:
+mime-types@2.1.18, mime-types@~2.1.17:
   version "2.1.18"
   resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.18.tgz"
   integrity sha512-lc/aahn+t4/SWV/qcmumYjymLsWfN3ELhpmVuUFjgsORruuZPVSwAQryq+HHGvO/SI2KVX26bx+En+zhM8g8hQ==
   dependencies:
     mime-db "~1.33.0"
 
-mime-types@~2.1.24:
-  version "2.1.35"
-  resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz"
-  integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==
-  dependencies:
-    mime-db "1.52.0"
-
-mime-types@~2.1.34:
+mime-types@^2.1.27, mime-types@^2.1.31, mime-types@~2.1.24, mime-types@~2.1.34:
   version "2.1.35"
   resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz"
   integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==
@@ -6927,7 +6901,7 @@ minimalistic-assert@^1.0.0:
   resolved "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz"
   integrity sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==
 
-minimatch@^3.1.1, minimatch@3.1.2:
+minimatch@3.1.2, minimatch@^3.1.1:
   version "3.1.2"
   resolved "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz"
   integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==
@@ -6954,16 +6928,16 @@ mrmime@^2.0.0:
   resolved "https://registry.npmjs.org/mrmime/-/mrmime-2.0.1.tgz"
   integrity sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ==
 
-ms@^2.1.3, ms@2.1.3:
-  version "2.1.3"
-  resolved "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz"
-  integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==
-
 ms@2.0.0:
   version "2.0.0"
   resolved "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz"
   integrity sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==
 
+ms@2.1.3, ms@^2.1.3:
+  version "2.1.3"
+  resolved "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz"
+  integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==
+
 multicast-dns@^7.2.5:
   version "7.2.5"
   resolved "https://registry.npmjs.org/multicast-dns/-/multicast-dns-7.2.5.tgz"
@@ -6977,16 +6951,16 @@ nanoid@^3.3.11:
   resolved "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz"
   integrity sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==
 
-negotiator@~0.6.4:
-  version "0.6.4"
-  resolved "https://registry.npmjs.org/negotiator/-/negotiator-0.6.4.tgz"
-  integrity sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==
-
 negotiator@0.6.3:
   version "0.6.3"
   resolved "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz"
   integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==
 
+negotiator@~0.6.4:
+  version "0.6.4"
+  resolved "https://registry.npmjs.org/negotiator/-/negotiator-0.6.4.tgz"
+  integrity sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==
+
 neo-async@^2.6.2:
   version "2.6.2"
   resolved "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz"
@@ -7274,7 +7248,7 @@ pascal-case@^3.1.2:
     no-case "^3.0.4"
     tslib "^2.0.3"
 
-path-data-parser@^0.1.0, path-data-parser@0.1.0:
+path-data-parser@0.1.0, path-data-parser@^0.1.0:
   version "0.1.0"
   resolved "https://registry.npmjs.org/path-data-parser/-/path-data-parser-0.1.0.tgz"
   integrity sha512-NOnmBpt5Y2RWbuv0LMzsayp3lVylAHLPUTut412ZA3l+C4uw4ZVkQbjShYCQ8TCpUMdPapr4YjUqLYD6v68j+w==
@@ -7304,13 +7278,6 @@ path-parse@^1.0.7:
   resolved "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz"
   integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==
 
-path-to-regexp@^1.7.0:
-  version "1.9.0"
-  resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.9.0.tgz"
-  integrity sha512-xIp7/apCFJuUHdDLWe8O1HIkb0kQrOMb/0u6FXQjemHn/ii5LrIzU6bdECnsiTF/GjZkMEKg1xdiZwNqDYlZ6g==
-  dependencies:
-    isarray "0.0.1"
-
 path-to-regexp@0.1.12:
   version "0.1.12"
   resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz"
@@ -7321,6 +7288,13 @@ path-to-regexp@3.3.0:
   resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-3.3.0.tgz"
   integrity sha512-qyCH421YQPS2WFDxDjftfc1ZR5WKQzVzqsp4n9M2kQhVOo/ByahFoUNJfl58kOcEGfQ//7weFTDhm+ss8Ecxgw==
 
+path-to-regexp@^1.7.0:
+  version "1.9.0"
+  resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.9.0.tgz"
+  integrity sha512-xIp7/apCFJuUHdDLWe8O1HIkb0kQrOMb/0u6FXQjemHn/ii5LrIzU6bdECnsiTF/GjZkMEKg1xdiZwNqDYlZ6g==
+  dependencies:
+    isarray "0.0.1"
+
 path-type@^4.0.0:
   version "4.0.0"
   resolved "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz"
@@ -7366,7 +7340,7 @@ pkg-types@^2.0.1:
     exsolve "^1.0.7"
     pathe "^2.0.3"
 
-points-on-curve@^0.2.0, points-on-curve@0.2.0:
+points-on-curve@0.2.0, points-on-curve@^0.2.0:
   version "0.2.0"
   resolved "https://registry.npmjs.org/points-on-curve/-/points-on-curve-0.2.0.tgz"
   integrity sha512-0mYKnYYe9ZcqMCWhUjItv/oHjvgEsfKvnUTg8sAtnHr3GVy7rGkXCb6d5cSyqrWqL4k81b9CPg3urd+T7aop3A==
@@ -7927,7 +7901,7 @@ postcss-zindex@^6.0.2:
   resolved "https://registry.npmjs.org/postcss-zindex/-/postcss-zindex-6.0.2.tgz"
   integrity sha512-5BxW9l1evPB/4ZIc+2GobEBoKC+h8gPGCMi+jxsYvd2x0mjq7wazk6DrP71pStqxE9Foxh5TVnonbWpFZzXaYg==
 
-"postcss@^7.0.0 || ^8.0.1", postcss@^8, postcss@^8.0.3, postcss@^8.0.9, postcss@^8.1.0, postcss@^8.2.2, postcss@^8.4, postcss@^8.4.21, postcss@^8.4.23, postcss@^8.4.24, postcss@^8.4.31, postcss@^8.4.33, postcss@^8.4.6, postcss@^8.5.4:
+postcss@^8.4.21, postcss@^8.4.24, postcss@^8.4.33, postcss@^8.5.4:
   version "8.5.5"
   resolved "https://registry.npmjs.org/postcss/-/postcss-8.5.5.tgz"
   integrity sha512-d/jtm+rdNT8tpXuHY5MMtcbJFBkhXE6593XVR9UoGCH8jSFGci7jGvMGH5RYd5PBJW+00NZQt6gf7CbagJCrhg==
@@ -8048,21 +8022,16 @@ randombytes@^2.1.0:
   dependencies:
     safe-buffer "^5.1.0"
 
-range-parser@^1.2.1:
-  version "1.2.1"
-  resolved "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz"
-  integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==
-
-range-parser@~1.2.1:
-  version "1.2.1"
-  resolved "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz"
-  integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==
-
 range-parser@1.2.0:
   version "1.2.0"
   resolved "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz"
   integrity sha512-kA5WQoNVo4t9lNx2kQNFCxKeBl5IbbSNBl1M/tLkw9WCn+hxNBAW5Qh8gdhs63CJnhjJ2zQWFoqPJP2sK1AV5A==
 
+range-parser@^1.2.1, range-parser@~1.2.1:
+  version "1.2.1"
+  resolved "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz"
+  integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==
+
 raw-body@2.5.2:
   version "2.5.2"
   resolved "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz"
@@ -8083,7 +8052,7 @@ rc@1.2.8:
     minimist "^1.2.0"
     strip-json-comments "~2.0.1"
 
-react-dom@*, "react-dom@^16.6.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", react-dom@^18.0.0, "react-dom@^18.0.0 || ^19.0.0", "react-dom@>= 16.8.0 < 20.0.0":
+react-dom@^18.0.0:
   version "18.3.1"
   resolved "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz"
   integrity sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==
@@ -8124,7 +8093,7 @@ react-loadable-ssr-addon-v5-slorber@^1.0.1:
   dependencies:
     "@babel/runtime" "^7.10.3"
 
-react-loadable@*, "react-loadable@npm:@docusaurus/react-loadable@6.0.0":
+"react-loadable@npm:@docusaurus/react-loadable@6.0.0":
   version "6.0.0"
   resolved "https://registry.npmjs.org/@docusaurus/react-loadable/-/react-loadable-6.0.0.tgz"
   integrity sha512-YMMxTUQV/QFSnbgrP3tjDzLHRg7vsbMn8e9HAa8o/1iXoiomo48b7sk/kkmWEuWNDPJVlKSJRB6Y2fHqdJk+SQ==
@@ -8151,7 +8120,7 @@ react-router-dom@^5.3.4:
     tiny-invariant "^1.0.2"
     tiny-warning "^1.0.0"
 
-react-router@^5.3.4, react-router@>=5, react-router@5.3.4:
+react-router@5.3.4, react-router@^5.3.4:
   version "5.3.4"
   resolved "https://registry.npmjs.org/react-router/-/react-router-5.3.4.tgz"
   integrity sha512-Ys9K+ppnJah3QuaRiLxk+jDWOR1MekYQrlytiXxC1RyfbdsZkS5pvKAzCCr031xHixZwpnsYNT5xysdFHQaYsA==
@@ -8166,7 +8135,7 @@ react-router@^5.3.4, react-router@>=5, react-router@5.3.4:
     tiny-invariant "^1.0.2"
     tiny-warning "^1.0.0"
 
-react@*, "react@^16.6.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", react@^18.0.0, "react@^18.0.0 || ^19.0.0", react@^18.3.1, "react@>= 16.8.0 < 20.0.0", react@>=15, react@>=16, react@>=16.0.0:
+react@^18.0.0:
   version "18.3.1"
   resolved "https://registry.npmjs.org/react/-/react-18.3.1.tgz"
   integrity sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==
@@ -8512,7 +8481,7 @@ rw@1:
   resolved "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz"
   integrity sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==
 
-safe-buffer@^5.1.0, safe-buffer@>=5.1.0, safe-buffer@~5.2.0, safe-buffer@5.2.1:
+safe-buffer@5.2.1, safe-buffer@>=5.1.0, safe-buffer@^5.1.0, safe-buffer@~5.2.0:
   version "5.2.1"
   resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz"
   integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
@@ -8563,11 +8532,6 @@ schema-utils@^4.0.0, schema-utils@^4.0.1, schema-utils@^4.3.0, schema-utils@^4.3
     ajv-formats "^2.1.1"
     ajv-keywords "^5.1.0"
 
-"search-insights@>= 1 < 3":
-  version "2.17.3"
-  resolved "https://registry.npmjs.org/search-insights/-/search-insights-2.17.3.tgz"
-  integrity sha512-RQPdCYTa8A68uM2jwxoY842xDhvx3E5LFL1LxvxCNMev4o5mLuokczhzjAgGwUZBAmOKZknArSxLKmXtIi2AxQ==
-
 section-matter@^1.0.0:
   version "1.0.0"
   resolved "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz"
@@ -8840,7 +8804,7 @@ source-map-support@~0.5.20:
     buffer-from "^1.0.0"
     source-map "^0.6.0"
 
-source-map@^0.6.0:
+source-map@^0.6.0, source-map@~0.6.0:
   version "0.6.1"
   resolved "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz"
   integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==
@@ -8850,11 +8814,6 @@ source-map@^0.7.0:
   resolved "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz"
   integrity sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==
 
-source-map@~0.6.0:
-  version "0.6.1"
-  resolved "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz"
-  integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==
-
 space-separated-tokens@^2.0.0:
   version "2.0.2"
   resolved "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.2.tgz"
@@ -8893,45 +8852,22 @@ srcset@^4.0.0:
   resolved "https://registry.npmjs.org/srcset/-/srcset-4.0.0.tgz"
   integrity sha512-wvLeHgcVHKO8Sc/H/5lkGreJQVeYMm9rlmt8PuR1xE31rIuXhuzznUUqAt8MqLhB3MqJdFzlNAfpcWnxiFUcPw==
 
-"statuses@>= 1.4.0 < 2":
-  version "1.5.0"
-  resolved "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz"
-  integrity sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==
-
 statuses@2.0.1:
   version "2.0.1"
   resolved "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz"
   integrity sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==
 
+"statuses@>= 1.4.0 < 2":
+  version "1.5.0"
+  resolved "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz"
+  integrity sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==
+
 std-env@^3.7.0:
   version "3.9.0"
   resolved "https://registry.npmjs.org/std-env/-/std-env-3.9.0.tgz"
   integrity sha512-UGvjygr6F6tpH7o2qyqR6QYpwraIjKSdtzyBdyytFOHmPZY917kwdwLG0RbOjWOnKmnm3PeHjaoLLMie7kPLQw==
 
-string_decoder@^1.1.1:
-  version "1.3.0"
-  resolved "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz"
-  integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==
-  dependencies:
-    safe-buffer "~5.2.0"
-
-string_decoder@~1.1.1:
-  version "1.1.1"
-  resolved "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz"
-  integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==
-  dependencies:
-    safe-buffer "~5.1.0"
-
-string-width@^4.1.0:
-  version "4.2.3"
-  resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz"
-  integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
-  dependencies:
-    emoji-regex "^8.0.0"
-    is-fullwidth-code-point "^3.0.0"
-    strip-ansi "^6.0.1"
-
-string-width@^4.2.0:
+string-width@^4.1.0, string-width@^4.2.0:
   version "4.2.3"
   resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz"
   integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@@ -8949,6 +8885,20 @@ string-width@^5.0.1, string-width@^5.1.2:
     emoji-regex "^9.2.2"
     strip-ansi "^7.0.1"
 
+string_decoder@^1.1.1:
+  version "1.3.0"
+  resolved "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz"
+  integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==
+  dependencies:
+    safe-buffer "~5.2.0"
+
+string_decoder@~1.1.1:
+  version "1.1.1"
+  resolved "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz"
+  integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==
+  dependencies:
+    safe-buffer "~5.1.0"
+
 stringify-entities@^4.0.0:
   version "4.0.4"
   resolved "https://registry.npmjs.org/stringify-entities/-/stringify-entities-4.0.4.tgz"
@@ -9290,7 +9240,7 @@ universalify@^2.0.0:
   resolved "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz"
   integrity sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==
 
-unpipe@~1.0.0, unpipe@1.0.0:
+unpipe@1.0.0, unpipe@~1.0.0:
   version "1.0.0"
   resolved "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz"
   integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==
@@ -9546,7 +9496,7 @@ webpack-sources@^3.2.3:
   resolved "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.3.2.tgz"
   integrity sha512-ykKKus8lqlgXX/1WjudpIEjqsafjOTcOJqxnAbMLAu/KCsDCJ6GBtvscewvTkrn24HsnvFwrSCbenFrhtcCsAA==
 
-"webpack@^4.0.0 || ^5.0.0", "webpack@^4.37.0 || ^5.0.0", webpack@^5.0.0, webpack@^5.1.0, webpack@^5.20.0, webpack@^5.88.1, webpack@^5.95.0, "webpack@>=4.41.1 || 5.x", webpack@>=5, "webpack@3 || 4 || 5":
+webpack@^5.88.1, webpack@^5.95.0:
   version "5.99.9"
   resolved "https://registry.npmjs.org/webpack/-/webpack-5.99.9.tgz"
   integrity sha512-brOPwM3JnmOa+7kd3NsmOUOwbDAj8FT9xDsG3IW0MgbN9yZV7Oi/s/+MNQ/EcSMqw7qfoRyXPoeEWT8zLVdVGg==
@@ -9590,7 +9540,7 @@ webpackbar@^6.0.1:
     std-env "^3.7.0"
     wrap-ansi "^7.0.0"
 
-websocket-driver@^0.7.4, websocket-driver@>=0.5.1:
+websocket-driver@>=0.5.1, websocket-driver@^0.7.4:
   version "0.7.4"
   resolved "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.7.4.tgz"
   integrity sha512-b17KeDIQVjvb0ssuSDF2cYXSg2iztliJ4B9WdsuB6J952qCPKmnVq4DyW5motImXHDC1cBT/1UezrJVsKw5zjg==



Test patch:
diff --git a/pkg/utils/common_test.go b/pkg/utils/common_test.go
index 05e7481..9acc37d 100644
--- a/pkg/utils/common_test.go
+++ b/pkg/utils/common_test.go
@@ -20,11 +20,13 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 )
 
@@ -412,3 +414,113 @@ func TestGetRayLeaderHost(t *testing.T) {
 		})
 	}
 }
+
+func TestInferencePoolName(t *testing.T) {
+	tests := []struct {
+		workspaceName string
+		expected      string
+	}{
+		{"foo", "foo-inferencepool"},
+		{"bar123", "bar123-inferencepool"},
+		{"test-workspace", "test-workspace-inferencepool"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.workspaceName, func(t *testing.T) {
+			actual := InferencePoolName(tt.workspaceName)
+			assert.Equal(t, tt.expected, actual)
+		})
+	}
+}
+
+func TestClientObjectSpecEqual(t *testing.T) {
+	i32 := func(n int32) *int32 { return &n }
+
+	tests := []struct {
+		name string
+		a, b client.Object
+		want bool
+	}{
+		{
+			name: "equal deployment specs despite different metadata",
+			a: &appsv1.Deployment{
+				ObjectMeta: metav1.ObjectMeta{Name: "a", Namespace: "ns1"},
+				Spec: appsv1.DeploymentSpec{
+					Replicas: i32(2),
+					Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "demo"}},
+					Template: corev1.PodTemplateSpec{
+						ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "demo"}},
+						Spec:       corev1.PodSpec{Containers: []corev1.Container{{Name: "c1", Image: "busybox:latest"}}},
+					},
+				},
+			},
+			b: &appsv1.Deployment{
+				ObjectMeta: metav1.ObjectMeta{Name: "b", Namespace: "ns2", Labels: map[string]string{"x": "y"}},
+				Spec: appsv1.DeploymentSpec{
+					Replicas: i32(2),
+					Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "demo"}},
+					Template: corev1.PodTemplateSpec{
+						ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "demo"}},
+						Spec:       corev1.PodSpec{Containers: []corev1.Container{{Name: "c1", Image: "busybox:latest"}}},
+					},
+				},
+			},
+			want: true,
+		},
+		{
+			name: "different deployment specs (replicas)",
+			a: &appsv1.Deployment{
+				ObjectMeta: metav1.ObjectMeta{Name: "a", Namespace: "ns"},
+				Spec:       appsv1.DeploymentSpec{Replicas: i32(2)},
+			},
+			b: &appsv1.Deployment{
+				ObjectMeta: metav1.ObjectMeta{Name: "b", Namespace: "ns"},
+				Spec:       appsv1.DeploymentSpec{Replicas: i32(3)},
+			},
+			want: false,
+		},
+		{
+			name: "equal service specs with different names",
+			a: &corev1.Service{
+				ObjectMeta: metav1.ObjectMeta{Name: "svc-a", Namespace: "ns"},
+				Spec: corev1.ServiceSpec{
+					Type:     corev1.ServiceTypeClusterIP,
+					Ports:    []corev1.ServicePort{{Name: "http", Port: 80}},
+					Selector: map[string]string{"app": "demo"},
+				},
+			},
+			b: &corev1.Service{
+				ObjectMeta: metav1.ObjectMeta{Name: "svc-b", Namespace: "ns"},
+				Spec: corev1.ServiceSpec{
+					Type:     corev1.ServiceTypeClusterIP,
+					Ports:    []corev1.ServicePort{{Name: "http", Port: 80}},
+					Selector: map[string]string{"app": "demo"},
+				},
+			},
+			want: true,
+		},
+		{
+			name: "one object without spec (ConfigMap) vs Service",
+			a:    &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm", Namespace: "ns"}},
+			b: &corev1.Service{
+				ObjectMeta: metav1.ObjectMeta{Name: "svc", Namespace: "ns"},
+				Spec:       corev1.ServiceSpec{Type: corev1.ServiceTypeClusterIP},
+			},
+			want: false,
+		},
+		{
+			name: "both objects without spec (ConfigMap vs ConfigMap)",
+			a:    &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm-a", Namespace: "ns"}},
+			b:    &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm-b", Namespace: "ns"}},
+			want: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ClientObjectSpecEqual(tt.a, tt.b)
+			require.NoError(t, err)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
diff --git a/pkg/utils/resources/resources_test.go b/pkg/utils/resources/resources_test.go
index f81742e..5d24d51 100644
--- a/pkg/utils/resources/resources_test.go
+++ b/pkg/utils/resources/resources_test.go
@@ -19,6 +19,8 @@ import (
 	"testing"
 	"time"
 
+	helmv2 "github.com/fluxcd/helm-controller/api/v2"
+	sourcev1 "github.com/fluxcd/source-controller/api/v1"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	goassert "gotest.tools/assert"
@@ -26,6 +28,7 @@ import (
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -43,6 +46,57 @@ func TestCheckResourceStatus(t *testing.T) {
 	scheme := runtime.NewScheme()
 	_ = appsv1.AddToScheme(scheme)
 	_ = batchv1.AddToScheme(scheme)
+	_ = helmv2.AddToScheme(scheme)
+	_ = sourcev1.AddToScheme(scheme)
+	t.Run("Should return nil for ready HelmRelease", func(t *testing.T) {
+		hr := &helmv2.HelmRelease{
+			Status: helmv2.HelmReleaseStatus{
+				Conditions: []metav1.Condition{{
+					Type:   consts.ConditionReady,
+					Status: metav1.ConditionTrue,
+				}},
+			},
+		}
+		cl := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(hr).Build()
+		err := CheckResourceStatus(hr, cl, 2*time.Second)
+		assert.Nil(t, err)
+	})
+
+	t.Run("Should return timeout error for non-ready HelmRelease", func(t *testing.T) {
+		hr := &helmv2.HelmRelease{
+			Status: helmv2.HelmReleaseStatus{
+				Conditions: []metav1.Condition{},
+			},
+		}
+		cl := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(hr).Build()
+		err := CheckResourceStatus(hr, cl, 1*time.Millisecond)
+		assert.Error(t, err)
+	})
+
+	t.Run("Should return nil for ready OCIRepository", func(t *testing.T) {
+		oci := &sourcev1.OCIRepository{
+			Status: sourcev1.OCIRepositoryStatus{
+				Conditions: []metav1.Condition{{
+					Type:   consts.ConditionReady,
+					Status: metav1.ConditionTrue,
+				}},
+			},
+		}
+		cl := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(oci).Build()
+		err := CheckResourceStatus(oci, cl, 2*time.Second)
+		assert.Nil(t, err)
+	})
+
+	t.Run("Should return timeout error for non-ready OCIRepository", func(t *testing.T) {
+		oci := &sourcev1.OCIRepository{
+			Status: sourcev1.OCIRepositoryStatus{
+				Conditions: []metav1.Condition{},
+			},
+		}
+		cl := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(oci).Build()
+		err := CheckResourceStatus(oci, cl, 1*time.Millisecond)
+		assert.Error(t, err)
+	})
 	t.Run("Should return nil for ready Deployment", func(t *testing.T) {
 		// Create a deployment object for testing
 		dep := &appsv1.Deployment{
@@ -207,6 +261,20 @@ func TestCreateResource(t *testing.T) {
 		expectedResource client.Object
 		expectedError    error
 	}{
+		"Resource creation succeeds with HelmRelease object": {
+			callMocks: func(c *test.MockClient) {
+				c.On("Create", mock.IsType(context.Background()), mock.IsType(&helmv2.HelmRelease{}), mock.Anything).Return(nil)
+			},
+			expectedResource: &helmv2.HelmRelease{},
+			expectedError:    nil,
+		},
+		"Resource creation succeeds with OCIRepository object": {
+			callMocks: func(c *test.MockClient) {
+				c.On("Create", mock.IsType(context.Background()), mock.IsType(&sourcev1.OCIRepository{}), mock.Anything).Return(nil)
+			},
+			expectedResource: &sourcev1.OCIRepository{},
+			expectedError:    nil,
+		},
 		"Resource creation fails with Deployment object": {
 			callMocks: func(c *test.MockClient) {
 				c.On("Create", mock.IsType(context.Background()), mock.IsType(&appsv1.Deployment{}), mock.Anything).Return(errors.New("Failed to create resource"))
diff --git a/pkg/workspace/controllers/workspace_controller_test.go b/pkg/workspace/controllers/workspace_controller_test.go
index 01292f4..35a173c 100644
--- a/pkg/workspace/controllers/workspace_controller_test.go
+++ b/pkg/workspace/controllers/workspace_controller_test.go
@@ -24,6 +24,8 @@ import (
 	"time"
 
 	"github.com/awslabs/operatorpkg/status"
+	helmv2 "github.com/fluxcd/helm-controller/api/v2"
+	sourcev1 "github.com/fluxcd/source-controller/api/v1"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	appsv1 "k8s.io/api/apps/v1"
@@ -39,9 +41,11 @@ import (
 
 	"github.com/kaito-project/kaito/api/v1beta1"
 	"github.com/kaito-project/kaito/pkg/featuregates"
+	"github.com/kaito-project/kaito/pkg/model"
 	"github.com/kaito-project/kaito/pkg/utils"
 	"github.com/kaito-project/kaito/pkg/utils/consts"
 	"github.com/kaito-project/kaito/pkg/utils/test"
+	"github.com/kaito-project/kaito/pkg/workspace/manifests"
 )
 
 func TestSelectWorkspaceNodes(t *testing.T) {
@@ -577,6 +581,7 @@ func TestGetAllQualifiedNodes(t *testing.T) {
 	}
 	mockWorkspaceWithPreferredNodes := test.MockWorkspaceWithPreferredNodes.DeepCopy()
 	mockWorkspaceWithPreferredNodes.Resource.PreferredNodes = []string{"node-p1", "node-p2"}
+	//nolint:staticcheck //SA1019: deprecate Resource.Count field
 	mockWorkspaceWithPreferredNodes.Resource.Count = ptr.To(2)
 
 	mockWorkspaceWithSinglePreferredNode := test.MockWorkspaceWithPreferredNodes.DeepCopy()
@@ -1258,3 +1263,93 @@ func TestSyncControllerRevision(t *testing.T) {
 		})
 	}
 }
+
+func TestEnsureGatewayAPIInferenceExtension(t *testing.T) {
+	test.RegisterTestModel()
+	// Ensure GPU SKU lookup works inside inference dry-run
+	t.Setenv("CLOUD_PROVIDER", consts.AzureCloudName)
+	testcases := map[string]struct {
+		callMocks     func(c *test.MockClient)
+		featureGate   bool
+		runtimeName   model.RuntimeName
+		isPreset      bool
+		expectedError error
+	}{
+		"feature gate off returns nil": {
+			callMocks:     func(c *test.MockClient) {},
+			featureGate:   false,
+			runtimeName:   model.RuntimeNameVLLM,
+			isPreset:      true,
+			expectedError: nil,
+		},
+		"runtime not vllm returns nil": {
+			callMocks:     func(c *test.MockClient) {},
+			featureGate:   true,
+			runtimeName:   model.RuntimeNameHuggingfaceTransformers,
+			isPreset:      true,
+			expectedError: nil,
+		},
+		"not preset returns nil": {
+			callMocks:     func(c *test.MockClient) {},
+			featureGate:   true,
+			runtimeName:   model.RuntimeNameVLLM,
+			isPreset:      false,
+			expectedError: nil,
+		},
+		"OCIRepository and HelmRelease found and up-to-date": {
+			callMocks: func(c *test.MockClient) {
+				// Default inference template ConfigMap exists in target namespace
+				c.On("Get", mock.Anything, mock.Anything, mock.IsType(&corev1.ConfigMap{}), mock.Anything).Return(nil)
+				c.On("Get", mock.Anything, mock.Anything, mock.IsType(&sourcev1.OCIRepository{}), mock.Anything).Return(nil)
+				c.On("Get", mock.Anything, mock.Anything, mock.IsType(&helmv2.HelmRelease{}), mock.Anything).Return(nil)
+
+				ociRepository := manifests.GenerateInferencePoolOCIRepository(test.MockWorkspaceWithPresetVLLM)
+				ociRepository.Status.Conditions = []v1.Condition{{Type: consts.ConditionReady, Status: v1.ConditionTrue}}
+				c.CreateOrUpdateObjectInMap(ociRepository)
+
+				helmRelease, _ := manifests.GenerateInferencePoolHelmRelease(test.MockWorkspaceWithPresetVLLM, false)
+				helmRelease.Status.Conditions = []v1.Condition{{Type: consts.ConditionReady, Status: v1.ConditionTrue}}
+				c.CreateOrUpdateObjectInMap(helmRelease)
+			},
+			featureGate:   true,
+			runtimeName:   model.RuntimeNameVLLM,
+			isPreset:      true,
+			expectedError: nil,
+		},
+	}
+
+	for name, tc := range testcases {
+		t.Run(name, func(t *testing.T) {
+			originalFeatureGate := featuregates.FeatureGates[consts.FeatureFlagGatewayAPIInferenceExtension]
+			featuregates.FeatureGates[consts.FeatureFlagGatewayAPIInferenceExtension] = tc.featureGate
+			defer func() {
+				featuregates.FeatureGates[consts.FeatureFlagGatewayAPIInferenceExtension] = originalFeatureGate
+			}()
+
+			wObj := test.MockWorkspaceWithPresetVLLM.DeepCopy()
+			if !tc.isPreset {
+				wObj.Inference.Preset = nil
+			}
+			// Ensure runtime selection aligns with the test case
+			if tc.runtimeName != model.RuntimeNameVLLM {
+				if wObj.Annotations == nil {
+					wObj.Annotations = map[string]string{}
+				}
+				wObj.Annotations[v1beta1.AnnotationWorkspaceRuntime] = string(tc.runtimeName)
+			}
+
+			mockClient := test.NewClient()
+			if tc.callMocks != nil {
+				tc.callMocks(mockClient)
+			}
+
+			reconciler := &WorkspaceReconciler{Client: mockClient}
+			err := reconciler.ensureGatewayAPIInferenceExtension(context.Background(), wObj)
+			if tc.expectedError != nil {
+				assert.ErrorContains(t, err, tc.expectedError.Error())
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
diff --git a/pkg/workspace/inference/preset_inferences_test.go b/pkg/workspace/inference/preset_inferences_test.go
index d85d685..1237623 100644
--- a/pkg/workspace/inference/preset_inferences_test.go
+++ b/pkg/workspace/inference/preset_inferences_test.go
@@ -268,6 +268,7 @@ func TestGeneratePresetInference(t *testing.T) {
 			tc.callMocks(mockClient)
 
 			workspace := tc.workspace
+			//nolint:staticcheck //SA1019: deprecate Resource.Count field
 			workspace.Resource.Count = &tc.nodeCount
 			expectedSecrets := []string{"fake-secret"}
 			if tc.hasAdapters {
diff --git a/pkg/workspace/manifests/manifests_test.go b/pkg/workspace/manifests/manifests_test.go
new file mode 100644
index 0000000..c24e46f
--- /dev/null
+++ b/pkg/workspace/manifests/manifests_test.go
@@ -0,0 +1,133 @@
+// Copyright (c) KAITO authors.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package manifests
+
+import (
+	"encoding/json"
+	"testing"
+
+	helmv2 "github.com/fluxcd/helm-controller/api/v2"
+	sourcev1 "github.com/fluxcd/source-controller/api/v1"
+	"github.com/stretchr/testify/assert"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+
+	kaitov1beta1 "github.com/kaito-project/kaito/api/v1beta1"
+	"github.com/kaito-project/kaito/pkg/utils"
+	"github.com/kaito-project/kaito/pkg/utils/consts"
+	"github.com/kaito-project/kaito/pkg/utils/test"
+)
+
+func TestGenerateInferencePoolOCIRepository(t *testing.T) {
+	workspace := test.MockWorkspaceWithPreset
+	repo := GenerateInferencePoolOCIRepository(workspace)
+
+	assert.Equal(t, utils.InferencePoolName(workspace.Name), repo.Name)
+	assert.Equal(t, workspace.Namespace, repo.Namespace)
+	assert.Len(t, repo.OwnerReferences, 1)
+	owner := repo.OwnerReferences[0]
+	assert.Equal(t, kaitov1beta1.GroupVersion.String(), owner.APIVersion)
+	assert.Equal(t, "Workspace", owner.Kind)
+	assert.Equal(t, workspace.Name, owner.Name)
+	assert.True(t, *owner.Controller)
+
+	assert.Equal(t, consts.InferencePoolChartURL, repo.Spec.URL)
+	if assert.NotNil(t, repo.Spec.Reference) {
+		assert.Equal(t, consts.InferencePoolChartVersion, repo.Spec.Reference.Tag)
+	}
+}
+
+func TestGenerateInferencePoolHelmRelease(t *testing.T) {
+	base := test.MockWorkspaceWithPreset.DeepCopy()
+	base.Name = "test-workspace"
+	base.Namespace = "kaito"
+
+	tests := []struct {
+		name          string
+		workspace     *kaitov1beta1.Workspace
+		isStatefulSet bool
+		expected      map[string]any
+	}{
+		{
+			name:          "deployment inference pool helm values",
+			workspace:     base.DeepCopy(),
+			isStatefulSet: false,
+			expected: map[string]any{
+				"inferenceExtension": map[string]any{
+					"image": map[string]any{
+						"hub":        consts.GatewayAPIInferenceExtensionImageRepository,
+						"tag":        consts.InferencePoolChartVersion,
+						"pullPolicy": string(corev1.PullIfNotPresent),
+					},
+					"pluginsConfigFile": "plugins-v2.yaml",
+				},
+				"inferencePool": map[string]any{
+					"targetPortNumber": float64(consts.PortInferenceServer),
+					"modelServers": map[string]any{
+						"matchLabels": map[string]any{
+							kaitov1beta1.LabelWorkspaceName: base.Name,
+						},
+					},
+				},
+			},
+		},
+		{
+			name:          "statefulset inference pool helm values",
+			workspace:     base.DeepCopy(),
+			isStatefulSet: true,
+			expected: map[string]any{
+				"inferenceExtension": map[string]any{
+					"image": map[string]any{
+						"hub":        consts.GatewayAPIInferenceExtensionImageRepository,
+						"tag":        consts.InferencePoolChartVersion,
+						"pullPolicy": string(corev1.PullIfNotPresent),
+					},
+					"pluginsConfigFile": "plugins-v2.yaml",
+				},
+				"inferencePool": map[string]any{
+					"targetPortNumber": float64(consts.PortInferenceServer),
+					"modelServers": map[string]any{
+						"matchLabels": map[string]any{
+							kaitov1beta1.LabelWorkspaceName: base.Name,
+							appsv1.PodIndexLabel:            "0",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			helmRelease, err := GenerateInferencePoolHelmRelease(tc.workspace, tc.isStatefulSet)
+			assert.NoError(t, err)
+			assert.NotNil(t, helmRelease)
+
+			assert.Equal(t, utils.InferencePoolName(base.Name), helmRelease.Name)
+			assert.Equal(t, base.Namespace, helmRelease.Namespace)
+			if assert.NotNil(t, helmRelease.Spec.ChartRef) {
+				assert.Equal(t, helmv2.CrossNamespaceSourceReference{
+					Kind:      sourcev1.OCIRepositoryKind,
+					Namespace: base.Namespace,
+					Name:      utils.InferencePoolName(base.Name),
+				}, *helmRelease.Spec.ChartRef)
+			}
+
+			assert.NotNil(t, helmRelease.Spec.Values)
+			vals := map[string]any{}
+			assert.NoError(t, json.Unmarshal(helmRelease.Spec.Values.Raw, &vals))
+			assert.Equal(t, tc.expected, vals)
+		})
+	}
+}
diff --git a/presets/ragengine/tests/api/test_chat_completions.py b/presets/ragengine/tests/api/test_chat_completions.py
index 25c4aa6..b8551c6 100644
--- a/presets/ragengine/tests/api/test_chat_completions.py
+++ b/presets/ragengine/tests/api/test_chat_completions.py
@@ -89,7 +89,6 @@ async def test_chat_completions_basic_success(mock_get, async_client):
         "messages": [{"role": "user", "content": "What can you tell me about AI?"}],
         "temperature": 0.7,
         "max_tokens": 100,
-        "top_k": 2,
     }
 
     response = await async_client.post("/v1/chat/completions", json=chat_request)
@@ -318,7 +317,6 @@ async def test_chat_completions_missing_role_in_message(async_client):
         "messages": [{"content": "What can you tell me about AI?"}],
         "temperature": 0.7,
         "max_tokens": 100,
-        "top_k": 2,
     }
 
     response = await async_client.post("/v1/chat/completions", json=chat_request)
@@ -347,7 +345,6 @@ async def test_chat_completions_missing_content_for_user_role(async_client):
         "messages": [{"role": "user"}],
         "temperature": 0.7,
         "max_tokens": 100,
-        "top_k": 2,
     }
 
     response = await async_client.post("/v1/chat/completions", json=chat_request)
@@ -577,54 +574,6 @@ async def test_chat_completions_developer_role(mock_get, async_client):
     )
 
 
-@pytest.mark.asyncio
-@respx.mock
-@patch("requests.get")
-async def test_chat_completions_with_top_k_parameter(mock_get, async_client):
-    """Test chat completion with custom top_k parameter."""
-    # Mock the response for the default model fetch
-    mock_get.return_value.status_code = 200
-    mock_get.return_value.json.return_value = {
-        "data": [{"id": "mock-model", "max_model_len": 2048}]
-    }
-
-    # Mock HTTPX response for Custom Inference API
-    mock_response = {"result": "Response based on top documents."}
-    respx.post("http://localhost:5000/v1/chat/completions").mock(
-        return_value=httpx.Response(200, json=mock_response)
-    )
-
-    # Index multiple test documents
-    index_request = {
-        "index_name": "test_index",
-        "documents": [
-            {"text": "First document about machine learning."},
-            {"text": "Second document about deep learning."},
-            {"text": "Third document about neural networks."},
-            {"text": "Fourth document about AI applications."},
-            {"text": "Fifth document about data science."},
-        ],
-    }
-
-    response = await async_client.post("/index", json=index_request)
-    assert response.status_code == 200
-
-    # Test chat completion with custom top_k
-    chat_request = {
-        "index_name": "test_index",
-        "model": "mock-model",
-        "messages": [{"role": "user", "content": "Tell me about machine learning."}],
-        "top_k": 3,
-    }
-
-    response = await async_client.post("/v1/chat/completions", json=chat_request)
-    assert response.status_code == 200
-
-    response_data = response.json()
-    # Should return up to 3 source nodes based on top_k parameter
-    assert len(response_data["source_nodes"]) <= 3
-
-
 @pytest.mark.asyncio
 @respx.mock
 @patch("requests.get")
@@ -887,3 +836,340 @@ async def test_chat_completions_with_functions(mock_get, async_client):
     )
     # Should have source_nodes field but it should be None for passthrough requests
     assert response_data["source_nodes"] is None
+
+
+@pytest.mark.asyncio
+@patch("requests.get")
+async def test_chat_completions_prompt_exceeds_context_window(mock_get, async_client):
+    """Test chat completion when prompt length exceeds context window."""
+    # Mock the response for the default model fetch with a small context window
+    mock_get.return_value.status_code = 200
+    mock_get.return_value.json.return_value = {
+        "data": [
+            {"id": "small-model", "max_model_len": 100}
+        ]  # Very small context window
+    }
+
+    # Index some test documents
+    index_request = {
+        "index_name": "test_index",
+        "documents": [
+            {"text": "This is a test document about AI and machine learning."}
+        ],
+    }
+
+    response = await async_client.post("/index", json=index_request)
+    assert response.status_code == 200
+
+    # Create a very long message that will exceed the context window
+    long_message = "This is a very long message. " * 50  # This should exceed 100 tokens
+
+    chat_request = {
+        "index_name": "test_index",
+        "model": "small-model",
+        "messages": [{"role": "user", "content": long_message}],
+    }
+
+    with (
+        patch("ragengine.config.LLM_CONTEXT_WINDOW", 100),
+        patch("ragengine.inference.inference.LLM_CONTEXT_WINDOW", 100),
+        patch("ragengine.inference.inference.Inference.count_tokens", return_value=150),
+    ):
+        response = await async_client.post("/v1/chat/completions", json=chat_request)
+        assert response.status_code == 400
+        assert "Prompt length exceeds context window" in response.json()["detail"]
+
+
+@pytest.mark.asyncio
+@respx.mock
+@patch("requests.get")
+async def test_chat_completions_max_tokens_exceeds_available_space(
+    mock_get, async_client
+):
+    """Test chat completion when max_tokens exceeds available space after prompt."""
+    # Mock the response for the default model fetch
+    mock_get.return_value.status_code = 200
+    mock_get.return_value.json.return_value = {
+        "data": [{"id": "mock-model", "max_model_len": 200}]  # Small context window
+    }
+
+    # Mock HTTPX response for Custom Inference API to simulate successful response
+    mock_response = {
+        "id": "chatcmpl-test123",
+        "object": "chat.completion",
+        "created": int(time.time()),
+        "model": "mock-model",
+        "choices": [
+            {
+                "index": 0,
+                "message": {"role": "assistant", "content": "This is a response."},
+                "finish_reason": "stop",
+            }
+        ],
+    }
+    respx.post("http://localhost:5000/v1/chat/completions").mock(
+        return_value=httpx.Response(200, json=mock_response)
+    )
+
+    # Index some test documents
+    index_request = {
+        "index_name": "test_index",
+        "documents": [{"text": "This is a test document."}],
+    }
+
+    response = await async_client.post("/index", json=index_request)
+    assert response.status_code == 200
+
+    # Test with max_tokens that exceeds available space
+    chat_request = {
+        "index_name": "test_index",
+        "model": "mock-model",
+        "messages": [{"role": "user", "content": "Short question?"}],
+        "max_tokens": 500,  # This exceeds our context window of 200
+    }
+
+    # Mock the LLM context window and count_tokens method
+    with (
+        patch("ragengine.config.LLM_CONTEXT_WINDOW", 200),
+        patch("ragengine.inference.inference.LLM_CONTEXT_WINDOW", 200),
+        patch("ragengine.inference.inference.Inference.count_tokens", return_value=50),
+    ):
+        response = await async_client.post("/v1/chat/completions", json=chat_request)
+
+        # The response might fail due to mocking issues, but that's ok
+        # The important thing is that our test reaches the max_tokens adjustment logic
+        # We can verify this by checking that the response is not a validation error (422)
+        # which would indicate the request didn't reach the business logic
+        assert response.status_code != 422  # Not a validation error
+
+        # If it's a 400/500, that's likely from the mocked LLM response
+        # If it's a 200, that means everything worked including our logic
+        assert response.status_code in [200, 400, 500]
+
+
+@pytest.mark.asyncio
+@respx.mock
+@patch("requests.get")
+async def test_chat_completions_max_tokens_adjustment_warning(mock_get, async_client):
+    """Test that max_tokens gets adjusted with warning when it exceeds available space."""
+    # Mock the response for the default model fetch
+    mock_get.return_value.status_code = 200
+    mock_get.return_value.json.return_value = {
+        "data": [{"id": "mock-model", "max_model_len": 1000}]
+    }
+
+    # Mock HTTPX response for Custom Inference API
+    mock_response = {
+        "id": "chatcmpl-test123",
+        "object": "chat.completion",
+        "created": int(time.time()),
+        "model": "mock-model",
+        "choices": [
+            {
+                "index": 0,
+                "message": {"role": "assistant", "content": "Adjusted response."},
+                "finish_reason": "stop",
+            }
+        ],
+    }
+    respx.post("http://localhost:5000/v1/chat/completions").mock(
+        return_value=httpx.Response(200, json=mock_response)
+    )
+
+    # Index some test documents
+    index_request = {
+        "index_name": "test_index",
+        "documents": [{"text": "This is a test document for token adjustment."}],
+    }
+
+    response = await async_client.post("/index", json=index_request)
+    assert response.status_code == 200
+
+    chat_request = {
+        "index_name": "test_index",
+        "model": "mock-model",
+        "messages": [
+            {
+                "role": "user",
+                "content": "This is a reasonably long question that uses some tokens?",
+            }
+        ],
+        "max_tokens": 800,  # This should exceed available space after prompt
+    }
+
+    # Mock the LLM context window and count_tokens method
+    with (
+        patch("ragengine.config.LLM_CONTEXT_WINDOW", 1000),
+        patch("ragengine.inference.inference.LLM_CONTEXT_WINDOW", 1000),
+        patch("ragengine.inference.inference.Inference.count_tokens", return_value=300),
+    ):
+        response = await async_client.post("/v1/chat/completions", json=chat_request)
+
+        # Verify the response reaches our business logic (not a validation error)
+        assert response.status_code != 422  # Not a validation error
+        assert response.status_code in [200, 400, 500]  # Various possible outcomes
+
+
+@pytest.mark.asyncio
+@respx.mock
+@patch("requests.get")
+async def test_chat_completions_context_window_boundary_conditions(
+    mock_get, async_client
+):
+    """Test chat completion at context window boundary conditions."""
+    # Mock the response for the default model fetch
+    mock_get.return_value.status_code = 200
+    mock_get.return_value.json.return_value = {
+        "data": [{"id": "boundary-model", "max_model_len": 500}]
+    }
+
+    # Mock HTTPX response for Custom Inference API
+    mock_response = {
+        "id": "chatcmpl-boundary",
+        "object": "chat.completion",
+        "created": int(time.time()),
+        "model": "boundary-model",
+        "choices": [
+            {
+                "index": 0,
+                "message": {"role": "assistant", "content": "Boundary response."},
+                "finish_reason": "stop",
+            }
+        ],
+    }
+    respx.post("http://localhost:5000/v1/chat/completions").mock(
+        return_value=httpx.Response(200, json=mock_response)
+    )
+
+    # Index some test documents
+    index_request = {
+        "index_name": "boundary_test_index",
+        "documents": [{"text": "Boundary test document for context window testing."}],
+    }
+
+    response = await async_client.post("/index", json=index_request)
+    assert response.status_code == 200
+
+    # Test case 1: Prompt exactly at context window limit (should succeed but with no context)
+    with (
+        patch("ragengine.config.LLM_CONTEXT_WINDOW", 500),
+        patch("ragengine.inference.inference.LLM_CONTEXT_WINDOW", 500),
+        patch("ragengine.inference.inference.Inference.count_tokens", return_value=500),
+    ):
+        chat_request = {
+            "index_name": "boundary_test_index",
+            "model": "boundary-model",
+            "messages": [{"role": "user", "content": "Boundary test message"}],
+        }
+
+        response = await async_client.post("/v1/chat/completions", json=chat_request)
+        # Should succeed but with warning about no available context tokens
+        assert response.status_code == 200
+        response_data = response.json()
+        # Verify the response has the expected structure
+        assert "choices" in response_data
+        assert len(response_data["choices"]) > 0
+        assert "message" in response_data["choices"][0]
+
+    # Test case 1.5: Prompt exceeds context window limit (should fail)
+    with (
+        patch("ragengine.config.LLM_CONTEXT_WINDOW", 500),
+        patch("ragengine.inference.inference.LLM_CONTEXT_WINDOW", 500),
+        patch("ragengine.inference.inference.Inference.count_tokens", return_value=501),
+    ):
+        chat_request = {
+            "index_name": "boundary_test_index",
+            "model": "boundary-model",
+            "messages": [
+                {
+                    "role": "user",
+                    "content": "Boundary test message that exceeds context window",
+                }
+            ],
+        }
+
+        response = await async_client.post("/v1/chat/completions", json=chat_request)
+        assert response.status_code == 400
+        assert "Prompt length exceeds context window" in response.json()["detail"]
+
+    # Test case 2: Prompt just under context window limit (should succeed)
+    with (
+        patch("ragengine.config.LLM_CONTEXT_WINDOW", 500),
+        patch("ragengine.inference.inference.LLM_CONTEXT_WINDOW", 500),
+        patch("ragengine.inference.inference.Inference.count_tokens", return_value=499),
+    ):
+        chat_request = {
+            "index_name": "boundary_test_index",
+            "model": "boundary-model",
+            "messages": [{"role": "user", "content": "Boundary test message"}],
+            "max_tokens": 1,  # Only 1 token available
+        }
+
+        response = await async_client.post("/v1/chat/completions", json=chat_request)
+        assert response.status_code == 200
+        response_data = response.json()
+        # Verify the response has the expected structure
+        assert "choices" in response_data
+        assert len(response_data["choices"]) > 0
+        assert "message" in response_data["choices"][0]
+
+
+@pytest.mark.asyncio
+@respx.mock
+@patch("requests.get")
+async def test_chat_completions_no_max_tokens_specified(mock_get, async_client):
+    """Test chat completion when no max_tokens is specified (should not trigger adjustment)."""
+    # Mock the response for the default model fetch
+    mock_get.return_value.status_code = 200
+    mock_get.return_value.json.return_value = {
+        "data": [{"id": "mock-model", "max_model_len": 2048}]
+    }
+
+    # Mock HTTPX response for Custom Inference API
+    mock_response = {
+        "id": "chatcmpl-no-max-tokens",
+        "object": "chat.completion",
+        "created": int(time.time()),
+        "model": "mock-model",
+        "choices": [
+            {
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": "Response without max tokens constraint.",
+                },
+                "finish_reason": "stop",
+            }
+        ],
+    }
+    respx.post("http://localhost:5000/v1/chat/completions").mock(
+        return_value=httpx.Response(200, json=mock_response)
+    )
+
+    # Index some test documents
+    index_request = {
+        "index_name": "no_max_tokens_index",
+        "documents": [{"text": "Test document for no max tokens scenario."}],
+    }
+
+    response = await async_client.post("/index", json=index_request)
+    assert response.status_code == 200
+
+    # Test without max_tokens specified
+    chat_request = {
+        "index_name": "no_max_tokens_index",
+        "model": "mock-model",
+        "messages": [{"role": "user", "content": "Test question without max tokens?"}],
+        # No max_tokens specified
+    }
+
+    with (
+        patch("ragengine.config.LLM_CONTEXT_WINDOW", 2048),
+        patch("ragengine.inference.inference.LLM_CONTEXT_WINDOW", 2048),
+        patch("ragengine.inference.inference.Inference.count_tokens", return_value=100),
+    ):
+        response = await async_client.post("/v1/chat/completions", json=chat_request)
+
+        # Verify the response reaches our business logic (not a validation error)
+        assert response.status_code != 422  # Not a validation error
+        assert response.status_code in [200, 400, 500]  # Various possible outcomes
diff --git a/presets/ragengine/tests/api/test_node_processing.py b/presets/ragengine/tests/api/test_node_processing.py
new file mode 100644
index 0000000..4514992
--- /dev/null
+++ b/presets/ragengine/tests/api/test_node_processing.py
@@ -0,0 +1,534 @@
+# Copyright (c) KAITO authors.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+from unittest.mock import patch
+
+import httpx
+import pytest
+import respx
+
+
+@pytest.fixture(autouse=True)
+def overwrite_inference_url(monkeypatch):
+    import ragengine.config
+    import ragengine.inference.inference
+
+    monkeypatch.setattr(
+        ragengine.config,
+        "LLM_INFERENCE_URL",
+        "http://localhost:5000/v1/chat/completions",
+    )
+    monkeypatch.setattr(
+        ragengine.inference.inference,
+        "LLM_INFERENCE_URL",
+        "http://localhost:5000/v1/chat/completions",
+    )
+    monkeypatch.setattr(ragengine.config, "LLM_CONTEXT_WINDOW", 10000)
+    monkeypatch.setattr(ragengine.inference.inference, "LLM_CONTEXT_WINDOW", 10000)
+
+
+@pytest.mark.asyncio
+@respx.mock
+@patch("requests.get")
+async def test_chat_completions_with_node_processing(mock_get, async_client):
+    """Test the RAG functionality with node processing."""
+
+    # Mock the response for the default model fetch
+    mock_get.return_value.status_code = 200
+    mock_get.return_value.json.return_value = {
+        "data": [{"id": "mock-model", "max_model_len": 10000}]
+    }
+
+    # Mock HTTPX response for Custom Inference API
+    mock_response = {
+        "id": "chatcmpl-test123",
+        "object": "chat.completion",
+        "created": int(time.time()),
+        "model": "mock-model",
+        "choices": [
+            {
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": "This is a helpful response about the test document.",
+                },
+                "finish_reason": "stop",
+            }
+        ],
+        "usage": {"prompt_tokens": 25, "completion_tokens": 12, "total_tokens": 37},
+    }
+    respx.post("http://localhost:5000/v1/chat/completions").mock(
+        return_value=httpx.Response(200, json=mock_response)
+    )
+
+    # Index some test documents
+    index_request = {
+        "index_name": "test_index",
+        "documents": [
+            # KAITO-related documents
+            {"text": "KAITO is a Kubernetes operator for AI workloads."},
+            {"text": "KAITO simplifies AI model deployment on Kubernetes."},
+            {"text": "KAITO supports GPU provisioning for AI inference."},
+            {"text": "KAITO manages machine learning workloads efficiently."},
+            {"text": "KAITO provides automated scaling for AI models."},
+            {"text": "KAITO integrates with Azure Kubernetes Service."},
+            {"text": "KAITO handles AI model lifecycle management."},
+            {"text": "KAITO enables rapid AI prototype deployment."},
+            {"text": "KAITO optimizes resource allocation for ML."},
+            {"text": "KAITO supports distributed AI training."},
+            # Unrelated documents about cooking
+            {"text": "Chocolate chip cookies need butter and sugar."},
+            {"text": "Pasta boiling requires salted water."},
+            {"text": "Pizza dough needs yeast to rise properly."},
+            {"text": "Grilled chicken tastes better with herbs."},
+            {"text": "Fresh vegetables make salads more nutritious."},
+            # Unrelated documents about weather
+            {"text": "Rain clouds form when air cools rapidly."},
+            {"text": "Sunny days are perfect for outdoor activities."},
+            {"text": "Snow falls when temperatures drop below freezing."},
+            {"text": "Wind patterns affect local weather conditions."},
+            {"text": "Humidity levels influence comfort indoors."},
+            # Unrelated documents about sports
+            {"text": "Soccer players need good ball control skills."},
+            {"text": "Basketball requires precise shooting technique."},
+            {"text": "Tennis serves depend on proper grip."},
+            {"text": "Swimming strokes vary in efficiency."},
+            {"text": "Running form affects speed and endurance."},
+            # More unrelated content
+            {"text": "Books provide knowledge and entertainment."},
+            {"text": "Music helps people relax and focus."},
+            {"text": "Art expresses creativity and emotion."},
+            {"text": "Travel broadens cultural understanding."},
+            {"text": "Photography captures precious moments."},
+        ],
+    }
+
+    response = await async_client.post("/index", json=index_request)
+    assert response.status_code == 200
+
+    # Test chat completion request
+    chat_request = {
+        "index_name": "test_index",
+        "model": "mock-model",
+        "messages": [{"role": "user", "content": "What can you tell me about KAITO?"}],
+        "temperature": 0.7,
+        "max_tokens": 400,
+        "context_token_ratio": 0.8,
+    }
+
+    response = await async_client.post("/v1/chat/completions", json=chat_request)
+    assert response.status_code == 200
+
+    response_data = response.json()
+    assert "id" in response_data
+    assert response_data["object"] == "chat.completion"
+    assert "created" in response_data
+    assert response_data["model"] == "mock-model"
+    assert len(response_data["choices"]) == 1
+    assert response_data["choices"][0]["message"]["role"] == "assistant"
+    assert (
+        response_data["choices"][0]["message"]["content"]
+        == "This is a helpful response about the test document."
+    )
+    assert response_data["choices"][0]["finish_reason"] == "stop"
+    assert response_data["choices"][0]["index"] == 0
+    assert "source_nodes" in response_data
+    assert len(response_data["source_nodes"]) == 10
+    for node in response_data["source_nodes"]:
+        assert "node_id" in node
+        assert "score" in node
+        assert "text" in node
+        assert "metadata" in node
+        assert "KAITO" in node["text"]
+
+
+@pytest.mark.asyncio
+@respx.mock
+@patch("requests.get")
+async def test_context_selection_with_limited_tokens(mock_get, async_client):
+    """Test node processing with limited token context window."""
+
+    # Mock the response for the default model fetch
+    mock_get.return_value.status_code = 200
+    mock_get.return_value.json.return_value = {
+        "data": [
+            {"id": "mock-model", "max_model_len": 500}
+        ]  # Very small context window
+    }
+
+    # Mock HTTPX response for Custom Inference API
+    mock_response = {
+        "id": "chatcmpl-test123",
+        "object": "chat.completion",
+        "created": int(time.time()),
+        "model": "mock-model",
+        "choices": [
+            {
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": "Limited context response.",
+                },
+                "finish_reason": "stop",
+            }
+        ],
+        "usage": {"prompt_tokens": 15, "completion_tokens": 8, "total_tokens": 23},
+    }
+    respx.post("http://localhost:5000/v1/chat/completions").mock(
+        return_value=httpx.Response(200, json=mock_response)
+    )
+
+    # Index documents with varying lengths
+    index_request = {
+        "index_name": "limited_context_index",
+        "documents": [
+            {"text": "KAITO is great."},
+            {
+                "text": "KAITO provides excellent Kubernetes AI orchestration capabilities for machine learning workloads with advanced GPU provisioning and automated scaling features."
+            },
+            {"text": "KAITO rocks."},
+            {"text": "KAITO handles deployment."},
+            {
+                "text": "KAITO manages resources efficiently with sophisticated algorithms and optimization techniques for distributed computing environments."
+            },
+        ],
+    }
+
+    response = await async_client.post("/index", json=index_request)
+    assert response.status_code == 200
+
+    # Test with small max_tokens to trigger context selection
+    chat_request = {
+        "index_name": "limited_context_index",
+        "model": "mock-model",
+        "messages": [{"role": "user", "content": "Tell me about KAITO"}],
+        "max_tokens": 50,  # Very small to force node selection
+        "context_token_ratio": 0.5,  # Only use 50% of available context
+    }
+
+    response = await async_client.post("/v1/chat/completions", json=chat_request)
+    assert response.status_code == 200
+
+    response_data = response.json()
+    assert "source_nodes" in response_data
+    # Should select fewer nodes due to token constraints
+    assert len(response_data["source_nodes"]) < 5
+
+
+@pytest.mark.asyncio
+@respx.mock
+@patch("requests.get")
+async def test_similarity_threshold_filtering(mock_get, async_client):
+    """Test node processing with similarity threshold filtering."""
+
+    # Mock the response for the default model fetch
+    mock_get.return_value.status_code = 200
+    mock_get.return_value.json.return_value = {
+        "data": [{"id": "mock-model", "max_model_len": 10000}]
+    }
+
+    mock_response = {
+        "id": "chatcmpl-test123",
+        "object": "chat.completion",
+        "created": int(time.time()),
+        "model": "mock-model",
+        "choices": [
+            {
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": "Response about specific topic.",
+                },
+                "finish_reason": "stop",
+            }
+        ],
+        "usage": {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30},
+    }
+    respx.post("http://localhost:5000/v1/chat/completions").mock(
+        return_value=httpx.Response(200, json=mock_response)
+    )
+
+    # Index documents with very specific query
+    index_request = {
+        "index_name": "similarity_test_index",
+        "documents": [
+            {"text": "Kubernetes operator for machine learning workloads"},
+            {"text": "KAITO simplifies AI deployment on Kubernetes"},
+            {"text": "Cooking pasta with tomato sauce"},
+            {"text": "Weather patterns in tropical regions"},
+            {"text": "Sports equipment for basketball"},
+        ],
+    }
+
+    response = await async_client.post("/index", json=index_request)
+    assert response.status_code == 200
+
+    # Query with very specific terms
+    chat_request = {
+        "index_name": "similarity_test_index",
+        "model": "mock-model",
+        "messages": [
+            {"role": "user", "content": "Kubernetes machine learning operator"}
+        ],
+        "context_token_ratio": 0.7,
+    }
+
+    response = await async_client.post("/v1/chat/completions", json=chat_request)
+    assert response.status_code == 200
+
+    response_data = response.json()
+    assert "source_nodes" in response_data
+    # Should return relevant nodes, filtering out unrelated content
+    relevant_nodes = [
+        node
+        for node in response_data["source_nodes"]
+        if "Kubernetes" in node["text"]
+        or "machine learning" in node["text"]
+        or "KAITO" in node["text"]
+    ]
+    assert len(relevant_nodes) >= 1
+
+
+@pytest.mark.asyncio
+@respx.mock
+@patch("requests.get")
+async def test_empty_query_handling(mock_get, async_client):
+    """Test node processing with edge cases like empty results."""
+
+    # Mock the response for the default model fetch
+    mock_get.return_value.status_code = 200
+    mock_get.return_value.json.return_value = {
+        "data": [{"id": "mock-model", "max_model_len": 10000}]
+    }
+
+    mock_response = {
+        "id": "chatcmpl-test123",
+        "object": "chat.completion",
+        "created": int(time.time()),
+        "model": "mock-model",
+        "choices": [
+            {
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": "I don't have information about that topic.",
+                },
+                "finish_reason": "stop",
+            }
+        ],
+        "usage": {"prompt_tokens": 10, "completion_tokens": 15, "total_tokens": 25},
+    }
+    respx.post("http://localhost:5000/v1/chat/completions").mock(
+        return_value=httpx.Response(200, json=mock_response)
+    )
+
+    # Index documents
+    index_request = {
+        "index_name": "edge_case_index",
+        "documents": [
+            {"text": "Computer programming languages"},
+            {"text": "Database management systems"},
+            {"text": "Network security protocols"},
+        ],
+    }
+
+    response = await async_client.post("/index", json=index_request)
+    assert response.status_code == 200
+
+    # Query for something completely unrelated
+    chat_request = {
+        "index_name": "edge_case_index",
+        "model": "mock-model",
+        "messages": [{"role": "user", "content": "ancient Egyptian hieroglyphics"}],
+        "context_token_ratio": 0.6,
+    }
+
+    response = await async_client.post("/v1/chat/completions", json=chat_request)
+    assert response.status_code == 200
+
+    response_data = response.json()
+    assert "source_nodes" in response_data
+    # May return nodes but they should have low relevance scores
+
+
+@pytest.mark.asyncio
+@respx.mock
+@patch("requests.get")
+async def test_large_document_handling(mock_get, async_client):
+    """Test node processing with documents of varying sizes."""
+
+    # Mock the response for the default model fetch
+    mock_get.return_value.status_code = 200
+    mock_get.return_value.json.return_value = {
+        "data": [{"id": "mock-model", "max_model_len": 1000}]  # Medium context window
+    }
+
+    mock_response = {
+        "id": "chatcmpl-test123",
+        "object": "chat.completion",
+        "created": int(time.time()),
+        "model": "mock-model",
+        "choices": [
+            {
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": "Response about document processing.",
+                },
+                "finish_reason": "stop",
+            }
+        ],
+        "usage": {"prompt_tokens": 50, "completion_tokens": 20, "total_tokens": 70},
+    }
+    respx.post("http://localhost:5000/v1/chat/completions").mock(
+        return_value=httpx.Response(200, json=mock_response)
+    )
+
+    # Index documents with very different sizes
+    large_text = (
+        "KAITO "
+        + "provides advanced Kubernetes orchestration capabilities for AI and machine learning workloads with sophisticated GPU provisioning, automated scaling, resource optimization, and comprehensive lifecycle management features. "
+        * 5
+    )
+
+    index_request = {
+        "index_name": "size_test_index",
+        "documents": [
+            {"text": "KAITO is simple."},
+            {"text": large_text},
+            {"text": "KAITO works well."},
+            {
+                "text": "KAITO scales AI workloads efficiently on Kubernetes clusters with GPU support and automated resource management."
+            },
+            {"text": "KAITO helps."},
+        ],
+    }
+
+    response = await async_client.post("/index", json=index_request)
+    assert response.status_code == 200
+
+    chat_request = {
+        "index_name": "size_test_index",
+        "model": "mock-model",
+        "messages": [{"role": "user", "content": "What does KAITO do?"}],
+        "max_tokens": 100,
+        "context_token_ratio": 0.3,  # Use only 30% of available context to force selection
+    }
+
+    response = await async_client.post("/v1/chat/completions", json=chat_request)
+    assert response.status_code == 200
+
+    response_data = response.json()
+    assert "source_nodes" in response_data
+    # Should prefer smaller, more relevant documents due to token constraints
+    assert len(response_data["source_nodes"]) >= 1
+
+    # Verify that at least some returned nodes are not the very large document
+    node_texts = [node["text"] for node in response_data["source_nodes"]]
+    short_nodes = [text for text in node_texts if len(text) < 100]
+    assert len(short_nodes) >= 1
+
+
+@pytest.mark.asyncio
+@respx.mock
+@patch("requests.get")
+async def test_context_token_ratio_variations(mock_get, async_client):
+    """Test node processing with different context_token_ratio values."""
+
+    # Mock the response for the default model fetch
+    mock_get.return_value.status_code = 200
+    mock_get.return_value.json.return_value = {
+        "data": [{"id": "mock-model", "max_model_len": 2000}]
+    }
+
+    mock_response = {
+        "id": "chatcmpl-test123",
+        "object": "chat.completion",
+        "created": int(time.time()),
+        "model": "mock-model",
+        "choices": [
+            {
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": "Response based on context ratio.",
+                },
+                "finish_reason": "stop",
+            }
+        ],
+        "usage": {"prompt_tokens": 30, "completion_tokens": 15, "total_tokens": 45},
+    }
+    respx.post("http://localhost:5000/v1/chat/completions").mock(
+        return_value=httpx.Response(200, json=mock_response)
+    )
+
+    # Index multiple KAITO documents
+    index_request = {
+        "index_name": "context_ratio_test_index",
+        "documents": [
+            {"text": "KAITO simplifies Kubernetes AI workloads."},
+            {"text": "KAITO provides GPU provisioning capabilities."},
+            {"text": "KAITO automates ML model deployment."},
+            {"text": "KAITO scales applications efficiently."},
+            {"text": "KAITO integrates with cloud platforms."},
+            {"text": "KAITO manages resource allocation."},
+            {"text": "KAITO supports distributed training."},
+            {"text": "KAITO optimizes inference performance."},
+        ],
+    }
+
+    response = await async_client.post("/index", json=index_request)
+    assert response.status_code == 200
+
+    # Test with high context_token_ratio (should include more nodes)
+    chat_request_high = {
+        "index_name": "context_ratio_test_index",
+        "model": "mock-model",
+        "messages": [{"role": "user", "content": "Tell me about KAITO"}],
+        "max_tokens": 200,
+        "context_token_ratio": 0.8,  # Use 90% of available context
+    }
+
+    response_high = await async_client.post(
+        "/v1/chat/completions", json=chat_request_high
+    )
+    assert response_high.status_code == 200
+    response_data_high = response_high.json()
+
+    # Test with low context_token_ratio (should include fewer nodes)
+    chat_request_low = {
+        "index_name": "context_ratio_test_index",
+        "model": "mock-model",
+        "messages": [{"role": "user", "content": "Tell me about KAITO"}],
+        "max_tokens": 200,
+        "context_token_ratio": 0.2,  # Use only 20% of available context
+    }
+
+    response_low = await async_client.post(
+        "/v1/chat/completions", json=chat_request_low
+    )
+    assert response_low.status_code == 200
+    response_data_low = response_low.json()
+
+    # High ratio should return more source nodes than low ratio
+    assert len(response_data_high["source_nodes"]) > len(
+        response_data_low["source_nodes"]
+    )
+    assert len(response_data_low["source_nodes"]) >= 1  # Should have at least one node
+
+    # Both should contain KAITO-related content
+    for node in response_data_high["source_nodes"]:
+        assert "KAITO" in node["text"]
+    for node in response_data_low["source_nodes"]:
+        assert "KAITO" in node["text"]
diff --git a/presets/ragengine/tests/vector_store/test_base_store.py b/presets/ragengine/tests/vector_store/test_base_store.py
index e7d5261..e39d3d6 100644
--- a/presets/ragengine/tests/vector_store/test_base_store.py
+++ b/presets/ragengine/tests/vector_store/test_base_store.py
@@ -184,7 +184,6 @@ class BaseVectorStoreTest(ABC):
                 "messages": [
                     {"role": "user", "content": "What is the first document?"}
                 ],
-                "top_k": 1,
                 "temperature": 0.7,
                 "max_tokens": 100,
             }
@@ -192,7 +191,7 @@ class BaseVectorStoreTest(ABC):
 
         assert chat_results is not None
         assert chat_results.source_nodes is not None
-        assert len(chat_results.source_nodes) == 1
+        assert len(chat_results.source_nodes) == 2
         assert chat_results.source_nodes[0].text == "First document"
         assert chat_results.source_nodes[0].doc_id == index_doc_resp[0]
         assert chat_results.id is not None
diff --git a/test/e2e/preset_vllm_test.go b/test/e2e/preset_vllm_test.go
index 450f4d2..21b2a04 100644
--- a/test/e2e/preset_vllm_test.go
+++ b/test/e2e/preset_vllm_test.go
@@ -18,11 +18,17 @@ import (
 	"math/rand"
 	"time"
 
+	helmv2 "github.com/fluxcd/helm-controller/api/v2"
+	sourcev1 "github.com/fluxcd/source-controller/api/v1"
 	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	kaitov1beta1 "github.com/kaito-project/kaito/api/v1beta1"
+	kaitoutils "github.com/kaito-project/kaito/pkg/utils"
+	"github.com/kaito-project/kaito/pkg/utils/consts"
 	"github.com/kaito-project/kaito/test/e2e/utils"
 )
 
@@ -63,6 +69,7 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateWorkspaceReadiness(workspaceObj)
 		validateModelsEndpoint(workspaceObj)
 		validateCompletionsEndpoint(workspaceObj)
+		validateGatewayAPIInferenceExtensionResources(workspaceObj)
 	})
 
 	It("should create a single-node llama-3.1-8b-instruct workspace with preset public mode successfully", utils.GinkgoLabelFastCheck, func() {
@@ -85,6 +92,7 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateWorkspaceReadiness(workspaceObj)
 		validateModelsEndpoint(workspaceObj)
 		validateCompletionsEndpoint(workspaceObj)
+		validateGatewayAPIInferenceExtensionResources(workspaceObj)
 	})
 
 	It("should create a multi-node llama-3.1-8b-instruct workspace with preset public mode successfully", utils.GinkgoLabelFastCheck, func() {
@@ -109,6 +117,7 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateWorkspaceReadiness(workspaceObj)
 		validateModelsEndpoint(workspaceObj)
 		validateCompletionsEndpoint(workspaceObj)
+		validateGatewayAPIInferenceExtensionResources(workspaceObj)
 	})
 
 	It("should create a deepseek-distilled-qwen-14b workspace with preset public mode successfully", utils.GinkgoLabelA100Required, func() {
@@ -131,6 +140,7 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateWorkspaceReadiness(workspaceObj)
 		validateModelsEndpoint(workspaceObj)
 		validateCompletionsEndpoint(workspaceObj)
+		validateGatewayAPIInferenceExtensionResources(workspaceObj)
 	})
 
 	It("should create a falcon workspace with preset public mode successfully", func() {
@@ -153,6 +163,7 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateWorkspaceReadiness(workspaceObj)
 		validateModelsEndpoint(workspaceObj)
 		validateCompletionsEndpoint(workspaceObj)
+		validateGatewayAPIInferenceExtensionResources(workspaceObj)
 	})
 
 	It("should create a mistral workspace with preset public mode successfully", func() {
@@ -175,6 +186,7 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateWorkspaceReadiness(workspaceObj)
 		validateModelsEndpoint(workspaceObj)
 		validateCompletionsEndpoint(workspaceObj)
+		validateGatewayAPIInferenceExtensionResources(workspaceObj)
 	})
 
 	It("should create a Phi-2 workspace with preset public mode successfully", func() {
@@ -197,6 +209,7 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateWorkspaceReadiness(workspaceObj)
 		validateModelsEndpoint(workspaceObj)
 		validateCompletionsEndpoint(workspaceObj)
+		validateGatewayAPIInferenceExtensionResources(workspaceObj)
 	})
 
 	It("should create a Phi-3-mini-128k-instruct workspace with preset public mode successfully", func() {
@@ -219,6 +232,7 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateWorkspaceReadiness(workspaceObj)
 		validateModelsEndpoint(workspaceObj)
 		validateCompletionsEndpoint(workspaceObj)
+		validateGatewayAPIInferenceExtensionResources(workspaceObj)
 	})
 
 	It("should create a qwen2.5 coder workspace with preset public mode and 2 gpu successfully", func() {
@@ -242,6 +256,7 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateWorkspaceReadiness(workspaceObj)
 		validateModelsEndpoint(workspaceObj)
 		validateCompletionsEndpoint(workspaceObj)
+		validateGatewayAPIInferenceExtensionResources(workspaceObj)
 	})
 
 	It("should create a phi4 workspace with adapter successfully", utils.GinkgoLabelA100Required, func() {
@@ -274,6 +289,7 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateInitContainers(workspaceObj, expectedInitContainers)
 
 		validateAdapterLoadedInVLLM(workspaceObj, phi4AdapterName)
+		validateGatewayAPIInferenceExtensionResources(workspaceObj)
 	})
 
 	It("should create a llama-3.3-70b-instruct workspace with preset public mode successfully", utils.GinkgoLabelA100Required, func() {
@@ -298,6 +314,7 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateWorkspaceReadiness(workspaceObj)
 		validateModelsEndpoint(workspaceObj)
 		validateCompletionsEndpoint(workspaceObj)
+		validateGatewayAPIInferenceExtensionResources(workspaceObj)
 	})
 })
 
@@ -440,3 +457,48 @@ func createLlama3_3_70BInstructWorkspaceWithPresetPublicModeAndVLLM(numOfNode in
 	})
 	return workspaceObj
 }
+
+func validateGatewayAPIInferenceExtensionResources(workspaceObj *kaitov1beta1.Workspace) {
+	// Only validate if the Inference Preset is set
+	if workspaceObj.Inference.Preset == nil {
+		return
+	}
+
+	By("Checking Flux OCIRepository is Ready", func() {
+		Eventually(func() bool {
+			ociRepository := &sourcev1.OCIRepository{}
+			err := utils.TestingCluster.KubeClient.Get(ctx, client.ObjectKey{
+				Namespace: workspaceObj.Namespace,
+				Name:      kaitoutils.InferencePoolName(workspaceObj.Name),
+			}, ociRepository, &client.GetOptions{})
+			if err != nil {
+				return false
+			}
+			for _, cond := range ociRepository.Status.Conditions {
+				if cond.Type == consts.ConditionReady && cond.Status == metav1.ConditionTrue {
+					return true
+				}
+			}
+			return false
+		}, utils.PollTimeout, utils.PollInterval).Should(BeTrue(), "Failed to validate Flux OCIRepository is Ready")
+	})
+
+	By("Checking Flux HelmRelease is Ready", func() {
+		Eventually(func() bool {
+			helmRelease := &helmv2.HelmRelease{}
+			err := utils.TestingCluster.KubeClient.Get(ctx, client.ObjectKey{
+				Namespace: workspaceObj.Namespace,
+				Name:      kaitoutils.InferencePoolName(workspaceObj.Name),
+			}, helmRelease, &client.GetOptions{})
+			if err != nil {
+				return false
+			}
+			for _, cond := range helmRelease.Status.Conditions {
+				if cond.Type == consts.ConditionReady && cond.Status == metav1.ConditionTrue {
+					return true
+				}
+			}
+			return false
+		}, utils.PollTimeout, utils.PollInterval).Should(BeTrue(), "Failed to validate Flux HelmRelease is Ready")
+	})
+}
diff --git a/test/e2e/utils/cluster.go b/test/e2e/utils/cluster.go
index a4204b6..a9db150 100644
--- a/test/e2e/utils/cluster.go
+++ b/test/e2e/utils/cluster.go
@@ -15,6 +15,8 @@ package utils
 
 import (
 	azurev1alpha2 "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2"
+	helmv2 "github.com/fluxcd/helm-controller/api/v2"
+	sourcev1 "github.com/fluxcd/source-controller/api/v1"
 	"github.com/onsi/gomega"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -58,6 +60,8 @@ func GetClusterClient(cluster *Cluster) {
 	utilruntime.Must(kaitoutils.KarpenterSchemeBuilder.AddToScheme(scheme))
 	utilruntime.Must(azurev1alpha2.SchemeBuilder.AddToScheme(scheme))
 	utilruntime.Must(kaitoutils.AwsSchemeBuilder.AddToScheme(scheme))
+	utilruntime.Must(helmv2.AddToScheme(scheme))
+	utilruntime.Must(sourcev1.AddToScheme(scheme))
 
 	restConfig := config.GetConfigOrDie()
 
@@ -68,5 +72,4 @@ func GetClusterClient(cluster *Cluster) {
 
 	cluster.DynamicClient, err = dynamic.NewForConfig(restConfig)
 	gomega.Expect(err).Should(gomega.Succeed(), "Failed to set up Dynamic Client")
-
 }