Repository: kaito-project/kaito

Issue:
Testing coverage for distributed inference
**Is your feature request related to a problem? Please describe.**

Add more test for distributed inference.

**Describe the solution you'd like**

Development workflow for adding a new runtime feature.
Build and push base image to acr -> run preset test against the new base image with distributed inference -> build mcr image -> bump image tag in controller side -> run e2e test. 

- [x] Add test [here](https://github.com/kaito-project/kaito/blob/main/.github/workflows/e2e-preset-test.yml) to test image before pushing to mcr    #1138 
- [x] e2e test against distributed inference
  - [x] ensure adapter works
  - [x] ensure health check works for both leader and worker pods
  - [x] ensure configfile works. (adjust model length and see if it takes effect)

Discussion:
@chewong: Currently, we have no preset model that uses distributed inference. To test changes in the kaito-base image, we need to create a new preset model with the following requirements:

- `downloadAtRuntime: true`
- Large enough that it cannot fit on a single node, but not too large that it requires a lot of time waiting to finish downloading. Llama 3.3 70B might be a good choice

Additionally, we need to update [`determine_models.py`](https://github.com/kaito-project/kaito/blob/039437a8ccc2158666a421fe55a9d6d8be7f77b2/.github/workflows/kind-cluster/determine_models.py) to output the list of models affected when the base image changes. All models with `downloadAtRuntime: true` will be affected and should be tested during e2e test.

Solution patch:
diff --git a/.github/actions/build-image-action/action.yml b/.github/actions/build-image-action/action.yml
index 38be081..0ab6550 100644
--- a/.github/actions/build-image-action/action.yml
+++ b/.github/actions/build-image-action/action.yml
@@ -171,6 +171,7 @@ runs:
         MODEL_VERSION=${{ inputs.model_version }} \
         WEIGHTS_DIR=$MODEL_DIR \
         HF_TOKEN=${{ inputs.hf_token }} \
+        HF_HOME=$MODEL_DIR \
         python3 docker/presets/models/tfs/download_and_push_model.py $ACR_NAME.azurecr.io/$IMAGE_NAME:$MODEL_TAG
 
       shell: bash
diff --git a/.github/workflows/preset-image-build-1ES.yml b/.github/workflows/preset-image-build-1ES.yml
index a5dfee8..6771b1b 100644
--- a/.github/workflows/preset-image-build-1ES.yml
+++ b/.github/workflows/preset-image-build-1ES.yml
@@ -146,10 +146,9 @@ jobs:
 
       - name: Ensure Python is Installed
         run: |
-          if ! command -v python3 &> /dev/null; then
-            sudo apt-get update
-            sudo apt-get install -y python3
-          fi
+          sudo apt-get update
+          sudo apt-get install -y python3
+          sudo apt install python3-pip -y
 
       - name: Ensure git and git LFS is Installed
         run: |
diff --git a/.github/workflows/ragengine-e2e-workflow.yml b/.github/workflows/ragengine-e2e-workflow.yml
index 23f5d13..0235149 100644
--- a/.github/workflows/ragengine-e2e-workflow.yml
+++ b/.github/workflows/ragengine-e2e-workflow.yml
@@ -77,6 +77,14 @@ jobs:
         env:
           REGISTRY: ${{ env.CLUSTER_NAME }}.azurecr.io
       
+      - name: build kaito RAG Engine service image
+        if: ${{ !inputs.isRelease }}
+        shell: bash
+        run: |
+          make docker-build-ragservice
+        env:
+          REGISTRY: ${{ env.CLUSTER_NAME }}.azurecr.io
+      
       - name: Remove existing Go modules directory
         run: sudo rm -rf ~/go/pkg/mod
       
@@ -91,7 +99,7 @@ jobs:
       - name: Install KAITO RAG Engine helm chart
         shell: bash
         run: |
-          make az-patch-install-ragengine-helm
+          make az-patch-install-ragengine-helm-e2e
           kubectl wait --for=condition=available deploy "kaito-ragengine" -n kaito-ragengine --timeout=300s
         env:
           REGISTRY: ${{ env.CLUSTER_NAME }}.azurecr.io
diff --git a/Makefile b/Makefile
index 8267186..d5fff44 100644
--- a/Makefile
+++ b/Makefile
@@ -235,6 +235,12 @@ BUILDKIT_VERSION ?= v0.18.1
 RAGENGINE_IMAGE_NAME ?= ragengine
 RAGENGINE_IMAGE_TAG ?= v0.0.1
 
+RAGENGINE_SERVICE_IMG_NAME ?= kaito-rag-service
+RAGENGINE_SERVICE_IMG_TAG ?= v0.0.1
+
+E2E_IMAGE_NAME ?= kaito-e2e
+E2E_IMAGE_TAG ?= v0.0.1
+
 
 .PHONY: docker-buildx
 docker-buildx: ## Build and push docker image for the manager for cross-platform support
@@ -269,7 +275,7 @@ docker-build-ragservice: docker-buildx
         --output=$(OUTPUT_TYPE) \
         --file ./docker/ragengine/service/Dockerfile \
         --pull \
-        -tag $(REGISTRY)/$(RAGENGINE_SERVICE_IMG_NAME):$(RAGENGINE_SERVICE_IMG_TAG) .
+        --tag $(REGISTRY)/$(RAGENGINE_SERVICE_IMG_NAME):$(RAGENGINE_SERVICE_IMG_TAG) .
 
 .PHONY: docker-build-adapter
 docker-build-adapter: docker-buildx
@@ -354,6 +360,19 @@ az-patch-install-ragengine-helm:
 
 	helm install kaito-ragengine ./charts/kaito/ragengine --namespace $(KAITO_RAGENGINE_NAMESPACE) --create-namespace
 
+.PHONY: az-patch-install-ragengine-helm-e2e
+az-patch-install-ragengine-helm-e2e:
+	az aks get-credentials --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP)
+
+	yq -i '(.image.repository)                                              = "$(REGISTRY)/ragengine"'                    ./charts/kaito/ragengine/values.yaml
+	yq -i '(.image.tag)                                                     = "$(IMG_TAG)"'                               ./charts/kaito/ragengine/values.yaml
+	yq -i '(.clusterName)                                                   = "$(AZURE_CLUSTER_NAME)"'                    ./charts/kaito/ragengine/values.yaml
+	yq -i '(.presetRagRegistryName)                                         = "$(REGISTRY)"'                              ./charts/kaito/ragengine/values.yaml
+	yq -i '(.presetRagImageName)                                         	= "$(RAGENGINE_SERVICE_IMG_NAME)"'            ./charts/kaito/ragengine/values.yaml
+	yq -i '(.presetRagImageTag)                                         	= "$(RAGENGINE_SERVICE_IMG_TAG)"'             ./charts/kaito/ragengine/values.yaml
+
+	helm install kaito-ragengine ./charts/kaito/ragengine --namespace $(KAITO_RAGENGINE_NAMESPACE) --create-namespace
+
 .PHONY: aws-patch-install-helm ##install kaito on AWS cluster
 aws-patch-install-helm:
 	yq -i '(.image.repository)                                              = "$(REGISTRY)/workspace"'                    	./charts/kaito/workspace/values.yaml
diff --git a/docker/presets/models/tfs/download_and_push_model.py b/docker/presets/models/tfs/download_and_push_model.py
index f2bcb3a..c53e7b3 100644
--- a/docker/presets/models/tfs/download_and_push_model.py
+++ b/docker/presets/models/tfs/download_and_push_model.py
@@ -74,7 +74,7 @@ def push_to_oras(source_dir, image_name):
     try:
         # Initialize command with oras push and image name
         # ORAS cli is much more stable than python sdk.
-        command = ["oras", "push", image_name]
+        command = ["oras", "push", "--concurrency", "2", image_name]
 
         # Gather all files with their relative paths, excluding hidden files
         files_to_push = []
diff --git a/docs/proposals/20250529-llama-3.3-70b-instruct.md b/docs/proposals/20250529-llama-3.3-70b-instruct.md
index fcd2708..7671160 100644
--- a/docs/proposals/20250529-llama-3.3-70b-instruct.md
+++ b/docs/proposals/20250529-llama-3.3-70b-instruct.md
@@ -83,5 +83,5 @@ This section describes how to configure the runtime framework to support the inf
 # History
 
 - [x] 2025-05-29: Open proposal PR.
-- [ ] YYYY-MM-DD: Start model integration.
-- [ ] YYYY-MM-DD: Complete model support.
+- [x] 2025-06-05: Start model integration.
+- [x] 2025-06-05: Complete model support.
diff --git a/examples/inference/kaito_workspace_llama-3.1_8b_instruct.yaml b/examples/inference/kaito_workspace_llama-3.1_8b_instruct.yaml
index 84c4004..f652723 100644
--- a/examples/inference/kaito_workspace_llama-3.1_8b_instruct.yaml
+++ b/examples/inference/kaito_workspace_llama-3.1_8b_instruct.yaml
@@ -10,3 +10,5 @@ resource:
 inference:
   preset:
     name: llama-3.1-8b-instruct
+    presetOptions:
+      modelAccessSecret: hf-token
diff --git a/examples/inference/kaito_workspace_llama-3.3_70b_instruct.yaml b/examples/inference/kaito_workspace_llama-3.3_70b_instruct.yaml
new file mode 100644
index 0000000..6b97e5b
--- /dev/null
+++ b/examples/inference/kaito_workspace_llama-3.3_70b_instruct.yaml
@@ -0,0 +1,15 @@
+apiVersion: kaito.sh/v1beta1
+kind: Workspace
+metadata:
+  name: workspace-llama-3-3-70b-instruct
+resource:
+  count: 2
+  instanceType: "Standard_NC48ads_A100_v4"
+  labelSelector:
+    matchLabels:
+      apps: llama-3-3-70b-instruct
+inference:
+  preset:
+    name: llama-3.3-70b-instruct
+    presetOptions:
+      modelAccessSecret: hf-token
diff --git a/pkg/workspace/controllers/workspace_controller.go b/pkg/workspace/controllers/workspace_controller.go
index f739b07..c760349 100644
--- a/pkg/workspace/controllers/workspace_controller.go
+++ b/pkg/workspace/controllers/workspace_controller.go
@@ -556,18 +556,20 @@ func (c *WorkspaceReconciler) ensureService(ctx context.Context, wObj *kaitov1be
 		return nil
 	}
 
-	supportsDistributedInference := false
+	isStatefulSet := false
 	if presetName := getPresetName(wObj); presetName != "" {
 		model := plugin.KaitoModelRegister.MustGet(presetName)
-		supportsDistributedInference = model.SupportDistributedInference()
+		// Dry-run the inference workload generation to determine if it will be a StatefulSet or not.
+		workloadObj, _ := inference.GeneratePresetInference(ctx, wObj, "", model, c.Client)
+		_, isStatefulSet = workloadObj.(*appsv1.StatefulSet)
 	}
 
-	serviceObj := manifests.GenerateServiceManifest(wObj, serviceType, supportsDistributedInference)
+	serviceObj := manifests.GenerateServiceManifest(wObj, serviceType, isStatefulSet)
 	if err := resources.CreateResource(ctx, serviceObj, c.Client); err != nil {
 		return err
 	}
 
-	if supportsDistributedInference {
+	if isStatefulSet {
 		headlessService := manifests.GenerateHeadlessServiceManifest(wObj)
 		if err := resources.CreateResource(ctx, headlessService, c.Client); err != nil {
 			return err
@@ -755,6 +757,7 @@ func (c *WorkspaceReconciler) SetupWithManager(mgr ctrl.Manager) error {
 
 	builder := ctrl.NewControllerManagedBy(mgr).
 		For(&kaitov1beta1.Workspace{}).
+		Owns(&corev1.Service{}).
 		Owns(&appsv1.ControllerRevision{}).
 		Owns(&appsv1.Deployment{}).
 		Owns(&appsv1.StatefulSet{}).
diff --git a/presets/ragengine/vector_store/base.py b/presets/ragengine/vector_store/base.py
index 09d7759..ea17919 100644
--- a/presets/ragengine/vector_store/base.py
+++ b/presets/ragengine/vector_store/base.py
@@ -13,7 +13,7 @@ from llama_index.core import Document as LlamaDocument
 from llama_index.core import (StorageContext, VectorStoreIndex, load_index_from_storage)
 from llama_index.core.postprocessor import LLMRerank  # Query with LLM Reranking
 
-from llama_index.vector_stores.faiss import FaissVectorStore
+from llama_index.vector_stores.faiss import FaissMapVectorStore
 
 from ragengine.models import Document
 from ragengine.embedding.base import BaseEmbeddingModel
@@ -415,7 +415,7 @@ class BaseVectorStore(ABC):
                 storage_context = StorageContext.from_defaults(persist_dir=path)
             except UnicodeDecodeError as ude:
                 # Failed to load the index in the default json format, trying faissdb
-                faiss_vs = FaissVectorStore.from_persist_dir(persist_dir=path)
+                faiss_vs = FaissMapVectorStore.from_persist_dir(persist_dir=path)
                 storage_context = StorageContext.from_defaults(persist_dir=path, vector_store=faiss_vs)
             except Exception as e:
                 logger.error(f"Failed to load index '{index_name}'. Error: {str(e)}")
diff --git a/presets/workspace/dependencies/requirements.txt b/presets/workspace/dependencies/requirements.txt
index fd9f4d8..868b296 100644
--- a/presets/workspace/dependencies/requirements.txt
+++ b/presets/workspace/dependencies/requirements.txt
@@ -1,9 +1,9 @@
 # Dependencies for TFS
 
 # Core Dependencies
-vllm==0.8.5
+vllm==0.9.0
 transformers == 4.51.3
-torch==2.6.0
+torch==2.7.0
 accelerate==1.0.0
 fastapi==0.115.0
 pydantic>=2.9
diff --git a/presets/workspace/models/llama3/model.go b/presets/workspace/models/llama3/model.go
index 0a1c8df..95ef2c8 100644
--- a/presets/workspace/models/llama3/model.go
+++ b/presets/workspace/models/llama3/model.go
@@ -16,10 +16,15 @@ func init() {
 		Name:     PresetLlama3_1_8BInstructModel,
 		Instance: &llama3_1_8b_instructA,
 	})
+	plugin.KaitoModelRegister.Register(&plugin.Registration{
+		Name:     PresetLlama3_3_70BInstructModel,
+		Instance: &llama3_3_70b_instruct,
+	})
 }
 
 const (
-	PresetLlama3_1_8BInstructModel = "llama-3.1-8b-instruct"
+	PresetLlama3_1_8BInstructModel  = "llama-3.1-8b-instruct"
+	PresetLlama3_3_70BInstructModel = "llama-3.3-70b-instruct"
 )
 
 var (
@@ -41,8 +46,6 @@ var llama3_1_8b_instructA llama3_1_8BInstruct
 
 type llama3_1_8BInstruct struct{}
 
-// TODO: Get a more exact GPU Memory requirement (currently we know it must be >16Gi)
-// https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct/discussions/77
 func (*llama3_1_8BInstruct) GetInferenceParameters() *model.PresetParam {
 	return &model.PresetParam{
 		Metadata:                  metadata.MustGet(PresetLlama3_1_8BInstructModel),
@@ -58,9 +61,11 @@ func (*llama3_1_8BInstruct) GetInferenceParameters() *model.PresetParam {
 				ModelRunParams:    llamaRunParams,
 			},
 			VLLM: model.VLLMParam{
-				BaseCommand:    inference.DefaultVLLMCommand,
-				ModelName:      PresetLlama3_1_8BInstructModel,
-				ModelRunParams: llamaRunParamsVLLM,
+				BaseCommand:          inference.DefaultVLLMCommand,
+				ModelName:            PresetLlama3_1_8BInstructModel,
+				ModelRunParams:       llamaRunParamsVLLM,
+				RayLeaderBaseCommand: inference.DefaultVLLMRayLeaderBaseCommand,
+				RayWorkerBaseCommand: inference.DefaultVLLMRayWorkerBaseCommand,
 			},
 		},
 		ReadinessTimeout: time.Duration(30) * time.Minute,
@@ -72,9 +77,53 @@ func (*llama3_1_8BInstruct) GetTuningParameters() *model.PresetParam {
 }
 
 func (*llama3_1_8BInstruct) SupportDistributedInference() bool {
-	return false
+	return true
 }
 
 func (*llama3_1_8BInstruct) SupportTuning() bool {
 	return false
 }
+
+var llama3_3_70b_instruct llama3_3_70Binstruct
+
+type llama3_3_70Binstruct struct{}
+
+// TODO: Get a more exact GPU Memory requirement (currently we know it must be >16Gi)
+// https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct/discussions/77
+func (*llama3_3_70Binstruct) GetInferenceParameters() *model.PresetParam {
+	return &model.PresetParam{
+		Metadata:                  metadata.MustGet(PresetLlama3_3_70BInstructModel),
+		DiskStorageRequirement:    "140Gi",
+		GPUCountRequirement:       "4",
+		TotalGPUMemoryRequirement: "320Gi",
+		PerGPUMemoryRequirement:   "80Gi",
+		RuntimeParam: model.RuntimeParam{
+			Transformers: model.HuggingfaceTransformersParam{
+				BaseCommand:       baseCommandPresetLlamaInference,
+				AccelerateParams:  inference.DefaultAccelerateParams,
+				InferenceMainFile: inference.DefaultTransformersMainFile,
+				ModelRunParams:    llamaRunParams,
+			},
+			VLLM: model.VLLMParam{
+				BaseCommand:          inference.DefaultVLLMCommand,
+				ModelName:            PresetLlama3_3_70BInstructModel,
+				ModelRunParams:       llamaRunParamsVLLM,
+				RayLeaderBaseCommand: inference.DefaultVLLMRayLeaderBaseCommand,
+				RayWorkerBaseCommand: inference.DefaultVLLMRayWorkerBaseCommand,
+			},
+		},
+		ReadinessTimeout: time.Duration(30) * time.Minute,
+	}
+}
+
+func (*llama3_3_70Binstruct) GetTuningParameters() *model.PresetParam {
+	return nil // It is not recommended/ideal to further fine-tune instruct models - Already been fine-tuned
+}
+
+func (*llama3_3_70Binstruct) SupportDistributedInference() bool {
+	return true
+}
+
+func (*llama3_3_70Binstruct) SupportTuning() bool {
+	return false
+}
diff --git a/presets/workspace/models/supported_models.yaml b/presets/workspace/models/supported_models.yaml
index c2114c3..f0c183f 100644
--- a/presets/workspace/models/supported_models.yaml
+++ b/presets/workspace/models/supported_models.yaml
@@ -15,6 +15,15 @@ models:
     version: https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct/commit/0e9e39f249a16976918f6564b8830bc894c89659
     runtime: tfs
     downloadAtRuntime: true
+    tag: 0.2.0
+    # Tag history:
+    # 0.2.0 - Convert to individual OCI artifacts
+    # 0.0.1 - Initial Release
+  - name: llama-3.3-70b-instruct
+    type: text-generation
+    version: https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct/commit/6f6073b423013f6a7d4d9f39144961bfbfbc386b
+    runtime: tfs
+    downloadAtRuntime: true
     tag: 0.0.1
     # Tag history:
     # 0.0.1 - Initial Release
@@ -24,13 +33,14 @@ models:
     type: text-generation
     version: https://huggingface.co/tiiuae/falcon-7b/commit/898df1396f35e447d5fe44e0a3ccaaaa69f30d36
     runtime: tfs
-    tag: 0.1.1
+    tag: 0.2.0
   - name: falcon-7b-instruct
     type: text-generation
     version: https://huggingface.co/tiiuae/falcon-7b-instruct/commit/cf4b3c42ce2fdfe24f753f0f0d179202fea59c99
     runtime: tfs
-    tag: 0.1.1
+    tag: 0.2.0
     # Tag history:
+    # 0.2.0 - Convert to individual OCI artifacts
     # 0.1.1 - bump vLLM to 0.8.5
     # 0.1.0 - vLLM, Transformers, and several other library updates and fixes
     # 0.0.8 - Support adapter and config file for VLLM runtime
@@ -45,13 +55,14 @@ models:
     type: text-generation
     version: https://huggingface.co/tiiuae/falcon-40b/commit/4a70170c215b36a3cce4b4253f6d0612bb7d4146
     runtime: tfs
-    tag: 0.1.1
+    tag: 0.2.0
   - name: falcon-40b-instruct
     type: text-generation
     version: https://huggingface.co/tiiuae/falcon-40b-instruct/commit/ecb78d97ac356d098e79f0db222c9ce7c5d9ee5f
     runtime: tfs
-    tag: 0.1.1
+    tag: 0.2.0
     # Tag history for 40b models:
+    # 0.2.0 - Convert to individual OCI artifacts
     # 0.1.1 - bump vLLM to 0.8.5
     # 0.1.0 - vLLM, Transformers, and several other library updates and fixes
     # 0.0.9 - Support adapter and config file for VLLM runtime
@@ -69,13 +80,14 @@ models:
     type: text-generation
     version: https://huggingface.co/mistralai/Mistral-7B-v0.3/commit/d8cadc02ac76bd617a919d50b092e59d2d110aff
     runtime: tfs
-    tag: 0.1.1
+    tag: 0.2.0
   - name: mistral-7b-instruct
     type: text-generation
     version: https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3/commit/e0bc86c23ce5aae1db576c8cca6f06f1f73af2db
     runtime: tfs
-    tag: 0.1.1
+    tag: 0.2.0
     # Tag history:
+    # 0.2.0 - Convert to individual OCI artifacts
     # 0.1.1 - bump vLLM to 0.8.5
     # 0.1.0 - vLLM, Transformers, and several other library updates and fixes
     # 0.0.9 - Support adapter and config file for VLLM runtime
@@ -93,8 +105,9 @@ models:
     type: text-generation
     version: https://huggingface.co/microsoft/phi-2/commit/ef382358ec9e382308935a992d908de099b64c23
     runtime: tfs
-    tag: 0.1.1
+    tag: 0.2.0
     # Tag history:
+    # 0.2.0 - Convert to individual OCI artifacts
     # 0.1.1 - bump vLLM to 0.8.5
     # 0.1.0 - vLLM, Transformers, and several other library updates and fixes
     # 0.0.7 - Support adapter and config file for VLLM runtime
@@ -110,23 +123,24 @@ models:
     type: text-generation
     version: https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/commit/0a67737cc96d2554230f90338b163bc6380a2a85
     runtime: tfs
-    tag: 0.1.1
+    tag: 0.2.0
   - name: phi-3-mini-128k-instruct
     type: text-generation
     version: https://huggingface.co/microsoft/Phi-3-mini-128k-instruct/commit/072cb7562cb8c4adf682a8e186aaafa49469eb5d
     runtime: tfs
-    tag: 0.1.1
+    tag: 0.2.0
   - name: phi-3-medium-4k-instruct
     type: text-generation
     version: https://huggingface.co/microsoft/Phi-3-medium-4k-instruct/commit/b64223aaea6fbf273c0c8cd0801d5e732dce8897
     runtime: tfs
-    tag: 0.1.1
+    tag: 0.2.0
   - name: phi-3-medium-128k-instruct
     type: text-generation
     version: https://huggingface.co/microsoft/Phi-3-medium-128k-instruct/commit/fa7d2aa4f5ea69b2e36b20d050cdae79c9bfbb3f
     runtime: tfs
-    tag: 0.1.1
+    tag: 0.2.0
     # Tag history:
+    # 0.2.0 - Convert to individual OCI artifacts
     # 0.1.1 - bump vLLM to 0.8.5
     # 0.1.0 - vLLM, Transformers, and several other library updates and fixes
     # 0.0.4 - Support adapter and config file for VLLM runtime
@@ -138,8 +152,9 @@ models:
     type: text-generation
     version: https://huggingface.co/microsoft/Phi-3.5-mini-instruct/commit/3145e03a9fd4cdd7cd953c34d9bbf7ad606122ca
     runtime: tfs
-    tag: 0.1.1
+    tag: 0.2.0
     # Tag history:
+    # 0.2.0 - Convert to individual OCI artifacts
     # 0.1.1 - bump vLLM to 0.8.5
     # 0.1.0 - vLLM, Transformers, and several other library updates and fixes
     # 0.0.2 - Support adapter and config file for VLLM runtime
@@ -149,13 +164,14 @@ models:
     type: text-generation
     version: https://huggingface.co/microsoft/Phi-4-mini-instruct/commit/c0fb9e74abda11b496b7907a9c6c9009a7a0488f
     runtime: tfs
-    tag: 0.1.1
+    tag: 0.2.0
   - name: phi-4
     type: text-generation
     version: https://huggingface.co/microsoft/phi-4/commit/187ef0342fff0eb3333be9f00389385e95ef0b61
     runtime: tfs
-    tag: 0.1.1
+    tag: 0.2.0
     # Tag history:
+    # 0.2.0 - Convert to individual OCI artifacts
     # 0.1.1 - bump vLLM to 0.8.5
     # 0.1.0 - New Model!
 
@@ -164,13 +180,14 @@ models:
     type: text-generation
     version: https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct/commit/c03e6d358207e414f1eca0bb1891e29f1db0e242
     runtime: tfs
-    tag: 0.1.1
+    tag: 0.2.0
   - name: qwen2.5-coder-32b-instruct
     type: text-generation
     version: https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct/commit/381fc969f78efac66bc87ff7ddeadb7e73c218a7
     runtime: tfs
-    tag: 0.1.1
+    tag: 0.2.0
     # Tag history:
+    # 0.2.0 - Convert to individual OCI artifacts
     # 0.1.1 - bump vLLM to 0.8.5
     # 0.1.0 - vLLM, Transformers, and several other library updates and fixes
     # 0.0.1 - New Model!
@@ -180,13 +197,14 @@ models:
     type: text-generation
     version: https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B/commit/1df8507178afcc1bef68cd8c393f61a886323761
     runtime: tfs
-    tag: 0.1.1
+    tag: 0.2.0
   - name: deepseek-r1-distill-llama-8b
     type: text-generation
     version: https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-8B/commit/6a6f4aa4197940add57724a7707d069478df56b1
     runtime: tfs
-    tag: 0.1.1
+    tag: 0.2.0
     # Tag history:
+    # 0.2.0 - Convert to individual OCI artifacts
     # 0.1.1 - bump vLLM to 0.8.5
     # 0.1.0 - vLLM, Transformers, and several other library updates and fixes
     # 0.0.1 - New Model!
\ No newline at end of file


Test patch:
diff --git a/pkg/workspace/controllers/workspace_controller_test.go b/pkg/workspace/controllers/workspace_controller_test.go
index 91ae462..f86a71f 100644
--- a/pkg/workspace/controllers/workspace_controller_test.go
+++ b/pkg/workspace/controllers/workspace_controller_test.go
@@ -420,6 +420,7 @@ func TestEnsureService(t *testing.T) {
 	}{
 		"Existing service is found for workspace": {
 			callMocks: func(c *test.MockClient) {
+				c.On("Get", mock.IsType(context.Background()), mock.Anything, mock.IsType(&corev1.ConfigMap{}), mock.Anything).Return(nil)
 				c.On("Get", mock.IsType(context.Background()), mock.Anything, mock.IsType(&corev1.Service{}), mock.Anything).Return(nil)
 			},
 			expectedError: nil,
@@ -427,6 +428,7 @@ func TestEnsureService(t *testing.T) {
 		},
 		"Service creation fails": {
 			callMocks: func(c *test.MockClient) {
+				c.On("Get", mock.IsType(context.Background()), mock.Anything, mock.IsType(&corev1.ConfigMap{}), mock.Anything).Return(nil)
 				c.On("Get", mock.IsType(context.Background()), mock.Anything, mock.IsType(&corev1.Service{}), mock.Anything).Return(test.NotFoundError())
 				c.On("Create", mock.IsType(context.Background()), mock.IsType(&corev1.Service{}), mock.Anything).Return(errors.New("cannot create service"))
 			},
@@ -435,6 +437,7 @@ func TestEnsureService(t *testing.T) {
 		},
 		"Successfully creates a new service": {
 			callMocks: func(c *test.MockClient) {
+				c.On("Get", mock.IsType(context.Background()), mock.Anything, mock.IsType(&corev1.ConfigMap{}), mock.Anything).Return(nil)
 				c.On("Get", mock.IsType(context.Background()), mock.Anything, mock.IsType(&corev1.Service{}), mock.Anything).Return(test.NotFoundError())
 				c.On("Create", mock.IsType(context.Background()), mock.IsType(&corev1.Service{}), mock.Anything).Return(nil)
 			},
@@ -443,6 +446,7 @@ func TestEnsureService(t *testing.T) {
 		},
 		"Successfully creates a new service for a custom model": {
 			callMocks: func(c *test.MockClient) {
+				c.On("Get", mock.IsType(context.Background()), mock.Anything, mock.IsType(&corev1.ConfigMap{}), mock.Anything).Return(nil)
 				c.On("Get", mock.IsType(context.Background()), mock.Anything, mock.IsType(&corev1.Service{}), mock.Anything).Return(test.NotFoundError())
 				c.On("Create", mock.IsType(context.Background()), mock.IsType(&corev1.Service{}), mock.Anything).Return(nil)
 			},
diff --git a/presets/ragengine/tests/vector_store/test_base_store.py b/presets/ragengine/tests/vector_store/test_base_store.py
index fdb2796..10c873b 100644
--- a/presets/ragengine/tests/vector_store/test_base_store.py
+++ b/presets/ragengine/tests/vector_store/test_base_store.py
@@ -348,6 +348,15 @@ func main() {}"""
         await vector_store_manager.delete_documents(second_index_name, [result[0]['doc_id']])
 
         first_index_result = await vector_store_manager.list_documents_in_index(index_name, limit=10, offset=0)
-        second_index_result = await vector_store_manager.list_documents_in_index("second_test_index", limit=10, offset=0)
+        second_index_result = await vector_store_manager.list_documents_in_index(second_index_name, limit=10, offset=0)
         assert len(first_index_result) == 10
         assert len(second_index_result) == 9
+
+        second_index_result[0]['text'] = "Modified text"
+        second_update_result = await vector_store_manager.update_documents(second_index_name, [Document(doc_id=second_index_result[0]['doc_id'], text="Modified text", metadata=second_index_result[0]['metadata'])])
+        assert len(second_update_result["updated_documents"]) == 1
+        assert second_update_result["updated_documents"][0].text == "Modified text"
+
+        second_delete_result = await vector_store_manager.delete_documents(second_index_name, [second_index_result[0]['doc_id']])
+        assert len(second_delete_result["deleted_doc_ids"]) == 1
+        assert second_delete_result["deleted_doc_ids"][0] == second_index_result[0]['doc_id']
\ No newline at end of file
diff --git a/test/e2e/preset_test.go b/test/e2e/preset_test.go
index 4a85b33..72f4f9b 100644
--- a/test/e2e/preset_test.go
+++ b/test/e2e/preset_test.go
@@ -33,6 +33,7 @@ import (
 
 const (
 	PresetLlama3_1_8BInstruct           = "llama-3.1-8b-instruct"
+	PresetLlama3_3_70BInstruct          = "llama-3.3-70b-instruct"
 	PresetFalcon7BModel                 = "falcon-7b"
 	PresetFalcon40BModel                = "falcon-40b"
 	PresetMistral7BInstructModel        = "mistral-7b-instruct"
diff --git a/test/e2e/preset_vllm_test.go b/test/e2e/preset_vllm_test.go
index 4621a6f..2678e7e 100644
--- a/test/e2e/preset_vllm_test.go
+++ b/test/e2e/preset_vllm_test.go
@@ -54,9 +54,9 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateCompletionsEndpoint(workspaceObj)
 	})
 
-	It("should create a llama-3.1-8b-instruct workspace with preset public mode successfully", utils.GinkgoLabelFastCheck, func() {
+	It("should create a single-node llama-3.1-8b-instruct workspace with preset public mode successfully", utils.GinkgoLabelFastCheck, func() {
 		numOfNode := 1
-		workspaceObj := createLlama3_1_8BInstructWorkspaceWithPresetPublicModeAndVLLM(numOfNode)
+		workspaceObj := createLlama3_1_8BInstructWorkspaceWithPresetPublicModeAndVLLM(numOfNode, "Standard_NC12s_v3")
 
 		defer cleanupResources(workspaceObj)
 		time.Sleep(30 * time.Second)
@@ -76,6 +76,30 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateCompletionsEndpoint(workspaceObj)
 	})
 
+	It("should create a multi-node llama-3.1-8b-instruct workspace with preset public mode successfully", utils.GinkgoLabelFastCheck, func() {
+		// Need 2 Standard_NC6s_v3 nodes to run Llama 3.1-8B Instruct model.
+		// Each node has 1 V100 GPU, so total 2 GPUs are used
+		numOfNode := 2
+		workspaceObj := createLlama3_1_8BInstructWorkspaceWithPresetPublicModeAndVLLM(numOfNode, "Standard_NC6s_v3")
+
+		defer cleanupResources(workspaceObj)
+		time.Sleep(30 * time.Second)
+
+		validateCreateNode(workspaceObj, numOfNode)
+		validateResourceStatus(workspaceObj)
+
+		time.Sleep(30 * time.Second)
+
+		validateAssociatedService(workspaceObj)
+		validateInferenceConfig(workspaceObj)
+
+		validateInferenceResource(workspaceObj, int32(numOfNode), true)
+
+		validateWorkspaceReadiness(workspaceObj)
+		validateModelsEndpoint(workspaceObj)
+		validateCompletionsEndpoint(workspaceObj)
+	})
+
 	It("should create a deepseek-distilled-qwen-14b workspace with preset public mode successfully", func() {
 		numOfNode := 1
 		workspaceObj := createDeepSeekQwen14BWorkspaceWithPresetPublicModeAndVLLM(numOfNode)
@@ -240,6 +264,30 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 
 		validateAdapterLoadedInVLLM(workspaceObj, phi4AdapterName)
 	})
+
+	It("should create a llama-3.3-70b-instruct workspace with preset public mode successfully", func() {
+		// Need 2 Standard_NC48ads_A100_v4 nodes to run Llama 3.3-70B Instruct model.
+		// Each node has 2 A100 GPUs, so total 4 GPUs are used
+		numOfNode := 2
+		workspaceObj := createLlama3_3_70BInstructWorkspaceWithPresetPublicModeAndVLLM(numOfNode)
+
+		defer cleanupResources(workspaceObj)
+		time.Sleep(30 * time.Second)
+
+		validateCreateNode(workspaceObj, numOfNode)
+		validateResourceStatus(workspaceObj)
+
+		time.Sleep(30 * time.Second)
+
+		validateAssociatedService(workspaceObj)
+		validateInferenceConfig(workspaceObj)
+
+		validateInferenceResource(workspaceObj, int32(numOfNode), true)
+
+		validateWorkspaceReadiness(workspaceObj)
+		validateModelsEndpoint(workspaceObj)
+		validateCompletionsEndpoint(workspaceObj)
+	})
 })
 
 func createDeepSeekLlama8BWorkspaceWithPresetPublicModeAndVLLM(numOfNode int) *kaitov1beta1.Workspace {
@@ -354,16 +402,30 @@ func createQwen2_5WorkspaceWithPresetPublicModeAndVLLMAndMultiGPU(numOfNode int)
 	return workspaceObj
 }
 
-func createLlama3_1_8BInstructWorkspaceWithPresetPublicModeAndVLLM(numOfNode int) *kaitov1beta1.Workspace {
+func createLlama3_1_8BInstructWorkspaceWithPresetPublicModeAndVLLM(numOfNode int, instanceType string) *kaitov1beta1.Workspace {
 	modelSecret := createAndValidateModelSecret()
 	workspaceObj := &kaitov1beta1.Workspace{}
 	By("Creating a workspace CR with Llama 3.1-8B Instruct preset public mode and vLLM", func() {
 		uniqueID := fmt.Sprint("preset-llama3-1-8b-", rand.Intn(1000))
-		workspaceObj = utils.GenerateInferenceWorkspaceManifestWithVLLM(uniqueID, namespaceName, "", numOfNode, "Standard_NC12s_v3",
+		workspaceObj = utils.GenerateInferenceWorkspaceManifestWithVLLM(uniqueID, namespaceName, "", numOfNode, instanceType,
 			&metav1.LabelSelector{
-				MatchLabels: map[string]string{"kaito-workspace": "public-preset-e2e-test-llama3-1-8b-vllm"},
+				MatchLabels: map[string]string{"kaito-workspace": uniqueID},
 			}, nil, PresetLlama3_1_8BInstruct, nil, nil, nil, modelSecret.Name) // Llama 3.1-8B Instruct model requires a model access secret
 		createAndValidateWorkspace(workspaceObj)
 	})
 	return workspaceObj
 }
+
+func createLlama3_3_70BInstructWorkspaceWithPresetPublicModeAndVLLM(numOfNode int) *kaitov1beta1.Workspace {
+	modelSecret := createAndValidateModelSecret()
+	workspaceObj := &kaitov1beta1.Workspace{}
+	By("Creating a workspace CR with Llama 3.3-70B Instruct preset public mode and vLLM", func() {
+		uniqueID := fmt.Sprint("preset-llama3-3-70b-", rand.Intn(1000))
+		workspaceObj = utils.GenerateInferenceWorkspaceManifestWithVLLM(uniqueID, namespaceName, "", numOfNode, "Standard_NC48ads_A100_v4",
+			&metav1.LabelSelector{
+				MatchLabels: map[string]string{"kaito-workspace": "public-preset-e2e-test-llama3-3-70b-vllm"},
+			}, nil, PresetLlama3_3_70BInstruct, nil, nil, nil, modelSecret.Name) // Llama 3.3-70B Instruct model requires a model access secret
+		createAndValidateWorkspace(workspaceObj)
+	})
+	return workspaceObj
+}
diff --git a/test/rage2e/rag_test.go b/test/rage2e/rag_test.go
index 655ead5..8a0241f 100644
--- a/test/rage2e/rag_test.go
+++ b/test/rage2e/rag_test.go
@@ -4,6 +4,7 @@
 package e2e
 
 import (
+	"encoding/json"
 	"fmt"
 	"math/rand"
 	"os"
@@ -91,8 +92,12 @@ var _ = Describe("RAGEngine", func() {
 		validateInferenceandRAGResource(ragengineObj.ObjectMeta, int32(numOfReplica), false)
 		validateRAGEngineCondition(ragengineObj, string(kaitov1alpha1.RAGEngineConditionTypeSucceeded), "ragengine to be ready")
 
-		err := createAndValidateIndexPod(ragengineObj)
+		indexDoc, err := createAndValidateIndexPod(ragengineObj)
 		Expect(err).NotTo(HaveOccurred(), "Failed to create and validate IndexPod")
+		Expect(indexDoc).NotTo(BeNil(), "Index document should not be nil")
+		Expect(indexDoc["doc_id"]).NotTo(BeNil(), "Index document ID should not be nil")
+		Expect(indexDoc["text"]).NotTo(BeNil(), "Index document text should not be nil")
+		docID := indexDoc["doc_id"].(string)
 
 		searchQuerySuccess := "\\n\\nKaito is an operator that is designed to automate the AI/ML model inference or tuning workload in a Kubernetes cluster."
 		err = createAndValidateQueryPod(ragengineObj, searchQuerySuccess, true)
@@ -105,6 +110,15 @@ var _ = Describe("RAGEngine", func() {
 		loadLogSuccess := "Successfully loaded index kaito"
 		err = createAndValidateLoadPod(ragengineObj, loadLogSuccess)
 		Expect(err).NotTo(HaveOccurred(), "Failed to create and validate LoadPod")
+
+		err = createAndValidateUpdateDocumentPod(ragengineObj, docID)
+		Expect(err).NotTo(HaveOccurred(), "Failed to create and validate UpdateDocumentPod")
+
+		err = createAndValidateDeleteDocumentPod(ragengineObj, docID)
+		Expect(err).NotTo(HaveOccurred(), "Failed to create and validate DeleteDocumentPod")
+
+		err = createAndValidateDeleteIndexPod(ragengineObj)
+		Expect(err).NotTo(HaveOccurred(), "Failed to create and validate DeleteIndexPod")
 	})
 
 	It("should create RAG with localembedding and kaito VLLM workspace successfully", func() {
@@ -141,8 +155,13 @@ var _ = Describe("RAGEngine", func() {
 		validateInferenceandRAGResource(ragengineObj.ObjectMeta, int32(numOfReplica), false)
 		validateRAGEngineCondition(ragengineObj, string(kaitov1alpha1.RAGEngineConditionTypeSucceeded), "ragengine to be ready")
 
-		err := createAndValidateIndexPod(ragengineObj)
+		indexDoc, err := createAndValidateIndexPod(ragengineObj)
 		Expect(err).NotTo(HaveOccurred(), "Failed to create and validate IndexPod")
+		Expect(indexDoc).NotTo(BeNil(), "Index document should not be nil")
+		Expect(indexDoc["doc_id"]).NotTo(BeNil(), "Index document ID should not be nil")
+		Expect(indexDoc["text"]).NotTo(BeNil(), "Index document text should not be nil")
+		docID := indexDoc["doc_id"].(string)
+
 		searchQuerySuccess := "\\nKaito is an operator that automates the AI/ML model inference or tuning workload in a Kubernetes cluster.\\n\\n\\n"
 		err = createAndValidateQueryPod(ragengineObj, searchQuerySuccess, false)
 		Expect(err).NotTo(HaveOccurred(), "Failed to create and validate QueryPod")
@@ -154,6 +173,15 @@ var _ = Describe("RAGEngine", func() {
 		loadLogSuccess := "Successfully loaded index kaito"
 		err = createAndValidateLoadPod(ragengineObj, loadLogSuccess)
 		Expect(err).NotTo(HaveOccurred(), "Failed to create and validate LoadPod")
+
+		err = createAndValidateUpdateDocumentPod(ragengineObj, docID)
+		Expect(err).NotTo(HaveOccurred(), "Failed to create and validate UpdateDocumentPod")
+
+		err = createAndValidateDeleteDocumentPod(ragengineObj, docID)
+		Expect(err).NotTo(HaveOccurred(), "Failed to create and validate DeleteDocumentPod")
+
+		err = createAndValidateDeleteIndexPod(ragengineObj)
+		Expect(err).NotTo(HaveOccurred(), "Failed to create and validate DeleteIndexPod")
 	})
 
 })
@@ -478,7 +506,8 @@ func deleteWorkspace(workspaceObj *kaitov1beta1.Workspace) error {
 	return nil
 }
 
-func createAndValidateIndexPod(ragengineObj *kaitov1alpha1.RAGEngine) error {
+func createAndValidateIndexPod(ragengineObj *kaitov1alpha1.RAGEngine) (map[string]any, error) {
+	indexResp := []map[string]any{}
 	podName := "index-pod"
 	By("Creating index pod", func() {
 		curlCommand := `curl -X POST ` + ragengineObj.Name + `:80/index \
@@ -513,10 +542,148 @@ func createAndValidateIndexPod(ragengineObj *kaitov1alpha1.RAGEngine) error {
 				return false
 			}
 
+			GinkgoWriter.Printf("Index pod logs: %s\n", logs)
+
+			startIndex := strings.Index(logs, "[")
+			endIndex := strings.LastIndex(logs, "]")
+			if startIndex == -1 || endIndex == -1 || startIndex >= endIndex {
+				GinkgoWriter.Printf("Invalid index logs format in pod %s: %s\n", podName, logs)
+				return false
+			}
+
+			apiResp := logs[startIndex : endIndex+1]
+			GinkgoWriter.Printf("Parsed API response: %s\n", apiResp)
+
+			err = json.Unmarshal([]byte(apiResp), &indexResp)
+			if err != nil {
+				GinkgoWriter.Printf("Failed to unmarshal pod logs to index response %s: %v\n", podName, err)
+				return false
+			}
+
+			if len(indexResp) == 0 {
+				GinkgoWriter.Printf("No index logs found in pod %s\n", podName)
+				return false
+			}
+
 			return strings.Contains(logs, "Kaito is an operator that automates the AI/ML model inference or tuning workload in a Kubernetes cluster")
 		}, 4*time.Minute, utils.PollInterval).Should(BeTrue(), "Failed to wait for index logs to be ready")
 	})
 
+	return indexResp[0], nil
+}
+
+func createAndValidateUpdateDocumentPod(ragengineObj *kaitov1alpha1.RAGEngine, docID string) error {
+	podName := "update-document-pod"
+	By("Creating update document pod", func() {
+		curlCommand := `curl -X POST ` + ragengineObj.Name + `:80/indexes/kaito/documents \
+-H "Content-Type: application/json" \
+-d '{
+    "documents": [
+        {
+			"doc_id": "` + docID + `",
+            "text": "Kaito is an operator that automates the AI/ML model inference or tuning workload in a Kubernetes cluster. It now has RAG capabilities.",
+            "metadata": {"author": "kaito", "category": "ai-ml"}
+        }
+    ]
+}'`
+		pod := GenerateCURLPodManifest(podName, curlCommand, ragengineObj.Namespace)
+		Eventually(func() error {
+			return utils.TestingCluster.KubeClient.Create(ctx, pod, &client.CreateOptions{})
+		}, utils.PollTimeout, utils.PollInterval).
+			Should(Succeed(), "Failed to create index pod")
+	})
+
+	By("Checking the index logs", func() {
+		Eventually(func() bool {
+			coreClient, err := utils.GetK8sClientset()
+			if err != nil {
+				GinkgoWriter.Printf("Failed to create core client: %v\n", err)
+				return false
+			}
+
+			logs, err := utils.GetPodLogs(coreClient, ragengineObj.Namespace, podName, "")
+			if err != nil {
+				GinkgoWriter.Printf("Failed to get logs from pod %s: %v\n", podName, err)
+				return false
+			}
+
+			GinkgoWriter.Printf("Index pod logs: %s\n", logs)
+
+			return strings.Contains(logs, `"updated_documents":[{"doc_id":"`+docID+`"`)
+		}, 4*time.Minute, utils.PollInterval).Should(BeTrue(), "Failed to wait for update document logs to be ready")
+	})
+
+	return nil
+}
+
+func createAndValidateDeleteDocumentPod(ragengineObj *kaitov1alpha1.RAGEngine, docID string) error {
+	podName := "delete-document-pod"
+	By("Creating delete document pod", func() {
+		curlCommand := `curl -X POST ` + ragengineObj.Name + `:80/indexes/kaito/documents/delete \
+-H "Content-Type: application/json" \
+-d '{"doc_ids": ["` + docID + `"]}'`
+		pod := GenerateCURLPodManifest(podName, curlCommand, ragengineObj.Namespace)
+		Eventually(func() error {
+			return utils.TestingCluster.KubeClient.Create(ctx, pod, &client.CreateOptions{})
+		}, utils.PollTimeout, utils.PollInterval).
+			Should(Succeed(), "Failed to create index pod")
+	})
+
+	By("Checking the index logs", func() {
+		Eventually(func() bool {
+			coreClient, err := utils.GetK8sClientset()
+			if err != nil {
+				GinkgoWriter.Printf("Failed to create core client: %v\n", err)
+				return false
+			}
+
+			logs, err := utils.GetPodLogs(coreClient, ragengineObj.Namespace, podName, "")
+			if err != nil {
+				GinkgoWriter.Printf("Failed to get logs from pod %s: %v\n", podName, err)
+				return false
+			}
+
+			GinkgoWriter.Printf("Index pod logs: %s\n", logs)
+
+			return strings.Contains(logs, `"deleted_doc_ids":["`+docID+`"]`)
+		}, 4*time.Minute, utils.PollInterval).Should(BeTrue(), "Failed to wait for delete document logs to be ready")
+	})
+
+	return nil
+}
+
+func createAndValidateDeleteIndexPod(ragengineObj *kaitov1alpha1.RAGEngine) error {
+	podName := "delete-index-pod"
+	By("Creating delete index pod", func() {
+		curlCommand := `curl -X DELETE ` + ragengineObj.Name + `:80/indexes/kaito \
+-H "Content-Type: application/json"`
+		pod := GenerateCURLPodManifest(podName, curlCommand, ragengineObj.Namespace)
+		Eventually(func() error {
+			return utils.TestingCluster.KubeClient.Create(ctx, pod, &client.CreateOptions{})
+		}, utils.PollTimeout, utils.PollInterval).
+			Should(Succeed(), "Failed to create delete index pod")
+	})
+
+	By("Checking the delete index logs", func() {
+		Eventually(func() bool {
+			coreClient, err := utils.GetK8sClientset()
+			if err != nil {
+				GinkgoWriter.Printf("Failed to create core client: %v\n", err)
+				return false
+			}
+
+			logs, err := utils.GetPodLogs(coreClient, ragengineObj.Namespace, podName, "")
+			if err != nil {
+				GinkgoWriter.Printf("Failed to get logs from pod %s: %v\n", podName, err)
+				return false
+			}
+
+			GinkgoWriter.Printf("Index pod logs: %s\n", logs)
+
+			return strings.Contains(logs, "Successfully deleted index kaito")
+		}, 4*time.Minute, utils.PollInterval).Should(BeTrue(), "Failed to wait for index logs to be ready")
+	})
+
 	return nil
 }