Repository: kaito-project/kaito

Issue:
RAGEngine returns "Empty Response" when no relevant context is found
**Describe the bug**

Both `CONTEXT` and `CONDENSE_PLUS_CONTEXT` Chat types in LlamaIndex use the same CompactAndRefine [response synthesizers](https://github.com/run-llama/llama_index/blob/main/llama-index-core/llama_index/core/response_synthesizers/compact_and_refine.py)
 
Which in turn, both `achat` functions would call `response = await synthesizer.asynthesize(message, nodes)` and return `Empty Response` if they have [no nodes](https://github.com/run-llama/llama_index/blob/8469a034226d20b70a667dc7faf013770716709f/llama-index-core/llama_index/core/response_synthesizers/base.py#L271), all without calling the LLM.

**Steps To Reproduce**
Create an index with a simple document "Cats and Dogs are animals". Then use the chat completions api to hit the rag with the created index and a query of "What is pasta made of?".

**Expected behavior**
The request should make it to the LLM, even if ther is no context added.


Hints from issue comments:

Solution patch:
diff --git a/presets/ragengine/vector_store/base.py b/presets/ragengine/vector_store/base.py
index abcc525..0b79b8a 100644
--- a/presets/ragengine/vector_store/base.py
+++ b/presets/ragengine/vector_store/base.py
@@ -470,6 +470,18 @@ class BaseVectorStore(ABC):
                     user_prompt, chat_history=chat_history
                 )
 
+            # If no relevant context is found, the llamaindex response synthesizers will send an empty response without hitting the llm.
+            # We will instead send the passthrough request directly to the llm rather than send an empty response to the user
+            # https://github.com/run-llama/llama_index/blob/8469a034226d20b70a667dc7faf013770716709f/llama-index-core/llama_index/core/response_synthesizers/base.py#L271
+            if (
+                len(chat_result.source_nodes) == 0
+                and chat_result.response.strip() == "Empty Response"
+            ):
+                logger.info(
+                    "No relevant context found for the query. Falling back to passthrough request"
+                )
+                return await self.llm.chat_completions_passthrough(openai_request)
+
             return ChatCompletionResponse(
                 id=uuid.uuid4().hex,
                 object="chat.completion",
diff --git a/presets/ragengine/vector_store/chat_completions_wrapper.md b/website/docs/rag-chat-completions-wrapper.md
similarity index 98%
rename from presets/ragengine/vector_store/chat_completions_wrapper.md
rename to website/docs/rag-chat-completions-wrapper.md
index 0e5f359..68369b3 100644
--- a/presets/ragengine/vector_store/chat_completions_wrapper.md
+++ b/website/docs/rag-chat-completions-wrapper.md
@@ -368,6 +368,8 @@ chat_result = await chat_engine.achat(user_prompt, chat_history=chat_history)
 4. **Prompt Construction**: Selected documents are formatted into the final prompt
 5. **LLM Generation**: Complete prompt (context + history + user query) sent to LLM
 
+**In the event that no relevant context is found, the LlamaIndex library will return an empty response. We look out for this and send the request directly to the LLM without context as described in the bypass handling above.**
+
 ### 4. Token Budget Example
 
 Here's how token allocation works in practice:
diff --git a/website/sidebars.js b/website/sidebars.js
index 365c951..705ca63 100644
--- a/website/sidebars.js
+++ b/website/sidebars.js
@@ -53,6 +53,7 @@ const sidebars = {
                     },
                     items: [
                         'rag-api',
+                        'rag-chat-completions-wrapper'
                     ],
                 },
                 'custom-model',



Test patch:
diff --git a/presets/ragengine/tests/vector_store/test_base_store.py b/presets/ragengine/tests/vector_store/test_base_store.py
index e39d3d6..afd9dc2 100644
--- a/presets/ragengine/tests/vector_store/test_base_store.py
+++ b/presets/ragengine/tests/vector_store/test_base_store.py
@@ -12,7 +12,9 @@
 # limitations under the License.
 
 
+import json
 import os
+import time
 from abc import ABC, abstractmethod
 from unittest.mock import patch
 
@@ -212,6 +214,110 @@ class BaseVectorStoreTest(ABC):
             respx.calls.call_count == 1
         )  # Ensure only one LLM inference request was made
 
+        # Validate the request being sent to the LLM
+        llm_req = respx.calls[0].request
+        json_request = json.loads(llm_req.content)
+        print(json_request)
+        assert json_request["model"] == "mock-model"
+        assert json_request["temperature"] == 0.7
+        assert len(json_request["messages"]) == 2
+        assert json_request["messages"][0]["role"] == "system"
+        assert (
+            "Use the context information below to assist the user."
+            in json_request["messages"][0]["content"]
+        )
+        assert json_request["messages"][1]["role"] == "user"
+        assert json_request["messages"][1]["content"] == "What is the first document?"
+
+    @pytest.mark.asyncio
+    @respx.mock
+    @patch("requests.get")
+    async def test_chat_completions_with_no_context(
+        self, mock_get, vector_store_manager, monkeypatch
+    ):
+        import ragengine.config
+        import ragengine.inference.inference
+
+        monkeypatch.setattr(
+            ragengine.config,
+            "LLM_INFERENCE_URL",
+            "http://localhost:5000/v1/chat/completions",
+        )
+        monkeypatch.setattr(
+            ragengine.inference.inference,
+            "LLM_INFERENCE_URL",
+            "http://localhost:5000/v1/chat/completions",
+        )
+        mock_get.return_value.status_code = 200
+        mock_get.return_value.json.return_value = {
+            "data": [{"id": "mock-model", "max_model_len": 2048}]
+        }
+
+        mock_response = {
+            "id": "chatcmpl-test123",
+            "created": int(time.time()),
+            "object": "chat.completion",
+            "model": "mock-model",
+            "choices": [
+                {
+                    "index": 0,
+                    "message": {
+                        "role": "assistant",
+                        "content": "This is a helpful response about the test document.",
+                    },
+                    "finish_reason": "stop",
+                }
+            ],
+        }
+        respx.post("http://localhost:5000/v1/chat/completions").mock(
+            return_value=httpx.Response(200, json=mock_response)
+        )
+
+        documents = [
+            Document(text="Cats and dogs are animals", metadata={"type": "text"}),
+        ]
+        await vector_store_manager.index_documents("test_index", documents)
+
+        chat_results = await vector_store_manager.chat_completion(
+            {
+                "index_name": "test_index",
+                "model": "mock-model",
+                "messages": [{"role": "user", "content": "What is pasta made of?"}],
+                "temperature": 0.7,
+                "max_tokens": 100,
+            }
+        )
+
+        assert chat_results is not None
+        assert chat_results.source_nodes is None
+        assert chat_results.id is not None
+        assert chat_results.model == "mock-model"
+        assert chat_results.object == "chat.completion"
+        assert chat_results.created is not None
+        assert chat_results.choices is not None
+        assert len(chat_results.choices) == 1
+        assert chat_results.choices[0].finish_reason == "stop"
+        assert chat_results.choices[0].index == 0
+        assert chat_results.choices[0].message.role == "assistant"
+        assert (
+            chat_results.choices[0].message.content
+            == "This is a helpful response about the test document."
+        )
+
+        assert (
+            respx.calls.call_count == 1
+        )  # Ensure only one LLM inference request was made
+
+        # Validate the request being sent to the LLM
+        llm_req = respx.calls[0].request
+        json_request = json.loads(llm_req.content)
+        print(json_request)
+        assert json_request["model"] == "mock-model"
+        assert json_request["temperature"] == 0.7
+        assert len(json_request["messages"]) == 1
+        assert json_request["messages"][0]["role"] == "user"
+        assert json_request["messages"][0]["content"] == "What is pasta made of?"
+
     @pytest.mark.asyncio
     async def test_add_document(self, vector_store_manager):
         documents = [Document(text="Third document", metadata={"type": "text"})]