added chunk details

api.py

@@ -12,6 +12,7 @@ from fastapi.responses import StreamingResponse
 from huggingface_hub import InferenceClient
 from pydantic import BaseModel, Field
 
+from config_loader import cfg
 from vector_db import get_index_by_name, load_chunks_with_local_cache
 from retriever.retriever import HybridRetriever
 from retriever.generator import RAGGenerator
@@ -40,7 +41,7 @@ class PredictResponse(BaseModel):
     model: str
     answer: str
     contexts: list[str]
-    metrics: dict[str, float]
+    retrieved_chunks: list[dict[str, Any]]
 
 
 class TitleRequest(BaseModel):
@@ -139,6 +140,34 @@ def _parse_title_model_candidates() -> list[str]:
     return models or ["meta-llama/Meta-Llama-3-8B-Instruct"]
 
 
+def _build_retrieved_chunks(
+    contexts: list[str],
+    chunk_lookup: dict[str, dict[str, Any]],
+) -> list[dict[str, Any]]:
+    if not contexts:
+        return []
+
+    retrieved_chunks: list[dict[str, Any]] = []
+
+    for idx, text in enumerate(contexts, start=1):
+        meta = chunk_lookup.get(text, {})
+        title = meta.get("title") or "Untitled"
+        url = meta.get("url") or ""
+        chunk_index = meta.get("chunk_index")
+
+        retrieved_chunks.append(
+            {
+                "rank": idx,
+                "text": text,
+                "source_title": title,
+                "source_url": url,
+                "chunk_index": chunk_index,
+            }
+        )
+
+    return retrieved_chunks
+
+
 
 # Fastapi setup
 # Fastapi allows us to define python based endpoint
@@ -209,7 +238,9 @@ def startup_event() -> None:
         raise RuntimeError("HF_TOKEN not found in environment variables")
 
     index_name = "cbt-book-recursive"
-    embed_model_name = "all-MiniLM-L6-v2"
+    # Keep retrieval embedding model aligned with the one used at ingest time
+    # to avoid Pinecone dimension mismatch errors (e.g., 384 vs 512).
+    embed_model_name = cfg.processing.get("embedding_model", "all-MiniLM-L6-v2")
     project_root = os.path.dirname(os.path.abspath(__file__))
     cache_dir = os.getenv("BM25_CACHE_DIR", os.path.join(project_root, ".cache"))
     force_cache_refresh = os.getenv("BM25_CACHE_REFRESH", "0").lower() in {"1", "true", "yes"}
@@ -251,10 +282,23 @@ def startup_event() -> None:
     models_time = time.perf_counter() - models_start
 
     state_start = time.perf_counter()
+    chunk_lookup: dict[str, dict[str, Any]] = {}
+    for chunk in final_chunks:
+        metadata = chunk.get("metadata", {})
+        text = metadata.get("text")
+        if not text or text in chunk_lookup:
+            continue
+        chunk_lookup[text] = {
+            "title": metadata.get("title", "Untitled"),
+            "url": metadata.get("url", ""),
+            "chunk_index": metadata.get("chunk_index"),
+        }
+
     state["index"] = index
     state["retriever"] = retriever
     state["rag_engine"] = rag_engine
     state["models"] = models
+    state["chunk_lookup"] = chunk_lookup
     state["title_model_ids"] = _parse_title_model_candidates()
     state["title_client"] = InferenceClient(token=hf_token)
     state_time = time.perf_counter() - state_start
@@ -338,6 +382,7 @@ def predict(payload: PredictRequest) -> PredictResponse:
     index = state["index"]
     rag_engine: RAGGenerator = state["rag_engine"]
     models: dict[str, Any] = state["models"]
+    chunk_lookup: dict[str, dict[str, Any]] = state.get("chunk_lookup", {})
     state_access_time = time.perf_counter() - state_access_start
 
     model_resolve_start = time.perf_counter()
@@ -364,33 +409,28 @@ def predict(payload: PredictRequest) -> PredictResponse:
     answer = rag_engine.get_answer(model_instance, query, contexts, temperature=0.1)
     inference_time = time.perf_counter() - inference_start
 
-    response_start = time.perf_counter()
-    metrics = {
-        "precheck_s": round(precheck_time, 3),
-        "state_access_s": round(state_access_time, 3),
-        "model_resolve_s": round(model_resolve_time, 3),
-        "retrieval_s": round(retrieval_time, 3),
-        "inference_s": round(inference_time, 3),
-    }
-    response_build_time = time.perf_counter() - response_start
+    mapping_start = time.perf_counter()
+    retrieved_chunks = _build_retrieved_chunks(
+        contexts=contexts,
+        chunk_lookup=chunk_lookup,
+    )
+    mapping_time = time.perf_counter() - mapping_start
 
     total_time = time.perf_counter() - req_start
-    metrics["response_build_s"] = round(response_build_time, 3)
-    metrics["total_s"] = round(total_time, 3)
 
     print(
         f"Predict timing | model={model_name} | mode={payload.mode} | "
         f"rerank={payload.rerank_strategy} | precheck={precheck_time:.3f}s | "
         f"state_access={state_access_time:.3f}s | model_resolve={model_resolve_time:.3f}s | "
         f"retrieval={retrieval_time:.3f}s | inference={inference_time:.3f}s | "
-        f"response_build={response_build_time:.3f}s | total={total_time:.3f}s"
+        f"context_map={mapping_time:.3f}s | total={total_time:.3f}s"
     )
 
     return PredictResponse(
         model=model_name,
         answer=answer,
         contexts=contexts,
-        metrics=metrics,
+        retrieved_chunks=retrieved_chunks,
     )
 
 # new endpoint for streaming response, allows frontend to render tokens as they come in instead of waiting for full answer
@@ -412,6 +452,7 @@ def predict_stream(payload: PredictRequest) -> StreamingResponse:
     index = state["index"]
     rag_engine: RAGGenerator = state["rag_engine"]
     models: dict[str, Any] = state["models"]
+    chunk_lookup: dict[str, dict[str, Any]] = state.get("chunk_lookup", {})
     state_access_time = time.perf_counter() - state_access_start
 
     model_resolve_start = time.perf_counter()
@@ -443,23 +484,19 @@ def predict_stream(payload: PredictRequest) -> StreamingResponse:
                 yield _to_ndjson({"type": "token", "token": token})
 
             inference_time = time.perf_counter() - inference_start
-            total_time = time.perf_counter() - req_start
             answer = "".join(answer_parts)
-            metrics = {
-                "precheck_s": round(precheck_time, 3),
-                "state_access_s": round(state_access_time, 3),
-                "model_resolve_s": round(model_resolve_time, 3),
-                "retrieval_s": round(retrieval_time, 3),
-                "inference_s": round(inference_time, 3),
-                "total_s": round(total_time, 3),
-            }
+            retrieved_chunks = _build_retrieved_chunks(
+                contexts=contexts,
+                chunk_lookup=chunk_lookup,
+            )
 
             yield _to_ndjson(
                 {
                     "type": "done",
                     "model": model_name,
                     "answer": answer,
-                    "metrics": metrics,
+                    "contexts": contexts,
+                    "retrieved_chunks": retrieved_chunks,
                 }
             )
         except Exception as exc:

frontend/components/AIAssistantUI.jsx

@@ -320,7 +320,7 @@ export default function AIAssistantUI() {
       )
     }
 
-    const finalizeAssistant = (finalText) => {
+    const finalizeAssistant = (finalText, details = null) => {
       upsertAssistantMessage(
         (m) => {
           const fallbackContent = m.content || "Sorry, I encountered an error."
@@ -328,6 +328,8 @@ export default function AIAssistantUI() {
             ...m,
             content: finalText != null ? finalText : fallbackContent,
             isStreaming: false,
+            retrievedChunks: details?.retrieved_chunks || [],
+            contexts: details?.contexts || [],
           }
         },
         finalText || "Sorry, I encountered an error.",
@@ -385,6 +387,7 @@ export default function AIAssistantUI() {
       let buffer = ""
       let firstTokenReceived = false
       let finalAnswer = null
+      let donePayload = null
 
       while (true) {
         const { value, done } = await reader.read()
@@ -414,6 +417,7 @@ export default function AIAssistantUI() {
           }
 
           if (evt.type === "done") {
+            donePayload = evt
             finalAnswer = typeof evt.answer === "string" ? evt.answer : null
           }
 
@@ -428,6 +432,7 @@ export default function AIAssistantUI() {
         try {
           const evt = JSON.parse(remainder)
           if (evt.type === "done") {
+            donePayload = evt
             finalAnswer = typeof evt.answer === "string" ? evt.answer : null
           }
           if (evt.type === "token") {
@@ -441,7 +446,7 @@ export default function AIAssistantUI() {
         }
       }
 
-      finalizeAssistant(finalAnswer)
+      finalizeAssistant(finalAnswer, donePayload)
     } catch (err) {
       console.error("predict request failed:", err)
       if (err?.name === "AbortError") {

frontend/components/ChatPane.jsx

@@ -1,7 +1,7 @@
 "use client"
 
 import { useState, forwardRef, useImperativeHandle, useRef } from "react"
-import { RefreshCw, Check, X, Square } from "lucide-react"
+import { RefreshCw, Check, X, Square, ChevronDown, ChevronUp } from "lucide-react"
 import Message from "./Message"
 import MarkdownMessage from "./MarkdownMessage"
 import Composer from "./Composer"
@@ -28,6 +28,41 @@ function ThinkingMessage({ onPause }) {
   )
 }
 
+function AssistantDetails({ message }) {
+  const chunks = Array.isArray(message?.retrievedChunks) ? message.retrievedChunks : []
+
+  if (!chunks.length) {
+    return (
+      <div className="rounded-xl border border-zinc-200 bg-zinc-50 p-3 text-xs text-zinc-500 dark:border-zinc-800 dark:bg-zinc-900/50 dark:text-zinc-400">
+        No retrieved chunks were returned for this response.
+      </div>
+    )
+  }
+
+  return (
+    <div className="space-y-3 rounded-xl border border-zinc-200 bg-zinc-50 p-3 text-xs dark:border-zinc-800 dark:bg-zinc-900/50">
+      <div className="space-y-2">
+        <div className="text-[10px] uppercase tracking-wide text-zinc-500">Retrieved Chunks ({chunks.length})</div>
+        <div className="max-h-80 space-y-2 overflow-y-auto pr-1">
+          {chunks.map((chunk) => (
+            <div
+              key={`${message.id}-${chunk.rank}`}
+              className="rounded-lg border border-zinc-200 bg-white p-2 dark:border-zinc-800 dark:bg-zinc-950"
+            >
+              <div className="mb-1 flex flex-wrap items-center gap-2 text-[10px] text-zinc-500">
+                <span className="rounded-full border border-zinc-300 px-1.5 py-0.5 dark:border-zinc-700">Chunk #{chunk.rank}</span>
+                {chunk.source_title && <span>{chunk.source_title}</span>}
+                {chunk.chunk_index !== null && chunk.chunk_index !== undefined && <span>Part {chunk.chunk_index}</span>}
+              </div>
+              <div className="whitespace-pre-wrap text-xs text-zinc-700 dark:text-zinc-300">{chunk.text}</div>
+            </div>
+          ))}
+        </div>
+      </div>
+    </div>
+  )
+}
+
 const ChatPane = forwardRef(function ChatPane(
   {
     conversation,
@@ -42,6 +77,7 @@ const ChatPane = forwardRef(function ChatPane(
   const [editingId, setEditingId] = useState(null)
   const [draft, setDraft] = useState("")
   const [busy, setBusy] = useState(false)
+  const [openDetailsId, setOpenDetailsId] = useState(null)
   const composerRef = useRef(null)
 
   useImperativeHandle(
@@ -174,8 +210,31 @@ const ChatPane = forwardRef(function ChatPane(
                 ) : (
                   <Message role={m.role} streaming={Boolean(m.isStreaming)}>
                     {m.role === "assistant" ? (
-                      <div className={cls(m.isStreaming && "streaming-text-reveal")}>
-                        <MarkdownMessage content={m.content} isStreaming={Boolean(m.isStreaming)} />
+                      <div className="space-y-2">
+                        <div className={cls(m.isStreaming && "streaming-text-reveal")}>
+                          <MarkdownMessage content={m.content} isStreaming={Boolean(m.isStreaming)} />
+                        </div>
+                        {!m.isStreaming && Array.isArray(m.retrievedChunks) && m.retrievedChunks.length > 0 && (
+                          <>
+                            <button
+                              onClick={() => setOpenDetailsId((prev) => (prev === m.id ? null : m.id))}
+                              className="inline-flex items-center gap-1 rounded-full border border-zinc-300 px-2 py-1 text-xs text-zinc-600 hover:bg-zinc-100 dark:border-zinc-700 dark:text-zinc-300 dark:hover:bg-zinc-800"
+                            >
+                              <span>{openDetailsId === m.id ? "Hide Details" : "Details"}</span>
+                              {openDetailsId === m.id ? <ChevronUp className="h-3.5 w-3.5" /> : <ChevronDown className="h-3.5 w-3.5" />}
+                            </button>
+                            <div
+                              className={cls(
+                                "grid overflow-hidden transition-all duration-300 ease-out",
+                                openDetailsId === m.id ? "grid-rows-[1fr] opacity-100" : "grid-rows-[0fr] opacity-0",
+                              )}
+                            >
+                              <div className="min-h-0 overflow-hidden">
+                                <AssistantDetails message={m} />
+                              </div>
+                            </div>
+                          </>
+                        )}
                       </div>
                     ) : (
                       <div className="whitespace-pre-wrap">{m.content}</div>

frontend/components/Header.tsx

@@ -21,15 +21,15 @@ export default function Header() {
           // see https://reui.io/docs/github-button for more variables
           initialStars={1}
           label=""
-          targetStars={5}
+          targetStars={3}
 
-          repoUrl="https://github.com/Qar-Raz/mlops_project.git"
+          repoUrl="https://github.com/ramailkk/RAG-AS3-NLP"
 
           filled = {true}
           animationDuration= {5}
           roundStars={true}
           // below line can be commented out for clear black button --@Qamar
-          className="bg-gray-900/50 border-gray-700 text-gray-200 hover:bg-gray-800/50 hover:border-gray-600"
+          className="bg-gray-900/50 border-gray-700 text-gray-200 hover:bg-gray-800/50 hover:border-gray-600 dark:bg-gray-900/50 dark:border-gray-700 dark:text-gray-200 dark:hover:bg-gray-800/50 dark:hover:border-gray-600"
         />
 
       </nav>

frontend/components/ui/github-button.tsx

@@ -160,7 +160,6 @@ function GithubButton({
   const ref = React.useRef(null);
   const isInView = useInView(ref, inViewOptions);
 
-  // Reset animation state when targetStars changes
   useEffect(() => {
     setHasAnimated(false);
     setCurrentStars(initialStars);