changed the retriever knowledge

chatbot_retriever.py

@@ -262,8 +262,9 @@ def build_or_load_indexes(force_reindex: bool = False):
         try:
             index = faiss.read_index(FAISS_INDEX_PATH)
             with open(FAISS_META_PATH, "rb") as f:
-                meta = pickle.load(f)
+               meta = pickle.load(f)
             texts = meta.get("texts", corpus_texts)
+            metadatas = meta.get("metadatas", metadatas)
             try:
                 index.nprobe = FAISS_NPROBE
             except Exception:
@@ -348,7 +349,10 @@ def build_or_load_indexes(force_reindex: bool = False):
     try:
         faiss.write_index(index, FAISS_INDEX_PATH)
         with open(FAISS_META_PATH, "wb") as f:
-            pickle.dump({"texts": corpus_texts}, f)
+            pickle.dump({
+                "texts": corpus_texts,
+                "metadatas": metadatas
+            }, f)
         logger.info("FAISS index saved to %s (entries=%d)", FAISS_INDEX_PATH, total)
     except Exception:
         logger.exception("Failed to persist FAISS index on disk")
@@ -391,8 +395,9 @@ def _faiss_search(query: str, top_k: int = TOP_K_DOCS, subject: Optional[str] =
         if idx < 0 or idx >= len(texts):
             continue
         meta = metadatas[idx]
-        if subject and meta.get("subject") != subject:
-            continue
+# subject filtering disabled because it blocks many relevant chunks
+#         if subject and meta.get("subject") != subject:
+#         continue
         score_like = float(-dist)
         results.append((score_like, meta, texts[idx]))
         if len(results) >= top_k:
@@ -433,13 +438,23 @@ def hybrid_retrieve(query: str, subject: Optional[str] = None, top_k: int = TOP_
     merged_texts = []
     merged_meta = []
     for score, meta, text in results_bm25:
-        if text and text.strip() and text not in merged_texts:
+        if text and text.strip():
             merged_texts.append(text)
-            merged_meta.append({"source": meta.get("filename"), "subject": meta.get("subject"), "score": score})
+            merged_meta.append({
+                "source": meta.get("filename"),
+                "subject": meta.get("subject"),
+                "score": score
+            })
+
     for score, meta, text in results_faiss:
-        if text and text.strip() and text not in merged_texts:
+        if text and text.strip():
             merged_texts.append(text)
-            merged_meta.append({"source": meta.get("filename") if isinstance(meta, dict) else None, "subject": meta.get("subject") if isinstance(meta, dict) else None, "score": score})
+            merged_meta.append({
+                "source": meta.get("filename") if isinstance(meta, dict) else None,
+                "subject": meta.get("subject") if isinstance(meta, dict) else None,
+                "score": score
+            })
+
 
     # compose context parts with headers
     context_parts = []
@@ -457,13 +472,12 @@ def hybrid_retrieve(query: str, subject: Optional[str] = None, top_k: int = TOP_
 
 
 # ---------- retrieve_node (for reuse) ----------
-def _last_n_user_messages(rows: List[tuple], n: int = 3) -> List[str]:
-    """Return only the latest user message for retrieval context."""
+def _last_n_user_messages(rows: List[tuple], n: int = 1) -> List[str]:
     users = [r[1] for r in rows if r[0] == "user"]
-    return users[-n:]  # only keep the last one
+    return users[-1:]  # always return ONLY the latest user query # only keep the last one
 
 def retrieve_node_from_rows(rows: List[tuple], top_k: int = TOP_K_DOCS) -> Dict[str, Any]:
-    last_users = _last_n_user_messages(rows, n=3)
+    last_users = _last_n_user_messages(rows, n=1)
     current_query = " ".join(last_users).strip() if last_users else ""
     if not current_query:
         return {"context": None, "direct": False}