Update agent.py

agent.py

@@ -37,7 +37,7 @@ import re
 from langchain_community.document_loaders import TextLoader, PyMuPDFLoader
 from docx import Document as DocxDocument
 import openpyxl
-
+from io import StringIO
 
 load_dotenv()
 
@@ -312,16 +312,59 @@ for task in tasks:
 # Step 4: Set up HuggingFace Embeddings and FAISS VectorStore
 # -------------------------------
 # Initialize HuggingFace Embedding model
-embedding_model = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")
+#embedding_model = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")
+embedding_model = HuggingFaceEmbeddings(model_name="BAAI/bge-base-en-v1.5")
 
 
-from langchain_community.document_loaders import WikipediaLoader
 
-wiki_docs = WikipediaLoader(query="Mercedes Sosa", load_max_docs=3).load()
-all_docs = docs + wiki_docs  # `docs` is your existing list
+# -----------------------------
+# Step 1: Load CSV Questions
+# -----------------------------
+csv_path = "questions.csv"  # Change to your CSV file
+df = pd.read_csv(csv_path)
+
+docs = []
+for _, row in df.iterrows():
+    question = str(row.get("question", "")).strip()
+    if question:
+        docs.append(Document(page_content=question, metadata={"source": "csv"}))
+docs = []
+for _, row in df.iterrows():
+    question = str(row.get("question", "")).strip()
+    if question:
+        docs.append(Document(page_content=question, metadata={"source": "csv"}))
+
+# -----------------------------
+# Step 2: Add Wikipedia Docs
+# -----------------------------
+wiki_docs = []
+for doc in docs:
+    try:
+        wiki_results = WikipediaLoader(query=doc.page_content, load_max_docs=1).load()
+        wiki_docs.extend(wiki_results)
+    except Exception as e:
+        print(f"Failed to load Wikipedia for: {doc.page_content} — {e}")
+
+all_docs = docs + wiki_docs
+
+# -----------------------------
+# Step 3: Build FAISS Index
+# -----------------------------
+embedding_model = HuggingFaceEmbeddings(model_name="BAAI/bge-base-en-v1.5")
 vector_store = FAISS.from_documents(all_docs, embedding_model)
 vector_store.save_local("faiss_index")
 
+# -----------------------------
+# Step 4: Create Retriever Tool
+# -----------------------------
+retriever = vector_store.as_retriever()
+
+question_retriever_tool = create_retriever_tool(
+    retriever=retriever,
+    name="Question_Search",
+    description="A tool to retrieve documents related to a user's question."
+)
+
 
 
 # -------------------------------
@@ -336,6 +379,30 @@ question_retriever_tool = create_retriever_tool(
     description="A tool to retrieve documents related to a user's question."
 )
 
+vector_store = FAISS.from_documents(all_docs, embedding_model)
+vector_store.save_local("faiss_index")
+
+
+
+def retriever(state: MessagesState):
+    """Retriever node using similarity scores for filtering"""
+    query = state["messages"][0].content
+    results = vector_store.similarity_search_with_score(query, k=4)  # top 4 matches
+
+    # Filter by score (lower is more similar; adjust threshold as needed)
+    threshold = 0.8
+    filtered = [doc for doc, score in results if score < threshold]
+
+    if not filtered:
+        example_msg = HumanMessage(content="No relevant documents found.")
+    else:
+        content = "\n\n".join(doc.page_content for doc in filtered)
+        example_msg = HumanMessage(
+            content=f"Here are relevant reference documents:\n\n{content}"
+        )
+
+    return {"messages": [sys_msg] + state["messages"] + [example_msg]}
+
 
 
 tools = [