some updates 2.0

pipelines.py

@@ -1,17 +1,15 @@
 import os
-os.environ["HAYSTACK_TELEMETRY_ENABLED"] = "False"
 import logging
 from haystack.utils import Secret
 from haystack.dataclasses import Document
 from haystack.document_stores.in_memory import InMemoryDocumentStore
 from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder
 from haystack.components.retrievers.in_memory import InMemoryEmbeddingRetriever
-
-# CORRECT IMPORT FOR HAYSTACK 2.1.0
-from haystack.nodes.ranker import SentenceTransformersRanker
-
-from haystack_integrations.components.generators.google_ai import GoogleAIGeminiGenerator
 from haystack.components.preprocessors import DocumentSplitter
+from haystack_integrations.components.generators.google_ai import GoogleAIGeminiGenerator
+
+# ✅ CORRECTED IMPORT FOR HAYSTACK 2.x
+from haystack.components.rankers import SentenceTransformersRanker
 
 # Set up logging
 logger = logging.getLogger(__name__)
@@ -29,12 +27,12 @@ text_embedder = SentenceTransformersTextEmbedder(
     model="BAAI/bge-base-en-v1.5",
     use_gpu=False
 )
-retriever = InMemoryEmbeddingRetriever(document_store=document_store, top_k=3)
+retriever = InMemoryEmbeddingRetriever(document_store=document_store, top_k=5) # Retrieve more to give reranker more options
 
-# Initialize ranker - DIFFERENT INITIALIZATION FOR 2.1.0
+# ✅ CORRECTED INITIALIZATION FOR HAYSTACK 2.x RERANKER
 reranker = SentenceTransformersRanker(
-    model_name_or_path="cross-encoder/ms-marco-TinyBERT-L-2-v2",
-    use_gpu=False
+    model="cross-encoder/ms-marco-TinyBERT-L-2-v2",
+    top_k=3 # Set top_k during initialization or run
 )
 
 # Initialize generator
@@ -60,7 +58,8 @@ try:
     logger.info("Warming up components...")
     doc_embedder.warm_up()
     text_embedder.warm_up()
-    reranker.prepared = True  # Different warmup for 2.1.0
+    # ✅ CORRECTED WARMUP FOR HAYSTACK 2.x
+    reranker.warm_up()
     logger.info("Components warmed up")
 except Exception as e:
     logger.error(f"Warmup failed: {e}")
@@ -76,22 +75,12 @@ def add_documents(texts: list[str], meta_list: list[dict]) -> int:
     if not docs:
         return 0
         
-    split_result = splitter.run(docs)
-    split_docs = split_result.get("documents", [])
+    split_docs = splitter.run(documents=docs).get("documents", [])
     
     if not split_docs:
         return 0
         
-    embedded_docs = []
-    batch_size = 8
-    
-    for i in range(0, len(split_docs), batch_size):
-        batch = split_docs[i:i+batch_size]
-        try:
-            embedded_batch = doc_embedder.run(batch).get("documents", [])
-            embedded_docs.extend(embedded_batch)
-        except Exception as e:
-            logger.error(f"Embedding failed: {e}")
+    embedded_docs = doc_embedder.run(documents=split_docs).get("documents", [])
     
     if embedded_docs:
         document_store.write_documents(embedded_docs)
@@ -103,7 +92,7 @@ def query_rag(question: str, session_id: str) -> dict:
         if not question.strip():
             return {"answer": "Please provide a non-empty question.", "sources": []}
             
-        embedding_result = text_embedder.run(question)
+        embedding_result = text_embedder.run(text=question)
         query_emb = embedding_result.get("embedding")
         
         if not query_emb:
@@ -115,23 +104,20 @@ def query_rag(question: str, session_id: str) -> dict:
         if not retrieved_docs:
             return {"answer": "No documents found. Upload a file first.", "sources": []}
         
-        # DIFFERENT USAGE FOR 2.1.0 RANKER
-        reranked_docs = reranker.predict(
-            query=question,
-            documents=retrieved_docs[:5],
-            top_k=3
-        )
+        # ✅ CORRECTED USAGE FOR HAYSTACK 2.x RERANKER
+        rerank_result = reranker.run(query=question, documents=retrieved_docs)
+        reranked_docs = rerank_result.get("documents", [])
         
         context = "\n\n".join([doc.content for doc in reranked_docs])
-        prompt = f"Context:\n{context}\n\nQuestion: {question}\nAnswer:"
+        prompt = f"Given the following context, please answer the question.\n\nContext:\n{context}\n\nQuestion: {question}"
         
-        response = generator.run(parts=[prompt])
-        answer = response.get("replies", [""])[0] if response and response.get("replies") else "No response"
+        response = generator.run(prompt=prompt)
+        answer = response["replies"][0] if response.get("replies") else "Sorry, I couldn't generate an answer."
         
         sources = [
             {
-                "filename": d.meta.get("filename", "Unknown"),
-                "page": d.meta.get("page", 1),
+                "filename": d.meta.get("file_name", "Unknown"), # Standardized meta key
+                "page": d.meta.get("page_number", "N/A"),
                 "snippet": d.content[:200] + "..." if len(d.content) > 200 else d.content
             } 
             for d in reranked_docs

requirements.txt

@@ -1,22 +1,13 @@
 # Core dependencies
-fastapi==0.111.0
-uvicorn==0.30.1
-python-multipart==0.0.9
-pillow==10.3.0
-pdfplumber==0.11.0
-pytesseract==0.3.10
+fastapi
+uvicorn
+python-multipart
+pillow
+pdfplumber
+pytesseract
+python-dotenv
 
-# Optimized sentence transformers
-sentence-transformers==3.0.1
-onnxruntime==1.17.3
-
-# Google AI and Haystack
-google-generativeai==0.7.2
-haystack-ai==2.1.0
-psutil==5.9.8
-
-# Compatible dependencies
-protobuf==4.25.3
-grpcio==1.64.0
-python-dotenv==1.0.1
-rpds-py==0.18.0
+# Haystack and Integrations
+haystack-ai
+google-ai-haystack
+sentence-transformers