changed to deep translator

app.py

@@ -7,6 +7,7 @@ from src.data_processing import load_huggingface_faq_data, load_faq_data, prepro
 from src.embedding import FAQEmbedder
 from src.llm_response import ResponseGenerator
 from src.utils import time_function, format_memory_stats, evaluate_response, evaluate_retrieval, baseline_keyword_search
+from deep_translator import GoogleTranslator  # Updated import
 
 # Suppress CUDA warning and Torch path errors
 os.environ["CUDA_VISIBLE_DEVICES"] = ""
@@ -146,10 +147,9 @@ def main():
     
     if submit_button and user_query:
         from src.data_processing import translate_faq
-        from googletrans import Translator
-        translator = Translator()
+        translator = GoogleTranslator(source='auto', target='en')  # Updated translator
         if target_lang != "en":
-            user_query_translated = translator.translate(user_query, dest="en").text
+            user_query_translated = translator.translate(user_query)
         else:
             user_query_translated = user_query
         
@@ -172,7 +172,7 @@ def main():
             generation_time = time.time() - start_time
             
             if target_lang != "en":
-                response = translator.translate(response, dest=target_lang).text
+                response = translator.translate(response, target=target_lang)
             
             st.session_state.query_cache[user_query_translated] = (response, relevant_faqs)
             st.session_state.retrieval_time = retrieval_time
@@ -210,11 +210,9 @@ def main():
             st.session_state.user_input = question
             st.session_state.chat_history.append({"role": "user", "content": question})
             
-            from src.data_processing import translate_faq
-            from googletrans import Translator
-            translator = Translator()
+            translator = GoogleTranslator(source='auto', target='en')  # Updated translator
             if target_lang != "en":
-                question_translated = translator.translate(question, dest="en").text
+                question_translated = translator.translate(question)
             else:
                 question_translated = question
             
@@ -237,7 +235,7 @@ def main():
                 generation_time = time.time() - start_time
                 
                 if target_lang != "en":
-                    response = translator.translate(response, dest=target_lang).text
+                    response = translator.translate(response, target=target_lang)
                 
                 st.session_state.query_cache[question_translated] = (response, relevant_faqs)
                 st.session_state.retrieval_time = retrieval_time
@@ -247,4 +245,257 @@ def main():
             st.session_state.chat_history.append({"role": "assistant", "content": response})
 
 if __name__ == "__main__":
-    main()
+    main()
+
+
+
+# import streamlit as st
+# import time
+# import os
+# import gc
+# import torch
+# from src.data_processing import load_huggingface_faq_data, load_faq_data, preprocess_faq, augment_faqs
+# from src.embedding import FAQEmbedder
+# from src.llm_response import ResponseGenerator
+# from src.utils import time_function, format_memory_stats, evaluate_response, evaluate_retrieval, baseline_keyword_search
+
+# # Suppress CUDA warning and Torch path errors
+# os.environ["CUDA_VISIBLE_DEVICES"] = ""
+# os.environ["TORCH_NO_PATH_CHECK"] = "1"
+
+# st.set_page_config(page_title="E-Commerce FAQ Chatbot", layout="wide", initial_sidebar_state="expanded")
+
+# @time_function
+# def initialize_components(use_huggingface: bool = True, model_name: str = "microsoft/phi-2", enable_augmentation: bool = True):
+#     """
+#     Initialize RAG system components
+#     """
+#     try:
+#         if use_huggingface:
+#             faqs = load_huggingface_faq_data("NebulaByte/E-Commerce_FAQs")
+#         else:
+#             faqs = load_faq_data("data/faq_data.csv")
+        
+#         processed_faqs = augment_faqs(preprocess_faq(faqs), enable_augmentation=enable_augmentation)
+#         embedder = FAQEmbedder()
+        
+#         if os.path.exists("embeddings"):
+#             embedder.load("embeddings")
+#         else:
+#             embedder.create_embeddings(processed_faqs)
+#             embedder.save("embeddings")
+        
+#         gc.collect()
+#         if torch.cuda.is_available():
+#             torch.cuda.empty_cache()
+        
+#         response_generator = ResponseGenerator(model_name=model_name)
+#         response_generator.generate_response("Warmup query", [{"question": "Test", "answer": "Test"}])
+        
+#         return embedder, response_generator, len(processed_faqs)
+#     except Exception as e:
+#         st.error(f"Initialization failed: {e}")
+#         raise
+
+# def main():
+#     st.title("E-Commerce Customer Support FAQ Chatbot")
+#     st.subheader("Ask about orders, shipping, returns, or other e-commerce queries")
+    
+#     st.sidebar.title("Configuration")
+#     use_huggingface = st.sidebar.checkbox("Use Hugging Face Dataset", value=True)
+#     enable_augmentation = st.sidebar.checkbox("Enable FAQ Augmentation", value=True, help="Generate paraphrased questions to expand dataset")
+#     target_lang = st.sidebar.selectbox("Language", ["en", "es", "fr"], index=0)
+    
+#     model_options = {
+#         "Phi-2 (Recommended for 16GB RAM)": "microsoft/phi-2",
+#         "TinyLlama-1.1B (Fastest)": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+#         "Mistral-7B (For 15GB+ GPU)": "mistralai/Mistral-7B-Instruct-v0.1"
+#     }
+#     selected_model = st.sidebar.selectbox("Select LLM Model", list(model_options.keys()), index=0)
+#     model_name = model_options[selected_model]
+    
+#     if st.sidebar.checkbox("Show Memory Usage", value=True):
+#         st.sidebar.subheader("Memory Usage")
+#         for key, value in format_memory_stats().items():
+#             st.sidebar.text(f"{key}: {value}")
+    
+#     if "chat_history" not in st.session_state:
+#         st.session_state.chat_history = []
+#     if "query_cache" not in st.session_state:
+#         st.session_state.query_cache = {}
+#     if "feedback" not in st.session_state:
+#         st.session_state.feedback = []
+    
+#     if "system_initialized" not in st.session_state or st.sidebar.button("Reload System"):
+#         with st.spinner("Initializing system..."):
+#             try:
+#                 st.session_state.embedder, st.session_state.response_generator, num_faqs = initialize_components(
+#                     use_huggingface=use_huggingface,
+#                     model_name=model_name,
+#                     enable_augmentation=enable_augmentation
+#                 )
+#                 st.session_state.system_initialized = True
+#                 st.sidebar.success(f"System initialized with {num_faqs} FAQs!")
+#             except Exception as e:
+#                 st.error(f"System initialization failed: {e}")
+#                 return
+    
+#     col1, col2 = st.columns([2, 1])
+    
+#     with col1:
+#         st.subheader("Conversation")
+#         chat_container = st.container(height=400)
+#         with chat_container:
+#             for i, message in enumerate(st.session_state.chat_history):
+#                 if message["role"] == "user":
+#                     st.markdown(f"**You**: {message['content']}")
+#                 else:
+#                     st.markdown(f"**Bot**: {message['content']}")
+#                 if i < len(st.session_state.chat_history) - 1:
+#                     st.markdown("---")
+        
+#         with st.form(key="chat_form"):
+#             user_query = st.text_input("Type your question:", key="user_input", placeholder="e.g., How do I track my order?")
+#             submit_button = st.form_submit_button("Ask")
+        
+#         if len(st.session_state.chat_history) > 0:
+#             with st.form(key=f"feedback_form_{len(st.session_state.chat_history)}"):
+#                 rating = st.slider("Rate this response (1-5)", 1, 5, key=f"rating_{len(st.session_state.chat_history)}")
+#                 comments = st.text_area("Comments", key=f"comments_{len(st.session_state.chat_history)}")
+#                 if st.form_submit_button("Submit Feedback"):
+#                     st.session_state.feedback.append({
+#                         "rating": rating,
+#                         "comments": comments,
+#                         "response": st.session_state.chat_history[-1]["content"]
+#                     })
+#                     with open("feedback.json", "w") as f:
+#                         json.dump(st.session_state.feedback, f)
+#                     st.success("Feedback submitted!")
+    
+#     with col2:
+#         if st.session_state.get("system_initialized", False):
+#             st.subheader("Retrieved Information")
+#             info_container = st.container(height=500)
+#             with info_container:
+#                 if "current_faqs" in st.session_state:
+#                     for i, faq in enumerate(st.session_state.current_faqs):
+#                         st.markdown(f"**Relevant FAQ #{i+1}**")
+#                         st.markdown(f"**Q**: {faq['question']}")
+#                         st.markdown(f"**A**: {faq['answer'][:150]}..." if len(faq['answer']) > 150 else f"**A**: {faq['answer']}")
+#                         st.markdown(f"*Similarity Score*: {faq['similarity']:.2f}")
+#                         if 'category' in faq and faq['category']:
+#                             st.markdown(f"*Category*: {faq['category']}")
+#                         st.markdown("---")
+#                 else:
+#                     st.markdown("Ask a question to see relevant FAQs.")
+    
+#     if "retrieval_time" in st.session_state and "generation_time" in st.session_state:
+#         st.sidebar.subheader("Performance Metrics")
+#         st.sidebar.markdown(f"Retrieval time: {st.session_state.retrieval_time:.2f} seconds")
+#         st.sidebar.markdown(f"Response generation: {st.session_state.generation_time:.2f} seconds")
+#         st.sidebar.markdown(f"Total time: {st.session_state.retrieval_time + st.session_state.generation_time:.2f} seconds")
+    
+#     if submit_button and user_query:
+#         from src.data_processing import translate_faq
+#         from googletrans import Translator
+#         translator = Translator()
+#         if target_lang != "en":
+#             user_query_translated = translator.translate(user_query, dest="en").text
+#         else:
+#             user_query_translated = user_query
+        
+#         if user_query_translated in st.session_state.query_cache:
+#             response, relevant_faqs = st.session_state.query_cache[user_query_translated]
+#         else:
+#             gc.collect()
+#             if torch.cuda.is_available():
+#                 torch.cuda.empty_cache()
+            
+#             start_time = time.time()
+#             relevant_faqs = st.session_state.embedder.retrieve_relevant_faqs(user_query_translated)
+#             retrieval_time = time.time() - start_time
+            
+#             if target_lang != "en":
+#                 relevant_faqs = [translate_faq(faq, target_lang) for faq in relevant_faqs]
+            
+#             start_time = time.time()
+#             response = st.session_state.response_generator.generate_response(user_query_translated, relevant_faqs)
+#             generation_time = time.time() - start_time
+            
+#             if target_lang != "en":
+#                 response = translator.translate(response, dest=target_lang).text
+            
+#             st.session_state.query_cache[user_query_translated] = (response, relevant_faqs)
+#             st.session_state.retrieval_time = retrieval_time
+#             st.session_state.generation_time = generation_time
+#             st.session_state.current_faqs = relevant_faqs
+        
+#         st.session_state.chat_history.append({"role": "user", "content": user_query})
+#         st.session_state.chat_history.append({"role": "assistant", "content": response})
+    
+#     if st.button("Clear Chat History"):
+#         st.session_state.chat_history = []
+#         st.session_state.query_cache = {}
+#         gc.collect()
+#         if torch.cuda.is_available():
+#             torch.cuda.empty_cache()
+    
+#     if st.session_state.get("system_initialized", False):
+#         st.sidebar.subheader("Baseline Comparison")
+#         baseline_faqs = baseline_keyword_search(user_query_translated if 'user_query_translated' in locals() else "", st.session_state.embedder.faqs)
+#         st.sidebar.write(f"RAG FAQs: {[faq['question'][:50] for faq in st.session_state.get('current_faqs', [])]}")
+#         st.sidebar.write(f"Keyword FAQs: {[faq['question'][:50] for faq in baseline_faqs]}")
+    
+#     st.subheader("Sample Questions")
+#     sample_questions = [
+#         "How do I track my order?",
+#         "What should I do if my delivery is delayed?",
+#         "How do I return a product?",
+#         "Can I cancel my order after placing it?",
+#         "How quickly will my order be delivered?"
+#     ]
+#     cols = st.columns(2)
+#     for i, question in enumerate(sample_questions):
+#         col_idx = i % 2
+#         if cols[col_idx].button(question, key=f"sample_{i}"):
+#             st.session_state.user_input = question
+#             st.session_state.chat_history.append({"role": "user", "content": question})
+            
+#             from src.data_processing import translate_faq
+#             from googletrans import Translator
+#             translator = Translator()
+#             if target_lang != "en":
+#                 question_translated = translator.translate(question, dest="en").text
+#             else:
+#                 question_translated = question
+            
+#             if question_translated in st.session_state.query_cache:
+#                 response, relevant_faqs = st.session_state.query_cache[question_translated]
+#             else:
+#                 gc.collect()
+#                 if torch.cuda.is_available():
+#                     torch.cuda.empty_cache()
+                
+#                 start_time = time.time()
+#                 relevant_faqs = st.session_state.embedder.retrieve_relevant_faqs(question_translated)
+#                 retrieval_time = time.time() - start_time
+                
+#                 if target_lang != "en":
+#                     relevant_faqs = [translate_faq(faq, target_lang) for faq in relevant_faqs]
+                
+#                 start_time = time.time()
+#                 response = st.session_state.response_generator.generate_response(question_translated, relevant_faqs)
+#                 generation_time = time.time() - start_time
+                
+#                 if target_lang != "en":
+#                     response = translator.translate(response, dest=target_lang).text
+                
+#                 st.session_state.query_cache[question_translated] = (response, relevant_faqs)
+#                 st.session_state.retrieval_time = retrieval_time
+#                 st.session_state.generation_time = generation_time
+#                 st.session_state.current_faqs = relevant_faqs
+            
+#             st.session_state.chat_history.append({"role": "assistant", "content": response})
+
+# if __name__ == "__main__":
+#     main()

requirements.txt

@@ -11,28 +11,7 @@ accelerate>=0.20.0
 evaluate>=0.4.0
 scikit-learn>=1.2.0
 nlpaug>=1.1.0
-googletrans==4.0.0-rc1
-httpx==0.23.0  # Pinned to compatible version
-httpcore==0.15.0  # Pinned to compatible version
+deep-translator>=1.9.0
 psutil>=5.9.0
 nltk>=3.8.0
 
-
-
-# torch>=2.0.0
-# transformers>=4.30.0
-# sentence-transformers>=2.2.2
-# faiss-cpu>=1.7.4
-# pandas>=1.5.0
-# streamlit>=1.36.0
-# numpy>=1.24.0
-# datasets>=2.10.0
-# bitsandbytes>=0.40.0
-# accelerate>=0.20.0
-# evaluate>=0.4.0
-# scikit-learn>=1.2.0
-# nlpaug>=1.1.0
-# googletrans==4.0.0-rc1
-# psutil>=5.9.0
-# nltk>=3.8.0
-

src/data_processing.py

@@ -5,7 +5,7 @@ import nltk
 from typing import List, Dict, Any
 from datasets import load_dataset
 import nlpaug.augmenter.word as naw
-from googletrans import Translator
+from deep_translator import GoogleTranslator  # Updated import
 
 # Configure NLTK data path and download required resources
 NLTK_DATA_PATH = os.path.join(os.path.dirname(__file__), "../nltk_data")
@@ -133,15 +133,166 @@ def augment_faqs(faqs: List[Dict[str, Any]], max_faqs: int = 1000, enable_augmen
 
 def translate_faq(faq: Dict[str, Any], target_lang: str = "es") -> Dict[str, Any]:
     """
-    Translate FAQ to a target language
+    Translate FAQ to a target language using deep-translator
     """
     try:
-        translator = Translator()
+        translator = GoogleTranslator(source='en', target=target_lang)
         translated = faq.copy()
-        translated["question"] = translator.translate(faq["question"], dest=target_lang).text
-        translated["answer"] = translator.translate(faq["answer"], dest=target_lang).text
+        translated["question"] = translator.translate(faq["question"])
+        translated["answer"] = translator.translate(faq["answer"])
         translated["language"] = target_lang
         return translated
     except Exception as e:
         print(f"Translation error: {e}")
-        return faq
+        return faq
+
+
+
+
+# import pandas as pd
+# import json
+# import os
+# import nltk
+# from typing import List, Dict, Any
+# from datasets import load_dataset
+# import nlpaug.augmenter.word as naw
+# from googletrans import Translator
+
+# # Configure NLTK data path and download required resources
+# NLTK_DATA_PATH = os.path.join(os.path.dirname(__file__), "../nltk_data")
+# os.makedirs(NLTK_DATA_PATH, exist_ok=True)
+# nltk.data.path.append(NLTK_DATA_PATH)
+
+# def ensure_nltk_resources():
+#     """
+#     Ensure NLTK resources are downloaded and available
+#     """
+#     try:
+#         nltk.download('averaged_perceptron_tagger', download_dir=NLTK_DATA_PATH)
+#         nltk.download('punkt', download_dir=NLTK_DATA_PATH)
+#         print(f"NLTK resources downloaded to {NLTK_DATA_PATH}")
+#         return True
+#     except Exception as e:
+#         print(f"Failed to download NLTK resources: {e}")
+#         return False
+
+# def load_huggingface_faq_data(dataset_name: str = "NebulaByte/E-Commerce_FAQs") -> List[Dict[str, Any]]:
+#     """
+#     Load FAQ data from Hugging Face datasets, cache locally
+#     """
+#     local_path = "data/ecommerce_faqs.json"
+#     if os.path.exists(local_path):
+#         print(f"Loading cached dataset from {local_path}")
+#         with open(local_path, 'r') as f:
+#             return json.load(f)
+    
+#     print(f"Loading dataset {dataset_name} from Hugging Face...")
+#     try:
+#         dataset = load_dataset(dataset_name)
+#         faqs = [{
+#             "question": item["question"],
+#             "answer": item["answer"],
+#             "category": item.get("category", ""),
+#             "question_id": item.get("question_id", ""),
+#             "faq_url": item.get("faq_url", "")
+#         } for item in dataset["train"]]
+#         with open(local_path, 'w') as f:
+#             json.dump(faqs, f)
+#         print(f"Saved dataset to {local_path}, loaded {len(faqs)} FAQs")
+#         return faqs
+#     except Exception as e:
+#         print(f"Error loading dataset: {e}")
+#         print("Falling back to local data...")
+#         return load_faq_data("data/faq_data.csv")
+
+# def load_faq_data(file_path: str) -> List[Dict[str, Any]]:
+#     """
+#     Load FAQ data from a local CSV or JSON file
+#     """
+#     print(f"Loading data from {file_path}")
+#     try:
+#         if file_path.endswith('.csv'):
+#             df = pd.read_csv(file_path)
+#             faqs = df.to_dict('records')
+#         elif file_path.endswith('.json'):
+#             with open(file_path, 'r') as f:
+#                 faqs = json.load(f)
+#         else:
+#             raise ValueError(f"Unsupported file format: {file_path}")
+#         print(f"Loaded {len(faqs)} FAQ entries")
+#         return faqs
+#     except Exception as e:
+#         print(f"Error loading data: {e}")
+#         print("Creating sample dataset as fallback")
+#         sample_faqs = [
+#             {"question": "How do I track my order?", "answer": "You can track your order by logging into your account and visiting the Order History section."},
+#             {"question": "How do I reset my password?", "answer": "To reset your password, click on the 'Forgot Password' link on the login page."}
+#         ]
+#         return sample_faqs
+
+# def preprocess_faq(faqs: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+#     """
+#     Preprocess FAQ data: clean text, handle formatting, and filter invalid entries
+#     """
+#     processed_faqs = []
+#     for faq in faqs:
+#         # Safely handle question and answer fields
+#         question = faq.get('question')
+#         answer = faq.get('answer')
+        
+#         # Convert to string and strip, handling None values
+#         question = str(question).strip() if question is not None else ""
+#         answer = str(answer).strip() if answer is not None else ""
+        
+#         # Update FAQ dictionary
+#         faq['question'] = question
+#         faq['answer'] = answer
+        
+#         # Only include FAQs with both question and answer
+#         if question and answer:
+#             processed_faqs.append(faq)
+#         else:
+#             print(f"Skipping invalid FAQ: question='{question}', answer='{answer}'")
+    
+#     print(f"After preprocessing: {len(processed_faqs)} valid FAQ entries")
+#     return processed_faqs
+
+# def augment_faqs(faqs: List[Dict[str, Any]], max_faqs: int = 1000, enable_augmentation: bool = True) -> List[Dict[str, Any]]:
+#     """
+#     Augment FAQs with paraphrased questions if enabled
+#     """
+#     if not enable_augmentation:
+#         print("Augmentation disabled; returning original FAQs")
+#         return faqs
+    
+#     if not ensure_nltk_resources():
+#         print("NLTK resources unavailable; skipping augmentation")
+#         return faqs
+    
+#     aug = naw.SynonymAug()
+#     augmented = []
+#     for faq in faqs:
+#         augmented.append(faq)
+#         if len(augmented) < max_faqs:
+#             try:
+#                 aug_question = aug.augment(faq['question'])[0]
+#                 augmented.append({"question": aug_question, "answer": faq['answer'], "category": faq.get("category", "")})
+#             except Exception as e:
+#                 print(f"Augmentation error for question '{faq['question'][:50]}...': {e}")
+#     print(f"Augmented to {len(augmented)} FAQs")
+#     return augmented
+
+# def translate_faq(faq: Dict[str, Any], target_lang: str = "es") -> Dict[str, Any]:
+#     """
+#     Translate FAQ to a target language
+#     """
+#     try:
+#         translator = Translator()
+#         translated = faq.copy()
+#         translated["question"] = translator.translate(faq["question"], dest=target_lang).text
+#         translated["answer"] = translator.translate(faq["answer"], dest=target_lang).text
+#         translated["language"] = target_lang
+#         return translated
+#     except Exception as e:
+#         print(f"Translation error: {e}")
+#         return faq