Add new files for Veda Bot 2.0

.streamlit/secrets.toml

@@ -0,0 +1,14 @@
+OPENAI_API_KEY="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"
+PINECONE_API_KEY_SAM = "a8f6c5af-e23c-4d57-8038-e47a3c0b3a42"
+#OPENAI_ORGANIZATION=YOUR_ORG_ID_HERE
+LANGCHAIN_API_KEY="ls__3dd443fbc5e5487b91b6d169692925c9"
+GROQ_API_KEY='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'
+NOMIC_API_KEY="nk-TsmqZHH1Y-rwWOuLgoUtJ7iihIljDeJTOIEalgnODhM"
+TAVILY_API_KEY="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"
+MIXED_API_KEY="emb_32ce4f9ad64a153cff0b41f865daf46ad87d3222b7084b6e"
+#database details
+DB_HOST="dev-veda-cms-database-1.cjopjxk5nm26.us-west-2.rds.amazonaws.com"
+DB_USER="veda_bot_user"
+DB_PASSWORD="Vdea#bp0t"
+DB_PORT=3306
+DB="veda_data_bot"

Docs/ramana_docs_ids.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b132a67107aed2df316c947a72bda5bcad9eae4917d0a062e82e00733eba31ed
+size 13480212

cache.py

@@ -0,0 +1,75 @@
+from langchain_openai import OpenAI
+import os
+import time
+from langchain.cache import SQLAlchemyCache
+from sqlalchemy import Column, Computed, Index, Integer, Sequence, String, create_engine
+from sqlalchemy.orm import sessionmaker, declarative_base
+from datetime import datetime
+
+
+
+
+#load postgres engine
+engine = create_engine("postgresql://postgres:sampath@localhost:5432/postgres")
+
+
+
+
+Base = declarative_base()
+
+
+class FeedBackCache(Base): 
+    """Postgres table for fulltext-indexed LLM Cache"""
+
+    __tablename__ = "veda_bot_feedback"
+    id = Column(Integer, Sequence("cache_id"), primary_key=True)
+    user_message = Column(String, nullable=True)
+    assistant_message = Column(String, nullable=True)
+    feedback_score = Column(String, nullable=True)
+    feedback_text = Column(String, nullable=True)
+    
+# Create the table in the database
+Base.metadata.create_all(engine)
+
+
+def write_to_db(u_message, a_message, f_score, f_text):
+    try:
+        # Create a sessionmaker bound to the engine
+        Session = sessionmaker(bind=engine)
+
+        # Create a session
+        session = Session()
+     
+        message = FeedBackCache(
+            user_message=u_message["content"],
+            assistant_message=a_message["content"],
+            feedback_score=f_score,
+            feedback_text=f_text
+            )
+        
+        # Add the instance to the session
+        session.add(message)
+
+        # Commit the session to persist the changes to the database
+        session.commit()
+        print("Feedback written to DB successfully!")
+
+    except Exception as e:
+        # If an error occurs, rollback the session and print the error message
+        session.rollback()
+        print("Error occurred while writing feedback to DB:", e)
+
+    finally:
+        # Close the session
+        session.close()
+
+    
+def current_time() -> str:
+    """Return the current time as a string. Used as part of the session UUID."""
+    # Get current date and time
+    current_datetime = datetime.now()
+
+    # Convert to a long number format
+    datetime_string = current_datetime.strftime("%Y%m%d%H%M%S")
+
+    return datetime_string

crag.py

@@ -0,0 +1,357 @@
+import os
+import streamlit as st
+from vector_tool import ensemble_retriever
+from langgraph.prebuilt import ToolInvocation
+from langchain_core.messages import ToolMessage
+import json
+# Set up the tools to execute them from the graph
+from langgraph.prebuilt import ToolExecutor
+# tools retrieval
+from function_tools import tool_chain
+from vector_tool import ensemble_retriever
+
+os.environ['OPENAI_API_KEY'] = st.secrets["OPENAI_API_KEY"]
+os.environ['TAVILY_API_KEY'] = st.secrets["TAVILY_API_KEY"]
+
+### Retrieval Grader 
+
+from langchain_openai import ChatOpenAI
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.pydantic_v1 import BaseModel, Field
+
+#LLM models
+llm_AI4 = ChatOpenAI(model="gpt-4-1106-preview", temperature=0)
+
+# Data model
+class GradeDocuments(BaseModel):
+    """Binary score for relevance check on retrieved documents."""
+
+    binary_score: str = Field(description="Documents are relevant to the question, 'yes' or 'no'")
+
+# LLM with function call 
+structured_llm_grader = llm_AI4.with_structured_output(GradeDocuments)
+
+# Prompt 
+system = """You are a grader assessing relevance of a retrieved document to a user question. \n 
+    If the document contains keyword(s) or semantic meaning related to the question, grade it as relevant. \n
+    Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question."""
+grade_prompt = ChatPromptTemplate.from_messages(
+    [
+        ("system", system),
+        ("human", "Retrieved document: \n\n {document} \n\n User question: {question}"),
+    ]
+)
+
+retrieval_grader = grade_prompt | structured_llm_grader
+
+### Generate
+from langchain import hub
+from langchain.prompts import MessagesPlaceholder
+from langchain.agents.output_parsers.openai_tools import OpenAIToolsAgentOutputParser
+from langchain.prompts import MessagesPlaceholder
+from langchain.agents.format_scratchpad.openai_tools import (
+    format_to_openai_tool_messages
+    )
+from langchain_core.messages import AIMessage, FunctionMessage, HumanMessage
+from langchain_core.output_parsers import StrOutputParser
+from typing import Any, List, Union
+# Prompt
+#prompt = hub.pull("rlm/rag-prompt")
+system_message = '''You are an AI assistant for answering questions about vedas and scriptures.
+                    \nYou are given the following extracted documents from Svarupa Knowledge Base (https://svarupa.org/) and other documents and a question. 
+                    Provide a conversational answer.\nIf you are not provided with any documents, say \"I did not get any relevant context for this but 
+                    I will reply to the best of my knowledge\" and then write your answer\nIf you don't know the answer, just say \"Hmm, I'm not sure. \" Don't try to make up an answer.
+                    \nIf the question is not about vedas and scriptures, politely inform them that you are tuned to only answer questions about that.\n\n'''
+'''
+prompt = ChatPromptTemplate.from_messages(
+    [
+        ("system",system_message),
+        # Please note the ordering of the fields in the prompt!
+        # The correct ordering is:
+        # 1. history - the past messages between the user and the agent
+        # 2. user - the user's current input
+        # 3. agent_scratchpad - the agent's working space for thinking and
+        #    invoking tools to respond to the user's input.
+        # If you change the ordering, the agent will not work correctly since
+        # the messages will be shown to the underlying LLM in the wrong order.
+        MessagesPlaceholder(variable_name="context"),
+        ("user", "{question}"),
+    ]
+)
+'''
+generate_prompt = ChatPromptTemplate.from_messages(
+    [
+        ("system", system_message),
+        ("human", "Here is the given context {context}, queation: {question} \n\n Formulate an answer."),
+    ]
+)
+# LLM
+llm_AI = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)
+
+# Post-processing
+def format_docs(docs):
+    return "\n\n".join(doc.page_content for doc in docs)
+
+# Chain
+rag_chain =  generate_prompt | llm_AI4 | StrOutputParser() #OpenAIToolsAgentOutputParser()
+
+####-----------------TESTING
+prompt = ChatPromptTemplate.from_messages(
+    [
+        (
+            "system",
+            "You are a helpful assistant. Answer all questions to the best of your ability.",
+        ),
+        MessagesPlaceholder(variable_name="chat_history"),
+        ("human", "{question}"),
+    ]
+)
+from langchain_core.runnables.history import RunnableWithMessageHistory
+from langchain.memory import ChatMessageHistory
+
+chat_history_for_chain = ChatMessageHistory()
+
+chain_with_message_history = RunnableWithMessageHistory(
+    rag_chain,
+    lambda session_id: chat_history_for_chain,
+    input_messages_key="question",
+    history_messages_key="chat_history",
+)
+
+### Question Re-writer
+
+# LLM 
+llm = ChatOpenAI(model="gpt-3.5-turbo-0125", temperature=0)
+
+# Prompt 
+system = """You a question re-writer that converts an input question to a better version that is optimized \n 
+     for a search. Look at the input and try to reason about the underlying sematic intent / meaning."""
+re_write_prompt = ChatPromptTemplate.from_messages(
+    [
+        ("system", system),
+        ("human", "Here is the initial question: \n\n {question} \n Formulate an improved question."),
+    ]
+)
+
+question_rewriter = re_write_prompt | llm | StrOutputParser()
+
+
+
+### Search
+
+from langchain_community.tools.tavily_search import TavilySearchResults
+web_search_tool = TavilySearchResults(k=2)
+
+from typing_extensions import TypedDict
+from typing import List
+from typing import TypedDict, Annotated, Sequence
+import operator
+from langchain_core.messages import BaseMessage
+
+class GraphState(TypedDict):
+    """
+    Represents the state of our graph.
+
+    Attributes:
+        question: question
+        generation: LLM generation
+        web_search: whether to add search
+        documents: list of documents 
+    """
+    question : str
+    generation : str
+    web_search : str
+    messages: List[str] #Union[dict[str, Any]] 
+
+from langchain.schema import Document
+
+
+
+def retrieve(state):
+    """
+    Retrieve documents
+
+    Args:
+        state (dict): The current graph state
+
+    Returns:
+        state (dict): New key added to state, documents, that contains retrieved documents
+    """
+    print("---VECTOR RETRIEVE---")
+    question = state["question"]
+    # Retrieval
+    documents = ensemble_retriever.get_relevant_documents(question)
+    #print(documents)
+    # Iterate over each document and update the 'metadata' field with the file name
+    for doc in documents:
+        try:
+            file_path = doc.metadata['source']
+            #print(file_path)
+            file_name = os.path.split(file_path)[1]  # Get the file name from the file path
+            doc.metadata['source'] = file_name
+        except KeyError:
+            # Handle the case where 'source' field is missing in the metadata
+            doc.metadata['source'] = 'unavailable'
+        except Exception as e:
+            # Handle any other exceptions that may occur
+            print(f"An error occurred while processing document: {e}")
+    return {"messages": documents, "question": question}
+
+
+def generate(state):
+    """
+    Generate answer
+
+    Args:
+        state (dict): The current graph state
+
+    Returns:
+        state (dict): New key added to state, generation, that contains LLM generation
+    """
+    print("---GENERATE---")
+    question = state["question"]
+    messages = state["messages"]
+    print(messages)
+    # RAG generation
+    generation = chain_with_message_history.invoke({"context": messages, "question": question},{"configurable": {"session_id": "unused"}})
+    return {"messages": messages, "question": question, "generation": generation}
+
+def grade_documents(state):
+    """
+    Determines whether the retrieved documents are relevant to the question.
+
+    Args:
+        state (dict): The current graph state
+
+    Returns:
+        state (dict): Updates documents key with only filtered relevant documents
+    """
+
+    print("---CHECK DOCUMENT RELEVANCE TO QUESTION---")
+    question = state["question"]
+    messages = state["messages"]
+    
+    # Score each doc
+    filtered_docs = []
+    web_search = "No"
+    for d in messages:
+        score = retrieval_grader.invoke({"question": question, "document": d.page_content})
+        grade = score.binary_score
+        if grade == "yes":
+            print("---GRADE: DOCUMENT RELEVANT---")
+            filtered_docs.append(d)
+        else:
+            print("---GRADE: DOCUMENT NOT RELEVANT---")
+            continue
+    print("---TOOLS RETRIEVE---")
+    tool_documents = tool_chain.invoke(question)
+    #print(tool_documents)
+    if tool_documents:
+        for item in tool_documents:
+            filtered_docs.append(Document(page_content=str(item['output']),metadata={"source": 'https://svarupa.org/home',"name":item['name']}))        
+    # If filtered_docs is empty, perform a web search
+    if not filtered_docs:
+        print("--PERFORMING WEB SEARCH--")
+        web_search = "Yes"
+
+    return {"messages": filtered_docs, "question": question, "web_search": web_search}
+
+
+
+def transform_query(state):
+    """
+    Transform the query to produce a better question.
+
+    Args:
+        state (dict): The current graph state
+
+    Returns:
+        state (dict): Updates question key with a re-phrased question
+    """
+
+    print("---TRANSFORM QUERY---")
+    question = state["question"]
+    messages = state["messages"]
+
+    # Re-write question
+    better_question = question_rewriter.invoke({"question": question})
+    return {"messages": messages, "question": better_question}
+    
+def web_search(state):
+    """
+    Web search based on the re-phrased question.
+
+    Args:
+        state (dict): The current graph state
+
+    Returns:
+        state (dict): Updates documents key with appended web results
+    """
+
+    print("---WEB SEARCH---")
+    question = state["question"]
+    messages = state["messages"]
+
+    # Web search
+    docs = web_search_tool.invoke({"query": question})
+    #web_results = "\n".join([d["content"] for d in docs])
+    web_results = [Document(page_content=d["content"], metadata={"source": d["url"]}) for d in docs]
+    print(f"Web Results: {web_results}")
+    messages.extend(web_results)
+    return {"messages": messages, "question": question}
+
+### Edges
+
+def decide_to_generate(state):
+    """
+    Determines whether to generate an answer, or re-generate a question.
+
+    Args:
+        state (dict): The current graph state
+
+    Returns:
+        str: Binary decision for next node to call
+    """
+
+    print("---ASSESS GRADED DOCUMENTS---")
+    question = state["question"]
+    web_search = state["web_search"]
+    filtered_documents = state["messages"]
+
+    if web_search == "Yes":
+        # All documents have been filtered check_relevance
+        # We will re-generate a new query
+        print("---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---")
+        return "transform_query"
+    else:
+        # We have relevant documents, so generate answer
+        print("---DECISION: GENERATE---")
+        return "generate"
+from langgraph.graph import END, StateGraph
+
+workflow = StateGraph(GraphState)
+
+# Define the nodes
+workflow.add_node("retrieve", retrieve)  # retrieve
+workflow.add_node("grade_documents", grade_documents)  # grade documents
+workflow.add_node("generate", generate)  # generatae
+workflow.add_node("transform_query", transform_query)  # transform_query
+workflow.add_node("web_search_node", web_search)  # web search
+
+# Build graph
+workflow.set_entry_point("retrieve")
+workflow.add_edge("retrieve", "grade_documents")
+workflow.add_conditional_edges(
+    "grade_documents",
+    decide_to_generate,
+    {
+        "transform_query": "transform_query",
+        "generate": "generate",
+    },
+)
+workflow.add_edge("transform_query", "web_search_node")
+workflow.add_edge("web_search_node", "generate")
+workflow.add_edge("generate", END)
+
+# Compile
+crag_app = workflow.compile()

crag_app.py

@@ -0,0 +1,158 @@
+import json
+import os
+import uuid
+from streamlit_feedback import streamlit_feedback
+import streamlit as st
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage
+from langchain.memory import ChatMessageHistory
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
+from langchain_core.prompts import HumanMessagePromptTemplate
+import tiktoken
+#from agent import app
+from crag import crag_app
+from datetime import timedelta
+from sqlalchemy import create_engine
+from cache import (write_to_db,
+                    current_time)
+
+
+#load postgres engine
+engine = create_engine("postgresql://postgres:sampath@localhost:5432/postgres")
+#load keys
+os.environ['OPENAI_API_KEY'] = st.secrets["OPENAI_API_KEY"]
+chat_history = ChatMessageHistory()
+
+system_message = '''You are an AI assistant for answering questions about vedas and scriptures.
+                    \nYou are given the following extracted documents from Svarupa Knowledge Base (https://svarupa.org/) and other documents and a question. 
+                    Provide a conversational answer. If there are any unicode characters in the final answer, please encode and provide readable answer to the user.
+                    \nIf you are not provided with any documents, say \"I did not get any relevant context for this but 
+                    I will reply to the best of my knowledge\" and then write your answer\nIf you don't know the answer, just say \"Hmm, I'm not sure. \" Don't try to make up an answer.
+                    \nIf the question is not about vedas and scriptures, politely inform them that you are tuned to only answer questions about that.\n\n'''
+generate_prompt = ChatPromptTemplate.from_messages(
+    [
+        ("system", system_message),
+        ("human", "Here is the given context {context}, queation: {question} \n\n Formulate an answer."),
+    ]
+)
+#@st.cache_resource(show_spinner=False)  # Set allow_output_mutation to True for mutable objects like instances
+def bot_response(user_input):
+    response = crag_app.invoke({"question": user_input})
+    return response
+
+
+##======
+# Main chatbot function
+def veda_bot(sidebar: bool = True) -> None:
+    # Define custom CSS
+    custom_css = """
+        <style>
+            /* Adjust the selector as needed */
+            .stHeadingContainer {
+                margin-top: -100px; /* Reduce the top margin */
+            }
+            #MainMenu {visibility: hidden;}
+            footer {visibility: hidden;}
+            header {visibility: hidden;}
+        </style>
+    """
+
+    # Apply the custom CSS
+    st.markdown(custom_css, unsafe_allow_html=True)
+
+    # Streamlit Components Initialization
+    st.title("Veda Bot")
+    st.write("This bot is developed based on the content from the [Svarupa](https://svarupa.org/home) website.")
+    chat_history.add_message(SystemMessage(content="Welcome! I am your Veda Bot. How can I assist you today?"))
+    # Initialize session state variables
+    if "messages" not in st.session_state.keys():
+        st.session_state.messages = [{"role": "assistant", "content": "Hi. I am an AI Assistant. Ask me a question about Vedas!"}]
+
+    if "session_uuid" not in st.session_state:
+        st.session_state["session_uuid"] = f"{current_time()}-{str(uuid.uuid4())}"
+
+    if "feedback" not in st.session_state:
+        st.session_state["feedback"] = None
+
+    if "chat_engine" not in st.session_state.keys():
+        st.session_state.chat_engine = bot_response
+
+    if "memory" not in st.session_state:
+        st.session_state["memory"] = ChatMessageHistory()
+        st.session_state["memory"].add_message(generate_prompt)
+        st.session_state["memory"].add_message({"role":"user","content":"Hi/Hello or Any Greating"})
+        st.session_state["memory"].add_message({"role":"assistant","content":"Hi. Please ask the question about vedas!"})
+    # Display chat history
+    for message in st.session_state.messages:
+        with st.chat_message(message["role"]):
+            st.markdown(message["content"])
+
+    # Get user input
+    prompt = st.chat_input("Enter your question!")
+
+    if prompt:
+        # Display user message in chat message container
+        with st.chat_message("user"):
+            st.markdown(prompt)
+
+        # Log user message
+        st.session_state["messages"].append({"role": "user", "content": prompt})
+        st.session_state["memory"].add_message({"role": "user", "content": prompt})
+
+    # Generate bot response
+    if st.session_state.messages[-1]["role"] != "assistant":
+        with st.spinner("Thinking..."):
+            references = []
+            message_placeholder = st.empty()
+            full_response = ""
+            
+            # Get bot response
+            response_bot = st.session_state.chat_engine(prompt)
+            generation = response_bot['generation']
+            full_response += generation
+            web_search = response_bot['web_search']
+
+            # Extract references from bot response
+            if response_bot['messages']:
+                try:
+                    references.extend([doc.metadata['source'] for doc in response_bot['messages']])
+                except Exception as e:
+                    print("Error:", e)
+            #message_placeholder.markdown(full_response + "▌")
+
+            # Add references to the full response
+            if references:
+                unique_references = set(references)
+                full_response += "\n\n**References:**\n\n"
+                for reference in unique_references:
+                    full_response += f"- {reference}\n"
+
+            #message_placeholder.markdown(full_response + "▌")
+            # Submit Feedback
+            streamlit_feedback(
+                feedback_type="faces",
+                on_submit=None,
+                optional_text_label="[Optional] Please provide an explanation",
+                key="feedback",	
+            )
+            message_placeholder.markdown(full_response)
+        st.session_state["messages"].append({"role": "assistant", "content": generation})
+        st.session_state["memory"].add_message({"role": "assistant", "content": generation})
+        print(f"Response added to memory: {full_response}")
+
+    # Log feedback and messages
+    if st.session_state['feedback']:
+        user_feedback ={
+            "user_message": st.session_state["messages"][-2],
+            "assistant_message": st.session_state["messages"][-1],
+            "feedback_score": st.session_state["feedback"]["score"],
+            "feedback_text": st.session_state["feedback"]["text"],
+        }
+        #write_to_db(u_message=user_feedback["user_message"],
+        #            a_message=user_feedback["assistant_message"],
+        #            f_score=user_feedback["feedback_score"],
+        #            f_text=user_feedback["feedback_text"])
+
+if __name__ == "__main__":
+    veda_bot()

database.py

@@ -0,0 +1,87 @@
+import pymysql
+import streamlit as st
+import logging
+import json
+import pandas as pd
+import re
+import os
+from langchain_community.utilities.sql_database import SQLDatabase
+
+
+db_params = {"host": st.secrets("DB_HOST"),
+                "user": os.getenv("DB_USER"),
+                "password": os.getenv("DB_PASSWORD"),
+                "port": 3306,
+                "database":os.getenv("DB")
+                }
+
+
+
+def initialize_database():
+    try:
+        # Database Connection
+        db = pymysql.connect(**db_params)
+        st.success("Database connection successful!")
+        return db
+    except Exception as e:
+        st.error(f"Database connection failed: {e}")
+        return None
+
+
+def get_db():
+    try:
+        db = SQLDatabase.from_uri(
+            f"mysql+pymysql://{db_params['user']}:{db_params['password']}@{db_params['host']}/{db_params['database']}",
+            include_tables=['term_details_modified', 'veda_content_details', 'veda_content_modified']
+        )
+        #st.success("Database connection successful!")
+        return db
+    except Exception as e:
+        st.error(f"Database connection failed: {e}")
+        return None
+
+
+def execute_query(query):
+    db = initialize_database()
+    cursor = db.cursor()
+    try:
+        cursor.execute(query)
+        description = cursor.description
+        result = cursor.fetchall()  # Fetch all rows from the result set
+        db.commit()
+        return description, result
+    except Exception as e:
+        print("Error executing query:", e)
+        db.rollback()
+        return None  # Return None if an error occurs
+    finally:
+        db.close()
+
+
+def execute_sql_query(query, parameters=None):
+    # Establish database connection and execute SQL query
+    db = initialize_database()
+    cursor = db.cursor(pymysql.cursors.DictCursor)  # Use dictionary cursor to retrieve data as dictionaries
+    try:
+        if parameters:
+            cursor.execute(query, parameters)
+        else:
+            cursor.execute(query)
+        results = cursor.fetchall()
+        return results
+    except Exception as e:
+        logging.error(f"Error executing SQL query: {e}")
+        return None
+    finally:
+        db.close()
+
+
+def get_details_mantra_json(query):
+    description, data = execute_query(query)
+    df = pd.DataFrame(data)
+    df.columns = [x[0] for x in description]
+    mantra_json = df['mantra_json'].values[0]
+    cleaned_data = re.sub('<[^<]+?>', '', mantra_json)
+    return json.loads(cleaned_data)
+
+

function_tools.py

@@ -0,0 +1,614 @@
+from typing import Optional
+import json
+import logging
+from utils import iast_process, get_list_meaning_word, get_details_mantra_json, word_sentence_similarity, extract_meaning_by_language
+import ast
+from langchain_core.tools import tool 
+from database import execute_sql_query, get_details_mantra_json
+from langchain.pydantic_v1 import BaseModel, Field
+from langchain.tools import StructuredTool
+from typing import Optional
+import streamlit as st
+from langchain_core.utils.function_calling import convert_to_openai_function
+from langchain_core.messages import AIMessage
+from langchain_core.runnables import Runnable
+import os
+
+os.environ['OPENAI_API_KEY'] = st.secrets["OPENAI_API_KEY"]
+from langchain_openai import ChatOpenAI
+
+#LLM
+llm = ChatOpenAI(model="gpt-3.5-turbo-0125")
+llm_AI4 = ChatOpenAI(model="gpt-4-1106-preview", temperature=0)
+
+
+
+#Classes
+class MantraInput(BaseModel):
+    mantraid: Optional[str] = Field(None, description="The mantra id. For example, 1.1.1.1, 2.1.1,3.1.1.2,4.2.3.1, and 5.0.1.1.2")
+    scripture_name: Optional[str] = Field(None, description="Name of the scripture like RigVeda, SamaVeda, AtharvaVeda, KrishnaYajurVeda, and ShuklaYajurVeda")
+    KandahNumber: Optional[int] = Field(None, description="Kandah Number of Vedamantra")
+    MandalaNumber: Optional[int] = Field(None, description="Mandala Number of Vedamantra")
+    ArchikahNumber: Optional[int] = Field(None, description="Archikah Number of Vedamantra")
+    ShuktaNumber: Optional[int] = Field(None, description="Shukta Number of Vedamantra")
+    PrapatakNumber: Optional[int] = Field(None, description="Prapatak Number of Vedamantra")
+    MantraNumber: Optional[int] = Field(None, description="Mantra Number of Vedamantra")    
+    AnuvakNumber: Optional[int] = Field(None, description="Anuvak Number of Vedamantra")
+    AdhyayaNumber: Optional[int] = Field(None, description="Adhyaya Number of Vedamantra")
+
+class PadaMeaningInput(BaseModel):
+    pada: str = Field(description="The pada or word that is being meaning checked")
+
+class PadaAAAInput(BaseModel):
+    pada: str = Field(description="The pada or word that is being meaning checked")
+    mantraid: Optional[str] = Field(None, description="The mantra id. For example, 1.1.1.1, 2.1.1,3.1.1.2,4.2.3.1, and 5.0.1.1.2")
+    scripture_name: Optional[str] = Field(None, description="Name of the scripture like RigVeda, SamaVeda, AtharvaVeda, KrishnaYajurVeda, and ShuklaYajurVeda")
+    KandahNumber: Optional[int] = Field(None, description="Kandah Number of Vedamantra")
+    MandalaNumber: Optional[int] = Field(None, description="Mandala Number of Vedamantra")
+    ArchikahNumber: Optional[int] = Field(None, description="Archikah Number of Vedamantra")
+    ShuktaNumber: Optional[int] = Field(None, description="Shukta Number of Vedamantra")
+    PrapatakNumber: Optional[int] = Field(None, description="Prapatak Number of Vedamantra")
+    MantraNumber: Optional[int] = Field(None, description="Mantra Number of Vedamantra")    
+    AnuvakNumber: Optional[int] = Field(None, description="Anuvak Number of Vedamantra")
+    AdhyayaNumber: Optional[int] = Field(None, description="Adhyaya Number of Vedamantra")
+
+class NLSQLResponse(BaseModel):
+    user_query: str = Field(description="user query")
+
+class VectorResponse(BaseModel):
+    query: str = Field(description="User query")
+
+class Response(BaseModel):
+    result: str = Field(description="The result based on the context. Provide the text in a readable format if there are unicode characters. Use only available context. If there is no context, return as 'unknown'. Do not use prior knowledge.")
+    explanation: str = Field(description="Explanation of the steps taken to get the result")
+
+#function tools for mantra level
+def _get_mantra_details(query):
+    try:
+        details = get_details_mantra_json(query)
+        return details['mantraHeader']['language'][1]
+    except Exception as e:
+        raise ValueError(f"Failed to get mantra details: {e}")
+
+
+def _get_mantra_details_by_scripture(scripture_name=None, KandahNumber=None, MandalaNumber=None, ArchikahNumber=None,
+                                      ShuktaNumber=None, PrapatakNumber=None, MantraNumber=None, AnuvakNumber=None,
+                                      AdhyayaNumber=None):
+    try:
+        # Construct the base SQL query
+        query = "SELECT * FROM veda_content_details WHERE 1 = 1"
+        parameters = []
+
+        # Add conditions based on provided parameters
+        if scripture_name:
+            query += " AND scripture_name = %s"
+            parameters.append(scripture_name.lower())
+        if KandahNumber:
+            query += " AND KandahNumber = %s"
+            parameters.append(KandahNumber)
+        if MandalaNumber:
+            query += " AND MandalaNumber = %s"
+            parameters.append(MandalaNumber)
+        if ArchikahNumber:
+            query += " AND ArchikahNumber = %s"
+            parameters.append(ArchikahNumber)
+        if ShuktaNumber:
+            query += " AND ShuktaNumber = %s"
+            parameters.append(ShuktaNumber)
+        if PrapatakNumber:
+            query += " AND PrapatakNumber = %s"
+            parameters.append(PrapatakNumber)
+        if MantraNumber:
+            query += " AND MantraNumber = %s"
+            parameters.append(MantraNumber)
+        if AnuvakNumber:
+            query += " AND AnuvakNumber = %s"
+            parameters.append(AnuvakNumber)
+        if AdhyayaNumber:
+            query += " AND AdhyayaNumber = %s"
+            parameters.append(AdhyayaNumber)
+
+        # Execute the SQL query
+        results = execute_sql_query(query, parameters)
+
+        if results:
+            return results
+        else:
+            return None
+    except Exception as e:
+        logging.error(f"Error in _get_mantra_details_by_scripture: {e}")
+
+
+
+def get_vedamantra_details(mantraid=None, scripture_name=None, KandahNumber=None,MandalaNumber=None, ArchikahNumber=None, 
+                                                        ShuktaNumber=None, PrapatakNumber=None, MantraNumber=None,
+                                                        AnuvakNumber=None, AdhyayaNumber=None):
+    '''
+    This function is used to get the vedamantra such as vedamantra, padapatha, devata, chandah, 
+    and rishi, from all Vedas (RigVeda, AtharvaVeda, SamaVeda, KrishnaYajurVeda, and ShuklaYajurVeda).
+    The Vedic scriptures has the following structure: \
+    RigVeda->Mandala->Shukta->Mantra\
+    SamaVeda->Archikah->Shukta->Mantra\
+    AtharvaVeda->Kandah->Shukta->Mantra\
+    ShuklaYajurVeda->Adhyaya->Mantra\
+    KrishnaYajurVeda->Kandah->Prapatak->Anuvak->Mantra\
+    Sample Questions:
+    1. Obtain the vedamantra of the mantra whose id is 1.1.1.1?
+    2. Retrieve the devata of the vedamantra from Rigveda, first mandala, first shukta, and first mantra.
+    3. Provide the meaning of the vedamantra from Rigveda, first mandala, first shukta, and first mantra written by Tulsi Ram.
+    4. Explain the adhibautic meaning of the first mantra from RigVeda, first mandala, and first shukta.
+    5. Identify the mantraVishaya of the vedamantra from RigVeda, first mandala, first shukta, and first mantra.
+    6. What is the adibhautic meaning of the mantra 1.1.1.9?
+    7. What is the adhyatmic meaning of the mantra 1.1.1.7?
+    8. What is the adhidyvic meaning of the 6th mantra from RigVeda, first mandala, and first shukta?
+    '''
+    try:
+        query = ""  # Initialize query outside of the if-else block
+        if mantraid:
+            query = f'''SELECT mantra_json FROM veda_content WHERE mantra_number = "{mantraid}"
+            '''
+        else:
+            filter_df = _get_mantra_details_by_scripture(scripture_name=scripture_name, KandahNumber=KandahNumber,MandalaNumber=MandalaNumber, ArchikahNumber=ArchikahNumber, 
+                                                        ShuktaNumber=ShuktaNumber, PrapatakNumber=PrapatakNumber, MantraNumber=MantraNumber,
+                                                        AnuvakNumber=AnuvakNumber, AdhyayaNumber=AdhyayaNumber)
+            if filter_df is not None:
+                mantra_id = filter_df[0]['mantra_id']
+                query = f'''SELECT mantra_json FROM veda_content WHERE mantra_number = "{mantra_id}"
+                '''
+        return _get_mantra_details(query)
+    except Exception as e:
+        return json.dumps({"error": str(e)})
+
+def get_vedamantra_summary(mantraid=None, scripture_name=None, KandahNumber=None,MandalaNumber=None, ArchikahNumber=None, 
+                                                        ShuktaNumber=None, PrapatakNumber=None, MantraNumber=None,
+                                                        AnuvakNumber=None, AdhyayaNumber=None):
+    '''
+    Use the function `get_vedamantra_summary` to access the information such as adibhautic meaning of the mantra, anvaya of the mantra,  mantraVishaya of the mantra, 
+    adhibautic (or adhyatmic or adhidyvic) meaning (or bhavarth) of the mantra, purpose of the mantra, usage of the mantra, and tippani of the mantra.
+    Sample Query:
+    1. Obtain the anvaya of the mantra whose id (mantraid) is 1.1.1.1?
+    2. Retrieve tha adibhautic meaning of the first mantra from RigVeda, first mandala, and first shukta.
+    '''
+    try:
+        if mantraid:
+            query = f"SELECT mantra_json FROM veda_content WHERE mantra_number = '{mantraid}'"
+        else:
+            filtered_df = _get_mantra_details_by_scripture(scripture_name=scripture_name, KandahNumber=KandahNumber,MandalaNumber=MandalaNumber, ArchikahNumber=ArchikahNumber, 
+                                                        ShuktaNumber=ShuktaNumber, PrapatakNumber=PrapatakNumber, MantraNumber=MantraNumber,
+                                                        AnuvakNumber=AnuvakNumber, AdhyayaNumber=AdhyayaNumber)   
+            if filtered_df is not None:
+                mantra_id = filtered_df[0]['mantra_id']
+                query = f"SELECT mantra_json FROM veda_content WHERE mantra_number = '{mantra_id}'"
+            else:
+                return None      
+        json_dict = get_details_mantra_json(query)
+        mantra_summary = json_dict['mantraSummary']['language']
+        summary_dict = {"Roman-IAST summary of vedamantra": json_dict['mantraSummary']['language'][1]}
+        for item in mantra_summary:
+            if item['languageName'] == 'English':
+                mahatma = item['mahatma']['mahatmaName']
+                summary_dict[f"English summary of vedamantra by {mahatma}"] = item
+        return summary_dict
+    except Exception as e:
+        return {"error": str(e)}
+
+
+
+def get_pada_meaning(pada):
+    '''
+    Purpose: For given sanskrit word, you have collection of meanings for available roots and stems of it.\
+    You need to process this information as context and provide possible meanings for given word.
+    Sample query:
+    1. What is the meaning of the word apratidhṛṣṭa-śavasam?
+    '''	
+    #pada=iast_process(pada)
+    try:
+        query = f'''
+        SELECT * FROM term_details_modified WHERE Pada = "{pada}"
+        '''
+            # Execute the query to get details from the database
+        details = execute_sql_query(query)
+        #print(details)
+        pada_details = details[0]
+        #print(pada_details['Morphology'])
+        meanings_list = []
+        for morphs in ast.literal_eval(pada_details['Morphology']):
+            for field in ['stem', 'root']:
+                word = morphs.get(field)
+                if word:
+                    meanings_list.append(get_list_meaning_word(word))
+        return meanings_list
+    except Exception as e:
+        logging.error(f"Error in get_pada_meaning: {e}")
+        return {"error": f"Required meaning associated with pada is not available. {e}"} 
+    
+
+
+def _get_pada_details_by_scripture(pada, scripture_name=None, KandahNumber=None, MandalaNumber=None, ArchikahNumber=None,
+                                   ShuktaNumber=None, PrapatakNumber=None, MantraNumber=None, AnuvakNumber=None,
+                                   AdhyayaNumber=None):
+    try:
+        # Construct the base SQL query
+        query = "SELECT * FROM term_details_modified WHERE Pada = %s"
+        parameters = [pada]
+
+        # Add conditions based on provided parameters
+        if scripture_name:
+            query += " AND scripture_name = %s"
+            parameters.append(scripture_name)
+        if KandahNumber:
+            query += " AND KandahNumber = %s"
+            parameters.append(KandahNumber)
+        if MandalaNumber:
+            query += " AND MandalaNumber = %s"
+            parameters.append(MandalaNumber)
+        if ArchikahNumber:
+            query += " AND ArchikahNumber = %s"
+            parameters.append(ArchikahNumber)
+        if ShuktaNumber:
+            query += " AND ShuktaNumber = %s"
+            parameters.append(ShuktaNumber)
+        if PrapatakNumber:
+            query += " AND PrapatakNumber = %s"
+            parameters.append(PrapatakNumber)
+        if MantraNumber:
+            query += " AND MantraNumber = %s"
+            parameters.append(MantraNumber)
+        if AnuvakNumber:
+            query += " AND AnuvakNumber = %s"
+            parameters.append(AnuvakNumber)
+        if AdhyayaNumber:
+            query += " AND AdhyayaNumber = %s"
+            parameters.append(AdhyayaNumber)
+
+        # Execute the SQL query
+        results = execute_sql_query(query, parameters)
+
+        if results:
+            return results
+        else:
+            return None
+
+    except Exception as e:
+        logging.error(f"Error in _get_pada_details_by_scripture: {e}")
+        return None
+    
+def _get_vedamantra_meaning(mantraID, MahatmaName=None):
+    try:
+        query = f"SELECT mantra_json FROM veda_content WHERE mantra_number = '{mantraID}'"
+        jsonDict = get_details_mantra_json(query)
+        mantraSummary = jsonDict['mantraSummary']['language']
+        if MahatmaName is not None:
+            filtered_summary = [data_dict for data_dict in mantraSummary if data_dict.get('mahatma', {}).get('mahatmaName') == MahatmaName]
+            if filtered_summary:
+                mantraSummary = filtered_summary
+        best_meaning = None
+        best_count = 0
+        for data_dict in mantraSummary:
+            if data_dict.get('languageName') == "English":
+                meanings = data_dict['mahatma']['bhavartha']
+                count = sum(bool(meanings.get(cat, None)) for cat in ['adibhautic', 'adidaivic', 'adhyatmic'])
+                if count >= best_count:
+                    best_meaning = {cat: meanings.get(cat, None) for cat in ['adibhautic', 'adidaivic', 'adhyatmic']}
+                    best_count = count
+        return best_meaning if best_meaning else json.dumps({"error": "Required meaning associated with vedamantra is not available."})
+    except Exception as e:
+        logging.error(f"Error in _get_vedamantra_meaning: {e}")
+        return json.dumps({"error": f"An error occurred: {e}"})
+
+def _get_pada_morphology(term_details, meanings):
+    try:
+        morphology_list = ast.literal_eval(term_details['Morphology'])
+        term_morph_list = []
+        for morphs in morphology_list:
+            term_info = {}
+            for field in ['stem', 'root']:
+                morph_word = morphs.get(field)
+                if morph_word:
+                    meaning = word_sentence_similarity(meanings, morph_word)
+                    term_info[f'{field}_word'] = morph_word
+                    term_info[f'{field}_meaning'] = meaning[0][0] if meaning else None
+                    term_info[f'{field}_score'] = meaning[0][1] if meaning else None
+            term_info['grammar'] = morphs['grammar']
+            term_morph_list.append(term_info)
+        return term_morph_list
+    except Exception as e:
+        logging.error(f"Error in _get_pada_morphology: {e}")
+        return []
+    
+def get_morphological_info_of_pada(pada, mantraid=None, scripture_name=None, KandahNumber=None, MandalaNumber=None,
+                                   ArchikahNumber=None, ShuktaNumber=None, PrapatakNumber=None, MantraNumber=None,
+                                   AnuvakNumber=None, AdhyayaNumber=None):
+    '''
+    This help to get segmentation and morphological information about the word.
+    '''
+    try:
+        if pada:
+            query = f'''SELECT * FROM term_details_modified WHERE Pada = "{pada}"
+            '''
+            details = execute_sql_query(query)
+        else:
+            # Placeholder for _get_pada_details_by_scripture function call
+            # Replace with your actual implementation
+            details = _get_pada_details_by_scripture(pada, scripture_name=scripture_name, KandahNumber=KandahNumber,
+                                                           MandalaNumber=MandalaNumber, ArchikahNumber=ArchikahNumber,
+                                                           ShuktaNumber=ShuktaNumber, PrapatakNumber=PrapatakNumber,
+                                                           MantraNumber=MantraNumber, AnuvakNumber=AnuvakNumber,
+                                                           AdhyayaNumber=AdhyayaNumber)
+        
+        if details:
+            if mantraid is not None:
+                for record in details:
+                    if record["mantra_id"] == mantraid:
+                        segmentation = record["Segmentation"]
+                        morphological_info = record["Morphology"]
+                        return {"morphology_info": {"segmentation": segmentation, "morphology": morphological_info}}
+                return {"error": f"No details found for mantraid '{mantraid}'"}
+            else:
+                pada_details = details[0]
+                segmentation = pada_details["Segmentation"]
+                morphological_info = pada_details["Morphology"]
+                return {"morphology_info": {"segmentation": segmentation, "morphology": morphological_info}}
+        else:
+            return {"error": "No details found for pada."}
+
+    except Exception as e:
+        logging.error(f"Error in get_morphological_info_of_pada: {e}")
+        return {"error": f"Failed to get meaning of the word {pada}. {e}"}
+        
+    
+def get_adibauatic_adidaivic_adhyatmic_meaning_of_pada(pada, mantraid=None, scripture_name=None, 
+                                                        KandahNumber=None,MandalaNumber=None, ArchikahNumber=None, 
+                                                        ShuktaNumber=None, PrapatakNumber=None, MantraNumber=None,
+                                                        AnuvakNumber=None, AdhyayaNumber=None,MahatmaName=None):
+    '''
+    Sample query:
+    1. What is the adibhautic meaning of pada 'agnim' from RigVeda, first mandala, first shukta and first mantra?
+    2. What is the adhyatmic meaning of the pada agnim in the context of the mantra whose id is '1.1.1.1?'
+    '''
+    try:
+        if mantraid:
+            query = f'''
+            SELECT * FROM term_details_modified WHERE mantra_id = '{mantraid}' AND Pada = "{pada}"
+            '''
+            # Execute the query to get details from the database
+            details = execute_sql_query(query)
+        else:
+            # Call the function to get details by scripture
+            details = _get_pada_details_by_scripture(pada, scripture_name=scripture_name, KandahNumber=KandahNumber,MandalaNumber=MandalaNumber, ArchikahNumber=ArchikahNumber, 
+                                                        ShuktaNumber=ShuktaNumber, PrapatakNumber=PrapatakNumber, MantraNumber=MantraNumber,
+                                                        AnuvakNumber=AnuvakNumber, AdhyayaNumber=AdhyayaNumber)
+    
+        if details:
+            pada_details = details[0]  # Assuming details is a list of dictionaries, select the first item
+            mantraID = pada_details['mantra_id']
+            meanings = _get_vedamantra_meaning(mantraID,MahatmaName=MahatmaName)
+            if 'error' in meanings:
+                return json.dumps(meanings)
+            ab_term_morph_list = _get_pada_morphology(pada_details, meanings['adibhautic'])
+            ad_term_morph_list = _get_pada_morphology(pada_details, meanings['adidaivic'])
+            at_term_morph_list = _get_pada_morphology(pada_details, meanings['adhyatmic'])
+            return json.dumps({
+                f'adibhautic_info_{pada}': ab_term_morph_list,
+                'vedamantra_adibhautic_meaning': meanings['adibhautic'],
+                f'adidavic_info_{pada}': ad_term_morph_list,
+                'vedamantra_adidavic_meaning': meanings['adidaivic'],
+                f'adhyatmic_info_{pada}': at_term_morph_list,
+                'vedamantra_adhyatmic_meaning': meanings['adhyatmic']
+            })
+        else:
+            return json.dumps({"error": f"No details found for pada '{pada}'"})
+    except Exception as e:
+        logging.error(f"Error in get_adibauatic_adidaivic_adhyatmic_meaning_of_pada: {e}")
+        return json.dumps({"error": f"Failed to get meaning of the word {pada}. {e}"})
+    
+# sql agent
+from langchain_community.utilities.sql_database import SQLDatabase
+from database import get_db
+from langchain_community.agent_toolkits import create_sql_agent
+from langchain_openai import ChatOpenAI
+from langchain.pydantic_v1 import BaseModel, Field
+from langchain.tools import StructuredTool
+from typing import Optional
+import json
+
+
+class NLSQLResponse(BaseModel):
+    user_query:str = Field(description="user query")
+
+db = get_db()
+agent_executor = create_sql_agent(llm_AI4, db=db, agent_type="openai-tools", verbose=False)
+
+def get_response(user_query):
+    response = agent_executor.invoke(user_query)
+    return response
+
+sql_tool  = StructuredTool.from_function(
+    func = get_response,
+    name = "nl_sql_query",
+    description="""To obtains a information using natural language query to sql query and then exceting sql query to get natural response.
+                Sample Query:
+                1. How many mantras are there in RigVeda?
+                2. What is the segmentation of the word 'prathasva' from KrishnaYajurVeda?""",
+    args_schema=NLSQLResponse,
+    return_direct=True,
+)
+
+pada_morphological_tool = StructuredTool.from_function(
+    func=get_morphological_info_of_pada,
+    name="pada_morphology",
+    description="""Purpose: To obtain morphological information such as segmentation, morphology, and grammar of a word.\
+                Sample query:
+                    1. What is the segmentation and morphology of the word 'apratidhṛṣṭa-śavasam' from RigVeda?
+                    2. What is the grammar of the word 'prathasva' from KrishnaYajurVeda?
+                    """,
+    args_schema=PadaAAAInput,
+    return_direct=False
+)
+
+pada_meaning_tool = StructuredTool.from_function(
+    func=get_pada_meaning,
+    name="pada_meaning",
+    description="""Purpose: For given sanskrit word, you have collection of meanings for available roots and stems of it.\
+    You need to process this information as context and provide possible meanings for given word.
+    Sample query:
+    1. What is the meaning of the word apratidhṛṣṭa-śavasam?
+    """,
+    args_schema=PadaMeaningInput,
+    return_direct=False
+    )
+
+pada_word_sense_tool = StructuredTool.from_function(
+    func=get_adibauatic_adidaivic_adhyatmic_meaning_of_pada,
+    name="pada_AAA_meaning",
+    description="""To obtain a complete or meaningful adibauatic/adhidaivic/adhyatmic meaning of a word or pada based on context information.\n
+                    Sample query:
+                    1. What is the adibhautic meaning of pada 'agnim' from RigVeda, first mandala, first shukta and first mantra?
+                    2. What is the adhyatmic meaning of the pada agnim in the context of the mantra whose id is '1.1.1.1'?
+                    """,
+    args_schema=PadaAAAInput,
+    return_direct=False
+    )
+
+vedamantra_tool = StructuredTool.from_function(
+    func=get_vedamantra_details,
+    name="vedamantra_details",
+    description='''This function is used to get the vedamantra such as vedamantra, padapatha, devata, chandah, 
+    and rishi, from all Vedas (RigVeda, AtharvaVeda, SamaVeda, KrishnaYajurVeda, and ShuklaYajurVeda).
+    Sample Questions:
+    1. Obtain the vedamantra of the mantra whose id is 1.1.1.1?
+    2. Retrieve the devata of the vedamantra from Rigveda, first mandala, first shukta, and first mantra.
+    3. Provide the meaning of the vedamantra from Rigveda, first mandala, first shukta, and first mantra written by Tulsi Ram.
+    4. Explain the adhibautic meaning of the first mantra from RigVeda, first mandala, and first shukta.
+    ''',
+    args_schema=MantraInput,
+    return_direct=False
+    )
+vedamantra_summary_tool =StructuredTool.from_function(
+    func=get_vedamantra_summary,
+    name="vedamantra_summary",
+    description="""Use the function `get_vedamantra_summary` to access the information such as adibhautic meaning of the mantra, anvaya of the mantra,  mantraVishaya of the mantra, 
+    adhibautic (or adhyatmic or adhidyvic) meaning (or bhavarth) of the mantra, purpose of the mantra, usage of the mantra, and tippani of the mantra.
+    Sample Query:
+    1. Obtain the anvaya of the mantra whose id (mantraid) is 1.1.1.1?
+    2. Retrieve tha adibhautic meaning of the first mantra from RigVeda, first mandala, and first shukta.
+    3. Provide the adhyatmic meaning of the mantra 1.1.1.9?
+    4. What is the tippani of the mantra 1.1.1.7?
+    5. What is the adhyatmic meaning of the mantra 1.1.1.7?
+    6. What is the mantravishaya of the 6th mantra from RigVeda, first mandala, and first shukta?""",
+    args_schema=MantraInput,
+    return_direct=False
+    )
+
+## vector tool
+import os
+import time
+import pickle
+import streamlit as st
+from dotenv import load_dotenv
+from pinecone import Pinecone, ServerlessSpec
+from utils import load_pickle, initialize_embedding_model
+from langchain_community.retrievers import BM25Retriever
+from langchain_pinecone import PineconeVectorStore
+from langchain.retrievers import EnsembleRetriever
+from langchain.tools.retriever import create_retriever_tool
+
+
+
+# Load .env file
+load_dotenv()
+
+# Constants
+INDEX_NAME = "veda-index-v2"
+MODEL_NAME = "BAAI/bge-large-en-v1.5"
+DOCS_DIRECTORY = r"Docs\ramana_docs_ids.pkl"
+CURRENT_DIRECTORY = os.getcwd()
+
+
+# Initialize Pinecone client
+PINECONE_API_KEY = os.getenv("PINECONE_API_KEY_SAM")
+pc = Pinecone(api_key=PINECONE_API_KEY)
+
+#@st.cache_resource
+def create_or_load_index():
+    # Check if index already exists
+    if INDEX_NAME not in pc.list_indexes().names():
+        # Create index if it does not exist
+        pc.create_index(
+            INDEX_NAME,
+            dimension=1024,
+            metric='dotproduct',
+            spec=ServerlessSpec(
+                cloud="aws",
+                region="us-east-1"
+            )
+        )
+        # Wait for index to be initialized
+        while not pc.describe_index(INDEX_NAME).status['ready']:
+            time.sleep(1)
+    # Connect to index
+    return pc.Index(INDEX_NAME)
+
+# Load documents
+docs = load_pickle(DOCS_DIRECTORY)
+# Initialize embedding model
+embedding = initialize_embedding_model(MODEL_NAME)
+# Create or load index
+index = create_or_load_index()
+
+# Initialize BM25 retriever
+bm25_retriever = BM25Retriever.from_texts(
+    [text['document'].page_content for text in docs],
+    metadatas=[text['document'].metadata for text in docs]
+)
+bm25_retriever.k = 2
+
+# Switch back to normal index for LangChain
+vector_store = PineconeVectorStore(index, embedding)
+retriever = vector_store.as_retriever(search_type="mmr")
+
+# Initialize the ensemble retriever
+ensemble_retriever = EnsembleRetriever(
+    retrievers=[bm25_retriever, retriever], weights=[0.2, 0.8]
+)
+
+class VectorResponse(BaseModel):
+    query:str = Field(description="user query")
+
+def vector_retrieve(query):
+    response = retriever.get_relevant_documents(query)
+    return response
+
+vector_tool = StructuredTool.from_function(
+    func = vector_retrieve,
+    name = "vector_retrieve",
+    description="Search and return documents related user query from the vector index.",
+    args_schema=VectorResponse,
+    return_direct=False
+)
+
+tools_list = [pada_morphological_tool, sql_tool, pada_meaning_tool, pada_word_sense_tool, vedamantra_tool, vedamantra_summary_tool]
+#vector_tool,
+
+# Convert tools to OpenAI functions
+tools_all = [convert_to_openai_function(tool) for tool in tools_list]
+# Set up the tools to execute them from the graph
+from langgraph.prebuilt import ToolExecutor
+
+tool_executor = ToolExecutor(tools_list)
+#tools_response = tools_all.append(convert_to_openai_function(Response))
+llm_with_tools = llm_AI4.bind_tools(tools_all)
+#tool_map = {tool.name: tool for tool in tools_list}
+
+
+def call_tools(msg: AIMessage) -> Runnable:
+    """Simple sequential tool calling helper."""
+    tool_map = {tool.name: tool for tool in tools_list}
+    tool_calls = msg.tool_calls.copy()
+    for tool_call in tool_calls:
+        tool_call["output"] = tool_map[tool_call["name"]].invoke(tool_call["args"])
+    return tool_calls
+
+#print("Invoking the chain")
+tool_chain = llm_with_tools | call_tools
+

requirements.txt

@@ -0,0 +1,19 @@
+langchain==0.1.12
+langsmith==0.1.29
+langgraph==0.0.28
+streamlit==1.31.1
+langchain_openai==0.0.8
+openai==1.14.1
+langchain-groq==0.0.1
+chardet==5.2.0
+aksharamukha==2.1.2
+sentence_transformers==2.4.0
+langchain-nomic==0.0.2
+beautifulsoup4==4.12.3
+pymysql==1.1.0
+langchain_core== 0.1.34
+mysql-connector-python==8.3.0
+pinecone-text==0.9.0
+rank-bm25==0.2.2
+langchain-mistralai==0.1.2
+momento==1.20.1

utils.py

@@ -0,0 +1,147 @@
+import os
+import re
+import requests
+import json
+import pickle
+import numpy as np
+import pandas as pd
+from typing import List
+from typing import Optional
+from typing import Union
+import streamlit as st
+from database import execute_sql_query
+from bs4 import BeautifulSoup
+from aksharamukha import transliterate
+from sentence_transformers import util
+from langchain_nomic.embeddings import NomicEmbeddings
+from langchain_community.embeddings import HuggingFaceBgeEmbeddings
+
+
+
+def load_pickle(path):
+    with open(path,'rb') as f:
+        docs = pickle.load(f)
+    return docs
+
+
+
+def initialize_embedding_model(model_name, device="cpu", normalize_embeddings=True):
+    model_kwargs = {"device": device}
+    encode_kwargs = {"normalize_embeddings": normalize_embeddings}
+    return HuggingFaceBgeEmbeddings(model_name=model_name, model_kwargs=model_kwargs, encode_kwargs=encode_kwargs)
+
+
+
+# embedding model for quick calculations
+nomic_api_key = os.getenv('NOMIC_API_KEY')
+#nomic embed model used for similarity scores
+nomic_embed_model = NomicEmbeddings(
+    dimensionality=128,
+    model="nomic-embed-text-v1.5",
+)
+
+
+
+def get_list_meaning_word(word):
+    pada_meanings = {'pada': word,
+                     'Monier-Williams Sanskrit-English Dictionary (1899)': [],
+                     'Shabda-Sagara (1900)': [],
+                     'Apte-Practical Sanskrit-English Dictionary (1890)': [],
+                     }
+    url = f"https://ambuda.org/tools/dictionaries/mw,shabdasagara,apte/{word}"
+
+    try:
+        # Fetch HTML content
+        response = requests.get(url)
+        response.raise_for_status()
+
+        # Parse HTML with BeautifulSoup
+        soup = BeautifulSoup(response.text, 'html.parser')
+
+        # Extracting text from different tags
+        divs = soup.find_all('div', class_='my-4', attrs={'x-show': 'show'})
+        
+        try:
+            # Find all list items <li> within the specified <ul> tag
+            div_items_0 = divs[0].find('ul').find_all('li', class_='dict-entry mw-entry')
+            # Print the text content of each list item
+            dive_text_0 = [li_tag.get_text(strip=True) for li_tag in div_items_0]
+            text_0_trans = [transliterate.process(src='Devanagari', tgt='IAST', txt=text) for text in dive_text_0]
+            pada_meanings['Monier-Williams Sanskrit-English Dictionary (1899)'] = text_0_trans
+        except :
+            print("Error: Unable to find Monier-Williams Sanskrit-English Dictionary (1899) data.")
+
+        try:
+            div_items_1 = divs[1].find_all('div')
+            dive_text_1 = [item.get_text(strip=True) for item in div_items_1]
+            text_1_trans = [transliterate.process(src='Devanagari', tgt='IAST', txt=text) for text in dive_text_1]
+            pada_meanings['Shabda-Sagara (1900)'] = text_1_trans
+        except :
+            print("Error: Unable to find Shabda-Sagara (1900) data.")
+
+        try:
+            apte_meanings = []
+            for tag in divs[2].find_all('b'):
+                if tag.text.strip() != '—':
+                    text1 = tag.text.strip()  # English text within <b> tag
+                    sibling = tag.find_next_sibling()  # Text following <b> tag
+                    text2 = tag.next_sibling.strip() + ' ' # English text following <b> tag
+                    while sibling.name != 'div':
+                        if sibling.name is None:  # Handling non-tag text
+                            text2 += " "
+                        elif sibling.name == 'span':  # Handling <b> tag
+                            IAST_text = transliterate.process(src='Devanagari', tgt='IAST', txt=sibling.text.strip()) 
+                            text2 += IAST_text  +  ' ' + sibling.next_sibling.strip()
+                        else:
+                            text2 += sibling.text.strip() +  ' ' +  sibling.next_sibling.strip()
+                        sibling = sibling.find_next_sibling()
+                    apte_meanings.append(text2)
+            pada_meanings['Apte-Practical Sanskrit-English Dictionary (1890)'] = apte_meanings[:-1]
+        except:
+            print("Error: Unable to find Apte-Practical Sanskrit-English Dictionary (1890) data.")
+
+    except requests.exceptions.RequestException as e:
+        print(f"Error: Failed to fetch data from {url}. {e}")
+
+    return pada_meanings
+
+#get similarity scores
+def word_sentence_similarity(meanings, root_stem_word):
+    # Check if the word embeddings are not empty
+    if not meanings or not root_stem_word:
+        return None
+    
+    meaning_embedding = np.array(nomic_embed_model.embed_query(meanings))
+    all_meanings = []
+    word_score_pair = []
+    all_meanings.extend(get_list_meaning_word(root_stem_word)['Monier-Williams Sanskrit-English Dictionary (1899)'])
+    all_meanings.extend(get_list_meaning_word(root_stem_word)['Shabda-Sagara (1900)'])
+    for word_meaning in all_meanings:
+        root_stem_word_meaning_embedding = np.array(nomic_embed_model.embed_query(word_meaning))
+        # Calculate cosine similarity
+        similarity_score = util.pytorch_cos_sim(meaning_embedding, root_stem_word_meaning_embedding).item()
+        word_score_pair.append((word_meaning,similarity_score))
+    # Sort the list in descending order based on similarity scores
+    sorted_word_score_pairs = sorted(word_score_pair, key=lambda x: x[1], reverse=True)
+    return sorted_word_score_pairs
+
+#extract the adhibautic meaning of the mantra from the vedamantra
+def extract_meaning_by_language(data_list, target_language='English'):
+    for data_dict in data_list:
+        if data_dict.get('languageName') == target_language:
+            return data_dict.get('mahatma', {})
+    return None  
+
+#mantra_json_details
+def get_details_mantra_json(query):
+    description, data = execute_sql_query(query)
+    df = pd.DataFrame(data)
+    df.columns = [x[0] for x in description]
+    mantra_json = df['mantra_json'].values[0]
+    cleaned_data = re.sub('<[^<]+?>', '', mantra_json)
+    return json.loads(cleaned_data)
+
+def iast_process(input_text):
+    output_text = re.sub('[\u0951-\u0954,\u200d,\u0331]', '', input_text)
+    return output_text
+