Update app.py

app.py

@@ -11,18 +11,20 @@ import time
 import shutil
 import json
 import nltk
-#commenting audio related code based on Arun's input
+import mysql.connector
+import fnmatch
+# audio related code is not included based on Arun's input
 # audio package
-"""import speech_recognition as sr
+import speech_recognition as sr
 from pydub import AudioSegment
-from pydub.playback import play"""
-# Commenting SMTP code since HFSpaces doesn't support it
+from pydub.playback import play
+# SMTP code is not included since HFSpaces doesn't support it
 # email library
-"""import smtplib, ssl
+import smtplib, ssl
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 from email.mime.base import MIMEBase
-from email import encoders"""
+from email import encoders
 # langchain
 from langchain_core.prompts import ChatPromptTemplate
 from langchain_core.output_parsers import StrOutputParser
@@ -34,12 +36,14 @@ from langchain_community.utilities import SQLDatabase
 from langchain.agents import create_tool_calling_agent, AgentExecutor, Tool
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.tools import StructuredTool
-from langchain.pydantic_v1 import BaseModel, Field
+#from langchain.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from PyPDF2 import PdfReader
 from nltk.tokenize import sent_tokenize
 from datetime import datetime
 from sqlalchemy import create_engine
 from sqlalchemy.sql import text
+import openai
 
 # pandas
 import pandas as pd
@@ -64,8 +68,24 @@ import threading
 from mailjet_rest import Client
 import base64
 
+#for PDF form filling
+from PyPDFForm import FormWrapper
+
+#Variables Initialization
+agent_executor = None
+vector_store1 = None
+texts1 = None
+excel_dataframe = None
+file_extension = None
+total_rows = ""
+docstatus = ""
+sample_table = ""
+#This is to define the summary of the runtime tool. This summary will be updated in prompt template and description of the new tool
+run_time_tool_summary=""
+
 # Define global variables for managing the thread and current_event
 executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
+
 current_event = None
 stop_event = threading.Event()
 
@@ -74,19 +94,25 @@ os.environ["LANGFUSE_PUBLIC_KEY"] = os.getenv("LANGFUSE_PUBLIC_KEY")
 os.environ["LANGFUSE_SECRET_KEY"] = os.getenv("LANGFUSE_SECRET_KEY")
 os.environ["LANGFUSE_HOST"] = os.getenv("LANGFUSE_HOST")
 
+DB_USER = 'u852023448_redmindgpt'
+DB_PASSWORD = 'redmindGpt@123'
+DB_HOST = '217.21.88.10'
+DB_NAME = 'u852023448_redmindgpt'
+
+
 langfuse_handler = CallbackHandler()
 langfuse_handler.auth_check()  # Optional: Checks if the authentication is successful
 
 nltk.download('punkt')
 
-open_api_key_token = os.getenv("OPEN_AI_API")
+open_api_key_token = os.getenv("OPENAI_API_KEY")
 
 os.environ['OPENAI_API_KEY'] = open_api_key_token
 pdf_path = "Inbound.pdf"
 
 db_uri = os.getenv("POSTGRESQL_CONNECTION")
-# Database setup
 
+# Database setup
 db = SQLDatabase.from_uri(db_uri)
 
 user_email = ""
@@ -94,6 +120,7 @@ warehouse_name = ""
 warehouse_id = ""
 # Today's date to be populated in inventory API
 inventory_date = datetime.today().strftime('%Y-%m-%d')
+
 apis = [
     # fetch warehouse ID
     {
@@ -112,7 +139,6 @@ apis = [
 llm = ChatOpenAI(model="gpt-4o-mini", max_tokens=300, temperature=0.1)
 llm_chart = OpenAI()
 
-
 def get_schema(_):
     schema_info = db.get_table_info()  # This should be a string of your SQL schema
     return schema_info
@@ -161,9 +187,12 @@ def database_tool(question):
 
 def get_ASN_data(question):
     base_url = os.getenv("ASN_API_URL")
+    print(f"base_url{base_url}")
     complete_url = f"{base_url}branchMaster.id=343&transactionUid={question}&userId=164&transactionType=ASN"
     try:
         response = requests.get(complete_url)
+        print(f"complete_url{complete_url}")
+        print(f"response{response}")
         data = response.json()
         response.raise_for_status()
 
@@ -246,6 +275,10 @@ def summarize_document(docs):
 texts = load_and_split_pdf(pdf_path)
 vector_store = create_vector_store(texts)
 
+def document_data_tool_runtime(question):
+    print(f"Document data runtime tool enter: {question} with {vector_store1}")
+    query_response = query_vector_store(vector_store1, question, config={"callbacks": [langfuse_handler]})
+    return query_response
 
 def document_data_tool(question):
     print(f"Document data tool enter: {question}")
@@ -256,8 +289,6 @@ def document_data_tool(question):
 
 # mailjet API since SMTP is not supported HF spaces
 def send_email_with_attachment_mailjet(recipient_email, subject, body, attach_img_base64=None):
-
-
     api_key = os.getenv("MAILJET_API_KEY")
     api_secret = os.getenv("MAILJET_API_SECRET")
     
@@ -350,8 +381,6 @@ def send_email_with_attachment(recipient_email, subject, body, attachment_path):
 def make_api_request(url, params):
     """Generic function to make API GET requests and return JSON data."""
     try:
-        print(url)
-        print(params)
         response = requests.get(url, params=params)
         response.raise_for_status()  # Raises an HTTPError if the response was an error
         return response.json()  # Return the parsed JSON data
@@ -377,18 +406,14 @@ def inventory_report(question):
     if data:
         # Extracting the id for the warehouse with the name "WH"
         warehouse_id = next((item['id'] for item in data['result'] if item['wareHouseId'] == warehouse_name), None)
-        #print(warehouse_id)
-        if (warehouse_id):
-           
-            # print(f"The id for the warehouse named {name} is: {warehouse_id}")
+       
+        if (warehouse_id):         
+            
             # Step 3: Update the placeholder with the actual warehouse_id
             for api in apis:
                 if "warehouseId" in api["params"]:
                     api["params"]["warehouseId"] = warehouse_id
-        
-            print(f"warehouseId: {warehouse_id}")
-            print(f"warehouseId: {apis[1]}")
-           
+                              
             data1 = make_api_request(apis[1]["url"], apis[1]["params"])
             if (data1):
                 headers = ["S.No", "Warehouse Code", "Warehouse Name", "Customer Code", "Customer Name", "Item Code", "Item Name",
@@ -416,20 +441,25 @@ def inventory_report(question):
         
                 # Convert to pandas DataFrame
                 df = pd.DataFrame(table_data, columns=headers)
+
+                chart_link = chat_with_llm(df,question)                  
         
-                sdf = SmartDataframe(df, config={"llm": llm_chart})
-        
-                # chart = sdf.chat("Can you draw a bar chart with all avaialble item name and quantity.")
-                chart = sdf.chat(question)
-        
-                return chart
+                return chart_link
             else:
                 return "There are no inventory details for the warehouse you have given."
         else:
             return "Please provide a warehouse name available in the database."
 
+def chat_with_llm(df,question):
+    sdf = SmartDataframe(df, config={"llm": llm_chart})
+    llm_response = sdf.chat(question)
+    return llm_response
 
-# inventory_report("WH:can you give me a bar chart with item name and quantity for the warehouse WH")
+def bind_llm(llm, tools,prompt_template):
+    llm = llm.bind()
+    agent = create_tool_calling_agent(llm, tools, ChatPromptTemplate.from_template(prompt_template))
+    agent_executor = AgentExecutor(agent=agent, tools=tools, verbose=True)
+    return agent_executor
 
 # Define input and output models using Pydantic
 class QueryInput(BaseModel):
@@ -446,6 +476,7 @@ class QueryOutput(BaseModel):
 
 # Wrap the function with StructuredTool for better parameter handling
 tools = [
+    
     StructuredTool(
         func=get_ASN_data,
         name="APIData",
@@ -486,6 +517,7 @@ tools = [
 ]
 
 prompt_template = f"""You are an assistant that helps with database queries, API information, and document retrieval.  Your job is to provide clear, complete, and detailed responses to the following queries. Please give the output response in an user friendly way and remove "**" from the response. For example, document related queries can be answered in a clear and concise way with numbering and not as a paragraph. Database related queries should be answered with proper indentation and use numbering for the rows. ASN id related queries should be answered with proper indentation and use numbering for the rows.
+
 For ASN id related questions, if the user specifies an ASN id, provide the information from the api tool. Pass only the id as input to the tool. Do not pass the entire question as input to the tool. If the details are not found, say it in a clear and concise way.
 You are an AI assistant trained to help with warehouse management questions based on a detailed document about our WMS. The document covers various processes such as ASN handling, purchase orders, cross docking, appointment scheduling for shipments, and yard management. Please provide a complete and concise response within 200 words and Ensure that the response is not truncated and covers the essential points. When answering, focus on providing actionable insights and clear explanations related to the specific query.  Please remove "**" from the response.
 For SQL database-related questions, only use the fields available in the warehouse schema, including tables such as customer_master, efs_company_master, efs_group_company_master, efs_region_master, party_address_detail, wms_warehouse_master.
@@ -493,18 +525,13 @@ For datavisualization, user will ask for inventory report of a particular wareho
 {{agent_scratchpad}}
 Here is the information you need to process:
 Question: {{input}}"""
-
-llm = llm.bind()
-agent = create_tool_calling_agent(llm, tools, ChatPromptTemplate.from_template(prompt_template))
-agent_executor = AgentExecutor(agent=agent, tools=tools, verbose=True)
-
+agent_executor = bind_llm(llm,tools,prompt_template)
 
 def ensure_temp_chart_dir():
     temp_chart_dir = os.getenv("IMAGE_MAIN_URL")
     if not os.path.exists(temp_chart_dir):
         os.makedirs(temp_chart_dir)
 
-
 def clean_gradio_tmp_dir():
     tmp_dir = os.getenv("IMAGE_GRADIO_PATH")
     if os.path.exists(tmp_dir):
@@ -529,9 +556,9 @@ def handle_query(user_question, chatbot, audio=None):
     # Clear previous stop event and current_event
     stop_event.clear()
 
-    #if current_event and not current_event.done():
-     #   chatbot.append(("","A query is already being processed. Please stop it before starting a new one."))
-      #  return gr.update(value=chatbot)
+    if current_event and not current_event.done():
+        chatbot.append(("","A query is already being processed. Please stop it before starting a new one."))
+        return gr.update(value=chatbot)
        
     # Start the processing in a new thread
     current_event = executor.submit(answer_question_thread, user_question, chatbot)
@@ -539,7 +566,13 @@ def handle_query(user_question, chatbot, audio=None):
     # Periodically check if current_event is done
     while not current_event.done():
         if stop_event.is_set():
-            current_event.cancel()
+            #current_event.task.cancel()  # Attempt to cancel the current_event         
+            current_event.set_result((user_question, "Sorry, we encountered an error while processing your request. Please try after some time."))
+            current_event.cancel()  # Attempt to cancel the current_event
+            executor.shutdown(wait=False)  # Shutdown the executor
+            print("Current event cancelled")
+            print(current_event.cancelled())
+           
             chatbot.append((user_question, "Sorry, we encountered an error while processing your request. Please try after some time."))
             return gr.update(value=chatbot)
             
@@ -574,7 +607,7 @@ def stop_processing(chatbot):
     return gr.update(value=chatbot)
 
 # This function is for agent executor invoke with the option of stop
-def answer_question_thread(user_question, chatbot, audio=None):
+def answer_question_thread(user_question, chatbot,audio=None):
      
     global iterations
     iterations = 0
@@ -609,18 +642,8 @@ def answer_question_thread(user_question, chatbot, audio=None):
 
     while iterations < max_iterations:
 
-        """if "send email to" in user_question:
-            email_match = re.search(r"send email to ([\w\.-]+@[\w\.-]+)", user_question)
-            if email_match:
-                user_email = email_match.group(1).strip()
-                user_question = user_question.replace(f"send email to {user_email}", "").strip()
-                user_question = f"{user_question}:{user_email}"
-                """
-
         response = agent_executor.invoke({"input": user_question}, config={"callbacks": [langfuse_handler]}, early_stopping_method="generate")
-        print("response generated")
-        print(response)
-
+        
         if isinstance(response, dict):
             response_text = response.get("output", "")
         else:
@@ -711,14 +734,6 @@ def answer_question(user_question, chatbot, audio=None):
 
     while iterations < max_iterations:
 
-        """if "send email to" in user_question:
-            email_match = re.search(r"send email to ([\w\.-]+@[\w\.-]+)", user_question)
-            if email_match:
-                user_email = email_match.group(1).strip()
-                user_question = user_question.replace(f"send email to {user_email}", "").strip()
-                user_question = f"{user_question}:{user_email}"
-                """
-
         response = agent_executor.invoke({"input": user_question}, config={"callbacks": [langfuse_handler]})
 
         if isinstance(response, dict):
@@ -731,6 +746,8 @@ def answer_question(user_question, chatbot, audio=None):
 
     if iterations == max_iterations:
         return "The agent could not generate a valid response within the iteration limit."
+    
+        
 
     if os.getenv("IMAGE_PATH") in response_text:
         # Open the image file
@@ -742,7 +759,7 @@ def answer_question(user_question, chatbot, audio=None):
         img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
 
         img = f'<img src="data:image/png;base64,{img_str}" style="width:450px; height:400px;">'
-        # image = gr.Image(value=img_str)
+        
         chatbot.append((user_question, img))
 
         email_pattern = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}'
@@ -778,21 +795,45 @@ def answer_question(user_question, chatbot, audio=None):
                 print(f"Error loading image file: {e}")
                 chatbot.append((user_question, "Chart generation failed. Please try again."))
         return gr.update(value=chatbot)
-
-
-    # return [(user_question,gr.Image("/home/user/app/exports/charts/temp_chart.png"))]
-    #  return "/home/user/app/exports/charts/temp_chart.png"
+    
     else:
         chatbot.append((user_question, response_text))
         return gr.update(value=chatbot)
-    # response_text = response_text.replace('\n', ' ').replace('  ', ' ').strip()
-    # return response_text
-        
-def submit_feedback(feedback, chatbot):
+    
+
+def submit_feedback(feedback, chatbot, request: gr.Request):
     gr.Info("Thank you for your feedback.")
+    #save feedback with user question and response in database
+    save_feedback(request.username,chatbot[-1][0], chatbot[-1][1], feedback)
     feedback_response = "User feedback: " + feedback
     return chatbot + [(feedback_response, None)], gr.update(visible=False), gr.update(visible=False)
 
+
+# Function to connect to MySQL database
+def connect_to_db():
+    return mysql.connector.connect(
+        host=DB_HOST,
+        user=DB_USER,
+        password=DB_PASSWORD,
+        database=DB_NAME
+    )
+
+# Function to save feedback to the database
+def save_feedback(username, user_question, user_response, feedback):
+    try:
+        conn = connect_to_db()
+        cursor = conn.cursor()
+        query = "INSERT INTO user_feedback (username, question, response, feedback) VALUES (%s, %s, %s, %s)"
+        cursor.execute(query, (username, user_question, user_response, feedback))
+        conn.commit()
+    except mysql.connector.Error as err:
+        print(f"Error: {err}")
+    finally:
+        if cursor:
+            cursor.close()
+        if conn:
+            conn.close()
+
 def handle_dislike(data: gr.LikeData):
     if not data.liked:
         print("downvote")
@@ -806,13 +847,369 @@ def handle_dislike(data: gr.LikeData):
 def update_message(request: gr.Request):
     return f"<h2 style=' font-family: Calibri;'>Welcome, {request.username}</h4>"
 
-def send_mail_with_history(req: gr.Request,chatbot):
-    if req.username:
-       send_email_with_attachment_mailjet("lakshmivairamani@gmail.com","conversation history" + req.username, chatbot)  
+# Function to generate a 50-word summary of the newly uploaded doc using OpenAI
+def generate_summary(text):
+    prompt = (
+         "You are an AI that helps with document analysis. Please provide a concise title and a summary of the following document. "
+        "The summary should be about 50 words and include key details that can help answer questions accurately:\n\n"
+        f"{text}\n\nTitle : Summary"
+    )
+    # Call the OpenAI API to generate a summary    
+    response = openai.chat.completions.create(
+        messages=[
+            {
+                "role": "user",
+                "content": prompt,
+            }
+        ],
+        model="gpt-4o-mini",
+    )
+    # Extract the title and summary from the response
+    response_content = response.choices[0].message.content
+    lines = response_content.split("\n")
+     # Extract title
+    title_line = lines[0]
+    title = title_line.split("**Title:**")[-1].strip()
+    
+    # Extract summary
+    summary_line = lines[2]
+    summary = summary_line.split("**Summary:**")[-1].strip()
+
+    return title, summary
+#function to handle file upload decide whether excel or doc is uploaded and respective tool will be created with appropriate prompts at runtime
+def upload_file(filepath):
+    global vector_store1, file_extension
+    
+    # Get the file extension
+    _, file_extension = os.path.splitext(filepath)
+    
+    if file_extension == ".pdf":
+        texts1 = load_and_split_pdf(filepath)
+       
+        vector_store1 = create_vector_store(texts1)   
+        # Generate a 50-word summary from the extracted text
+        title, summary = generate_summary(texts1)        
+        return title, summary, file_extension
+    elif file_extension == ".xlsx":
+        title, prompt = process_excel(filepath)        
+        return title, prompt
+
+def generate_example_questions(sheet_name, column_headers):
+    """
+    Generates natural language questions based on column headers.
+    
+    Args:
+        sheet_name (str): The name of the Excel sheet.
+        column_headers (list): List of column headers from the sheet.
+    
+    Returns:
+        questions (list): List of generated questions based on the columns.
+    """
+    questions = []
+
+    # Check for typical columns and create questions
+    if 'Product Name' in column_headers or 'Product' in column_headers:
+        questions.append(f"What is the total sales for a specific product in {sheet_name}?")
+            
+    if 'Sales Amount' in column_headers or 'Amount' in column_headers:
+        questions.append(f"What is the total sales amount for a specific region in {sheet_name}?")
+            
+    if 'Region' in column_headers:
+        questions.append(f"Which region had the highest sales in {sheet_name}?")
+    
+    if 'Date' in column_headers:
+        questions.append(f"What were the total sales during a specific month in {sheet_name}?")
+    
+    if 'Price' in column_headers:
+        questions.append(f"What is the price of a specific product in {sheet_name}?")
+    
+    if any(fnmatch.fnmatch(header, 'Employee*') for header in column_headers):
+        questions.append(f"What are the details of the distinct broker names?")
+        
+    return questions
+
+def generate_prompt_from_excel_file(df_dict):
+    """
+    Generates a prompt from an Excel file containing multiple sheets.
+    
+    Args:
+        excel_file_path (str): The path to the Excel file.
+    
+    Returns:
+        prompt (str): A detailed prompt including sheet names, column headers, sample data, 
+                      and example questions for each sheet.
+    """
+
+    # Initialize prompt with basic structure
+    prompt = "You have been provided with an Excel file containing data in several sheets.\n"
+    
+    # Loop through each sheet to extract column headers and sample data
+    for sheet_name, sheet_df in df_dict.items():
+        # Extract column headers
+        column_headers = list(sheet_df.columns)
+        
+        # Get a sample of the data (first few rows)
+        sample_data = sheet_df.head(3).to_string(index=False)
+
+        # Add sheet details to the prompt
+        prompt += f"For the sheet '{sheet_name}', the column headers are:"
+        prompt += f"{', '.join(column_headers)}\n\n"
+        #prompt += f"Example data from sheet '{sheet_name}':\n"
+        #prompt += f"{sample_data}\n\n"
+
+        # Generate example natural language questions based on columns
+        example_questions = generate_example_questions(sheet_name, column_headers)
+        #prompt += "### Example Questions:\n"
+        #for question in example_questions:
+         #   prompt += f"- {question}\n"
+        #prompt += "\n"
+
+    # Finalize the prompt with function call description
+
+    prompt += f"- Query: A natural language question (e.g., List all the employees with broker name ADP or Alerus). The question should be sent as 'What are the employee details with broker name ADP or Alerus :'."
+    prompt += f"""Output :  {docstatus}. Here is the sample table:
+            {sample_table}. 
+            """
+
+    prompt += f"- Query: A natural language question with request to create LOA document (e.g., can you create LOA document for all the employees with broker name ADP or Alerus). The question should be sent as 'What are the employee details with broker name ADP or Alerus : LOA document'."
+    prompt += f"""Output:  {docstatus}. Here is the sample table:
+            {sample_table}.             
+            If there is any error, please display the message returned by the function as response. """
+
+
+    return "Excel data", prompt
 
+# Function to handle "Add to RedMindGPT" button click
+def add_to_redmindgpt(title, summary):
+    """
+    Adds a document or Excel file to the RedmindGPT system and configures the appropriate runtime tool for handling related queries.
+    Parameters:
+    title (str): The title of the document or Excel file.
+    summary (str): A brief summary of the document or Excel file.
+    Returns:
+    str: A message indicating whether the file has been added successfully.
+    Behavior:
+    - If the file extension is ".pdf", it sets up a runtime tool for handling document-related queries.
+    - If the file extension is ".xlsx", it sets up a runtime tool for handling Excel data-related queries.
+    - Configures the prompt template for the agent executor based on the file type.
+    - Adds the configured runtime tool to the list of tools used by the agent executor.
+    """
+    
+    global agent_executor, file_extension
+    
+    if file_extension == ".pdf":
+        run_time_tool_summary = f"For {title} document related questions, Please refer runtimeDocumentData tool. {summary}. Please provide a complete and concise response within 200 words and Ensure that the response is not truncated and covers the essential points."
+  
+        run_time_tool = StructuredTool(
+            func=document_data_tool_runtime,
+            name="runtimeDocumentData",
+            args_schema=QueryInput,
+            output_schema=QueryOutput,
+            description=f"You are an AI assistant trained to help with the questions based on the uploaded document {title}. {summary}. Please provide a complete and concise response within 200 words and Ensure that the response is not truncated and covers the essential points."
+        )
+       
+        # Add the new tool to the beginning
+        tools.insert(0, run_time_tool)
+               
+        prompt_template = f"""You are an assistant that helps with database queries, API information, and document retrieval.  Your job is to provide clear, complete, and detailed responses to the following queries. Please give the output response in an user friendly way and remove "**" from the response. For example, document related queries can be answered in a clear and concise way with numbering and not as a paragraph. Database related queries should be answered with proper indentation and use numbering for the rows. ASN id related queries should be answered with proper indentation and use numbering for the rows.
+        {run_time_tool_summary}
+        For ASN id related questions, if the user specifies an ASN id, provide the information from the api tool. Pass only the id as input to the tool. Do not pass the entire question as input to the tool. If the details are not found, say it in a clear and concise way.
+        You are an AI assistant trained to help with warehouse management questions based on a detailed document about our WMS. The document covers various processes such as ASN handling, purchase orders, cross docking, appointment scheduling for shipments, and yard management. Please provide a complete and concise response within 200 words and Ensure that the response is not truncated and covers the essential points. When answering, focus on providing actionable insights and clear explanations related to the specific query.  Please remove "**" from the response.
+        For SQL database-related questions, only use the fields available in the warehouse schema, including tables such as customer_master, efs_company_master, efs_group_company_master, efs_region_master, party_address_detail, wms_warehouse_master.
+        For datavisualization, user will ask for inventory report of a particular warehouse. Your job is to return the image path to chat interface and display the image as output.
+    
+        {{agent_scratchpad}}
+        Here is the information you need to process:
+        Question: {{input}}"""
+        agent_executor = bind_llm(llm,tools,prompt_template)
+        return f"File has been added successfully."
+    elif file_extension == ".xlsx":
+        run_time_excel_tool_summary = f"For {title}  related questions, Please refer runtimeExcelData tool. {summary}. Display the response only in the format as mentioned in the tool description. "
+   
+        run_time_excel_tool = StructuredTool(
+            func=chat_with_excel_data_dataframe,
+            name="runtimeExcelData",
+            args_schema=QueryInput,
+            output_schema=QueryOutput,
+            description=f"""You are an AI assistant trained to handle Excel data and return meaningful insights. If user query is given with an option of generating the document with the result set dataframe, pass two inputs to the tool. First input is the user query and the second input will be the phrase "create document". display the response only in the below format. 
+             {docstatus}. Here is the sample data:
+            {sample_table}.
+            Please provide the total rows count from the {total_rows} values returned by the function and not the count of sample table rows. If there is any error, please display the message returned by the function as response. """
+        )
+       
+        # Add the new tool to the beginning
+        tools.insert(0, run_time_excel_tool)
+            
+        prompt_template = f"""You are an assistant that helps with database queries, API information, and document retrieval.  Your job is to provide clear, complete, and detailed responses to the following queries. Please give the output response in an user friendly way and remove "**" from the response. For example, document related queries can be answered in a clear and concise way with numbering and not as a paragraph. Database related queries should be answered with proper indentation and use numbering for the rows. ASN id related queries should be answered with proper indentation and use numbering for the rows.
+        {run_time_excel_tool_summary}
+        For ASN id related questions, if the user specifies an ASN id, provide the information from the api tool. Pass only the id as input to the tool. Do not pass the entire question as input to the tool. If the details are not found, say it in a clear and concise way.
+        You are an AI assistant trained to help with warehouse management questions based on a detailed document about our WMS. The document covers various processes such as ASN handling, purchase orders, cross docking, appointment scheduling for shipments, and yard management. Please provide a complete and concise response within 200 words and Ensure that the response is not truncated and covers the essential points. When answering, focus on providing actionable insights and clear explanations related to the specific query.  Please remove "**" from the response.
+        For SQL database-related questions, only use the fields available in the warehouse schema, including tables such as customer_master, efs_company_master, efs_group_company_master, efs_region_master, party_address_detail, wms_warehouse_master.
+        For datavisualization, user will ask for inventory report of a particular warehouse. Your job is to return the image path to chat interface and display the image as output.
+    
+        {{agent_scratchpad}}
+        Here is the information you need to process:
+        Question: {{input}}"""
+        agent_executor = bind_llm(llm,tools,prompt_template)
+        return f"File has been added successfully."
+
+def process_excel(file):
+    global excel_dataframe
+    # Check if the file is None
+    if file is None:
+        return "Excel file", "Your excel does not have values. Please upload a different file."  # Return an empty dataframe if no file is uploaded
+    else:
+        # Read the uploaded Excel file
+        excel_dataframe = pd.read_excel(file.name, sheet_name=None)  # 'file.name' to get the actual file path
+        
+        #to get title and summary of excel file
+        title, prompt = generate_prompt_from_excel_file(excel_dataframe)
+        excel_dataframe = pd.read_excel(file.name)
+
+        return title, prompt  # Return the success message.
+
+def chat_with_excel_data(question):
+    global excel_dataframe
+    response_dataframe = chat_with_llm(excel_dataframe,question)
+    print(response_dataframe)
+    return response_dataframe
+
+def chat_with_excel_data_dataframe(question):
+    isDataFrame = True
+    print(f"question for excel data frame  :  {question}")
+    if "LOA" in question:
+        #question = question.replace("create document", "").strip()
+        create_document = True
+    else:
+        create_document = False
+    print(f"create document : {create_document}")
+    response_dataframe = chat_with_excel_data(question)
+    if isinstance(response_dataframe, pd.DataFrame) == False:          
+        
+        print("The result is not a DataFrame.")
+        if ":" in response_dataframe:
+            isDataFrame = False
+            names_part = response_dataframe.split(":", 1)[1]  # Get everything after the colon and space
+
+            # Split the names by commas to create a list
+            names = names_part.split(",")
+
+            # Convert the list of names to a DataFrame
+            response_dataframe = pd.DataFrame(names, columns=["Result"])
+            
+        
+    #handle large dataset
+    response = handle_large_dataset(response_dataframe, create_document,isDataFrame) 
+      
+    return response
+
+    #Save the respnse dataframe to an Excel file in hostinger so that the user can download it
+    #save_file_path = "dataframe_output.xlsx"
+    #response_dataframe.to_excel(save_file_path, index=False)
+    #save_file_to_hostinger(save_file_path)
+    
+    # Check if the response is a DataFrame
+    """if isinstance(response_dataframe, pd.DataFrame):
+        # Convert DataFrame to HTML for display
+        df_html = response_dataframe.to_html(classes='dataframe', index=False)   
+        print(f"dfhtml:{df_html}")     
+        return df_html"""
+    
+    #return response_dataframe.head(10)#, len(response_dataframe)
+
+def save_file_to_hostinger(save_file_path):
+    from ftplib import FTP
+    # Step 2: FTP server credentials
+    ftp_host = 'ftp.redmindtechnologies.com'  # Replace with your FTP server address
+    ftp_user = 'u852023448.redmindGpt'         # Replace with your FTP username
+    ftp_pass = 'RedMind@505'         # Replace with your FTP password
+    remote_file_path = '/RedMindGPT/output.xlsx'  # Replace with the desired path on the server
+
+    # Create an FTP connection
+    ftp = FTP(ftp_host)
+    ftp.login(ftp_user, ftp_pass)
+
+    # Open the local file and upload it to the server
+    with open(save_file_path, 'rb') as file:
+        ftp.storbinary(f'STOR {remote_file_path}', file)
+
+    print(f'File {save_file_path} uploaded to {remote_file_path} on server.')
+
+    # Close the FTP connection
+    ftp.quit()
+
+def handle_large_dataset(df, create_document,isDataFrame):
+    
+    total_rows = len(df)
+    #print(df)
+    print(f"Total rows: {total_rows}")
+    docstatus = f"Download the complete dataset <a href='https://redmindtechnologies.com/RedMindGPT/output.xlsx' download> here.</a>.There are total of {total_rows} rows."
+    if total_rows < 4000:
+
+        # 1. Limit to first 10 rows
+       
+        
+        # 2. Handle missing values
+        #limited_data.fillna("N/A", inplace=True)
+        # 3. Drop the original first column
+        if len(df.columns) > 1:
+                # Skipping the original first column
+                limited_data = df.head(3)
+                limited_data_without_first_column = limited_data.iloc[:, 1:] 
+        else:
+                limited_data = df.head(20)
+                limited_data_without_first_column = limited_data 
+        #print( "range "+ len(limited_data_without_first_column))
+        # 4. Add SNo (serial number) as the first column, starting from 1
+        if isDataFrame :          
+            
+            limited_data_without_first_column.insert(0, 'SNo', range(1, len(limited_data_without_first_column) + 1))
+        else:
+            
+            limited_data_without_first_column.insert(0, 'SNo', range(1, len(limited_data) + 1))
+        # 3. Save the full dataset to a downloadable file
+        df.to_excel('output_data.xlsx', index=False)
+        save_file_to_hostinger('output_data.xlsx')
+        # 4. Create a summary and table of the first 10 rows for display
+    
+        #columns = list(df.columns)
+        sample_table = limited_data_without_first_column.to_markdown()
+        #print(sample_table)
+        if create_document:
+            #Logic to generate pdfs with employee name and account number
+            for index, row in df.iterrows():
+                # Create a PDF for each row
+                create_pdf(row['Account Name'], row['Account ID'])
+            create_document = False
+            docstatus += f" {total_rows} documents are created successfully."
+        print(sample_table)
+        # 5. Return the summary and downloadable link
+        #return f"""
+        #There are a total of {total_rows} rows. Please download the complete dataset here: <a href="https://redmindtechnologies.com/RedMindGPT/output.xlsx" download>Download</a>. Here are the first 3 rows:
+        #{sample_table} """
+
+        return sample_table, docstatus
+
+    else:
+        return "Your query returns a large dataset which is not supported in the current version. Please try a different query."
+def create_pdf(name,id):
+   
+
+    filled = FormWrapper("Goldman_LOA - Gold.pdf").fill(
+        {
+            "Title of Account": name,
+            "Account Number": id,
+            "Print Name and Title": name
+        },
+    )
+    output_file_name =  f"documents\\{name}.pdf"
+    with open(output_file_name, "wb+") as output:
+        output.write(filled.read())
+    return f"{output_file_name} is created successfully."
+    
 
-# CSS for styling the buttons and other elements
 css = """
+
 /* Example of custom button styling */
 .gr-button {
     background-color: #6366f1; /* Change to your desired button color */
@@ -822,7 +1219,6 @@ css = """
     padding: 10px 20px;
     font-size: 12px;
     cursor: pointer;
-   
 }
 
 .gr-button:hover {
@@ -837,14 +1233,12 @@ css = """
     padding: 10px 20px;
     font-size: 14px;
     cursor: pointer;
-    
 }
 
 .gr-buttonbig:hover {
     background-color: #8a92f7; /* Darker shade on hover */
 }
 
-
 /* Customizing the Logout link to be on the right */
 .logout-link {
     text-align: right;
@@ -857,57 +1251,106 @@ css = """
     text-decoration: none;
     font-size: 16px;
 }
+
 .chatbot_gpt {
-   /* width: 800px !important;   Adjust width as needed */
     height: 600px !important; /* Adjust height as needed */
 }
+
 .logout-link a:hover {
     text-decoration: underline; /* Underline on hover */
 }
+
 .message-buttons-right{
     display: none !important; 
 }
+
 body, .gradio-container {
     margin: 0;
     padding: 0;
 }
-"""
 
+/* Styling the tab header with a blue background */
+.gr-tab-header {
+    background-color: #4A90E2; /* Blue background for the tab header */
+    padding: 10px;
+    border-radius: 8px;
+    color: white;
+    font-size: 16px;
+}
+
+/* Styling the selected tab text color to be green */
+.gr-tab-header .gr-tab-active {
+    color: green; /* Change selected tab text to green */
+}
+
+/* Keep non-selected tab text color white */
+.gr-tab-header .gr-tab {
+    color: white;
+}
+
+/* Custom CSS for reducing the size of the video element */
+.video-player {
+    width: 500px;  /* Set a custom width for the video */
+    height: 350px; /* Set a custom height for the video */
+    margin: 0 auto; /* Center the video horizontally */
+}
+"""
 with gr.Blocks(css=css, theme=gr.themes.Soft()) as demo:
     gr.HTML("<CENTER><B><h1 style='font-size:30px; font-family: Calibri;'>RedMindGPT</h1></B></CENTER>")
+    # Logout link styled as text link in the right corner
+    gr.Markdown("<div class='logout-link'><a href='/logout'><b>Logout</b></a></div>")
+
+    # Unified RedMindGPT Interface
     with gr.Row():
         m = gr.Markdown()
         demo.load(update_message, None, m)
-        
-        # Logout link styled as text link in the right corner
-        gr.Markdown("<div class='logout-link'><a href='/logout'><b>Logout</b></a></div>")
-        
+
+    # Buttons for sample queries
     with gr.Row():
-        sample_button = gr.Button("What are the details of ASN24091600002",elem_classes="gr-buttonbig")
-        sample_button1 = gr.Button("What are the active warehouses available",elem_classes="gr-buttonbig")
-        sample_button2 = gr.Button("Explain Pre-Receiving Yard Management",elem_classes="gr-buttonbig")
-        sample_button3 = gr.Button("Can you generate a pie chart with item names and quantities in warehouse WH1000001",elem_classes="gr-buttonbig")
+        sample_button = gr.Button("What are the details of ASN24091600002", elem_classes="gr-buttonbig")
+        sample_button1 = gr.Button("What are the active warehouses available", elem_classes="gr-buttonbig")
+        sample_button2 = gr.Button("Explain Pre-Receiving Yard Management", elem_classes="gr-buttonbig")
+        sample_button3 = gr.Button("Can you generate a pie chart with item names and quantities in warehouse WH1000001", elem_classes="gr-buttonbig")
         sample_button4 = gr.Button("Analyze item name & quantity for different customers in a stacked bar chart for the warehouse WH1000001 & send email to meetarun@gmail.com", elem_classes="gr-button")
 
+    # Chatbot component
     with gr.Row():
-        chatbot = gr.Chatbot(label="Select any of the questions listed above to experience RedmindGPT in action.",elem_classes="chatbot_gpt")
+        chatbot = gr.Chatbot(label="Select any of the questions listed above to experience RedMindGPT in action.", elem_classes="chatbot_gpt")
 
+    # Textbox for user questions
     with gr.Row():
-        with gr.Column(scale=2):
+        with gr.Column(scale=1):
             message = gr.Textbox(show_label=False, container=False, placeholder="Please enter your question")
-           
+            
             with gr.Row():
                 feedback_textbox = gr.Textbox(visible=False, show_label=False, container=False, placeholder="Please enter your feedback.")
-                submit_feedback_button = gr.Button("Submit Feedback", visible=False,elem_classes="gr-buttonbig")
+                submit_feedback_button = gr.Button("Submit Feedback", visible=False, elem_classes="gr-buttonbig")
         with gr.Column(scale=1):
             with gr.Row():
-                button = gr.Button("Submit", elem_id="submit",elem_classes="gr-buttonbig")
-                # Button to stop the current processing
+                button = gr.Button("Submit", elem_id="submit", elem_classes="gr-buttonbig")
                 stop_button = gr.Button("Stop", elem_classes="gr-buttonbig")
-                #gr.ClearButton(message, elem_classes="gr-buttonbig")
-
-    stop_button.click(stop_processing, [chatbot],[chatbot])
+    # Rearranged to place Upload Doc and Upload Excel in the same row
+    with gr.Row():
+        with gr.Column(scale=1):
+            # File Upload Section
+            gr.Markdown("**Add a document or Excel for natural language interaction.**")
+        with gr.Column(scale=1):    
+            u = gr.UploadButton("Upload a doc/excel", file_count="single", elem_classes="gr-buttonbig")
+        #excel_file = gr.UploadButton("Upload an excel", file_count="single", elem_classes="gr-buttonbig", file_types=[".xlsx", ".xls"])
+        with gr.Column(scale=1):
+            add_button = gr.Button("Add to RedMindGPT", elem_classes="gr-buttonbig", visible=False)
+    with gr.Row():
+        title_textbox = gr.Textbox(label="Title", visible=False)
+        summary_textarea = gr.Textbox(label="Summary", lines=5, visible=False)
+        
     
+    output_message = gr.Markdown()  # Markdown to display output message
+    success_message = gr.Markdown()  # Placeholder for messages
+
+   
+    # Moved function calling lines to the end
+    stop_button.click(stop_processing, [chatbot], [chatbot])
+
     button.click(handle_query, [message, chatbot], [chatbot])
     message.submit(handle_query, [message, chatbot], [chatbot])
     message.submit(lambda x: gr.update(value=""), None, [message], queue=False)
@@ -917,14 +1360,15 @@ with gr.Blocks(css=css, theme=gr.themes.Soft()) as demo:
     submit_feedback_button.click(submit_feedback, [feedback_textbox, chatbot], [chatbot, feedback_textbox, submit_feedback_button])
     submit_feedback_button.click(lambda x: gr.update(value=''), [], [feedback_textbox])
 
-    #sample_button.click(answer_question, [sample_button, chatbot], [chatbot])
     sample_button.click(handle_query, [sample_button, chatbot], [chatbot])
     sample_button1.click(handle_query, [sample_button1, chatbot], [chatbot])
     sample_button2.click(handle_query, [sample_button2, chatbot], [chatbot])
     sample_button3.click(handle_query, [sample_button3, chatbot], [chatbot])
     sample_button4.click(handle_query, [sample_button4, chatbot], [chatbot])
 
-    #demo.unload(lambda: send_mail_with_history(chatbot))
-    demo.title = "RedmindGPT"
-#user_details for login page
-demo.launch(auth=[("lakshmi", "redmind"), ("arun", "redmind"), ("NewageGlobal", "Newage123$")], auth_message= "          RedmindGPT",inline=False)
+    u.upload(upload_file, u, [title_textbox, summary_textarea])
+    u.upload(lambda _: (gr.update(visible=True), gr.update(visible=True), gr.update(visible=True)), None, [title_textbox, summary_textarea, add_button])
+    add_button.click(add_to_redmindgpt, [title_textbox, summary_textarea], output_message)
+    add_button.click(lambda _: (gr.update(visible=False), gr.update(visible=False), gr.update(visible=False)), None, [title_textbox, summary_textarea, add_button])
+    
+demo.launch(auth=[("lakshmi", "redmind"), ("arun", "redmind"), ("NewageGlobal", "Newage123$")], auth_message="RedMindGPT", inline=False)