Updated Agent RAG

.gitignore

@@ -1 +1,2 @@
 __pycache__/
+venv/

app.py

@@ -11,9 +11,16 @@ from aimakerspace.openai_utils.embedding import EmbeddingModel
 from aimakerspace.vectordatabase import VectorDatabase
 from aimakerspace.openai_utils.chatmodel import ChatOpenAI
 import chainlit as cl
+import tempfile
+import pandas as pd
+import pdfplumber
 
 system_template = """\
-Use the following context to answer a users question. If you cannot find the answer in the context, say you don't know the answer."""
+Use the following context to answer the user's question. If you cannot find the answer in the context, 
+say you don't know the answer. Additionally, if the user requests a summary or context overview, 
+generate an engaging and concise summary that captures the main ideas with an interesting and appealing tone.
+
+"""
 system_role_prompt = SystemRolePrompt(system_template)
 
 user_prompt_template = """\
@@ -49,74 +56,107 @@ class RetrievalAugmentedQAPipeline:
 
 text_splitter = CharacterTextSplitter()
 
-
 def process_text_file(file: AskFileResponse):
-    import tempfile
-
-    with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as temp_file:
+    with tempfile.NamedTemporaryFile(mode="wb", delete=False, suffix=".txt") as temp_file:
         temp_file_path = temp_file.name
-
-    with open(temp_file_path, "wb") as f:
-        f.write(file.content)
+        temp_file.write(file.content)
 
     text_loader = TextFileLoader(temp_file_path)
     documents = text_loader.load_documents()
     texts = text_splitter.split_texts(documents)
     return texts
 
+def process_pdf_file(file: AskFileResponse):
+    with tempfile.NamedTemporaryFile(mode="wb", delete=False, suffix=".pdf") as temp_file:
+        temp_file_path = temp_file.name
+        temp_file.write(file.content)
+
+    extracted_text = ""
+    with pdfplumber.open(temp_file_path) as pdf:
+        for page in pdf.pages:
+            extracted_text += page.extract_text()
+
+    texts = text_splitter.split_texts([extracted_text])
+    return texts
+
+def process_csv_file(file: AskFileResponse):
+    with tempfile.NamedTemporaryFile(mode="wb", delete=False, suffix=".csv") as temp_file:
+        temp_file_path = temp_file.name
+        temp_file.write(file.content)
+
+    df = pd.read_csv(temp_file_path)
+    texts = df.apply(lambda row: ' '.join(row.astype(str)), axis=1).tolist()
+    return text_splitter.split_texts(texts)
 
 @cl.on_chat_start
 async def on_chat_start():
-    files = None
-
-    # Wait for the user to upload a file
-    while files == None:
-        files = await cl.AskFileMessage(
-            content="Please upload a Text File file to begin!",
-            accept=["text/plain"],
-            max_size_mb=2,
-            timeout=180,
-        ).send()
-
-    file = files[0]
-
-    msg = cl.Message(
-        content=f"Processing `{file.name}`...", disable_human_feedback=True
-    )
-    await msg.send()
+    cl.user_session.set("all_texts", [])
+
+    files = await cl.AskFileMessage(
+        content="Please upload one or more Text, PDF, or CSV files to begin!",
+        accept=["text/plain", "application/pdf", "text/csv"],
+        max_size_mb=20,
+        timeout=180,
+    ).send()
+
+    if not files:
+        await cl.Message(content="No files were uploaded. Please upload at least one file to proceed.").send()
+        return
+
+    all_texts = cl.user_session.get("all_texts", [])
 
-    # load the file
-    texts = process_text_file(file)
+    for file in files:
+        file_type = file.name.split(".")[-1].lower()
 
-    print(f"Processing {len(texts)} text chunks")
+        msg = cl.Message(content=f"Processing `{file.name}`...", disable_human_feedback=True)
+        await msg.send()
 
-    # Create a dict vector store
-    vector_db = VectorDatabase()
-    vector_db = await vector_db.abuild_from_list(texts)
-    
-    chat_openai = ChatOpenAI()
+        # Process each file based on its type
+        if file_type == "txt":
+            texts = process_text_file(file)
+        elif file_type == "pdf":
+            texts = process_pdf_file(file)
+        elif file_type == "csv":
+            texts = process_csv_file(file)
+        else:
+            await cl.Message(content=f"Unsupported file type: `{file.name}`. Please upload text, PDF, or CSV files.").send()
+            continue
 
-    # Create a chain
-    retrieval_augmented_qa_pipeline = RetrievalAugmentedQAPipeline(
-        vector_db_retriever=vector_db,
-        llm=chat_openai
-    )
-    
-    # Let the user know that the system is ready
-    msg.content = f"Processing `{file.name}` done. You can now ask questions!"
-    await msg.update()
+        all_texts.extend(texts)  # Combine texts from all uploaded files
 
-    cl.user_session.set("chain", retrieval_augmented_qa_pipeline)
+    cl.user_session.set("all_texts", all_texts)
 
+    await cl.Message(content="Files processed! You can now start asking questions.").send()
 
 @cl.on_message
 async def main(message):
     chain = cl.user_session.get("chain")
 
+    if not chain:
+        all_texts = cl.user_session.get("all_texts")
+        if not all_texts:
+            await cl.Message(content="Please upload at least one file before asking questions.").send()
+            return
+
+        # Create a dict vector store
+        vector_db = VectorDatabase()
+        vector_db = await vector_db.abuild_from_list(all_texts)
+        
+        chat_openai = ChatOpenAI()
+
+        # Create a chain
+        retrieval_augmented_qa_pipeline = RetrievalAugmentedQAPipeline(
+            vector_db_retriever=vector_db,
+            llm=chat_openai
+        )
+
+        cl.user_session.set("chain", retrieval_augmented_qa_pipeline)
+        chain = retrieval_augmented_qa_pipeline
+
     msg = cl.Message(content="")
     result = await chain.arun_pipeline(message.content)
 
     async for stream_resp in result["response"]:
         await msg.stream_token(stream_resp)
 
-    await msg.send()
+    await msg.send()

chainlit.md

@@ -1,3 +1,17 @@
-# Welcome to Chat with Your Text File
+# ✨🔮 Dive into Data Magic with Chat-to-Chart! 📊💬
 
-With this application, you can chat with an uploaded text file that is smaller than 2MB!
+Welcome to the ultimate AI-powered assistant, designed especially for **Business Owners**, **Stakeholders**, **CEOs**, and other **non-technical professionals**! 💼🚀
+
+Seamlessly interact with your data by uploading:
+- 📄 **Text Files** (under 2MB)
+- 📚 **PDFs**
+- 📊 **CSV Files**
+
+🔗 **Connect directly to your database** and let the insights flow! Whether you need:
+- 🧠 **Instant answers** and insights from your files or database
+- 🔍 **Effortless SQL querying** in plain language
+- 📈 **Dynamic charts** (bar, line, pie) to visualize your data
+
+This tool is all about transforming complex data into clear, actionable insights — as easy as having a conversation! 🌐🤖
+
+It’s not just data analysis; it’s **Chat-to-Chart**! 📊💬 Upload your files, connect your database, and watch the magic happen. ✨

venv/Scripts/activate

@@ -35,7 +35,7 @@ deactivate () {
 # unset irrelevant variables
 deactivate nondestructive
 
-VIRTUAL_ENV="D:\DataTicon\AIE4 Pythonic RAG\AIE4-DeployPythonicRAG\venv"
+VIRTUAL_ENV="D:\DataTicon\AIE4 Pythonic RAG - Copy\AIE4-DeployPythonicRAG\venv"
 export VIRTUAL_ENV
 
 _OLD_VIRTUAL_PATH="$PATH"

venv/Scripts/activate.bat

@@ -8,7 +8,7 @@ if defined _OLD_CODEPAGE (
     "%SystemRoot%\System32\chcp.com" 65001 > nul
 )
 
-set VIRTUAL_ENV=D:\DataTicon\AIE4 Pythonic RAG\AIE4-DeployPythonicRAG\venv
+set VIRTUAL_ENV=D:\DataTicon\AIE4 Pythonic RAG - Copy\AIE4-DeployPythonicRAG\venv
 
 if not defined PROMPT set PROMPT=$P$G
 

venv/pyvenv.cfg

@@ -2,4 +2,4 @@ home = C:\Users\USER\anaconda3
 include-system-site-packages = false
 version = 3.11.7
 executable = C:\Users\USER\anaconda3\python.exe
-command = C:\Users\USER\anaconda3\python.exe -m venv D:\DataTicon\AIE4 Pythonic RAG\AIE4-DeployPythonicRAG\venv
+command = C:\Users\USER\anaconda3\python.exe -m venv D:\DataTicon\AIE4 Pythonic RAG - Copy\AIE4-DeployPythonicRAG\venv