Upload 25 files

data/parsed_data.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0a86749f9c9e30411fdd0846439e91c7e23eb925cedd20c715f2200a7f4184d
+size 850770

media/ouput_file.mp3

@@ -0,0 +1 @@
+

src/OCR.py

@@ -0,0 +1,85 @@
+import numpy as np
+import pandas as pd
+import easyocr
+import streamlit as st
+from PIL import Image
+import cv2
+from utils.qa import chain
+from langchain.memory import ConversationBufferWindowMemory
+from langchain_community.chat_message_histories import StreamlitChatMessageHistory
+
+def I_OCR():
+    # Function to display the OCR image with bounding boxes and text
+    def display_ocr_image(img, results):
+        img_np = np.array(img)
+        for detection in results:
+            top_left = tuple([int(val) for val in detection[0][0]])
+            bottom_right = tuple([int(val) for val in detection[0][2]])
+            text = detection[1]
+            font = cv2.FONT_HERSHEY_COMPLEX
+            cv2.rectangle(img_np, top_left, bottom_right, (0, 255, 0), 5)
+            cv2.putText(img_np, text, top_left, font, 1, (125, 29, 241), 2, cv2.LINE_AA)
+        st.image(img_np, channels="BGR", use_column_width=True)
+
+    # Function to extract text from DataFrame column
+    def extracted_text(col):
+        return " , ".join(img_df[col])
+
+    # Function to initialize session state
+    def initialize_session_state():
+        if "messages" not in st.session_state:
+            st.session_state.messages = [
+                {"role": "assistant", "content": "Hi! How may I assist you today?"}
+            ]
+
+    # Function to get answer from QA model
+    def get_answer(query):
+        response = chain.invoke(query)
+        return response["result"]
+
+    # Streamlit app
+    st.title("Question in image")
+
+    file = st.file_uploader(label= "Upload Image Here (png/jpg/jpeg) : ", type=['png', 'jpg', 'jpeg'])
+
+    if file is not None:
+        image = Image.open(file)
+        st.image(image)
+
+        reader = easyocr.Reader(['en', 'hi'], gpu=False)
+        results = reader.readtext(np.array(image))
+
+        img_df = pd.DataFrame(results, columns=['bbox', 'Predicted Text', 'Prediction Confidence'])
+
+        text_combined = extracted_text(col='Predicted Text')
+        st.write("Text Generated :- ", text_combined)
+
+        display_ocr_image(image, results)
+
+    else:
+        st.warning("!! Please Upload your image !!")
+
+    initialize_session_state()
+
+    memory_storage = StreamlitChatMessageHistory(key="chat_messages")
+    memory = ConversationBufferWindowMemory(memory_key="chat_history", human_prefix="User", chat_memory=memory_storage, k=3)
+
+    for i, msg in enumerate(memory_storage.messages):
+        name = "user" if i % 2 == 0 else "assistant"
+        st.chat_message(name).markdown(msg.content)
+
+    if user_input := st.chat_input("User Input"):
+        with st.chat_message("user"):
+            st.markdown(user_input)
+
+        with st.spinner("Generating Response..."):
+            with st.chat_message("assistant"):
+                response = get_answer(user_input)
+                answer = response
+                st.markdown(answer)
+
+    #if st.sidebar.button("Clear Chat History"):
+    #    memory_storage.clear()
+
+# Run the OCR function
+#OCR()

src/about.py

@@ -0,0 +1,29 @@
+virtual_tutor_markdown = """
+# Virtual Tutor Project
+
+Welcome to the Virtual Tutor Project! Our mission is to revolutionize education through personalized and interactive virtual tutoring experiences.
+
+## Introduction
+The Virtual Tutor Project utilizes cutting-edge technology, including Large Language Models (LLMs), to create a dynamic learning environment tailored to each student's needs.
+
+## Key Features
+- **Personalized Learning:** Our virtual tutor adapts to the individual learning pace and style of each student, providing customized learning experiences.
+- **Interactive Sessions:** Engaging lessons with interactive elements such as quizzes, games, and simulations enhance learning retention.
+- **Real-time Feedback:** Immediate feedback on assignments and assessments helps students track their progress and identify areas for improvement.
+- **24/7 Availability:** Accessible anytime, anywhere, our virtual tutor ensures learning continuity and flexibility.
+- **Comprehensive Subjects:** Covering a wide range of subjects and topics, from mathematics and science to languages and humanities, catering to diverse educational needs.
+- **OCR for Questionnaires:** Utilize Optical Character Recognition (OCR) technology to facilitate the processing of questionnaires for improved assessment and feedback.
+- **Audio-to-Audio Query Window:** Incorporate an audio-to-audio query window feature, enabling students to ask questions verbally and receive audio responses from the virtual tutor.
+
+## Benefits
+- **Enhanced Learning Outcomes:** Personalized learning experiences foster deeper understanding and improved academic performance.
+- **Convenience and Flexibility:** Students can learn at their own pace and schedule, eliminating barriers to education.
+- **Engagement and Motivation:** Interactive lessons and real-time feedback keep students engaged and motivated to learn.
+- **Accessibility:** The virtual tutor provides access to quality education to students worldwide, regardless of geographical location or socioeconomic background.
+
+## Get in Touch
+Have questions or feedback? Feel free to contact us at [virtual-tutor@vt.com](mailto:naresh.is21@bmsce.ac.in).
+
+## Join the Virtual Tutor Revolution!
+Experience the future of education with our Virtual Tutor Project. Start your journey to academic success today!
+"""

src/audio.py

@@ -0,0 +1,72 @@
+import streamlit as st
+import base64
+from audio_recorder_streamlit import audio_recorder
+from streamlit_float import *
+from utils.stt import speech_to_text
+from utils.tts import text_to_speech
+from utils.qa import chain
+
+recorded_audio = r".\media\recorded.mp3"
+output_audio = r".\media\ouput_file.mp3"
+
+def autoplay_audio(file_path: str):
+    with open(file_path, "rb") as f:
+        data = f.read()
+    b64 = base64.b64encode(data).decode("utf-8")
+    md = f"""
+    <audio autoplay>
+    <source src="data:audio/mp3;base64,{b64}" type="audio/mp3">
+    </audio>
+    """
+    st.markdown(md, unsafe_allow_html=True)
+
+def get_answer(query):
+    response = chain.invoke(query)
+    return response['result']
+
+def audio_d():
+    float_init()
+
+    st.title("Ai Doubt resolver")
+
+    # Initialize session state
+    if "messages" not in st.session_state:
+        st.session_state.messages = [
+            {"role": "assistant", "content": "Hi! How may I assist you today?"}
+        ]
+
+    footer_container = st.container()
+
+    with footer_container:
+        audio_bytes = audio_recorder()
+
+    for message in st.session_state.messages:
+        with st.chat_message(message["role"]):
+            st.write(message["content"])
+
+    if audio_bytes:
+        with st.spinner("Transcribing..."):
+            webm_file_path = recorded_audio
+            with open(webm_file_path, "wb") as f:
+                f.write(audio_bytes)
+
+            transcript = speech_to_text()
+            if transcript:
+                st.session_state.messages.append({"role": "user", "content": transcript})
+                with st.chat_message("user"):
+                    st.write(transcript)
+
+    if st.session_state.messages[-1]["role"] != "assistant":
+        with st.chat_message("assistant"):
+            with st.spinner("Thinking🤔..."):
+                final_response = get_answer(str(st.session_state.messages))
+            with st.spinner("Generating audio response..."):    
+                text_to_speech(final_response)
+                audio_file = output_audio
+                autoplay_audio(audio_file)
+            st.write(final_response)
+            st.session_state.messages.append({"role": "assistant", "content": final_response})
+            os.remove(audio_file)
+
+    # Float the footer container and provide CSS to target it with
+    footer_container.float("bottom: 0rem;")

src/chainlit.md

@@ -0,0 +1,27 @@
+# Virtual Tutor Project
+
+Welcome to the Virtual Tutor Project! Our mission is to revolutionize education through personalized and interactive virtual tutoring experiences.
+
+## Introduction
+The Virtual Tutor Project utilizes cutting-edge technology, including Large Language Models (LLMs), to create a dynamic learning environment tailored to each student's needs.
+
+## Key Features
+- **Personalized Learning:** Our virtual tutor adapts to the individual learning pace and style of each student, providing customized learning experiences.
+- **Interactive Sessions:** Engaging lessons with interactive elements such as quizzes, games, and simulations enhance learning retention.
+- **Real-time Feedback:** Immediate feedback on assignments and assessments helps students track their progress and identify areas for improvement.
+- **24/7 Availability:** Accessible anytime, anywhere, our virtual tutor ensures learning continuity and flexibility.
+- **Comprehensive Subjects:** Covering a wide range of subjects and topics, from mathematics and science to languages and humanities, catering to diverse educational needs.
+- **OCR for Questionnaires:** Utilize Optical Character Recognition (OCR) technology to facilitate the processing of questionnaires for improved assessment and feedback.
+- **Audio-to-Audio Query Window:** Incorporate an audio-to-audio query window feature, enabling students to ask questions verbally and receive audio responses from the virtual tutor.
+
+## Benefits
+- **Enhanced Learning Outcomes:** Personalized learning experiences foster deeper understanding and improved academic performance.
+- **Convenience and Flexibility:** Students can learn at their own pace and schedule, eliminating barriers to education.
+- **Engagement and Motivation:** Interactive lessons and real-time feedback keep students engaged and motivated to learn.
+- **Accessibility:** The virtual tutor provides access to quality education to students worldwide, regardless of geographical location or socioeconomic background.
+
+## Get in Touch
+Have questions or feedback? Feel free to contact us at [naresh.is21@bmsce.ac.in](mailto:naresh.is21@bmsce.ac.in).
+
+## Join the Virtual Tutor Revolution!
+Experience the future of education with our Virtual Tutor Project. Start your journey to academic success today!

src/chat.py

@@ -0,0 +1,76 @@
+import streamlit as st
+import base64
+from utils.qa import chain
+from langchain.memory import ConversationBufferWindowMemory
+from langchain_community.chat_message_histories import StreamlitChatMessageHistory
+
+def virtual_tutor():
+    #st.set_page_config(layout='wide')
+    #st.set_page_config(page_title="Virtual Tutor")
+
+    st.markdown("""
+        <svg width="600" height="100">
+            <text x="50%" y="50%" font-family="San serif" font-size="42px" fill="Black" text-anchor="middle" stroke="white"
+             stroke-width="0.3" stroke-linejoin="round">Virtual Tutor - CHAT
+            </text>
+        </svg>
+    """, unsafe_allow_html=True)
+
+    def add_bg_from_local(image_file):
+        with open(image_file, "rb") as image_file:
+            encoded_string = base64.b64encode(image_file.read())
+        st.markdown(
+            f"""<style>.stApp {{background-image: url(data:image/{"png"};base64,{encoded_string.decode()});
+            background-size: cover}}</style>""",
+            unsafe_allow_html=True)
+
+    #add_bg_from_local(r'C:\Users\Naresh Kumar Lahajal\Desktop\Capstone-streamlit\STREAMCHAT\freepik-export-20240425023906eVmL.jpeg')
+
+    def initialize_session_state():
+        if "messages" not in st.session_state:
+            st.session_state.messages = [
+                {"role": "assistant", "content": "Hi! How may I assist you today?"}
+            ]
+
+    initialize_session_state()
+
+    m = st.markdown("""
+        <style> 
+        .stChatInputContainer > div {
+        background-color: #000000;
+        }
+        </style>
+        """, unsafe_allow_html=True)
+
+    def get_answer(query):
+        response = chain.invoke(query)
+        return response
+
+    memory_storage = StreamlitChatMessageHistory(key="chat_messages")
+    memory = ConversationBufferWindowMemory(memory_key="chat_history", human_prefix="User", chat_memory=memory_storage, k=3)
+
+    for message in st.session_state.messages:  # Display the prior chat messages
+        with st.chat_message(message["role"]):
+            st.write(message["content"])
+
+    for i, msg in enumerate(memory_storage.messages):
+        name = "user" if i % 2 == 0 else "assistant"
+        st.chat_message(name).markdown(msg.content)
+
+    if user_input := st.chat_input("User Input"):
+        with st.chat_message("user"):
+            st.markdown(user_input)
+
+        with st.spinner("Generating Response..."):
+
+            with st.chat_message("assistant"):
+                response = get_answer(user_input)
+                answer = response['result']
+                st.markdown(answer)
+                message = {"role": "assistant", "content": answer}
+                message_u = {"role": "user", "content": user_input}
+                st.session_state.messages.append(message_u)
+                st.session_state.messages.append(message)
+
+
+#virtual_tutor()

src/pdf_up.py

@@ -0,0 +1,41 @@
+import tempfile
+import streamlit as st
+from PIL import Image
+import os
+from utils.ingest1 import create_vector_database
+
+def process_uploaded_file():
+    st.title("Upload File to Chat")
+    uploaded_file = st.file_uploader("File upload", type="pdf")
+    if uploaded_file:
+        temp_dir = tempfile.mkdtemp()
+        path = os.path.join(temp_dir, uploaded_file.name)
+    #with open(path, "wb") as f:
+    #    f.write(uploaded_file.getvalue())
+        print(path)    
+        st.write("Document uploaded successfully!")
+        # Display the uploaded document
+        st.write("Preview of the document:")
+        st.write(uploaded_file)
+
+        # Button to start parsing and vector database creation
+        if st.button("Start Processing"):
+            # Placeholder for processing logic
+            st.write("Processing...")
+
+            # Placeholder for progress bar
+            with st.spinner('Processing...'):
+                # Call your function to parse data and create vector database
+                create_vector_database(path)
+
+            st.success("Processing completed!")
+
+            # Display success message
+            st.write("Vector database created successfully!")
+
+            # Show success image
+            success_image = Image.open("success_image.jpg")
+            st.image(success_image, caption="Success!", use_column_width=True)
+
+    # Add a footer
+    #st.text("Built with Streamlit")

utils/ingest1.py

@@ -0,0 +1,107 @@
+import os
+import nest_asyncio  # noqa: E402
+nest_asyncio.apply()
+
+# bring in our LLAMA_CLOUD_API_KEY
+from dotenv import load_dotenv
+load_dotenv()
+
+##### LLAMAPARSE #####
+from llama_parse import LlamaParse
+
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.embeddings.fastembed import FastEmbedEmbeddings
+from langchain_community.vectorstores import Qdrant
+from langchain_community.document_loaders import DirectoryLoader
+
+
+llamaparse_api_key = os.getenv("LLAMA_CLOUD_API_KEY")
+qdrant_url = os.getenv("QDRANT_URL")
+qdrant_api_key = os.getenv("QDRANT_API_KEY")
+
+#to_parse_documents = ["./data/example.pdf", "./data/uber_10q_march_2022.pdf"]
+
+parsed_data_file = r".\data\parsed_data.pkl"
+output_md = r".\data\output.md"
+loki = r".\data"
+
+import pickle
+# Define a function to load parsed data if available, or parse if not
+def load_or_parse_data(loc):
+    data_file = parsed_data_file
+    
+    if os.path.exists(data_file):
+        # Load the parsed data from the file
+        with open(data_file, "rb") as f:
+            parsed_data = pickle.load(f)
+    else:
+        # Perform the parsing step and store the result in llama_parse_documents
+        parsingInstructiontest10k = """The provided document is an entry level machine learning textbook with example code and outputs.
+        It contains many images and tables.
+        Try to be precise while answering the questions"""
+        parser = LlamaParse(api_key=llamaparse_api_key, result_type="markdown", parsing_instruction=parsingInstructiontest10k)
+        llama_parse_documents = parser.load_data(loc)
+        
+
+        # Save the parsed data to a file
+        with open(data_file, "wb") as f:
+            pickle.dump(llama_parse_documents, f)
+        
+        # Set the parsed data to the variable
+        parsed_data = llama_parse_documents
+    
+    return parsed_data
+
+
+# Create vector database
+def create_vector_database(loc):
+    """
+    Creates a vector database using document loaders and embeddings.
+
+    This function loads urls,
+    splits the loaded documents into chunks, transforms them into embeddings using OllamaEmbeddings,
+    and finally persists the embeddings into a Chroma vector database.
+
+    """
+    # Call the function to either load or parse the data
+    llama_parse_documents = load_or_parse_data(loc)
+    #print(llama_parse_documents[1].text[:100])
+    
+    #with open('data/output.md', 'a') as f:  # Open the file in append mode ('a')
+      #  for doc in llama_parse_documents:
+       #     f.write(doc.text + '\n')
+    with open(output_md, 'a', encoding='utf-8') as f:  # Open the file in append mode ('a')
+        for doc in llama_parse_documents:
+            f.write(doc.text + '\n')
+
+    loader = DirectoryLoader(loki, glob="**/*.md", show_progress=True)
+    documents = loader.load()
+    # Split loaded documents into chunks
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=100)
+    docs = text_splitter.split_documents(documents)
+    
+    #len(docs)
+    #docs[0]
+    
+    # Initialize Embeddings
+    embeddings = FastEmbedEmbeddings()
+    
+    # Create and persist a Chroma vector database from the chunked documents
+    qdrant = Qdrant.from_documents(
+        documents=docs,
+        embedding=embeddings,
+        url=qdrant_url,
+        collection_name="rag",
+        api_key=qdrant_api_key
+    )
+
+    #query it
+    #query = "what is the agend of Financial Statements for 2022 ?"
+    #found_doc = qdrant.similarity_search(query, k=3)
+    #print(found_doc[0][:100])
+    
+    print('Vector DB created successfully !')
+
+
+if __name__ == "__main__":
+    create_vector_database()

utils/qa.py

@@ -0,0 +1,87 @@
+import os
+from typing import List
+from langchain_groq import ChatGroq
+from langchain.prompts import PromptTemplate
+from langchain_community.vectorstores import Qdrant
+from langchain_community.embeddings.fastembed import FastEmbedEmbeddings
+from qdrant_client import QdrantClient
+#from langchain_community.chat_models import ChatOllama
+
+
+#import chainlit as cl
+from langchain.chains import RetrievalQA
+
+# bring in our GROQ_API_KEY
+from dotenv import load_dotenv
+load_dotenv()
+
+groq_api_key = os.getenv("GROQ_API_KEY")
+qdrant_url = os.getenv("QDRANT_URL")
+qdrant_api_key = os.getenv("QDRANT_API_KEY")
+
+custom_prompt_template = """Use the following pieces of information to answer the user's question.
+If you don't know the answer, just say that you don't know,if it is out of context say that it is out of context and also try to provide the answer and don't be rude.
+
+Context: {context}
+Question: {question}
+
+Only return the helpful answer below and nothing else.
+Helpful answer:
+"""
+
+def set_custom_prompt():
+    """
+    Prompt template for QA retrieval for each vectorstore
+    """
+    prompt = PromptTemplate(template=custom_prompt_template,
+                            input_variables=['context', 'question'])
+    return prompt
+
+
+chat_model = ChatGroq(temperature=0, model_name="mixtral-8x7b-32768")
+#chat_model = ChatGroq(temperature=0, model_name="Llama2-70b-4096")
+#chat_model = ChatOllama(model="llama2", request_timeout=30.0)
+
+client = QdrantClient(api_key=qdrant_api_key, url=qdrant_url,)
+
+
+def retrieval_qa_chain(llm, prompt, vectorstore):
+    qa_chain = RetrievalQA.from_chain_type(
+        llm=llm,
+        chain_type="stuff",
+        retriever=vectorstore.as_retriever(search_kwargs={'k': 2}),
+        return_source_documents=True,
+        chain_type_kwargs={'prompt': prompt}
+    )
+    return qa_chain
+
+
+def qa_bot():
+    embeddings = FastEmbedEmbeddings()
+    vectorstore = Qdrant(client=client, embeddings=embeddings, collection_name="rag")
+    llm = chat_model
+    qa_prompt=set_custom_prompt()
+    qa = retrieval_qa_chain(llm, qa_prompt, vectorstore)
+    return qa
+
+#---------------------------------------------------------------------#
+ 
+#qdrant_cloud_api_key="your_qdrant_cloud_api_key"
+#qdrant_url="your_qdrant_url"
+
+#qdrant_cloud = Qdrant.from_documents(
+#    docs,
+#   embeddings,
+#   url=qdrant_url,
+#   prefer_grpc=True,
+#   api_key=qdrant_cloud_api_key,
+#  collection_name="qdrant_cloud_documents",
+#)
+
+#---------------------------------------------------------------------#
+query="how to make coffee"
+print(query)
+
+chain = qa_bot()
+
+

utils/stt.py

@@ -0,0 +1,45 @@
+# main.py (python example)
+
+import os
+from dotenv import load_dotenv
+
+from deepgram import (
+    DeepgramClient,
+    PrerecordedOptions,
+    FileSource,
+)
+
+load_dotenv()
+
+# Path to the audio file
+AUDIO_FILE = r".\media\recorded.mp3"
+API_KEY = os.getenv("DG_API_KEY")
+
+
+def speech_to_text():
+    try:
+        # STEP 1 Create a Deepgram client using the API key
+        deepgram = DeepgramClient(API_KEY)
+
+        with open(AUDIO_FILE, "rb") as file:
+            buffer_data = file.read()
+
+        payload: FileSource = {
+            "buffer": buffer_data,
+        }
+
+        #STEP 2: Configure Deepgram options for audio analysis
+        options = PrerecordedOptions(
+            model="nova-2",
+            smart_format=True,
+        )
+
+        # STEP 3: Call the transcribe_file method with the text payload and options
+        response = deepgram.listen.prerecorded.v("1").transcribe_file(payload, options)
+        # STEP 4: Print the response
+        #print(response.to_json(indent=4))
+        #print(response["results"]["channels"][0]["alternatives"][0]["transcript"])
+        return str(response["results"]["channels"][0]["alternatives"][0]["transcript"])
+
+    except Exception as e:
+        print(f"Exception: {e}")

utils/tts.py

@@ -0,0 +1,39 @@
+import requests
+import os
+from dotenv import load_dotenv
+import json
+
+load_dotenv()
+API_KEY = os.getenv("DG_API_KEY")
+AUDIO_FILE=r".\media\ouput_file.mp3"
+
+def text_to_speech(llm_response):
+    # Define the API endpoint
+    url = "https://api.deepgram.com/v1/speak?model=aura-asteria-en"
+
+    # Define the headers
+    headers = {
+        "Authorization": f"Token {API_KEY}",
+        "Content-Type": "application/json"
+    }
+
+    # Define the payload
+    payload = {
+        "text": llm_response
+    }
+
+    # Make the POST request
+    response = requests.post(url, headers=headers, json=payload)
+
+    # Check if the request was successful
+    if response.status_code == 200:
+        # Save the response content to a file
+        with open(AUDIO_FILE, "wb") as f:
+            f.write(response.content)
+        print("File saved successfully.")
+    else:
+        print(f"Error: {response.status_code} - {response.text}")
+
+# Example usage
+#transcribed_text = "Hello, how can I help you today?"
+#tts(transcribed_text)