Delete utils

utils/ingest1.py

@@ -1,107 +0,0 @@
-import os
-import nest_asyncio  # noqa: E402
-nest_asyncio.apply()
-
-# bring in our LLAMA_CLOUD_API_KEY
-from dotenv import load_dotenv
-load_dotenv()
-
-##### LLAMAPARSE #####
-from llama_parse import LlamaParse
-
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain_community.embeddings.fastembed import FastEmbedEmbeddings
-from langchain_community.vectorstores import Qdrant
-from langchain_community.document_loaders import DirectoryLoader
-
-
-llamaparse_api_key = os.getenv("LLAMA_CLOUD_API_KEY")
-qdrant_url = os.getenv("QDRANT_URL")
-qdrant_api_key = os.getenv("QDRANT_API_KEY")
-
-#to_parse_documents = ["./data/example.pdf", "./data/uber_10q_march_2022.pdf"]
-
-parsed_data_file = r"C:\Users\Naresh Kumar Lahajal\Desktop\FINAL\data\parsed_data.pkl"
-output_md = r"C:\Users\Naresh Kumar Lahajal\Desktop\FINAL\data\output.md"
-loki = r"C:\Users\Naresh Kumar Lahajal\Desktop\FINAL\data"
-
-import pickle
-# Define a function to load parsed data if available, or parse if not
-def load_or_parse_data(loc):
-    data_file = parsed_data_file
-    
-    if os.path.exists(data_file):
-        # Load the parsed data from the file
-        with open(data_file, "rb") as f:
-            parsed_data = pickle.load(f)
-    else:
-        # Perform the parsing step and store the result in llama_parse_documents
-        parsingInstructiontest10k = """The provided document is an entry level machine learning textbook with example code and outputs.
-        It contains many images and tables.
-        Try to be precise while answering the questions"""
-        parser = LlamaParse(api_key=llamaparse_api_key, result_type="markdown", parsing_instruction=parsingInstructiontest10k)
-        llama_parse_documents = parser.load_data(loc)
-        
-
-        # Save the parsed data to a file
-        with open(data_file, "wb") as f:
-            pickle.dump(llama_parse_documents, f)
-        
-        # Set the parsed data to the variable
-        parsed_data = llama_parse_documents
-    
-    return parsed_data
-
-
-# Create vector database
-def create_vector_database(loc):
-    """
-    Creates a vector database using document loaders and embeddings.
-
-    This function loads urls,
-    splits the loaded documents into chunks, transforms them into embeddings using OllamaEmbeddings,
-    and finally persists the embeddings into a Chroma vector database.
-
-    """
-    # Call the function to either load or parse the data
-    llama_parse_documents = load_or_parse_data(loc)
-    #print(llama_parse_documents[1].text[:100])
-    
-    #with open('data/output.md', 'a') as f:  # Open the file in append mode ('a')
-      #  for doc in llama_parse_documents:
-       #     f.write(doc.text + '\n')
-    with open(output_md, 'a', encoding='utf-8') as f:  # Open the file in append mode ('a')
-        for doc in llama_parse_documents:
-            f.write(doc.text + '\n')
-
-    loader = DirectoryLoader(loki, glob="**/*.md", show_progress=True)
-    documents = loader.load()
-    # Split loaded documents into chunks
-    text_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=100)
-    docs = text_splitter.split_documents(documents)
-    
-    #len(docs)
-    #docs[0]
-    
-    # Initialize Embeddings
-    embeddings = FastEmbedEmbeddings()
-    
-    # Create and persist a Chroma vector database from the chunked documents
-    qdrant = Qdrant.from_documents(
-        documents=docs,
-        embedding=embeddings,
-        url=qdrant_url,
-        collection_name="rag",
-        api_key=qdrant_api_key
-    )
-
-    #query it
-    #query = "what is the agend of Financial Statements for 2022 ?"
-    #found_doc = qdrant.similarity_search(query, k=3)
-    #print(found_doc[0][:100])
-    
-    print('Vector DB created successfully !')
-
-
-if __name__ == "__main__":
-    create_vector_database()

utils/qa.py

@@ -1,87 +0,0 @@
-import os
-from typing import List
-from langchain_groq import ChatGroq
-from langchain.prompts import PromptTemplate
-from langchain_community.vectorstores import Qdrant
-from langchain_community.embeddings.fastembed import FastEmbedEmbeddings
-from qdrant_client import QdrantClient
-from langchain_community.chat_models import ChatOllama
-
-
-#import chainlit as cl
-from langchain.chains import RetrievalQA
-
-# bring in our GROQ_API_KEY
-from dotenv import load_dotenv
-load_dotenv()
-
-groq_api_key = os.getenv("GROQ_API_KEY")
-qdrant_url = os.getenv("QDRANT_URL")
-qdrant_api_key = os.getenv("QDRANT_API_KEY")
-
-custom_prompt_template = """Use the following pieces of information to answer the user's question.
-If you don't know the answer, just say that you don't know,if it is out of context say that it is out of context and also try to provide the answer and don't be rude.
-
-Context: {context}
-Question: {question}
-
-Only return the helpful answer below and nothing else.
-Helpful answer:
-"""
-
-def set_custom_prompt():
-    """
-    Prompt template for QA retrieval for each vectorstore
-    """
-    prompt = PromptTemplate(template=custom_prompt_template,
-                            input_variables=['context', 'question'])
-    return prompt
-
-
-chat_model = ChatGroq(temperature=0, model_name="mixtral-8x7b-32768")
-#chat_model = ChatGroq(temperature=0, model_name="Llama2-70b-4096")
-#chat_model = ChatOllama(model="llama2", request_timeout=30.0)
-
-client = QdrantClient(api_key=qdrant_api_key, url=qdrant_url,)
-
-
-def retrieval_qa_chain(llm, prompt, vectorstore):
-    qa_chain = RetrievalQA.from_chain_type(
-        llm=llm,
-        chain_type="stuff",
-        retriever=vectorstore.as_retriever(search_kwargs={'k': 2}),
-        return_source_documents=True,
-        chain_type_kwargs={'prompt': prompt}
-    )
-    return qa_chain
-
-
-def qa_bot():
-    embeddings = FastEmbedEmbeddings()
-    vectorstore = Qdrant(client=client, embeddings=embeddings, collection_name="rag")
-    llm = chat_model
-    qa_prompt=set_custom_prompt()
-    qa = retrieval_qa_chain(llm, qa_prompt, vectorstore)
-    return qa
-
-#---------------------------------------------------------------------#
- 
-#qdrant_cloud_api_key="your_qdrant_cloud_api_key"
-#qdrant_url="your_qdrant_url"
-
-#qdrant_cloud = Qdrant.from_documents(
-#    docs,
-#   embeddings,
-#   url=qdrant_url,
-#   prefer_grpc=True,
-#   api_key=qdrant_cloud_api_key,
-#  collection_name="qdrant_cloud_documents",
-#)
-
-#---------------------------------------------------------------------#
-query="how to make coffee"
-print(query)
-
-chain = qa_bot()
-
-

utils/stt.py

@@ -1,45 +0,0 @@
-# main.py (python example)
-
-import os
-from dotenv import load_dotenv
-
-from deepgram import (
-    DeepgramClient,
-    PrerecordedOptions,
-    FileSource,
-)
-
-load_dotenv()
-
-# Path to the audio file
-AUDIO_FILE = r"C:\Users\Naresh Kumar Lahajal\Desktop\FINAL\media\recorded.mp3"
-API_KEY = os.getenv("DG_API_KEY")
-
-
-def speech_to_text():
-    try:
-        # STEP 1 Create a Deepgram client using the API key
-        deepgram = DeepgramClient(API_KEY)
-
-        with open(AUDIO_FILE, "rb") as file:
-            buffer_data = file.read()
-
-        payload: FileSource = {
-            "buffer": buffer_data,
-        }
-
-        #STEP 2: Configure Deepgram options for audio analysis
-        options = PrerecordedOptions(
-            model="nova-2",
-            smart_format=True,
-        )
-
-        # STEP 3: Call the transcribe_file method with the text payload and options
-        response = deepgram.listen.prerecorded.v("1").transcribe_file(payload, options)
-        # STEP 4: Print the response
-        #print(response.to_json(indent=4))
-        #print(response["results"]["channels"][0]["alternatives"][0]["transcript"])
-        return str(response["results"]["channels"][0]["alternatives"][0]["transcript"])
-
-    except Exception as e:
-        print(f"Exception: {e}")

utils/tts.py

@@ -1,38 +0,0 @@
-import requests
-import os
-from dotenv import load_dotenv
-
-load_dotenv()
-API_KEY = os.getenv("DG_API_KEY")
-AUDIO_FILE=r"C:\Users\Naresh Kumar Lahajal\Desktop\FINAL\media\ouput_file.mp3"
-
-def text_to_speech(llm_response):
-    # Define the API endpoint
-    url = "https://api.deepgram.com/v1/speak?model=aura-asteria-en"
-
-    # Define the headers
-    headers = {
-        "Authorization": f"Token {API_KEY}",
-        "Content-Type": "application/json"
-    }
-
-    # Define the payload
-    payload = {
-        "text": llm_response
-    }
-
-    # Make the POST request
-    response = requests.post(url, headers=headers, json=payload)
-
-    # Check if the request was successful
-    if response.status_code == 200:
-        # Save the response content to a file
-        with open(AUDIO_FILE, "wb") as f:
-            f.write(response.content)
-        print("File saved successfully.")
-    else:
-        print(f"Error: {response.status_code} - {response.text}")
-
-# Example usage
-#transcribed_text = "Hello, how can I help you today?"
-#tts(transcribed_text)