initial commit

.env

@@ -0,0 +1 @@
+OPENAI_API_KEY=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

.gitattributes

@@ -0,0 +1,42 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.tsv filter=lfs diff=lfs merge=lfs -text
+data/ filter=lfs diff=lfs merge=lfs -text
+vectorstore/ filter=lfs diff=lfs merge=lfs -text
+documents/ filter=lfs diff=lfs merge=lfs -text
+*.faiss filter=lfs diff=lfs merge=lfs -text
+document/* filter=lfs diff=lfs merge=lfs -text
+*.pdf filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Rajat Bansal
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,29 @@
+---
+title: Publisher Chatbot 50k
+emoji: 📈
+colorFrom: purple
+colorTo: blue
+sdk: gradio
+sdk_version: 4.31.5
+app_file: rag.py
+pinned: false
+---
+
+## Steps for running rag -:
+1. Create .env file in root folder and add the following environment variables
+```
+OPENAI_API_KEY=<YOUR OPENAI KEY>
+```
+2. Run the following commands:
+```
+    pip3 install -r requirements.txt
+    python3 rag.py
+``` 
+
+
+Neo4j RAG course - https://www.deeplearning.ai/short-courses/knowledge-graphs-rag/
+1. langchain documentation retreiver - https://python.langchain.com/v0.1/docs/modules/data_connection/retrievers/vectorstore/
+2. https://medium.com/@shaktikanungo2019/3. conversational-ai-unveiling-the-first-rag-chatbot-with-langchain-8b9b04ee4b63
+3. https://medium.com/@vikrambhat2/building-a-rag-system-and-conversational-chatbot-with-custom-data-793e9617a865
+4. https://abvijaykumar.medium.com/retrieval-augmented-generation-rag-with-llamaindex-1828ef80314c
+5. https://medium.com/the-ai-forum/implementing-agentic-rag-using-langchain-b22af7f6a3b5

backupcodes/llamaIndexSingleQuery.py

@@ -0,0 +1,32 @@
+from llama_index.core import (
+    VectorStoreIndex,
+    SimpleDirectoryReader,
+    StorageContext,
+    load_index_from_storage,
+)
+from llama_index.llms.openai import OpenAI
+from llama_index.core.settings import Settings
+import os
+from dotenv import load_dotenv
+
+load_dotenv(override=True)
+Settings.llm = OpenAI(model="gpt-3.5-turbo", temperature=0.1)
+
+storage_path = "../vectorstore"
+documents_path = "../../documents"
+
+
+def initialize():
+    if not os.path.exists(storage_path):
+        documents = SimpleDirectoryReader(documents_path).load_data()
+        index = VectorStoreIndex.from_documents(documents)
+        index.storage_context.persist(persist_dir=storage_path)
+    else:
+        storage_context = StorageContext.from_defaults(persist_dir=storage_path)
+        index = load_index_from_storage(storage_context)
+    return index
+index = initialize()
+
+chat_engine = index.as_chat_engine(chat_mode="condense_question", verbose=True)
+response = chat_engine.chat("hi tell me what i can ask you")
+print(response.response)

backupcodes/pdfreader.py

@@ -0,0 +1,18 @@
+docs = []
+metadata = []
+
+# Read PDF documents from the given path
+pdf_docs = [os.path.join(data_path, f) for f in os.listdir(data_path) if f.endswith('.pdf')]
+for pdf_path in pdf_docs:
+    with open(pdf_path, "rb") as pdf_file:
+        pdf_reader = PyPDF2.PdfReader(pdf_file)
+        for index, page in enumerate(pdf_reader.pages):
+            doc_page = {
+                "title": os.path.basename(pdf_path) + " page " + str(index + 1),
+                "content": page.extract_text(),
+            }
+            docs.append(doc_page)
+
+content = [doc["content"] for doc in docs]
+metadata = [{"title": doc["title"]} for doc in docs]
+print("Content and metadata are extracted from the documents")

backupcodes/streamlist_LLAMA.py

@@ -0,0 +1,154 @@
+import PyPDF2
+import streamlit as st
+from dotenv import load_dotenv
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.chains.conversational_retrieval.base import ConversationalRetrievalChain
+from langchain_community.vectorstores import FAISS
+from langchain.memory import ConversationBufferMemory
+from langchain.prompts import PromptTemplate
+
+# from langchain_community.llms import llamacpp
+# from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_openai import OpenAIEmbeddings
+from langchain_openai import ChatOpenAI
+import os
+
+llmtemplate = """[INST]
+As an AI, provide accurate and relevant information based on the provided document. Your responses should adhere to the following guidelines:
+- Answer the question based on the provided documents.
+- Be direct and factual, limited to 50 words and 2-3 sentences. Begin your response without using introductory phrases like yes, no etc.
+- Maintain an ethical and unbiased tone, avoiding harmful or offensive content.
+- Avoid using confirmatory phrases like "Yes, you are correct" or any similar validation in your responses.
+- Do not fabricate information or include questions in your responses.
+- do not prompt to select answers. do not ask me questions
+{question}
+
+[/INST]
+"""
+prompt_template = """Use the following pieces of context and previous questions and answers to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer.
+
+{context}
+
+Previous Q&A: {previous_qa}
+
+Question: {question}
+Helpful Answer:"""
+
+# PDF_MODEL_NAME = "sentence-transformers/all-MiniLM-L6-v2"
+# LLAMA_MODEL_PATH = "llama-2-7b-chat.Q4_K_M.gguf"
+DB_FAISS_PATH = "../vectorstore/db_faiss"
+CHUNK_SIZE = 512
+CHUNK_OVERLAP = 256
+SIMILARITY_THRESHOLD = 0.5
+
+
+def prepare_db(pdf_docs):
+    embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
+    if not os.path.exists(DB_FAISS_PATH):
+        docs = []
+        metadata = []
+        content = []
+
+        for pdf in pdf_docs:
+            pdf_reader = PyPDF2.PdfReader(pdf)
+            for index, page in enumerate(pdf_reader.pages):
+                doc_page = {
+                    "title": pdf.name + " page " + str(index + 1),
+                    "content": page.extract_text(),
+                }
+                docs.append(doc_page)
+        for doc in docs:
+            content.append(doc["content"])
+            metadata.append({"title": doc["title"]})
+        print("Content and metadata are extracted from the documents")
+        text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
+            chunk_size=CHUNK_SIZE,
+            chunk_overlap=CHUNK_OVERLAP,
+        )
+        split_docs = text_splitter.create_documents(content, metadatas=metadata)
+        print(f"Documents are split into {len(split_docs)} passages")
+        # embeddings = HuggingFaceEmbeddings(
+        #     model_name=PDF_MODEL_NAME,
+        #     model_kwargs={"device": "cpu"},
+        # )
+        db = FAISS.from_documents(split_docs, embeddings)
+        print(f"Document saved in db")
+        db.save_local(DB_FAISS_PATH)
+    else:
+        print(f"Db already exists")
+        db = FAISS.load_local("./vectorstore/db_faiss", embeddings, allow_dangerous_deserialization=True)
+    return db
+
+
+def get_conversation_chain(vectordb):
+    conversation_chain = ConversationalRetrievalChain.from_llm(
+        # llm=llamacpp.LlamaCpp(
+        #     model_path=LLAMA_MODEL_PATH,
+        #     temperature=0.75,
+        #     max_tokens=200,
+        #     top_p=1,
+        #     n_ctx=3000,
+        #     verbose=False,
+        # ),
+        llm = ChatOpenAI(model="gpt-3.5-turbo"),
+        retriever=vectordb.as_retriever(),
+        condense_question_prompt= PromptTemplate.from_template(llmtemplate),
+        
+        memory=ConversationBufferMemory(
+            memory_key="chat_history", return_messages=True, output_key="answer"
+        ),
+        return_source_documents=True,
+    )
+    print("Conversation chain created")
+    return conversation_chain
+
+
+# def validate_answer_against_sources(response_answer, source_documents):
+#     model = SentenceTransformer(PDF_MODEL_NAME)
+#     source_texts = [doc.page_content for doc in source_documents]
+#     answer_embedding = model.encode(response_answer, convert_to_tensor=True)
+#     source_embeddings = model.encode(source_texts, convert_to_tensor=True)
+#     cosine_scores = util.pytorch_cos_sim(answer_embedding, source_embeddings)
+#     if any(score.item() > SIMILARITY_THRESHOLD for score in cosine_scores[0]):
+#         return True
+#     return False
+
+
+def handle_userinput(user_question):
+    response = st.session_state.conversation({"question": user_question})
+    st.session_state.chat_history = response["chat_history"]
+
+    for i, message in enumerate(st.session_state.chat_history):
+        template = "<div style='color: blue;'>{{MSG}}</div>"
+        if i%2 != 0:
+            template = "<div style='color: green;'>{{MSG}}</div>"
+        st.write(
+            template.replace("{{MSG}}", str(i) + ': ' + message.content),
+            unsafe_allow_html=True,
+        )
+
+
+def main():
+    load_dotenv(override=True)
+    st.set_page_config(page_title="Chat with your PDFs", page_icon=":books:")
+    if "conversation" not in st.session_state:
+        st.session_state.conversation = None
+    if "chat_history" not in st.session_state:
+        st.session_state.chat_history = []
+    st.header("Chat with multiple PDFs :books:")
+    user_question = st.text_input("Ask a question about your documents:")
+    if user_question:
+        handle_userinput(user_question)
+    with st.sidebar:
+        st.subheader("Your documents")
+        pdf_docs = st.file_uploader(
+            "Upload your PDFs here and click on 'Process'", accept_multiple_files=True
+        )
+        if st.button("Process"):
+            with st.spinner("Processing"):
+                vectorstore = prepare_db(pdf_docs)
+                # print(vectorstore.similarity_search("Tell me about add and adhd"))
+                st.session_state.conversation = get_conversation_chain(vectorstore)
+
+if __name__ == "__main__":
+    main()

data/131_webmd_vogon_sample1000_urlsContent_cleaned.tsv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ece612a60d11e19ca95d1ae6af58e6c968d98b014459c8afbd298b57afae4cf4
+size 157522

data/132_webmd_vogon_urlsContent_cleaned.tsv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb0b5985d1bc8610d28603f556c91194c2f755939dc235bdf5d3b77524f10b05
+size 197658055

documents/ADD and ADHD (Attention Deficit Hyperactivity Disorder) Health Center.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:872fa6d12d05e40655b03de827150868f2233828843e5363e89fcfdb2dd66c57
+size 2362135

documents/Media.net — WebMD - Better information. Better health..pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bec7051b94443c051b75f1aade0904b3d413c7ca11420787b93d152512dd7487
+size 5966263

documents/WebMD Allergies Health Center - Find allergy information and latest health news.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d1591c0ddd6b184c3cfc4514d5613017a7a9054dbc7021ee4777a8a9eca0608
+size 1367936

documents/WebMD Arthritis and Joint Pain Center: Symptoms, Causes, Tests, and Treatments.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a9c402c5f7c8d8ae0c02dc6c36426d1cf2ed538c5d8ae3e41203e1fed4d095c
+size 672808

documents/WebMD Health News Center - The latest breaking health news and alerts.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:119e2f7dd9de99275ac13d7ba06f7984dc43c7b0e44006360ef784e3b44076e7
+size 1921177

rag.py

@@ -0,0 +1,134 @@
+from dotenv import load_dotenv
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.vectorstores import FAISS
+# from langchain_openai import OpenAIEmbeddings
+from langchain_community.embeddings import HuggingFaceEmbeddings
+import os
+import pandas as pd
+
+import gradio as gr
+from openai import OpenAI
+
+load_dotenv(override=True)
+client = OpenAI()
+DB_FAISS_PATH = "./vectorstore/db_faiss_50k"
+data_file_path = "./data/132_webmd_vogon_urlsContent_cleaned.tsv"
+
+# DB_FAISS_PATH = "./vectorstore/db_faiss_10"
+# data_file_path = "./data/131_webmd_vogon_sample1000_urlsContent_cleaned.tsv"
+
+CHUNK_SIZE = 512
+CHUNK_OVERLAP = 128
+# embedding_model_oa = "text-embedding-3-small"
+embedding_model_hf = "BAAI/bge-m3"
+# embedding_model_hf = "sentence-transformers/all-mpnet-base-v2"
+qa_model_name = "gpt-3.5-turbo"
+bestReformulationPrompt = "Given a chat history and the latest user question, which may reference context from the chat history, you must formulate a standalone question that can be understood without the chat history. You are strictly forbidden from using any outside knowledge. Do not, under any circumstances, answer the question. Reformulate it if necessary; otherwise, return it as is."
+bestSystemPrompt = "You're an assistant for question-answering tasks. Under absolutely no circumstances should you use external knowledge or go beyond the provided preknowledge. Your approach must be systematic and meticulous. First, identify CLUES such as keywords, phrases, contextual information, semantic relations, tones, and references that aid in determining the context of the input. Second, construct a concise diagnostic REASONING process (limiting to 130 words) based on premises supporting the INPUT relevance within the provided context. Third, utilizing the identified clues, reasoning, and input, furnish the pertinent answer for the question. Remember, you are required to use ONLY the provided context to answer the questions. If the question does not align with the preknowledge or if the preknowledge is absent, state that you don't know the answer. External knowledge is strictly prohibited. Failure to adhere will result in incorrect answers. The preknowledge is as follows:"
+
+# embeddings_oa = OpenAIEmbeddings(model=embedding_model_oa)
+embeddings_hf = HuggingFaceEmbeddings(model_name = embedding_model_hf, show_progress = True)
+
+def setupDb(data_path):
+    df = pd.read_csv(data_path, sep="\t")
+    relevant_content = df["url"].values
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=CHUNK_SIZE,
+        chunk_overlap=CHUNK_OVERLAP,
+    )
+
+    if not os.path.exists(DB_FAISS_PATH):
+        split_docs = text_splitter.create_documents(
+            df["url_content"].tolist(),
+            metadatas=[
+                {"title": row["url_title"], "url": row["url"]}
+                for _, row in df.iterrows()
+            ],
+        )
+        print(f"Documents are split into {len(split_docs)} passages")
+
+        db = FAISS.from_documents(split_docs, embeddings_hf)
+        print(f"Document saved in db")
+        db.save_local(DB_FAISS_PATH + "/index_1")
+    else:
+        print(f"Db already exists")
+        db = FAISS.load_local(
+            DB_FAISS_PATH, embeddings_hf, allow_dangerous_deserialization=True
+        )
+    return db, relevant_content
+
+def reformulate_question(chat_history, latest_question, reformulationPrompt):
+    system_message = {
+        "role": "system",
+        "content": reformulationPrompt
+    }
+
+    formatted_history = []
+    for i, chat in enumerate(chat_history):
+        formatted_history.append({"role": "user", "content": chat[0]})
+        formatted_history.append({"role": "assistant", "content": chat[1]})
+    # print("History -------------->", formatted_history)
+
+    formatted_history.append({"role": "user", "content": latest_question})
+    response = client.chat.completions.create(
+        model="gpt-3.5-turbo",
+        messages=[system_message] + formatted_history,
+        temperature=0
+    )
+
+    reformulated_question = response.choices[0].message.content
+    return reformulated_question
+
+def getQuestionAnswerOnTheBasisOfContext(question, context, systemPrompt):
+    system_message = {
+        "role": "system",
+        "content": systemPrompt + context
+    }
+
+    response = client.chat.completions.create(
+        model=qa_model_name,
+        messages=[system_message] + [{"role": "user", "content": question}],
+        temperature=0
+    )
+    answer = response.choices[0].message.content
+    return answer
+
+
+def chatWithRag(reformulationPrompt, QAPrompt, question):
+    global curr_question_no, chat_history
+    curr_question_prompt = bestSystemPrompt
+    if QAPrompt != None or len(QAPrompt):
+        curr_question_prompt = QAPrompt
+
+    # reformulated_query = reformulate_question(chat_history, question, reformulationPrompt)
+    reformulated_query = question
+    retreived_documents = [doc for doc in db.similarity_search_with_score(reformulated_query) if doc[1] < 1.3]
+    answer = getQuestionAnswerOnTheBasisOfContext(reformulated_query, '. '.join([doc[0].page_content for doc in retreived_documents]), curr_question_prompt)
+    chat_history.append((question, answer))
+    curr_question_no += 1
+    docs_info = "\n\n".join([
+        f"Title: {doc[0].metadata['title']}\nUrl: {doc[0].metadata['url']}\nContent: {doc[0].page_content}\nValue: {doc[1]}" for doc in retreived_documents
+    ])
+    full_response = f"Answer: {answer}\n\nReformulated question: {reformulated_query}\nRetrieved Documents:\n{docs_info}"
+    # print(question, full_response)
+    return full_response
+
+db, relevant_content = setupDb(data_file_path)
+chat_history = []
+curr_question_no = 1
+
+with gr.Blocks() as demo:
+    gr.Markdown("# RAG on webmd")
+    with gr.Row():
+        reformulationPrompt = gr.Textbox(bestReformulationPrompt, lines=1, placeholder="Enter the system prompt for reformulation of query", label="Reformulation System prompt")
+        QAPrompt = gr.Textbox(bestSystemPrompt, lines=1, placeholder="Enter the system prompt for QA.", label="QA System prompt")
+        question = gr.Textbox(lines=1, placeholder="Enter the question asked", label="Question")
+    output = gr.Textbox(label="Output")
+    submit_btn = gr.Button("Submit")
+    submit_btn.click(chatWithRag, inputs=[reformulationPrompt, QAPrompt, question], outputs=output)
+    question.submit(chatWithRag, [reformulationPrompt, QAPrompt, question], [output])
+    with gr.Accordion("Urls", open=False):
+        gr.Markdown(', '.join(relevant_content))
+
+gr.close_all()
+demo.launch()

requirements.txt

@@ -0,0 +1,6 @@
+gradio
+python-dotenv
+langchain
+langchain_community
+langchain_openai
+faiss-cpu

vectorstore/db_faiss_10/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e15f7b6feed6be9100fe75f074fa861e6d80fec0b10ac60f902c6b0980aa280
+size 3704877

vectorstore/db_faiss_10/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f4c9e2ecd619a27680b0ddede61f3745ee764284fab71ad0f95a83487f699f6
+size 372279

vectorstore/db_faiss_50k/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7454ff939aed934f1a9741bda8d9d4ad0962c1d6c22737c5bd9d98f3a91b25e
+size 2059517997

vectorstore/db_faiss_50k/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62959ed49b05c3081c612a3b951c427238098bafc7b7ad74da67e61da6af1167
+size 321147051