Upload 8 files

Pipfile

@@ -0,0 +1,20 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+pypdf = "*"
+langchain = "*"
+pinecone-client = "*"
+python-dotenv = "*"
+openai = "*"
+tiktoken = "*"
+streamlit = "*"
+typing = "*"
+streamlit-chat = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.9"

app.py

@@ -0,0 +1,85 @@
+from typing import Set
+
+from backend.core import run_llm
+import streamlit as st
+from streamlit_chat import message
+from PIL import Image
+from io import BytesIO
+import base64
+
+# def add_bg_from_local(image_file):
+#     with open(image_file, "rb") as image_file:
+#         encoded_string = base64.b64encode(image_file.read())
+#     st.markdown(
+#     f"""
+#     <style>
+#     .stApp {{
+#         background-image: url(data:{"jpeg"};base64,{encoded_string.decode()});
+#         background-size: cover
+#     }}
+#     </style>
+#     """,
+#     unsafe_allow_html=True
+#     )
+# background_image = "bg2.jpeg"
+# add_bg_from_local(background_image)
+
+
+st.header("Get to know Anuj !!!")
+
+if "user_prompt_history" not in st.session_state:
+    st.session_state["user_prompt_history"] = []
+
+if "chat_answers_history" not in st.session_state:
+    st.session_state["chat_answers_history"] = []
+
+if "chat_history" not in st.session_state:
+    st.session_state["chat_history"] = []
+
+
+def create_sources_string(source_urls: Set[str]) -> str:
+    if not source_urls:
+        return ""
+    sources_list = list(source_urls)
+    sources_list.sort()
+    sources_string = "sources:\n"
+    for i, source in enumerate(sources_list):
+        sources_string += f"{i+1}. {source}\n"
+    return sources_string
+
+
+prompt = st.text_input("Prompt", placeholder="Enter your question here (eg. Who is Anuj Mahajan, What is his work experience ?)...")
+
+
+if prompt:
+    with st.spinner("Generating response..."):
+        generated_response = run_llm(
+            query=prompt, chat_history=st.session_state["chat_history"]
+        )
+        sources = set(
+            [doc.metadata["source"] for doc in generated_response["source_documents"]]
+        )
+        formatted_response = (
+            f"{generated_response['answer']} \n\n {create_sources_string(sources)}"
+        )
+
+        st.session_state.user_prompt_history.append(prompt)
+        st.session_state.chat_answers_history.append(formatted_response)
+        st.session_state.chat_history.append((prompt, generated_response["answer"]))
+
+if st.session_state["chat_answers_history"]:
+    for generated_response, user_query in zip(
+        st.session_state["chat_answers_history"],
+        st.session_state["user_prompt_history"],
+    ):
+        message(
+            user_query,
+            is_user=True,
+            avatar_style="adventurer",
+            seed=123,
+        )
+        # message(generated_response)
+        st.write(
+            f'<div style="word-wrap: break-word;">{generated_response}</div>',
+            unsafe_allow_html=True,
+        )

backend/core.py

@@ -0,0 +1,28 @@
+import os
+from typing import Any, Dict, List
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.chat_models import ChatOpenAI
+from langchain.chains import ConversationalRetrievalChain
+from langchain.vectorstores import Pinecone
+import pinecone
+
+from dotenv import load_dotenv
+
+from consts import INDEX_NAME
+load_dotenv()
+
+pinecone.init(
+    api_key=os.environ.get("PINECONE_API_KEY"),
+    environment=os.environ.get("PINECONE_ENVIRONMENT_REGION"),
+)
+
+def run_llm(query, chat_history):
+    embeddings = OpenAIEmbeddings()
+    docSearch = Pinecone.from_existing_index(index_name = INDEX_NAME, embedding = embeddings)
+    chat = ChatOpenAI(verbose = True, temperature = 0)
+    
+    qa = ConversationalRetrievalChain.from_llm(llm = chat, retriever = docSearch.as_retriever(), return_source_documents = True)
+
+    return qa({"question" : query, "chat_history" : chat_history})
+
+

consts.py

@@ -0,0 +1 @@
+INDEX_NAME = "pdf-parser"

dataIngestionFromPdfToPinecone.py

@@ -0,0 +1,38 @@
+import os
+
+from langchain.document_loaders import PyPDFLoader
+from langchain.text_splitter import CharacterTextSplitter, RecursiveCharacterTextSplitter
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.chains import RetrievalQA
+from langchain.llms import OpenAI
+
+from langchain.vectorstores import Pinecone
+import pinecone
+from dotenv import load_dotenv
+
+from consts import INDEX_NAME
+
+load_dotenv()
+
+pinecone.init(
+    api_key=os.environ.get("PINECONE_API_KEY"),
+    environment=os.environ.get("PINECONE_ENVIRONMENT_REGION"),
+)
+
+def ingestDataFromPdfIntoPinecone():
+    print('Reading Data from PDF')
+    pdf_path = "/Users/anujmahajan/Desktop/Anuj Documents/Resume/PDF/Amazon/Anuj Mahajan - IUB MS CS - CV.pdf"
+    loader = PyPDFLoader(file_path=pdf_path)
+    documents = loader.load()
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=1000, chunk_overlap=30, separators=["\n\n", "\n", " ", ""]
+    )
+    documents = text_splitter.split_documents(documents=documents)
+
+    print(f"Going to insert {len(documents)} to Pinecone")
+    embeddings = OpenAIEmbeddings()
+    Pinecone.from_documents(documents, embeddings, index_name=INDEX_NAME)
+    print("****** Added to Pinecone vectorstore vectors")
+
+if __name__ == "__main__":
+    ingestDataFromPdfIntoPinecone()

requirements.txt

@@ -0,0 +1,84 @@
+aiohttp==3.9.1
+aiosignal==1.3.1
+altair==5.2.0
+annotated-types==0.6.0
+anyio==4.2.0
+async-timeout==4.0.3
+attrs==23.2.0
+blinker==1.7.0
+cachetools==5.3.2
+certifi==2023.11.17
+charset-normalizer==3.3.2
+click==8.1.7
+dataclasses-json==0.6.3
+distro==1.9.0
+dnspython==2.4.2
+exceptiongroup==1.2.0
+frozenlist==1.4.1
+gitdb==4.0.11
+GitPython==3.1.40
+greenlet==3.0.3
+h11==0.14.0
+httpcore==1.0.2
+httpx==0.26.0
+idna==3.6
+importlib-metadata==6.11.0
+Jinja2==3.1.2
+jsonpatch==1.33
+jsonpointer==2.4
+jsonschema==4.20.0
+jsonschema-specifications==2023.12.1
+langchain==0.0.354
+langchain-community==0.0.8
+langchain-core==0.1.5
+langsmith==0.0.77
+loguru==0.7.2
+markdown-it-py==3.0.0
+MarkupSafe==2.1.3
+marshmallow==3.20.1
+mdurl==0.1.2
+multidict==6.0.4
+mypy-extensions==1.0.0
+numpy==1.26.3
+openai==1.6.1
+packaging==23.2
+pandas==2.1.4
+pillow==10.2.0
+pinecone-client==2.2.4
+protobuf==4.25.1
+pyarrow==14.0.2
+pydantic==2.5.3
+pydantic_core==2.14.6
+pydeck==0.8.1b0
+Pygments==2.17.2
+pypdf==3.17.4
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+pytz==2023.3.post1
+PyYAML==6.0.1
+referencing==0.32.0
+regex==2023.12.25
+requests==2.31.0
+rich==13.7.0
+rpds-py==0.16.2
+six==1.16.0
+smmap==5.0.1
+sniffio==1.3.0
+SQLAlchemy==2.0.25
+streamlit==1.29.0
+streamlit-chat==0.1.1
+tenacity==8.2.3
+tiktoken==0.5.2
+toml==0.10.2
+toolz==0.12.0
+tornado==6.4
+tqdm==4.66.1
+typing==3.7.4.3
+typing-inspect==0.9.0
+typing_extensions==4.9.0
+tzdata==2023.4
+tzlocal==5.2
+urllib3==2.1.0
+validators==0.22.0
+yarl==1.9.4
+zipp==3.17.0