File size: 4,363 Bytes
3573cc9 ec90e85 c49b9cc 3573cc9 ec90e85 3573cc9 520da56 3573cc9 520da56 3573cc9 520da56 3573cc9 c241fe7 f89d622 f5808e4 2733653 f89d622 3573cc9 f89d622 3573cc9 520da56 3573cc9 c241fe7 f435a3d 3573cc9 520da56 3573cc9 520da56 3573cc9 528cec9 520da56 3573cc9 528cec9 3573cc9 047284f 3573cc9 047284f 3573cc9 047284f 3573cc9 047284f 520da56 047284f 3573cc9 520da56 3573cc9 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 |
import streamlit as st
from dotenv import load_dotenv
from PyPDF2 import PdfReader
from langchain.text_splitter import CharacterTextSplitter
from langchain_community.embeddings import OpenAIEmbeddings, HuggingFaceInstructEmbeddings
from langchain_community.vectorstores import FAISS
from langchain_community.chat_models import ChatOpenAI
from langchain.memory import ConversationBufferMemory
from langchain.chains import ConversationalRetrievalChain
from htmlTemplates import css, bot_template, user_template, hide_st_style, footer
from langchain_community.llms import HuggingFaceHub
from matplotlib import style
def get_pdf_text(pdf_docs):
text = ""
for pdf in pdf_docs:
pdf_reader = PdfReader(pdf)
for page in pdf_reader.pages:
text += page.extract_text()
return text
def get_text_chunks(text):
text_splitter = CharacterTextSplitter(
separator="\n",
chunk_size=1000,
chunk_overlap=200,
length_function=len
)
chunks = text_splitter.split_text(text)
return chunks
def get_vectorstore(text_chunks):
# embeddings = OpenAIEmbeddings()
print("HuggingFaceInstructEmbeddings")
model_kwargs = {'device': 'cpu', 'weights_only': True}
# embeddings = HuggingFaceInstructEmbeddings(model_name="hkunlp/instructor-xl", model_kwargs=model_kwargs)
embeddings = HuggingFaceInstructEmbeddings(model_name="hkunlp/instructor-xl")
print("FAISS.from_texts")
vectorstore = FAISS.from_texts(texts=text_chunks, embedding=embeddings)
print("returning vectorstore")
return vectorstore
def get_conversation_chain(vectorstore):
# llm = ChatOpenAI()
llm = HuggingFaceHub(repo_id="google/flan-t5-base", model_kwargs={"temperature":0.5, "max_length":512})
memory = ConversationBufferMemory(
memory_key='chat_history', return_messages=True)
conversation_chain = ConversationalRetrievalChain.from_llm(
llm=llm,
retriever=vectorstore.as_retriever(),
memory=memory
)
return conversation_chain
def handle_userinput(user_question):
if st.session_state.conversation is None:
st.error("Please upload PDF data before starting the chat.")
return
response = st.session_state.conversation({'question': user_question})
st.session_state.chat_history = response['chat_history']
for i, message in enumerate(st.session_state.chat_history):
if i % 2 == 0:
st.write(user_template.replace(
"{{MSG}}", message.content), unsafe_allow_html=True)
else:
st.write(bot_template.replace(
"{{MSG}}", message.content), unsafe_allow_html=True)
def main():
load_dotenv()
st.set_page_config(page_title="Talk with PDF",
page_icon="icon.png")
st.write(css, unsafe_allow_html=True)
if "conversation" not in st.session_state:
st.session_state.conversation = None
if "chat_history" not in st.session_state:
st.session_state.chat_history = None
st.header("Chat with AI with Custom Data 🚀")
user_question = st.text_input("Ask a question about your Data:")
with st.sidebar:
st.subheader("Your documents")
pdf_docs = st.file_uploader(
"Upload your Data here in PDF format and click on 'Process'", accept_multiple_files=True, type=['pdf'])
if st.button("Process"):
if pdf_docs is None:
st.error("Please upload at least one PDF file.")
else:
with st.spinner("Processing"):
print("get_pdf_text")
raw_text = get_pdf_text(pdf_docs)
print("get_text_chunks")
text_chunks = get_text_chunks(raw_text)
print("get_vectorstore")
vectorstore = get_vectorstore(text_chunks)
print("get_conversation_chain")
st.session_state.conversation = get_conversation_chain(
vectorstore)
print("success")
st.success("Your Data has been processed successfully")
if user_question:
handle_userinput(user_question)
st.markdown(hide_st_style, unsafe_allow_html=True)
st.markdown(footer, unsafe_allow_html=True)
if __name__ == '__main__':
main()
|