|
import streamlit as st |
|
from langchain_community.document_loaders import WebBaseLoader |
|
from langchain.text_splitter import CharacterTextSplitter, RecursiveCharacterTextSplitter |
|
from langchain_community.vectorstores import Chroma |
|
from langchain_nomic.embeddings import NomicEmbeddings |
|
|
|
from langchain_community.llms import HuggingFaceHub |
|
from langchain_community.embeddings import HuggingFaceEmbeddings |
|
from bs4 import BeautifulSoup |
|
|
|
from langchain_core.runnables import RunnablePassthrough |
|
from langchain_core.output_parsers import StrOutputParser |
|
from langchain_core.prompts import ChatPromptTemplate |
|
|
|
|
|
|
|
def method_get_website_text(urls): |
|
urls_list = urls.split("\n") |
|
docs = [WebBaseLoader(url).load() for url in urls_list] |
|
docs_list = [item for sublist in docs for item in sublist] |
|
return docs_list |
|
|
|
|
|
def method_get_text_chunks(text): |
|
|
|
text_splitter = RecursiveCharacterTextSplitter(chunk_size=7500, chunk_overlap=100) |
|
doc_splits = text_splitter.split_documents(text) |
|
return doc_splits |
|
|
|
|
|
def method_get_vectorstore(document_chunks): |
|
embeddings = HuggingFaceEmbeddings() |
|
|
|
|
|
|
|
vector_store = Chroma.from_documents(document_chunks, embeddings) |
|
return vector_store |
|
|
|
|
|
def get_context_retriever_chain(vector_store,question): |
|
|
|
retriever = vector_store.as_retriever() |
|
|
|
|
|
after_rag_template = """Answer the question based only on the following context: |
|
{context} |
|
Question: {question} |
|
""" |
|
|
|
|
|
after_rag_prompt = ChatPromptTemplate.from_template(after_rag_template) |
|
|
|
|
|
llm = HuggingFaceHub(repo_id="mistralai/Mistral-7B-Instruct-v0.2", model_kwargs={"temperature":0.6, "max_length":1024}) |
|
|
|
|
|
after_rag_chain = ( |
|
{"context": retriever, "question": RunnablePassthrough()} |
|
| after_rag_prompt |
|
| llm |
|
| StrOutputParser() |
|
) |
|
|
|
return after_rag_chain.invoke(question) |
|
|
|
def main(): |
|
st.set_page_config(page_title="Chat with websites", page_icon="🤖") |
|
st.title("Chat with websites") |
|
|
|
|
|
with st.sidebar: |
|
st.header("Settings") |
|
website_url = st.text_input("Website URL") |
|
|
|
if website_url is None or website_url == "": |
|
st.info("Please enter a website URL") |
|
|
|
else: |
|
|
|
question = st.text_input("Question") |
|
|
|
|
|
if st.button('Query Documents'): |
|
with st.spinner('Processing...'): |
|
|
|
raw_text = method_get_website_text(website_url) |
|
|
|
doc_splits = method_get_text_chunks(raw_text) |
|
|
|
vector_store = method_get_vectorstore(doc_splits) |
|
|
|
answer = get_context_retriever_chain(vector_store,question) |
|
|
|
split_string = "Question: " + str(question) |
|
result = answer.split(split_string)[-1] |
|
st.text_area("Answer", value=result, height=300, disabled=True) |
|
|
|
if __name__ == '__main__': |
|
main() |