Spaces:
Paused
Paused
File size: 2,050 Bytes
bd2cf7b 8421d14 9359dde bd2cf7b 8421d14 bd2cf7b 8421d14 bd2cf7b 8421d14 bd2cf7b 8421d14 ea780f0 8421d14 9359dde bd2cf7b 9359dde 8421d14 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 |
import streamlit as st
import pickle
import os
import torch
from tqdm.auto import tqdm
from langchain.text_splitter import RecursiveCharacterTextSplitter
# from langchain.vectorstores import Chroma
from langchain.vectorstores import FAISS
from langchain.embeddings import HuggingFaceInstructEmbeddings
from langchain import HuggingFacePipeline
from langchain.chains import RetrievalQA
st.set_page_config(
page_title = 'aitGPT',
page_icon = '✅')
st.markdown("# Hello")
@st.cache_data
def load_scraped_web_info():
with open("/Users/carlosito/Library/CloudStorage/OneDrive-Personal/AIT material/99-AIT-thesis/aitGPT/ait-web-document", "rb") as fp:
ait_web_documents = pickle.load(fp)
text_splitter = RecursiveCharacterTextSplitter(
# Set a really small chunk size, just to show.
chunk_size = 500,
chunk_overlap = 100,
length_function = len,
)
chunked_text = text_splitter.create_documents([doc for doc in tqdm(ait_web_documents)])
st.markdown(f"Number of Documents: {len(ait_web_documents)}")
st.markdown(f"Number of chunked texts: {len(chunked_text)}")
@st.cache_resource
def load_embedding_model():
embedding_model = HuggingFaceInstructEmbeddings(model_name='hkunlp/instructor-base',
model_kwargs = {'device': torch.device('cuda' if torch.cuda.is_available() else 'cpu')})
return embedding_model
@st.cache_data
def load_faiss_index():
vector_database = FAISS.load_local("faiss_index", embedding_model)
return vector_database
#--------------
load_scraped_web_info()
embedding_model = load_embedding_model()
vector_database = load_faiss_index()
print("load done")
query_input = st.text_input(label= 'your question')
def retrieve_document(query_input):
related_doc = vector_database.similarity_search(query_input)
return related_doc
output = st.text_area(label = "Here is the relevant documents",
value = retrieve_document(query_input))
|