|
import streamlit as st |
|
from langchain.embeddings import HuggingFaceInstructEmbeddings |
|
from langchain.vectorstores import FAISS |
|
from langchain.text_splitter import CharacterTextSplitter |
|
from langchain.document_loaders import DirectoryLoader, PyPDFLoader |
|
import os |
|
from PyPDF2 import PdfReader |
|
from transformers import pipeline |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_pdf_text(folder_path): |
|
text = "" |
|
|
|
for filename in os.listdir(folder_path): |
|
filepath = os.path.join(folder_path, filename) |
|
|
|
|
|
if os.path.isfile(filepath) and filename.lower().endswith(".pdf"): |
|
pdf_reader = PdfReader(filepath) |
|
for page in pdf_reader.pages: |
|
text += page.extract_text() |
|
|
|
|
|
return text |
|
|
|
|
|
def get_text_chunks(text): |
|
|
|
text_splitter = CharacterTextSplitter( |
|
separator="\n", |
|
chunk_size=1000, |
|
chunk_overlap=200, |
|
length_function=len |
|
) |
|
chunks = text_splitter.split_text(text) |
|
return chunks |
|
|
|
|
|
def create_vectorstore_and_store(): |
|
folder_path = './files' |
|
pdf_text = get_pdf_text(folder_path) |
|
text_chunks = get_text_chunks(pdf_text) |
|
embeddings = HuggingFaceInstructEmbeddings(model_name="hkunlp/instructor-base") |
|
|
|
|
|
vectorstoreDB = FAISS.from_texts(texts=text_chunks,embedding=embeddings) |
|
|
|
save_directory = "Store" |
|
|
|
vectorstoreDB.save_local(save_directory) |
|
print(vectorstoreDB) |
|
return None |
|
|
|
|
|
|
|
def get_vectorstore(): |
|
embeddings = HuggingFaceInstructEmbeddings(model_name="hkunlp/instructor-base") |
|
|
|
|
|
save_directory = "Store" |
|
vectorstoreDB = FAISS.load_local(save_directory, embeddings) |
|
return vectorstoreDB |
|
|
|
|
|
def (): |
|
|
|
user_question = st.text_area("Stell mir eine Frage: ") |
|
|
|
retriever=get_vectorstore().as_retriever() |
|
retrieved_docs=retriever.invoke( |
|
user_question |
|
) |
|
|
|
return retrieved_docs[0].page_content |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__': |
|
main() |