Streamlit_Docker_Format_Output

Sleeping

App Files Files Community

Streamlit_Docker_Format_Output / pages /bot.py

ErikH

Update pages/bot.py

a52aa9f 7 months ago

raw history blame contribute delete

No virus

4.79 kB

	import streamlit as st
	from langchain.embeddings import HuggingFaceInstructEmbeddings
	from langchain.vectorstores import FAISS
	from langchain.text_splitter import CharacterTextSplitter
	from langchain.document_loaders import DirectoryLoader, PyPDFLoader
	import os
	from PyPDF2 import PdfReader
	from transformers import pipeline
	from transformers import AutoModel

	#Retriever erweiterung
	from langchain.prompts import ChatPromptTemplate
	from langchain.schema import StrOutputParser
	from langchain.schema.runnable import RunnablePassthrough
	from langchain.chains import ConversationalRetrievalChain
	from langchain.llms import HuggingFaceHub

	###########
	#pip install faiss-cpu
	#pip install langchain
	#pip install pypdf
	#pip tiktoken
	#pip install InstructorEmbedding
	###############


	# PDF in String umwandeln
	def get_pdf_text(folder_path):
	text = ""
	# Durchsuche alle Dateien im angegebenen Verzeichnis
	for filename in os.listdir(folder_path):
	filepath = os.path.join(folder_path, filename)

	# Überprüfe, ob die Datei die Erweiterung ".pdf" hat
	if os.path.isfile(filepath) and filename.lower().endswith(".pdf"):
	pdf_reader = PdfReader(filepath)
	for page in pdf_reader.pages:
	text += page.extract_text()
	#text += '\n'
	translator = pipeline("translation_de_to_en", model="t5-small")
	text=text.replace("\n", " ")
	text=text.replace("- ", "")
	st.text(text)
	text=translator(""+text)
	st.text(text)
	return text

	#Chunks erstellen
	def get_text_chunks(text):
	#Arbeitsweise Textsplitter definieren
	text_splitter = CharacterTextSplitter(
	separator="\n",
	chunk_size=1000,
	chunk_overlap=200,
	length_function=len
	)
	chunks = text_splitter.split_text(text)
	return chunks

	# nur zum Anlegen des lokalen Verzeichnisses "Store" und speichern der Vektor-Datenbank
	def create_vectorstore_and_store():
	folder_path = './files'
	pdf_text = get_pdf_text(folder_path)
	text_chunks = get_text_chunks(pdf_text)
	embeddings = HuggingFaceInstructEmbeddings(model_name="deutsche-telekom/bert-multi-english-german-squad2")
	#embeddings = HuggingFaceInstructEmbeddings(model_name="aari1995/German_Semantic_STS_V2")
	# Initiate Faiss DB
	vectorstoreDB = FAISS.from_texts(texts=text_chunks,embedding=embeddings)#texts=text_chunks,
	# Verzeichnis in dem die VektorDB gespeichert werden soll
	save_directory = "Store"
	#VektorDB lokal speichern
	vectorstoreDB.save_local(save_directory)
	print(vectorstoreDB)
	return None

	########

	def get_vectorstore():
	embeddings = HuggingFaceInstructEmbeddings(model_name="deutsche-telekom/bert-multi-english-german-squad2")
	#embeddings = HuggingFaceInstructEmbeddings(model_name="aari1995/German_Semantic_STS_V2")
	#Abruf lokaler Vektordatenbank
	save_directory = "Store"
	vectorstoreDB = FAISS.load_local(save_directory, embeddings)
	return vectorstoreDB

	######


	#####
	def main():
	#if os.path.exists("./Store"): #Nutzereingabe nur eingelesen, wenn vectorstore angelegt
	user_question = st.text_area("Stell mir eine Frage: ")
	#if os.path.exists("./Store"): #Nutzereingabe nur eingelesen, wenn vectorstore angelegt
	retriever=get_vectorstore().as_retriever()
	retrieved_docs=retriever.invoke(
	user_question
	)
	if user_question:

	question=user_question
	st.text(user_question)
	context=""+retrieved_docs[0].page_content+retrieved_docs[1].page_content+retrieved_docs[3].page_content
	context=context.replace("\n", " ")
	context=context.replace("- ", "")
	st.text("Das ist der Textausschnitt der durch den Retriever herausgesucht wird:")
	st.text(context)

	# Erstelle die Question Answering-Pipeline für Deutsch
	qa_pipeline = pipeline("question-answering", model="deutsche-telekom/bert-multi-english-german-squad2", tokenizer="deutsche-telekom/bert-multi-english-german-squad2")

	# Frage beantworten
	#answer = qa_pipeline(question=question, context=context, top_k=3)
	answer = qa_pipeline(question=question, context=context)

	# Gib die Antwort aus
	st.text("Basisantwort:")
	st.text(answer["answer"])
	st.text(answer)

	"""
	#Die Basisantwort müsste man jetzt ausformulieren
	text2text_generator = pipeline("text2text-generation", model="google/flan-t5-xxl")
	#newText=text2text_generator(question=question, context=answer)
	newText=text2text_generator("Formuliere einen neuen Satz. Frage: "+question+ " Antwort: " + answer["answer"])
	st.text(newText)
	"""

	if __name__ == '__main__':
	main()