import os import gradio as gr from langchain.document_loaders import PDFMinerLoader,CSVLoader ,UnstructuredWordDocumentLoader,TextLoader,OnlinePDFLoader from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain.embeddings import SentenceTransformerEmbeddings from langchain.vectorstores import FAISS from langchain import HuggingFaceHub from langchain.chains import RetrievalQA from langchain.prompts import PromptTemplate DEVICE = 'cpu' FILE_EXT = ['pdf','text','csv','word','wav'] def loading_file(): return "Loading..." def get_openai_chat_model(API_key): try: from langchain.llms import OpenAI except ImportError as err: raise "{}, unable to load openAI. Please install openai and add OPENAIAPI_KEY" os.environ["OPENAI_API_KEY"] = API_key llm = OpenAI() return llm def process_documents(documents,data_chunk=1000,chunk_overlap=50): text_splitter = RecursiveCharacterTextSplitter(chunk_size=data_chunk, chunk_overlap=chunk_overlap) texts = text_splitter.split_documents(documents[0]) return texts def get_hugging_face_model(model_id,API_key,temperature=0.1): chat_llm = HuggingFaceHub(huggingfacehub_api_token=API_key, repo_id=model_id, model_kwargs={"temperature": temperature, "max_new_tokens": 2048}) return chat_llm def chat_application(llm_service,key): if llm_model == 'HuggingFace': llm = get_hugging_face_model(model_id='tiiuae/falcon-7b-instruct',API_key=key) else: llm_model = get_openai_chat_model(API_key=key) def document_loader(file_data,api_key,doc_type='pdf',llm='Huggingface'): embedding_model = SentenceTransformerEmbeddings(model_name='all-mpnet-base-v2',model_kwargs={"device": DEVICE}) document = None if doc_type == 'pdf': document = process_pdf_document(document_file_name=file_data) elif doc_type == 'text': document = process_text_document(document_file_name=file_data) elif doc_type == 'csv': document = process_csv_document(document_file_name=file_data) elif doc_type == 'word': document = process_word_document(document_file_name=file_data) print(document) if document: texts = process_documents(documents=document) vector_db = FAISS.from_documents(documents=texts, embedding= embedding_model) global qa qa = RetrievalQA.from_chain_type(llm=chat_application(llm_service=llm,key=api_key), chain_type='stuff', retriever=vector_db.as_retriever(), # chain_type_kwargs=chain_type_kwargs, return_source_documents=True ) else: return "Error in loading Documents " return "Ready..." def process_text_document(document_file_name): loader = TextLoader(document_file_name) document = loader.load() return document def process_csv_document(document_file_name): loader = CSVLoader(file_path=document_file_name) document = loader.load() return document def process_word_document(document_file_name): loader = UnstructuredWordDocumentLoader(file_path=document_file_name) document = loader.load() return document def process_pdf_document(document_file_name): loader = OnlinePDFLoader(document_file_name) document = loader.load()[0] return document css=""" #col-container {max-width: 700px; margin-left: auto; margin-right: auto;} """ title = """
Upload a file from your computer, click the "Load data to LangChain" button,
when everything is ready, you can start asking questions about the data you uploaded ;)
This version is just for QA retrival so it will not use chat history, and uses Hugging face as LLM,
so you don't need any key