from langchain.embeddings.openai import OpenAIEmbeddings from langchain.vectorstores import Chroma from langchain.text_splitter import CharacterTextSplitter from langchain.chains.question_answering import load_qa_chain from langchain.llms import OpenAI import os from glob import glob import shutil import gradio as gr import git git.Repo.clone_from("https://github.com/TheMITTech/shakespeare", "shakespeare") files = glob("./shakespeare/**/*.html") os.mkdir('./data') destination_folder = './data/' for html_file in files: shutil.move(html_file, destination_folder + html_file.split("/")[-1]) from langchain.document_loaders import BSHTMLLoader, DirectoryLoader bshtml_dir_loader = DirectoryLoader('./data/', loader_cls=BSHTMLLoader) data = bshtml_dir_loader.load() from langchain.text_splitter import RecursiveCharacterTextSplitter text_splitter = RecursiveCharacterTextSplitter( chunk_size = 1000, chunk_overlap = 20, length_function = len, ) documents = text_splitter.split_documents(data) from langchain.embeddings.openai import OpenAIEmbeddings embeddings = OpenAIEmbeddings() from chromadb.segment.impl.vector.local_hnsw import HnswParams from langchain.vectorstores import Chroma persist_directory = "vector_db" vectordb = Chroma.from_documents(documents=documents, embedding=embeddings, persist_directory=persist_directory) vectordb.persist() vectordb = None vectordb = Chroma(persist_directory=persist_directory, embedding_function=embeddings) from langchain.chat_models import ChatOpenAI llm = ChatOpenAI(temperature=0, model="gpt-4") doc_retriever = vectordb.as_retriever() from langchain.chains import RetrievalQA shakespeare_qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=doc_retriever) from langchain.utilities import SerpAPIWrapper search = SerpAPIWrapper() from langchain.agents import initialize_agent, Tool from langchain.agents import AgentType from langchain.tools import BaseTool from langchain.llms import OpenAI from langchain import LLMMathChain, SerpAPIWrapper tools = [ Tool( name = "Shakespeare QA System", func=shakespeare_qa.run, description="useful for when you need to answer questions about Shakespeare's works. Input should be a fully formed question." ), Tool( name = "SERP API Search", func=search.run, description="useful for when you need to answer questions about ruff (a python linter). Input should be a fully formed question." ), ] agent = initialize_agent(tools, llm, agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, verbose=True) def make_inference(query): return(agent.run(query)) if __name__ == "__main__": gr.Interface( make_inference, [ gr.inputs.Textbox(lines=2, label="Query"), ], gr.outputs.Textbox(label="Response"), title="Shakespeare", description="Shakespeare is a tool that allows you to ask questions about Shakespeare.", ).launch()