Spaces:
Runtime error
Runtime error
| """ | |
| Python Backend API to chat with private data | |
| 08/14/2023 | |
| D.M. Theekshana Samaradiwakara | |
| """ | |
| import os | |
| import time | |
| from dotenv import load_dotenv | |
| from langchain.chains import RetrievalQA | |
| from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler | |
| from langchain.llms import GPT4All | |
| from langchain.llms import HuggingFaceHub | |
| from langchain.chat_models import ChatOpenAI | |
| from langchain.chat_models import ChatAnyscale | |
| # from langchain.retrievers.self_query.base import SelfQueryRetriever | |
| # from langchain.chains.query_constructor.base import AttributeInfo | |
| # from chromaDb import load_store | |
| from faissDb import load_FAISS_store | |
| from langchain.agents import ZeroShotAgent, Tool, AgentExecutor | |
| from langchain.prompts import PromptTemplate | |
| from langchain.chains import LLMChain, ConversationalRetrievalChain | |
| from conversationBufferWindowMemory import ConversationBufferWindowMemory | |
| from langchain.memory import ReadOnlySharedMemory | |
| load_dotenv() | |
| #gpt4 all model | |
| gpt4all_model_path = os.environ.get('GPT4ALL_MODEL_PATH') | |
| model_n_ctx = os.environ.get('MODEL_N_CTX') | |
| model_n_batch = int(os.environ.get('MODEL_N_BATCH',8)) | |
| target_source_chunks = int(os.environ.get('TARGET_SOURCE_CHUNKS',4)) | |
| openai_api_key = os.environ.get('OPENAI_API_KEY') | |
| anyscale_api_key = os.environ.get('ANYSCALE_ENDPOINT_TOKEN') | |
| verbose = os.environ.get('VERBOSE') | |
| # activate/deactivate the streaming StdOut callback for LLMs | |
| callbacks = [StreamingStdOutCallbackHandler()] | |
| memory = ConversationBufferWindowMemory( | |
| memory_key="chat_history", | |
| input_key="question", | |
| return_messages=True, | |
| k=3 | |
| ) | |
| readonlymemory = ReadOnlySharedMemory(memory=memory) | |
| class Singleton: | |
| __instance = None | |
| def getInstance(): | |
| """ Static access method. """ | |
| if Singleton.__instance == None: | |
| Singleton() | |
| return Singleton.__instance | |
| def __init__(self): | |
| """ Virtually private constructor. """ | |
| if Singleton.__instance != None: | |
| raise Exception("This class is a singleton!") | |
| else: | |
| Singleton.__instance = QAPipeline() | |
| def get_local_LLAMA2(): | |
| import torch | |
| from transformers import AutoTokenizer, AutoModelForCausalLM | |
| tokenizer = AutoTokenizer.from_pretrained("NousResearch/Llama-2-13b-chat-hf", | |
| # use_auth_token=True, | |
| ) | |
| model = AutoModelForCausalLM.from_pretrained("NousResearch/Llama-2-13b-chat-hf", | |
| device_map='auto', | |
| torch_dtype=torch.float16, | |
| use_auth_token=True, | |
| # load_in_8bit=True, | |
| # load_in_4bit=True | |
| ) | |
| from transformers import pipeline | |
| pipe = pipeline("text-generation", | |
| model=model, | |
| tokenizer= tokenizer, | |
| torch_dtype=torch.bfloat16, | |
| device_map="auto", | |
| max_new_tokens = 512, | |
| do_sample=True, | |
| top_k=30, | |
| num_return_sequences=1, | |
| eos_token_id=tokenizer.eos_token_id | |
| ) | |
| from langchain import HuggingFacePipeline | |
| LLAMA2 = HuggingFacePipeline(pipeline = pipe, model_kwargs = {'temperature':0}) | |
| print(f"\n\n> torch.cuda.is_available(): {torch.cuda.is_available()}") | |
| print("\n\n> local LLAMA2 loaded") | |
| return LLAMA2 | |
| class QAPipeline: | |
| def __init__(self): | |
| print("\n\n> Initializing QAPipeline:") | |
| self.llm_name = None | |
| self.llm = None | |
| self.dataset_name = None | |
| self.vectorstore = None | |
| self.qa_chain = None | |
| self.agent = None | |
| def run(self,query, model, dataset): | |
| if (self.llm_name != model) or (self.dataset_name != dataset) or (self.qa_chain == None): | |
| self.set_model(model) | |
| self.set_vectorstore(dataset) | |
| self.set_qa_chain() | |
| # Get the answer from the chain | |
| start = time.time() | |
| res = self.qa_chain(query) | |
| # answer, docs = res['result'],res['source_documents'] | |
| end = time.time() | |
| # Print the result | |
| print("\n\n> Question:") | |
| print(query) | |
| print(f"\n> Answer (took {round(end - start, 2)} s.):") | |
| print( res) | |
| return res | |
| def run_agent(self,query, model, dataset): | |
| try: | |
| if (self.llm_name != model) or (self.dataset_name != dataset) or (self.agent == None): | |
| self.set_model(model) | |
| self.set_vectorstore(dataset) | |
| self.set_qa_chain_with_agent() | |
| # Get the answer from the chain | |
| start = time.time() | |
| res = self.agent(query) | |
| # answer, docs = res['result'],res['source_documents'] | |
| end = time.time() | |
| # Print the result | |
| print("\n\n> Question:") | |
| print(query) | |
| print(f"\n> Answer (took {round(end - start, 2)} s.):") | |
| print( res) | |
| return res["output"] | |
| except Exception as e: | |
| # logger.error(f"Answer retrieval failed with {e}") | |
| print(f"> QAPipeline run_agent Error : {e}")#, icon=":books:") | |
| return | |
| def set_model(self,model_type): | |
| if model_type != self.llm_name: | |
| match model_type: | |
| case "gpt4all": | |
| # self.llm = GPT4All(model=gpt4all_model_path, n_ctx=model_n_ctx, backend='gptj', n_batch=model_n_batch, callbacks=callbacks, verbose=verbose) | |
| self.llm = GPT4All(model=gpt4all_model_path, max_tokens=model_n_ctx, backend='gptj', n_batch=model_n_batch, callbacks=callbacks, verbose=verbose) | |
| # self.llm = HuggingFaceHub(repo_id="nomic-ai/gpt4all-j", model_kwargs={"temperature":0.001, "max_length":1024}) | |
| case "google/flan-t5-xxl": | |
| self.llm = HuggingFaceHub(repo_id="google/flan-t5-xxl", model_kwargs={"temperature":0.001, "max_length":1024}) | |
| case "tiiuae/falcon-7b-instruct": | |
| self.llm = HuggingFaceHub(repo_id=model_type, model_kwargs={"temperature":0.001, "max_length":1024}) | |
| case "openai": | |
| self.llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0) | |
| case "Deci/DeciLM-6b-instruct": | |
| self.llm = ChatOpenAI(model_name="Deci/DeciLM-6b-instruct", temperature=0) | |
| case "Deci/DeciLM-6b": | |
| self.llm = ChatOpenAI(model_name="Deci/DeciLM-6b", temperature=0) | |
| case "local/LLAMA2": | |
| self.llm = get_local_LLAMA2() | |
| case "anyscale/Llama-2-13b-chat-hf": | |
| self.llm = ChatAnyscale(anyscale_api_key=anyscale_api_key,temperature=0, model_name='meta-llama/Llama-2-13b-chat-hf', streaming=False) | |
| case "anyscale/Llama-2-70b-chat-hf": | |
| self.llm = ChatAnyscale(anyscale_api_key=anyscale_api_key,temperature=0, model_name='meta-llama/Llama-2-70b-chat-hf', streaming=False) | |
| case _default: | |
| # raise exception if model_type is not supported | |
| raise Exception(f"Model type {model_type} is not supported. Please choose a valid one") | |
| self.llm_name = model_type | |
| def set_vectorstore(self, dataset): | |
| if dataset != self.dataset_name: | |
| # self.vectorstore = load_store(dataset) | |
| self.vectorstore = load_FAISS_store() | |
| print("\n\n> vectorstore loaded:") | |
| self.dataset_name = dataset | |
| def set_qa_chain(self): | |
| self.qa_chain = RetrievalQA.from_chain_type( | |
| llm=self.llm, | |
| chain_type="stuff", | |
| retriever = self.vectorstore.as_retriever(), | |
| # retriever = self.vectorstore.as_retriever(search_kwargs={"k": target_source_chunks} | |
| return_source_documents= True | |
| ) | |
| def set_qa_chain_with_agent(self): | |
| try: | |
| # Define a custom prompt | |
| general_qa_template = ( | |
| """[INST]<<SYS>> You are the AI of company boardpac which provide services to company board members related to banking and financial sector. You should only continue the conversation and reply to users questions like welcomes, greetings and goodbyes. | |
| If you dont know the answer say you dont know, dont try to makeup answers. Answer should be short and simple as possible. Start the answer with code word Boardpac AI (chat): <</SYS>> | |
| Conversation: {chat_history} | |
| Question: {question} [/INST]""" | |
| ) | |
| general_qa_chain_prompt = PromptTemplate(input_variables=["question", "chat_history"], template=general_qa_template) | |
| general_qa_chain = LLMChain( | |
| llm=self.llm, | |
| prompt=general_qa_chain_prompt, | |
| verbose=True, | |
| memory=readonlymemory, # use the read-only memory to prevent the tool from modifying the memory | |
| ) | |
| general_qa_chain_tool = Tool( | |
| name="general qa", | |
| func= general_qa_chain.run, | |
| description='''use this when only you need to answer questions like welcomes, greetings and goodbyes. | |
| Input should be a fully formed question.''', | |
| return_direct=True, | |
| ) | |
| # Define a custom prompt | |
| retrieval_qa_template = ( | |
| """[INST]<<SYS>> You are the AI of company boardpac which provide services to company board members. Only answer questions related to Banking and Financial Services Sector like Banking & Financial regulations, legal framework, governance framework, compliance requirements as per Central Bank regulations. | |
| please answer the question based on the chat history and context information provided below related to central bank acts published in various years. The published year is mentioned as the metadata 'year' of each source document. | |
| The content of a bank act of a past year can updated by a bank act from a latest year. Always try to answer with latest information and mention the year which information extracted. | |
| If you dont know the answer say you dont know, dont try to makeup answers. Answer should be short and simple as possible. Start the answer with code word Boardpac AI (QA): <</SYS>> | |
| Conversation: {chat_history} | |
| Context: {context} | |
| Question : {question} [/INST]""" | |
| ) | |
| retrieval_qa_chain_prompt = PromptTemplate( | |
| input_variables=["question", "context", "chat_history"], | |
| template=retrieval_qa_template | |
| ) | |
| document_combine_prompt = PromptTemplate( | |
| input_variables=["source","year", "page","page_content"], | |
| template= | |
| """<doc> source: {source}, year: {year}, page: {page}, page content: {page_content} </doc>""" | |
| ) | |
| bank_regulations_qa = ConversationalRetrievalChain.from_llm( | |
| llm=self.llm, | |
| chain_type="stuff", | |
| retriever = self.vectorstore.as_retriever(), | |
| # retriever = self.vectorstore.as_retriever( | |
| # search_type="mmr", | |
| # search_kwargs={ | |
| # 'k': 6, | |
| # # 'lambda_mult': 0.1, | |
| # 'fetch_k': 50}, | |
| # # search_type="similarity_score_threshold", | |
| # # search_kwargs={"score_threshold": .5} | |
| # ), | |
| return_source_documents= True, | |
| return_generated_question= True, | |
| get_chat_history=lambda h : h, | |
| combine_docs_chain_kwargs={ | |
| "prompt": retrieval_qa_chain_prompt, | |
| "document_prompt": document_combine_prompt, | |
| }, | |
| verbose=True, | |
| memory=readonlymemory, # use the read-only memory to prevent the tool from modifying the memory | |
| ) | |
| bank_regulations_qa_tool = Tool( | |
| name="bank regulations", | |
| func= lambda question: bank_regulations_qa({"question": question}), | |
| description='''Use this more when you need to answer questions about Banking and Financial Services Sector like Banking & Financial regulations, legal framework, governance framework, compliance requirements as per Central Bank regulations. | |
| Input should be a fully formed question.''', | |
| return_direct=True, | |
| ) | |
| tools = [ | |
| bank_regulations_qa_tool, | |
| general_qa_chain_tool | |
| ] | |
| prefix = """<<SYS>> You are the AI of company boardpac which provide services to company board members related to banking and financial sector. Have a conversation with the user, answering the following questions as best you can. You have access to the following tools:""" | |
| suffix = """Begin! " | |
| {agent_scratchpad} | |
| <chat history>: {chat_history} | |
| <</SYS>> | |
| [INST] | |
| <Question>: {question} | |
| [/INST]""" | |
| agent_prompt = ZeroShotAgent.create_prompt( | |
| tools, | |
| prefix=prefix, | |
| suffix=suffix, | |
| input_variables=["question", "chat_history", "agent_scratchpad"], | |
| ) | |
| llm_chain = LLMChain(llm=self.llm, prompt=agent_prompt) | |
| agent = ZeroShotAgent( | |
| llm_chain=llm_chain, | |
| tools=tools, | |
| verbose=True, | |
| ) | |
| agent_chain = AgentExecutor.from_agent_and_tools( | |
| agent=agent, | |
| tools=tools, | |
| verbose=True, | |
| memory=memory, | |
| handle_parsing_errors=True, | |
| ) | |
| self.agent = agent_chain | |
| print(f"\n> agent_chain created") | |
| except Exception as e: | |
| # logger.error(f"Answer retrieval failed with {e}") | |
| print(f"> QAPipeline set_qa_chain_with_agent Error : {e}")#, icon=":books:") | |
| return | |