changed to ConversationalRetrievalChain

.env

@@ -11,7 +11,7 @@ TARGET_SOURCE_CHUNKS=4
 
 #API token keys
 HUGGINGFACEHUB_API_TOKEN=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
-OPENAI_API_KEY=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
+OPENAI_API_KEY=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
 ANYSCALE_ENDPOINT_TOKEN=esecret_n1svfld85uklyx5ebaasyiw2m9
 
 #api app

app.py

@@ -17,9 +17,7 @@ logger = get_logger(__name__)
 from ui.htmlTemplates import css, bot_template, user_template, source_template
 from config import MODELS, DATASETS
 
-from qaPipeline import QAPipeline
-import qaPipeline_functions
-from faissDb import create_faiss
+from qaPipeline_chain_only import QAPipeline
 
 # loads environment variables
 from dotenv import load_dotenv
@@ -117,6 +115,7 @@ def side_bar():
 
 def chat_body():
     st.header("Chat with your own data:")
+    # st.text("Implemented using ConversationalRetrievalChain")
     with st.form('chat_body'):
 
         user_question = st.text_input(
@@ -195,11 +194,13 @@ def show_query_response(query, response, show_source_files):
         #         "{{MSG}}",  "source files" ), unsafe_allow_html=True)
 
         if len(docs)>0 :
-            st.markdown("#### source files : ")
-            for source in docs:
-                # st.info(source.metadata)
-                with st.expander(source.metadata["source"]):
-                    st.markdown(source.page_content)
+            code_word = 'Boardpac AI(QA):'
+            if code_word in answer:
+                st.markdown("#### source files : ")
+                for source in docs:
+                    # st.info(source.metadata)
+                    with st.expander(source.metadata["source"]):
+                        st.markdown(source.page_content)
             
         # st.write(response)
     

app_agent.py

@@ -0,0 +1,246 @@
+"""
+Python Backend API to chat with private data  
+
+08/16/2023
+D.M. Theekshana Samaradiwakara
+
+python -m streamlit run app.py
+"""
+
+import os
+import time
+import streamlit as st
+from streamlit.logger import get_logger
+
+logger = get_logger(__name__)
+
+from ui.htmlTemplates import css, bot_template, user_template, source_template
+from config import MODELS, DATASETS
+
+from qaPipeline import QAPipeline
+import qaPipeline_functions
+from faissDb import create_faiss
+
+# loads environment variables
+from dotenv import load_dotenv
+load_dotenv()
+
+isHuggingFaceHubEnabled = os.environ.get('ENABLE_HUGGINGFSCE_HUB_MODELS')
+isOpenAiApiEnabled = os.environ.get('ENABLE_OPENAI_API_MODELS')
+
+st.set_page_config(page_title="Chat with data",
+                       page_icon=":books:")
+st.write(css, unsafe_allow_html=True)
+
+qaPipeline = QAPipeline()
+# qaPipeline = qaPipeline_functions
+
+def initialize_session_state():
+    # Initialise all session state variables with defaults
+    SESSION_DEFAULTS = {
+        "model": MODELS["DEFAULT"],
+        "dataset": DATASETS["DEFAULT"],
+        "chat_history": None,
+        "is_parameters_changed":False,
+        "show_source_files": False,
+        "user_question":'',
+    }
+
+    for k, v in SESSION_DEFAULTS.items():
+        if k not in st.session_state:
+            st.session_state[k] = v
+
+def side_bar():
+    with st.sidebar:
+        st.subheader("Chat parameters")
+
+        with st.form('param_form'):
+            st.info('Info: use openai chat model for best results')
+            chat_model = st.selectbox(
+                "Chat model",
+                MODELS,
+                key="chat_model",
+                help="Select the LLM model for the chat",
+                # on_change=update_parameters_change,
+            )
+
+            # data_source = st.selectbox(
+            #     "dataset",
+            #     DATASETS,
+            #     key="data_source",
+            #     help="Select the private data_source for the chat",
+            #     on_change=update_parameters_change,
+            # )
+
+            st.session_state.dataset =  "DEFAULT"
+
+            show_source = st.checkbox(
+                label="show source files",
+                key="show_source",
+                help="Select this to show relavant source files for the query",
+                # on_change=update_parameters_change,
+            )
+
+            submitted = st.form_submit_button(
+                "Save Parameters",
+                # on_click=update_parameters_change
+                )
+       
+            if submitted:
+                parameters_change_button(chat_model, show_source)
+
+
+            # if st.session_state.is_parameters_changed:
+            #     st.button("Update", 
+            #             on_click=parameters_change_button,
+            #             args=[chat_model, show_source]
+            #     )
+                    
+        st.markdown("\n")
+
+        # if st.button("Create FAISS db"):
+        #     try:
+        #         with st.spinner('creating faiss vector store'):
+        #             create_faiss()
+        #             st.success('faiss saved')
+        #     except Exception as e:
+        #         st.error(f"Error : {e}")#, icon=":books:")
+        #         return
+                
+        st.markdown(
+            "### How to use\n"
+            "1. Select the chat model\n"  # noqa: E501
+            "2. Select \"show source files\" to show the source files related to the answer.📄\n"
+            "3. Ask a question about the documents💬\n"
+        )
+
+
+def chat_body():
+    st.header("Chat with your own data:")
+    with st.form('chat_body'):
+
+        user_question = st.text_input(
+            "Ask a question about your documents:",
+            placeholder="enter question",
+            key='user_question',
+            # on_change=submit_user_question,
+        )
+
+        submitted = st.form_submit_button(
+            "Submit",
+            # on_click=update_parameters_change
+            )
+    
+        if submitted:
+            submit_user_question()
+
+    # if user_question:
+    #     submit_user_question()
+    #     # user_question = False
+
+    
+
+def submit_user_question():
+    with st.spinner("Processing"):
+        user_question = st.session_state.user_question
+        # st.success(user_question)
+        handle_userinput(user_question)
+        # st.session_state.user_question=''
+
+
+def main():
+   
+    initialize_session_state()
+    
+    side_bar()
+
+    chat_body()
+
+                
+def update_parameters_change():
+    st.session_state.is_parameters_changed = True
+
+
+def parameters_change_button(chat_model, show_source):
+    st.session_state.model = chat_model
+    st.session_state.dataset = "DEFAULT"
+    st.session_state.show_source_files = show_source
+    st.session_state.is_parameters_changed = False
+
+    alert =  st.success("chat parameters updated")
+    time.sleep(1) # Wait for 3 seconds
+    alert.empty() # Clear the alert
+
+@st.cache_data
+def get_answer_from_backend(query, model, dataset):
+    # response = qaPipeline.run(query=query, model=model, dataset=dataset)
+    response = qaPipeline.run_agent(query=query, model=model, dataset=dataset)
+    return response
+
+
+def show_query_response(query, response, show_source_files):
+    docs = []
+    if isinstance(response, dict):
+        answer, docs = response['answer'], response['source_documents']
+    else:
+        answer = response
+
+    st.write(user_template.replace(
+                "{{MSG}}", query), unsafe_allow_html=True)
+    st.write(bot_template.replace(
+                "{{MSG}}",  answer ), unsafe_allow_html=True)
+    
+    if show_source_files:
+        # st.write(source_template.replace(
+        #         "{{MSG}}",  "source files" ), unsafe_allow_html=True)
+
+        if len(docs)>0 :
+            st.markdown("#### source files : ")
+            for source in docs:
+                # st.info(source.metadata)
+                with st.expander(source.metadata["source"]):
+                    st.markdown(source.page_content)
+            
+        # st.write(response)
+    
+
+def is_query_valid(query: str) -> bool:
+    if (not query) or (query.strip() == ''):
+        st.error("Please enter a question!")
+        return False
+    return True
+
+
+def handle_userinput(query):
+    # Get the answer from the chain
+    try:
+        if not is_query_valid(query):
+            st.stop()
+
+        model = MODELS[st.session_state.model]
+        dataset = DATASETS[st.session_state.dataset]
+        show_source_files = st.session_state.show_source_files
+
+        # Try to access openai and deeplake
+        print(f">\n model: {model} \n dataset : {dataset} \n show_source_files : {show_source_files}")
+
+        response = get_answer_from_backend(query, model, dataset)
+
+        show_query_response(query, response, show_source_files)
+
+
+    except Exception as e:
+        # logger.error(f"Answer retrieval failed with {e}")
+        st.error(f"Error ocuured! see log info for more details.")#, icon=":books:")
+        print(f"Streamlit handle_userinput Error : {e}")#, icon=":books:")
+        return
+
+
+if __name__ == "__main__":
+    main()
+
+# initialize_session_state()
+
+# side_bar()
+
+# chat_body()

config.py

@@ -2,14 +2,17 @@ MODELS={
     "DEFAULT":"tiiuae/falcon-7b-instruct",
     # "gpt4all":"gpt4all",
     # "flan-t5-xxl":"google/flan-t5-xxl",
-    "falcon-7b-instruct":"tiiuae/falcon-7b-instruct",
+    "hf/falcon-7b-instruct":"tiiuae/falcon-7b-instruct",
+    
     "anyscale/Llama-2-13b":"anyscale/Llama-2-13b-chat-hf",
     "anyscale/Llama-2-70b":"anyscale/Llama-2-70b-chat-hf",
-    "openai gpt-3.5":"openai",
+
+    "local/Llama-2-13b":"local/LLAMA2",
+
+    "openai/gpt-3.5":"openai",
     # "Deci/DeciLM-6b-instruct":"Deci/DeciLM-6b-instruct",
     # "Deci/DeciLM-6b":"Deci/DeciLM-6b",
-    "local/LLAMA2":"local/LLAMA2",
-
+    
 }
 
 DATASETS={

qaPipeline.py

@@ -309,12 +309,15 @@ class QAPipeline:
                 general_qa_chain_tool
             ]
 
-            prefix = """[INST]<<SYS>> You are the AI of company boardpac which provide services to company board members related to banking and financial sector. Have a conversation with the user, answering the following questions as best you can. You have access to the following tools:"""
-            suffix = """Begin! <</SYS>>"
-
-            {chat_history}
-            Question: {question}
-            {agent_scratchpad} [/INST]"""
+            prefix = """<<SYS>> You are the AI of company boardpac which provide services to company board members related to banking and financial sector. Have a conversation with the user, answering the following questions as best you can. You have access to the following tools:"""
+            suffix = """Begin! "
+            {agent_scratchpad} 
+            <chat history>: {chat_history}
+            <</SYS>>
+
+            [INST]
+            <Question>: {question}
+            [/INST]"""
 
             agent_prompt = ZeroShotAgent.create_prompt(
                 tools,

qaPipeline_chain_only.py

@@ -0,0 +1,241 @@
+"""
+Python Backend API to chat with private data  
+
+08/14/2023
+D.M. Theekshana Samaradiwakara
+"""
+
+import os
+import time
+
+from dotenv import load_dotenv
+
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+
+from langchain.llms import GPT4All
+from langchain.llms import HuggingFaceHub
+from langchain.chat_models import ChatOpenAI
+from langchain.chat_models import ChatAnyscale
+
+# from langchain.retrievers.self_query.base import SelfQueryRetriever
+# from langchain.chains.query_constructor.base import AttributeInfo
+
+# from chromaDb import load_store
+from faissDb import load_FAISS_store
+
+
+
+from langchain.prompts import PromptTemplate
+from langchain.chains import LLMChain, ConversationalRetrievalChain
+from conversationBufferWindowMemory import ConversationBufferWindowMemory
+
+load_dotenv()
+
+#gpt4 all model
+gpt4all_model_path = os.environ.get('GPT4ALL_MODEL_PATH')
+model_n_ctx = os.environ.get('MODEL_N_CTX')
+model_n_batch = int(os.environ.get('MODEL_N_BATCH',8))
+target_source_chunks = int(os.environ.get('TARGET_SOURCE_CHUNKS',4))
+
+openai_api_key = os.environ.get('OPENAI_API_KEY')
+anyscale_api_key = os.environ.get('ANYSCALE_ENDPOINT_TOKEN')
+
+verbose = os.environ.get('VERBOSE')
+
+# activate/deactivate the streaming StdOut callback for LLMs
+callbacks = [StreamingStdOutCallbackHandler()]
+
+def get_local_LLAMA2():
+    import torch
+    from transformers import AutoTokenizer, AutoModelForCausalLM
+
+    tokenizer = AutoTokenizer.from_pretrained("NousResearch/Llama-2-13b-chat-hf",
+                                        # use_auth_token=True,
+                                        )
+
+    model = AutoModelForCausalLM.from_pretrained("NousResearch/Llama-2-13b-chat-hf",
+                                            device_map='auto',
+                                            torch_dtype=torch.float16,
+                                            use_auth_token=True,
+                                        #  load_in_8bit=True,
+                                        #  load_in_4bit=True
+                                        )
+    from transformers import pipeline
+
+    pipe = pipeline("text-generation",
+                    model=model,
+                    tokenizer= tokenizer,
+                    torch_dtype=torch.bfloat16,
+                    device_map="auto",
+                    max_new_tokens = 512,
+                    do_sample=True,
+                    top_k=30,
+                    num_return_sequences=1,
+                    eos_token_id=tokenizer.eos_token_id
+                    )
+    
+    from langchain import HuggingFacePipeline
+    LLAMA2 = HuggingFacePipeline(pipeline = pipe, model_kwargs = {'temperature':0})
+    print(f"\n\n> torch.cuda.is_available(): {torch.cuda.is_available()}")
+    print("\n\n> local LLAMA2 loaded")
+    return LLAMA2
+
+class QAPipeline:
+
+    def __init__(self):
+
+        print("\n\n> Initializing QAPipeline:")
+        self.llm_name = None
+        self.llm = None
+
+        self.dataset_name = None
+        self.vectorstore = None
+
+        self.qa_chain = None
+    
+    def run_agent(self,query, model, dataset):
+    
+        try:
+            if (self.llm_name != model) or (self.dataset_name != dataset) or (self.qa_chain == None):
+                self.set_model(model)
+                self.set_vectorstore(dataset)
+                self.set_qa_chain()
+
+            # Get the answer from the chain
+            start = time.time()
+            res = self.qa_chain(query)
+            # answer, docs = res['result'],res['source_documents']
+            end = time.time()
+
+            # Print the result
+            print("\n\n> Question:")
+            print(query)
+            print(f"\n> Answer (took {round(end - start, 2)} s.):")
+            print( res)
+
+            return res
+    
+        except Exception as e:
+        # logger.error(f"Answer retrieval failed with {e}")
+            print(f"> QAPipeline run_agent Error : {e}")#, icon=":books:")
+            return 
+
+
+    def set_model(self,model_type):
+        if model_type != self.llm_name:
+            match model_type:
+                case "gpt4all":
+                    # self.llm = GPT4All(model=gpt4all_model_path, n_ctx=model_n_ctx, backend='gptj', n_batch=model_n_batch, callbacks=callbacks, verbose=verbose)
+                    self.llm = GPT4All(model=gpt4all_model_path, max_tokens=model_n_ctx, backend='gptj', n_batch=model_n_batch, callbacks=callbacks, verbose=verbose)
+                    # self.llm = HuggingFaceHub(repo_id="nomic-ai/gpt4all-j", model_kwargs={"temperature":0.001, "max_length":1024})
+                case "google/flan-t5-xxl":
+                    self.llm = HuggingFaceHub(repo_id="google/flan-t5-xxl", model_kwargs={"temperature":0.001, "max_length":1024})
+                case "tiiuae/falcon-7b-instruct":
+                    self.llm = HuggingFaceHub(repo_id=model_type, model_kwargs={"temperature":0.001, "max_length":1024})
+                case "openai":
+                    self.llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)
+                case "Deci/DeciLM-6b-instruct":
+                    self.llm = ChatOpenAI(model_name="Deci/DeciLM-6b-instruct", temperature=0)
+                case "Deci/DeciLM-6b":
+                    self.llm = ChatOpenAI(model_name="Deci/DeciLM-6b", temperature=0)
+                case "local/LLAMA2":
+                    self.llm = get_local_LLAMA2()
+                case "anyscale/Llama-2-13b-chat-hf":
+                    self.llm = ChatAnyscale(anyscale_api_key=anyscale_api_key,temperature=0, model_name='meta-llama/Llama-2-13b-chat-hf', streaming=False)
+                case "anyscale/Llama-2-70b-chat-hf":
+                    self.llm = ChatAnyscale(anyscale_api_key=anyscale_api_key,temperature=0, model_name='meta-llama/Llama-2-70b-chat-hf', streaming=False)
+                case _default:
+                    # raise exception if model_type is not supported
+                    raise Exception(f"Model type {model_type} is not supported. Please choose a valid one")
+                 
+            self.llm_name = model_type
+    
+
+
+    def set_vectorstore(self, dataset):
+        if dataset != self.dataset_name:
+            # self.vectorstore = load_store(dataset)
+            self.vectorstore = load_FAISS_store()
+            print("\n\n> vectorstore loaded:")
+            self.dataset_name = dataset
+
+
+    def set_qa_chain(self):
+        
+        try:
+            memory = ConversationBufferWindowMemory(
+            memory_key="chat_history",
+            input_key="question",
+            output_key = "answer",
+            return_messages=True,
+            k=3
+            )
+
+            # Define a custom prompt
+            B_INST, E_INST = "[INST]", "[/INST]"
+            B_SYS, E_SYS = "<<SYS>>\n", "\n<</SYS>>\n\n"
+
+            retrieval_qa_template = (
+            """<<SYS>> 
+            You are the AI assistant of company boardpac which provide services to company board members related to banking and financial sector.
+            You have 2 tasks to do.
+
+            Task 1: combine the given chat history and user question to come up with a follow-up question
+            <chat history>: {chat_history}
+
+            Task 2:
+            Identify the type of the follow-up question using following 3 types and answer accordingly.
+            Answer should be short and simple as possible.
+            Dont add any extra details that is not mentioned in the context.
+
+            <Type 1>
+            If the user asks questions like welcome messages, greetings and goodbyes.
+            Just reply accordingly with a short and simple answer as possible.
+            Dont use context information provided below to answer the question. 
+            Start the answer with code word Boardpac AI(chat): 
+            </Type 1>
+
+            <Type 2>
+            If the follow-up question doesn't belong to type 1 or type 3, that means if the question is not about greetings or Banking and Financial Services say that the question is out of your domain.
+            Start the answer with code word Boardpac AI(OD): 
+            </Type 2> 
+
+            <Type 3>
+            If the follow-up question is related to Banking and Financial Services Sector like Banking & Financial regulations, legal framework, governance framework, compliance requirements as per Central Bank regulations.
+            please answer the question based on the context information provided in bottom containing few related documents of central bank acts published in various years.
+            The published year is mentioned as the  metadata 'year' of each source document.
+            The content of a bank act of a past year can updated by a bank act from a latest year.
+            Always try to answer with latest information and mention the year which information extracted.
+            If you dont know the answer say you dont know, dont try to makeup answers.
+            Start the answer with code word Boardpac AI(QA):
+            </Type 3>
+   
+            <Context information>: {context}
+            <</SYS>>
+
+            [INST]Question : {question}[/INST]"""
+            )
+
+            retrieval_qa_chain_prompt = PromptTemplate(
+                input_variables=["question", "context", "chat_history"], 
+                template=retrieval_qa_template
+            )
+
+            self.qa_chain = ConversationalRetrievalChain.from_llm(
+                llm=self.llm,
+                chain_type="stuff",
+                retriever = self.vectorstore.as_retriever(), 
+                # retriever = self.vectorstore.as_retriever(search_kwargs={"k": target_source_chunks}
+                return_source_documents= True,
+                get_chat_history=lambda h : h,
+                combine_docs_chain_kwargs={"prompt": retrieval_qa_chain_prompt},
+                verbose=True,
+                memory=memory,
+            )
+
+            print(f"\n> agent_chain created")
+
+        except Exception as e:
+            # logger.error(f"Answer retrieval failed with {e}")
+            print(f"> QAPipeline set_qa_chain_with_agent Error : {e}")#, icon=":books:")
+            return