local llama2

app.py

@@ -3,6 +3,8 @@ Python Backend API to chat with private data
 
 08/16/2023
 D.M. Theekshana Samaradiwakara
+
+python -m streamlit run app.py
 """
 
 import os
@@ -229,7 +231,8 @@ def handle_userinput(query):
 
     except Exception as e:
         # logger.error(f"Answer retrieval failed with {e}")
-        st.error(f"Streamlit handle_userinput Error : {e}")#, icon=":books:")
+        st.error(f"Error ocuured! see log info for more details.")#, icon=":books:")
+        print(f"Streamlit handle_userinput Error : {e}")#, icon=":books:")
         return
 
 

config.py

@@ -6,6 +6,7 @@ MODELS={
     "openai gpt-3.5":"openai",
     # "Deci/DeciLM-6b-instruct":"Deci/DeciLM-6b-instruct",
     # "Deci/DeciLM-6b":"Deci/DeciLM-6b",
+    "local/LLAMA2":"local/LLAMA2",
 
 }
 

qaPipeline.py

@@ -69,6 +69,41 @@ class Singleton:
       else:
          Singleton.__instance = QAPipeline()
 
+def get_local_LLAMA2():
+    import torch
+    from transformers import AutoTokenizer, AutoModelForCausalLM
+
+    tokenizer = AutoTokenizer.from_pretrained("NousResearch/Llama-2-13b-chat-hf",
+                                        # use_auth_token=True,
+                                        )
+
+    model = AutoModelForCausalLM.from_pretrained("NousResearch/Llama-2-13b-chat-hf",
+                                            device_map='auto',
+                                            torch_dtype=torch.float16,
+                                            use_auth_token=True,
+                                        #  load_in_8bit=True,
+                                        #  load_in_4bit=True
+                                        )
+    from transformers import pipeline
+
+    pipe = pipeline("text-generation",
+                    model=model,
+                    tokenizer= tokenizer,
+                    torch_dtype=torch.bfloat16,
+                    device_map="auto",
+                    max_new_tokens = 512,
+                    do_sample=True,
+                    top_k=30,
+                    num_return_sequences=1,
+                    eos_token_id=tokenizer.eos_token_id
+                    )
+    
+    from langchain import HuggingFacePipeline
+    LLAMA2 = HuggingFacePipeline(pipeline = pipe, model_kwargs = {'temperature':0})
+    print(f"\n\n> torch.cuda.is_available(): {torch.cuda.is_available()}")
+    print("\n\n> local LLAMA2 loaded")
+    return LLAMA2
+
 class QAPipeline:
 
     def __init__(self):
@@ -134,7 +169,6 @@ class QAPipeline:
             return 
 
 
-    
     def set_model(self,model_type):
         if model_type != self.llm_name:
             match model_type:
@@ -152,12 +186,16 @@ class QAPipeline:
                     self.llm = ChatOpenAI(model_name="Deci/DeciLM-6b-instruct", temperature=0)
                 case "Deci/DeciLM-6b":
                     self.llm = ChatOpenAI(model_name="Deci/DeciLM-6b", temperature=0)
+                case "local/LLAMA2":
+                    self.llm = get_local_LLAMA2()
                 case _default:
                     # raise exception if model_type is not supported
                     raise Exception(f"Model type {model_type} is not supported. Please choose a valid one")
                  
             self.llm_name = model_type
     
+
+
     def set_vectorstore(self, dataset):
         if dataset != self.dataset_name:
             # self.vectorstore = load_store(dataset)

requirements.txt

@@ -1,4 +1,4 @@
-langchain == 0.0.287
+# langchain == 0.0.287
 # openai == 0.27.8
 # streamlit == 1.25.0
 
@@ -21,3 +21,6 @@ huggingface-hub
 sentence_transformers
 
 python-dotenv
+
+#local llama2
+accelerate