pure mistral testing

app.py

@@ -1,25 +1,16 @@
 import gradio as gr
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from inference import get_bot_response
-from rag import get_context
-from config import config
+from huggingface_hub import InferenceClient
 
+"""
+For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
+"""
+# client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
 
-model_name = "TheBloke/Mistral-7B-Instruct-v0.2-GPTQ"
-tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
+model_name = "mistralai/Mistral-7B-Instruct-v0.2"
+client = InferenceClient(model_name)
 
-print("model start loading")
-model = AutoModelForCausalLM.from_pretrained(model_name,
-                                             device_map="auto",
-                                             trust_remote_code=False,
-                                             revision="main")
-print("model loaded")
-
-# model = AutoModelForCausalLM.from_pretrained(config["model_checkpoint"],
-#                                                 device_map="auto",
-#                                                 trust_remote_code=False,
-#                                                 revision="main")
 
+print("test")
 
 def respond(
     message,
@@ -31,12 +22,31 @@ def respond(
 ):
     messages = [{"role": "system", "content": system_message}]
 
-    request = message
-    context = get_context(request, config["top_k"])
-    response = get_bot_response(request, context, model, tokenizer)
+    for val in history:
+        if val[0]:
+            messages.append({"role": "user", "content": val[0]})
+        if val[1]:
+            messages.append({"role": "assistant", "content": val[1]})
+
+    messages.append({"role": "user", "content": message})
+
+    response = ""
+
+    for message in client.chat_completion(
+        messages,
+        max_tokens=max_tokens,
+        stream=True,
+        temperature=temperature,
+        top_p=top_p,
+    ):
+        token = message.choices[0].delta.content
 
-    return response
+        response += token
+        yield response
 
+"""
+For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
+"""
 demo = gr.ChatInterface(
     respond,
     additional_inputs=[

app_old.py

@@ -1,14 +1,30 @@
 import gradio as gr
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from inference import get_bot_response
+from rag import get_context
+from config import config
 from huggingface_hub import InferenceClient
 
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
-# client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
-client = InferenceClient("TheBloke/Mistral-7B-Instruct-v0.2-GPTQ")
 
+model_name = "mistralai/Mistral-7B-Instruct-v0.2"
+
+
+client = InferenceClient(model_name)
+
+tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
+
+print("model start loading")
+model = AutoModelForCausalLM.from_pretrained(model_name,
+                                             device_map="auto",
+                                             trust_remote_code=False,
+                                             revision="main")
+print("model loaded")
+
+# model = AutoModelForCausalLM.from_pretrained(config["model_checkpoint"],
+#                                                 device_map="auto",
+#                                                 trust_remote_code=False,
+#                                                 revision="main")
 
-print("test")
 
 def respond(
     message,
@@ -20,31 +36,12 @@ def respond(
 ):
     messages = [{"role": "system", "content": system_message}]
 
-    for val in history:
-        if val[0]:
-            messages.append({"role": "user", "content": val[0]})
-        if val[1]:
-            messages.append({"role": "assistant", "content": val[1]})
-
-    messages.append({"role": "user", "content": message})
-
-    response = ""
-
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
+    request = message
+    context = get_context(request, config["top_k"])
+    response = get_bot_response(request, context, model, tokenizer)
 
-        response += token
-        yield response
+    return response
 
-"""
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-"""
 demo = gr.ChatInterface(
     respond,
     additional_inputs=[

requirements.txt

@@ -3,8 +3,3 @@ torch
 transformers
 llama_index
 llama-index-embeddings-huggingface
-accelerate
-optimum
-bitsandbytes
-auto-gptq
-peft