Update app.py

app.py

@@ -1,9 +1,13 @@
 import gradio as gr
 import torch
-from transformers import pipeline
+from transformers import AutoTokenizer
+from awq import AutoAWQForCausalLM
+
+model_path = "bragour/Camel-7b-chat-awq"
+
+model  = AutoAWQForCausalLM.from_quantized(model_path, fuse_layers=True, trust_remote_code=False, safetensors=True)
+tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=False)
 
-# Initialize the inference client with the model ID
-client = pipeline(model="bragour/Camel-7b-chat")
 
 def respond(
     message,
@@ -11,15 +15,20 @@ def respond(
     temperature,
     top_p,
 ):
+    formatted_prompt = f"<s>[INST]{message}[/INST]"
+
+    tokens = tokenizer(formatted_prompt,return_tensors='pt').input_ids.cuda()
+        
     # Generate the response from the API
-    result = client(
-        message,
+    result = model.generate(
+        tokens,
+        do_sample=False
         max_new_tokens=max_tokens,
         temperature=temperature,
         top_p=top_p,
     )
     
-    response = result[0]['generated_text']
+    response = tokenizer.decode(result[0], skip_special_tokens=True)
 
     return response