Update app.py

app.py

@@ -4,7 +4,13 @@ import spaces
 from transformers import GemmaTokenizer, AutoModelForCausalLM
 from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
 from threading import Thread
+import gradio as gr
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+import transformers
+import torch
+from peft import PeftModel, PeftConfig
 
+access_token = os.getenv('HF_TOKEN')
 # Set an environment variable
 HF_TOKEN = os.environ.get("HF_TOKEN", None)
 
@@ -48,13 +54,12 @@ h1 {
 }
 """
 
-# Load the tokenizer and model
-tokenizer = AutoTokenizer.from_pretrained("physician-ai/mistral-finetuned1")
-model = AutoModelForCausalLM.from_pretrained("physician-ai/mistral-finetuned1", device_map="auto")  # to("cuda:0") 
-terminators = [
-    tokenizer.eos_token_id,
-    tokenizer.convert_tokens_to_ids("<|eot_id|>")
-]
+
+#config = PeftConfig.from_pretrained("physician-ai/mistral-finetuned1",use_auth_token=access_token)
+model = AutoModelForCausalLM.from_pretrained("mistralai/Mistral-7B-Instruct-v0.2",use_auth_token=access_token)
+model = PeftModel.from_pretrained(model, "physician-ai/mistral-finetuned1",use_auth_token=access_token)
+tokenizer = AutoTokenizer.from_pretrained("physician-ai/mistral-finetuned1",use_auth_token=access_token)
+text_pipeline = pipeline("text-generation", model=model, tokenizer=tokenizer, max_new_tokens=1024, temperature=0.8, top_p=0.95, repetition_penalty=1.15)
 
 @spaces.GPU(duration=120)
 def chat_llama3_8b(message: str,