Update app.py

checking unsloth

app.py

@@ -1,23 +1,50 @@
 import os
 import gradio as gr
-from huggingface_hub import InferenceClient
-from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
-import torch
+from transformers import TextStreamer
+from peft import PeftModel
+from unsloth import FastLanguageModel
+
 # Load your model and tokenizer
 model_name = "Renjith95/renj-portfolio-finetuned-model"  # Replace with your model name
-auth_token = os.getenv("HF_TOKEN")  # Get token from environment variable
-tokenizer = AutoTokenizer.from_pretrained(model_name, use_auth_token=auth_token)
-model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, use_auth_token=auth_token)
+auth_token = os.getenv("HF_TOKEN")   # Now this should work
+# print("Auth token:", auth_token)  # To verify it's loaded
+
+# Loading the base model and applying the local adapter.
+max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!
+dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+
+load_in_4bit = True # Use 4bit quantization to reduce memory usage. Can be False.
+
+# 4bit pre quantized models we support for 4x faster downloading + no OOMs.
+fourbit_models = [
+    "unsloth/mistral-7b-bnb-4bit",
+    "unsloth/mistral-7b-instruct-v0.2-bnb-4bit",
+    "unsloth/llama-2-7b-bnb-4bit",
+    "unsloth/llama-2-13b-bnb-4bit",
+    "unsloth/codellama-34b-bnb-4bit",
+    "unsloth/tinyllama-bnb-4bit",
+    "unsloth/gemma-7b-bnb-4bit", # New Google 6 trillion tokens model 2.5x faster!
+    "unsloth/gemma-2b-bnb-4bit",
+] # More models at https://huggingface.co/unsloth
+
+model, tokenizer = FastLanguageModel.from_pretrained(
+    model_name = "unsloth/mistral-7b-instruct-v0.3-bnb-4bit", # Choose ANY! eg teknium/OpenHermes-2.5-Mistral-7B
+    max_seq_length = max_seq_length,
+    dtype = dtype,
+    load_in_4bit = load_in_4bit,
+    token = auth_token, # use one if using gated models like meta-llama/Llama-2-7b-hf
+)
+model = PeftModel.from_pretrained(model, "Renjith95/renj-portfolio-finetuned-adapter", use_auth_token=auth_token)
+FastLanguageModel.for_inference(model)
 
 
+# tokenizer = AutoTokenizer.from_pretrained(model_name, use_auth_token=auth_token)
+# model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, use_auth_token=auth_token)
+text_streamer = TextStreamer(tokenizer, skip_prompt = True)
 """
 For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
 """
-# client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
-
-
-def respond(message, history, system_message, max_tokens, temperature, top_p):
-    messages = [{"role": "system", "content": system_message}]
+def respond(message, history):
+    messages = []
     for user_msg, assistant_msg in history:
         messages.append({"role": "user", "content": user_msg})
         messages.append({"role": "assistant", "content": assistant_msg})
@@ -32,35 +59,20 @@ def respond(message, history, system_message, max_tokens, temperature, top_p):
 
     outputs = model.generate(
         input_ids=inputs,
-        max_new_tokens=max_tokens,
+        max_new_tokens=512,
         use_cache=True,
-        temperature=temperature,
-        top_p=top_p,
+        temperature=0.7,
+        top_p=0.95,
     )
-    response = tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]
-    
-    # Assuming your model's response is the last part after the user's message
-    response = response.split(message)[-1].strip()  
-    yield response
-"""
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-"""
+
+    response = tokenizer.decode(outputs[0][inputs.shape[1]:], skip_special_tokens=True)
+    return response
+
 demo = gr.ChatInterface(
     respond,
-    additional_inputs=[
-        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
-        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p (nucleus sampling)",
-        ),
-    ],
+    title="Renj Chatbot",
+    description="Ask me anything about my portfolio and projects."
 )
 
-
 if __name__ == "__main__":
     demo.launch(share = True)