Spaces:
Paused
Paused
File size: 1,721 Bytes
b49b83b d8b2749 11a35d1 b49b83b 11a35d1 e02030a b49b83b 11a35d1 46d9167 11a35d1 e02030a b49b83b 11a35d1 46d9167 11a35d1 e02030a b49b83b e622ac4 e02030a 3ea1454 e622ac4 e02030a e622ac4 e02030a b49b83b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 |
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM
from peft import PeftModel
import gradio as gr
# Load model and tokenizer
base_model_name = "unsloth/gemma-3-12b-it-unsloth-bnb-4bit"
adapter_name = "adarsh3601/my_gemma3_pt"
device = "cuda" if torch.cuda.is_available() else "cpu"
base_model = AutoModelForCausalLM.from_pretrained(
base_model_name,
device_map="auto",
torch_dtype=torch.float16,
load_in_4bit=True
)
tokenizer = AutoTokenizer.from_pretrained(base_model_name)
model = PeftModel.from_pretrained(base_model, adapter_name)
model.to(device)
# Chat function with debug/error handling
def chat(message):
if not message or not message.strip():
return "Please enter a message."
try:
# Tokenize
inputs = tokenizer(message, return_tensors="pt")
inputs = {k: v.to(device) for k, v in inputs.items()}
# Cast to float16 only if model is on CUDA
if device == "cuda":
inputs = {k: v.half() for k, v in inputs.items()}
# Generate
outputs = model.generate(
**inputs,
max_new_tokens=150,
do_sample=True,
temperature=0.7,
top_k=50,
top_p=0.95
)
response = tokenizer.decode(outputs[0], skip_special_tokens=True)
return response
except RuntimeError as e:
if "CUDA error" in str(e):
return "⚠️ CUDA error during generation. Try restarting or changing your input."
return f"Unexpected error: {e}"
except Exception as e:
return f"Error: {e}"
# Gradio UI
iface = gr.Interface(fn=chat, inputs="text", outputs="text", title="Gemma Chatbot")
iface.launch()
|