updated app.py

app.py

@@ -1,7 +1,7 @@
 import gradio as gr
 import torch
 from peft import PeftModel, PeftConfig
-from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
+from transformers import AutoTokenizer, AutoModelForCausalLM
 
 
 #Loading model
@@ -10,7 +10,7 @@ quantization_config = BitsAndBytesConfig(load_in_4bit=True)
 model_path = "parsanna17/finetune_starcoder2_with_R_data"
 checkpoint = "bigcode/starcoder2-3b"
 config = PeftConfig.from_pretrained(model_path)
-model = AutoModelForCausalLM.from_pretrained(checkpoint, quantization_config=quantization_config)
+model = AutoModelForCausalLM.from_pretrained(checkpoint, torch_dtype=torch.bfloat16)
 model = PeftModel.from_pretrained(model, model_path).to(device)
 tokenizer = AutoTokenizer.from_pretrained(checkpoint)