Update app.py

app.py

@@ -22,7 +22,7 @@ tokenizer = AutoTokenizer.from_pretrained(model, token = access_token, trust_rem
 tokenizer.pad_token = tokenizer.eos_token
 tokenizer.padding_side = "right"
 
-model = AutoModelForCausalLM.from_pretrained(model, trust_remote_code=True, token = access_token, device_map = 'cuda', offload_folder="offload/")
+model = AutoModelForCausalLM.from_pretrained(model, trust_remote_code=True, token = access_token, device_map="auto", load_in_8bit=True, offload_folder="offload/")
 model = PeftModel.from_pretrained(model, peft_model, offload_folder="offload/")
 
 model = model.eval()