Update app.py

app.py

@@ -18,10 +18,7 @@ model_name = "microsoft/Orca-2-13b"
 #     os.makedirs(offload_folder)
 
 tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False)
-model = AutoModelForCausalLM.from_pretrained(model_name)
-model = model.to(torch.bfloat16)
-model = model.to(device)
-
+model = transformers.AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", load_in_8bit=True)
 
 class OrcaChatBot:
     def __init__(self, model, tokenizer, system_message="You are Orca, an AI language model created by Microsoft. You are a cautious assistant. You carefully follow instructions. You are helpful and harmless and you follow ethical guidelines and promote positive behavior."):