Tonic commited on
Commit
7a71692
1 Parent(s): 83b24ab

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +1 -2
app.py CHANGED
@@ -14,8 +14,7 @@ os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'max_split_size_mb:50'
14
  device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
15
  model_name = "allenai/tulu-2-dpo-70b"
16
  tokenizer = AutoTokenizer.from_pretrained("allenai/tulu-2-dpo-70b")
17
- model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto")
18
- model = model.to(dtype=torch.bfloat16)
19
 
20
  class XgenChatBot:
21
  def __init__(self, model, tokenizer, system_message="You are 🌷Tulu, an AI language model created by Tonic-AI. You are a cautious assistant. You carefully follow instructions. You are helpful and harmless and you follow ethical guidelines and promote positive behavior."):
 
14
  device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
15
  model_name = "allenai/tulu-2-dpo-70b"
16
  tokenizer = AutoTokenizer.from_pretrained("allenai/tulu-2-dpo-70b")
17
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.bfloat16, use_flash_attention_2=True, device_map="auto")
 
18
 
19
  class XgenChatBot:
20
  def __init__(self, model, tokenizer, system_message="You are 🌷Tulu, an AI language model created by Tonic-AI. You are a cautious assistant. You carefully follow instructions. You are helpful and harmless and you follow ethical guidelines and promote positive behavior."):