Use chat template (#1)

- Use chat template (2530c083203c9428de90dcd889365c43f396f06e)


Co-authored-by: Jade <euclaise@users.noreply.huggingface.co>

app.py

@@ -26,10 +26,9 @@ def generate_text(usertitle, content, temperature, max_length, N=3):
     # 'title': usertitle,
     # 'content': content
     # }
-    input_text = f"[[[title:]]] {usertitle}\n[[[content:]]]{content}\n\n"
-    inputs = tokenizer(input_text, return_tensors='pt').to('cuda')
-    attention_mask = torch.ones(inputs['input_ids'].shape, dtype=torch.long, device='cuda')
-    generated_sequences = model.generate(inputs['input_ids'], attention_mask=attention_mask, temperature=temperature, max_length=max_length, pad_token_id=tokenizer.eos_token_id, num_return_sequences=N, do_sample=True)
+    input_ids = tokenizer.apply_chat_template(input_text, return_tensors='pt').to('cuda')
+    attention_mask = torch.ones(input_ids.shape, dtype=torch.long, device='cuda')
+    generated_sequences = model.generate(input_ids, attention_mask=attention_mask, temperature=temperature, max_length=max_length, pad_token_id=tokenizer.eos_token_id, num_return_sequences=N, do_sample=True)
     decoded_sequences = [tokenizer.decode(g, skip_special_tokens=True) for g in generated_sequences]
     def score(sequence):
         inputs = rm_tokenizer(sequence, return_tensors='pt', padding=True, truncation=True, max_length=512).to('cuda')