attention mask is not set Warning!
#6
by
LearnToGrow
- opened
I am using transformers 4.46.3
from transformers import AutoTokenizer, AutoModelForCausalLM
from transformers import BitsAndBytesConfig
import torch
nf4_config = BitsAndBytesConfig(
load_in_4bit=True,
bnb_4bit_use_double_quant=True,
bnb_4bit_quant_type="nf4",
bnb_4bit_compute_dtype=torch.bfloat16)
# Load the tokenizer and model
tokenizer = AutoTokenizer.from_pretrained(
"nvidia/Mistral-NeMo-Minitron-8B-Instruct")
model = AutoModelForCausalLM.from_pretrained(
"nvidia/Mistral-NeMo-Minitron-8B-Instruct", quantization_config=nf4_config)
messages = [
{
"role": "system",
"content": "You are an annotator to extract verbs from sentences in english",
},
{"role": "user", "content": "I like pasta. I eat pasta. I enjoy football\n Based on these sentences, extract the result in a json format {{'verb':[]}}"},
{"role": "assistant", "content": "\{'verb':[like, eat, enjoy]\}"},
{"role": "user", "content": "I am going home. I travel to USA. I vote for election.\n Based on these sentences, extract the result in a json format {{'verb':[]}}"}
]
tokenized_chat = tokenizer.apply_chat_template(
messages, tokenize=True, add_generation_prompt=False, return_tensors="pt")
tokenized_chat = tokenized_chat.to(model.device)
outputs = model.generate(tokenized_chat, stop_strings=[
"<extra_id_1>"], tokenizer=tokenizer, max_new_tokens=1024,
pad_token_id=tokenizer.eos_token_id
)
print(tokenizer.decode(outputs[0]))
I got this warning:
The attention mask is not set and cannot be inferred from input because pad token is same as eos token. As a consequence, you may observe unexpected behavior. Please pass your input's attention_mask
to obtain reliable results.