Can't run the example in https://huggingface.co/transformers/v4.9.2/model_doc/blenderbot.html#transformers.BlenderbotModel
Author: OliverZijia
Pull request: no

Hi,

I just encountered some issue while running this example code:
from transformers import BlenderbotSmallTokenizer, BlenderbotSmallForConditionalGeneration
mname = 'facebook/blenderbot_small-90M'
model = BlenderbotSmallForConditionalGeneration.from_pretrained(mname)
tokenizer = BlenderbotSmallTokenizer.from_pretrained(mname)
UTTERANCE = "My friends are cool but they eat too many carbs."
print("Human: ", UTTERANCE)
inputs = tokenizer([UTTERANCE], return_tensors='pt')
inputs.keys()
inputs.pop("token_type_ids")
reply_ids = model.generate(**inputs)
print("Bot: ", tokenizer.batch_decode(reply_ids, skip_special_tokens=True)[0])

REPLY = "I'm not sure"
print("Human: ", REPLY)
NEXT_UTTERANCE = (
"My friends are cool but they eat too many carbs."
)
inputs = tokenizer([NEXT_UTTERANCE], return_tensors='pt')
inputs.pop("token_type_ids")
next_reply_ids = model.generate(**inputs)
print("Bot: ", tokenizer.batch_decode(next_reply_ids, skip_special_tokens=True)[0])

Then I got this error:
KeyError                                  Traceback (most recent call last)
Input In [1], in <cell line: 9>()
      7 inputs = tokenizer([UTTERANCE], return_tensors='pt')
      8 inputs.keys()
      9 inputs.pop("token_type_ids")
     10 # inputs.pop("input_ids")
     11 reply_ids = model.generate(**inputs)

File /opt/conda/lib/python3.8/_collections_abc.py:795, in MutableMapping.pop(self, key, default)
    791 '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value.
    792   If key is not found, d is returned if given, otherwise KeyError is raised.
    793 '''
    794 try:
--> 795     value = self[key]
    796 except KeyError:
    797     if default is self.__marker:

File /opt/conda/lib/python3.8/site-packages/transformers/tokenization_utils_base.py:236, in BatchEncoding.__getitem__(self, item)
    229 """
    230 If the key is a string, returns the value of the dict associated to `key` ('input_ids', 'attention_mask',
    231 etc.).
    232 
    233 If the key is an integer, get the `tokenizers.Encoding` for batch item with index `key`.
    234 """
    235 if isinstance(item, str):
--> 236     return self.data[item]
    237 elif self._encodings is not None:
    238     return self._encodings[item]

KeyError: 'token_type_ids'

I found the keys in input are:
dict_keys(['input_ids', 'attention_mask'])
then i change 'token_type_ides ' to 'input_ids', then i got this error:
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Input In [2], in <cell line: 11>()
      9 # inputs.pop("token_type_ids")
     10 inputs.pop("input_ids")
---> 11 reply_ids = model.generate(**inputs)
     12 print("Bot: ", tokenizer.batch_decode(reply_ids, skip_special_tokens=True)[0])
     14 REPLY = "I'm not sure"

File /opt/conda/lib/python3.8/site-packages/torch/autograd/grad_mode.py:27, in _DecoratorContextManager.__call__.<locals>.decorate_context(*args, **kwargs)
     24 @functools.wraps(func)
     25 def decorate_context(*args, **kwargs):
     26     with self.clone():
---> 27         return func(*args, **kwargs)

File /opt/conda/lib/python3.8/site-packages/transformers/generation_utils.py:1182, in GenerationMixin.generate(self, inputs, max_length, min_length, do_sample, early_stopping, num_beams, temperature, top_k, top_p, typical_p, repetition_penalty, bad_words_ids, force_words_ids, bos_token_id, pad_token_id, eos_token_id, length_penalty, no_repeat_ngram_size, encoder_no_repeat_ngram_size, num_return_sequences, max_time, max_new_tokens, decoder_start_token_id, use_cache, num_beam_groups, diversity_penalty, prefix_allowed_tokens_fn, logits_processor, renormalize_logits, stopping_criteria, constraints, output_attentions, output_hidden_states, output_scores, return_dict_in_generate, forced_bos_token_id, forced_eos_token_id, remove_invalid_values, synced_gpus, exponential_decay_length_penalty, **model_kwargs)
   1175     model_kwargs["attention_mask"] = self._prepare_attention_mask_for_generation(
   1176         inputs_tensor, pad_token_id, eos_token_id
   1177     )
   1179 if self.config.is_encoder_decoder and "encoder_outputs" not in model_kwargs:
   1180     # if model is encoder decoder encoder_outputs are created
   1181     # and added to `model_kwargs`
-> 1182     model_kwargs = self._prepare_encoder_decoder_kwargs_for_generation(
   1183         inputs_tensor, model_kwargs, model_input_name
   1184     )
   1186 # 4. Prepare `input_ids` which will be used for auto-regressive generation
   1187 if self.config.is_encoder_decoder:

File /opt/conda/lib/python3.8/site-packages/transformers/generation_utils.py:525, in GenerationMixin._prepare_encoder_decoder_kwargs_for_generation(self, inputs_tensor, model_kwargs, model_input_name)
    523 encoder_kwargs["return_dict"] = True
    524 encoder_kwargs[model_input_name] = inputs_tensor
--> 525 model_kwargs["encoder_outputs"]: ModelOutput = encoder(**encoder_kwargs)
    527 return model_kwargs

File /opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py:1110, in Module._call_impl(self, *input, **kwargs)
   1106 # If we don't have any hooks, we want to skip the rest of the logic in
   1107 # this function, and just call forward.
   1108 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1109         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1110     return forward_call(*input, **kwargs)
   1111 # Do not call functions when jit is used
   1112 full_backward_hooks, non_full_backward_hooks = [], []

File /opt/conda/lib/python3.8/site-packages/transformers/models/blenderbot_small/modeling_blenderbot_small.py:780, in BlenderbotSmallEncoder.forward(self, input_ids, attention_mask, head_mask, inputs_embeds, output_attentions, output_hidden_states, return_dict)
    773         layer_outputs = torch.utils.checkpoint.checkpoint(
    774             create_custom_forward(encoder_layer),
    775             hidden_states,
    776             attention_mask,
    777             (head_mask[idx] if head_mask is not None else None),
    778         )
    779     else:
--> 780         layer_outputs = encoder_layer(
    781             hidden_states,
    782             attention_mask,
    783             layer_head_mask=(head_mask[idx] if head_mask is not None else None),
    784             output_attentions=output_attentions,
    785         )
    787     hidden_states = layer_outputs[0]
    789 if output_attentions:

File /opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py:1110, in Module._call_impl(self, *input, **kwargs)
   1106 # If we don't have any hooks, we want to skip the rest of the logic in
   1107 # this function, and just call forward.
   1108 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1109         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1110     return forward_call(*input, **kwargs)
   1111 # Do not call functions when jit is used
   1112 full_backward_hooks, non_full_backward_hooks = [], []

File /opt/conda/lib/python3.8/site-packages/transformers/models/blenderbot_small/modeling_blenderbot_small.py:311, in BlenderbotSmallEncoderLayer.forward(self, hidden_states, attention_mask, layer_head_mask, output_attentions)
    299 """
    300 Args:
    301     hidden_states (`torch.FloatTensor`): input to the layer of shape `(seq_len, batch, embed_dim)`
   (...)
    308         returned tensors for more detail.
    309 """
    310 residual = hidden_states
--> 311 hidden_states, attn_weights, _ = self.self_attn(
    312     hidden_states=hidden_states,
    313     attention_mask=attention_mask,
    314     layer_head_mask=layer_head_mask,
    315     output_attentions=output_attentions,
    316 )
    317 hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
    318 hidden_states = residual + hidden_states

File /opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py:1110, in Module._call_impl(self, *input, **kwargs)
   1106 # If we don't have any hooks, we want to skip the rest of the logic in
   1107 # this function, and just call forward.
   1108 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1109         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1110     return forward_call(*input, **kwargs)
   1111 # Do not call functions when jit is used
   1112 full_backward_hooks, non_full_backward_hooks = [], []

File /opt/conda/lib/python3.8/site-packages/transformers/models/blenderbot_small/modeling_blenderbot_small.py:225, in BlenderbotSmallAttention.forward(self, hidden_states, key_value_states, past_key_value, attention_mask, layer_head_mask, output_attentions)
    223 if attention_mask is not None:
    224     if attention_mask.size() != (bsz, 1, tgt_len, src_len):
--> 225         raise ValueError(
    226             f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}"
    227         )
    228     attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask
    229     attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)

ValueError: Attention mask should be of size (1, 1, 1, 1), but is torch.Size([1, 1, 12, 12])


Guess should be triggered by the version of torch or transformers? Could you give me the version that can run this code properly?

Best regards,
Zijia


Comments:
Remove this line:
`inputs.pop("token_type_ids")`
As for the `inputs.keys()`, the keys aren't assigned to any variable or used for any form of auxiliary calculation, so it effectively does nothing. 

In other words, just run this:
```python
from transformers import BlenderbotSmallTokenizer, BlenderbotSmallForConditionalGeneration

mname = 'facebook/blenderbot_small-90M'
model = BlenderbotSmallForConditionalGeneration.from_pretrained(mname)
tokenizer = BlenderbotSmallTokenizer.from_pretrained(mname)
UTTERANCE = "My friends are cool but they eat too many carbs."
print("Human: ", UTTERANCE)
inputs = tokenizer([UTTERANCE], max_length=512, truncation=True, return_tensors='pt')
reply_ids = model.generate(**inputs)
print("Bot: ", tokenizer.batch_decode(reply_ids, skip_special_tokens=True)[0])
```

Thanks @shermansiu, the problem is solved, thanks for your prompt reply :-)

You're welcome! :smile: