RuntimeError: The size of tensor a (1024) must match the size of tensor b (1025) at non-singleton dimension 3
Author: yanan1116
Pull request: no

Here I try to use gpt2 to generation the text under the prompt text. I have several datasets, some of them, such as AG_NEWS and POP_NEWS, are made of short sentences while when I use YAHOO_NEWS, consisting of longer sentences, the error came out.
Anything to modify for my codes? 
Thanks.

```
from transformers import (
    CTRLLMHeadModel,
    CTRLTokenizer,
    GPT2LMHeadModel,
    GPT2Tokenizer,
    OpenAIGPTLMHeadModel,
    OpenAIGPTTokenizer,
    TransfoXLLMHeadModel,
    TransfoXLTokenizer,
    XLMTokenizer,
    XLMWithLMHeadModel,
    XLNetLMHeadModel,
    XLNetTokenizer,
)


class generation():
    def __init__(self, model_name='gpt2',num_return_sequences=1):
        self.model_name = model_name
        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        self.MODEL_CLASSES = {
                    "gpt2": (GPT2LMHeadModel, GPT2Tokenizer),
                    "ctrl": (CTRLLMHeadModel, CTRLTokenizer),
                    "openai-gpt": (OpenAIGPTLMHeadModel, OpenAIGPTTokenizer),
                    "xlnet-base-cased": (XLNetLMHeadModel, XLNetTokenizer),
                    "transfo-xl": (TransfoXLLMHeadModel, TransfoXLTokenizer),
                    "xlm": (XLMWithLMHeadModel, XLMTokenizer),
                }
        self.length = 100
        self.k = 0
        self.p = 0.9
        self.num_return_sequences = num_return_sequences
        self.model_class, self.tokenizer_class = self.MODEL_CLASSES[self.model_name]
        self.tokenizer = self.tokenizer_class.from_pretrained(self.model_name)
        self.model = self.model_class.from_pretrained(self.model_name)
        self.model.to(self.device)
        if self.model_name == "xlnet-base-cased":
            self.p=0.95
            self.k=60

        self.length = self.adjust_length_to_model(self.length, max_sequence_length=self.model.config.max_position_embeddings)

        if self.model_name == 'ctrl':
            self.temperature = 0.3
            self.repetition_penalty = 1.2
        else:
            self.temperature = 1.0
            self.repetition_penalty = 1.0

    def adjust_length_to_model(self, length, max_sequence_length):
        if length < 0 and max_sequence_length > 0:
            length = max_sequence_length
        elif 0 < max_sequence_length < length:
            length = max_sequence_length  # No generation bigger than model size
        elif length < 0:
            length = 1000  # avoid infinite loop
        return length


    def ctrl_label2prefix(self, label):
        # https://github.com/salesforce/ctrl/blob/master/control_codes.py
        '''
        'Pregnancy Christianity Explain Fitness Saving Ask Ass Joke Questions Thoughts Retail 
        Feminism Writing Atheism Netflix Computing Opinion Alone Funny Gaming Human India Joker Diet 
        Legal Norman Tip Weight Movies Running Science Horror Confession Finance Politics Scary Support 
        Technologies Teenage Event Learned Notion Wikipedia Books Extract Confessions Conspiracy Links 
        Narcissus Relationship Relationships Reviews News Translation multilingual'
        '''
        return 'News'

        if label in ('Sci/Tech', 'tech'):
            return 'Technologies'
        elif label in ('politics'):
            return 'Politics'
        elif label in ('Sports', 'sport'):
            return 'Fitness'
        else:
            return 'News'

    def augment(self, prompt_text):
        if self.model_name == 'ctrl':
            prefix = 'News '
        else:
            prefix = ''
        encoded_prompt = self.tokenizer.encode(prefix  + prompt_text, add_special_tokens=False, return_tensors="pt")
        
        encoded_prompt = encoded_prompt.to(self.device)

        if encoded_prompt.size()[-1] == 0:
            input_ids = None
        else:
            input_ids = encoded_prompt

        output_sequences = self.model.generate(
            input_ids=input_ids,
            max_length= self.length + len(encoded_prompt[0]),
            temperature=self.temperature,
            top_k=self.k,
            top_p=self.p,
            repetition_penalty=self.repetition_penalty,
            do_sample=True,
            num_return_sequences=self.num_return_sequences,
        )

        # Decode text
        text_generated = self.tokenizer.decode(output_sequences[0][len(encoded_prompt[0]):], clean_up_tokenization_spaces=True)
        return text_generated


# unit test
'''
augmentor = generation('gpt2')

prompt_text = "Microsoft has said it will replace more than 14 million power cables for its Xbox consoles due to safety concerns."
prompt_text = "Versace art portfolio up for sale The art collection of murdered fashion designer Gianni Versace could fetch \
up to £9m ($17m) when it is auctioned in New York and \
London later this year. <eod> 
 <eos>"

augmentor.augment(prompt_text)
'''
```

ERROR information:

> File "baseline_classifier.py", line 45, in run_benchmark
>     ds.df_train['content_aug'] = ds.df_train['content'].map(lambda x: augmentor.augment(x))
>   File "/workspace/.conda/miniconda/lib/python3.7/site-packages/pandas/core/series.py", line 3382, in map
>     arg, na_action=na_action)
>   File "/workspace/.conda/miniconda/lib/python3.7/site-packages/pandas/core/base.py", line 1218, in _map_values
>     new_values = map_f(values, mapper)
>   File "pandas/_libs/lib.pyx", line 2217, in pandas._libs.lib.map_infer
>   File "baseline_classifier.py", line 45, in <lambda>
>     ds.df_train['content_aug'] = ds.df_train['content'].map(lambda x: augmentor.augment(x))
>   File "/workspace/user-workspace/topic_classification_augmentation/aug_generation.py", line 110, in augment
>     num_return_sequences=self.num_return_sequences,
>   File "/workspace/.conda/miniconda/lib/python3.7/site-packages/torch/autograd/grad_mode.py", line 15, in decorate_context
>     return func(*args, **kwargs)
>   File "/workspace/.conda/miniconda/lib/python3.7/site-packages/transformers/generation_utils.py", line 1019, in generate
>     **model_kwargs,
>   File "/workspace/.conda/miniconda/lib/python3.7/site-packages/transformers/generation_utils.py", line 1486, in sample
>     output_hidden_states=output_hidden_states,
>   File "/workspace/.conda/miniconda/lib/python3.7/site-packages/torch/nn/modules/module.py", line 550, in __call__
>     result = self.forward(*input, **kwargs)
>   File "/workspace/.conda/miniconda/lib/python3.7/site-packages/transformers/models/gpt2/modeling_gpt2.py", line 917, in forward
>     return_dict=return_dict,
>   File "/workspace/.conda/miniconda/lib/python3.7/site-packages/torch/nn/modules/module.py", line 550, in __call__
>     result = self.forward(*input, **kwargs)
>   File "/workspace/.conda/miniconda/lib/python3.7/site-packages/transformers/models/gpt2/modeling_gpt2.py", line 760, in forward
>     output_attentions=output_attentions,
>   File "/workspace/.conda/miniconda/lib/python3.7/site-packages/torch/nn/modules/module.py", line 550, in __call__
>     result = self.forward(*input, **kwargs)
>   File "/workspace/.conda/miniconda/lib/python3.7/site-packages/transformers/models/gpt2/modeling_gpt2.py", line 296, in forward
>     output_attentions=output_attentions,
>   File "/workspace/.conda/miniconda/lib/python3.7/site-packages/torch/nn/modules/module.py", line 550, in __call__
>     result = self.forward(*input, **kwargs)
>   File "/workspace/.conda/miniconda/lib/python3.7/site-packages/transformers/models/gpt2/modeling_gpt2.py", line 241, in forward
>     attn_outputs = self._attn(query, key, value, attention_mask, head_mask, output_attentions)
>   File "/workspace/.conda/miniconda/lib/python3.7/site-packages/transformers/models/gpt2/modeling_gpt2.py", line 176, in _attn
>     w = torch.where(mask.bool(), w, self.masked_bias.to(w.dtype))
> RuntimeError: The size of tensor a (1024) must match the size of tensor b (1025) at non-singleton dimension 3

Comments:
This issue has been automatically marked as stale because it has not had recent activity. If you think this still needs to be addressed please comment on this thread.

Please note that issues that do not follow the [contributing guidelines](https://github.com/huggingface/transformers/blob/master/CONTRIBUTING.md) are likely to be ignored.

I am also having the same issue and I don't have input/output longer than 1024/.

Experiencing the same problem using the 2048 size with GPT-J.