IndexError with Reformer Model when padding the sequence
Author: RobinGeibel
Pull request: no

Hi, 

I am trying to use Reformer for text classification, but I am getting the following RuntimeError. I tried both using a custom head and Huggingface's Reformer for sequence classification. 

The code runs fine when I swap Reformer for BigBird. I am using the google/reformer-crime-and-punishment checkpoint and have padded the text to max_length.

```
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
[<ipython-input-27-5c75e8f9ad52>](https://a55g9zay06c-496ff2e9c6d22116-0-colab.googleusercontent.com/outputframe.html?vrz=colab-20220616-060045-RC00_455344893#) in <module>()
      9   for batch in training_loader:
     10       batch = {k: v.to(config['device']) for k, v in batch.items()}
---> 11       outputs = model(**batch)
     12       loss = criterion(outputs, batch['labels'])
     13       loss.backward()

8 frames
[/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py](https://a55g9zay06c-496ff2e9c6d22116-0-colab.googleusercontent.com/outputframe.html?vrz=colab-20220616-060045-RC00_455344893#) in _call_impl(self, *input, **kwargs)
   1108         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1109                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1110             return forward_call(*input, **kwargs)
   1111         # Do not call functions when jit is used
   1112         full_backward_hooks, non_full_backward_hooks = [], []

[/usr/local/lib/python3.7/dist-packages/transformers/models/reformer/modeling_reformer.py](https://a55g9zay06c-496ff2e9c6d22116-0-colab.googleusercontent.com/outputframe.html?vrz=colab-20220616-060045-RC00_455344893#) in forward(self, input_ids, position_ids, attention_mask, head_mask, inputs_embeds, num_hashes, labels, output_hidden_states, output_attentions, return_dict)
   2554             output_hidden_states=output_hidden_states,
   2555             output_attentions=output_attentions,
-> 2556             return_dict=return_dict,
   2557         )
   2558 

[/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py](https://a55g9zay06c-496ff2e9c6d22116-0-colab.googleusercontent.com/outputframe.html?vrz=colab-20220616-060045-RC00_455344893#) in _call_impl(self, *input, **kwargs)
   1108         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1109                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1110             return forward_call(*input, **kwargs)
   1111         # Do not call functions when jit is used
   1112         full_backward_hooks, non_full_backward_hooks = [], []

[/usr/local/lib/python3.7/dist-packages/transformers/models/reformer/modeling_reformer.py](https://a55g9zay06c-496ff2e9c6d22116-0-colab.googleusercontent.com/outputframe.html?vrz=colab-20220616-060045-RC00_455344893#) in forward(self, input_ids, attention_mask, position_ids, head_mask, inputs_embeds, num_hashes, past_buckets_states, use_cache, output_hidden_states, output_attentions, return_dict)
   2102             position_ids=position_ids,
   2103             inputs_embeds=inputs_embeds,
-> 2104             start_idx_pos_encodings=start_idx_pos_encodings,
   2105         )
   2106 

[/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py](https://a55g9zay06c-496ff2e9c6d22116-0-colab.googleusercontent.com/outputframe.html?vrz=colab-20220616-060045-RC00_455344893#) in _call_impl(self, *input, **kwargs)
   1108         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1109                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1110             return forward_call(*input, **kwargs)
   1111         # Do not call functions when jit is used
   1112         full_backward_hooks, non_full_backward_hooks = [], []

[/usr/local/lib/python3.7/dist-packages/transformers/models/reformer/modeling_reformer.py](https://a55g9zay06c-496ff2e9c6d22116-0-colab.googleusercontent.com/outputframe.html?vrz=colab-20220616-060045-RC00_455344893#) in forward(self, input_ids, position_ids, inputs_embeds, start_idx_pos_encodings)
    253 
    254         if inputs_embeds is None:
--> 255             inputs_embeds = self.word_embeddings(input_ids)
    256 
    257         if position_ids.shape[-1] > self.max_position_embeddings:

[/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py](https://a55g9zay06c-496ff2e9c6d22116-0-colab.googleusercontent.com/outputframe.html?vrz=colab-20220616-060045-RC00_455344893#) in _call_impl(self, *input, **kwargs)
   1108         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1109                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1110             return forward_call(*input, **kwargs)
   1111         # Do not call functions when jit is used
   1112         full_backward_hooks, non_full_backward_hooks = [], []

[/usr/local/lib/python3.7/dist-packages/torch/nn/modules/sparse.py](https://a55g9zay06c-496ff2e9c6d22116-0-colab.googleusercontent.com/outputframe.html?vrz=colab-20220616-060045-RC00_455344893#) in forward(self, input)
    158         return F.embedding(
    159             input, self.weight, self.padding_idx, self.max_norm,
--> 160             self.norm_type, self.scale_grad_by_freq, self.sparse)
    161 
    162     def extra_repr(self) -> str:

[/usr/local/lib/python3.7/dist-packages/torch/nn/functional.py](https://a55g9zay06c-496ff2e9c6d22116-0-colab.googleusercontent.com/outputframe.html?vrz=colab-20220616-060045-RC00_455344893#) in embedding(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)
   2181         # remove once script supports set_grad_enabled
   2182         _no_grad_embedding_renorm_(weight, input, max_norm, norm_type)
-> 2183     return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)
   2184 
   2185 

RuntimeError: CUDA error: device-side assert triggered
```

I have tried the following example on CPU and I think padding is the problem:

```
This works:

from transformers import ReformerTokenizer, ReformerModel
import torch

tokenizer = ReformerTokenizer.from_pretrained("google/reformer-crime-and-punishment")
tokenizer.add_special_tokens({'pad_token': '[PAD]'})
model = ReformerModel.from_pretrained("google/reformer-crime-and-punishment")

inputs = tokenizer(["Hello, my dog is cute", "Hello, my dog is cute"], return_tensors="pt")
outputs = model(**inputs)

last_hidden_states = outputs.last_hidden_state
```

This doesn't:

```
from transformers import ReformerTokenizer, ReformerModel
import torch

tokenizer = ReformerTokenizer.from_pretrained("google/reformer-crime-and-punishment")
tokenizer.add_special_tokens({'pad_token': '[PAD]'})
model = ReformerModel.from_pretrained("google/reformer-crime-and-punishment")

inputs = tokenizer(["Hello, my dog is cute", "Hello, my dog is cute"], return_tensors="pt", padding='max_length', max_length=524288)
outputs = model(**inputs)

last_hidden_states = outputs.last_hidden_state
```

Error:

```
IndexError                                Traceback (most recent call last)
[<ipython-input-68-d9b515284501>](https://a55g9zay06c-496ff2e9c6d22116-0-colab.googleusercontent.com/outputframe.html?vrz=colab-20220616-060045-RC00_455344893#) in <module>()
      7 
      8 inputs = tokenizer(["Hello, my dog is cute", "Hello, my dog is cute"], return_tensors="pt", padding='max_length', max_length=524288)
----> 9 outputs = model(**inputs)
     10 
     11 last_hidden_states = outputs.last_hidden_state

6 frames
[/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py](https://a55g9zay06c-496ff2e9c6d22116-0-colab.googleusercontent.com/outputframe.html?vrz=colab-20220616-060045-RC00_455344893#) in _call_impl(self, *input, **kwargs)
   1108         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1109                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1110             return forward_call(*input, **kwargs)
   1111         # Do not call functions when jit is used
   1112         full_backward_hooks, non_full_backward_hooks = [], []

[/usr/local/lib/python3.7/dist-packages/transformers/models/reformer/modeling_reformer.py](https://a55g9zay06c-496ff2e9c6d22116-0-colab.googleusercontent.com/outputframe.html?vrz=colab-20220616-060045-RC00_455344893#) in forward(self, input_ids, attention_mask, position_ids, head_mask, inputs_embeds, num_hashes, past_buckets_states, use_cache, output_hidden_states, output_attentions, return_dict)
   2102             position_ids=position_ids,
   2103             inputs_embeds=inputs_embeds,
-> 2104             start_idx_pos_encodings=start_idx_pos_encodings,
   2105         )
   2106 

[/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py](https://a55g9zay06c-496ff2e9c6d22116-0-colab.googleusercontent.com/outputframe.html?vrz=colab-20220616-060045-RC00_455344893#) in _call_impl(self, *input, **kwargs)
   1108         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1109                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1110             return forward_call(*input, **kwargs)
   1111         # Do not call functions when jit is used
   1112         full_backward_hooks, non_full_backward_hooks = [], []

[/usr/local/lib/python3.7/dist-packages/transformers/models/reformer/modeling_reformer.py](https://a55g9zay06c-496ff2e9c6d22116-0-colab.googleusercontent.com/outputframe.html?vrz=colab-20220616-060045-RC00_455344893#) in forward(self, input_ids, position_ids, inputs_embeds, start_idx_pos_encodings)
    253 
    254         if inputs_embeds is None:
--> 255             inputs_embeds = self.word_embeddings(input_ids)
    256 
    257         if position_ids.shape[-1] > self.max_position_embeddings:

[/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py](https://a55g9zay06c-496ff2e9c6d22116-0-colab.googleusercontent.com/outputframe.html?vrz=colab-20220616-060045-RC00_455344893#) in _call_impl(self, *input, **kwargs)
   1108         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1109                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1110             return forward_call(*input, **kwargs)
   1111         # Do not call functions when jit is used
   1112         full_backward_hooks, non_full_backward_hooks = [], []

[/usr/local/lib/python3.7/dist-packages/torch/nn/modules/sparse.py](https://a55g9zay06c-496ff2e9c6d22116-0-colab.googleusercontent.com/outputframe.html?vrz=colab-20220616-060045-RC00_455344893#) in forward(self, input)
    158         return F.embedding(
    159             input, self.weight, self.padding_idx, self.max_norm,
--> 160             self.norm_type, self.scale_grad_by_freq, self.sparse)
    161 
    162     def extra_repr(self) -> str:

[/usr/local/lib/python3.7/dist-packages/torch/nn/functional.py](https://a55g9zay06c-496ff2e9c6d22116-0-colab.googleusercontent.com/outputframe.html?vrz=colab-20220616-060045-RC00_455344893#) in embedding(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)
   2181         # remove once script supports set_grad_enabled
   2182         _no_grad_embedding_renorm_(weight, input, max_norm, norm_type)
-> 2183     return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)
   2184 
   2185 

IndexError: index out of range in self

```








Comments:
If I add the Eos token as padding token it works. In any case, the tokenizer tells me to add a padding token. I do believe that the documentation says that one should exit however.

This issue has been automatically marked as stale because it has not had recent activity. If you think this still needs to be addressed please comment on this thread.

Please note that issues that do not follow the [contributing guidelines](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md) are likely to be ignored.