Dimensional weight error
Author: noskidr
Pull request: no

## Environment info
<!-- You can run the command `transformers-cli env` and copy-and-paste its output below.
     Don't forget to fill out the missing fields in that output! -->

- `transformers` version: 4.7.0
- Platform: Windows
- Python version: 3.7.5
- PyTorch version (GPU?): 1.8.1+cu111
- Tensorflow version (GPU?): 2.4.0
- Using GPU in script?: yes
- Using distributed or parallel set-up in script?: no

### Who can help

Models:
wav2vec2: @patrickvonplaten


## To reproduce

Steps to reproduce the behavior:

1. Same steps as in the fine-tuning wav2vec2 : https://huggingface.co/blog/fine-tune-wav2vec2-english

```
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-51-9db903a616c9> in <module>
----> 1 trainer.train()
      2 trainer.save_model('content/wav2vec2-nepali-openslr-54_10000')
      3 trainer.tokenizer.save_pretrained('content/wav2vec2-nepali-openslr-54_10000')
      4 processor.save_pretrained('content/wav2vec2-nepali-openslr-54_10000')

d:\work\asr\transformer_test\env\lib\site-packages\transformers\trainer.py in train(self, resume_from_checkpoint, trial, **kwargs)
   1261                         tr_loss += self.training_step(model, inputs)
   1262                 else:
-> 1263                     tr_loss += self.training_step(model, inputs)
   1264                 self.current_flos += float(self.floating_point_ops(inputs))
   1265 

d:\work\asr\transformer_test\env\lib\site-packages\transformers\trainer.py in training_step(self, model, inputs)
   1744         if self.use_amp:
   1745             with autocast():
-> 1746                 loss = self.compute_loss(model, inputs)
   1747         else:
   1748             loss = self.compute_loss(model, inputs)

d:\work\asr\transformer_test\env\lib\site-packages\transformers\trainer.py in compute_loss(self, model, inputs, return_outputs)
   1778         else:
   1779             labels = None
-> 1780         outputs = model(**inputs)
   1781         # Save past state if it exists
   1782         # TODO: this needs to be fixed and made cleaner later.

d:\work\asr\transformer_test\env\lib\site-packages\torch\nn\modules\module.py in _call_impl(self, *input, **kwargs)
    887             result = self._slow_forward(*input, **kwargs)
    888         else:
--> 889             result = self.forward(*input, **kwargs)
    890         for hook in itertools.chain(
    891                 _global_forward_hooks.values(),

d:\work\asr\transformer_test\env\lib\site-packages\transformers\models\wav2vec2\modeling_wav2vec2.py in forward(self, input_values, attention_mask, output_attentions, output_hidden_states, return_dict, labels)
   1470             output_attentions=output_attentions,
   1471             output_hidden_states=output_hidden_states,
-> 1472             return_dict=return_dict,
   1473         )
   1474 

d:\work\asr\transformer_test\env\lib\site-packages\torch\nn\modules\module.py in _call_impl(self, *input, **kwargs)
    887             result = self._slow_forward(*input, **kwargs)
    888         else:
--> 889             result = self.forward(*input, **kwargs)
    890         for hook in itertools.chain(
    891                 _global_forward_hooks.values(),

d:\work\asr\transformer_test\env\lib\site-packages\transformers\models\wav2vec2\modeling_wav2vec2.py in forward(self, input_values, attention_mask, mask_time_indices, output_attentions, output_hidden_states, return_dict)
   1042         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
   1043 
-> 1044         extract_features = self.feature_extractor(input_values)
   1045         extract_features = extract_features.transpose(1, 2)
   1046 

d:\work\asr\transformer_test\env\lib\site-packages\torch\nn\modules\module.py in _call_impl(self, *input, **kwargs)
    887             result = self._slow_forward(*input, **kwargs)
    888         else:
--> 889             result = self.forward(*input, **kwargs)
    890         for hook in itertools.chain(
    891                 _global_forward_hooks.values(),

d:\work\asr\transformer_test\env\lib\site-packages\transformers\models\wav2vec2\modeling_wav2vec2.py in forward(self, input_values)
    329         hidden_states = input_values[:, None]
    330         for conv_layer in self.conv_layers:
--> 331             hidden_states = conv_layer(hidden_states)
    332 
    333         return hidden_states

d:\work\asr\transformer_test\env\lib\site-packages\torch\nn\modules\module.py in _call_impl(self, *input, **kwargs)
    887             result = self._slow_forward(*input, **kwargs)
    888         else:
--> 889             result = self.forward(*input, **kwargs)
    890         for hook in itertools.chain(
    891                 _global_forward_hooks.values(),

d:\work\asr\transformer_test\env\lib\site-packages\transformers\models\wav2vec2\modeling_wav2vec2.py in forward(self, hidden_states)
    222 
    223     def forward(self, hidden_states):
--> 224         hidden_states = self.conv(hidden_states)
    225 
    226         hidden_states = hidden_states.transpose(-2, -1)

d:\work\asr\transformer_test\env\lib\site-packages\torch\nn\modules\module.py in _call_impl(self, *input, **kwargs)
    887             result = self._slow_forward(*input, **kwargs)
    888         else:
--> 889             result = self.forward(*input, **kwargs)
    890         for hook in itertools.chain(
    891                 _global_forward_hooks.values(),

d:\work\asr\transformer_test\env\lib\site-packages\torch\nn\modules\conv.py in forward(self, input)
    261 
    262     def forward(self, input: Tensor) -> Tensor:
--> 263         return self._conv_forward(input, self.weight, self.bias)
    264 
    265 

d:\work\asr\transformer_test\env\lib\site-packages\torch\nn\modules\conv.py in _conv_forward(self, input, weight, bias)
    258                             _single(0), self.dilation, self.groups)
    259         return F.conv1d(input, weight, bias, self.stride,
--> 260                         self.padding, self.dilation, self.groups)
    261 
    262     def forward(self, input: Tensor) -> Tensor:

RuntimeError: Expected 3-dimensional input for 3-dimensional weight [512, 1, 10], but got 4-dimensional input of size [1, 1, 1, 43200] instead
```

Comments:
 I am currently getting the same error - how did you solve it? Thanks in advance!