| 0%| | 0/23265 [00:00<?, ?it/s]Traceback (most recent call last): | |
| File "/mnt/lv_ai_1_dante/javierr/wav2vec2-xls-r-1b-npsc/run_speech_recognition_ctc.py", line 792, in <module> | |
| main() | |
| File "/mnt/lv_ai_1_dante/javierr/wav2vec2-xls-r-1b-npsc/run_speech_recognition_ctc.py", line 743, in main | |
| train_result = trainer.train(resume_from_checkpoint=checkpoint) | |
| File "/mnt/lv_ai_1_dante/javierr/audio/lib/python3.9/site-packages/transformers/trainer.py", line 1384, in train | |
| tr_loss_step = self.training_step(model, inputs) | |
| File "/mnt/lv_ai_1_dante/javierr/audio/lib/python3.9/site-packages/transformers/trainer.py", line 1959, in training_step | |
| loss = self.compute_loss(model, inputs) | |
| File "/mnt/lv_ai_1_dante/javierr/audio/lib/python3.9/site-packages/transformers/trainer.py", line 1991, in compute_loss | |
| outputs = model(**inputs) | |
| File "/mnt/lv_ai_1_dante/javierr/audio/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/mnt/lv_ai_1_dante/javierr/audio/lib/python3.9/site-packages/transformers/models/wav2vec2/modeling_wav2vec2.py", line 1756, in forward | |
| outputs = self.wav2vec2( | |
| File "/mnt/lv_ai_1_dante/javierr/audio/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/mnt/lv_ai_1_dante/javierr/audio/lib/python3.9/site-packages/transformers/models/wav2vec2/modeling_wav2vec2.py", line 1346, in forward | |
| extract_features = self.feature_extractor(input_values) | |
| File "/mnt/lv_ai_1_dante/javierr/audio/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/mnt/lv_ai_1_dante/javierr/audio/lib/python3.9/site-packages/transformers/models/wav2vec2/modeling_wav2vec2.py", line 514, in forward | |
| hidden_states = conv_layer(hidden_states) | |
| File "/mnt/lv_ai_1_dante/javierr/audio/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/mnt/lv_ai_1_dante/javierr/audio/lib/python3.9/site-packages/transformers/models/wav2vec2/modeling_wav2vec2.py", line 386, in forward | |
| hidden_states = self.conv(hidden_states) | |
| File "/mnt/lv_ai_1_dante/javierr/audio/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/mnt/lv_ai_1_dante/javierr/audio/lib/python3.9/site-packages/torch/nn/modules/conv.py", line 301, in forward | |
| return self._conv_forward(input, self.weight, self.bias) | |
| File "/mnt/lv_ai_1_dante/javierr/audio/lib/python3.9/site-packages/torch/nn/modules/conv.py", line 297, in _conv_forward | |
| return F.conv1d(input, weight, bias, self.stride, | |
| RuntimeError: CUDA error: no kernel image is available for execution on the device | |
| CUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect. | |
| For debugging consider passing CUDA_LAUNCH_BLOCKING=1. |