|
Reading metadata...: 2165it [00:00, 13200.08it/s] | 0/60000 [00:00<?, ?it/s] |
|
Reading metadata...: 1it [00:00, 6.42it/s] |
|
|
|
[INFO|trainer_utils.py:759] 2023-11-18 13:12:05,720 >> The following columns in the training set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length. If input_length are not expected by `WhisperForConditionalGeneration.forward`, you can safely ignore this message. |
|
Traceback (most recent call last): |
|
File "/mnt/e/run_speech_recognition_seq2seq_streaming.py", line 679, in <module> |
|
main() |
|
File "/mnt/e/run_speech_recognition_seq2seq_streaming.py", line 628, in main |
|
train_result = trainer.train(resume_from_checkpoint=checkpoint) |
|
File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/transformers/trainer.py", line 1547, in train |
|
return inner_training_loop( |
|
File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/transformers/trainer.py", line 1872, in _inner_training_loop |
|
tr_loss_step = self.training_step(model, inputs) |
|
File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/transformers/trainer.py", line 2739, in training_step |
|
loss = self.compute_loss(model, inputs) |
|
File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/transformers/trainer.py", line 2762, in compute_loss |
|
outputs = model(**inputs) |
|
File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl |
|
return self._call_impl(*args, **kwargs) |
|
File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl |
|
return forward_call(*args, **kwargs) |
|
File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 15, in wrapped_fn |
|
ret_val = func(*args, **kwargs) |
|
File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 1818, in forward |
|
loss = self.module(*inputs, **kwargs) |
|
File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl |
|
return self._call_impl(*args, **kwargs) |
|
File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl |
|
return forward_call(*args, **kwargs) |
|
File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/transformers/models/whisper/modeling_whisper.py", line 1683, in forward |
|
outputs = self.model( |
|
File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl |
|
return self._call_impl(*args, **kwargs) |
|
File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl |
|
return forward_call(*args, **kwargs) |
|
File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/transformers/models/whisper/modeling_whisper.py", line 1543, in forward |
|
encoder_outputs = self.encoder( |
|
File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl |
|
return self._call_impl(*args, **kwargs) |
|
File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl |
|
return forward_call(*args, **kwargs) |
|
File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/transformers/models/whisper/modeling_whisper.py", line 1119, in forward |
|
inputs_embeds = nn.functional.gelu(self.conv1(input_features)) |
|
File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl |
|
return self._call_impl(*args, **kwargs) |
|
File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl |
|
return forward_call(*args, **kwargs) |
|
File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/conv.py", line 310, in forward |
|
return self._conv_forward(input, self.weight, self.bias) |
|
File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/conv.py", line 306, in _conv_forward |
|
return F.conv1d(input, weight, bias, self.stride, |
|
RuntimeError: Input type (torch.cuda.FloatTensor) and weight type (CUDABFloat16Type) should be the same |