| 0%| | 0/30000 [00:00<?, ?it/s] | |
| Reading metadata...: 1it [00:01, 1.06s/it] | |
| [INFO|trainer_utils.py:744] 2023-11-18 11:40:21,343 >> The following columns in the training set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length. If input_length are not expected by `WhisperForConditionalGeneration.forward`, you can safely ignore this message. | |
| Traceback (most recent call last): | |
| File "/mnt/e/run_speech_recognition_seq2seq_streaming.py", line 679, in <module> | |
| main() | |
| File "/mnt/e/run_speech_recognition_seq2seq_streaming.py", line 628, in main | |
| train_result = trainer.train(resume_from_checkpoint=checkpoint) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/transformers/trainer.py", line 1546, in train | |
| return inner_training_loop( | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/transformers/trainer.py", line 1860, in _inner_training_loop | |
| tr_loss_step = self.training_step(model, inputs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/transformers/trainer.py", line 2725, in training_step | |
| loss = self.compute_loss(model, inputs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/transformers/trainer.py", line 2748, in compute_loss | |
| outputs = model(**inputs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/accelerate/utils/operations.py", line 659, in forward | |
| return model_forward(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/accelerate/utils/operations.py", line 647, in __call__ | |
| return convert_to_fp32(self.model_forward(*args, **kwargs)) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/amp/autocast_mode.py", line 16, in decorate_autocast | |
| return func(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/transformers/models/whisper/modeling_whisper.py", line 1683, in forward | |
| outputs = self.model( | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/transformers/models/whisper/modeling_whisper.py", line 1543, in forward | |
| encoder_outputs = self.encoder( | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/transformers/models/whisper/modeling_whisper.py", line 1151, in forward | |
| layer_outputs = self._gradient_checkpointing_func( | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/_compile.py", line 24, in inner | |
| return torch._dynamo.disable(fn, recursive)(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 328, in _fn | |
| return fn(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/_dynamo/external_utils.py", line 17, in inner | |
| return fn(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 451, in checkpoint | |
| return CheckpointFunction.apply(function, preserve, *args) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/autograd/function.py", line 539, in apply | |
| return super().apply(*args, **kwargs) # type: ignore[misc] | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 230, in forward | |
| outputs = run_function(*args) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/transformers/models/whisper/modeling_whisper.py", line 722, in forward | |
| hidden_states, attn_weights, _ = self.self_attn( | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/transformers/models/whisper/modeling_whisper.py", line 429, in forward | |
| attn_weights = nn.functional.softmax(attn_weights, dim=-1) | |
| File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/nn/functional.py", line 1856, in softmax | |
| ret = input.softmax(dim) | |
| torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 2.68 GiB. GPU 0 has a total capacty of 15.99 GiB of which 0 bytes is free. Including non-PyTorch memory, this process has 17179869184.00 GiB memory in use. Of the allocated memory 10.73 GiB is allocated by PyTorch, and 2.87 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation. See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF |