sanchit-gandhi's picture
Training in progress, step 500
fc53a59
raw
history blame
3.36 kB
0%| | 0/446 [00:00<?, ?it/s][WARNING|modeling_utils.py:388] 2022-03-02 08:50:05,594 >> Could not estimate the number of tokens of the input, floating-point operations will not be computed
Traceback (most recent call last):
File "/home/sanchit_huggingface_co/wav2vec2-gpt2-wandb-grid-search/run_speech_recognition_seq2seq.py", line 539, in <module>
main()
File "/home/sanchit_huggingface_co/wav2vec2-gpt2-wandb-grid-search/run_speech_recognition_seq2seq.py", line 491, in main
train_result = trainer.train(resume_from_checkpoint=checkpoint)
File "/home/sanchit_huggingface_co/transformers/src/transformers/trainer.py", line 1384, in train
tr_loss_step = self.training_step(model, inputs)
File "/home/sanchit_huggingface_co/transformers/src/transformers/trainer.py", line 1959, in training_step
loss = self.compute_loss(model, inputs)
File "/home/sanchit_huggingface_co/transformers/src/transformers/trainer.py", line 1991, in compute_loss
outputs = model(**inputs)
File "/home/sanchit_huggingface_co/gcp/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/home/sanchit_huggingface_co/transformers/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py", line 503, in forward
encoder_outputs = self.encoder(
File "/home/sanchit_huggingface_co/gcp/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/home/sanchit_huggingface_co/transformers/src/transformers/models/wav2vec2/modeling_wav2vec2.py", line 1346, in forward
extract_features = self.feature_extractor(input_values)
File "/home/sanchit_huggingface_co/gcp/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/home/sanchit_huggingface_co/transformers/src/transformers/models/wav2vec2/modeling_wav2vec2.py", line 514, in forward
hidden_states = conv_layer(hidden_states)
File "/home/sanchit_huggingface_co/gcp/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/home/sanchit_huggingface_co/transformers/src/transformers/models/wav2vec2/modeling_wav2vec2.py", line 389, in forward
hidden_states = self.layer_norm(hidden_states)
File "/home/sanchit_huggingface_co/gcp/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/home/sanchit_huggingface_co/gcp/lib/python3.9/site-packages/torch/nn/modules/normalization.py", line 189, in forward
return F.layer_norm(
File "/home/sanchit_huggingface_co/gcp/lib/python3.9/site-packages/torch/nn/functional.py", line 2347, in layer_norm
return torch.layer_norm(input, normalized_shape, weight, bias, eps, torch.backends.cudnn.enabled)
RuntimeError: CUDA out of memory. Tried to allocate 1.64 GiB (GPU 0; 15.78 GiB total capacity; 10.11 GiB already allocated; 707.31 MiB free; 13.39 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation. See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF