Erro trainer.fit(char_model)
State: closed | Pull request: no
Labels: name: bug

Hi
With a supposed Colab update yesterday, I have been having the following issue during training.

I say supposed update because I noticed changes in the Colab interface, but I didn't find any version talking about updating

I didn't make changes to the training process, the processes I used worked and didn't give this error before

As a way to try to solve this problem, I changed the torch versions from 1.9.0 to 1.8.0 and torchvision versions from 0.10.0 to 0.9.0 and torchtext from 0.10.0 to 0.9.0

I use nemo version "r1.1.0" but the same error occurred in versions "r1.0.2" and "r1.0.1"

The problem I had since yesterday when running the training is this:


```
/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/trainer.py in fit(self, model, train_dataloader, val_dataloaders, datamodule)
    456         )
    457 
--> 458         self._run(model)
    459 
    460         assert self.state.stopped

/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/trainer.py in _run(self, model)
    754 
    755         # dispatch `start_training` or `start_evaluating` or `start_predicting`
--> 756         self.dispatch()
    757 
    758         # plugin will finalized fitting (e.g. ddp_spawn will load trained model)

/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/trainer.py in dispatch(self)
    795             self.accelerator.start_predicting(self)
    796         else:
--> 797             self.accelerator.start_training(self)
    798 
    799     def run_stage(self):

/usr/local/lib/python3.7/dist-packages/pytorch_lightning/accelerators/accelerator.py in start_training(self, trainer)
     94 
     95     def start_training(self, trainer: 'pl.Trainer') -> None:
---> 96         self.training_type_plugin.start_training(trainer)
     97 
     98     def start_evaluating(self, trainer: 'pl.Trainer') -> None:

/usr/local/lib/python3.7/dist-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py in start_training(self, trainer)
    142     def start_training(self, trainer: 'pl.Trainer') -> None:
    143         # double dispatch to initiate the training loop
--> 144         self._results = trainer.run_stage()
    145 
    146     def start_evaluating(self, trainer: 'pl.Trainer') -> None:

/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/trainer.py in run_stage(self)
    805         if self.predicting:
    806             return self.run_predict()
--> 807         return self.run_train()
    808 
    809     def _pre_training_routine(self):

/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/trainer.py in run_train(self)
    867                 with self.profiler.profile("run_training_epoch"):
    868                     # run train epoch
--> 869                     self.train_loop.run_training_epoch()
    870 
    871                 if self.max_steps and self.max_steps <= self.global_step:

/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/training_loop.py in run_training_epoch(self)
    497             # ------------------------------------
    498             with self.trainer.profiler.profile("run_training_batch"):
--> 499                 batch_output = self.run_training_batch(batch, batch_idx, dataloader_idx)
    500 
    501             # when returning -1 from train_step, we end epoch early

/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/training_loop.py in run_training_batch(self, batch, batch_idx, dataloader_idx)
    736 
    737                         # optimizer step
--> 738                         self.optimizer_step(optimizer, opt_idx, batch_idx, train_step_and_backward_closure)
    739                         if len(self.trainer.optimizers) > 1:
    740                             # revert back to previous state

/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/training_loop.py in optimizer_step(self, optimizer, opt_idx, batch_idx, train_step_and_backward_closure)
    440             on_tpu=self.trainer._device_type == DeviceType.TPU and _TPU_AVAILABLE,
    441             using_native_amp=using_native_amp,
--> 442             using_lbfgs=is_lbfgs,
    443         )
    444 

/usr/local/lib/python3.7/dist-packages/pytorch_lightning/core/lightning.py in optimizer_step(self, epoch, batch_idx, optimizer, optimizer_idx, optimizer_closure, on_tpu, using_native_amp, using_lbfgs)
   1401 
   1402         """
-> 1403         optimizer.step(closure=optimizer_closure)
   1404 
   1405     def optimizer_zero_grad(self, epoch: int, batch_idx: int, optimizer: Optimizer, optimizer_idx: int):

/usr/local/lib/python3.7/dist-packages/pytorch_lightning/core/optimizer.py in step(self, closure, *args, **kwargs)
    212             profiler_name = f"optimizer_step_and_closure_{self._optimizer_idx}"
    213 
--> 214         self.__optimizer_step(*args, closure=closure, profiler_name=profiler_name, **kwargs)
    215         self._total_optimizer_step_calls += 1
    216 

/usr/local/lib/python3.7/dist-packages/pytorch_lightning/core/optimizer.py in __optimizer_step(self, closure, profiler_name, **kwargs)
    132 
    133         with trainer.profiler.profile(profiler_name):
--> 134             trainer.accelerator.optimizer_step(optimizer, self._optimizer_idx, lambda_closure=closure, **kwargs)
    135 
    136     def step(self, *args, closure: Optional[Callable] = None, **kwargs):

/usr/local/lib/python3.7/dist-packages/pytorch_lightning/accelerators/accelerator.py in optimizer_step(self, optimizer, opt_idx, lambda_closure, **kwargs)
    327         )
    328         if make_optimizer_step:
--> 329             self.run_optimizer_step(optimizer, opt_idx, lambda_closure, **kwargs)
    330         self.precision_plugin.post_optimizer_step(optimizer, opt_idx)
    331         self.training_type_plugin.post_optimizer_step(optimizer, opt_idx, **kwargs)

/usr/local/lib/python3.7/dist-packages/pytorch_lightning/accelerators/accelerator.py in run_optimizer_step(self, optimizer, optimizer_idx, lambda_closure, **kwargs)
    334         self, optimizer: Optimizer, optimizer_idx: int, lambda_closure: Callable, **kwargs: Any
    335     ) -> None:
--> 336         self.training_type_plugin.optimizer_step(optimizer, lambda_closure=lambda_closure, **kwargs)
    337 
    338     def optimizer_zero_grad(self, current_epoch: int, batch_idx: int, optimizer: Optimizer, opt_idx: int) -> None:

/usr/local/lib/python3.7/dist-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py in optimizer_step(self, optimizer, lambda_closure, **kwargs)
    191 
    192     def optimizer_step(self, optimizer: torch.optim.Optimizer, lambda_closure: Callable, **kwargs):
--> 193         optimizer.step(closure=lambda_closure, **kwargs)
    194 
    195     @property

/usr/local/lib/python3.7/dist-packages/torch/optim/lr_scheduler.py in wrapper(*args, **kwargs)
     63                 instance._step_count += 1
     64                 wrapped = func.__get__(instance, cls)
---> 65                 return wrapped(*args, **kwargs)
     66 
     67             # Note that the returned function here is no longer a bound method,

/usr/local/lib/python3.7/dist-packages/torch/optim/optimizer.py in wrapper(*args, **kwargs)
     87                 profile_name = "Optimizer.step#{}.step".format(obj.__class__.__name__)
     88                 with torch.autograd.profiler.record_function(profile_name):
---> 89                     return func(*args, **kwargs)
     90             return wrapper
     91 

/usr/local/lib/python3.7/dist-packages/nemo/core/optim/novograd.py in step(self, closure)
     81         loss = None
     82         if closure is not None:
---> 83             loss = closure()
     84 
     85         for group in self.param_groups:

/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/training_loop.py in train_step_and_backward_closure()
    731                         def train_step_and_backward_closure():
    732                             result = self.training_step_and_backward(
--> 733                                 split_batch, batch_idx, opt_idx, optimizer, self.trainer.hiddens
    734                             )
    735                             return None if result is None else result.loss

/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/training_loop.py in training_step_and_backward(self, split_batch, batch_idx, opt_idx, optimizer, hiddens)
    821         with self.trainer.profiler.profile("training_step_and_backward"):
    822             # lightning module hook
--> 823             result = self.training_step(split_batch, batch_idx, opt_idx, hiddens)
    824             self._curr_step_result = result
    825 

/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/training_loop.py in training_step(self, split_batch, batch_idx, opt_idx, hiddens)
    288             model_ref._results = Result()
    289             with self.trainer.profiler.profile("training_step"):
--> 290                 training_step_output = self.trainer.accelerator.training_step(args)
    291                 self.trainer.accelerator.post_training_step()
    292 

/usr/local/lib/python3.7/dist-packages/pytorch_lightning/accelerators/accelerator.py in training_step(self, args)
    202 
    203         with self.precision_plugin.train_step_context(), self.training_type_plugin.train_step_context():
--> 204             return self.training_type_plugin.training_step(*args)
    205 
    206     def post_training_step(self) -> None:

/usr/local/lib/python3.7/dist-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py in training_step(self, *args, **kwargs)
    153 
    154     def training_step(self, *args, **kwargs):
--> 155         return self.lightning_module.training_step(*args, **kwargs)
    156 
    157     def post_training_step(self):

/usr/local/lib/python3.7/dist-packages/nemo/utils/model_utils.py in wrap_training_step(wrapped, instance, args, kwargs)
    337 @wrapt.decorator
    338 def wrap_training_step(wrapped, instance: pl.LightningModule, args, kwargs):
--> 339     output_dict = wrapped(*args, **kwargs)
    340 
    341     if isinstance(output_dict, dict) and output_dict is not None and 'log' in output_dict:

/usr/local/lib/python3.7/dist-packages/nemo/collections/asr/models/ctc_models.py in training_step(self, batch, batch_nb)
    557             )
    558         else:
--> 559             log_probs, encoded_len, predictions = self.forward(input_signal=signal, input_signal_length=signal_len)
    560 
    561         loss_value = self.loss(

/usr/local/lib/python3.7/dist-packages/nemo/core/classes/common.py in __call__(self, wrapped, instance, args, kwargs)
    790 
    791         # Call the method - this can be forward, or any other callable method
--> 792         outputs = wrapped(*args, **kwargs)
    793 
    794         instance._attach_and_validate_output_types(

/usr/local/lib/python3.7/dist-packages/nemo/collections/asr/models/ctc_models.py in forward(self, input_signal, input_signal_length, processed_signal, processed_signal_length)
    541 
    542         if self.spec_augmentation is not None and self.training:
--> 543             processed_signal = self.spec_augmentation(input_spec=processed_signal, length=processed_signal_length)
    544 
    545         encoded, encoded_len = self.encoder(audio_signal=processed_signal, length=processed_signal_length)

/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    887             result = self._slow_forward(*input, **kwargs)
    888         else:
--> 889             result = self.forward(*input, **kwargs)
    890         for hook in itertools.chain(
    891                 _global_forward_hooks.values(),

/usr/local/lib/python3.7/dist-packages/nemo/core/classes/common.py in __call__(self, wrapped, instance, args, kwargs)
    790 
    791         # Call the method - this can be forward, or any other callable method
--> 792         outputs = wrapped(*args, **kwargs)
    793 
    794         instance._attach_and_validate_output_types(

/usr/local/lib/python3.7/dist-packages/nemo/collections/asr/modules/audio_preprocessing.py in forward(self, input_spec, length)
    492     @typecheck()
    493     def forward(self, input_spec, length):
--> 494         augmented_spec = self.spec_cutout(input_spec=input_spec)
    495 
    496         # To run the Numba kernel, correct numba version is required as well as

/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    887             result = self._slow_forward(*input, **kwargs)
    888         else:
--> 889             result = self.forward(*input, **kwargs)
    890         for hook in itertools.chain(
    891                 _global_forward_hooks.values(),

/usr/local/lib/python3.7/dist-packages/nemo/core/classes/common.py in __call__(self, wrapped, instance, args, kwargs)
    790 
    791         # Call the method - this can be forward, or any other callable method
--> 792         outputs = wrapped(*args, **kwargs)
    793 
    794         instance._attach_and_validate_output_types(

/usr/local/lib/python3.7/dist-packages/torch/autograd/grad_mode.py in decorate_context(*args, **kwargs)
     25         def decorate_context(*args, **kwargs):
     26             with self.__class__():
---> 27                 return func(*args, **kwargs)
     28         return cast(F, decorate_context)
     29 

/usr/local/lib/python3.7/dist-packages/nemo/collections/asr/parts/submodules/spectr_augment.py in forward(self, input_spec)
    146             for i in range(self.rect_masks):
    147                 rect_x = self._rng.randint(0, sh[1] - self.rect_freq)
--> 148                 rect_y = self._rng.randint(0, sh[2] - self.rect_time)
    149 
    150                 w_x = self._rng.randint(0, self.rect_freq)

/usr/lib/python3.7/random.py in randint(self, a, b)
    220         """
    221 
--> 222         return self.randrange(a, b+1)
    223 
    224     def _randbelow(self, n, int=int, maxsize=1<<BPF, type=type,

/usr/lib/python3.7/random.py in randrange(self, start, stop, step, _int)
    198             return istart + self._randbelow(width)
    199         if step == 1:
--> 200             raise ValueError("empty range for randrange() (%d,%d, %d)" % (istart, istop, width))
    201 
    202         # Non-unit step argument supplied.

ValueError: empty range for randrange() (0,-7, -7)
```